Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .buildkite/test-amd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1275,10 +1275,12 @@ steps:
- vllm/
- tests/entrypoints/openai
- tests/entrypoints/test_chat_utils
- tests/entrypoints/generate
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
- pytest -v -s entrypoints/test_chat_utils.py
- pytest -v -s tests/entrypoints/generate

- label: Entrypoints Integration (API Server openai - Part 3) # TBD
timeout_in_minutes: 180
Expand Down Expand Up @@ -1368,7 +1370,7 @@ steps:
- vllm/platforms/rocm.py
commands:
- pytest -v -s entrypoints/openai/tool_parsers
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate

- label: OpenAI API correctness # TBD
timeout_in_minutes: 180
Expand Down Expand Up @@ -2782,10 +2784,12 @@ steps:
- vllm/
- tests/entrypoints/openai
- tests/entrypoints/test_chat_utils
- tests/entrypoints/generate
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
- pytest -v -s entrypoints/test_chat_utils.py
- pytest -v -s tests/entrypoints/generate

- label: Entrypoints Integration (API Server openai - Part 3) # TBD
timeout_in_minutes: 180
Expand Down
4 changes: 3 additions & 1 deletion .buildkite/test_areas/entrypoints.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ steps:
- tests/entrypoints/
commands:
- pytest -v -s entrypoints/openai/tool_parsers
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text
Comment thread
noooop marked this conversation as resolved.
Resolved
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate

- label: Entrypoints Integration (LLM)
key: entrypoints-integration-llm
Expand Down Expand Up @@ -60,9 +60,11 @@ steps:
- vllm/
- tests/entrypoints/openai
- tests/entrypoints/test_chat_utils
- tests/entrypoints/generate
commands:
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
- pytest -v -s entrypoints/test_chat_utils.py
- pytest -v -s tests/entrypoints/generate
mirror:
amd:
device: mi325_1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@
import pytest

from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.generative_scoring.serving import (
from vllm.entrypoints.generate.generative_scoring.serving import (
GenerativeScoringItemResult,
GenerativeScoringRequest,
GenerativeScoringResponse,
OpenAIServingGenerativeScoring,
ServingGenerativeScoring,
)
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.logprobs import Logprob
Expand Down Expand Up @@ -86,13 +86,13 @@ def _create_mock_engine():
return mock_engine


def _create_serving(mock_engine) -> OpenAIServingGenerativeScoring:
"""Create an OpenAIServingGenerativeScoring instance with mocks."""
def _create_serving(mock_engine) -> ServingGenerativeScoring:
"""Create an ServingGenerativeScoring instance with mocks."""
models = OpenAIServingModels(
engine_client=mock_engine,
base_model_paths=BASE_MODEL_PATHS,
)
return OpenAIServingGenerativeScoring(mock_engine, models, request_logger=None)
return ServingGenerativeScoring(mock_engine, models, request_logger=None)


def _create_mock_request_output(logprobs_dict: dict[int, float]) -> RequestOutput:
Expand Down Expand Up @@ -186,7 +186,7 @@ def test_compute_probabilities(
self, label_logprobs, apply_softmax, should_sum_to_one
):
"""Test probability computation for softmax and true probability modes."""
serving = OpenAIServingGenerativeScoring.__new__(OpenAIServingGenerativeScoring)
serving = ServingGenerativeScoring.__new__(ServingGenerativeScoring)
probs = serving._compute_probabilities(
label_logprobs, apply_softmax=apply_softmax
)
Expand All @@ -211,7 +211,7 @@ def test_compute_probabilities(

def test_score_formula(self):
"""Test the score formula: P(token[0]) / (P(token[0]) + P(token[1]))."""
serving = OpenAIServingGenerativeScoring.__new__(OpenAIServingGenerativeScoring)
serving = ServingGenerativeScoring.__new__(ServingGenerativeScoring)

# With logprobs -0.5 and -2.0, softmax gives higher prob to first token
logprobs = {9454: -0.5, 2753: -2.0}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import pytest
import requests

from ....utils import RemoteOpenAIServer
from tests.utils import RemoteOpenAIServer

MODEL_NAME = "Qwen/Qwen3-0.6B"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ def register_generate_api_routers(app: FastAPI):

register_anthropic_api_router(app)

from .generative_scoring.api_router import register_generative_scoring_api_router

register_generative_scoring_api_router(app)


async def init_generate_state(
engine_client: "EngineClient",
Expand Down Expand Up @@ -185,3 +189,11 @@ async def init_generate_state(
if "generate" in supported_tasks
else None
)

from .generative_scoring.serving import ServingGenerativeScoring

state.generative_scoring = ServingGenerativeScoring(
Comment thread
noooop marked this conversation as resolved.
Resolved
engine_client,
state.openai_serving_models,
request_logger=request_logger,
)
Original file line number Diff line number Diff line change
@@ -1,34 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from typing import TYPE_CHECKING

from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse

from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.generative_scoring.serving import (
from vllm.entrypoints.generate.generative_scoring.serving import (
GenerativeScoringResponse,
OpenAIServingGenerativeScoring,
ServingGenerativeScoring,
)
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.utils import validate_json_request
from vllm.entrypoints.utils import load_aware_call, with_cancellation
from vllm.logger import init_logger

if TYPE_CHECKING:
from argparse import Namespace

from starlette.datastructures import State

from vllm.engine.protocol import EngineClient
from vllm.entrypoints.logger import RequestLogger

router = APIRouter()

logger = init_logger(__name__)


def generative_scoring(request: Request) -> OpenAIServingGenerativeScoring | None:
def generative_scoring(request: Request) -> ServingGenerativeScoring | None:
return request.app.state.serving_generative_scoring


Expand All @@ -51,7 +42,7 @@ async def create_generative_scoring(raw_request: Request):

raw_body = await raw_request.json()

from vllm.entrypoints.openai.generative_scoring.serving import (
from vllm.entrypoints.generate.generative_scoring.serving import (
GenerativeScoringRequest,
)

Expand All @@ -68,20 +59,3 @@ async def create_generative_scoring(raw_request: Request):

def register_generative_scoring_api_router(app: FastAPI):
app.include_router(router)


async def init_generative_scoring_state(
engine_client: "EngineClient",
state: "State",
args: "Namespace",
request_logger: "RequestLogger | None",
):
from vllm.entrypoints.openai.generative_scoring.serving import (
OpenAIServingGenerativeScoring,
)

state.serving_generative_scoring = OpenAIServingGenerativeScoring(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
)
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ class GenerativeScoringResponse(OpenAIBaseModel):
# ============================================================================


class OpenAIServingGenerativeScoring(OpenAIServing):
class ServingGenerativeScoring(OpenAIServing):
"""Serving class for generative scoring computation.

This class handles computing the probability of specified token IDs
Expand Down
16 changes: 2 additions & 14 deletions vllm/entrypoints/openai/api_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ def build_app(
register_sagemaker_api_router(app, supported_tasks, model_config)

if "generate" in supported_tasks:
from vllm.entrypoints.openai.generate.api_router import (
from vllm.entrypoints.generate.api_router import (
register_generate_api_routers,
)

Expand All @@ -220,12 +220,6 @@ def build_app(

elastic_ep_attach_router(app)

from vllm.entrypoints.openai.generative_scoring.api_router import (
register_generative_scoring_api_router,
)

register_generative_scoring_api_router(app)

if "generate" in supported_tasks or "render" in supported_tasks:
from vllm.entrypoints.serve.render.api_router import (
attach_router as attach_render_router,
Expand Down Expand Up @@ -402,18 +396,12 @@ async def init_app_state(
)

if "generate" in supported_tasks:
from vllm.entrypoints.openai.generate.api_router import init_generate_state
from vllm.entrypoints.generate.api_router import init_generate_state

await init_generate_state(
engine_client, state, args, request_logger, supported_tasks
)

from vllm.entrypoints.openai.generative_scoring.api_router import (
init_generative_scoring_state,
)

await init_generative_scoring_state(engine_client, state, args, request_logger)

if "transcription" in supported_tasks or "realtime" in supported_tasks:
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state

Expand Down
2 changes: 1 addition & 1 deletion vllm/entrypoints/sagemaker/api_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@
from fastapi.responses import JSONResponse, Response

from vllm.config import ModelConfig
from vllm.entrypoints.generate.factories import get_generate_invocation_types
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.engine.serving import OpenAIServing
from vllm.entrypoints.openai.generate.factories import get_generate_invocation_types
from vllm.entrypoints.openai.utils import validate_json_request
from vllm.entrypoints.pooling.base.serving import PoolingServingBase
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
Expand Down
Loading