Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/pre-merge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ jobs:
docs-website-composition:
name: Docs Website Composition Check
needs: changed-files
if: needs.changed-files.outputs.docs == 'true'
if: needs.changed-files.outputs.docs == 'true' && github.base_ref == 'main'
runs-on: ubuntu-latest
permissions:
contents: read
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,8 @@ jobs:
# Base version pinned in pyproject.toml at the source SHA. Same value
# cargo and maturin stamp into the wheel filename / METADATA.
BASE_VERSION=$(awk '/^version = / {gsub(/"/, "", $3); print $3; exit}' pyproject.toml)
if [[ ! "${BASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
# Accept a plain X.Y.Z release or a PEP 440 development release.
if [[ ! "${BASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9]+)?$ ]]; then
echo "Error: cannot parse base version from pyproject.toml (got '${BASE_VERSION}')"
exit 1
fi
Expand Down
139 changes: 109 additions & 30 deletions components/src/dynamo/frontend/prepost.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from concurrent.futures import ThreadPoolExecutor
from copy import deepcopy
from dataclasses import dataclass
from typing import Any, Protocol
from typing import Any, Protocol, cast

from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.entrypoints.openai.chat_completion.protocol import (
Expand Down Expand Up @@ -43,6 +43,13 @@ async def render_messages_async(
...


class _EngineBasedStreamingParser(Protocol):
engine_based_streaming: bool

def finish_streaming(self) -> DeltaMessage | None:
...


@dataclass
class PreprocessResult:
request_for_sampling: ChatCompletionRequest
Expand Down Expand Up @@ -636,13 +643,15 @@ def __init__(
tool_parser: ToolParser | None,
reasoning_parser_class: type[ReasoningParser] | None,
chat_template_kwargs: dict[str, Any],
reasoning_ended: bool | None = None,
stream_response: bool = True,
) -> None:
self.tokenizer = tokenizer
self.request_for_sampling = request_for_sampling
self.sampling_params = sampling_params
self.tool_parser = tool_parser
self.stream_response = stream_response
self.reasoning_is_done = bool(reasoning_ended)
# See https://github.com/ai-dynamo/dynamo/issues/8636 —
# when the chat template runs with enable_thinking=False,
# the reasoning open/close tags live in the prompt and the generated
Expand All @@ -658,20 +667,30 @@ def __init__(
tokenizer,
chat_template_kwargs=chat_template_kwargs,
)
if reasoning_parser_class and not thinking_disabled
if reasoning_parser_class
and not thinking_disabled
and reasoning_ended is not True
else None
)
if self.reasoning_parser is not None:
if reasoning_ended is None:
self.reasoning_is_done = self.reasoning_parser.is_reasoning_end(
prompt_token_ids
)
if not self.reasoning_is_done:
self.reasoning_parser.adjust_initial_state_from_prompt(prompt_token_ids)
self._fast_plain_text = (
self.tool_parser is None and self.reasoning_parser is None
)
self._reasoning_parser_streaming_started = False
self._tool_parser_streaming_started = False

self._control_markers = tuple(
t for t in getattr(tokenizer, "all_special_tokens", ()) if t
)

self.previous_text = ""
self.previous_token_ids: list[int] = []
self.reasoning_is_done = False
self.in_progress_tool_calls: dict[int, DeltaToolCall] = {}
# Per-choice tracking (https://github.com/ai-dynamo/dynamo/issues/8636) of whether a tool_call delta was
# emitted on that choice, keyed by `output.index`. Required because
Expand Down Expand Up @@ -778,6 +797,30 @@ def _compose_delta_message(
return None
return delta_message

@staticmethod
def _merge_delta_messages(
current: DeltaMessage | None,
incoming: DeltaMessage | None,
) -> DeltaMessage | None:
"""Append a parser flush delta to the delta produced for this chunk."""
if incoming is None:
return current
if current is None:
return incoming
if incoming.content:
current.content = (current.content or "") + incoming.content
if incoming.reasoning:
current.reasoning = (current.reasoning or "") + incoming.reasoning
if incoming.tool_calls:
current.tool_calls = (current.tool_calls or []) + incoming.tool_calls
return current

@staticmethod
def _finish_engine_parser(parser: Any) -> DeltaMessage | None:
if parser is None or not parser.engine_based_streaming:
return None
return cast(_EngineBasedStreamingParser, parser).finish_streaming()

def _add_tool_call_from_extracted(self, index: int, tool_call: Any) -> None:
tool_delta = DeltaToolCall(
index=index,
Expand All @@ -797,6 +840,9 @@ def _extract_tool_calls_from_text(
if self.tool_parser is None:
return self._compose_delta_message(saved_reasoning, None)

# A buffered parse replaces the streaming path; do not flush stale
# engine-stream state again when finish_reason is processed.
self._tool_parser_streaming_started = False
extracted = self.tool_parser.extract_tool_calls(text, self.request_for_sampling)
if extracted.tools_called:
for i, tool_call in enumerate(extracted.tool_calls):
Expand All @@ -817,6 +863,7 @@ def _extract_tool_calls_streaming(
) -> DeltaMessage | None:
if self.tool_parser is None:
return None
self._tool_parser_streaming_started = True
return self.tool_parser.extract_tool_calls_streaming(
previous_text=self.previous_text,
current_text=current_text,
Expand Down Expand Up @@ -975,6 +1022,7 @@ def process_output(self, output: Any) -> dict[str, Any] | None:
return None

elif not self.reasoning_is_done and self.reasoning_parser:
self._reasoning_parser_streaming_started = True
delta_message = self.reasoning_parser.extract_reasoning_streaming(
self.previous_text,
current_text,
Expand All @@ -989,9 +1037,21 @@ def process_output(self, output: Any) -> dict[str, Any] | None:
# buffer it for non-streaming extraction rather than feeding it
# to the streaming tool parser which cannot handle the combined
# reasoning-end + tool-start in a single chunk.
if self.reasoning_parser.is_reasoning_end_streaming(
current_token_ids, delta_token_ids
):
if self.reasoning_parser.engine_based_streaming:
reasoning_ended = (
self.reasoning_parser.has_engine_confirmed_reasoning_end()
)
else:
reasoning_ended = self.reasoning_parser.is_reasoning_end_streaming(
current_token_ids, delta_token_ids
)

if reasoning_ended:
delta_message = self._merge_delta_messages(
delta_message,
self._finish_engine_parser(self.reasoning_parser),
)
self._reasoning_parser_streaming_started = False
self.reasoning_is_done = True
saved_reasoning = delta_message.reasoning if delta_message else None
post_content = (delta_message.content if delta_message else None) or ""
Expand Down Expand Up @@ -1061,37 +1121,56 @@ def process_output(self, output: Any) -> dict[str, Any] | None:
delta_token_ids=delta_token_ids,
)

if output.finish_reason:
if self._reasoning_parser_streaming_started:
delta_message = self._merge_delta_messages(
delta_message,
self._finish_engine_parser(self.reasoning_parser),
)
self._reasoning_parser_streaming_started = False
if self._tool_parser_streaming_started:
delta_message = self._merge_delta_messages(
delta_message,
self._finish_engine_parser(self.tool_parser),
)
self._tool_parser_streaming_started = False

choice = None
if delta_message is None:
if self.in_progress_tool_calls:
choice = self._emit_tool_calls_choice(output)
elif output.finish_reason:
choice = self._build_choice(output, {})
elif delta_message.tool_calls:
self._merge_streaming_tool_calls(delta_message.tool_calls)
if output.finish_reason and self.in_progress_tool_calls:
# Tool calls and finish_reason arrived in the same chunk.
# Emit now — there will be no subsequent process_output call
# to drain the buffer.
else:
if delta_message.tool_calls:
self._merge_streaming_tool_calls(delta_message.tool_calls)

if delta_message.content or delta_message.reasoning:
delta = {"role": "assistant"}
content = delta_message.content
if self.in_progress_tool_calls and self._is_control_only_content(
content
):
content = None
if content:
delta["content"] = content
if delta_message.reasoning:
delta["reasoning_content"] = delta_message.reasoning
if self.in_progress_tool_calls:
delta["tool_calls"] = self._dump_in_progress_tool_calls()
self.in_progress_tool_calls.clear()
if len(delta) > 1:
choice = self._build_choice(output, delta)
elif delta_message.tool_calls:
if output.finish_reason and self.in_progress_tool_calls:
# Tool calls and finish_reason arrived in the same chunk.
# Emit now — there will be no subsequent process_output call
# to drain the buffer.
choice = self._emit_tool_calls_choice(output)
elif self.in_progress_tool_calls:
choice = self._emit_tool_calls_choice(output)
elif delta_message.content or delta_message.reasoning:
delta = {"role": "assistant"}
content = delta_message.content
if self.in_progress_tool_calls and self._is_control_only_content(content):
content = None
if content:
delta["content"] = content
if delta_message.reasoning:
delta["reasoning_content"] = delta_message.reasoning
if self.in_progress_tool_calls:
delta["tool_calls"] = self._dump_in_progress_tool_calls()
self.in_progress_tool_calls.clear()
if len(delta) > 1:
choice = self._build_choice(output, delta)
elif self.in_progress_tool_calls:
choice = self._emit_tool_calls_choice(output)
elif output.finish_reason:
choice = self._build_choice(output, {})
elif output.finish_reason:
choice = self._build_choice(output, {})

self.previous_text = current_text
self.previous_token_ids = current_token_ids
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -743,7 +743,7 @@ async def test_generator_preserves_zero_top_logprobs(
caplog,
):
class RequestForSampling(SimpleNamespace):
model_fields = {}
model_fields = frozenset()

monkeypatch.setattr(
vllm_processor_module,
Expand Down
32 changes: 28 additions & 4 deletions components/src/dynamo/frontend/vllm_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from msgspec.structs import replace as msgspec_replace
from vllm.config import CacheConfig, LoadConfig, ModelConfig, VllmConfig
from vllm.entrypoints.chat_utils import load_chat_template
from vllm.exceptions import VLLMClientError
from vllm.reasoning import ReasoningParser, ReasoningParserManager
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.tasks import GENERATION_TASKS
Expand All @@ -37,6 +38,7 @@
from dynamo.common.utils import nvtx_utils as _nvtx
from dynamo.frontend.frontend_args import FrontendConfig
from dynamo.llm import ModelCardInstanceId, PythonAsyncEngine, RoutedEngine
from dynamo.vllm.errors import vllm_client_error_to_http_error

from .prepost import StreamingPostProcessor, preprocess_chat_request
from .thinking import runtime_default_thinking_mode
Expand Down Expand Up @@ -186,6 +188,8 @@ def _build_reasoning_parser_metadata(
return None, None

parser_kwargs = {"chat_template_kwargs": chat_template_kwargs}
if chat_template_kwargs.get("enable_thinking") is False:
return True, parser_kwargs
Comment thread
ynpandey-nv marked this conversation as resolved.
Resolved
if not getattr(request_for_sampling, "include_reasoning", True):
return True, parser_kwargs
if getattr(request_for_sampling, "_grammar_from_tool_parser", False):
Expand Down Expand Up @@ -472,9 +476,15 @@ async def generator(
Run a single request through the engine. Does pre and post processing on this machine, delegates
model inference to a backend using the router.
"""
with _nvtx.annotate("mm_frontend:generator", color="blue"):
async for item in self._generator_inner(request, context=context):
yield item
try:
with _nvtx.annotate("mm_frontend:generator", color="blue"):
async for item in self._generator_inner(request, context=context):
yield item
except VLLMClientError as exc:
# vLLM 0.27 replaced many request-side ValueError/TypeError raises
# with this hierarchy. Preserve vLLM's 400/404/422 distinction at
# Dynamo's HTTP boundary.
raise vllm_client_error_to_http_error(exc) from exc

async def _generator_inner(
self, request: dict[str, Any], context: Any | None = None
Expand Down Expand Up @@ -678,14 +688,23 @@ async def _generator_inner(
] = request_for_sampling.mm_processor_kwargs

def new_post_processor() -> StreamingPostProcessor:
# vLLM tool parsers keep mutable streaming state. Give every
# n>1 choice its own parser instead of reusing the parser that
# adjusted the shared request during preprocessing.
choice_tool_parser = (
self.tool_parser_class(self.tokenizer, request_for_sampling.tools)
if tool_parser is not None and self.tool_parser_class is not None
else None
)
Comment thread
ynpandey-nv marked this conversation as resolved.
Resolved
return StreamingPostProcessor(
tokenizer=self.tokenizer,
request_for_sampling=request_for_sampling,
sampling_params=sampling_params,
prompt_token_ids=tokens,
tool_parser=tool_parser,
tool_parser=choice_tool_parser,
reasoning_parser_class=self.reasoning_parser_class,
chat_template_kwargs=chat_template_kwargs,
reasoning_ended=reasoning_ended,
stream_response=bool(request.get("stream", False)),
)

Expand Down Expand Up @@ -917,6 +936,11 @@ async def _generate_and_stream(

yield envelope
_nvtx.end_range(rng_stream)
except VLLMClientError:
# Preserve request-side 400/404/422 errors for generator(), which
# translates them at Dynamo's HTTP boundary. The generic handler
# below is reserved for genuine internal failures.
raise
except Exception as e:
logger.exception("Error generating response for request %s", request_id)
yield make_internal_error(request_id, str(e))
Expand Down
23 changes: 23 additions & 0 deletions components/src/dynamo/vllm/errors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Translate vLLM request errors into Dynamo's HTTP error boundary."""

from vllm.exceptions import (
VLLMClientError,
VLLMNotFoundError,
VLLMUnprocessableEntityError,
)

from dynamo.llm.exceptions import HttpError


def vllm_client_error_to_http_error(exc: VLLMClientError) -> HttpError:
"""Preserve the HTTP status assigned by vLLM's client-error hierarchy."""
if isinstance(exc, VLLMUnprocessableEntityError):
status_code = 422
elif isinstance(exc, VLLMNotFoundError):
status_code = 404
else:
status_code = 400
return HttpError(status_code, str(exc))
Loading
Loading