Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
84c91c0
chore(vllm): bump vLLM to v0.27.1
JulienDarve Aug 11, 2026
fe7622b
fix(gpu-memory-service): load canonical vLLM NIXL connector
JulienDarve Aug 11, 2026
ab47c68
fix(vllm): address v0.27.1 compatibility failures
JulienDarve Aug 12, 2026
e1a636e
test(vllm): extend XPU guided decoding timeout
JulienDarve Aug 12, 2026
b0c4b59
Merge remote-tracking branch 'origin/main' into jdarve/vllm-bump-v0.2…
JulienDarve Aug 12, 2026
94aaee6
Merge remote-tracking branch 'origin/main' into jdarve/vllm-bump-v0.2…
JulienDarve Aug 12, 2026
ada026a
fix(vllm): pin Transformers for Gemma 4 compatibility
JulienDarve Aug 12, 2026
ad5ae03
Merge remote-tracking branch 'origin/main' into jdarve/vllm-bump-v0.2…
JulienDarve Aug 12, 2026
6933255
fix(vllm): address 0.27 review feedback
JulienDarve Aug 12, 2026
6a896ee
build(vllm): upgrade vLLM-Omni to 0.27.0rc1
JulienDarve Aug 12, 2026
eae8684
Merge remote-tracking branch 'origin/main' into jdarve/vllm-bump-v0.2…
JulienDarve Aug 12, 2026
d682cbd
fix(gms): simplify vLLM NIXL compatibility
JulienDarve Aug 12, 2026
413df59
Merge remote-tracking branch 'origin/main' into jdarve/vllm-bump-v0.2…
JulienDarve Aug 13, 2026
ec1a7a0
fix(vllm): support Omni text encoder tensor parallelism
JulienDarve Aug 13, 2026
76fee56
Merge remote-tracking branch 'origin/main' into jdarve/vllm-bump-v0.2…
JulienDarve Aug 13, 2026
73c3e0d
test(gms): remove patch-specific regression test
JulienDarve Aug 13, 2026
56bab09
fix(vllm): harden streaming compatibility
JulienDarve Aug 14, 2026
4a5fb69
fix(protocol): reject unsupported stream interval
JulienDarve Aug 14, 2026
017b13f
fix(vllm-omni): forward parallel topology
JulienDarve Aug 14, 2026
94355c6
fix(gms): patch vLLM NIXL base connector
JulienDarve Aug 14, 2026
4c84219
fix(sidecar): restore compatible vLLM image
JulienDarve Aug 14, 2026
560308d
fix(container): align vLLM dependency constraints
JulienDarve Aug 14, 2026
832eddf
Merge origin/main into jdarve/vllm-bump-v0.27.1__DYN-3836
JulienDarve Aug 14, 2026
25042ea
test(vllm): guard reasoning parser prompt API
JulienDarve Aug 14, 2026
32a9132
fix(container): scope vLLM dependency validation
JulienDarve Aug 14, 2026
9bbda48
docs(api): regenerate frontend reference
JulienDarve Aug 14, 2026
54a8322
Merge origin/main into jdarve/vllm-bump-v0.27.1__DYN-3836
JulienDarve Aug 14, 2026
3dfd1cc
fix(sidecar): align vLLM wire contract
JulienDarve Aug 14, 2026
2f850d8
Merge branch 'main' into jdarve/vllm-bump-v0.27.1__DYN-3836
JulienDarve Aug 17, 2026
aff6aa4
revert(sidecar): defer vLLM sidecar bump
JulienDarve Aug 17, 2026
bac9646
Merge branch 'main' into jdarve/vllm-bump-v0.27.1__DYN-3836
JulienDarve Aug 17, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 94 additions & 28 deletions components/src/dynamo/frontend/prepost.py
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,7 @@ def __init__(
self.sampling_params = sampling_params
self.tool_parser = tool_parser
self.stream_response = stream_response
self.reasoning_is_done = False
# See https://github.com/ai-dynamo/dynamo/issues/8636 —
# when the chat template runs with enable_thinking=False,
# the reasoning open/close tags live in the prompt and the generated
Expand All @@ -661,17 +662,24 @@ def __init__(
if reasoning_parser_class and not thinking_disabled
else None
)
if self.reasoning_parser is not None:
Comment thread
dmitry-tokarev-nv marked this conversation as resolved.
Resolved
self.reasoning_is_done = self.reasoning_parser.is_reasoning_end(
prompt_token_ids
)
if not self.reasoning_is_done:
self.reasoning_parser.adjust_initial_state_from_prompt(prompt_token_ids)
Comment thread
JulienDarve marked this conversation as resolved.
Resolved
self._fast_plain_text = (
self.tool_parser is None and self.reasoning_parser is None
)
self._reasoning_parser_streaming_started = False
self._tool_parser_streaming_started = False

self._control_markers = tuple(
t for t in getattr(tokenizer, "all_special_tokens", ()) if t
)

self.previous_text = ""
self.previous_token_ids: list[int] = []
self.reasoning_is_done = False
self.in_progress_tool_calls: dict[int, DeltaToolCall] = {}
# Per-choice tracking (https://github.com/ai-dynamo/dynamo/issues/8636) of whether a tool_call delta was
# emitted on that choice, keyed by `output.index`. Required because
Expand Down Expand Up @@ -778,6 +786,31 @@ def _compose_delta_message(
return None
return delta_message

@staticmethod
def _merge_delta_messages(
current: DeltaMessage | None,
incoming: DeltaMessage | None,
) -> DeltaMessage | None:
"""Append a parser flush delta to the delta produced for this chunk."""
if incoming is None:
return current
if current is None:
return incoming
if incoming.content:
current.content = (current.content or "") + incoming.content
if incoming.reasoning:
current.reasoning = (current.reasoning or "") + incoming.reasoning
if incoming.tool_calls:
current.tool_calls = (current.tool_calls or []) + incoming.tool_calls
return current

@staticmethod
def _finish_engine_parser(parser: Any) -> DeltaMessage | None:
if parser is None or not getattr(parser, "engine_based_streaming", False):
Comment thread
dmitry-tokarev-nv marked this conversation as resolved.
Resolved
return None
finish = getattr(parser, "finish_streaming", None)
return finish() if finish is not None else None

def _add_tool_call_from_extracted(self, index: int, tool_call: Any) -> None:
tool_delta = DeltaToolCall(
index=index,
Expand Down Expand Up @@ -817,6 +850,7 @@ def _extract_tool_calls_streaming(
) -> DeltaMessage | None:
if self.tool_parser is None:
return None
self._tool_parser_streaming_started = True
return self.tool_parser.extract_tool_calls_streaming(
previous_text=self.previous_text,
current_text=current_text,
Expand Down Expand Up @@ -975,6 +1009,7 @@ def process_output(self, output: Any) -> dict[str, Any] | None:
return None

elif not self.reasoning_is_done and self.reasoning_parser:
self._reasoning_parser_streaming_started = True
delta_message = self.reasoning_parser.extract_reasoning_streaming(
self.previous_text,
current_text,
Expand All @@ -989,9 +1024,21 @@ def process_output(self, output: Any) -> dict[str, Any] | None:
# buffer it for non-streaming extraction rather than feeding it
# to the streaming tool parser which cannot handle the combined
# reasoning-end + tool-start in a single chunk.
if self.reasoning_parser.is_reasoning_end_streaming(
current_token_ids, delta_token_ids
):
if getattr(self.reasoning_parser, "engine_based_streaming", False):
reasoning_ended = (
self.reasoning_parser.has_engine_confirmed_reasoning_end()
)
else:
reasoning_ended = self.reasoning_parser.is_reasoning_end_streaming(
current_token_ids, delta_token_ids
)

if reasoning_ended:
delta_message = self._merge_delta_messages(
delta_message,
self._finish_engine_parser(self.reasoning_parser),
)
self._reasoning_parser_streaming_started = False
self.reasoning_is_done = True
saved_reasoning = delta_message.reasoning if delta_message else None
post_content = (delta_message.content if delta_message else None) or ""
Expand Down Expand Up @@ -1061,37 +1108,56 @@ def process_output(self, output: Any) -> dict[str, Any] | None:
delta_token_ids=delta_token_ids,
)

if output.finish_reason:
if self._reasoning_parser_streaming_started:
delta_message = self._merge_delta_messages(
delta_message,
self._finish_engine_parser(self.reasoning_parser),
)
self._reasoning_parser_streaming_started = False
if self._tool_parser_streaming_started:
Comment thread
dmitry-tokarev-nv marked this conversation as resolved.
Resolved
delta_message = self._merge_delta_messages(
delta_message,
self._finish_engine_parser(self.tool_parser),
)
self._tool_parser_streaming_started = False
Comment thread
JulienDarve marked this conversation as resolved.
Resolved

choice = None
if delta_message is None:
if self.in_progress_tool_calls:
choice = self._emit_tool_calls_choice(output)
elif output.finish_reason:
choice = self._build_choice(output, {})
elif delta_message.tool_calls:
self._merge_streaming_tool_calls(delta_message.tool_calls)
if output.finish_reason and self.in_progress_tool_calls:
# Tool calls and finish_reason arrived in the same chunk.
# Emit now — there will be no subsequent process_output call
# to drain the buffer.
else:
if delta_message.tool_calls:
self._merge_streaming_tool_calls(delta_message.tool_calls)

if delta_message.content or delta_message.reasoning:
delta = {"role": "assistant"}
content = delta_message.content
if self.in_progress_tool_calls and self._is_control_only_content(
content
):
content = None
if content:
delta["content"] = content
if delta_message.reasoning:
delta["reasoning_content"] = delta_message.reasoning
if self.in_progress_tool_calls:
delta["tool_calls"] = self._dump_in_progress_tool_calls()
self.in_progress_tool_calls.clear()
if len(delta) > 1:
choice = self._build_choice(output, delta)
elif delta_message.tool_calls:
if output.finish_reason and self.in_progress_tool_calls:
# Tool calls and finish_reason arrived in the same chunk.
# Emit now — there will be no subsequent process_output call
# to drain the buffer.
choice = self._emit_tool_calls_choice(output)
elif self.in_progress_tool_calls:
choice = self._emit_tool_calls_choice(output)
elif delta_message.content or delta_message.reasoning:
delta = {"role": "assistant"}
content = delta_message.content
if self.in_progress_tool_calls and self._is_control_only_content(content):
content = None
if content:
delta["content"] = content
if delta_message.reasoning:
delta["reasoning_content"] = delta_message.reasoning
if self.in_progress_tool_calls:
delta["tool_calls"] = self._dump_in_progress_tool_calls()
self.in_progress_tool_calls.clear()
if len(delta) > 1:
choice = self._build_choice(output, delta)
elif self.in_progress_tool_calls:
choice = self._emit_tool_calls_choice(output)
elif output.finish_reason:
choice = self._build_choice(output, {})
elif output.finish_reason:
choice = self._build_choice(output, {})

self.previous_text = current_text
self.previous_token_ids = current_token_ids
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -737,13 +737,13 @@ def __init__(self, **kwargs):


@pytest.mark.asyncio
async def test_generator_preserves_zero_top_logprobs(
async def test_generator_preserves_zero_top_logprobs_and_stream_interval(
vllm_processor_module,
monkeypatch,
caplog,
):
class RequestForSampling(SimpleNamespace):
model_fields = {}
model_fields = {"stream_interval": object()}

monkeypatch.setattr(
vllm_processor_module,
Expand All @@ -757,6 +757,7 @@ class RequestForSampling(SimpleNamespace):
top_logprobs=0,
cache_salt=None,
mm_processor_kwargs=None,
stream_interval=7,
),
tool_parser=None,
chat_template_kwargs={},
Expand All @@ -772,6 +773,7 @@ class ProjectionObserved(Exception):

def process_inputs(request_id, engine_inputs, sampling_params, supported_tasks):
assert sampling_params.logprobs == 0
assert sampling_params.stream_interval == 7
raise ProjectionObserved

input_processor = SimpleNamespace(
Expand Down
14 changes: 11 additions & 3 deletions components/src/dynamo/frontend/vllm_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from msgspec.structs import replace as msgspec_replace
from vllm.config import CacheConfig, LoadConfig, ModelConfig, VllmConfig
from vllm.entrypoints.chat_utils import load_chat_template
from vllm.exceptions import VLLMClientError
from vllm.reasoning import ReasoningParser, ReasoningParserManager
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.tasks import GENERATION_TASKS
Expand All @@ -37,6 +38,7 @@
from dynamo.common.utils import nvtx_utils as _nvtx
from dynamo.frontend.frontend_args import FrontendConfig
from dynamo.llm import ModelCardInstanceId, PythonAsyncEngine, RoutedEngine
from dynamo.llm.exceptions import InvalidArgument

from .prepost import StreamingPostProcessor, preprocess_chat_request
from .thinking import runtime_default_thinking_mode
Expand Down Expand Up @@ -472,9 +474,15 @@ async def generator(
Run a single request through the engine. Does pre and post processing on this machine, delegates
model inference to a backend using the router.
"""
with _nvtx.annotate("mm_frontend:generator", color="blue"):
async for item in self._generator_inner(request, context=context):
yield item
try:
with _nvtx.annotate("mm_frontend:generator", color="blue"):
async for item in self._generator_inner(request, context=context):
yield item
Comment thread
dmitry-tokarev-nv marked this conversation as resolved.
Resolved
except VLLMClientError as exc:
# vLLM 0.27 replaced many request-side ValueError/TypeError raises
# with this hierarchy. Preserve Dynamo's client-error boundary so
# invalid sampling, rendering, and multimodal inputs remain HTTP 400.
raise InvalidArgument(str(exc)) from exc

async def _generator_inner(
self, request: dict[str, Any], context: Any | None = None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,7 @@ def test_vllm_processor_cache_handles_uuid_only_unified_vision_chunk():
uuid_items,
)
processor_inputs = ProcessorInputs([], data_items, uuid_items)
mm_hashes = processor_inputs.get_mm_hashes("test-model")
mm_hashes = processor_inputs.get_mm_hashes("test-model", "blake3")
Comment thread
karen-sy marked this conversation as resolved.
Resolved

assert mm_hashes == {"vision_chunk": ["catalog/image:v2"]}

Expand Down
16 changes: 16 additions & 0 deletions components/src/dynamo/vllm/tests/test_vllm_api_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,22 @@ def test_new_request_data_has_fields_instrumented_scheduler_sets():
)


def test_scheduler_output_new_connector_fields_remain_optional():
"""Dynamo constructs SchedulerOutput directly without v0.27's new fields."""
from vllm.v1.core.sched.output import SchedulerOutput

fields = {field.name: field for field in dataclasses.fields(SchedulerOutput)}
for field_name in ("ec_manager_metadata", "partial_tail_offloads"):
assert field_name in fields, (
f"vLLM SchedulerOutput.{field_name} is gone — re-audit Dynamo's "
"direct SchedulerOutput construction."
)
assert fields[field_name].default is None, (
f"vLLM SchedulerOutput.{field_name} became required — update every "
"InstrumentedScheduler constructor."
)


def test_async_scheduler_has_methods_instrumented_scheduler_overrides():
"""Guard the ``AsyncScheduler`` methods ``InstrumentedScheduler`` overrides /
calls via ``super()``."""
Expand Down
22 changes: 11 additions & 11 deletions container/compliance/base_sboms/manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -119,16 +119,6 @@
"from_tag": "25.11-cuda13.0-devel-ubuntu24.04",
"platform": "linux/amd64"
},
{
"baseline_digest": "sha256:e2ab50957fc309d3fb110e6331ab8bfb7dd4f0074df41cbbf0bc3c64c8bc0e52",
"baseline_image": "vllm/vllm-openai-xpu",
"baseline_sbom": "vllm-openai-xpu@e2ab5095-amd64.cdx.json",
"baseline_tag": "v0.26.0",
"from_digest": "sha256:e2ab50957fc309d3fb110e6331ab8bfb7dd4f0074df41cbbf0bc3c64c8bc0e52",
"from_image": "vllm/vllm-openai-xpu",
"from_tag": "v0.26.0",
"platform": "linux/amd64"
},
{
"baseline_digest": "sha256:316b840a08a8174fc3f6b5716828bdfe1daaf629ee1ac2a8b7a22526d141a007",
"baseline_image": "nvcr.io/nvidia/tensorrt-llm/release",
Expand All @@ -148,6 +138,16 @@
"from_image": "nvcr.io/nvidia/tensorrt-llm/release",
"from_tag": "1.3.0rc23",
"platform": "linux/arm64"
},
{
"baseline_digest": "sha256:3309c2e1f7b1048fea9f392423afd91080c9f9e63796692872cff477e2562220",
"baseline_image": "vllm/vllm-openai-xpu",
"baseline_sbom": "vllm-openai-xpu@3309c2e1-amd64.cdx.json",
"baseline_tag": "v0.27.1",
"from_digest": "sha256:3309c2e1f7b1048fea9f392423afd91080c9f9e63796692872cff477e2562220",
"from_image": "vllm/vllm-openai-xpu",
"from_tag": "v0.27.1",
"platform": "linux/amd64"
}
],
"entry_schema": {
Expand All @@ -162,6 +162,6 @@
"platform": "platform pinned for layer-prefix verification, e.g. linux/amd64"
},
"format": "CycloneDX 1.6 JSON, slim filter (drop properties/hashes/dependencies; keep evidence). Hard cap 5 MB per file.",
"generated_at": "2026-08-05T00:00:00+00:00",
"generated_at": "2026-08-11T20:41:13+00:00",
"schema_version": 2
}
Loading
Loading