diff --git a/.github/actions/setup-dynamo-operator/action.yml b/.github/actions/setup-dynamo-operator/action.yml
index b9ca9dd2f5cb..44fa18956d5d 100644
--- a/.github/actions/setup-dynamo-operator/action.yml
+++ b/.github/actions/setup-dynamo-operator/action.yml
@@ -470,6 +470,29 @@ runs:
run: |
echo "${DOCKERHUB_PASS}" | helm registry login registry-1.docker.io -u "${DOCKERHUB_USER}" --password-stdin
+ # The operator refuses to start when checkpoint is enabled unless the
+ # PodSnapshot API exists. Snapshot-agent (which owns that CRD) is installed
+ # in a later job, so apply the CRDs first and let that later job adopt them.
+ - name: Install snapshot CRDs for checkpoint
+ if: inputs.checkpoint_enabled == 'true'
+ shell: bash
+ env:
+ # Keep in lockstep with .github/actions/setup-snapshot-agent/action.yml.
+ SNAPSHOT_CHART_VERSION: 0.1.0-alpha.1
+ run: |
+ echo "::group::Install snapshot CRDs before operator start"
+ set -euo pipefail
+ VKUBECONFIG="${{ github.workspace }}/.kubeconfig-vcluster"
+ helm show crds oci://ghcr.io/ai-dynamo/snapshot/snapshot \
+ --version "${SNAPSHOT_CHART_VERSION}" \
+ | kubectl --kubeconfig="${VKUBECONFIG}" \
+ apply --server-side --force-conflicts --field-manager=ci-setup -f -
+ kubectl --kubeconfig="${VKUBECONFIG}" wait --for=condition=established \
+ crd/podsnapshots.nvidia.com \
+ crd/podsnapshotcontents.nvidia.com \
+ --timeout=120s
+ echo "::endgroup::"
+
- name: Install Dynamo platform via Helm
shell: bash
env:
diff --git a/components/src/dynamo/sglang/CLAUDE.md b/components/src/dynamo/sglang/CLAUDE.md
index e0e8689ca0fa..802f13b53274 100644
--- a/components/src/dynamo/sglang/CLAUDE.md
+++ b/components/src/dynamo/sglang/CLAUDE.md
@@ -20,7 +20,7 @@ support the current version plus 1 version back (N and N-1). The pattern:
enough surface area to cover what Dynamo actually calls.
4. Each fallback branch in `_compat.py` MUST have a comment noting which SGLang
version it supports and when it can be removed, e.g.:
- `# Fallback for sglang <= 0.5.12. Remove when min supported version is 0.5.14+`
+ `# Fallback for sglang <= 0.5.16. Remove when min supported version is 0.5.18+`
5. When a new SGLang version is released and the old N-1 falls outside the support
window, delete the corresponding fallback branches and polyfills from `_compat.py`.
If `_compat.py` becomes trivial re-exports, inline the imports and delete the file.
@@ -70,6 +70,11 @@ Worker dispatch (main.py:60-132):
have `max_running_requests`, `dllm_algorithm_config`, or other LLM-specific fields.
Use `getattr()` when accessing fields that may not exist on the stub.
+SGLang 0.5.17 makes a resolved `ServerArgs` unconditionally read-only. Apply Dynamo's
+post-resolution startup overrides through `_compat.override_server_args()`; control-plane
+updates after engine creation should use the tokenizer manager's update API instead of
+assigning fields on `server_args`.
+
**DynamoConfig** combines `DynamoRuntimeConfig` (common flags like `--namespace`,
`--output-modalities`, `--media-output-fs-url`) with `DynamoSGLangConfig` (sglang-specific
flags like `--enable-multimodal`, `--dedicated-mm-encoder`, `--embedding-worker`).
diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py
index d644143adf57..bd242c5cd35e 100644
--- a/components/src/dynamo/sglang/_compat.py
+++ b/components/src/dynamo/sglang/_compat.py
@@ -38,35 +38,10 @@ def _warn_require_reasoning_unsupported() -> None:
)
-# ---------------------------------------------------------------------------
-# Top-level sglang exports: Engine, ServerArgs
-#
-# Some SGLang dev builds (including 0.5.x snapshots) do not re-export these
-# from sglang/__init__.py, while Dynamo historically uses `import sglang as sgl`
-# followed by `sgl.Engine(...)` throughout this backend.
-# ---------------------------------------------------------------------------
-def ensure_sglang_top_level_exports() -> None:
- """Restore top-level SGLang exports omitted by some install flavors."""
- import sglang as sgl
-
- if not hasattr(sgl, "Engine"):
- from sglang.srt.entrypoints.engine import Engine
-
- sgl.Engine = Engine
-
- if not hasattr(sgl, "ServerArgs"):
- from sglang.srt.server_args import ServerArgs
-
- sgl.ServerArgs = ServerArgs
-
-
-ensure_sglang_top_level_exports()
-
-
def ensure_sglang_tensor_image_size() -> None:
"""Allow SGLang's image-token resolver to handle decoded image tensors.
- SGLang 0.5.13 through 0.5.16 assume every decoded image exposes the PIL
+ SGLang 0.5.13 through 0.5.17 assume every decoded image exposes the PIL
``height``/``width`` attributes. Its CUDA JPEG decoder instead returns a
CHW tensor, causing multimodal requests to fall back to retokenization.
@@ -106,6 +81,27 @@ def resolve_image_token_counts(self: Any, images: list[Any]) -> list[int]:
BaseMultimodalProcessor.resolve_image_token_counts = resolve_image_token_counts
+def override_server_args(server_args: Any, source: str, **fields: Any) -> None:
+ """Apply a post-resolution SGLang configuration update.
+
+ SGLang 0.5.17 makes ``ServerArgs`` unconditionally read-only after
+ resolution. Both supported CUDA releases expose ``ServerArgs.override`` as
+ the audited mutation API, so Dynamo must use it instead of assigning fields.
+ The separately pinned XPU image still uses SGLang 0.5.11, which predates
+ that API; preserve its legacy assignment behavior until its engine pin is
+ upgraded.
+ """
+ override = getattr(server_args, "override", None)
+ if callable(override):
+ override(source, **fields)
+ return
+
+ # XPU compatibility for SGLang 0.5.11. Remove when the XPU SGLang pin is
+ # upgraded to 0.5.16+.
+ for name, value in fields.items():
+ setattr(server_args, name, value)
+
+
@lru_cache(maxsize=32)
def _get_async_generate_supported_kwarg_names(
async_generate: Any,
@@ -138,10 +134,12 @@ def filter_supported_async_generate_kwargs(
) -> dict[str, Any]:
"""Return only async_generate kwargs accepted by this SGLang engine.
- SGLang occasionally adds optional Engine.async_generate kwargs before every
- supported install flavor has them. Keep the compatibility boundary narrow:
- callers decide which kwargs are optional, and this helper only drops those
- optional kwargs when the installed engine cannot accept them.
+ Both supported CUDA releases accept Dynamo's optional kwargs. The separately
+ pinned XPU image still uses SGLang 0.5.11, which predates ``mm_hashes`` and
+ ``require_reasoning``. Keep the compatibility boundary narrow: callers
+ decide which kwargs are optional, and this helper only drops those optional
+ kwargs when the installed engine cannot accept them. Remove this filtering
+ when the XPU SGLang pin is upgraded to 0.5.16+.
"""
async_generate = engine.async_generate
signature_source = getattr(async_generate, "__func__", async_generate)
@@ -175,7 +173,7 @@ def require_reasoning_kwargs(engine: Any, request: Mapping[str, Any]) -> dict[st
__all__ = [
"ensure_sglang_tensor_image_size",
- "ensure_sglang_top_level_exports",
"filter_supported_async_generate_kwargs",
+ "override_server_args",
"require_reasoning_kwargs",
]
diff --git a/components/src/dynamo/sglang/init_llm.py b/components/src/dynamo/sglang/init_llm.py
index 8c0e19af8f27..2279d10481e5 100644
--- a/components/src/dynamo/sglang/init_llm.py
+++ b/components/src/dynamo/sglang/init_llm.py
@@ -14,6 +14,7 @@
from dynamo.common.utils.endpoint_types import parse_endpoint_types
from dynamo.llm import ModelInput, ModelType, WorkerType
from dynamo.runtime import DistributedRuntime
+from dynamo.sglang._compat import override_server_args
from dynamo.sglang.args import Config
from dynamo.sglang.health_check import (
SglangDisaggHealthCheckPayload,
@@ -71,7 +72,11 @@ async def init_decode(
"created before the endpoint existed, so its FPM publisher bound "
"a different IPC path than the relay would subscribe to."
)
- server_args.enable_forward_pass_metrics = False
+ override_server_args(
+ server_args,
+ "dynamo.snapshot",
+ enable_forward_pass_metrics=False,
+ )
else:
set_forward_pass_metrics_worker_id(server_args, generate_endpoint)
start_time = time.time()
@@ -227,7 +232,11 @@ async def init_prefill(
"created before the endpoint existed, so its FPM publisher bound "
"a different IPC path than the relay would subscribe to."
)
- server_args.enable_forward_pass_metrics = False
+ override_server_args(
+ server_args,
+ "dynamo.snapshot",
+ enable_forward_pass_metrics=False,
+ )
else:
set_forward_pass_metrics_worker_id(server_args, generate_endpoint)
start_time = time.time()
diff --git a/components/src/dynamo/sglang/init_multimodal.py b/components/src/dynamo/sglang/init_multimodal.py
index 5ce1d9e4cef7..9c010e445f0b 100644
--- a/components/src/dynamo/sglang/init_multimodal.py
+++ b/components/src/dynamo/sglang/init_multimodal.py
@@ -253,7 +253,7 @@ async def init_multimodal_prefill_worker(
logging.error(f"Failed to serve endpoints: {e}")
raise
finally:
- handler.cleanup()
+ await handler.cleanup_async()
if run_deferred_handlers is not None:
logging.info("Running deferred handlers")
await run_deferred_handlers()
diff --git a/components/src/dynamo/sglang/main.py b/components/src/dynamo/sglang/main.py
index f91bfbfc7c29..91d02e166a95 100644
--- a/components/src/dynamo/sglang/main.py
+++ b/components/src/dynamo/sglang/main.py
@@ -15,6 +15,7 @@
)
from dynamo.common.utils.runtime import create_runtime
from dynamo.runtime.logging import configure_dynamo_logging
+from dynamo.sglang._compat import override_server_args
from dynamo.sglang.args import parse_args
from dynamo.sglang.init_diffusion import (
init_image_diffusion,
@@ -44,7 +45,11 @@ async def worker(argv: list[str] | None = None):
if config.server_args.load_format == "gms":
from gpu_memory_service.integrations.sglang import setup_gms
- config.server_args.load_format = setup_gms(config.server_args)
+ override_server_args(
+ config.server_args,
+ "dynamo.gms",
+ load_format=setup_gms(config.server_args),
+ )
# Snapshot mode: engine must be created before runtime so CRIU captures no
# NATS/etcd connections.
diff --git a/components/src/dynamo/sglang/publisher.py b/components/src/dynamo/sglang/publisher.py
index 49cb6c38aedd..54892c47f468 100644
--- a/components/src/dynamo/sglang/publisher.py
+++ b/components/src/dynamo/sglang/publisher.py
@@ -24,6 +24,7 @@
)
from dynamo.llm import KvEventPublisher, WorkerMetricsPublisher
from dynamo.runtime import Endpoint
+from dynamo.sglang._compat import override_server_args
from dynamo.sglang._disagg import SGLANG_WORKER_GROUP_ID_KEY, get_sglang_worker_group_id
from dynamo.sglang.args import Config
from dynamo.sglang.capacity import (
@@ -49,9 +50,13 @@ def set_forward_pass_metrics_worker_id(
import tempfile
- server_args.forward_pass_metrics_worker_id = str(generate_endpoint.connection_id())
ipc_path = tempfile.NamedTemporaryFile(delete=False).name
- server_args.forward_pass_metrics_ipc_name = f"ipc://{ipc_path}"
+ override_server_args(
+ server_args,
+ "dynamo.forward_pass_metrics",
+ forward_pass_metrics_worker_id=str(generate_endpoint.connection_id()),
+ forward_pass_metrics_ipc_name=f"ipc://{ipc_path}",
+ )
async def _resolve_multinode_leader_worker_id(
diff --git a/components/src/dynamo/sglang/request_handlers/handler_base.py b/components/src/dynamo/sglang/request_handlers/handler_base.py
index f2dd5a3e5a56..b9a9d1fb27da 100644
--- a/components/src/dynamo/sglang/request_handlers/handler_base.py
+++ b/components/src/dynamo/sglang/request_handlers/handler_base.py
@@ -871,7 +871,9 @@ async def update_weight_version(self, body: dict) -> dict:
if req.abort_all_requests:
self.engine.tokenizer_manager.abort_request(abort_all=True)
- self.engine.tokenizer_manager.server_args.weight_version = req.new_version
+ self.engine.tokenizer_manager._update_weight_version_if_provided(
+ req.new_version
+ )
return {
"success": True,
"message": f"Weight version updated to {req.new_version}",
diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py
index 256514c3d68a..7aa25668bfc3 100644
--- a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py
+++ b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py
@@ -4,6 +4,7 @@
import asyncio
import json
import logging
+import sys
from typing import Any, AsyncIterator, Callable, Literal, Optional, Protocol
import sglang as sgl
@@ -677,6 +678,8 @@ class MultimodalPrefillWorkerHandler(
Processes multimodal inputs and coordinates with decode worker.
"""
+ _REQUEST_REGISTRATION_TIMEOUT_SECONDS = 5.0
+
def __init__(
self,
engine: sgl.Engine,
@@ -692,6 +695,7 @@ def __init__(
# Get bootstrap info using BootstrapManager
self.bootstrap_host, self.bootstrap_port = self._get_bootstrap_info(engine)
+ self._consume_tasks: set[asyncio.Task[Any]] = set()
logger.info(
f"Multimodal prefill worker handler initialized - bootstrap host: {self.bootstrap_host}, bootstrap port: {self.bootstrap_port}"
@@ -721,21 +725,96 @@ def _end_bootstrap() -> None:
# Validate and parse request
disagg_request = self._validate_and_parse_disagg_request(disagg_request)
- # Generate and return bootstrap info first (like regular SGLang)
+ rid = context.trace_id or context.id()
bootstrap_room = self._generate_bootstrap_room()
- bootstrap_info = {
- "bootstrap_host": self.bootstrap_host,
- "bootstrap_port": self.bootstrap_port,
- "bootstrap_room": bootstrap_room,
- }
-
- _end_bootstrap()
- yield json.dumps(bootstrap_info)
-
- # Process prefill generation
- await self._process_prefill_generation(
- disagg_request, bootstrap_room, context=context
+ results, tensor_id = await self._start_prefill_or_cancel(
+ disagg_request,
+ bootstrap_room,
+ rid,
+ context,
)
+ consumer_owns_tensor = asyncio.Event()
+ request_started = asyncio.Event()
+ task: asyncio.Task[Any] | None = None
+ try:
+ task = asyncio.create_task(
+ self._consume_results(
+ results,
+ tensor_id,
+ rid,
+ context,
+ consumer_owns_tensor,
+ request_started,
+ )
+ )
+ self._consume_tasks.add(task)
+ task.add_done_callback(self._consume_tasks.discard)
+
+ started_wait = asyncio.create_task(request_started.wait())
+ try:
+ await asyncio.wait(
+ (task, started_wait),
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+ finally:
+ if not started_wait.done():
+ started_wait.cancel()
+ try:
+ await started_wait
+ except asyncio.CancelledError:
+ pass
+
+ # Surface an immediate submission/cancellation failure before
+ # decode is authorized. Once consumer_owns_tensor is set, the
+ # consumer's try/finally owns the transferred tensor.
+ if task.done():
+ await task
+ # Do not authorize decode until embeddings have been received and
+ # the result consumer has advanced SGLang's lazy request iterator.
+ # Otherwise decode can start waiting before prefill is submitted.
+ bootstrap_info = {
+ "bootstrap_host": self.bootstrap_host,
+ "bootstrap_port": self.bootstrap_port,
+ "bootstrap_room": bootstrap_room,
+ }
+
+ _end_bootstrap()
+ yield json.dumps(bootstrap_info)
+
+ await task
+ except BaseException:
+ if not consumer_owns_tensor.is_set():
+ if task is not None and not task.done():
+ task.cancel()
+ if tensor_id is not None:
+ self.embeddings_processor.release_embeddings(tensor_id)
+ raise
+ finally:
+ pending_exception = sys.exc_info()[1]
+ if task is not None:
+ if not task.done():
+ task.cancel()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+ except Exception as task_error:
+ if pending_exception is None:
+ raise
+ if (
+ task_error is not pending_exception
+ and task_error
+ is not getattr(pending_exception, "__cause__", None)
+ ):
+ logger.error(
+ "Multimodal prefill consumer failed during request "
+ "cleanup",
+ exc_info=(
+ type(task_error),
+ task_error,
+ task_error.__traceback__,
+ ),
+ )
except Exception as e:
logger.error(f"Error in prefill generation: {e}", exc_info=True)
@@ -761,13 +840,14 @@ def _validate_and_parse_disagg_request(
)
return disagg_request
- async def _process_prefill_generation(
+ async def _start_prefill_generation(
self,
disagg_request: DisaggSglangMultimodalRequest,
bootstrap_room: int,
+ rid: Optional[str] = None,
context=None,
- ):
- """Process multimodal input and start prefill generation"""
+ ) -> tuple[AsyncIterator[Any], Optional[int]]:
+ """Receive multimodal embeddings and submit the prefill to SGLang."""
# Get the SglangMultimodalRequest from the DisaggSglangMultimodalRequest
request = disagg_request.request
input_ids = request.request.token_ids
@@ -787,42 +867,302 @@ async def _process_prefill_generation(
context.trace_headers() if context and self.enable_trace else None
)
- # Start SGLang prefill generation (like regular SGLang)
- with _nvtx.annotate("mm:prefill:engine_async_generate", color="blue"):
- gen_params = {
- "input_ids": input_ids,
- "sampling_params": sampling_params,
- "stream": True,
- "bootstrap_host": self.bootstrap_host,
- "bootstrap_port": self.bootstrap_port,
- "bootstrap_room": bootstrap_room,
- "external_trace_header": trace_header,
- "rid": context.trace_id if context else None,
- }
+ try:
+ # Start SGLang prefill generation (like regular SGLang)
+ with _nvtx.annotate("mm:prefill:engine_async_generate", color="blue"):
+ gen_params = {
+ "input_ids": input_ids,
+ "sampling_params": sampling_params,
+ "stream": True,
+ "bootstrap_host": self.bootstrap_host,
+ "bootstrap_port": self.bootstrap_port,
+ "bootstrap_room": bootstrap_room,
+ "external_trace_header": trace_header,
+ "rid": rid,
+ }
+
+ if image_mm_items:
+ gen_params["image_data"] = image_mm_items
+ if video_data:
+ gen_params["video_data"] = video_data
+
+ results = await self.engine.async_generate(**gen_params)
+ except BaseException:
+ if tensor_id is not None:
+ self.embeddings_processor.release_embeddings(tensor_id)
+ raise
- if image_mm_items:
- gen_params["image_data"] = image_mm_items
- if video_data:
- gen_params["video_data"] = video_data
+ return results, tensor_id
+
+ async def _start_prefill_or_cancel(
+ self,
+ disagg_request: DisaggSglangMultimodalRequest,
+ bootstrap_room: int,
+ rid: str,
+ context: Context,
+ ) -> tuple[AsyncIterator[Any], Optional[int]]:
+ """Cancel local preprocessing/submission if the client stops early."""
+ start_task = asyncio.create_task(
+ self._start_prefill_generation(
+ disagg_request,
+ bootstrap_room,
+ rid=rid,
+ context=context,
+ )
+ )
+ cancellation_future = context.async_killed_or_stopped()
+ try:
+ done, _ = await asyncio.wait(
+ (start_task, cancellation_future),
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+ # Prefer a completed submission if both signals arrive together. The
+ # registered-request phase will observe the sticky context state and
+ # abort by the same RID without dropping tensor ownership.
+ if start_task in done:
+ return await start_task
+
+ start_task.cancel()
+ try:
+ await start_task
+ except asyncio.CancelledError:
+ pass
+ raise asyncio.CancelledError
+ finally:
+ if not start_task.done():
+ start_task.cancel()
+ try:
+ await start_task
+ except asyncio.CancelledError:
+ pass
+ if not cancellation_future.done():
+ cancellation_future.cancel()
+ try:
+ await cancellation_future
+ except asyncio.CancelledError:
+ pass
+
+ async def _wait_for_request_registration(self, rid: str) -> None:
+ """Wait until SGLang owns the RID, without waiting for engine output."""
+ tokenizer_manager = getattr(self.engine, "tokenizer_manager", None)
+ rid_to_state = getattr(tokenizer_manager, "rid_to_state", None)
+ if rid_to_state is None:
+ raise RuntimeError("SGLang tokenizer manager has no request registry")
- results = await self.engine.async_generate(**gen_params)
+ async def poll_registry() -> None:
+ while rid not in rid_to_state:
+ await asyncio.sleep(0.001)
- # Consume results without yielding (prefill doesn't return text, just coordinates)
- asyncio.create_task(self._consume_results(results, tensor_id))
+ try:
+ await asyncio.wait_for(
+ poll_registry(),
+ timeout=self._REQUEST_REGISTRATION_TIMEOUT_SECONDS,
+ )
+ except TimeoutError as e:
+ raise RuntimeError(
+ f"SGLang did not register prefill request {rid} within "
+ f"{self._REQUEST_REGISTRATION_TIMEOUT_SECONDS:g}s"
+ ) from e
- async def _consume_results(self, results, tensor_id: Optional[int]):
- """Consume prefill results without returning them (like regular SGLang)"""
+ async def _consume_results(
+ self,
+ results,
+ tensor_id: Optional[int],
+ rid: str,
+ context: Context,
+ owns_tensor: asyncio.Event,
+ request_started: asyncio.Event,
+ ) -> None:
+ """Consume prefill output while honoring request cancellation."""
released = False
+ request_id_future: asyncio.Future[str] = asyncio.Future()
+ first_result_task: asyncio.Task[Any] | None = None
+ next_result_task: asyncio.Task[Any] | None = None
+ registration_task: asyncio.Task[None] | None = None
+ pre_registration_cancellation: asyncio.Future[Any] | None = None
+
+ def process_result(result: dict[str, Any]) -> None:
+ nonlocal released
+ if tensor_id is not None and not released:
+ self.embeddings_processor.release_embeddings(tensor_id)
+ released = True
+
try:
- async for _ in results:
- if tensor_id is not None and not released:
- self.embeddings_processor.release_embeddings(tensor_id)
- released = True
+ owns_tensor.set()
+ registration_task = asyncio.create_task(
+ self._wait_for_request_registration(rid)
+ )
+ first_result_task = asyncio.create_task(anext(results))
+ pre_registration_cancellation = context.async_killed_or_stopped()
+
+ first_result: Any = None
+ first_result_ready = False
+ while not registration_task.done():
+ wait_for: set[asyncio.Future[Any]] = {
+ registration_task,
+ pre_registration_cancellation,
+ }
+ if first_result_task is not None and not first_result_task.done():
+ wait_for.add(first_result_task)
+ done, _ = await asyncio.wait(
+ wait_for,
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+ if registration_task in done:
+ break
+ if first_result_task is not None:
+ first_result_is_done = first_result_task in done
+ else:
+ first_result_is_done = False
+ if first_result_is_done and not first_result_ready:
+ assert first_result_task is not None
+ try:
+ first_result = await first_result_task
+ except StopAsyncIteration as e:
+ raise RuntimeError(
+ "SGLang prefill stream ended before producing a result"
+ ) from e
+ finally:
+ first_result_task = None
+ first_result_ready = True
+ if pre_registration_cancellation in done:
+ raise asyncio.CancelledError
+
+ await registration_task
+ request_id_future.set_result(rid)
+ if not pre_registration_cancellation.done():
+ pre_registration_cancellation.cancel()
+ try:
+ await pre_registration_cancellation
+ except asyncio.CancelledError:
+ pass
+
+ async with self._cancellation_monitor(
+ request_id_future, context
+ ) as cancellation_task:
+ request_started.set()
+ if not first_result_ready:
+ assert first_result_task is not None
+ done, _ = await asyncio.wait(
+ (first_result_task, cancellation_task),
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+ if cancellation_task in done:
+ await cancellation_task
+ raise asyncio.CancelledError
+ try:
+ first_result = await first_result_task
+ except StopAsyncIteration as e:
+ raise RuntimeError(
+ "SGLang prefill stream ended before producing a result"
+ ) from e
+ finally:
+ first_result_task = None
+ process_result(first_result)
+
+ while True:
+ next_result_task = asyncio.create_task(anext(results))
+ done, _ = await asyncio.wait(
+ (next_result_task, cancellation_task),
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+ if cancellation_task in done:
+ if not next_result_task.done():
+ next_result_task.cancel()
+ try:
+ await next_result_task
+ except asyncio.CancelledError:
+ pass
+ await cancellation_task
+ raise asyncio.CancelledError
+ try:
+ result = await next_result_task
+ except StopAsyncIteration:
+ break
+ finally:
+ next_result_task = None
+ process_result(result)
finally:
+ pending_exception = sys.exc_info()[1]
+ for task in (registration_task, pre_registration_cancellation):
+ if task is None:
+ continue
+ if not task.done():
+ task.cancel()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+ except Exception as task_error:
+ if pending_exception is None:
+ raise
+ if task_error is not pending_exception:
+ logger.error(
+ "SGLang prefill registration task failed during cleanup",
+ exc_info=(
+ type(task_error),
+ task_error,
+ task_error.__traceback__,
+ ),
+ )
+ if next_result_task is not None:
+ if not next_result_task.done():
+ next_result_task.cancel()
+ try:
+ await next_result_task
+ except asyncio.CancelledError:
+ pass
+ except Exception as task_error:
+ if pending_exception is None:
+ raise
+ if task_error is not pending_exception:
+ logger.error(
+ "SGLang prefill next-result task failed during cleanup",
+ exc_info=(
+ type(task_error),
+ task_error,
+ task_error.__traceback__,
+ ),
+ )
+ if first_result_task is not None:
+ if not first_result_task.done():
+ first_result_task.cancel()
+ try:
+ await first_result_task
+ except asyncio.CancelledError:
+ pass
+ except Exception as task_error:
+ if pending_exception is None:
+ raise
+ if task_error is not pending_exception:
+ logger.error(
+ "SGLang prefill first-result task failed during cleanup",
+ exc_info=(
+ type(task_error),
+ task_error,
+ task_error.__traceback__,
+ ),
+ )
if tensor_id is not None and not released:
self.embeddings_processor.release_embeddings(tensor_id)
- def cleanup(self):
+ async def cleanup_async(self) -> None:
+ tasks = list(self._consume_tasks)
+ for task in tasks:
+ if not task.done():
+ task.cancel()
+ if tasks:
+ results = await asyncio.gather(*tasks, return_exceptions=True)
+ for result in results:
+ if isinstance(result, Exception) and not isinstance(
+ result, asyncio.CancelledError
+ ):
+ logger.error(
+ "Multimodal prefill consumer failed during handler cleanup",
+ exc_info=(type(result), result, result.__traceback__),
+ )
+ self._consume_tasks.clear()
+
super().cleanup()
self.engine.shutdown()
logger.info("Multimodal prefill engine shutdown")
diff --git a/components/src/dynamo/sglang/snapshot.py b/components/src/dynamo/sglang/snapshot.py
index b280bd9eed4a..eb65065f7b8a 100644
--- a/components/src/dynamo/sglang/snapshot.py
+++ b/components/src/dynamo/sglang/snapshot.py
@@ -16,6 +16,7 @@
SnapshotConfig,
configure_snapshot_capture_env,
)
+from dynamo.sglang._compat import override_server_args
from .pause import SGLangEnginePauseController
@@ -138,7 +139,7 @@ async def prepare_snapshot_engine(
# Enable memory_saver so GPU memory can be released for CRIU.
# When using GMS, weights use VA-stable unmap/remap (no CPU backup); GMS
# forbids enable_weights_cpu_backup. Otherwise use CPU backup for weights.
- server_args.enable_memory_saver = True
+ snapshot_overrides = {"enable_memory_saver": True}
try:
from gpu_memory_service.integrations.sglang import is_gms_active
@@ -146,7 +147,8 @@ async def prepare_snapshot_engine(
except ImportError:
_using_gms = False
if not _using_gms:
- server_args.enable_weights_cpu_backup = True
+ snapshot_overrides["enable_weights_cpu_backup"] = True
+ override_server_args(server_args, "dynamo.snapshot", **snapshot_overrides)
start_time = time.time()
engine = sgl.Engine(server_args=server_args)
diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py
index a62e2258f1f8..bd0e1c805499 100644
--- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py
+++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py
@@ -1,7 +1,11 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
+import asyncio
import importlib
+import json
+from contextlib import asynccontextmanager
+from types import SimpleNamespace
import numpy as np
import pytest
@@ -21,6 +25,7 @@
)
from dynamo.sglang.request_handlers.multimodal.worker_handler import (
EmbeddingsProcessor,
+ MultimodalPrefillWorkerHandler,
_build_mm_items,
)
@@ -35,6 +40,35 @@
]
+class _FakeContext:
+ def __init__(self, context_id: str, trace_id: str | None = None):
+ self.trace_id = trace_id
+ self._context_id = context_id
+ self._cancelled = asyncio.Event()
+
+ def id(self) -> str:
+ return self._context_id
+
+ def async_killed_or_stopped(self) -> asyncio.Task[bool]:
+ return asyncio.create_task(self._cancelled.wait())
+
+ def cancel(self) -> None:
+ self._cancelled.set()
+
+
+@asynccontextmanager
+async def _never_cancels(request_id_future, context):
+ task = asyncio.create_task(asyncio.Event().wait())
+ try:
+ yield task
+ finally:
+ task.cancel()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+
+
def test_extract_media_inputs_supports_video_urls():
handler = MultimodalEncodeWorkerHandler.__new__(MultimodalEncodeWorkerHandler)
@@ -148,6 +182,239 @@ def create_multimodal_video_item(
assert mm_item["video_timestamps"] == [[0.25, 0.75]]
+@pytest.mark.asyncio
+async def test_multimodal_prefill_starts_before_returning_bootstrap():
+ handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler)
+ handler.bootstrap_host = "prefill-host"
+ handler.bootstrap_port = 1234
+ handler._consume_tasks = set()
+ handler.embeddings_processor = SimpleNamespace(release_embeddings=lambda _: None)
+ events = []
+
+ handler._validate_and_parse_disagg_request = lambda request: request
+ handler._generate_bootstrap_room = lambda: 17
+
+ async def start_prefill(request, bootstrap_room, rid=None, context=None):
+ events.append(("start", bootstrap_room, rid))
+
+ async def results():
+ events.append(("iterate", None))
+ yield {"meta_info": {"id": "request-id"}}
+
+ return results(), None
+
+ async def wait_for_registration(rid):
+ events.append(("registered", rid))
+
+ handler._start_prefill_generation = start_prefill
+ handler._wait_for_request_registration = wait_for_registration
+ handler._cancellation_monitor = _never_cancels
+
+ stream = handler.generate(object(), _FakeContext("request-id"))
+ bootstrap = json.loads(await anext(stream))
+
+ assert events == [
+ ("start", 17, "request-id"),
+ ("registered", "request-id"),
+ ("iterate", None),
+ ]
+ assert bootstrap == {
+ "bootstrap_host": "prefill-host",
+ "bootstrap_port": 1234,
+ "bootstrap_room": 17,
+ }
+
+ with pytest.raises(StopAsyncIteration):
+ await anext(stream)
+ assert events == [
+ ("start", 17, "request-id"),
+ ("registered", "request-id"),
+ ("iterate", None),
+ ]
+
+
+@pytest.mark.asyncio
+async def test_multimodal_prefill_releases_embeddings_when_submission_fails(
+ monkeypatch,
+):
+ import dynamo.sglang.request_handlers.multimodal.worker_handler as worker_handler
+
+ handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler)
+ handler.bootstrap_host = "prefill-host"
+ handler.bootstrap_port = 1234
+ handler.enable_trace = False
+
+ released = []
+ handler.embeddings_processor = SimpleNamespace(release_embeddings=released.append)
+
+ async def build_mm_items(request, embeddings_processor):
+ return [], [], torch.empty(0), 23
+
+ class FailingEngine:
+ async def async_generate(self, **kwargs):
+ raise RuntimeError("submission failed")
+
+ handler.engine = FailingEngine()
+ monkeypatch.setattr(worker_handler, "_build_mm_items", build_mm_items)
+
+ request = SimpleNamespace(
+ request=SimpleNamespace(request=SimpleNamespace(token_ids=[1, 2, 3])),
+ sampling_params={"max_new_tokens": 1},
+ )
+
+ with pytest.raises(RuntimeError, match="submission failed"):
+ await handler._start_prefill_generation(request, 17)
+
+ assert released == [23]
+
+
+@pytest.mark.asyncio
+async def test_multimodal_prefill_rejects_empty_engine_stream():
+ handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler)
+ released = []
+ handler.embeddings_processor = SimpleNamespace(release_embeddings=released.append)
+
+ async def wait_for_registration(rid):
+ return None
+
+ handler._wait_for_request_registration = wait_for_registration
+ handler._cancellation_monitor = _never_cancels
+
+ async def empty_results():
+ if False:
+ yield None
+
+ owns_tensor = asyncio.Event()
+ request_started = asyncio.Event()
+
+ with pytest.raises(RuntimeError, match="ended before producing a result"):
+ await handler._consume_results(
+ empty_results(),
+ 23,
+ "request-id",
+ _FakeContext("request-id"),
+ owns_tensor,
+ request_started,
+ )
+
+ assert owns_tensor.is_set()
+ assert request_started.is_set()
+ assert released == [23]
+
+
+@pytest.mark.asyncio
+async def test_multimodal_prefill_cancellation_awaits_consumer_cleanup():
+ handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler)
+ handler.bootstrap_host = "prefill-host"
+ handler.bootstrap_port = 1234
+ handler._consume_tasks = set()
+ released = []
+ handler.embeddings_processor = SimpleNamespace(release_embeddings=released.append)
+
+ handler._validate_and_parse_disagg_request = lambda request: request
+ handler._generate_bootstrap_room = lambda: 17
+ result_stopped = asyncio.Event()
+
+ async def results():
+ try:
+ await asyncio.Event().wait()
+ yield {}
+ finally:
+ result_stopped.set()
+
+ async def start_prefill(request, bootstrap_room, rid=None, context=None):
+ return results(), 23
+
+ async def wait_for_registration(rid):
+ return None
+
+ handler._start_prefill_generation = start_prefill
+ handler._wait_for_request_registration = wait_for_registration
+ handler._cancellation_monitor = _never_cancels
+
+ stream = handler.generate(object(), _FakeContext("request-id"))
+ await anext(stream)
+ await stream.aclose()
+
+ assert result_stopped.is_set()
+ assert released == [23]
+ assert not handler._consume_tasks
+
+
+@pytest.mark.asyncio
+async def test_multimodal_prefill_cancels_registered_rid_before_first_result():
+ rid = "dynamo-request-id"
+ abort_calls = []
+ result_stopped = asyncio.Event()
+
+ class _TokenizerManager:
+ def __init__(self):
+ self.rid_to_state = {}
+
+ def abort_request(self, *, rid, abort_all):
+ abort_calls.append((rid, abort_all))
+
+ tokenizer_manager = _TokenizerManager()
+ handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler)
+ handler.engine = SimpleNamespace(tokenizer_manager=tokenizer_manager)
+ handler.shutdown_event = None
+ handler.embeddings_processor = SimpleNamespace(release_embeddings=lambda _: None)
+
+ async def results():
+ tokenizer_manager.rid_to_state[rid] = object()
+ try:
+ await asyncio.Event().wait()
+ yield {}
+ finally:
+ result_stopped.set()
+
+ owns_tensor = asyncio.Event()
+ request_started = asyncio.Event()
+ context = _FakeContext(rid)
+ consumer = asyncio.create_task(
+ handler._consume_results(
+ results(),
+ None,
+ rid,
+ context,
+ owns_tensor,
+ request_started,
+ )
+ )
+
+ await asyncio.wait_for(request_started.wait(), timeout=1)
+ context.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(consumer, timeout=1)
+
+ assert abort_calls == [(rid, False)]
+ assert result_stopped.is_set()
+
+
+@pytest.mark.asyncio
+async def test_multimodal_prefill_cleanup_awaits_consumers_before_engine_shutdown():
+ handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler)
+ handler.publisher = None
+ events = []
+
+ async def consumer():
+ try:
+ await asyncio.Event().wait()
+ finally:
+ events.append("consumer stopped")
+
+ task = asyncio.create_task(consumer())
+ await asyncio.sleep(0)
+ handler._consume_tasks = {task}
+ handler.engine = SimpleNamespace(shutdown=lambda: events.append("engine shutdown"))
+
+ await handler.cleanup_async()
+
+ assert events == ["consumer stopped", "engine shutdown"]
+ assert not handler._consume_tasks
+
+
async def test_nvdec_video_metadata_shim_stamps_valid_metadata():
"""The metadata shim gives pre-decoded ndarray frames a valid metadata dict.
diff --git a/components/src/dynamo/sglang/tests/test_sglang_publisher.py b/components/src/dynamo/sglang/tests/test_sglang_publisher.py
index d774b1605ea1..0aea2168f9ee 100644
--- a/components/src/dynamo/sglang/tests/test_sglang_publisher.py
+++ b/components/src/dynamo/sglang/tests/test_sglang_publisher.py
@@ -51,11 +51,25 @@ def test_get_local_dp_rank_range_respects_multinode_dp_attention():
def test_set_forward_pass_metrics_worker_id_uses_endpoint_identity():
- server_args = SimpleNamespace(enable_forward_pass_metrics=True)
+ class ReadOnlyServerArgs:
+ def __init__(self):
+ object.__setattr__(self, "enable_forward_pass_metrics", True)
+ object.__setattr__(self, "override_source", None)
+
+ def __setattr__(self, name, value):
+ raise AttributeError("server args are read-only")
+
+ def override(self, source, **fields):
+ object.__setattr__(self, "override_source", source)
+ for name, value in fields.items():
+ object.__setattr__(self, name, value)
+
+ server_args = ReadOnlyServerArgs()
endpoint = SimpleNamespace(connection_id=lambda: "endpoint-9")
set_forward_pass_metrics_worker_id(server_args, endpoint)
+ assert server_args.override_source == "dynamo.forward_pass_metrics"
assert server_args.forward_pass_metrics_worker_id == "endpoint-9"
assert server_args.forward_pass_metrics_ipc_name.startswith("ipc://")
diff --git a/components/src/dynamo/sglang/tests/test_sglang_unit.py b/components/src/dynamo/sglang/tests/test_sglang_unit.py
index f113d9a38ff6..19597fa35f67 100644
--- a/components/src/dynamo/sglang/tests/test_sglang_unit.py
+++ b/components/src/dynamo/sglang/tests/test_sglang_unit.py
@@ -22,8 +22,8 @@
from dynamo.common.snapshot.constants import SNAPSHOT_CONTROL_DIR_ENV
from dynamo.sglang._compat import (
ensure_sglang_tensor_image_size,
- ensure_sglang_top_level_exports,
filter_supported_async_generate_kwargs,
+ override_server_args,
require_reasoning_kwargs,
)
from dynamo.sglang.args import (
@@ -68,6 +68,20 @@
mock_sglang_cli = make_cli_args_fixture("dynamo.sglang")
+def test_override_server_args_supports_legacy_xpu_pin():
+ server_args = SimpleNamespace(enable_memory_saver=False)
+
+ override_server_args(
+ server_args,
+ "dynamo.test",
+ enable_memory_saver=True,
+ load_format="legacy-loader",
+ )
+
+ assert server_args.enable_memory_saver is True
+ assert server_args.load_format == "legacy-loader"
+
+
@pytest.fixture(autouse=True)
def _cpu_engine_when_no_accelerator(monkeypatch):
"""Honor the file's gpu_0 contract on hosts with no accelerator.
@@ -206,40 +220,6 @@ def _make_sglang_config(**overrides):
return config
-def test_compat_restores_sglang_top_level_exports():
- """Dynamo supports SGLang builds that omit top-level Engine/ServerArgs."""
- import sglang as sgl
- from sglang.srt.entrypoints.engine import Engine
- from sglang.srt.server_args import ServerArgs
-
- missing = object()
- original_engine = getattr(sgl, "Engine", missing)
- original_server_args = getattr(sgl, "ServerArgs", missing)
-
- try:
- if hasattr(sgl, "Engine"):
- delattr(sgl, "Engine")
- if hasattr(sgl, "ServerArgs"):
- delattr(sgl, "ServerArgs")
-
- ensure_sglang_top_level_exports()
-
- assert sgl.Engine is Engine
- assert sgl.ServerArgs is ServerArgs
- finally:
- if original_engine is missing:
- if hasattr(sgl, "Engine"):
- delattr(sgl, "Engine")
- else:
- sgl.Engine = original_engine
-
- if original_server_args is missing:
- if hasattr(sgl, "ServerArgs"):
- delattr(sgl, "ServerArgs")
- else:
- sgl.ServerArgs = original_server_args
-
-
def test_compat_supports_tensor_image_sizes_and_is_idempotent(caplog, monkeypatch):
from sglang.srt.multimodal.processors.base_processor import (
BaseMultimodalProcessor,
@@ -266,6 +246,10 @@ async def process_mm_data_async(self, *args, **kwargs):
processor = object.__new__(ConcreteMultimodalProcessor)
processor._processor = Processor()
+ # SGLang 0.5.17 resolves the processor and tokenizer together before
+ # handling raw multimodal items. This test stubs the processing path,
+ # so a tokenizer is not exercised, but the attribute must exist.
+ processor._tokenizer = None
processor.use_cuda_ipc = False
image_token_id = 99
processor._process_and_collect_mm_items = lambda **kwargs: (
@@ -581,6 +565,32 @@ async def start_profile(self, request):
assert response == {"status": "ok", "message": "Profiling started"}
+@pytest.mark.asyncio
+async def test_update_weight_version_uses_tokenizer_manager_control_api():
+ class TokenizerManager:
+ updated_version = None
+
+ def _update_weight_version_if_provided(self, version):
+ self.updated_version = version
+
+ tokenizer_manager = TokenizerManager()
+ handler = SimpleNamespace(
+ engine=SimpleNamespace(tokenizer_manager=tokenizer_manager)
+ )
+
+ response = await BaseWorkerHandler.update_weight_version(
+ handler,
+ {"new_version": "step-42", "abort_all_requests": False},
+ )
+
+ assert tokenizer_manager.updated_version == "step-42"
+ assert response == {
+ "success": True,
+ "message": "Weight version updated to step-42",
+ "new_version": "step-42",
+ }
+
+
@pytest.mark.asyncio
async def test_custom_jinja_template_invalid_path(mock_sglang_cli):
"""Test that invalid file path raises FileNotFoundError."""
diff --git a/container/context.yaml b/container/context.yaml
index d6708877de13..604d33133265 100644
--- a/container/context.yaml
+++ b/container/context.yaml
@@ -107,7 +107,7 @@ sglang:
base_image: nvcr.io/nvidia/cuda-dl-base
runtime_image: lmsysorg/sglang
base_image_tag: 25.11-cuda13.0-devel-ubuntu24.04
- runtime_image_tag: v0.5.16-cu130-runtime
+ runtime_image_tag: v0.5.17-cu130-runtime
# Baseline is the TRUE base of the third-party lmsysorg/sglang image,
# nvidia/cuda:13.0.1-cudnn-devel-ubuntu24.04 (Docker Hub; CUDA_VERSION=13.0.1,
# cuDNN 9.13.0.50, ubuntu24.04), NOT a self-baseline of lmsysorg/sglang.
@@ -135,7 +135,7 @@ sglang:
# (see templates/dev.Dockerfile). Runtime stays on the upstream lmsysorg/sglang
# NIXL Python stack — its wheel COPY is narrowed to ai_dynamo*.whl so the SDK
# build doesn't leak into the runtime image.
- nixl_ref: v1.3.0
+ nixl_ref: v1.3.2
enable_media_ffmpeg: "false"
enable_gpu_memory_service: "true"
enable_kvbm: "false"
diff --git a/docs/fern/components/releases.data.ts b/docs/fern/components/releases.data.ts
index 531a067791e9..f2030b272e17 100644
--- a/docs/fern/components/releases.data.ts
+++ b/docs/fern/components/releases.data.ts
@@ -71,10 +71,10 @@ export const CURRENT_TAG = "1.4.0";
export const CURRENT_WHEEL = "1.4.0";
export const MAIN_TOT: BackendPins = {
- sglang: "0.5.16",
+ sglang: "0.5.17",
trtllm: "1.3.0rc24",
vllm: "0.27.1",
- nixlSglang: "1.3.0",
+ nixlSglang: "1.3.2",
nixlTrtllm: "1.3.1",
nixlVllm: "1.3.2",
};
@@ -1177,7 +1177,7 @@ export const FEATURE_INTERACTIONS: BackendInteractions[] = [
// KV Block Manager
[{ status: "wip" }, { status: "wip" }, { status: "wip" }, { status: "na" }],
// Multimodal
- [{ status: "yes", label: "Supported serving patterns", note: "Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported.", source: "/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal" }, { status: "yes", label: "Image-aware routing on Dynamo's SGLang image", note: "Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.16, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing.", source: "/dynamo/dev/multimodal/multimodal-kv-routing" }, { status: "na" }, { status: "wip" }, { status: "na" }],
+ [{ status: "yes", label: "Supported serving patterns", note: "Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported.", source: "/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal" }, { status: "yes", label: "Image-aware routing on Dynamo's SGLang image", note: "Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.17, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing.", source: "/dynamo/dev/multimodal/multimodal-kv-routing" }, { status: "na" }, { status: "wip" }, { status: "na" }],
// Request Migration
[{ status: "yes" }, { status: "yes" }, { status: "yes" }, { status: "wip" }, { status: "yes" }, { status: "na" }],
// Request Cancellation
diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md
index 50ea8014c771..12da98dbc3ef 100644
--- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md
+++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md
@@ -122,7 +122,7 @@ The launcher configures KV events on each worker and sets `--router-mode kv` wit
The Dynamo SGLang image includes both routing prerequisites:
- Dynamo is built with the `mm-routing` Rust feature.
-- SGLang 0.5.13 or later includes `GenerateReqInput.mm_hashes` support. Dynamo currently pins 0.5.16.
+- SGLang 0.5.13 or later includes `GenerateReqInput.mm_hashes` support. Dynamo currently pins 0.5.17.
Custom installations on SGLang 0.5.12 or earlier need the `mm_hashes` change
from [sgl-project/sglang#25300](https://github.com/sgl-project/sglang/pull/25300).
diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/observability.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/observability.md
index 77f76baf9c98..2a992dda6719 100644
--- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/observability.md
+++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/observability.md
@@ -115,7 +115,7 @@ For the complete and authoritative list of all SGLang metrics, see the [official
## Forward Pass Metrics (FPM)
-> **Availability.** Forward Pass Metrics require SGLang's upstream `sglang.srt.observability.forward_pass_metrics` module and the `ServerArgs` fields (`enable_forward_pass_metrics`, `forward_pass_metrics_worker_id`, `forward_pass_metrics_ipc_name`). These landed in **SGLang v0.5.13** and ship in the current Dynamo runtime (`sglang==0.5.15`), so setting `DYN_FORWARDPASS_METRIC_PORT` enables SGLang-side FPM emission to the NATS event plane via `FpmEventRelay`. The wire-format contract is guarded by `dynamo/sglang/tests/test_fpm_contract.py`. On runtimes older than v0.5.13 the module is absent: the Dynamo-side relay still starts and the worker serves normally, but no SGLang-side FPM payloads are emitted.
+> **Availability.** Forward Pass Metrics require SGLang's upstream `sglang.srt.observability.forward_pass_metrics` module and the `ServerArgs` fields (`enable_forward_pass_metrics`, `forward_pass_metrics_worker_id`, `forward_pass_metrics_ipc_name`). These landed in **SGLang v0.5.13** and ship in the current CUDA Dynamo runtime (`sglang==0.5.17`), so setting `DYN_FORWARDPASS_METRIC_PORT` enables SGLang-side FPM emission to the NATS event plane via `FpmEventRelay`. The wire-format contract is guarded by `dynamo/sglang/tests/test_fpm_contract.py`. On runtimes older than v0.5.13 the module is absent: the Dynamo-side relay still starts and the worker serves normally, but no SGLang-side FPM payloads are emitted.
Forward Pass Metrics provide **per-iteration scheduler telemetry** pushed over ZMQ, giving the [Planner](../../planner/overview.md) real-time visibility into batch composition, queue depth, and GPU forward pass duration. Unlike Prometheus metrics (which are scraped asynchronously and reflect only the latest gauge value), FPM emits a structured message after every scheduler iteration with the exact batch state.
diff --git a/docs/fern/pages/reference/general/compatibility.mdx b/docs/fern/pages/reference/general/compatibility.mdx
index 7f243d6e2283..a01fbccd0e2f 100644
--- a/docs/fern/pages/reference/general/compatibility.mdx
+++ b/docs/fern/pages/reference/general/compatibility.mdx
@@ -156,7 +156,7 @@ Current stable release: v1.4.0 (container tag `1.4.0`, wheel version `1.4.0`).
| Dynamo | Type | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX |
| --- | --- | --- | --- | --- | --- | --- |
-| main (ToT) | development head | 0.5.16 | 1.3.0rc24 | 0.27.1 | 1.3.0 / 1.3.1 / 1.3.2 | - |
+| main (ToT) | development head | 0.5.17 | 1.3.0rc24 | 0.27.1 | 1.3.2 / 1.3.1 / 1.3.2 | - |
| v1.4.0 | stable | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x |
| v1.3.1 | patch | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.2 / 1.0.1 / 1.1.0 | 1.20.x |
| v1.3.0 | stable | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.0 / 1.0.1 / 1.1.0 | 1.20.x |
@@ -306,10 +306,10 @@ Each cell states whether the row feature works together with the column feature.
| Feature | Disaggregated Serving | KV-Aware Routing | SLA-Based Planner | KV Block Manager | Multimodal | Request Migration | Request Cancellation | LoRA | Tool Calling | Speculative Decoding |
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
| Disaggregated Serving | n/a | Yes | Yes | Experimental | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes | Experimental — Cancellation during remote prefill is not supported in disaggregated mode. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental — Code hooks exist, but examples and documentation are not yet available. |
-| KV-Aware Routing | Yes | n/a | Yes | Experimental | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.16, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | Yes | Yes | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental |
+| KV-Aware Routing | Yes | n/a | Yes | Experimental | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.17, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | Yes | Yes | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental |
| SLA-Based Planner | Yes | Yes | n/a | Experimental | n/a | Yes | Yes | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a |
| KV Block Manager | Experimental | Experimental | Experimental | n/a | Experimental | Experimental | Experimental | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Experimental | Experimental |
-| Multimodal | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.16, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | n/a | Experimental | n/a | Yes | Experimental | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a |
+| Multimodal | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.17, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | n/a | Experimental | n/a | Yes | Experimental | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a |
| Request Migration | Yes | Yes | Yes | Experimental | Yes | n/a | Yes | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental |
| Request Cancellation | Experimental — Cancellation during remote prefill is not supported in disaggregated mode. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Yes | Experimental | Experimental | Yes | n/a | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a |
| LoRA | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | n/a | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) |
diff --git a/docs/fern/pages/reference/general/releases-machine-readable.mdx b/docs/fern/pages/reference/general/releases-machine-readable.mdx
index 0299a753c8fa..2c6241aa2e5b 100644
--- a/docs/fern/pages/reference/general/releases-machine-readable.mdx
+++ b/docs/fern/pages/reference/general/releases-machine-readable.mdx
@@ -15,7 +15,7 @@ Current stable release: v1.4.0 (Aug 14, 2026; container tag `1.4.0`, wheel versi
| Version | Kind | Date | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX | Notes | Delta |
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| main (ToT) | development head | - | 0.5.16 | 1.3.0rc24 | 0.27.1 | 1.3.0 / 1.3.1 / 1.3.2 | - | - | - |
+| main (ToT) | development head | - | 0.5.17 | 1.3.0rc24 | 0.27.1 | 1.3.2 / 1.3.1 / 1.3.2 | - | - | - |
| v1.4.0 | stable | Aug 14, 2026 | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-4-0) | Audit subsystem migrated into request trace (DYN_AUDIT_* honored as legacy aliases); HTTP header capture in trace records is an explicit fail-closed allowlist; deprecated multimodal worker flags and vLLM worker-role flags removed; runtime images no longer bundle software video decoders (H.264/H.265 decodes via NVDEC); UCX 1.21.x. |
| v1.3.1 | patch | Aug 5, 2026 | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.2 / 1.0.1 / 1.1.0 | 1.20.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-3-0#v131) | Patch release. Fixes disaggregated SGLang serving over AWS EFA on GB200: the SGLang EFA runtime moves to NIXL 1.3.2 and all three EFA images to EFA Installer 1.49.0. Backend pins are otherwise unchanged from v1.3.0. |
| v1.3.0 | stable | Jul 20, 2026 | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.0 / 1.0.1 / 1.1.0 | 1.20.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-3-0) | CUDA 12 container images discontinued; EFA variants retagged from -efa-amd64 to -efa (the images were already multi-arch — the old suffix was misleading); GA wheels published as 1.3.0.post1 (containers stay :1.3.0); UCX 1.20.x. |
diff --git a/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md b/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md
index 6787ca42eff7..9a16e907d127 100644
--- a/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md
+++ b/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md
@@ -113,5 +113,5 @@ URI string, while frontend decoding hashes the decoded bytes.
|---------|--------------|--------|-------|
| [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Supported families include Qwen2-VL, Qwen2.5-VL, Qwen3-VL, LLaVA 1.5, LLaVA-NeXT, Llama 4, Kimi K2.5/K2.6, Qwen3.5, and Qwen3.6. The rest use text-prefix-only routing. |
| [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md#multimodal-kv-routing) | Python chat processor | Yes | Uses vLLM’s own multimodal processor — supports any VLM that vLLM supports. |
-| [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Hash forwarding is upstream in SGLang 0.5.13+; Dynamo pins 0.5.16. Older custom installations need the upstream patch. |
+| [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Hash forwarding is upstream in SGLang 0.5.13+; Dynamo pins 0.5.17. Older custom installations need the upstream patch. |
| [TensorRT-LLM](../../developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Supported model scope is the Qwen2-VL family (Qwen2-VL / Qwen2.5-VL / Qwen3-VL) and Kimi (Kimi-K2.5 / Kimi-K2.6). Other multimodal models fall back to text-prefix routing. |
diff --git a/examples/backends/sglang/launch/agg_agent.sh b/examples/backends/sglang/launch/agg_agent.sh
index 8b25d8ebdff6..c0f4b25beddb 100755
--- a/examples/backends/sglang/launch/agg_agent.sh
+++ b/examples/backends/sglang/launch/agg_agent.sh
@@ -72,7 +72,6 @@ python3 -m dynamo.sglang \
--tp "$TP" \
--trust-remote-code \
--radix-eviction-policy priority \
- --skip-tokenizer-init \
--dyn-reasoning-parser glm45 \
--dyn-tool-call-parser glm47 \
--kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' \
diff --git a/lib/gpu_memory_service/integrations/sglang/__init__.py b/lib/gpu_memory_service/integrations/sglang/__init__.py
index 28a460212bc6..cb073ce0af82 100644
--- a/lib/gpu_memory_service/integrations/sglang/__init__.py
+++ b/lib/gpu_memory_service/integrations/sglang/__init__.py
@@ -7,7 +7,8 @@
from gpu_memory_service.integrations.sglang import setup_gms
if server_args.load_format == "gms":
- server_args.load_format = setup_gms(server_args)
+ load_format = setup_gms(server_args)
+ server_args.override("dynamo.gms", load_format=load_format)
"""
from __future__ import annotations
@@ -57,7 +58,13 @@ def setup_gms(server_args) -> Type["GMSModelLoader"]:
"Cannot use --enable-draft-weights-cpu-backup with --load-format gms."
)
- server_args.enable_memory_saver = True
+ override = getattr(server_args, "override", None)
+ if callable(override):
+ override("dynamo.gms", enable_memory_saver=True)
+ else:
+ # The separately pinned XPU image still uses SGLang 0.5.11, which
+ # predates ServerArgs.override. Remove after that pin reaches 0.5.16+.
+ server_args.enable_memory_saver = True
# Resolve lock mode and RO reconnect timeout from model_loader_extra_config
# before patches fire.
global _gms_lock_mode
diff --git a/lib/sidecar/sglang/README.md b/lib/sidecar/sglang/README.md
index de7c1f541702..246b1ede8a0c 100644
--- a/lib/sidecar/sglang/README.md
+++ b/lib/sidecar/sglang/README.md
@@ -61,8 +61,8 @@ There is no published sidecar image yet, so you build and push your own from
> [!NOTE]
> The engine image must be a stock SGLang **v0.5.16+** build: the native gRPC
-> server (`--grpc-port`) landed there, and Dynamo's `sglang-runtime` pins an
-> older SGLang without it. `deploy/agg.yaml` uses `lmsysorg/sglang:v0.5.16`.
+> server (`--grpc-port`) landed there. The deployment examples use
+> `lmsysorg/sglang:v0.5.17`, matching Dynamo main's SGLang pin.
### Prerequisites
diff --git a/lib/sidecar/sglang/deploy/agg.yaml b/lib/sidecar/sglang/deploy/agg.yaml
index 375c1662f0e5..8899d0de220f 100644
--- a/lib/sidecar/sglang/deploy/agg.yaml
+++ b/lib/sidecar/sglang/deploy/agg.yaml
@@ -38,10 +38,10 @@ spec:
spec:
# sglang-engine as a native sidecar (initContainer + restartPolicy: Always):
# starts before `main`, stops after. Native gRPC `--grpc-port` needs SGLang
- # v0.5.16+ (Dynamo's sglang-runtime pins an older SGLang without it).
+ # v0.5.16+; keep this example aligned with Dynamo main's SGLang pin.
initContainers:
- name: sglang-engine
- image: lmsysorg/sglang:v0.5.16
+ image: lmsysorg/sglang:v0.5.17
restartPolicy: Always
# gRPC HealthCheck via exec (tcpSocket/grpc: dial the pod IP, not the
# loopback engine). startupProbe gates `main`; readinessProbe drops dead workers.
diff --git a/lib/sidecar/sglang/deploy/disagg.yaml b/lib/sidecar/sglang/deploy/disagg.yaml
index a20428d1da06..4142966c1357 100644
--- a/lib/sidecar/sglang/deploy/disagg.yaml
+++ b/lib/sidecar/sglang/deploy/disagg.yaml
@@ -52,7 +52,7 @@ spec:
sizeLimit: 10Gi
initContainers:
- name: sglang-engine
- image: lmsysorg/sglang:v0.5.16
+ image: lmsysorg/sglang:v0.5.17
restartPolicy: Always
securityContext:
capabilities:
@@ -140,7 +140,7 @@ spec:
sizeLimit: 10Gi
initContainers:
- name: sglang-engine
- image: lmsysorg/sglang:v0.5.16
+ image: lmsysorg/sglang:v0.5.17
restartPolicy: Always
securityContext:
capabilities:
diff --git a/pyproject.toml b/pyproject.toml
index 9e1e8fad1ac5..d1d3d4fbc64b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -78,11 +78,11 @@ vllm = [
sglang = [
"uvloop",
- "sglang[diffusion]==0.5.16",
+ "sglang[diffusion]==0.5.17",
# sglang[diffusion] dropped accelerate in 0.5.12; diffusers still needs it.
"accelerate>=0.17.0",
"blake3>=1.0.0,<2.0.0",
- "nixl[cu13]==1.3.0",
+ "nixl[cu13]==1.3.2",
"cupy-cuda12x>=13.0.0",
]