diff --git a/.github/actions/setup-dynamo-operator/action.yml b/.github/actions/setup-dynamo-operator/action.yml index b9ca9dd2f5cb..44fa18956d5d 100644 --- a/.github/actions/setup-dynamo-operator/action.yml +++ b/.github/actions/setup-dynamo-operator/action.yml @@ -470,6 +470,29 @@ runs: run: | echo "${DOCKERHUB_PASS}" | helm registry login registry-1.docker.io -u "${DOCKERHUB_USER}" --password-stdin + # The operator refuses to start when checkpoint is enabled unless the + # PodSnapshot API exists. Snapshot-agent (which owns that CRD) is installed + # in a later job, so apply the CRDs first and let that later job adopt them. + - name: Install snapshot CRDs for checkpoint + if: inputs.checkpoint_enabled == 'true' + shell: bash + env: + # Keep in lockstep with .github/actions/setup-snapshot-agent/action.yml. + SNAPSHOT_CHART_VERSION: 0.1.0-alpha.1 + run: | + echo "::group::Install snapshot CRDs before operator start" + set -euo pipefail + VKUBECONFIG="${{ github.workspace }}/.kubeconfig-vcluster" + helm show crds oci://ghcr.io/ai-dynamo/snapshot/snapshot \ + --version "${SNAPSHOT_CHART_VERSION}" \ + | kubectl --kubeconfig="${VKUBECONFIG}" \ + apply --server-side --force-conflicts --field-manager=ci-setup -f - + kubectl --kubeconfig="${VKUBECONFIG}" wait --for=condition=established \ + crd/podsnapshots.nvidia.com \ + crd/podsnapshotcontents.nvidia.com \ + --timeout=120s + echo "::endgroup::" + - name: Install Dynamo platform via Helm shell: bash env: diff --git a/components/src/dynamo/sglang/CLAUDE.md b/components/src/dynamo/sglang/CLAUDE.md index e0e8689ca0fa..802f13b53274 100644 --- a/components/src/dynamo/sglang/CLAUDE.md +++ b/components/src/dynamo/sglang/CLAUDE.md @@ -20,7 +20,7 @@ support the current version plus 1 version back (N and N-1). The pattern: enough surface area to cover what Dynamo actually calls. 4. Each fallback branch in `_compat.py` MUST have a comment noting which SGLang version it supports and when it can be removed, e.g.: - `# Fallback for sglang <= 0.5.12. Remove when min supported version is 0.5.14+` + `# Fallback for sglang <= 0.5.16. Remove when min supported version is 0.5.18+` 5. When a new SGLang version is released and the old N-1 falls outside the support window, delete the corresponding fallback branches and polyfills from `_compat.py`. If `_compat.py` becomes trivial re-exports, inline the imports and delete the file. @@ -70,6 +70,11 @@ Worker dispatch (main.py:60-132): have `max_running_requests`, `dllm_algorithm_config`, or other LLM-specific fields. Use `getattr()` when accessing fields that may not exist on the stub. +SGLang 0.5.17 makes a resolved `ServerArgs` unconditionally read-only. Apply Dynamo's +post-resolution startup overrides through `_compat.override_server_args()`; control-plane +updates after engine creation should use the tokenizer manager's update API instead of +assigning fields on `server_args`. + **DynamoConfig** combines `DynamoRuntimeConfig` (common flags like `--namespace`, `--output-modalities`, `--media-output-fs-url`) with `DynamoSGLangConfig` (sglang-specific flags like `--enable-multimodal`, `--dedicated-mm-encoder`, `--embedding-worker`). diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py index d644143adf57..bd242c5cd35e 100644 --- a/components/src/dynamo/sglang/_compat.py +++ b/components/src/dynamo/sglang/_compat.py @@ -38,35 +38,10 @@ def _warn_require_reasoning_unsupported() -> None: ) -# --------------------------------------------------------------------------- -# Top-level sglang exports: Engine, ServerArgs -# -# Some SGLang dev builds (including 0.5.x snapshots) do not re-export these -# from sglang/__init__.py, while Dynamo historically uses `import sglang as sgl` -# followed by `sgl.Engine(...)` throughout this backend. -# --------------------------------------------------------------------------- -def ensure_sglang_top_level_exports() -> None: - """Restore top-level SGLang exports omitted by some install flavors.""" - import sglang as sgl - - if not hasattr(sgl, "Engine"): - from sglang.srt.entrypoints.engine import Engine - - sgl.Engine = Engine - - if not hasattr(sgl, "ServerArgs"): - from sglang.srt.server_args import ServerArgs - - sgl.ServerArgs = ServerArgs - - -ensure_sglang_top_level_exports() - - def ensure_sglang_tensor_image_size() -> None: """Allow SGLang's image-token resolver to handle decoded image tensors. - SGLang 0.5.13 through 0.5.16 assume every decoded image exposes the PIL + SGLang 0.5.13 through 0.5.17 assume every decoded image exposes the PIL ``height``/``width`` attributes. Its CUDA JPEG decoder instead returns a CHW tensor, causing multimodal requests to fall back to retokenization. @@ -106,6 +81,27 @@ def resolve_image_token_counts(self: Any, images: list[Any]) -> list[int]: BaseMultimodalProcessor.resolve_image_token_counts = resolve_image_token_counts +def override_server_args(server_args: Any, source: str, **fields: Any) -> None: + """Apply a post-resolution SGLang configuration update. + + SGLang 0.5.17 makes ``ServerArgs`` unconditionally read-only after + resolution. Both supported CUDA releases expose ``ServerArgs.override`` as + the audited mutation API, so Dynamo must use it instead of assigning fields. + The separately pinned XPU image still uses SGLang 0.5.11, which predates + that API; preserve its legacy assignment behavior until its engine pin is + upgraded. + """ + override = getattr(server_args, "override", None) + if callable(override): + override(source, **fields) + return + + # XPU compatibility for SGLang 0.5.11. Remove when the XPU SGLang pin is + # upgraded to 0.5.16+. + for name, value in fields.items(): + setattr(server_args, name, value) + + @lru_cache(maxsize=32) def _get_async_generate_supported_kwarg_names( async_generate: Any, @@ -138,10 +134,12 @@ def filter_supported_async_generate_kwargs( ) -> dict[str, Any]: """Return only async_generate kwargs accepted by this SGLang engine. - SGLang occasionally adds optional Engine.async_generate kwargs before every - supported install flavor has them. Keep the compatibility boundary narrow: - callers decide which kwargs are optional, and this helper only drops those - optional kwargs when the installed engine cannot accept them. + Both supported CUDA releases accept Dynamo's optional kwargs. The separately + pinned XPU image still uses SGLang 0.5.11, which predates ``mm_hashes`` and + ``require_reasoning``. Keep the compatibility boundary narrow: callers + decide which kwargs are optional, and this helper only drops those optional + kwargs when the installed engine cannot accept them. Remove this filtering + when the XPU SGLang pin is upgraded to 0.5.16+. """ async_generate = engine.async_generate signature_source = getattr(async_generate, "__func__", async_generate) @@ -175,7 +173,7 @@ def require_reasoning_kwargs(engine: Any, request: Mapping[str, Any]) -> dict[st __all__ = [ "ensure_sglang_tensor_image_size", - "ensure_sglang_top_level_exports", "filter_supported_async_generate_kwargs", + "override_server_args", "require_reasoning_kwargs", ] diff --git a/components/src/dynamo/sglang/init_llm.py b/components/src/dynamo/sglang/init_llm.py index 8c0e19af8f27..2279d10481e5 100644 --- a/components/src/dynamo/sglang/init_llm.py +++ b/components/src/dynamo/sglang/init_llm.py @@ -14,6 +14,7 @@ from dynamo.common.utils.endpoint_types import parse_endpoint_types from dynamo.llm import ModelInput, ModelType, WorkerType from dynamo.runtime import DistributedRuntime +from dynamo.sglang._compat import override_server_args from dynamo.sglang.args import Config from dynamo.sglang.health_check import ( SglangDisaggHealthCheckPayload, @@ -71,7 +72,11 @@ async def init_decode( "created before the endpoint existed, so its FPM publisher bound " "a different IPC path than the relay would subscribe to." ) - server_args.enable_forward_pass_metrics = False + override_server_args( + server_args, + "dynamo.snapshot", + enable_forward_pass_metrics=False, + ) else: set_forward_pass_metrics_worker_id(server_args, generate_endpoint) start_time = time.time() @@ -227,7 +232,11 @@ async def init_prefill( "created before the endpoint existed, so its FPM publisher bound " "a different IPC path than the relay would subscribe to." ) - server_args.enable_forward_pass_metrics = False + override_server_args( + server_args, + "dynamo.snapshot", + enable_forward_pass_metrics=False, + ) else: set_forward_pass_metrics_worker_id(server_args, generate_endpoint) start_time = time.time() diff --git a/components/src/dynamo/sglang/init_multimodal.py b/components/src/dynamo/sglang/init_multimodal.py index 5ce1d9e4cef7..9c010e445f0b 100644 --- a/components/src/dynamo/sglang/init_multimodal.py +++ b/components/src/dynamo/sglang/init_multimodal.py @@ -253,7 +253,7 @@ async def init_multimodal_prefill_worker( logging.error(f"Failed to serve endpoints: {e}") raise finally: - handler.cleanup() + await handler.cleanup_async() if run_deferred_handlers is not None: logging.info("Running deferred handlers") await run_deferred_handlers() diff --git a/components/src/dynamo/sglang/main.py b/components/src/dynamo/sglang/main.py index f91bfbfc7c29..91d02e166a95 100644 --- a/components/src/dynamo/sglang/main.py +++ b/components/src/dynamo/sglang/main.py @@ -15,6 +15,7 @@ ) from dynamo.common.utils.runtime import create_runtime from dynamo.runtime.logging import configure_dynamo_logging +from dynamo.sglang._compat import override_server_args from dynamo.sglang.args import parse_args from dynamo.sglang.init_diffusion import ( init_image_diffusion, @@ -44,7 +45,11 @@ async def worker(argv: list[str] | None = None): if config.server_args.load_format == "gms": from gpu_memory_service.integrations.sglang import setup_gms - config.server_args.load_format = setup_gms(config.server_args) + override_server_args( + config.server_args, + "dynamo.gms", + load_format=setup_gms(config.server_args), + ) # Snapshot mode: engine must be created before runtime so CRIU captures no # NATS/etcd connections. diff --git a/components/src/dynamo/sglang/publisher.py b/components/src/dynamo/sglang/publisher.py index 49cb6c38aedd..54892c47f468 100644 --- a/components/src/dynamo/sglang/publisher.py +++ b/components/src/dynamo/sglang/publisher.py @@ -24,6 +24,7 @@ ) from dynamo.llm import KvEventPublisher, WorkerMetricsPublisher from dynamo.runtime import Endpoint +from dynamo.sglang._compat import override_server_args from dynamo.sglang._disagg import SGLANG_WORKER_GROUP_ID_KEY, get_sglang_worker_group_id from dynamo.sglang.args import Config from dynamo.sglang.capacity import ( @@ -49,9 +50,13 @@ def set_forward_pass_metrics_worker_id( import tempfile - server_args.forward_pass_metrics_worker_id = str(generate_endpoint.connection_id()) ipc_path = tempfile.NamedTemporaryFile(delete=False).name - server_args.forward_pass_metrics_ipc_name = f"ipc://{ipc_path}" + override_server_args( + server_args, + "dynamo.forward_pass_metrics", + forward_pass_metrics_worker_id=str(generate_endpoint.connection_id()), + forward_pass_metrics_ipc_name=f"ipc://{ipc_path}", + ) async def _resolve_multinode_leader_worker_id( diff --git a/components/src/dynamo/sglang/request_handlers/handler_base.py b/components/src/dynamo/sglang/request_handlers/handler_base.py index f2dd5a3e5a56..b9a9d1fb27da 100644 --- a/components/src/dynamo/sglang/request_handlers/handler_base.py +++ b/components/src/dynamo/sglang/request_handlers/handler_base.py @@ -871,7 +871,9 @@ async def update_weight_version(self, body: dict) -> dict: if req.abort_all_requests: self.engine.tokenizer_manager.abort_request(abort_all=True) - self.engine.tokenizer_manager.server_args.weight_version = req.new_version + self.engine.tokenizer_manager._update_weight_version_if_provided( + req.new_version + ) return { "success": True, "message": f"Weight version updated to {req.new_version}", diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py index 256514c3d68a..7aa25668bfc3 100644 --- a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py +++ b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py @@ -4,6 +4,7 @@ import asyncio import json import logging +import sys from typing import Any, AsyncIterator, Callable, Literal, Optional, Protocol import sglang as sgl @@ -677,6 +678,8 @@ class MultimodalPrefillWorkerHandler( Processes multimodal inputs and coordinates with decode worker. """ + _REQUEST_REGISTRATION_TIMEOUT_SECONDS = 5.0 + def __init__( self, engine: sgl.Engine, @@ -692,6 +695,7 @@ def __init__( # Get bootstrap info using BootstrapManager self.bootstrap_host, self.bootstrap_port = self._get_bootstrap_info(engine) + self._consume_tasks: set[asyncio.Task[Any]] = set() logger.info( f"Multimodal prefill worker handler initialized - bootstrap host: {self.bootstrap_host}, bootstrap port: {self.bootstrap_port}" @@ -721,21 +725,96 @@ def _end_bootstrap() -> None: # Validate and parse request disagg_request = self._validate_and_parse_disagg_request(disagg_request) - # Generate and return bootstrap info first (like regular SGLang) + rid = context.trace_id or context.id() bootstrap_room = self._generate_bootstrap_room() - bootstrap_info = { - "bootstrap_host": self.bootstrap_host, - "bootstrap_port": self.bootstrap_port, - "bootstrap_room": bootstrap_room, - } - - _end_bootstrap() - yield json.dumps(bootstrap_info) - - # Process prefill generation - await self._process_prefill_generation( - disagg_request, bootstrap_room, context=context + results, tensor_id = await self._start_prefill_or_cancel( + disagg_request, + bootstrap_room, + rid, + context, ) + consumer_owns_tensor = asyncio.Event() + request_started = asyncio.Event() + task: asyncio.Task[Any] | None = None + try: + task = asyncio.create_task( + self._consume_results( + results, + tensor_id, + rid, + context, + consumer_owns_tensor, + request_started, + ) + ) + self._consume_tasks.add(task) + task.add_done_callback(self._consume_tasks.discard) + + started_wait = asyncio.create_task(request_started.wait()) + try: + await asyncio.wait( + (task, started_wait), + return_when=asyncio.FIRST_COMPLETED, + ) + finally: + if not started_wait.done(): + started_wait.cancel() + try: + await started_wait + except asyncio.CancelledError: + pass + + # Surface an immediate submission/cancellation failure before + # decode is authorized. Once consumer_owns_tensor is set, the + # consumer's try/finally owns the transferred tensor. + if task.done(): + await task + # Do not authorize decode until embeddings have been received and + # the result consumer has advanced SGLang's lazy request iterator. + # Otherwise decode can start waiting before prefill is submitted. + bootstrap_info = { + "bootstrap_host": self.bootstrap_host, + "bootstrap_port": self.bootstrap_port, + "bootstrap_room": bootstrap_room, + } + + _end_bootstrap() + yield json.dumps(bootstrap_info) + + await task + except BaseException: + if not consumer_owns_tensor.is_set(): + if task is not None and not task.done(): + task.cancel() + if tensor_id is not None: + self.embeddings_processor.release_embeddings(tensor_id) + raise + finally: + pending_exception = sys.exc_info()[1] + if task is not None: + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + except Exception as task_error: + if pending_exception is None: + raise + if ( + task_error is not pending_exception + and task_error + is not getattr(pending_exception, "__cause__", None) + ): + logger.error( + "Multimodal prefill consumer failed during request " + "cleanup", + exc_info=( + type(task_error), + task_error, + task_error.__traceback__, + ), + ) except Exception as e: logger.error(f"Error in prefill generation: {e}", exc_info=True) @@ -761,13 +840,14 @@ def _validate_and_parse_disagg_request( ) return disagg_request - async def _process_prefill_generation( + async def _start_prefill_generation( self, disagg_request: DisaggSglangMultimodalRequest, bootstrap_room: int, + rid: Optional[str] = None, context=None, - ): - """Process multimodal input and start prefill generation""" + ) -> tuple[AsyncIterator[Any], Optional[int]]: + """Receive multimodal embeddings and submit the prefill to SGLang.""" # Get the SglangMultimodalRequest from the DisaggSglangMultimodalRequest request = disagg_request.request input_ids = request.request.token_ids @@ -787,42 +867,302 @@ async def _process_prefill_generation( context.trace_headers() if context and self.enable_trace else None ) - # Start SGLang prefill generation (like regular SGLang) - with _nvtx.annotate("mm:prefill:engine_async_generate", color="blue"): - gen_params = { - "input_ids": input_ids, - "sampling_params": sampling_params, - "stream": True, - "bootstrap_host": self.bootstrap_host, - "bootstrap_port": self.bootstrap_port, - "bootstrap_room": bootstrap_room, - "external_trace_header": trace_header, - "rid": context.trace_id if context else None, - } + try: + # Start SGLang prefill generation (like regular SGLang) + with _nvtx.annotate("mm:prefill:engine_async_generate", color="blue"): + gen_params = { + "input_ids": input_ids, + "sampling_params": sampling_params, + "stream": True, + "bootstrap_host": self.bootstrap_host, + "bootstrap_port": self.bootstrap_port, + "bootstrap_room": bootstrap_room, + "external_trace_header": trace_header, + "rid": rid, + } + + if image_mm_items: + gen_params["image_data"] = image_mm_items + if video_data: + gen_params["video_data"] = video_data + + results = await self.engine.async_generate(**gen_params) + except BaseException: + if tensor_id is not None: + self.embeddings_processor.release_embeddings(tensor_id) + raise - if image_mm_items: - gen_params["image_data"] = image_mm_items - if video_data: - gen_params["video_data"] = video_data + return results, tensor_id + + async def _start_prefill_or_cancel( + self, + disagg_request: DisaggSglangMultimodalRequest, + bootstrap_room: int, + rid: str, + context: Context, + ) -> tuple[AsyncIterator[Any], Optional[int]]: + """Cancel local preprocessing/submission if the client stops early.""" + start_task = asyncio.create_task( + self._start_prefill_generation( + disagg_request, + bootstrap_room, + rid=rid, + context=context, + ) + ) + cancellation_future = context.async_killed_or_stopped() + try: + done, _ = await asyncio.wait( + (start_task, cancellation_future), + return_when=asyncio.FIRST_COMPLETED, + ) + # Prefer a completed submission if both signals arrive together. The + # registered-request phase will observe the sticky context state and + # abort by the same RID without dropping tensor ownership. + if start_task in done: + return await start_task + + start_task.cancel() + try: + await start_task + except asyncio.CancelledError: + pass + raise asyncio.CancelledError + finally: + if not start_task.done(): + start_task.cancel() + try: + await start_task + except asyncio.CancelledError: + pass + if not cancellation_future.done(): + cancellation_future.cancel() + try: + await cancellation_future + except asyncio.CancelledError: + pass + + async def _wait_for_request_registration(self, rid: str) -> None: + """Wait until SGLang owns the RID, without waiting for engine output.""" + tokenizer_manager = getattr(self.engine, "tokenizer_manager", None) + rid_to_state = getattr(tokenizer_manager, "rid_to_state", None) + if rid_to_state is None: + raise RuntimeError("SGLang tokenizer manager has no request registry") - results = await self.engine.async_generate(**gen_params) + async def poll_registry() -> None: + while rid not in rid_to_state: + await asyncio.sleep(0.001) - # Consume results without yielding (prefill doesn't return text, just coordinates) - asyncio.create_task(self._consume_results(results, tensor_id)) + try: + await asyncio.wait_for( + poll_registry(), + timeout=self._REQUEST_REGISTRATION_TIMEOUT_SECONDS, + ) + except TimeoutError as e: + raise RuntimeError( + f"SGLang did not register prefill request {rid} within " + f"{self._REQUEST_REGISTRATION_TIMEOUT_SECONDS:g}s" + ) from e - async def _consume_results(self, results, tensor_id: Optional[int]): - """Consume prefill results without returning them (like regular SGLang)""" + async def _consume_results( + self, + results, + tensor_id: Optional[int], + rid: str, + context: Context, + owns_tensor: asyncio.Event, + request_started: asyncio.Event, + ) -> None: + """Consume prefill output while honoring request cancellation.""" released = False + request_id_future: asyncio.Future[str] = asyncio.Future() + first_result_task: asyncio.Task[Any] | None = None + next_result_task: asyncio.Task[Any] | None = None + registration_task: asyncio.Task[None] | None = None + pre_registration_cancellation: asyncio.Future[Any] | None = None + + def process_result(result: dict[str, Any]) -> None: + nonlocal released + if tensor_id is not None and not released: + self.embeddings_processor.release_embeddings(tensor_id) + released = True + try: - async for _ in results: - if tensor_id is not None and not released: - self.embeddings_processor.release_embeddings(tensor_id) - released = True + owns_tensor.set() + registration_task = asyncio.create_task( + self._wait_for_request_registration(rid) + ) + first_result_task = asyncio.create_task(anext(results)) + pre_registration_cancellation = context.async_killed_or_stopped() + + first_result: Any = None + first_result_ready = False + while not registration_task.done(): + wait_for: set[asyncio.Future[Any]] = { + registration_task, + pre_registration_cancellation, + } + if first_result_task is not None and not first_result_task.done(): + wait_for.add(first_result_task) + done, _ = await asyncio.wait( + wait_for, + return_when=asyncio.FIRST_COMPLETED, + ) + if registration_task in done: + break + if first_result_task is not None: + first_result_is_done = first_result_task in done + else: + first_result_is_done = False + if first_result_is_done and not first_result_ready: + assert first_result_task is not None + try: + first_result = await first_result_task + except StopAsyncIteration as e: + raise RuntimeError( + "SGLang prefill stream ended before producing a result" + ) from e + finally: + first_result_task = None + first_result_ready = True + if pre_registration_cancellation in done: + raise asyncio.CancelledError + + await registration_task + request_id_future.set_result(rid) + if not pre_registration_cancellation.done(): + pre_registration_cancellation.cancel() + try: + await pre_registration_cancellation + except asyncio.CancelledError: + pass + + async with self._cancellation_monitor( + request_id_future, context + ) as cancellation_task: + request_started.set() + if not first_result_ready: + assert first_result_task is not None + done, _ = await asyncio.wait( + (first_result_task, cancellation_task), + return_when=asyncio.FIRST_COMPLETED, + ) + if cancellation_task in done: + await cancellation_task + raise asyncio.CancelledError + try: + first_result = await first_result_task + except StopAsyncIteration as e: + raise RuntimeError( + "SGLang prefill stream ended before producing a result" + ) from e + finally: + first_result_task = None + process_result(first_result) + + while True: + next_result_task = asyncio.create_task(anext(results)) + done, _ = await asyncio.wait( + (next_result_task, cancellation_task), + return_when=asyncio.FIRST_COMPLETED, + ) + if cancellation_task in done: + if not next_result_task.done(): + next_result_task.cancel() + try: + await next_result_task + except asyncio.CancelledError: + pass + await cancellation_task + raise asyncio.CancelledError + try: + result = await next_result_task + except StopAsyncIteration: + break + finally: + next_result_task = None + process_result(result) finally: + pending_exception = sys.exc_info()[1] + for task in (registration_task, pre_registration_cancellation): + if task is None: + continue + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + except Exception as task_error: + if pending_exception is None: + raise + if task_error is not pending_exception: + logger.error( + "SGLang prefill registration task failed during cleanup", + exc_info=( + type(task_error), + task_error, + task_error.__traceback__, + ), + ) + if next_result_task is not None: + if not next_result_task.done(): + next_result_task.cancel() + try: + await next_result_task + except asyncio.CancelledError: + pass + except Exception as task_error: + if pending_exception is None: + raise + if task_error is not pending_exception: + logger.error( + "SGLang prefill next-result task failed during cleanup", + exc_info=( + type(task_error), + task_error, + task_error.__traceback__, + ), + ) + if first_result_task is not None: + if not first_result_task.done(): + first_result_task.cancel() + try: + await first_result_task + except asyncio.CancelledError: + pass + except Exception as task_error: + if pending_exception is None: + raise + if task_error is not pending_exception: + logger.error( + "SGLang prefill first-result task failed during cleanup", + exc_info=( + type(task_error), + task_error, + task_error.__traceback__, + ), + ) if tensor_id is not None and not released: self.embeddings_processor.release_embeddings(tensor_id) - def cleanup(self): + async def cleanup_async(self) -> None: + tasks = list(self._consume_tasks) + for task in tasks: + if not task.done(): + task.cancel() + if tasks: + results = await asyncio.gather(*tasks, return_exceptions=True) + for result in results: + if isinstance(result, Exception) and not isinstance( + result, asyncio.CancelledError + ): + logger.error( + "Multimodal prefill consumer failed during handler cleanup", + exc_info=(type(result), result, result.__traceback__), + ) + self._consume_tasks.clear() + super().cleanup() self.engine.shutdown() logger.info("Multimodal prefill engine shutdown") diff --git a/components/src/dynamo/sglang/snapshot.py b/components/src/dynamo/sglang/snapshot.py index b280bd9eed4a..eb65065f7b8a 100644 --- a/components/src/dynamo/sglang/snapshot.py +++ b/components/src/dynamo/sglang/snapshot.py @@ -16,6 +16,7 @@ SnapshotConfig, configure_snapshot_capture_env, ) +from dynamo.sglang._compat import override_server_args from .pause import SGLangEnginePauseController @@ -138,7 +139,7 @@ async def prepare_snapshot_engine( # Enable memory_saver so GPU memory can be released for CRIU. # When using GMS, weights use VA-stable unmap/remap (no CPU backup); GMS # forbids enable_weights_cpu_backup. Otherwise use CPU backup for weights. - server_args.enable_memory_saver = True + snapshot_overrides = {"enable_memory_saver": True} try: from gpu_memory_service.integrations.sglang import is_gms_active @@ -146,7 +147,8 @@ async def prepare_snapshot_engine( except ImportError: _using_gms = False if not _using_gms: - server_args.enable_weights_cpu_backup = True + snapshot_overrides["enable_weights_cpu_backup"] = True + override_server_args(server_args, "dynamo.snapshot", **snapshot_overrides) start_time = time.time() engine = sgl.Engine(server_args=server_args) diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py index a62e2258f1f8..bd0e1c805499 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py +++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py @@ -1,7 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import asyncio import importlib +import json +from contextlib import asynccontextmanager +from types import SimpleNamespace import numpy as np import pytest @@ -21,6 +25,7 @@ ) from dynamo.sglang.request_handlers.multimodal.worker_handler import ( EmbeddingsProcessor, + MultimodalPrefillWorkerHandler, _build_mm_items, ) @@ -35,6 +40,35 @@ ] +class _FakeContext: + def __init__(self, context_id: str, trace_id: str | None = None): + self.trace_id = trace_id + self._context_id = context_id + self._cancelled = asyncio.Event() + + def id(self) -> str: + return self._context_id + + def async_killed_or_stopped(self) -> asyncio.Task[bool]: + return asyncio.create_task(self._cancelled.wait()) + + def cancel(self) -> None: + self._cancelled.set() + + +@asynccontextmanager +async def _never_cancels(request_id_future, context): + task = asyncio.create_task(asyncio.Event().wait()) + try: + yield task + finally: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + def test_extract_media_inputs_supports_video_urls(): handler = MultimodalEncodeWorkerHandler.__new__(MultimodalEncodeWorkerHandler) @@ -148,6 +182,239 @@ def create_multimodal_video_item( assert mm_item["video_timestamps"] == [[0.25, 0.75]] +@pytest.mark.asyncio +async def test_multimodal_prefill_starts_before_returning_bootstrap(): + handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler) + handler.bootstrap_host = "prefill-host" + handler.bootstrap_port = 1234 + handler._consume_tasks = set() + handler.embeddings_processor = SimpleNamespace(release_embeddings=lambda _: None) + events = [] + + handler._validate_and_parse_disagg_request = lambda request: request + handler._generate_bootstrap_room = lambda: 17 + + async def start_prefill(request, bootstrap_room, rid=None, context=None): + events.append(("start", bootstrap_room, rid)) + + async def results(): + events.append(("iterate", None)) + yield {"meta_info": {"id": "request-id"}} + + return results(), None + + async def wait_for_registration(rid): + events.append(("registered", rid)) + + handler._start_prefill_generation = start_prefill + handler._wait_for_request_registration = wait_for_registration + handler._cancellation_monitor = _never_cancels + + stream = handler.generate(object(), _FakeContext("request-id")) + bootstrap = json.loads(await anext(stream)) + + assert events == [ + ("start", 17, "request-id"), + ("registered", "request-id"), + ("iterate", None), + ] + assert bootstrap == { + "bootstrap_host": "prefill-host", + "bootstrap_port": 1234, + "bootstrap_room": 17, + } + + with pytest.raises(StopAsyncIteration): + await anext(stream) + assert events == [ + ("start", 17, "request-id"), + ("registered", "request-id"), + ("iterate", None), + ] + + +@pytest.mark.asyncio +async def test_multimodal_prefill_releases_embeddings_when_submission_fails( + monkeypatch, +): + import dynamo.sglang.request_handlers.multimodal.worker_handler as worker_handler + + handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler) + handler.bootstrap_host = "prefill-host" + handler.bootstrap_port = 1234 + handler.enable_trace = False + + released = [] + handler.embeddings_processor = SimpleNamespace(release_embeddings=released.append) + + async def build_mm_items(request, embeddings_processor): + return [], [], torch.empty(0), 23 + + class FailingEngine: + async def async_generate(self, **kwargs): + raise RuntimeError("submission failed") + + handler.engine = FailingEngine() + monkeypatch.setattr(worker_handler, "_build_mm_items", build_mm_items) + + request = SimpleNamespace( + request=SimpleNamespace(request=SimpleNamespace(token_ids=[1, 2, 3])), + sampling_params={"max_new_tokens": 1}, + ) + + with pytest.raises(RuntimeError, match="submission failed"): + await handler._start_prefill_generation(request, 17) + + assert released == [23] + + +@pytest.mark.asyncio +async def test_multimodal_prefill_rejects_empty_engine_stream(): + handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler) + released = [] + handler.embeddings_processor = SimpleNamespace(release_embeddings=released.append) + + async def wait_for_registration(rid): + return None + + handler._wait_for_request_registration = wait_for_registration + handler._cancellation_monitor = _never_cancels + + async def empty_results(): + if False: + yield None + + owns_tensor = asyncio.Event() + request_started = asyncio.Event() + + with pytest.raises(RuntimeError, match="ended before producing a result"): + await handler._consume_results( + empty_results(), + 23, + "request-id", + _FakeContext("request-id"), + owns_tensor, + request_started, + ) + + assert owns_tensor.is_set() + assert request_started.is_set() + assert released == [23] + + +@pytest.mark.asyncio +async def test_multimodal_prefill_cancellation_awaits_consumer_cleanup(): + handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler) + handler.bootstrap_host = "prefill-host" + handler.bootstrap_port = 1234 + handler._consume_tasks = set() + released = [] + handler.embeddings_processor = SimpleNamespace(release_embeddings=released.append) + + handler._validate_and_parse_disagg_request = lambda request: request + handler._generate_bootstrap_room = lambda: 17 + result_stopped = asyncio.Event() + + async def results(): + try: + await asyncio.Event().wait() + yield {} + finally: + result_stopped.set() + + async def start_prefill(request, bootstrap_room, rid=None, context=None): + return results(), 23 + + async def wait_for_registration(rid): + return None + + handler._start_prefill_generation = start_prefill + handler._wait_for_request_registration = wait_for_registration + handler._cancellation_monitor = _never_cancels + + stream = handler.generate(object(), _FakeContext("request-id")) + await anext(stream) + await stream.aclose() + + assert result_stopped.is_set() + assert released == [23] + assert not handler._consume_tasks + + +@pytest.mark.asyncio +async def test_multimodal_prefill_cancels_registered_rid_before_first_result(): + rid = "dynamo-request-id" + abort_calls = [] + result_stopped = asyncio.Event() + + class _TokenizerManager: + def __init__(self): + self.rid_to_state = {} + + def abort_request(self, *, rid, abort_all): + abort_calls.append((rid, abort_all)) + + tokenizer_manager = _TokenizerManager() + handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler) + handler.engine = SimpleNamespace(tokenizer_manager=tokenizer_manager) + handler.shutdown_event = None + handler.embeddings_processor = SimpleNamespace(release_embeddings=lambda _: None) + + async def results(): + tokenizer_manager.rid_to_state[rid] = object() + try: + await asyncio.Event().wait() + yield {} + finally: + result_stopped.set() + + owns_tensor = asyncio.Event() + request_started = asyncio.Event() + context = _FakeContext(rid) + consumer = asyncio.create_task( + handler._consume_results( + results(), + None, + rid, + context, + owns_tensor, + request_started, + ) + ) + + await asyncio.wait_for(request_started.wait(), timeout=1) + context.cancel() + + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(consumer, timeout=1) + + assert abort_calls == [(rid, False)] + assert result_stopped.is_set() + + +@pytest.mark.asyncio +async def test_multimodal_prefill_cleanup_awaits_consumers_before_engine_shutdown(): + handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler) + handler.publisher = None + events = [] + + async def consumer(): + try: + await asyncio.Event().wait() + finally: + events.append("consumer stopped") + + task = asyncio.create_task(consumer()) + await asyncio.sleep(0) + handler._consume_tasks = {task} + handler.engine = SimpleNamespace(shutdown=lambda: events.append("engine shutdown")) + + await handler.cleanup_async() + + assert events == ["consumer stopped", "engine shutdown"] + assert not handler._consume_tasks + + async def test_nvdec_video_metadata_shim_stamps_valid_metadata(): """The metadata shim gives pre-decoded ndarray frames a valid metadata dict. diff --git a/components/src/dynamo/sglang/tests/test_sglang_publisher.py b/components/src/dynamo/sglang/tests/test_sglang_publisher.py index d774b1605ea1..0aea2168f9ee 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_publisher.py +++ b/components/src/dynamo/sglang/tests/test_sglang_publisher.py @@ -51,11 +51,25 @@ def test_get_local_dp_rank_range_respects_multinode_dp_attention(): def test_set_forward_pass_metrics_worker_id_uses_endpoint_identity(): - server_args = SimpleNamespace(enable_forward_pass_metrics=True) + class ReadOnlyServerArgs: + def __init__(self): + object.__setattr__(self, "enable_forward_pass_metrics", True) + object.__setattr__(self, "override_source", None) + + def __setattr__(self, name, value): + raise AttributeError("server args are read-only") + + def override(self, source, **fields): + object.__setattr__(self, "override_source", source) + for name, value in fields.items(): + object.__setattr__(self, name, value) + + server_args = ReadOnlyServerArgs() endpoint = SimpleNamespace(connection_id=lambda: "endpoint-9") set_forward_pass_metrics_worker_id(server_args, endpoint) + assert server_args.override_source == "dynamo.forward_pass_metrics" assert server_args.forward_pass_metrics_worker_id == "endpoint-9" assert server_args.forward_pass_metrics_ipc_name.startswith("ipc://") diff --git a/components/src/dynamo/sglang/tests/test_sglang_unit.py b/components/src/dynamo/sglang/tests/test_sglang_unit.py index f113d9a38ff6..19597fa35f67 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_unit.py +++ b/components/src/dynamo/sglang/tests/test_sglang_unit.py @@ -22,8 +22,8 @@ from dynamo.common.snapshot.constants import SNAPSHOT_CONTROL_DIR_ENV from dynamo.sglang._compat import ( ensure_sglang_tensor_image_size, - ensure_sglang_top_level_exports, filter_supported_async_generate_kwargs, + override_server_args, require_reasoning_kwargs, ) from dynamo.sglang.args import ( @@ -68,6 +68,20 @@ mock_sglang_cli = make_cli_args_fixture("dynamo.sglang") +def test_override_server_args_supports_legacy_xpu_pin(): + server_args = SimpleNamespace(enable_memory_saver=False) + + override_server_args( + server_args, + "dynamo.test", + enable_memory_saver=True, + load_format="legacy-loader", + ) + + assert server_args.enable_memory_saver is True + assert server_args.load_format == "legacy-loader" + + @pytest.fixture(autouse=True) def _cpu_engine_when_no_accelerator(monkeypatch): """Honor the file's gpu_0 contract on hosts with no accelerator. @@ -206,40 +220,6 @@ def _make_sglang_config(**overrides): return config -def test_compat_restores_sglang_top_level_exports(): - """Dynamo supports SGLang builds that omit top-level Engine/ServerArgs.""" - import sglang as sgl - from sglang.srt.entrypoints.engine import Engine - from sglang.srt.server_args import ServerArgs - - missing = object() - original_engine = getattr(sgl, "Engine", missing) - original_server_args = getattr(sgl, "ServerArgs", missing) - - try: - if hasattr(sgl, "Engine"): - delattr(sgl, "Engine") - if hasattr(sgl, "ServerArgs"): - delattr(sgl, "ServerArgs") - - ensure_sglang_top_level_exports() - - assert sgl.Engine is Engine - assert sgl.ServerArgs is ServerArgs - finally: - if original_engine is missing: - if hasattr(sgl, "Engine"): - delattr(sgl, "Engine") - else: - sgl.Engine = original_engine - - if original_server_args is missing: - if hasattr(sgl, "ServerArgs"): - delattr(sgl, "ServerArgs") - else: - sgl.ServerArgs = original_server_args - - def test_compat_supports_tensor_image_sizes_and_is_idempotent(caplog, monkeypatch): from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor, @@ -266,6 +246,10 @@ async def process_mm_data_async(self, *args, **kwargs): processor = object.__new__(ConcreteMultimodalProcessor) processor._processor = Processor() + # SGLang 0.5.17 resolves the processor and tokenizer together before + # handling raw multimodal items. This test stubs the processing path, + # so a tokenizer is not exercised, but the attribute must exist. + processor._tokenizer = None processor.use_cuda_ipc = False image_token_id = 99 processor._process_and_collect_mm_items = lambda **kwargs: ( @@ -581,6 +565,32 @@ async def start_profile(self, request): assert response == {"status": "ok", "message": "Profiling started"} +@pytest.mark.asyncio +async def test_update_weight_version_uses_tokenizer_manager_control_api(): + class TokenizerManager: + updated_version = None + + def _update_weight_version_if_provided(self, version): + self.updated_version = version + + tokenizer_manager = TokenizerManager() + handler = SimpleNamespace( + engine=SimpleNamespace(tokenizer_manager=tokenizer_manager) + ) + + response = await BaseWorkerHandler.update_weight_version( + handler, + {"new_version": "step-42", "abort_all_requests": False}, + ) + + assert tokenizer_manager.updated_version == "step-42" + assert response == { + "success": True, + "message": "Weight version updated to step-42", + "new_version": "step-42", + } + + @pytest.mark.asyncio async def test_custom_jinja_template_invalid_path(mock_sglang_cli): """Test that invalid file path raises FileNotFoundError.""" diff --git a/container/context.yaml b/container/context.yaml index d6708877de13..604d33133265 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -107,7 +107,7 @@ sglang: base_image: nvcr.io/nvidia/cuda-dl-base runtime_image: lmsysorg/sglang base_image_tag: 25.11-cuda13.0-devel-ubuntu24.04 - runtime_image_tag: v0.5.16-cu130-runtime + runtime_image_tag: v0.5.17-cu130-runtime # Baseline is the TRUE base of the third-party lmsysorg/sglang image, # nvidia/cuda:13.0.1-cudnn-devel-ubuntu24.04 (Docker Hub; CUDA_VERSION=13.0.1, # cuDNN 9.13.0.50, ubuntu24.04), NOT a self-baseline of lmsysorg/sglang. @@ -135,7 +135,7 @@ sglang: # (see templates/dev.Dockerfile). Runtime stays on the upstream lmsysorg/sglang # NIXL Python stack — its wheel COPY is narrowed to ai_dynamo*.whl so the SDK # build doesn't leak into the runtime image. - nixl_ref: v1.3.0 + nixl_ref: v1.3.2 enable_media_ffmpeg: "false" enable_gpu_memory_service: "true" enable_kvbm: "false" diff --git a/docs/fern/components/releases.data.ts b/docs/fern/components/releases.data.ts index 531a067791e9..f2030b272e17 100644 --- a/docs/fern/components/releases.data.ts +++ b/docs/fern/components/releases.data.ts @@ -71,10 +71,10 @@ export const CURRENT_TAG = "1.4.0"; export const CURRENT_WHEEL = "1.4.0"; export const MAIN_TOT: BackendPins = { - sglang: "0.5.16", + sglang: "0.5.17", trtllm: "1.3.0rc24", vllm: "0.27.1", - nixlSglang: "1.3.0", + nixlSglang: "1.3.2", nixlTrtllm: "1.3.1", nixlVllm: "1.3.2", }; @@ -1177,7 +1177,7 @@ export const FEATURE_INTERACTIONS: BackendInteractions[] = [ // KV Block Manager [{ status: "wip" }, { status: "wip" }, { status: "wip" }, { status: "na" }], // Multimodal - [{ status: "yes", label: "Supported serving patterns", note: "Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported.", source: "/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal" }, { status: "yes", label: "Image-aware routing on Dynamo's SGLang image", note: "Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.16, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing.", source: "/dynamo/dev/multimodal/multimodal-kv-routing" }, { status: "na" }, { status: "wip" }, { status: "na" }], + [{ status: "yes", label: "Supported serving patterns", note: "Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported.", source: "/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal" }, { status: "yes", label: "Image-aware routing on Dynamo's SGLang image", note: "Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.17, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing.", source: "/dynamo/dev/multimodal/multimodal-kv-routing" }, { status: "na" }, { status: "wip" }, { status: "na" }], // Request Migration [{ status: "yes" }, { status: "yes" }, { status: "yes" }, { status: "wip" }, { status: "yes" }, { status: "na" }], // Request Cancellation diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md index 50ea8014c771..12da98dbc3ef 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md @@ -122,7 +122,7 @@ The launcher configures KV events on each worker and sets `--router-mode kv` wit The Dynamo SGLang image includes both routing prerequisites: - Dynamo is built with the `mm-routing` Rust feature. -- SGLang 0.5.13 or later includes `GenerateReqInput.mm_hashes` support. Dynamo currently pins 0.5.16. +- SGLang 0.5.13 or later includes `GenerateReqInput.mm_hashes` support. Dynamo currently pins 0.5.17. Custom installations on SGLang 0.5.12 or earlier need the `mm_hashes` change from [sgl-project/sglang#25300](https://github.com/sgl-project/sglang/pull/25300). diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/observability.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/observability.md index 77f76baf9c98..2a992dda6719 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/observability.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/observability.md @@ -115,7 +115,7 @@ For the complete and authoritative list of all SGLang metrics, see the [official ## Forward Pass Metrics (FPM) -> **Availability.** Forward Pass Metrics require SGLang's upstream `sglang.srt.observability.forward_pass_metrics` module and the `ServerArgs` fields (`enable_forward_pass_metrics`, `forward_pass_metrics_worker_id`, `forward_pass_metrics_ipc_name`). These landed in **SGLang v0.5.13** and ship in the current Dynamo runtime (`sglang==0.5.15`), so setting `DYN_FORWARDPASS_METRIC_PORT` enables SGLang-side FPM emission to the NATS event plane via `FpmEventRelay`. The wire-format contract is guarded by `dynamo/sglang/tests/test_fpm_contract.py`. On runtimes older than v0.5.13 the module is absent: the Dynamo-side relay still starts and the worker serves normally, but no SGLang-side FPM payloads are emitted. +> **Availability.** Forward Pass Metrics require SGLang's upstream `sglang.srt.observability.forward_pass_metrics` module and the `ServerArgs` fields (`enable_forward_pass_metrics`, `forward_pass_metrics_worker_id`, `forward_pass_metrics_ipc_name`). These landed in **SGLang v0.5.13** and ship in the current CUDA Dynamo runtime (`sglang==0.5.17`), so setting `DYN_FORWARDPASS_METRIC_PORT` enables SGLang-side FPM emission to the NATS event plane via `FpmEventRelay`. The wire-format contract is guarded by `dynamo/sglang/tests/test_fpm_contract.py`. On runtimes older than v0.5.13 the module is absent: the Dynamo-side relay still starts and the worker serves normally, but no SGLang-side FPM payloads are emitted. Forward Pass Metrics provide **per-iteration scheduler telemetry** pushed over ZMQ, giving the [Planner](../../planner/overview.md) real-time visibility into batch composition, queue depth, and GPU forward pass duration. Unlike Prometheus metrics (which are scraped asynchronously and reflect only the latest gauge value), FPM emits a structured message after every scheduler iteration with the exact batch state. diff --git a/docs/fern/pages/reference/general/compatibility.mdx b/docs/fern/pages/reference/general/compatibility.mdx index 7f243d6e2283..a01fbccd0e2f 100644 --- a/docs/fern/pages/reference/general/compatibility.mdx +++ b/docs/fern/pages/reference/general/compatibility.mdx @@ -156,7 +156,7 @@ Current stable release: v1.4.0 (container tag `1.4.0`, wheel version `1.4.0`). | Dynamo | Type | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX | | --- | --- | --- | --- | --- | --- | --- | -| main (ToT) | development head | 0.5.16 | 1.3.0rc24 | 0.27.1 | 1.3.0 / 1.3.1 / 1.3.2 | - | +| main (ToT) | development head | 0.5.17 | 1.3.0rc24 | 0.27.1 | 1.3.2 / 1.3.1 / 1.3.2 | - | | v1.4.0 | stable | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | | v1.3.1 | patch | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.2 / 1.0.1 / 1.1.0 | 1.20.x | | v1.3.0 | stable | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.0 / 1.0.1 / 1.1.0 | 1.20.x | @@ -306,10 +306,10 @@ Each cell states whether the row feature works together with the column feature. | Feature | Disaggregated Serving | KV-Aware Routing | SLA-Based Planner | KV Block Manager | Multimodal | Request Migration | Request Cancellation | LoRA | Tool Calling | Speculative Decoding | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | Disaggregated Serving | n/a | Yes | Yes | Experimental | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes | Experimental — Cancellation during remote prefill is not supported in disaggregated mode. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental — Code hooks exist, but examples and documentation are not yet available. | -| KV-Aware Routing | Yes | n/a | Yes | Experimental | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.16, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | Yes | Yes | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental | +| KV-Aware Routing | Yes | n/a | Yes | Experimental | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.17, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | Yes | Yes | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental | | SLA-Based Planner | Yes | Yes | n/a | Experimental | n/a | Yes | Yes | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a | | KV Block Manager | Experimental | Experimental | Experimental | n/a | Experimental | Experimental | Experimental | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Experimental | Experimental | -| Multimodal | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.16, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | n/a | Experimental | n/a | Yes | Experimental | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a | +| Multimodal | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.17, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | n/a | Experimental | n/a | Yes | Experimental | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a | | Request Migration | Yes | Yes | Yes | Experimental | Yes | n/a | Yes | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental | | Request Cancellation | Experimental — Cancellation during remote prefill is not supported in disaggregated mode. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Yes | Experimental | Experimental | Yes | n/a | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a | | LoRA | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | n/a | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | diff --git a/docs/fern/pages/reference/general/releases-machine-readable.mdx b/docs/fern/pages/reference/general/releases-machine-readable.mdx index 0299a753c8fa..2c6241aa2e5b 100644 --- a/docs/fern/pages/reference/general/releases-machine-readable.mdx +++ b/docs/fern/pages/reference/general/releases-machine-readable.mdx @@ -15,7 +15,7 @@ Current stable release: v1.4.0 (Aug 14, 2026; container tag `1.4.0`, wheel versi | Version | Kind | Date | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX | Notes | Delta | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| main (ToT) | development head | - | 0.5.16 | 1.3.0rc24 | 0.27.1 | 1.3.0 / 1.3.1 / 1.3.2 | - | - | - | +| main (ToT) | development head | - | 0.5.17 | 1.3.0rc24 | 0.27.1 | 1.3.2 / 1.3.1 / 1.3.2 | - | - | - | | v1.4.0 | stable | Aug 14, 2026 | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-4-0) | Audit subsystem migrated into request trace (DYN_AUDIT_* honored as legacy aliases); HTTP header capture in trace records is an explicit fail-closed allowlist; deprecated multimodal worker flags and vLLM worker-role flags removed; runtime images no longer bundle software video decoders (H.264/H.265 decodes via NVDEC); UCX 1.21.x. | | v1.3.1 | patch | Aug 5, 2026 | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.2 / 1.0.1 / 1.1.0 | 1.20.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-3-0#v131) | Patch release. Fixes disaggregated SGLang serving over AWS EFA on GB200: the SGLang EFA runtime moves to NIXL 1.3.2 and all three EFA images to EFA Installer 1.49.0. Backend pins are otherwise unchanged from v1.3.0. | | v1.3.0 | stable | Jul 20, 2026 | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.0 / 1.0.1 / 1.1.0 | 1.20.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-3-0) | CUDA 12 container images discontinued; EFA variants retagged from -efa-amd64 to -efa (the images were already multi-arch — the old suffix was misleading); GA wheels published as 1.3.0.post1 (containers stay :1.3.0); UCX 1.20.x. | diff --git a/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md b/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md index 6787ca42eff7..9a16e907d127 100644 --- a/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md +++ b/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md @@ -113,5 +113,5 @@ URI string, while frontend decoding hashes the decoded bytes. |---------|--------------|--------|-------| | [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Supported families include Qwen2-VL, Qwen2.5-VL, Qwen3-VL, LLaVA 1.5, LLaVA-NeXT, Llama 4, Kimi K2.5/K2.6, Qwen3.5, and Qwen3.6. The rest use text-prefix-only routing. | | [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md#multimodal-kv-routing) | Python chat processor | Yes | Uses vLLM’s own multimodal processor — supports any VLM that vLLM supports. | -| [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Hash forwarding is upstream in SGLang 0.5.13+; Dynamo pins 0.5.16. Older custom installations need the upstream patch. | +| [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Hash forwarding is upstream in SGLang 0.5.13+; Dynamo pins 0.5.17. Older custom installations need the upstream patch. | | [TensorRT-LLM](../../developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Supported model scope is the Qwen2-VL family (Qwen2-VL / Qwen2.5-VL / Qwen3-VL) and Kimi (Kimi-K2.5 / Kimi-K2.6). Other multimodal models fall back to text-prefix routing. | diff --git a/examples/backends/sglang/launch/agg_agent.sh b/examples/backends/sglang/launch/agg_agent.sh index 8b25d8ebdff6..c0f4b25beddb 100755 --- a/examples/backends/sglang/launch/agg_agent.sh +++ b/examples/backends/sglang/launch/agg_agent.sh @@ -72,7 +72,6 @@ python3 -m dynamo.sglang \ --tp "$TP" \ --trust-remote-code \ --radix-eviction-policy priority \ - --skip-tokenizer-init \ --dyn-reasoning-parser glm45 \ --dyn-tool-call-parser glm47 \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' \ diff --git a/lib/gpu_memory_service/integrations/sglang/__init__.py b/lib/gpu_memory_service/integrations/sglang/__init__.py index 28a460212bc6..cb073ce0af82 100644 --- a/lib/gpu_memory_service/integrations/sglang/__init__.py +++ b/lib/gpu_memory_service/integrations/sglang/__init__.py @@ -7,7 +7,8 @@ from gpu_memory_service.integrations.sglang import setup_gms if server_args.load_format == "gms": - server_args.load_format = setup_gms(server_args) + load_format = setup_gms(server_args) + server_args.override("dynamo.gms", load_format=load_format) """ from __future__ import annotations @@ -57,7 +58,13 @@ def setup_gms(server_args) -> Type["GMSModelLoader"]: "Cannot use --enable-draft-weights-cpu-backup with --load-format gms." ) - server_args.enable_memory_saver = True + override = getattr(server_args, "override", None) + if callable(override): + override("dynamo.gms", enable_memory_saver=True) + else: + # The separately pinned XPU image still uses SGLang 0.5.11, which + # predates ServerArgs.override. Remove after that pin reaches 0.5.16+. + server_args.enable_memory_saver = True # Resolve lock mode and RO reconnect timeout from model_loader_extra_config # before patches fire. global _gms_lock_mode diff --git a/lib/sidecar/sglang/README.md b/lib/sidecar/sglang/README.md index de7c1f541702..246b1ede8a0c 100644 --- a/lib/sidecar/sglang/README.md +++ b/lib/sidecar/sglang/README.md @@ -61,8 +61,8 @@ There is no published sidecar image yet, so you build and push your own from > [!NOTE] > The engine image must be a stock SGLang **v0.5.16+** build: the native gRPC -> server (`--grpc-port`) landed there, and Dynamo's `sglang-runtime` pins an -> older SGLang without it. `deploy/agg.yaml` uses `lmsysorg/sglang:v0.5.16`. +> server (`--grpc-port`) landed there. The deployment examples use +> `lmsysorg/sglang:v0.5.17`, matching Dynamo main's SGLang pin. ### Prerequisites diff --git a/lib/sidecar/sglang/deploy/agg.yaml b/lib/sidecar/sglang/deploy/agg.yaml index 375c1662f0e5..8899d0de220f 100644 --- a/lib/sidecar/sglang/deploy/agg.yaml +++ b/lib/sidecar/sglang/deploy/agg.yaml @@ -38,10 +38,10 @@ spec: spec: # sglang-engine as a native sidecar (initContainer + restartPolicy: Always): # starts before `main`, stops after. Native gRPC `--grpc-port` needs SGLang - # v0.5.16+ (Dynamo's sglang-runtime pins an older SGLang without it). + # v0.5.16+; keep this example aligned with Dynamo main's SGLang pin. initContainers: - name: sglang-engine - image: lmsysorg/sglang:v0.5.16 + image: lmsysorg/sglang:v0.5.17 restartPolicy: Always # gRPC HealthCheck via exec (tcpSocket/grpc: dial the pod IP, not the # loopback engine). startupProbe gates `main`; readinessProbe drops dead workers. diff --git a/lib/sidecar/sglang/deploy/disagg.yaml b/lib/sidecar/sglang/deploy/disagg.yaml index a20428d1da06..4142966c1357 100644 --- a/lib/sidecar/sglang/deploy/disagg.yaml +++ b/lib/sidecar/sglang/deploy/disagg.yaml @@ -52,7 +52,7 @@ spec: sizeLimit: 10Gi initContainers: - name: sglang-engine - image: lmsysorg/sglang:v0.5.16 + image: lmsysorg/sglang:v0.5.17 restartPolicy: Always securityContext: capabilities: @@ -140,7 +140,7 @@ spec: sizeLimit: 10Gi initContainers: - name: sglang-engine - image: lmsysorg/sglang:v0.5.16 + image: lmsysorg/sglang:v0.5.17 restartPolicy: Always securityContext: capabilities: diff --git a/pyproject.toml b/pyproject.toml index 9e1e8fad1ac5..d1d3d4fbc64b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,11 +78,11 @@ vllm = [ sglang = [ "uvloop", - "sglang[diffusion]==0.5.16", + "sglang[diffusion]==0.5.17", # sglang[diffusion] dropped accelerate in 0.5.12; diffusers still needs it. "accelerate>=0.17.0", "blake3>=1.0.0,<2.0.0", - "nixl[cu13]==1.3.0", + "nixl[cu13]==1.3.2", "cupy-cuda12x>=13.0.0", ]