From 3f3e1267e7869941c87fb465938cbbb7a7846e86 Mon Sep 17 00:00:00 2001 From: jain-ria Date: Sun, 16 Aug 2026 18:58:46 -0700 Subject: [PATCH 01/11] build(sglang): bump runtime to 0.5.17 Signed-off-by: jain-ria --- components/src/dynamo/sglang/CLAUDE.md | 2 +- components/src/dynamo/sglang/_compat.py | 2 +- components/src/dynamo/sglang/tests/test_sglang_unit.py | 4 ++++ container/context.yaml | 2 +- docs/fern/components/releases.data.ts | 4 ++-- .../modular-components/backends/sglang/multimodal.md | 2 +- docs/fern/pages/reference/general/compatibility.mdx | 7 +++---- .../pages/reference/general/releases-machine-readable.mdx | 2 +- .../use-cases/multimodal-serving/multimodal-kv-routing.md | 2 +- pyproject.toml | 2 +- 10 files changed, 16 insertions(+), 13 deletions(-) diff --git a/components/src/dynamo/sglang/CLAUDE.md b/components/src/dynamo/sglang/CLAUDE.md index e0e8689ca0fa..9f6f923ca237 100644 --- a/components/src/dynamo/sglang/CLAUDE.md +++ b/components/src/dynamo/sglang/CLAUDE.md @@ -20,7 +20,7 @@ support the current version plus 1 version back (N and N-1). The pattern: enough surface area to cover what Dynamo actually calls. 4. Each fallback branch in `_compat.py` MUST have a comment noting which SGLang version it supports and when it can be removed, e.g.: - `# Fallback for sglang <= 0.5.12. Remove when min supported version is 0.5.14+` + `# Fallback for sglang <= 0.5.16. Remove when min supported version is 0.5.18+` 5. When a new SGLang version is released and the old N-1 falls outside the support window, delete the corresponding fallback branches and polyfills from `_compat.py`. If `_compat.py` becomes trivial re-exports, inline the imports and delete the file. diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py index d644143adf57..3cf111012e5b 100644 --- a/components/src/dynamo/sglang/_compat.py +++ b/components/src/dynamo/sglang/_compat.py @@ -66,7 +66,7 @@ def ensure_sglang_top_level_exports() -> None: def ensure_sglang_tensor_image_size() -> None: """Allow SGLang's image-token resolver to handle decoded image tensors. - SGLang 0.5.13 through 0.5.16 assume every decoded image exposes the PIL + SGLang 0.5.13 through 0.5.17 assume every decoded image exposes the PIL ``height``/``width`` attributes. Its CUDA JPEG decoder instead returns a CHW tensor, causing multimodal requests to fall back to retokenization. diff --git a/components/src/dynamo/sglang/tests/test_sglang_unit.py b/components/src/dynamo/sglang/tests/test_sglang_unit.py index f113d9a38ff6..c7a646cd142d 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_unit.py +++ b/components/src/dynamo/sglang/tests/test_sglang_unit.py @@ -266,6 +266,10 @@ async def process_mm_data_async(self, *args, **kwargs): processor = object.__new__(ConcreteMultimodalProcessor) processor._processor = Processor() + # SGLang 0.5.17 resolves the processor and tokenizer together before + # handling raw multimodal items. This test stubs the processing path, + # so a tokenizer is not exercised, but the attribute must exist. + processor._tokenizer = None processor.use_cuda_ipc = False image_token_id = 99 processor._process_and_collect_mm_items = lambda **kwargs: ( diff --git a/container/context.yaml b/container/context.yaml index b4ac92939be1..64d5e008c3eb 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -104,7 +104,7 @@ sglang: base_image: nvcr.io/nvidia/cuda-dl-base runtime_image: lmsysorg/sglang base_image_tag: 25.11-cuda13.0-devel-ubuntu24.04 - runtime_image_tag: v0.5.16-cu130-runtime + runtime_image_tag: v0.5.17-cu130-runtime # Baseline is the TRUE base of the third-party lmsysorg/sglang image, # nvidia/cuda:13.0.1-cudnn-devel-ubuntu24.04 (Docker Hub; CUDA_VERSION=13.0.1, # cuDNN 9.13.0.50, ubuntu24.04), NOT a self-baseline of lmsysorg/sglang. diff --git a/docs/fern/components/releases.data.ts b/docs/fern/components/releases.data.ts index 464a4ad2cc1d..0364e363a09b 100644 --- a/docs/fern/components/releases.data.ts +++ b/docs/fern/components/releases.data.ts @@ -71,7 +71,7 @@ export const CURRENT_TAG = "1.4.0"; export const CURRENT_WHEEL = "1.4.0"; export const MAIN_TOT: BackendPins = { - sglang: "0.5.16", + sglang: "0.5.17", trtllm: "1.3.0rc23", vllm: "0.26.0", nixlSglang: "1.3.0", @@ -1177,7 +1177,7 @@ export const FEATURE_INTERACTIONS: BackendInteractions[] = [ // KV Block Manager [{ status: "wip" }, { status: "wip" }, { status: "wip" }, { status: "na" }], // Multimodal - [{ status: "yes", label: "Supported serving patterns", note: "Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported.", source: "/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal" }, { status: "yes", label: "Image-aware routing on Dynamo's SGLang image", note: "Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.16, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing.", source: "/dynamo/dev/multimodal/multimodal-kv-routing" }, { status: "na" }, { status: "wip" }, { status: "na" }], + [{ status: "yes", label: "Supported serving patterns", note: "Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported.", source: "/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal" }, { status: "yes", label: "Image-aware routing on Dynamo's SGLang image", note: "Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.17, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing.", source: "/dynamo/dev/multimodal/multimodal-kv-routing" }, { status: "na" }, { status: "wip" }, { status: "na" }], // Request Migration [{ status: "yes" }, { status: "yes" }, { status: "yes" }, { status: "wip" }, { status: "yes" }, { status: "na" }], // Request Cancellation diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md index 50ea8014c771..12da98dbc3ef 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md @@ -122,7 +122,7 @@ The launcher configures KV events on each worker and sets `--router-mode kv` wit The Dynamo SGLang image includes both routing prerequisites: - Dynamo is built with the `mm-routing` Rust feature. -- SGLang 0.5.13 or later includes `GenerateReqInput.mm_hashes` support. Dynamo currently pins 0.5.16. +- SGLang 0.5.13 or later includes `GenerateReqInput.mm_hashes` support. Dynamo currently pins 0.5.17. Custom installations on SGLang 0.5.12 or earlier need the `mm_hashes` change from [sgl-project/sglang#25300](https://github.com/sgl-project/sglang/pull/25300). diff --git a/docs/fern/pages/reference/general/compatibility.mdx b/docs/fern/pages/reference/general/compatibility.mdx index b70ca1c4d46e..f5817989aefe 100644 --- a/docs/fern/pages/reference/general/compatibility.mdx +++ b/docs/fern/pages/reference/general/compatibility.mdx @@ -156,7 +156,7 @@ Current stable release: v1.4.0 (container tag `1.4.0`, wheel version `1.4.0`). | Dynamo | Type | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX | | --- | --- | --- | --- | --- | --- | --- | -| main (ToT) | development head | 0.5.16 | 1.3.0rc23 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | - | +| main (ToT) | development head | 0.5.17 | 1.3.0rc23 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | - | | v1.4.0 | stable | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | | v1.3.1 | patch | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.2 / 1.0.1 / 1.1.0 | 1.20.x | | v1.3.0 | stable | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.0 / 1.0.1 / 1.1.0 | 1.20.x | @@ -306,10 +306,10 @@ Each cell states whether the row feature works together with the column feature. | Feature | Disaggregated Serving | KV-Aware Routing | SLA-Based Planner | KV Block Manager | Multimodal | Request Migration | Request Cancellation | LoRA | Tool Calling | Speculative Decoding | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | Disaggregated Serving | n/a | Yes | Yes | Experimental | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes | Experimental — Cancellation during remote prefill is not supported in disaggregated mode. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental — Code hooks exist, but examples and documentation are not yet available. | -| KV-Aware Routing | Yes | n/a | Yes | Experimental | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.16, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | Yes | Yes | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental | +| KV-Aware Routing | Yes | n/a | Yes | Experimental | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.17, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | Yes | Yes | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental | | SLA-Based Planner | Yes | Yes | n/a | Experimental | n/a | Yes | Yes | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a | | KV Block Manager | Experimental | Experimental | Experimental | n/a | Experimental | Experimental | Experimental | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Experimental | Experimental | -| Multimodal | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.16, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | n/a | Experimental | n/a | Yes | Experimental | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a | +| Multimodal | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.17, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | n/a | Experimental | n/a | Yes | Experimental | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a | | Request Migration | Yes | Yes | Yes | Experimental | Yes | n/a | Yes | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental | | Request Cancellation | Experimental — Cancellation during remote prefill is not supported in disaggregated mode. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Yes | Experimental | Experimental | Yes | n/a | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a | | LoRA | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | n/a | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | No — SGLang does not support LoRA in Dynamo, so every LoRA pairing is unsupported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | @@ -342,4 +342,3 @@ Each cell states whether the row feature works together with the column feature. {/* llms-tables:end */} - diff --git a/docs/fern/pages/reference/general/releases-machine-readable.mdx b/docs/fern/pages/reference/general/releases-machine-readable.mdx index 3f276f464f8a..2f0a477053cc 100644 --- a/docs/fern/pages/reference/general/releases-machine-readable.mdx +++ b/docs/fern/pages/reference/general/releases-machine-readable.mdx @@ -15,7 +15,7 @@ Current stable release: v1.4.0 (Aug 14, 2026; container tag `1.4.0`, wheel versi | Version | Kind | Date | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX | Notes | Delta | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| main (ToT) | development head | - | 0.5.16 | 1.3.0rc23 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | - | - | - | +| main (ToT) | development head | - | 0.5.17 | 1.3.0rc23 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | - | - | - | | v1.4.0 | stable | Aug 14, 2026 | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-4-0) | Audit subsystem migrated into request trace (DYN_AUDIT_* honored as legacy aliases); HTTP header capture in trace records is an explicit fail-closed allowlist; deprecated multimodal worker flags and vLLM worker-role flags removed; runtime images no longer bundle software video decoders (H.264/H.265 decodes via NVDEC); UCX 1.21.x. | | v1.3.1 | patch | Aug 5, 2026 | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.2 / 1.0.1 / 1.1.0 | 1.20.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-3-0#v131) | Patch release. Fixes disaggregated SGLang serving over AWS EFA on GB200: the SGLang EFA runtime moves to NIXL 1.3.2 and all three EFA images to EFA Installer 1.49.0. Backend pins are otherwise unchanged from v1.3.0. | | v1.3.0 | stable | Jul 20, 2026 | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.0 / 1.0.1 / 1.1.0 | 1.20.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-3-0) | CUDA 12 container images discontinued; EFA variants retagged from -efa-amd64 to -efa (the images were already multi-arch — the old suffix was misleading); GA wheels published as 1.3.0.post1 (containers stay :1.3.0); UCX 1.20.x. | diff --git a/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md b/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md index 6787ca42eff7..9a16e907d127 100644 --- a/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md +++ b/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md @@ -113,5 +113,5 @@ URI string, while frontend decoding hashes the decoded bytes. |---------|--------------|--------|-------| | [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Supported families include Qwen2-VL, Qwen2.5-VL, Qwen3-VL, LLaVA 1.5, LLaVA-NeXT, Llama 4, Kimi K2.5/K2.6, Qwen3.5, and Qwen3.6. The rest use text-prefix-only routing. | | [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md#multimodal-kv-routing) | Python chat processor | Yes | Uses vLLM’s own multimodal processor — supports any VLM that vLLM supports. | -| [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Hash forwarding is upstream in SGLang 0.5.13+; Dynamo pins 0.5.16. Older custom installations need the upstream patch. | +| [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Hash forwarding is upstream in SGLang 0.5.13+; Dynamo pins 0.5.17. Older custom installations need the upstream patch. | | [TensorRT-LLM](../../developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Supported model scope is the Qwen2-VL family (Qwen2-VL / Qwen2.5-VL / Qwen3-VL) and Kimi (Kimi-K2.5 / Kimi-K2.6). Other multimodal models fall back to text-prefix routing. | diff --git a/pyproject.toml b/pyproject.toml index 591aee30c0c3..feadda90f18a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ vllm = [ sglang = [ "uvloop", - "sglang[diffusion]==0.5.16", + "sglang[diffusion]==0.5.17", # sglang[diffusion] dropped accelerate in 0.5.12; diffusers still needs it. "accelerate>=0.17.0", "blake3>=1.0.0,<2.0.0", From f3777acba589fa46997cbd272ae96090efd70cfe Mon Sep 17 00:00:00 2001 From: jain-ria Date: Mon, 17 Aug 2026 09:15:31 -0700 Subject: [PATCH 02/11] fix(sglang): support read-only server args Signed-off-by: jain-ria --- components/src/dynamo/sglang/CLAUDE.md | 5 ++++ components/src/dynamo/sglang/_compat.py | 11 ++++++++ components/src/dynamo/sglang/init_llm.py | 13 ++++++++-- components/src/dynamo/sglang/main.py | 7 ++++- components/src/dynamo/sglang/publisher.py | 9 +++++-- .../sglang/request_handlers/handler_base.py | 4 ++- components/src/dynamo/sglang/snapshot.py | 6 +++-- .../sglang/tests/test_sglang_publisher.py | 16 +++++++++++- .../dynamo/sglang/tests/test_sglang_unit.py | 26 +++++++++++++++++++ 9 files changed, 88 insertions(+), 9 deletions(-) diff --git a/components/src/dynamo/sglang/CLAUDE.md b/components/src/dynamo/sglang/CLAUDE.md index 9f6f923ca237..802f13b53274 100644 --- a/components/src/dynamo/sglang/CLAUDE.md +++ b/components/src/dynamo/sglang/CLAUDE.md @@ -70,6 +70,11 @@ Worker dispatch (main.py:60-132): have `max_running_requests`, `dllm_algorithm_config`, or other LLM-specific fields. Use `getattr()` when accessing fields that may not exist on the stub. +SGLang 0.5.17 makes a resolved `ServerArgs` unconditionally read-only. Apply Dynamo's +post-resolution startup overrides through `_compat.override_server_args()`; control-plane +updates after engine creation should use the tokenizer manager's update API instead of +assigning fields on `server_args`. + **DynamoConfig** combines `DynamoRuntimeConfig` (common flags like `--namespace`, `--output-modalities`, `--media-output-fs-url`) with `DynamoSGLangConfig` (sglang-specific flags like `--enable-multimodal`, `--dedicated-mm-encoder`, `--embedding-worker`). diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py index 3cf111012e5b..d8ca03b1d41f 100644 --- a/components/src/dynamo/sglang/_compat.py +++ b/components/src/dynamo/sglang/_compat.py @@ -106,6 +106,16 @@ def resolve_image_token_counts(self: Any, images: list[Any]) -> list[int]: BaseMultimodalProcessor.resolve_image_token_counts = resolve_image_token_counts +def override_server_args(server_args: Any, source: str, **fields: Any) -> None: + """Apply a post-resolution SGLang configuration update. + + SGLang 0.5.17 makes ``ServerArgs`` unconditionally read-only after + resolution. Both supported releases expose ``ServerArgs.override`` as the + audited mutation API, so Dynamo must use it instead of assigning fields. + """ + server_args.override(source, **fields) + + @lru_cache(maxsize=32) def _get_async_generate_supported_kwarg_names( async_generate: Any, @@ -177,5 +187,6 @@ def require_reasoning_kwargs(engine: Any, request: Mapping[str, Any]) -> dict[st "ensure_sglang_tensor_image_size", "ensure_sglang_top_level_exports", "filter_supported_async_generate_kwargs", + "override_server_args", "require_reasoning_kwargs", ] diff --git a/components/src/dynamo/sglang/init_llm.py b/components/src/dynamo/sglang/init_llm.py index 8c0e19af8f27..2279d10481e5 100644 --- a/components/src/dynamo/sglang/init_llm.py +++ b/components/src/dynamo/sglang/init_llm.py @@ -14,6 +14,7 @@ from dynamo.common.utils.endpoint_types import parse_endpoint_types from dynamo.llm import ModelInput, ModelType, WorkerType from dynamo.runtime import DistributedRuntime +from dynamo.sglang._compat import override_server_args from dynamo.sglang.args import Config from dynamo.sglang.health_check import ( SglangDisaggHealthCheckPayload, @@ -71,7 +72,11 @@ async def init_decode( "created before the endpoint existed, so its FPM publisher bound " "a different IPC path than the relay would subscribe to." ) - server_args.enable_forward_pass_metrics = False + override_server_args( + server_args, + "dynamo.snapshot", + enable_forward_pass_metrics=False, + ) else: set_forward_pass_metrics_worker_id(server_args, generate_endpoint) start_time = time.time() @@ -227,7 +232,11 @@ async def init_prefill( "created before the endpoint existed, so its FPM publisher bound " "a different IPC path than the relay would subscribe to." ) - server_args.enable_forward_pass_metrics = False + override_server_args( + server_args, + "dynamo.snapshot", + enable_forward_pass_metrics=False, + ) else: set_forward_pass_metrics_worker_id(server_args, generate_endpoint) start_time = time.time() diff --git a/components/src/dynamo/sglang/main.py b/components/src/dynamo/sglang/main.py index f91bfbfc7c29..91d02e166a95 100644 --- a/components/src/dynamo/sglang/main.py +++ b/components/src/dynamo/sglang/main.py @@ -15,6 +15,7 @@ ) from dynamo.common.utils.runtime import create_runtime from dynamo.runtime.logging import configure_dynamo_logging +from dynamo.sglang._compat import override_server_args from dynamo.sglang.args import parse_args from dynamo.sglang.init_diffusion import ( init_image_diffusion, @@ -44,7 +45,11 @@ async def worker(argv: list[str] | None = None): if config.server_args.load_format == "gms": from gpu_memory_service.integrations.sglang import setup_gms - config.server_args.load_format = setup_gms(config.server_args) + override_server_args( + config.server_args, + "dynamo.gms", + load_format=setup_gms(config.server_args), + ) # Snapshot mode: engine must be created before runtime so CRIU captures no # NATS/etcd connections. diff --git a/components/src/dynamo/sglang/publisher.py b/components/src/dynamo/sglang/publisher.py index 49cb6c38aedd..54892c47f468 100644 --- a/components/src/dynamo/sglang/publisher.py +++ b/components/src/dynamo/sglang/publisher.py @@ -24,6 +24,7 @@ ) from dynamo.llm import KvEventPublisher, WorkerMetricsPublisher from dynamo.runtime import Endpoint +from dynamo.sglang._compat import override_server_args from dynamo.sglang._disagg import SGLANG_WORKER_GROUP_ID_KEY, get_sglang_worker_group_id from dynamo.sglang.args import Config from dynamo.sglang.capacity import ( @@ -49,9 +50,13 @@ def set_forward_pass_metrics_worker_id( import tempfile - server_args.forward_pass_metrics_worker_id = str(generate_endpoint.connection_id()) ipc_path = tempfile.NamedTemporaryFile(delete=False).name - server_args.forward_pass_metrics_ipc_name = f"ipc://{ipc_path}" + override_server_args( + server_args, + "dynamo.forward_pass_metrics", + forward_pass_metrics_worker_id=str(generate_endpoint.connection_id()), + forward_pass_metrics_ipc_name=f"ipc://{ipc_path}", + ) async def _resolve_multinode_leader_worker_id( diff --git a/components/src/dynamo/sglang/request_handlers/handler_base.py b/components/src/dynamo/sglang/request_handlers/handler_base.py index f2dd5a3e5a56..b9a9d1fb27da 100644 --- a/components/src/dynamo/sglang/request_handlers/handler_base.py +++ b/components/src/dynamo/sglang/request_handlers/handler_base.py @@ -871,7 +871,9 @@ async def update_weight_version(self, body: dict) -> dict: if req.abort_all_requests: self.engine.tokenizer_manager.abort_request(abort_all=True) - self.engine.tokenizer_manager.server_args.weight_version = req.new_version + self.engine.tokenizer_manager._update_weight_version_if_provided( + req.new_version + ) return { "success": True, "message": f"Weight version updated to {req.new_version}", diff --git a/components/src/dynamo/sglang/snapshot.py b/components/src/dynamo/sglang/snapshot.py index b280bd9eed4a..eb65065f7b8a 100644 --- a/components/src/dynamo/sglang/snapshot.py +++ b/components/src/dynamo/sglang/snapshot.py @@ -16,6 +16,7 @@ SnapshotConfig, configure_snapshot_capture_env, ) +from dynamo.sglang._compat import override_server_args from .pause import SGLangEnginePauseController @@ -138,7 +139,7 @@ async def prepare_snapshot_engine( # Enable memory_saver so GPU memory can be released for CRIU. # When using GMS, weights use VA-stable unmap/remap (no CPU backup); GMS # forbids enable_weights_cpu_backup. Otherwise use CPU backup for weights. - server_args.enable_memory_saver = True + snapshot_overrides = {"enable_memory_saver": True} try: from gpu_memory_service.integrations.sglang import is_gms_active @@ -146,7 +147,8 @@ async def prepare_snapshot_engine( except ImportError: _using_gms = False if not _using_gms: - server_args.enable_weights_cpu_backup = True + snapshot_overrides["enable_weights_cpu_backup"] = True + override_server_args(server_args, "dynamo.snapshot", **snapshot_overrides) start_time = time.time() engine = sgl.Engine(server_args=server_args) diff --git a/components/src/dynamo/sglang/tests/test_sglang_publisher.py b/components/src/dynamo/sglang/tests/test_sglang_publisher.py index d774b1605ea1..0aea2168f9ee 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_publisher.py +++ b/components/src/dynamo/sglang/tests/test_sglang_publisher.py @@ -51,11 +51,25 @@ def test_get_local_dp_rank_range_respects_multinode_dp_attention(): def test_set_forward_pass_metrics_worker_id_uses_endpoint_identity(): - server_args = SimpleNamespace(enable_forward_pass_metrics=True) + class ReadOnlyServerArgs: + def __init__(self): + object.__setattr__(self, "enable_forward_pass_metrics", True) + object.__setattr__(self, "override_source", None) + + def __setattr__(self, name, value): + raise AttributeError("server args are read-only") + + def override(self, source, **fields): + object.__setattr__(self, "override_source", source) + for name, value in fields.items(): + object.__setattr__(self, name, value) + + server_args = ReadOnlyServerArgs() endpoint = SimpleNamespace(connection_id=lambda: "endpoint-9") set_forward_pass_metrics_worker_id(server_args, endpoint) + assert server_args.override_source == "dynamo.forward_pass_metrics" assert server_args.forward_pass_metrics_worker_id == "endpoint-9" assert server_args.forward_pass_metrics_ipc_name.startswith("ipc://") diff --git a/components/src/dynamo/sglang/tests/test_sglang_unit.py b/components/src/dynamo/sglang/tests/test_sglang_unit.py index c7a646cd142d..129857ef4ae8 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_unit.py +++ b/components/src/dynamo/sglang/tests/test_sglang_unit.py @@ -585,6 +585,32 @@ async def start_profile(self, request): assert response == {"status": "ok", "message": "Profiling started"} +@pytest.mark.asyncio +async def test_update_weight_version_uses_tokenizer_manager_control_api(): + class TokenizerManager: + updated_version = None + + def _update_weight_version_if_provided(self, version): + self.updated_version = version + + tokenizer_manager = TokenizerManager() + handler = SimpleNamespace( + engine=SimpleNamespace(tokenizer_manager=tokenizer_manager) + ) + + response = await BaseWorkerHandler.update_weight_version( + handler, + {"new_version": "step-42", "abort_all_requests": False}, + ) + + assert tokenizer_manager.updated_version == "step-42" + assert response == { + "success": True, + "message": "Weight version updated to step-42", + "new_version": "step-42", + } + + @pytest.mark.asyncio async def test_custom_jinja_template_invalid_path(mock_sglang_cli): """Test that invalid file path raises FileNotFoundError.""" From 063048ae8e3e2282a28ce164e5f0213d668abc40 Mon Sep 17 00:00:00 2001 From: jain-ria Date: Mon, 17 Aug 2026 09:41:45 -0700 Subject: [PATCH 03/11] build(sglang): align sidecar examples with 0.5.17 Signed-off-by: jain-ria --- lib/sidecar/sglang/README.md | 4 ++-- lib/sidecar/sglang/deploy/agg.yaml | 4 ++-- lib/sidecar/sglang/deploy/disagg.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/sidecar/sglang/README.md b/lib/sidecar/sglang/README.md index de7c1f541702..246b1ede8a0c 100644 --- a/lib/sidecar/sglang/README.md +++ b/lib/sidecar/sglang/README.md @@ -61,8 +61,8 @@ There is no published sidecar image yet, so you build and push your own from > [!NOTE] > The engine image must be a stock SGLang **v0.5.16+** build: the native gRPC -> server (`--grpc-port`) landed there, and Dynamo's `sglang-runtime` pins an -> older SGLang without it. `deploy/agg.yaml` uses `lmsysorg/sglang:v0.5.16`. +> server (`--grpc-port`) landed there. The deployment examples use +> `lmsysorg/sglang:v0.5.17`, matching Dynamo main's SGLang pin. ### Prerequisites diff --git a/lib/sidecar/sglang/deploy/agg.yaml b/lib/sidecar/sglang/deploy/agg.yaml index 375c1662f0e5..8899d0de220f 100644 --- a/lib/sidecar/sglang/deploy/agg.yaml +++ b/lib/sidecar/sglang/deploy/agg.yaml @@ -38,10 +38,10 @@ spec: spec: # sglang-engine as a native sidecar (initContainer + restartPolicy: Always): # starts before `main`, stops after. Native gRPC `--grpc-port` needs SGLang - # v0.5.16+ (Dynamo's sglang-runtime pins an older SGLang without it). + # v0.5.16+; keep this example aligned with Dynamo main's SGLang pin. initContainers: - name: sglang-engine - image: lmsysorg/sglang:v0.5.16 + image: lmsysorg/sglang:v0.5.17 restartPolicy: Always # gRPC HealthCheck via exec (tcpSocket/grpc: dial the pod IP, not the # loopback engine). startupProbe gates `main`; readinessProbe drops dead workers. diff --git a/lib/sidecar/sglang/deploy/disagg.yaml b/lib/sidecar/sglang/deploy/disagg.yaml index a20428d1da06..4142966c1357 100644 --- a/lib/sidecar/sglang/deploy/disagg.yaml +++ b/lib/sidecar/sglang/deploy/disagg.yaml @@ -52,7 +52,7 @@ spec: sizeLimit: 10Gi initContainers: - name: sglang-engine - image: lmsysorg/sglang:v0.5.16 + image: lmsysorg/sglang:v0.5.17 restartPolicy: Always securityContext: capabilities: @@ -140,7 +140,7 @@ spec: sizeLimit: 10Gi initContainers: - name: sglang-engine - image: lmsysorg/sglang:v0.5.16 + image: lmsysorg/sglang:v0.5.17 restartPolicy: Always securityContext: capabilities: From cc2cf09dd12d381997a770b788c2b8864cab15c1 Mon Sep 17 00:00:00 2001 From: jain-ria Date: Mon, 17 Aug 2026 10:35:05 -0700 Subject: [PATCH 04/11] fix(sglang): preserve server arg updates across runtimes Signed-off-by: jain-ria --- components/src/dynamo/sglang/_compat.py | 17 ++++++++-- .../dynamo/sglang/tests/test_sglang_unit.py | 15 +++++++++ .../backends/sglang/observability.md | 2 +- .../integrations/sglang/__init__.py | 11 +++++-- .../tests/test_sglang_patches.py | 32 ++++++++++++++++++- 5 files changed, 70 insertions(+), 7 deletions(-) diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py index d8ca03b1d41f..69a2f84ff997 100644 --- a/components/src/dynamo/sglang/_compat.py +++ b/components/src/dynamo/sglang/_compat.py @@ -110,10 +110,21 @@ def override_server_args(server_args: Any, source: str, **fields: Any) -> None: """Apply a post-resolution SGLang configuration update. SGLang 0.5.17 makes ``ServerArgs`` unconditionally read-only after - resolution. Both supported releases expose ``ServerArgs.override`` as the - audited mutation API, so Dynamo must use it instead of assigning fields. + resolution. Both supported CUDA releases expose ``ServerArgs.override`` as + the audited mutation API, so Dynamo must use it instead of assigning fields. + The separately pinned XPU image still uses SGLang 0.5.11, which predates + that API; preserve its legacy assignment behavior until its engine pin is + upgraded. """ - server_args.override(source, **fields) + override = getattr(server_args, "override", None) + if callable(override): + override(source, **fields) + return + + # XPU compatibility for SGLang 0.5.11. Remove when the XPU SGLang pin is + # upgraded to 0.5.16+. + for name, value in fields.items(): + setattr(server_args, name, value) @lru_cache(maxsize=32) diff --git a/components/src/dynamo/sglang/tests/test_sglang_unit.py b/components/src/dynamo/sglang/tests/test_sglang_unit.py index 129857ef4ae8..b1bc50120665 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_unit.py +++ b/components/src/dynamo/sglang/tests/test_sglang_unit.py @@ -24,6 +24,7 @@ ensure_sglang_tensor_image_size, ensure_sglang_top_level_exports, filter_supported_async_generate_kwargs, + override_server_args, require_reasoning_kwargs, ) from dynamo.sglang.args import ( @@ -68,6 +69,20 @@ mock_sglang_cli = make_cli_args_fixture("dynamo.sglang") +def test_override_server_args_supports_legacy_xpu_pin(): + server_args = SimpleNamespace(enable_memory_saver=False) + + override_server_args( + server_args, + "dynamo.test", + enable_memory_saver=True, + load_format="legacy-loader", + ) + + assert server_args.enable_memory_saver is True + assert server_args.load_format == "legacy-loader" + + @pytest.fixture(autouse=True) def _cpu_engine_when_no_accelerator(monkeypatch): """Honor the file's gpu_0 contract on hosts with no accelerator. diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/observability.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/observability.md index 77f76baf9c98..2a992dda6719 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/observability.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/observability.md @@ -115,7 +115,7 @@ For the complete and authoritative list of all SGLang metrics, see the [official ## Forward Pass Metrics (FPM) -> **Availability.** Forward Pass Metrics require SGLang's upstream `sglang.srt.observability.forward_pass_metrics` module and the `ServerArgs` fields (`enable_forward_pass_metrics`, `forward_pass_metrics_worker_id`, `forward_pass_metrics_ipc_name`). These landed in **SGLang v0.5.13** and ship in the current Dynamo runtime (`sglang==0.5.15`), so setting `DYN_FORWARDPASS_METRIC_PORT` enables SGLang-side FPM emission to the NATS event plane via `FpmEventRelay`. The wire-format contract is guarded by `dynamo/sglang/tests/test_fpm_contract.py`. On runtimes older than v0.5.13 the module is absent: the Dynamo-side relay still starts and the worker serves normally, but no SGLang-side FPM payloads are emitted. +> **Availability.** Forward Pass Metrics require SGLang's upstream `sglang.srt.observability.forward_pass_metrics` module and the `ServerArgs` fields (`enable_forward_pass_metrics`, `forward_pass_metrics_worker_id`, `forward_pass_metrics_ipc_name`). These landed in **SGLang v0.5.13** and ship in the current CUDA Dynamo runtime (`sglang==0.5.17`), so setting `DYN_FORWARDPASS_METRIC_PORT` enables SGLang-side FPM emission to the NATS event plane via `FpmEventRelay`. The wire-format contract is guarded by `dynamo/sglang/tests/test_fpm_contract.py`. On runtimes older than v0.5.13 the module is absent: the Dynamo-side relay still starts and the worker serves normally, but no SGLang-side FPM payloads are emitted. Forward Pass Metrics provide **per-iteration scheduler telemetry** pushed over ZMQ, giving the [Planner](../../planner/overview.md) real-time visibility into batch composition, queue depth, and GPU forward pass duration. Unlike Prometheus metrics (which are scraped asynchronously and reflect only the latest gauge value), FPM emits a structured message after every scheduler iteration with the exact batch state. diff --git a/lib/gpu_memory_service/integrations/sglang/__init__.py b/lib/gpu_memory_service/integrations/sglang/__init__.py index 28a460212bc6..cb073ce0af82 100644 --- a/lib/gpu_memory_service/integrations/sglang/__init__.py +++ b/lib/gpu_memory_service/integrations/sglang/__init__.py @@ -7,7 +7,8 @@ from gpu_memory_service.integrations.sglang import setup_gms if server_args.load_format == "gms": - server_args.load_format = setup_gms(server_args) + load_format = setup_gms(server_args) + server_args.override("dynamo.gms", load_format=load_format) """ from __future__ import annotations @@ -57,7 +58,13 @@ def setup_gms(server_args) -> Type["GMSModelLoader"]: "Cannot use --enable-draft-weights-cpu-backup with --load-format gms." ) - server_args.enable_memory_saver = True + override = getattr(server_args, "override", None) + if callable(override): + override("dynamo.gms", enable_memory_saver=True) + else: + # The separately pinned XPU image still uses SGLang 0.5.11, which + # predates ServerArgs.override. Remove after that pin reaches 0.5.16+. + server_args.enable_memory_saver = True # Resolve lock mode and RO reconnect timeout from model_loader_extra_config # before patches fire. global _gms_lock_mode diff --git a/lib/gpu_memory_service/tests/test_sglang_patches.py b/lib/gpu_memory_service/tests/test_sglang_patches.py index 07c521c80051..6315ec726c4c 100644 --- a/lib/gpu_memory_service/tests/test_sglang_patches.py +++ b/lib/gpu_memory_service/tests/test_sglang_patches.py @@ -18,7 +18,7 @@ if not HAS_TORCH: pytest.skip("torch is required", allow_module_level=True) -from gpu_memory_service.integrations.sglang import patches +from gpu_memory_service.integrations.sglang import patches, setup_gms pytestmark = [ pytest.mark.pre_merge, @@ -79,3 +79,33 @@ def alloc_memory_pool(self): assert runner.alloc_memory_pool() == 10.0 assert runner.pre_model_load_memory == 10.0 + + +def test_setup_gms_uses_read_only_server_args_override(monkeypatch): + class GMSModelLoader: + pass + + model_loader_module = ModuleType( + "gpu_memory_service.integrations.sglang.model_loader" + ) + model_loader_module.GMSModelLoader = GMSModelLoader + monkeypatch.setitem(sys.modules, model_loader_module.__name__, model_loader_module) + + class ReadOnlyServerArgs: + enable_weights_cpu_backup = False + enable_draft_weights_cpu_backup = False + model_loader_extra_config = None + + def __init__(self): + object.__setattr__(self, "overrides", []) + + def __setattr__(self, name, value): + raise AttributeError(f"{name} is read-only") + + def override(self, source, **fields): + self.overrides.append((source, fields)) + + server_args = ReadOnlyServerArgs() + + assert setup_gms(server_args) is GMSModelLoader + assert server_args.overrides == [("dynamo.gms", {"enable_memory_saver": True})] From 8c6f5b690d2d80f35e0516c6c63ac32c443e62ea Mon Sep 17 00:00:00 2001 From: jain-ria Date: Mon, 17 Aug 2026 10:35:10 -0700 Subject: [PATCH 05/11] fix(sglang): order multimodal prefill bootstrap safely Signed-off-by: jain-ria --- .../multimodal/worker_handler.py | 166 ++++++++++++++---- .../tests/test_sglang_multimodal_video.py | 116 ++++++++++++ 2 files changed, 247 insertions(+), 35 deletions(-) diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py index 256514c3d68a..affee75f9a1a 100644 --- a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py +++ b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py @@ -692,6 +692,7 @@ def __init__( # Get bootstrap info using BootstrapManager self.bootstrap_host, self.bootstrap_port = self._get_bootstrap_info(engine) + self._consume_tasks: set[asyncio.Task[Any]] = set() logger.info( f"Multimodal prefill worker handler initialized - bootstrap host: {self.bootstrap_host}, bootstrap port: {self.bootstrap_port}" @@ -721,8 +722,56 @@ def _end_bootstrap() -> None: # Validate and parse request disagg_request = self._validate_and_parse_disagg_request(disagg_request) - # Generate and return bootstrap info first (like regular SGLang) bootstrap_room = self._generate_bootstrap_room() + results, tensor_id = await self._start_prefill_generation( + disagg_request, bootstrap_room, context=context + ) + consumer_owns_tensor = asyncio.Event() + request_started = asyncio.Event() + task: asyncio.Task[Any] | None = None + try: + task = asyncio.create_task( + self._consume_results( + results, + tensor_id, + context, + consumer_owns_tensor, + request_started, + ) + ) + self._consume_tasks.add(task) + task.add_done_callback(self._consume_tasks.discard) + + started_wait = asyncio.create_task(request_started.wait()) + try: + await asyncio.wait( + (task, started_wait), + return_when=asyncio.FIRST_COMPLETED, + ) + finally: + if not started_wait.done(): + started_wait.cancel() + try: + await started_wait + except asyncio.CancelledError: + pass + + # Surface an immediate submission/cancellation failure before + # decode is authorized. Once consumer_owns_tensor is set, the + # consumer's try/finally owns the transferred tensor. + if task.done(): + await task + except BaseException: + if not consumer_owns_tensor.is_set(): + if task is not None and not task.done(): + task.cancel() + if tensor_id is not None: + self.embeddings_processor.release_embeddings(tensor_id) + raise + + # Do not authorize decode until embeddings have been received and + # the result consumer has advanced SGLang's lazy request iterator. + # Otherwise decode can start waiting before prefill is submitted. bootstrap_info = { "bootstrap_host": self.bootstrap_host, "bootstrap_port": self.bootstrap_port, @@ -732,10 +781,7 @@ def _end_bootstrap() -> None: _end_bootstrap() yield json.dumps(bootstrap_info) - # Process prefill generation - await self._process_prefill_generation( - disagg_request, bootstrap_room, context=context - ) + await task except Exception as e: logger.error(f"Error in prefill generation: {e}", exc_info=True) @@ -761,13 +807,13 @@ def _validate_and_parse_disagg_request( ) return disagg_request - async def _process_prefill_generation( + async def _start_prefill_generation( self, disagg_request: DisaggSglangMultimodalRequest, bootstrap_room: int, context=None, - ): - """Process multimodal input and start prefill generation""" + ) -> tuple[AsyncIterator[Any], Optional[int]]: + """Receive multimodal embeddings and submit the prefill to SGLang.""" # Get the SglangMultimodalRequest from the DisaggSglangMultimodalRequest request = disagg_request.request input_ids = request.request.token_ids @@ -787,42 +833,92 @@ async def _process_prefill_generation( context.trace_headers() if context and self.enable_trace else None ) - # Start SGLang prefill generation (like regular SGLang) - with _nvtx.annotate("mm:prefill:engine_async_generate", color="blue"): - gen_params = { - "input_ids": input_ids, - "sampling_params": sampling_params, - "stream": True, - "bootstrap_host": self.bootstrap_host, - "bootstrap_port": self.bootstrap_port, - "bootstrap_room": bootstrap_room, - "external_trace_header": trace_header, - "rid": context.trace_id if context else None, - } - - if image_mm_items: - gen_params["image_data"] = image_mm_items - if video_data: - gen_params["video_data"] = video_data - - results = await self.engine.async_generate(**gen_params) + try: + # Start SGLang prefill generation (like regular SGLang) + with _nvtx.annotate("mm:prefill:engine_async_generate", color="blue"): + gen_params = { + "input_ids": input_ids, + "sampling_params": sampling_params, + "stream": True, + "bootstrap_host": self.bootstrap_host, + "bootstrap_port": self.bootstrap_port, + "bootstrap_room": bootstrap_room, + "external_trace_header": trace_header, + "rid": context.trace_id if context else None, + } + + if image_mm_items: + gen_params["image_data"] = image_mm_items + if video_data: + gen_params["video_data"] = video_data + + results = await self.engine.async_generate(**gen_params) + except BaseException: + if tensor_id is not None: + self.embeddings_processor.release_embeddings(tensor_id) + raise - # Consume results without yielding (prefill doesn't return text, just coordinates) - asyncio.create_task(self._consume_results(results, tensor_id)) + return results, tensor_id - async def _consume_results(self, results, tensor_id: Optional[int]): - """Consume prefill results without returning them (like regular SGLang)""" + async def _consume_results( + self, + results, + tensor_id: Optional[int], + context: Context, + owns_tensor: asyncio.Event, + request_started: asyncio.Event, + ) -> None: + """Consume prefill output while honoring request cancellation.""" released = False + request_id_future: asyncio.Future[str] = asyncio.Future() + first_result_task: asyncio.Task[Any] | None = None + + def process_result(result: dict[str, Any]) -> None: + nonlocal released + if not request_id_future.done(): + request_id = result.get("meta_info", {}).get("id") + if request_id: + request_id_future.set_result(request_id) + if tensor_id is not None and not released: + self.embeddings_processor.release_embeddings(tensor_id) + released = True + try: - async for _ in results: - if tensor_id is not None and not released: - self.embeddings_processor.release_embeddings(tensor_id) - released = True + owns_tensor.set() + async with self._cancellation_monitor(request_id_future, context): + first_result_task = asyncio.create_task(anext(results)) + # SGLang's stream is a lazy async generator. Give its first + # iteration a scheduler turn before authorizing decode so the + # request reaches TokenizerManager.generate_request. + await asyncio.sleep(0) + request_started.set() + try: + first_result = await first_result_task + except StopAsyncIteration as e: + raise RuntimeError( + "SGLang prefill stream ended before producing a result" + ) from e + process_result(first_result) + + async for result in results: + process_result(result) finally: + if first_result_task is not None: + if not first_result_task.done(): + first_result_task.cancel() + try: + await first_result_task + except (asyncio.CancelledError, Exception): + pass if tensor_id is not None and not released: self.embeddings_processor.release_embeddings(tensor_id) def cleanup(self): + for task in self._consume_tasks: + if not task.done(): + task.cancel() + self._consume_tasks.clear() + super().cleanup() self.engine.shutdown() logger.info("Multimodal prefill engine shutdown") diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py index a62e2258f1f8..d35899820bc3 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py +++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py @@ -1,7 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import asyncio import importlib +import json +from contextlib import asynccontextmanager +from types import SimpleNamespace import numpy as np import pytest @@ -21,6 +25,7 @@ ) from dynamo.sglang.request_handlers.multimodal.worker_handler import ( EmbeddingsProcessor, + MultimodalPrefillWorkerHandler, _build_mm_items, ) @@ -148,6 +153,117 @@ def create_multimodal_video_item( assert mm_item["video_timestamps"] == [[0.25, 0.75]] +@pytest.mark.asyncio +async def test_multimodal_prefill_starts_before_returning_bootstrap(): + handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler) + handler.bootstrap_host = "prefill-host" + handler.bootstrap_port = 1234 + handler._consume_tasks = set() + handler.embeddings_processor = SimpleNamespace(release_embeddings=lambda _: None) + events = [] + + handler._validate_and_parse_disagg_request = lambda request: request + handler._generate_bootstrap_room = lambda: 17 + + async def start_prefill(request, bootstrap_room, context=None): + events.append(("start", bootstrap_room)) + + async def results(): + events.append(("iterate", None)) + yield {"meta_info": {"id": "request-id"}} + + return results(), None + + @asynccontextmanager + async def cancellation_monitor(request_id_future, context): + yield + + handler._start_prefill_generation = start_prefill + handler._cancellation_monitor = cancellation_monitor + + stream = handler.generate(object(), SimpleNamespace()) + bootstrap = json.loads(await anext(stream)) + + assert events == [("start", 17), ("iterate", None)] + assert bootstrap == { + "bootstrap_host": "prefill-host", + "bootstrap_port": 1234, + "bootstrap_room": 17, + } + + with pytest.raises(StopAsyncIteration): + await anext(stream) + assert events == [("start", 17), ("iterate", None)] + + +@pytest.mark.asyncio +async def test_multimodal_prefill_releases_embeddings_when_submission_fails( + monkeypatch, +): + import dynamo.sglang.request_handlers.multimodal.worker_handler as worker_handler + + handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler) + handler.bootstrap_host = "prefill-host" + handler.bootstrap_port = 1234 + handler.enable_trace = False + + released = [] + handler.embeddings_processor = SimpleNamespace(release_embeddings=released.append) + + async def build_mm_items(request, embeddings_processor): + return [], [], torch.empty(0), 23 + + class FailingEngine: + async def async_generate(self, **kwargs): + raise RuntimeError("submission failed") + + handler.engine = FailingEngine() + monkeypatch.setattr(worker_handler, "_build_mm_items", build_mm_items) + + request = SimpleNamespace( + request=SimpleNamespace(request=SimpleNamespace(token_ids=[1, 2, 3])), + sampling_params={"max_new_tokens": 1}, + ) + + with pytest.raises(RuntimeError, match="submission failed"): + await handler._start_prefill_generation(request, 17) + + assert released == [23] + + +@pytest.mark.asyncio +async def test_multimodal_prefill_rejects_empty_engine_stream(): + handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler) + released = [] + handler.embeddings_processor = SimpleNamespace(release_embeddings=released.append) + + @asynccontextmanager + async def cancellation_monitor(request_id_future, context): + yield + + handler._cancellation_monitor = cancellation_monitor + + async def empty_results(): + if False: + yield None + + owns_tensor = asyncio.Event() + request_started = asyncio.Event() + + with pytest.raises(RuntimeError, match="ended before producing a result"): + await handler._consume_results( + empty_results(), + 23, + SimpleNamespace(), + owns_tensor, + request_started, + ) + + assert owns_tensor.is_set() + assert request_started.is_set() + assert released == [23] + + async def test_nvdec_video_metadata_shim_stamps_valid_metadata(): """The metadata shim gives pre-decoded ndarray frames a valid metadata dict. From 43927289a546aa7578373cd7fa8fd07c4f3e403c Mon Sep 17 00:00:00 2001 From: jain-ria Date: Mon, 17 Aug 2026 20:32:50 -0700 Subject: [PATCH 06/11] fix(sglang): complete 0.5.17 compatibility validation Signed-off-by: jain-ria --- components/src/dynamo/sglang/_compat.py | 36 ++++--------------- .../dynamo/sglang/tests/test_sglang_unit.py | 35 ------------------ examples/backends/sglang/launch/agg_agent.sh | 1 - 3 files changed, 6 insertions(+), 66 deletions(-) diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py index 69a2f84ff997..bd242c5cd35e 100644 --- a/components/src/dynamo/sglang/_compat.py +++ b/components/src/dynamo/sglang/_compat.py @@ -38,31 +38,6 @@ def _warn_require_reasoning_unsupported() -> None: ) -# --------------------------------------------------------------------------- -# Top-level sglang exports: Engine, ServerArgs -# -# Some SGLang dev builds (including 0.5.x snapshots) do not re-export these -# from sglang/__init__.py, while Dynamo historically uses `import sglang as sgl` -# followed by `sgl.Engine(...)` throughout this backend. -# --------------------------------------------------------------------------- -def ensure_sglang_top_level_exports() -> None: - """Restore top-level SGLang exports omitted by some install flavors.""" - import sglang as sgl - - if not hasattr(sgl, "Engine"): - from sglang.srt.entrypoints.engine import Engine - - sgl.Engine = Engine - - if not hasattr(sgl, "ServerArgs"): - from sglang.srt.server_args import ServerArgs - - sgl.ServerArgs = ServerArgs - - -ensure_sglang_top_level_exports() - - def ensure_sglang_tensor_image_size() -> None: """Allow SGLang's image-token resolver to handle decoded image tensors. @@ -159,10 +134,12 @@ def filter_supported_async_generate_kwargs( ) -> dict[str, Any]: """Return only async_generate kwargs accepted by this SGLang engine. - SGLang occasionally adds optional Engine.async_generate kwargs before every - supported install flavor has them. Keep the compatibility boundary narrow: - callers decide which kwargs are optional, and this helper only drops those - optional kwargs when the installed engine cannot accept them. + Both supported CUDA releases accept Dynamo's optional kwargs. The separately + pinned XPU image still uses SGLang 0.5.11, which predates ``mm_hashes`` and + ``require_reasoning``. Keep the compatibility boundary narrow: callers + decide which kwargs are optional, and this helper only drops those optional + kwargs when the installed engine cannot accept them. Remove this filtering + when the XPU SGLang pin is upgraded to 0.5.16+. """ async_generate = engine.async_generate signature_source = getattr(async_generate, "__func__", async_generate) @@ -196,7 +173,6 @@ def require_reasoning_kwargs(engine: Any, request: Mapping[str, Any]) -> dict[st __all__ = [ "ensure_sglang_tensor_image_size", - "ensure_sglang_top_level_exports", "filter_supported_async_generate_kwargs", "override_server_args", "require_reasoning_kwargs", diff --git a/components/src/dynamo/sglang/tests/test_sglang_unit.py b/components/src/dynamo/sglang/tests/test_sglang_unit.py index b1bc50120665..19597fa35f67 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_unit.py +++ b/components/src/dynamo/sglang/tests/test_sglang_unit.py @@ -22,7 +22,6 @@ from dynamo.common.snapshot.constants import SNAPSHOT_CONTROL_DIR_ENV from dynamo.sglang._compat import ( ensure_sglang_tensor_image_size, - ensure_sglang_top_level_exports, filter_supported_async_generate_kwargs, override_server_args, require_reasoning_kwargs, @@ -221,40 +220,6 @@ def _make_sglang_config(**overrides): return config -def test_compat_restores_sglang_top_level_exports(): - """Dynamo supports SGLang builds that omit top-level Engine/ServerArgs.""" - import sglang as sgl - from sglang.srt.entrypoints.engine import Engine - from sglang.srt.server_args import ServerArgs - - missing = object() - original_engine = getattr(sgl, "Engine", missing) - original_server_args = getattr(sgl, "ServerArgs", missing) - - try: - if hasattr(sgl, "Engine"): - delattr(sgl, "Engine") - if hasattr(sgl, "ServerArgs"): - delattr(sgl, "ServerArgs") - - ensure_sglang_top_level_exports() - - assert sgl.Engine is Engine - assert sgl.ServerArgs is ServerArgs - finally: - if original_engine is missing: - if hasattr(sgl, "Engine"): - delattr(sgl, "Engine") - else: - sgl.Engine = original_engine - - if original_server_args is missing: - if hasattr(sgl, "ServerArgs"): - delattr(sgl, "ServerArgs") - else: - sgl.ServerArgs = original_server_args - - def test_compat_supports_tensor_image_sizes_and_is_idempotent(caplog, monkeypatch): from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor, diff --git a/examples/backends/sglang/launch/agg_agent.sh b/examples/backends/sglang/launch/agg_agent.sh index 8b25d8ebdff6..c0f4b25beddb 100755 --- a/examples/backends/sglang/launch/agg_agent.sh +++ b/examples/backends/sglang/launch/agg_agent.sh @@ -72,7 +72,6 @@ python3 -m dynamo.sglang \ --tp "$TP" \ --trust-remote-code \ --radix-eviction-policy priority \ - --skip-tokenizer-init \ --dyn-reasoning-parser glm45 \ --dyn-tool-call-parser glm47 \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' \ From 9b2988a9b2ec04275df6d02b0d4141ca394abce7 Mon Sep 17 00:00:00 2001 From: jain-ria Date: Mon, 17 Aug 2026 20:50:04 -0700 Subject: [PATCH 07/11] fix(sglang): await multimodal prefill cleanup Signed-off-by: jain-ria --- .../src/dynamo/sglang/init_multimodal.py | 2 +- .../multimodal/worker_handler.py | 84 +++++++++++++++---- .../tests/test_sglang_multimodal_video.py | 62 ++++++++++++++ 3 files changed, 130 insertions(+), 18 deletions(-) diff --git a/components/src/dynamo/sglang/init_multimodal.py b/components/src/dynamo/sglang/init_multimodal.py index 5ce1d9e4cef7..d8d70812bd34 100644 --- a/components/src/dynamo/sglang/init_multimodal.py +++ b/components/src/dynamo/sglang/init_multimodal.py @@ -253,7 +253,7 @@ async def init_multimodal_prefill_worker( logging.error(f"Failed to serve endpoints: {e}") raise finally: - handler.cleanup() + await handler.cleanup() if run_deferred_handlers is not None: logging.info("Running deferred handlers") await run_deferred_handlers() diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py index affee75f9a1a..0bdaa024f1f1 100644 --- a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py +++ b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py @@ -4,6 +4,7 @@ import asyncio import json import logging +import sys from typing import Any, AsyncIterator, Callable, Literal, Optional, Protocol import sglang as sgl @@ -761,6 +762,19 @@ def _end_bootstrap() -> None: # consumer's try/finally owns the transferred tensor. if task.done(): await task + # Do not authorize decode until embeddings have been received and + # the result consumer has advanced SGLang's lazy request iterator. + # Otherwise decode can start waiting before prefill is submitted. + bootstrap_info = { + "bootstrap_host": self.bootstrap_host, + "bootstrap_port": self.bootstrap_port, + "bootstrap_room": bootstrap_room, + } + + _end_bootstrap() + yield json.dumps(bootstrap_info) + + await task except BaseException: if not consumer_owns_tensor.is_set(): if task is not None and not task.done(): @@ -768,20 +782,32 @@ def _end_bootstrap() -> None: if tensor_id is not None: self.embeddings_processor.release_embeddings(tensor_id) raise - - # Do not authorize decode until embeddings have been received and - # the result consumer has advanced SGLang's lazy request iterator. - # Otherwise decode can start waiting before prefill is submitted. - bootstrap_info = { - "bootstrap_host": self.bootstrap_host, - "bootstrap_port": self.bootstrap_port, - "bootstrap_room": bootstrap_room, - } - - _end_bootstrap() - yield json.dumps(bootstrap_info) - - await task + finally: + pending_exception = sys.exc_info()[1] + if task is not None: + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + except Exception as task_error: + if pending_exception is None: + raise + if ( + task_error is not pending_exception + and task_error + is not getattr(pending_exception, "__cause__", None) + ): + logger.error( + "Multimodal prefill consumer failed during request " + "cleanup", + exc_info=( + type(task_error), + task_error, + task_error.__traceback__, + ), + ) except Exception as e: logger.error(f"Error in prefill generation: {e}", exc_info=True) @@ -903,20 +929,44 @@ def process_result(result: dict[str, Any]) -> None: async for result in results: process_result(result) finally: + pending_exception = sys.exc_info()[1] if first_result_task is not None: if not first_result_task.done(): first_result_task.cancel() try: await first_result_task - except (asyncio.CancelledError, Exception): + except asyncio.CancelledError: pass + except Exception as task_error: + if pending_exception is None: + raise + if task_error is not pending_exception: + logger.error( + "SGLang prefill first-result task failed during cleanup", + exc_info=( + type(task_error), + task_error, + task_error.__traceback__, + ), + ) if tensor_id is not None and not released: self.embeddings_processor.release_embeddings(tensor_id) - def cleanup(self): - for task in self._consume_tasks: + async def cleanup(self) -> None: + tasks = list(self._consume_tasks) + for task in tasks: if not task.done(): task.cancel() + if tasks: + results = await asyncio.gather(*tasks, return_exceptions=True) + for result in results: + if isinstance(result, Exception) and not isinstance( + result, asyncio.CancelledError + ): + logger.error( + "Multimodal prefill consumer failed during handler cleanup", + exc_info=(type(result), result, result.__traceback__), + ) self._consume_tasks.clear() super().cleanup() diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py index d35899820bc3..91e84090420e 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py +++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py @@ -264,6 +264,68 @@ async def empty_results(): assert released == [23] +@pytest.mark.asyncio +async def test_multimodal_prefill_cancellation_awaits_consumer_cleanup(): + handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler) + handler.bootstrap_host = "prefill-host" + handler.bootstrap_port = 1234 + handler._consume_tasks = set() + released = [] + handler.embeddings_processor = SimpleNamespace(release_embeddings=released.append) + + handler._validate_and_parse_disagg_request = lambda request: request + handler._generate_bootstrap_room = lambda: 17 + result_stopped = asyncio.Event() + + async def results(): + try: + await asyncio.Event().wait() + yield {} + finally: + result_stopped.set() + + async def start_prefill(request, bootstrap_room, context=None): + return results(), 23 + + @asynccontextmanager + async def cancellation_monitor(request_id_future, context): + yield + + handler._start_prefill_generation = start_prefill + handler._cancellation_monitor = cancellation_monitor + + stream = handler.generate(object(), SimpleNamespace()) + await anext(stream) + await stream.aclose() + + assert result_stopped.is_set() + assert released == [23] + assert not handler._consume_tasks + + +@pytest.mark.asyncio +async def test_multimodal_prefill_cleanup_awaits_consumers_before_engine_shutdown(): + handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler) + handler.publisher = None + events = [] + + async def consumer(): + try: + await asyncio.Event().wait() + finally: + events.append("consumer stopped") + + task = asyncio.create_task(consumer()) + await asyncio.sleep(0) + handler._consume_tasks = {task} + handler.engine = SimpleNamespace(shutdown=lambda: events.append("engine shutdown")) + + await handler.cleanup() + + assert events == ["consumer stopped", "engine shutdown"] + assert not handler._consume_tasks + + async def test_nvdec_video_metadata_shim_stamps_valid_metadata(): """The metadata shim gives pre-decoded ndarray frames a valid metadata dict. From 2a0b096a376720f7cce3fba6b4ce3c004a245cc3 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Tue, 18 Aug 2026 17:22:11 +0000 Subject: [PATCH 08/11] chore(recipes): refresh generated OpenAPI schema Signed-off-by: Ishan Dhanani --- .../dynamo-openapi/dynamo-openapi.json | 42 ------------------- 1 file changed, 42 deletions(-) diff --git a/recipes/kustomize/components/dynamo-openapi/dynamo-openapi.json b/recipes/kustomize/components/dynamo-openapi/dynamo-openapi.json index 87c8c0449d02..8e99d445c76c 100644 --- a/recipes/kustomize/components/dynamo-openapi/dynamo-openapi.json +++ b/recipes/kustomize/components/dynamo-openapi/dynamo-openapi.json @@ -1283,48 +1283,6 @@ } ] }, - "nvidia.com.v1alpha1.PodSnapshot": { - "properties": { - "apiVersion": { - "type": "string" - }, - "kind": { - "type": "string" - }, - "metadata": { - "type": "object" - } - }, - "type": "object", - "x-kubernetes-group-version-kind": [ - { - "group": "nvidia.com", - "kind": "PodSnapshot", - "version": "v1alpha1" - } - ] - }, - "nvidia.com.v1alpha1.PodSnapshotContent": { - "properties": { - "apiVersion": { - "type": "string" - }, - "kind": { - "type": "string" - }, - "metadata": { - "type": "object" - } - }, - "type": "object", - "x-kubernetes-group-version-kind": [ - { - "group": "nvidia.com", - "kind": "PodSnapshotContent", - "version": "v1alpha1" - } - ] - }, "nvidia.com.v1beta1.DynamoComponentDeployment": { "properties": { "apiVersion": { From 6534c353bbf2f52c37d26c57b4ce82fb749259ba Mon Sep 17 00:00:00 2001 From: jain-ria Date: Tue, 18 Aug 2026 18:36:27 -0700 Subject: [PATCH 09/11] fix(sglang): arm multimodal cancellation before output Signed-off-by: jain-ria --- .../multimodal/worker_handler.py | 229 ++++++++++++++++-- .../tests/test_sglang_multimodal_video.py | 129 ++++++++-- 2 files changed, 318 insertions(+), 40 deletions(-) diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py index 0bdaa024f1f1..32310eaee097 100644 --- a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py +++ b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py @@ -678,6 +678,8 @@ class MultimodalPrefillWorkerHandler( Processes multimodal inputs and coordinates with decode worker. """ + _REQUEST_REGISTRATION_TIMEOUT_SECONDS = 5.0 + def __init__( self, engine: sgl.Engine, @@ -723,9 +725,13 @@ def _end_bootstrap() -> None: # Validate and parse request disagg_request = self._validate_and_parse_disagg_request(disagg_request) + rid = context.trace_id or context.id() bootstrap_room = self._generate_bootstrap_room() - results, tensor_id = await self._start_prefill_generation( - disagg_request, bootstrap_room, context=context + results, tensor_id = await self._start_prefill_or_cancel( + disagg_request, + bootstrap_room, + rid, + context, ) consumer_owns_tensor = asyncio.Event() request_started = asyncio.Event() @@ -735,6 +741,7 @@ def _end_bootstrap() -> None: self._consume_results( results, tensor_id, + rid, context, consumer_owns_tensor, request_started, @@ -837,6 +844,7 @@ async def _start_prefill_generation( self, disagg_request: DisaggSglangMultimodalRequest, bootstrap_room: int, + rid: Optional[str] = None, context=None, ) -> tuple[AsyncIterator[Any], Optional[int]]: """Receive multimodal embeddings and submit the prefill to SGLang.""" @@ -870,7 +878,7 @@ async def _start_prefill_generation( "bootstrap_port": self.bootstrap_port, "bootstrap_room": bootstrap_room, "external_trace_header": trace_header, - "rid": context.trace_id if context else None, + "rid": rid, } if image_mm_items: @@ -886,10 +894,81 @@ async def _start_prefill_generation( return results, tensor_id + async def _start_prefill_or_cancel( + self, + disagg_request: DisaggSglangMultimodalRequest, + bootstrap_room: int, + rid: str, + context: Context, + ) -> tuple[AsyncIterator[Any], Optional[int]]: + """Cancel local preprocessing/submission if the client stops early.""" + start_task = asyncio.create_task( + self._start_prefill_generation( + disagg_request, + bootstrap_room, + rid=rid, + context=context, + ) + ) + cancellation_future = context.async_killed_or_stopped() + try: + done, _ = await asyncio.wait( + (start_task, cancellation_future), + return_when=asyncio.FIRST_COMPLETED, + ) + # Prefer a completed submission if both signals arrive together. The + # registered-request phase will observe the sticky context state and + # abort by the same RID without dropping tensor ownership. + if start_task in done: + return await start_task + + start_task.cancel() + try: + await start_task + except asyncio.CancelledError: + pass + raise asyncio.CancelledError + finally: + if not start_task.done(): + start_task.cancel() + try: + await start_task + except asyncio.CancelledError: + pass + if not cancellation_future.done(): + cancellation_future.cancel() + try: + await cancellation_future + except asyncio.CancelledError: + pass + + async def _wait_for_request_registration(self, rid: str) -> None: + """Wait until SGLang owns the RID, without waiting for engine output.""" + tokenizer_manager = getattr(self.engine, "tokenizer_manager", None) + rid_to_state = getattr(tokenizer_manager, "rid_to_state", None) + if rid_to_state is None: + raise RuntimeError("SGLang tokenizer manager has no request registry") + + async def poll_registry() -> None: + while rid not in rid_to_state: + await asyncio.sleep(0.001) + + try: + await asyncio.wait_for( + poll_registry(), + timeout=self._REQUEST_REGISTRATION_TIMEOUT_SECONDS, + ) + except TimeoutError as e: + raise RuntimeError( + f"SGLang did not register prefill request {rid} within " + f"{self._REQUEST_REGISTRATION_TIMEOUT_SECONDS:g}s" + ) from e + async def _consume_results( self, results, tensor_id: Optional[int], + rid: str, context: Context, owns_tensor: asyncio.Event, request_started: asyncio.Event, @@ -898,38 +977,148 @@ async def _consume_results( released = False request_id_future: asyncio.Future[str] = asyncio.Future() first_result_task: asyncio.Task[Any] | None = None + next_result_task: asyncio.Task[Any] | None = None + registration_task: asyncio.Task[None] | None = None + pre_registration_cancellation: asyncio.Future[Any] | None = None def process_result(result: dict[str, Any]) -> None: nonlocal released - if not request_id_future.done(): - request_id = result.get("meta_info", {}).get("id") - if request_id: - request_id_future.set_result(request_id) if tensor_id is not None and not released: self.embeddings_processor.release_embeddings(tensor_id) released = True try: owns_tensor.set() - async with self._cancellation_monitor(request_id_future, context): - first_result_task = asyncio.create_task(anext(results)) - # SGLang's stream is a lazy async generator. Give its first - # iteration a scheduler turn before authorizing decode so the - # request reaches TokenizerManager.generate_request. - await asyncio.sleep(0) - request_started.set() + registration_task = asyncio.create_task( + self._wait_for_request_registration(rid) + ) + first_result_task = asyncio.create_task(anext(results)) + pre_registration_cancellation = context.async_killed_or_stopped() + + first_result: Any = None + first_result_ready = False + while not registration_task.done(): + wait_for: set[asyncio.Future[Any]] = { + registration_task, + pre_registration_cancellation, + } + if first_result_task is not None and not first_result_task.done(): + wait_for.add(first_result_task) + done, _ = await asyncio.wait( + wait_for, + return_when=asyncio.FIRST_COMPLETED, + ) + if registration_task in done: + break + if first_result_task in done and not first_result_ready: + try: + first_result = await first_result_task + except StopAsyncIteration as e: + raise RuntimeError( + "SGLang prefill stream ended before producing a result" + ) from e + finally: + first_result_task = None + first_result_ready = True + if pre_registration_cancellation in done: + raise asyncio.CancelledError + + await registration_task + request_id_future.set_result(rid) + if not pre_registration_cancellation.done(): + pre_registration_cancellation.cancel() try: - first_result = await first_result_task - except StopAsyncIteration as e: - raise RuntimeError( - "SGLang prefill stream ended before producing a result" - ) from e + await pre_registration_cancellation + except asyncio.CancelledError: + pass + + async with self._cancellation_monitor( + request_id_future, context + ) as cancellation_task: + request_started.set() + if not first_result_ready: + assert first_result_task is not None + done, _ = await asyncio.wait( + (first_result_task, cancellation_task), + return_when=asyncio.FIRST_COMPLETED, + ) + if cancellation_task in done: + await cancellation_task + raise asyncio.CancelledError + try: + first_result = await first_result_task + except StopAsyncIteration as e: + raise RuntimeError( + "SGLang prefill stream ended before producing a result" + ) from e + finally: + first_result_task = None process_result(first_result) - async for result in results: + while True: + next_result_task = asyncio.create_task(anext(results)) + done, _ = await asyncio.wait( + (next_result_task, cancellation_task), + return_when=asyncio.FIRST_COMPLETED, + ) + if cancellation_task in done: + if not next_result_task.done(): + next_result_task.cancel() + try: + await next_result_task + except asyncio.CancelledError: + pass + await cancellation_task + raise asyncio.CancelledError + try: + result = await next_result_task + except StopAsyncIteration: + break + finally: + next_result_task = None process_result(result) finally: pending_exception = sys.exc_info()[1] + for task in (registration_task, pre_registration_cancellation): + if task is None: + continue + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + except Exception as task_error: + if pending_exception is None: + raise + if task_error is not pending_exception: + logger.error( + "SGLang prefill registration task failed during cleanup", + exc_info=( + type(task_error), + task_error, + task_error.__traceback__, + ), + ) + if next_result_task is not None: + if not next_result_task.done(): + next_result_task.cancel() + try: + await next_result_task + except asyncio.CancelledError: + pass + except Exception as task_error: + if pending_exception is None: + raise + if task_error is not pending_exception: + logger.error( + "SGLang prefill next-result task failed during cleanup", + exc_info=( + type(task_error), + task_error, + task_error.__traceback__, + ), + ) if first_result_task is not None: if not first_result_task.done(): first_result_task.cancel() diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py index 91e84090420e..9184b0d4427c 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py +++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py @@ -40,6 +40,35 @@ ] +class _FakeContext: + def __init__(self, context_id: str, trace_id: str | None = None): + self.trace_id = trace_id + self._context_id = context_id + self._cancelled = asyncio.Event() + + def id(self) -> str: + return self._context_id + + def async_killed_or_stopped(self) -> asyncio.Task[bool]: + return asyncio.create_task(self._cancelled.wait()) + + def cancel(self) -> None: + self._cancelled.set() + + +@asynccontextmanager +async def _never_cancels(request_id_future, context): + task = asyncio.create_task(asyncio.Event().wait()) + try: + yield task + finally: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + def test_extract_media_inputs_supports_video_urls(): handler = MultimodalEncodeWorkerHandler.__new__(MultimodalEncodeWorkerHandler) @@ -165,8 +194,8 @@ async def test_multimodal_prefill_starts_before_returning_bootstrap(): handler._validate_and_parse_disagg_request = lambda request: request handler._generate_bootstrap_room = lambda: 17 - async def start_prefill(request, bootstrap_room, context=None): - events.append(("start", bootstrap_room)) + async def start_prefill(request, bootstrap_room, rid=None, context=None): + events.append(("start", bootstrap_room, rid)) async def results(): events.append(("iterate", None)) @@ -174,17 +203,21 @@ async def results(): return results(), None - @asynccontextmanager - async def cancellation_monitor(request_id_future, context): - yield + async def wait_for_registration(rid): + events.append(("registered", rid)) handler._start_prefill_generation = start_prefill - handler._cancellation_monitor = cancellation_monitor + handler._wait_for_request_registration = wait_for_registration + handler._cancellation_monitor = _never_cancels - stream = handler.generate(object(), SimpleNamespace()) + stream = handler.generate(object(), _FakeContext("request-id")) bootstrap = json.loads(await anext(stream)) - assert events == [("start", 17), ("iterate", None)] + assert events == [ + ("start", 17, "request-id"), + ("registered", "request-id"), + ("iterate", None), + ] assert bootstrap == { "bootstrap_host": "prefill-host", "bootstrap_port": 1234, @@ -193,7 +226,11 @@ async def cancellation_monitor(request_id_future, context): with pytest.raises(StopAsyncIteration): await anext(stream) - assert events == [("start", 17), ("iterate", None)] + assert events == [ + ("start", 17, "request-id"), + ("registered", "request-id"), + ("iterate", None), + ] @pytest.mark.asyncio @@ -237,11 +274,11 @@ async def test_multimodal_prefill_rejects_empty_engine_stream(): released = [] handler.embeddings_processor = SimpleNamespace(release_embeddings=released.append) - @asynccontextmanager - async def cancellation_monitor(request_id_future, context): - yield + async def wait_for_registration(rid): + return None - handler._cancellation_monitor = cancellation_monitor + handler._wait_for_request_registration = wait_for_registration + handler._cancellation_monitor = _never_cancels async def empty_results(): if False: @@ -254,7 +291,8 @@ async def empty_results(): await handler._consume_results( empty_results(), 23, - SimpleNamespace(), + "request-id", + _FakeContext("request-id"), owns_tensor, request_started, ) @@ -284,17 +322,17 @@ async def results(): finally: result_stopped.set() - async def start_prefill(request, bootstrap_room, context=None): + async def start_prefill(request, bootstrap_room, rid=None, context=None): return results(), 23 - @asynccontextmanager - async def cancellation_monitor(request_id_future, context): - yield + async def wait_for_registration(rid): + return None handler._start_prefill_generation = start_prefill - handler._cancellation_monitor = cancellation_monitor + handler._wait_for_request_registration = wait_for_registration + handler._cancellation_monitor = _never_cancels - stream = handler.generate(object(), SimpleNamespace()) + stream = handler.generate(object(), _FakeContext("request-id")) await anext(stream) await stream.aclose() @@ -303,6 +341,57 @@ async def cancellation_monitor(request_id_future, context): assert not handler._consume_tasks +@pytest.mark.asyncio +async def test_multimodal_prefill_cancels_registered_rid_before_first_result(): + rid = "dynamo-request-id" + abort_calls = [] + result_stopped = asyncio.Event() + + class _TokenizerManager: + def __init__(self): + self.rid_to_state = {} + + def abort_request(self, *, rid, abort_all): + abort_calls.append((rid, abort_all)) + + tokenizer_manager = _TokenizerManager() + handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler) + handler.engine = SimpleNamespace(tokenizer_manager=tokenizer_manager) + handler.shutdown_event = None + handler.embeddings_processor = SimpleNamespace(release_embeddings=lambda _: None) + + async def results(): + tokenizer_manager.rid_to_state[rid] = object() + try: + await asyncio.Event().wait() + yield {} + finally: + result_stopped.set() + + owns_tensor = asyncio.Event() + request_started = asyncio.Event() + context = _FakeContext(rid) + consumer = asyncio.create_task( + handler._consume_results( + results(), + None, + rid, + context, + owns_tensor, + request_started, + ) + ) + + await asyncio.wait_for(request_started.wait(), timeout=1) + context.cancel() + + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(consumer, timeout=1) + + assert abort_calls == [(rid, False)] + assert result_stopped.is_set() + + @pytest.mark.asyncio async def test_multimodal_prefill_cleanup_awaits_consumers_before_engine_shutdown(): handler = MultimodalPrefillWorkerHandler.__new__(MultimodalPrefillWorkerHandler) From 51bdae88c4bf6fe2beff2a06d6148b85e072236e Mon Sep 17 00:00:00 2001 From: jain-ria Date: Thu, 20 Aug 2026 07:56:29 -0700 Subject: [PATCH 10/11] build(sglang): bump NIXL to 1.3.2 Signed-off-by: jain-ria --- container/context.yaml | 2 +- docs/fern/components/releases.data.ts | 2 +- .../pages/reference/general/compatibility.mdx | 2 +- .../general/releases-machine-readable.mdx | 2 +- .../tests/test_sglang_patches.py | 32 +------------------ pyproject.toml | 2 +- 6 files changed, 6 insertions(+), 36 deletions(-) diff --git a/container/context.yaml b/container/context.yaml index 64d5e008c3eb..02ff33f12241 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -132,7 +132,7 @@ sglang: # (see templates/dev.Dockerfile). Runtime stays on the upstream lmsysorg/sglang # NIXL Python stack — its wheel COPY is narrowed to ai_dynamo*.whl so the SDK # build doesn't leak into the runtime image. - nixl_ref: v1.3.0 + nixl_ref: v1.3.2 enable_media_ffmpeg: "false" enable_gpu_memory_service: "true" enable_kvbm: "false" diff --git a/docs/fern/components/releases.data.ts b/docs/fern/components/releases.data.ts index 0364e363a09b..6207174523e2 100644 --- a/docs/fern/components/releases.data.ts +++ b/docs/fern/components/releases.data.ts @@ -74,7 +74,7 @@ export const MAIN_TOT: BackendPins = { sglang: "0.5.17", trtllm: "1.3.0rc23", vllm: "0.26.0", - nixlSglang: "1.3.0", + nixlSglang: "1.3.2", nixlTrtllm: "1.3.1", nixlVllm: "1.3.2", }; diff --git a/docs/fern/pages/reference/general/compatibility.mdx b/docs/fern/pages/reference/general/compatibility.mdx index f5817989aefe..bec1b5a4cfdf 100644 --- a/docs/fern/pages/reference/general/compatibility.mdx +++ b/docs/fern/pages/reference/general/compatibility.mdx @@ -156,7 +156,7 @@ Current stable release: v1.4.0 (container tag `1.4.0`, wheel version `1.4.0`). | Dynamo | Type | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX | | --- | --- | --- | --- | --- | --- | --- | -| main (ToT) | development head | 0.5.17 | 1.3.0rc23 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | - | +| main (ToT) | development head | 0.5.17 | 1.3.0rc23 | 0.26.0 | 1.3.2 / 1.3.1 / 1.3.2 | - | | v1.4.0 | stable | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | | v1.3.1 | patch | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.2 / 1.0.1 / 1.1.0 | 1.20.x | | v1.3.0 | stable | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.0 / 1.0.1 / 1.1.0 | 1.20.x | diff --git a/docs/fern/pages/reference/general/releases-machine-readable.mdx b/docs/fern/pages/reference/general/releases-machine-readable.mdx index 2f0a477053cc..042304295056 100644 --- a/docs/fern/pages/reference/general/releases-machine-readable.mdx +++ b/docs/fern/pages/reference/general/releases-machine-readable.mdx @@ -15,7 +15,7 @@ Current stable release: v1.4.0 (Aug 14, 2026; container tag `1.4.0`, wheel versi | Version | Kind | Date | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX | Notes | Delta | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| main (ToT) | development head | - | 0.5.17 | 1.3.0rc23 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | - | - | - | +| main (ToT) | development head | - | 0.5.17 | 1.3.0rc23 | 0.26.0 | 1.3.2 / 1.3.1 / 1.3.2 | - | - | - | | v1.4.0 | stable | Aug 14, 2026 | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-4-0) | Audit subsystem migrated into request trace (DYN_AUDIT_* honored as legacy aliases); HTTP header capture in trace records is an explicit fail-closed allowlist; deprecated multimodal worker flags and vLLM worker-role flags removed; runtime images no longer bundle software video decoders (H.264/H.265 decodes via NVDEC); UCX 1.21.x. | | v1.3.1 | patch | Aug 5, 2026 | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.2 / 1.0.1 / 1.1.0 | 1.20.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-3-0#v131) | Patch release. Fixes disaggregated SGLang serving over AWS EFA on GB200: the SGLang EFA runtime moves to NIXL 1.3.2 and all three EFA images to EFA Installer 1.49.0. Backend pins are otherwise unchanged from v1.3.0. | | v1.3.0 | stable | Jul 20, 2026 | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.0 / 1.0.1 / 1.1.0 | 1.20.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-3-0) | CUDA 12 container images discontinued; EFA variants retagged from -efa-amd64 to -efa (the images were already multi-arch — the old suffix was misleading); GA wheels published as 1.3.0.post1 (containers stay :1.3.0); UCX 1.20.x. | diff --git a/lib/gpu_memory_service/tests/test_sglang_patches.py b/lib/gpu_memory_service/tests/test_sglang_patches.py index 6315ec726c4c..07c521c80051 100644 --- a/lib/gpu_memory_service/tests/test_sglang_patches.py +++ b/lib/gpu_memory_service/tests/test_sglang_patches.py @@ -18,7 +18,7 @@ if not HAS_TORCH: pytest.skip("torch is required", allow_module_level=True) -from gpu_memory_service.integrations.sglang import patches, setup_gms +from gpu_memory_service.integrations.sglang import patches pytestmark = [ pytest.mark.pre_merge, @@ -79,33 +79,3 @@ def alloc_memory_pool(self): assert runner.alloc_memory_pool() == 10.0 assert runner.pre_model_load_memory == 10.0 - - -def test_setup_gms_uses_read_only_server_args_override(monkeypatch): - class GMSModelLoader: - pass - - model_loader_module = ModuleType( - "gpu_memory_service.integrations.sglang.model_loader" - ) - model_loader_module.GMSModelLoader = GMSModelLoader - monkeypatch.setitem(sys.modules, model_loader_module.__name__, model_loader_module) - - class ReadOnlyServerArgs: - enable_weights_cpu_backup = False - enable_draft_weights_cpu_backup = False - model_loader_extra_config = None - - def __init__(self): - object.__setattr__(self, "overrides", []) - - def __setattr__(self, name, value): - raise AttributeError(f"{name} is read-only") - - def override(self, source, **fields): - self.overrides.append((source, fields)) - - server_args = ReadOnlyServerArgs() - - assert setup_gms(server_args) is GMSModelLoader - assert server_args.overrides == [("dynamo.gms", {"enable_memory_saver": True})] diff --git a/pyproject.toml b/pyproject.toml index 5683f17b7591..73c670df2bc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ sglang = [ # sglang[diffusion] dropped accelerate in 0.5.12; diffusers still needs it. "accelerate>=0.17.0", "blake3>=1.0.0,<2.0.0", - "nixl[cu13]==1.3.0", + "nixl[cu13]==1.3.2", "cupy-cuda12x>=13.0.0", ] From c7cb8e47b5c709884dc4982ac9f7faf69405d354 Mon Sep 17 00:00:00 2001 From: jain-ria Date: Thu, 20 Aug 2026 15:58:01 -0700 Subject: [PATCH 11/11] fix(ci): unblock SGLang bump checks Signed-off-by: jain-ria --- .../actions/setup-dynamo-operator/action.yml | 23 +++++++++++++++++++ .../src/dynamo/sglang/init_multimodal.py | 2 +- .../multimodal/worker_handler.py | 9 ++++++-- .../tests/test_sglang_multimodal_video.py | 2 +- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup-dynamo-operator/action.yml b/.github/actions/setup-dynamo-operator/action.yml index b9ca9dd2f5cb..44fa18956d5d 100644 --- a/.github/actions/setup-dynamo-operator/action.yml +++ b/.github/actions/setup-dynamo-operator/action.yml @@ -470,6 +470,29 @@ runs: run: | echo "${DOCKERHUB_PASS}" | helm registry login registry-1.docker.io -u "${DOCKERHUB_USER}" --password-stdin + # The operator refuses to start when checkpoint is enabled unless the + # PodSnapshot API exists. Snapshot-agent (which owns that CRD) is installed + # in a later job, so apply the CRDs first and let that later job adopt them. + - name: Install snapshot CRDs for checkpoint + if: inputs.checkpoint_enabled == 'true' + shell: bash + env: + # Keep in lockstep with .github/actions/setup-snapshot-agent/action.yml. + SNAPSHOT_CHART_VERSION: 0.1.0-alpha.1 + run: | + echo "::group::Install snapshot CRDs before operator start" + set -euo pipefail + VKUBECONFIG="${{ github.workspace }}/.kubeconfig-vcluster" + helm show crds oci://ghcr.io/ai-dynamo/snapshot/snapshot \ + --version "${SNAPSHOT_CHART_VERSION}" \ + | kubectl --kubeconfig="${VKUBECONFIG}" \ + apply --server-side --force-conflicts --field-manager=ci-setup -f - + kubectl --kubeconfig="${VKUBECONFIG}" wait --for=condition=established \ + crd/podsnapshots.nvidia.com \ + crd/podsnapshotcontents.nvidia.com \ + --timeout=120s + echo "::endgroup::" + - name: Install Dynamo platform via Helm shell: bash env: diff --git a/components/src/dynamo/sglang/init_multimodal.py b/components/src/dynamo/sglang/init_multimodal.py index d8d70812bd34..9c010e445f0b 100644 --- a/components/src/dynamo/sglang/init_multimodal.py +++ b/components/src/dynamo/sglang/init_multimodal.py @@ -253,7 +253,7 @@ async def init_multimodal_prefill_worker( logging.error(f"Failed to serve endpoints: {e}") raise finally: - await handler.cleanup() + await handler.cleanup_async() if run_deferred_handlers is not None: logging.info("Running deferred handlers") await run_deferred_handlers() diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py index 32310eaee097..7aa25668bfc3 100644 --- a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py +++ b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py @@ -1010,7 +1010,12 @@ def process_result(result: dict[str, Any]) -> None: ) if registration_task in done: break - if first_result_task in done and not first_result_ready: + if first_result_task is not None: + first_result_is_done = first_result_task in done + else: + first_result_is_done = False + if first_result_is_done and not first_result_ready: + assert first_result_task is not None try: first_result = await first_result_task except StopAsyncIteration as e: @@ -1141,7 +1146,7 @@ def process_result(result: dict[str, Any]) -> None: if tensor_id is not None and not released: self.embeddings_processor.release_embeddings(tensor_id) - async def cleanup(self) -> None: + async def cleanup_async(self) -> None: tasks = list(self._consume_tasks) for task in tasks: if not task.done(): diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py index 9184b0d4427c..bd0e1c805499 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py +++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py @@ -409,7 +409,7 @@ async def consumer(): handler._consume_tasks = {task} handler.engine = SimpleNamespace(shutdown=lambda: events.append("engine shutdown")) - await handler.cleanup() + await handler.cleanup_async() assert events == ["consumer stopped", "engine shutdown"] assert not handler._consume_tasks