From 465a9dd5b7f7f24fa36e2ff5725556571be7b3d0 Mon Sep 17 00:00:00 2001 From: Rohan Varma Date: Sun, 25 Oct 2026 04:22:14 -0400 Subject: [PATCH 1/8] [Feat] [NVIDIA] megakernels: DSA / MLA binding view, EAGLE draft capture checks, early draft embedding sharing A megakernel that replaces the decode or verify step of a DSA (sparse MLA) model reads the DSA backend's layout and decode buffers and writes the MLA and index-K rows of its KV pool. Offer them through the binding: KvPoolView gains the MLA key and index-K buffers, the index page size and the MLA key write; ForwardView.dsa is a DsaView of the running DSA backend (MLA dimensions, decode implementation, top-k and DCP settings, the decode workspace and the multi-CTA KV counter as sealed data, and the indexer metadata). TOLERATED_DSA_CALLABLES / TOLERATED_DSA_WRITES list what a megakernel still calls or writes; the existing TOLERATED_* lists are unchanged. The EAGLE draft-decode and draft-extend graph runners now call the capture tail hooks at the end of their captured function, as the decode graph runner does, so a megakernel spec installed on a draft model is checked after capture. The draft-decode runner runs them under its first per-step attention backend and checks that the per-step backends share one KV pool. spec.EAGLE_DRAFT_CAPTURE_CHECK marks an SGLang that does this; Step documents the draft modes. Under SGLANG_ENABLE_EMBED_REPLICATION the NextN draft loads its own full embedding table on every rank. EagleDraftWorker replaced it with the target's in alloc_memory_pool, which the scheduler calls after it has sized the target KV pool, so the draft's copy (and its lm_head shard) was still allocated when the pool was sized. Under replication the draft worker now binds the target's embedding and lm_head at the end of its construction, so the draft's own copies are freed before the pool is sized and the pool can use that memory. Without replication the calls stay where they were. With --enable-nvidia-megakernel, argument resolution now sets a variable of spec.FLAG_SETS_ENV (SGLANG_ENABLE_EMBED_REPLICATION) that the served spec requires and the user left unset, to the spec's value, and logs it. SGLang reads that variable only when a model is built, in the model processes started after resolution, so every rank sees the value. A value the user set is kept and checked as before, and a refused setup leaves the environment as it was. --- .../kernels/plugin_kernels/nvidia/spec.py | 21 +- .../srt/kernel_plugins/nvidia/binding.py | 188 +++++++++++++++- .../eagle_draft_cuda_graph_runner.py | 20 ++ .../eagle_draft_extend_cuda_graph_runner.py | 7 + .../sglang/srt/speculative/eagle_worker_v2.py | 19 +- .../unit/kernel_plugins/nvidia/_fakes.py | 10 + .../nvidia/test_megakernel_binding_dsa.py | 124 +++++++++++ .../nvidia/test_megakernel_draft_capture.py | 202 ++++++++++++++++++ .../nvidia/test_megakernel_guards.py | 67 +++++- .../unit/spec/test_eagle_draft_embed_share.py | 70 ++++++ 10 files changed, 714 insertions(+), 14 deletions(-) create mode 100644 test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding_dsa.py create mode 100644 test/registered/unit/kernel_plugins/nvidia/test_megakernel_draft_capture.py create mode 100644 test/registered/unit/spec/test_eagle_draft_embed_share.py diff --git a/python/sglang/kernels/plugin_kernels/nvidia/spec.py b/python/sglang/kernels/plugin_kernels/nvidia/spec.py index 312e1c95e985..357cca55fc20 100644 --- a/python/sglang/kernels/plugin_kernels/nvidia/spec.py +++ b/python/sglang/kernels/plugin_kernels/nvidia/spec.py @@ -18,6 +18,13 @@ # the sglang.srt.plugins entry point that registers the specs ENTRY_POINT = "nvidia_megakernel" FLAG = "--enable-nvidia-megakernel" +# The EAGLE draft graph runners run a draft model's capture check (Step); a draft +# spec that serves steps can require it +EAGLE_DRAFT_CAPTURE_CHECK = True +# The variables of Requirements.env that the flag sets to the spec's value when they +# are unset, at argument resolution: SGLang reads each only when a model is built, in +# the model processes started after it. A value the user set is kept and checked. +FLAG_SETS_ENV = frozenset({"SGLANG_ENABLE_EMBED_REPLICATION"}) class Feature(enum.Enum): @@ -59,7 +66,8 @@ class Requirements(msgspec.Struct, frozen=True): server_args: Mapping[str, frozenset] = {} # an HF config field -> the value the kernels are built for hf_config: Mapping[str, Any] = {} - # an SGLang environment variable in envs -> its required value + # an SGLang environment variable in envs -> its required value (the flag sets one + # in FLAG_SETS_ENV that is unset) env: Mapping[str, Any] = {} @@ -88,9 +96,16 @@ class BuildContext(msgspec.Struct, frozen=True): class Step(msgspec.Struct, frozen=True): - """A covered step, decided from host state alike on every rank.""" + """A covered step, decided from host state alike on every rank. - mode: str # "decode" or "target_verify" + A draft spec's steps may be off its width grid (an EAGLE draft decode step has + one row a request); the dispatch does not check rows against the widths. The + capture check of a draft model runs once, when the first of its EAGLE draft + graph runners has captured every batch size. + """ + + # "decode" or "target_verify"; a draft spec's "draft_decode" or "draft_extend" + mode: str rows: int # the running forward's ForwardView, set by SGLang before forward runs the step view: Any = None diff --git a/python/sglang/srt/kernel_plugins/nvidia/binding.py b/python/sglang/srt/kernel_plugins/nvidia/binding.py index 1d70291219b9..90a2edcf5d10 100644 --- a/python/sglang/srt/kernel_plugins/nvidia/binding.py +++ b/python/sglang/srt/kernel_plugins/nvidia/binding.py @@ -4,11 +4,12 @@ It checks a plugin's spec against the resolved configuration at startup, builds it once the model is prepared, and installs the whole-step dispatch on the model. It is also the one place a megakernel reads SGLang's runtime state: the KV pool, the -attention metadata of the running forward and the Engram tables. A spec's build gets -a ``MegakernelBinding`` as ``BuildContext.binding``; a covered forward gets the -running forward's ``ForwardView`` as ``Step.view`` (a draft entry asks -``binding.forward_view()``). Everything else a megakernel touches is the model it -replaces, SGLang's kernel ops, and the reads listed below. +attention metadata of the running forward, the DSA (sparse MLA) backend's layout and +buffers and the Engram tables. A spec's build gets a ``MegakernelBinding`` as +``BuildContext.binding``; a covered forward gets the running forward's +``ForwardView`` as ``Step.view`` (a draft entry asks ``binding.forward_view()``). +Everything else a megakernel touches is the model it replaces, SGLang's kernel ops, +and the reads listed below. """ from __future__ import annotations @@ -26,6 +27,7 @@ from sglang.kernels.plugin_kernels.nvidia.spec import ( ENTRY_POINT, FLAG, + FLAG_SETS_ENV, BuildContext, BuiltMegakernel, Feature, @@ -82,6 +84,18 @@ "capture-mode queries (get_is_capture_mode, is_in_breakable_cuda_graph)", "MegakernelBinding.engram_host_table", # the per-layer host table object ) +# Called by a megakernel on a DSA (sparse MLA) backend, tolerated until offered as +# data. +TOLERATED_DSA_CALLABLES = ( + "KvPoolView.set_mla_kv_buffer", # the MLA key write + "DsaView.indexer_metadata", # the indexer's metadata of the running forward +) +# Written into the DSA pool by a megakernel (the step's rows, at its out_cache_loc), +# tolerated until SGLang owns the writes. +TOLERATED_DSA_WRITES = ( + "KvPoolView.get_key_buffer", # the MLA key rows + "KvPoolView.get_index_k_with_scale_buffer", # the index-K rows and their scales +) class KvPoolView: @@ -121,6 +135,100 @@ def get_extra_key_bytes_per_token(self, layer_id: int) -> int: def set_swa_key_buffer_radix_fused_norm_rope(self) -> Callable[..., None]: return self.pool.set_swa_key_buffer_radix_fused_norm_rope + # MLA / DSA pools + def get_key_buffer(self, layer_id: int) -> torch.Tensor: + return self.pool.get_key_buffer(layer_id) + + def get_index_k_with_scale_buffer(self, layer_id: int) -> torch.Tensor: + return self.pool.get_index_k_with_scale_buffer(layer_id) + + @property + def index_page_size(self) -> Optional[int]: + """Tokens per page of the index-K buffer; None for a pool without one.""" + return getattr(self.pool, "index_page_size", None) + + @property + def set_mla_kv_buffer(self) -> Callable[..., None]: + return self.pool.set_mla_kv_buffer + + +class DsaView: + """The running DSA (sparse MLA) backend's layout and the buffers its decode + uses, under names a megakernel can rely on.""" + + def __init__(self, backend: Any): + self._backend = backend + + @property + def kv_lora_rank(self) -> int: + return self._backend.kv_lora_rank + + @property + def qk_nope_head_dim(self) -> int: + return self._backend.qk_nope_head_dim + + @property + def qk_rope_head_dim(self) -> int: + return self._backend.qk_rope_head_dim + + @property + def kv_cache_dim(self) -> int: + return self._backend.kv_cache_dim + + @property + def physical_page_size(self) -> int: + return self._backend.physical_page_size + + @property + def decode_impl(self) -> str: + return self._backend.dsa_decode_impl + + @property + def use_mha(self) -> bool: + """Whether the running forward takes the MHA path instead of MLA.""" + return self._backend.use_mha + + @property + def kv_cache_dtype(self) -> torch.dtype: + return self._backend.kv_cache_dtype + + @property + def use_fused_topk(self) -> bool: + return self._backend.use_fused_topk + + @property + def topk_backend(self) -> Any: + return self._backend.dsa_topk_backend + + @property + def dcp_size(self) -> int: + return self._backend.dcp_size + + @property + def index_kpool(self) -> int: + return self._backend.dsa_index_kpool + + @property + def hisparse(self) -> bool: + return self._backend.hisparse_coordinator is not None + + @property + def workspace_buffer(self) -> Optional[torch.Tensor]: + """The decode kernels' workspace, or None when the backend has none.""" + return self._backend.workspace_buffer + + @property + def multi_ctas_kv_counter_buffer(self) -> Optional[torch.Tensor]: + """The TRT-LLM decode's multi-CTA KV counter, sized before the decode graphs + are captured; None when the backend has none.""" + return self._backend._multi_ctas_kv_counter_buffer + + @property + def indexer_metadata(self) -> Callable[[int, Any], Any]: + """The backend's indexer metadata of the running forward, by + (layer_id, forward_batch).""" + return self._backend.get_indexer_metadata + class ForwardView: """The running forward's attention data, read from its attention backend; built @@ -155,6 +263,11 @@ def low_ratio_sources(self) -> Callable[..., Any]: def core_compressor(self) -> Callable[..., Any]: return self._backend.forward_core_compressor + @property + def dsa(self) -> Optional[DsaView]: + """The DSA backend's view, or None when the running backend is not DSA.""" + return _dsa_view(self._backend) + class MegakernelBinding: """Built at install, before the KV pools exist; read at build or forward. Each @@ -171,6 +284,11 @@ def forward_view(self) -> ForwardView: view = ForwardView(get_attn_backend()) self._seal("kv_pool", view.kv_pool.pool) + dsa = view.dsa + if dsa is not None: + for name, buffer in _dsa_buffers(dsa).items(): + if buffer is not None: + self._seal(name, buffer) return view def engram_history(self) -> Optional[torch.Tensor]: @@ -186,7 +304,12 @@ def engram_host_table(self, engram: torch.nn.Module) -> Any: return engram.embed.host_table def assert_live(self) -> None: - current = {"kv_pool": self._current_pool, "engram_history": self._history} + current = { + "kv_pool": self._current_pool, + "dsa_workspace": lambda: self._current_dsa_buffer("dsa_workspace"), + "dsa_kv_counter": lambda: self._current_dsa_buffer("dsa_kv_counter"), + "engram_history": self._history, + } for name, sealed in self._sealed.items(): if _identity(current[name]()) != sealed: raise RuntimeError( @@ -201,12 +324,33 @@ def _current_pool(self) -> Any: return get_attn_backend().token_to_kv_pool + def _current_dsa_buffer(self, name: str) -> Optional[torch.Tensor]: + from sglang.srt.model_executor.forward_context import get_attn_backend + + dsa = _dsa_view(get_attn_backend()) + return None if dsa is None else _dsa_buffers(dsa)[name] + def _history(self) -> Optional[torch.Tensor]: # only models with Engram layers have a hasher hasher = getattr(self._model, "engram_hasher", None) return None if hasher is None else hasher.history +def _dsa_view(backend: Any) -> Optional[DsaView]: + # only the DSA backend resolves a DSA decode implementation + if getattr(backend, "dsa_decode_impl", None) is None: + return None + return DsaView(backend) + + +def _dsa_buffers(dsa: DsaView) -> dict[str, Optional[torch.Tensor]]: + """The DSA backend's buffers a captured step reads by address, sealed by name.""" + return { + "dsa_workspace": dsa.workspace_buffer, + "dsa_kv_counter": dsa.multi_ctas_kv_counter_buffer, + } + + def _identity(value: Any) -> tuple: if isinstance(value, torch.Tensor): return (value.data_ptr(), tuple(value.shape), value.dtype) @@ -360,28 +504,56 @@ def _dtype_reasons(spec: Optional[MegakernelSpec], model_config: Any) -> list[st return [f"dtype {dtype!r} is not one of {sorted(spec.requires.dtypes)}"] +def set_flag_env(spec: Optional[MegakernelSpec]) -> list[str]: + """Set each variable of FLAG_SETS_ENV that the spec requires and the user left + unset to the spec's value; return their names.""" + if spec is None: + return [] + names = [ + name + for name in spec.requires.env + if name in FLAG_SETS_ENV and not getattr(envs, name).is_set() + ] + for name in names: + getattr(envs, name).set(spec.requires.env[name]) + return names + + def handle_nvidia_megakernel(server_args: Any) -> None: """Resolution step: fail before the weights load when the flag cannot be served, - listing every reason.""" + listing every reason. The model processes, started after it, inherit what it sets + for the spec (FLAG_SETS_ENV).""" cfg = resolving_view(server_args) if not cfg.enable_nvidia_megakernel: return + env_set = [] reasons = plugin_reasons() if not reasons: model_config = model_config_of(server_args) + hf_config, isa = model_config.hf_config, device_isa() + env_set = set_flag_env( + find_spec(architectures=hf_config.architectures, isa=isa)[0] + ) spec, reasons = megakernel_disable_reasons( - cfg=cfg, hf_config=model_config.hf_config, isa=device_isa() + cfg=cfg, hf_config=hf_config, isa=isa ) reasons += _dtype_reasons(spec, model_config) if cfg.enable_amd_monokernel: # both replace the model's forward, which would fail only after weights load reasons.append("--enable-amd-monokernel is also set; use one of the two") if reasons: + # a refused setup leaves the environment as it was + for name in env_set: + getattr(envs, name).clear() raise ValueError( f"{FLAG} cannot run this setup:\n - " + "\n - ".join(reasons) ) logger.warning("%s is experimental. Its kernels and coverage may change.", FLAG) logger.info("%s: %s on %s", FLAG, spec.name, device_isa()) + for name in env_set: + logger.info( + "%s: set %s=%s for %s", FLAG, name, getattr(envs, name).get(), spec.name + ) def megakernel_enabled() -> bool: diff --git a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py index 8d57fdd45095..56016ecc61b6 100644 --- a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py @@ -377,6 +377,25 @@ def can_run_graph(self, forward_batch: ForwardBatch): # ----------------------------------------------------------------- # Capture (per-shape) # ----------------------------------------------------------------- + def _run_capture_tail_hooks(self, out, forward_batch, num_tokens: int) -> None: + """The model runner's capture tail hooks, after the draft forwards of a captured + batch size. The draft steps ran under the per-step attention backends, which + share the draft's KV pool (checked here when a hook exists); a hook that reads + the running backend sees the first.""" + hooks = self.model_runner.capture_tail_hooks + if not hooks: + return + backends = self.draft_attn_backend.attn_backends + pool = getattr(backends[0], "token_to_kv_pool", None) + if any(getattr(b, "token_to_kv_pool", None) is not pool for b in backends): + raise RuntimeError( + "the draft's per-step attention backends do not share one KV pool; " + "the capture tail hooks check the running backend's" + ) + with forward_context(ForwardContext(attn_backend=backends[0])): + for capture_hook in hooks: + capture_hook(self, out, forward_batch, num_tokens) + def capture_one_shape( self, size: int, @@ -522,6 +541,7 @@ def run_once(): forward_batch.spec_info.hidden_states = hidden_states_backup forward_batch.spec_info.dsa_topk_indices = dsa_topk_indices_backup forward_batch.positions.sub_(self.eagle_worker.speculative_num_steps - 1) + self._run_capture_tail_hooks(ret, forward_batch, num_tokens) return ret with forward_context( diff --git a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py index 9a3f7b7a8222..ecc5b5a872cc 100644 --- a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py @@ -347,6 +347,12 @@ def can_run_graph(self, forward_batch: ForwardBatch): return is_bs_supported + def _run_capture_tail_hooks(self, out, forward_batch, num_tokens: int) -> None: + """The model runner's capture tail hooks, after the draft-extend forward of a + captured batch size (under the runner's own attention backend).""" + for capture_hook in self.model_runner.capture_tail_hooks: + capture_hook(self, out, forward_batch, num_tokens) + def capture_one_shape( self, size: int, @@ -479,6 +485,7 @@ def run_once(): forward_batch.out_cache_loc = output_cache_loc_backup forward_batch.spec_info.hidden_states = hidden_states_backup + self._run_capture_tail_hooks(ret, forward_batch, num_tokens) return ret with forward_context( diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 4aaa01e5ea1d..36cfe6825373 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -240,6 +240,21 @@ def __init__( self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device) + self.target_embed_and_head_shared = False + self._maybe_share_embed_before_pool_sizing() + + def _maybe_share_embed_before_pool_sizing(self): + """Under embedding replication the draft loads its own full embedding table on + every rank. Share the target's now, so that copy is freed before the scheduler + sizes the KV pool instead of after.""" + if envs.SGLANG_ENABLE_EMBED_REPLICATION.get(): + self._share_target_embed_and_head() + + def _share_target_embed_and_head(self): + self.init_token_map() + self.init_lm_head() + self.target_embed_and_head_shared = True + def alloc_memory_pool( self, memory_pool_config=None, @@ -254,8 +269,8 @@ def alloc_memory_pool( req_to_token_pool=req_to_token_pool, token_to_kv_pool_allocator=token_to_kv_pool_allocator, ) - self.init_token_map() - self.init_lm_head() + if not self.target_embed_and_head_shared: + self._share_target_embed_and_head() if get_spec().speculative_use_rejection_sampling: target_vocab_size = self.target_worker.model_config.vocab_size diff --git a/test/registered/unit/kernel_plugins/nvidia/_fakes.py b/test/registered/unit/kernel_plugins/nvidia/_fakes.py index a75fd9fb2c5e..4bb362de0ca2 100644 --- a/test/registered/unit/kernel_plugins/nvidia/_fakes.py +++ b/test/registered/unit/kernel_plugins/nvidia/_fakes.py @@ -63,6 +63,16 @@ def _build(model, hf_config, context): ) # an empty field accepts any value ANY_TP = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset())) +# specs that need the replicated input embedding, which the flag sets when it is unset +# (FLAG_SETS_ENV); the second also needs a variable the flag never sets +REPLICATION = "SGLANG_ENABLE_EMBED_REPLICATION" +REPLICATED = replace(SPEC, requires=replace(SPEC.requires, env={REPLICATION: True})) +REPLICATED_AND_BROADCAST = replace( + SPEC, + requires=replace( + SPEC.requires, env={REPLICATION: True, "SGLANG_DSA_TOPK_BROADCAST": True} + ), +) # a draft spec: builds no width, offers only the draft sampler DRAFT = replace( SPEC, diff --git a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding_dsa.py b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding_dsa.py new file mode 100644 index 000000000000..200d37185cac --- /dev/null +++ b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding_dsa.py @@ -0,0 +1,124 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU checks of the megakernel binding's DSA (sparse MLA) view: it is the running +DSA backend and its pool, other backends have none, and a replaced decode buffer is +caught.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.kernel_plugins.nvidia import binding as mb +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") + +BACKEND = "sglang.srt.model_executor.forward_context.get_attn_backend" + + +class DsaPool: + index_page_size = 64 + + def __init__(self): + self.keys = [torch.zeros(4, 1, 8) for _ in range(2)] + self.index_k = [torch.zeros(4, 12) for _ in range(2)] + + def get_key_buffer(self, layer_id): + return self.keys[layer_id] + + def get_index_k_with_scale_buffer(self, layer_id): + return self.index_k[layer_id] + + def set_mla_kv_buffer(self, layer, loc, k, k_rope): + return layer, loc, k, k_rope + + +def dsa_backend(pool): + return SimpleNamespace( + token_to_kv_pool=pool, + forward_metadata="metadata", + dsa_decode_impl="trtllm", + kv_lora_rank=512, + qk_nope_head_dim=192, + qk_rope_head_dim=64, + kv_cache_dim=656, + physical_page_size=64, + use_mha=False, + kv_cache_dtype=torch.float8_e4m3fn, + use_fused_topk=True, + dsa_topk_backend="topk backend", + dcp_size=1, + dsa_index_kpool=1, + hisparse_coordinator=None, + workspace_buffer=torch.zeros(16, dtype=torch.uint8), + _multi_ctas_kv_counter_buffer=torch.zeros(8, dtype=torch.uint8), + get_indexer_metadata=lambda layer_id, fb: ("indexer", layer_id, fb), + ) + + +class TestMegakernelBindingDsa(CustomTestCase): + def test_the_dsa_view_is_the_running_dsa_backend(self): + backend = dsa_backend(DsaPool()) + with patch(BACKEND, return_value=backend): + dsa = mb.MegakernelBinding(model=torch.nn.Module()).forward_view().dsa + same = { + "kv_lora_rank": "kv_lora_rank", + "qk_nope_head_dim": "qk_nope_head_dim", + "qk_rope_head_dim": "qk_rope_head_dim", + "kv_cache_dim": "kv_cache_dim", + "physical_page_size": "physical_page_size", + "decode_impl": "dsa_decode_impl", + "use_mha": "use_mha", + "kv_cache_dtype": "kv_cache_dtype", + "use_fused_topk": "use_fused_topk", + "topk_backend": "dsa_topk_backend", + "dcp_size": "dcp_size", + "index_kpool": "dsa_index_kpool", + "workspace_buffer": "workspace_buffer", + "multi_ctas_kv_counter_buffer": "_multi_ctas_kv_counter_buffer", + "indexer_metadata": "get_indexer_metadata", + } + for member, attr in same.items(): + self.assertIs(getattr(dsa, member), getattr(backend, attr), member) + self.assertFalse(dsa.hisparse) + backend.use_mha = True # read live: the backend sets it per forward + self.assertTrue(dsa.use_mha) + self.assertEqual(dsa.indexer_metadata(3, "fb"), ("indexer", 3, "fb")) + + def test_a_backend_without_dsa_has_no_dsa_view(self): + backend = SimpleNamespace(token_to_kv_pool=DsaPool(), forward_metadata=None) + with patch(BACKEND, return_value=backend): + binding = mb.MegakernelBinding(model=torch.nn.Module()) + self.assertIsNone(binding.forward_view().dsa) + binding.assert_live() + + def test_the_pool_view_offers_the_mla_and_index_k_buffers(self): + pool = DsaPool() + view = mb.KvPoolView(pool) + self.assertIs(view.get_key_buffer(1), pool.keys[1]) + self.assertIs(view.get_index_k_with_scale_buffer(0), pool.index_k[0]) + self.assertEqual(view.index_page_size, 64) + self.assertEqual(view.set_mla_kv_buffer(1, 2, 3, 4), (1, 2, 3, 4)) + # a pool without index-K pages + self.assertIsNone(mb.KvPoolView(SimpleNamespace()).index_page_size) + + def test_a_replaced_dsa_buffer_fails_assert_live(self): + for attr, name in ( + ("workspace_buffer", "dsa_workspace"), + ("_multi_ctas_kv_counter_buffer", "dsa_kv_counter"), + ): + with self.subTest(buffer=attr): + backend = dsa_backend(DsaPool()) + binding = mb.MegakernelBinding(model=torch.nn.Module()) + with patch(BACKEND, return_value=backend): + binding.forward_view() + binding.assert_live() + setattr(backend, attr, torch.zeros(32, dtype=torch.uint8)) + with self.assertRaisesRegex(RuntimeError, name): + binding.assert_live() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_draft_capture.py b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_draft_capture.py new file mode 100644 index 000000000000..41c66669d9c1 --- /dev/null +++ b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_draft_capture.py @@ -0,0 +1,202 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU checks of the capture check of a draft spec, fired by the EAGLE draft graph +runners' capture hook call: after the draft forwards of each captured batch size, +with the runner's batch sizes, under the draft decode's first per-step backend.""" + +import ast +import inspect +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import torch +from msgspec.structs import replace + +from sglang.kernels.plugin_kernels.nvidia import spec as registry +from sglang.kernels.plugin_kernels.nvidia.spec import Step +from sglang.srt.kernel_plugins.nvidia import binding +from sglang.srt.kernel_plugins.nvidia.binding import MegakernelBinding +from sglang.srt.model_executor.forward_context import get_attn_backend +from sglang.srt.runtime_context import publish, reset_context +from sglang.srt.speculative.eagle_draft_cuda_graph_runner import ( + EAGLEDraftCudaGraphRunner, +) +from sglang.srt.speculative.eagle_draft_extend_cuda_graph_runner import ( + EAGLEDraftExtendCudaGraphRunner, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from _fakes import ARCH, ISA, SPEC, Lm, batch, server_args # noqa: E402 + +register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") + + +class DraftBuilt: + """Covers draft decode steps of up to max_bs rows: one row a request, so off the + width grid.""" + + draft_sampler = None + + def __init__(self, widths, max_bs): + self.widths, self.max_bs = widths, max_bs + + def summary(self): + return "fake draft" + + def plan(self, input_ids, forward_batch, *args, **kwargs): + rows = int(input_ids.shape[0]) + return Step(mode="draft_decode", rows=rows) if rows <= self.max_bs else None + + def forward(self, step, original, input_ids, positions, forward_batch, **kw): + return "megakernel" + + +def draft_spec(max_bs): + return replace( + SPEC, build=lambda model, hf_config, ctx: DraftBuilt(ctx.widths, max_bs) + ) + + +def draft_decode_runner(hooks, capture_bs=(), backends=None): + """An EAGLE draft decode runner with only what its capture hook call reads.""" + pool = object() + runner = EAGLEDraftCudaGraphRunner.__new__(EAGLEDraftCudaGraphRunner) + runner.capture_bs = list(capture_bs) + runner.model_runner = SimpleNamespace(capture_tail_hooks=hooks) + runner.draft_attn_backend = SimpleNamespace( + attn_backends=backends + or [SimpleNamespace(token_to_kv_pool=pool, step=i) for i in range(3)] + ) + return runner + + +def capture_draft_graphs(spec, capture_bs, steps=2): + """Install spec for a draft model, then capture its graphs as the EAGLE draft + decode runner does: the draft forwards of a batch size, then its capture hook + call.""" + lm, hooks = Lm(), [] + gpu = SimpleNamespace(multi_processor_count=160) + with ( + patch.object(binding, "find_spec", return_value=(spec, "")), + patch.object(binding, "device_isa", return_value=ISA), + patch.object(binding, "deadline_s", return_value=30.0), + patch.object(binding, "_gather_tp", side_effect=lambda x, group=None: [x]), + patch.object(binding, "_tokens_per_request", return_value=4), + patch.object(binding, "_decode_graphs_enabled", return_value=True), + patch.object(torch.cuda, "current_device", return_value=0), + patch.object(torch.cuda, "get_device_properties", return_value=gpu), + patch.object(torch.cuda, "mem_get_info", return_value=(0, 0)), + ): + binding.install_megakernel( + model=lm, + hf_config=SimpleNamespace(architectures=[ARCH]), + capture_tail_hooks=hooks, + is_draft=True, + ) + runner = draft_decode_runner(hooks, capture_bs) + with patch.object(torch.cuda, "is_current_stream_capturing", return_value=True): + for bs in capture_bs: + out = [lm(batch(bs).input_ids, None, batch(bs)) for _ in range(steps)] + runner._run_capture_tail_hooks(out, batch(bs), bs) + return hooks + + +class TestMegakernelDraftCapture(CustomTestCase): + def setUp(self): + registry_patch = patch.dict(registry._REGISTRY, clear=True) + registry_patch.start() + self.addCleanup(registry_patch.stop) + registry.register(architecture=ARCH, path="_fakes:SPEC") + publish(server_args(enable_nvidia_megakernel=True), role="test") + self.addCleanup(reset_context) + view_patch = patch.object( + MegakernelBinding, "forward_view", return_value="view" + ) + view_patch.start() + self.addCleanup(view_patch.stop) + + def test_a_draft_runner_capture_is_checked(self): + with ( + patch.object(MegakernelBinding, "assert_live") as assert_live, + self.assertLogs(binding.logger, "INFO") as logs, + ): + hooks = capture_draft_graphs(draft_spec(max_bs=2), capture_bs=[1, 2, 4]) + self.assertEqual(len(hooks), 1) + assert_live.assert_called_once_with() + self.assertIn( + "runs the decode graphs of batch sizes [1, 2] (2 of 3)", + "".join(logs.output), + ) + + def test_a_draft_runner_with_no_routed_graph_is_refused(self): + with self.assertRaisesRegex(ValueError, "none of the 2 captured decode graphs"): + capture_draft_graphs(draft_spec(max_bs=0), capture_bs=[1, 2]) + + +class TestEagleDraftRunnersFireTheHooks(CustomTestCase): + def test_the_draft_decode_runner_uses_its_first_step_backend(self): + seen = [] + runner = draft_decode_runner( + [lambda *args: seen.append((args, get_attn_backend()))] + ) + runner._run_capture_tail_hooks("out", "forward batch", 8) + first = runner.draft_attn_backend.attn_backends[0] + self.assertEqual(seen, [((runner, "out", "forward batch", 8), first)]) + + def test_step_backends_on_different_pools_are_refused(self): + backends = [SimpleNamespace(token_to_kv_pool=object()) for _ in range(2)] + runner = draft_decode_runner([lambda *args: None], backends=backends) + with self.assertRaisesRegex(RuntimeError, "do not share one KV pool"): + runner._run_capture_tail_hooks("out", "forward batch", 8) + + def test_without_hooks_nothing_is_read(self): + runner = draft_decode_runner([], backends=[None]) + runner._run_capture_tail_hooks("out", "forward batch", 8) + + def test_the_draft_extend_runner_fires_the_hooks(self): + seen = [] + runner = EAGLEDraftExtendCudaGraphRunner.__new__( + EAGLEDraftExtendCudaGraphRunner + ) + runner.model_runner = SimpleNamespace( + capture_tail_hooks=[lambda *args: seen.append(args)] + ) + runner._run_capture_tail_hooks("out", "forward batch", 16) + self.assertEqual(seen, [(runner, "out", "forward batch", 16)]) + + def test_the_captured_functions_call_the_hooks(self): + """capture_one_shape's captured function ends with the hook call, with its + output, batch and token count.""" + for cls in (EAGLEDraftCudaGraphRunner, EAGLEDraftExtendCudaGraphRunner): + with self.subTest(runner=cls.__name__): + tree = ast.parse(inspect.getsource(inspect.getmodule(cls))) + owner = next( + n + for n in ast.walk(tree) + if isinstance(n, ast.ClassDef) and n.name == cls.__name__ + ) + capture = next( + n + for n in owner.body + if isinstance(n, ast.FunctionDef) and n.name == "capture_one_shape" + ) + run_once = next( + n + for n in ast.walk(capture) + if isinstance(n, ast.FunctionDef) and n.name == "run_once" + ) + calls = [ + [ast.unparse(a) for a in n.args] + for n in ast.walk(run_once) + if isinstance(n, ast.Call) + and ast.unparse(n.func) == "self._run_capture_tail_hooks" + ] + self.assertEqual(calls, [["ret", "forward_batch", "num_tokens"]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_guards.py b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_guards.py index cce17e415524..7673752840a7 100644 --- a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_guards.py +++ b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_guards.py @@ -1,5 +1,6 @@ """CPU checks of --enable-nvidia-megakernel's startup resolution.""" +import os import sys import unittest from pathlib import Path @@ -7,6 +8,7 @@ from unittest.mock import patch from sglang.kernels.plugin_kernels.nvidia import spec as registry +from sglang.srt.environ import envs from sglang.srt.kernel_plugins.nvidia import binding from sglang.srt.kernel_plugins.nvidia.binding import ( megakernel_disable_reasons, @@ -15,7 +17,7 @@ from sglang.test.test_utils import CustomTestCase sys.path.insert(0, str(Path(__file__).resolve().parent)) -from _fakes import ARCH, ISA, server_args # noqa: E402 +from _fakes import ARCH, ISA, REPLICATION, server_args # noqa: E402 register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") @@ -52,5 +54,68 @@ def tp_reasons(path): self.assertEqual(tp_reasons("_fakes:ANY_TP"), []) +class TestFlagSetsEnv(CustomTestCase): + """With the flag, a variable of FLAG_SETS_ENV that the served spec requires and + the user left unset is set to the spec's value; a set value is kept.""" + + def setUp(self): + for patcher in ( + patch.dict(registry._REGISTRY, clear=True), + patch.dict(os.environ), + ): + patcher.start() + self.addCleanup(patcher.stop) + for name in (REPLICATION, "SGLANG_DSA_TOPK_BROADCAST"): + os.environ.pop(name, None) + + def resolve(self, path="_fakes:REPLICATED", **kw): + registry.register(architecture=ARCH, path=path) + hf_config = SimpleNamespace(architectures=[ARCH], model_type="fake") + model_config = SimpleNamespace(hf_config=hf_config, dtype="bfloat16") + with ( + patch.object(binding, "plugin_reasons", return_value=[]), + patch.object(binding, "device_isa", return_value=ISA), + patch.object(binding, "model_config_of", return_value=model_config), + ): + binding.handle_nvidia_megakernel( + server_args(**{"enable_nvidia_megakernel": True, **kw}) + ) + + def test_an_unset_variable_is_set_and_logged(self): + with self.assertLogs(binding.logger, "INFO") as logs: + self.resolve() + self.assertTrue(envs.SGLANG_ENABLE_EMBED_REPLICATION.get()) + self.assertIn(f"set {REPLICATION}=True for fake", "\n".join(logs.output)) + + def test_a_set_value_is_kept_and_checked(self): + os.environ[REPLICATION] = "1" + self.resolve() + self.assertEqual(os.environ[REPLICATION], "1") + os.environ[REPLICATION] = "0" + with self.assertRaisesRegex(ValueError, f"{REPLICATION} False is not True"): + self.resolve() + self.assertEqual(os.environ[REPLICATION], "0") + + def test_without_the_flag_nothing_is_set(self): + self.resolve(enable_nvidia_megakernel=False) + self.assertNotIn(REPLICATION, os.environ) + + def test_a_spec_that_does_not_require_it_leaves_it_unset(self): + self.resolve(path="_fakes:SPEC") + self.assertNotIn(REPLICATION, os.environ) + + def test_a_refused_setup_leaves_the_environment_as_it_was(self): + # SGLANG_DSA_TOPK_BROADCAST is not in FLAG_SETS_ENV: checked, never set + with self.assertRaisesRegex( + ValueError, "SGLANG_DSA_TOPK_BROADCAST False is not True" + ): + self.resolve(path="_fakes:REPLICATED_AND_BROADCAST") + self.assertNotIn(REPLICATION, os.environ) + self.assertNotIn("SGLANG_DSA_TOPK_BROADCAST", os.environ) + + def test_the_flag_sets_only_variables_read_when_a_model_is_built(self): + self.assertEqual(registry.FLAG_SETS_ENV, {REPLICATION}) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/spec/test_eagle_draft_embed_share.py b/test/registered/unit/spec/test_eagle_draft_embed_share.py new file mode 100644 index 000000000000..966599b4afee --- /dev/null +++ b/test/registered/unit/spec/test_eagle_draft_embed_share.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU checks of when the EAGLE draft worker binds the target's embedding and lm_head: +under embedding replication before the scheduler sizes the KV pool (so the draft's own +full table is freed first), otherwise in ``alloc_memory_pool`` as before.""" + +import ast +import inspect +import textwrap +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from sglang.srt.environ import envs +from sglang.srt.speculative.eagle_worker_v2 import EagleDraftWorker +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _run(replicate: bool) -> list: + """The order of the draft worker's calls from the end of ``__init__`` through + ``alloc_memory_pool``.""" + calls = [] + worker = EagleDraftWorker.__new__(EagleDraftWorker) + worker.draft_worker = MagicMock() + worker.draft_worker.alloc_memory_pool.side_effect = lambda **_: calls.append( + "draft_pool" + ) + with ( + envs.SGLANG_ENABLE_EMBED_REPLICATION.override(replicate), + patch.object( + EagleDraftWorker, "init_token_map", lambda self: calls.append("token_map") + ), + patch.object( + EagleDraftWorker, "init_lm_head", lambda self: calls.append("lm_head") + ), + patch( + "sglang.srt.speculative.eagle_worker_v2.get_spec", + return_value=SimpleNamespace(speculative_use_rejection_sampling=False), + ), + ): + worker.target_embed_and_head_shared = False + worker._maybe_share_embed_before_pool_sizing() + calls.append("target_pool") + worker.alloc_memory_pool() + return calls + + +class TestEagleDraftEmbedShare(CustomTestCase): + def test_replicated_embedding_is_shared_before_the_pool_is_sized(self): + self.assertEqual( + _run(True), ["token_map", "lm_head", "target_pool", "draft_pool"] + ) + + def test_otherwise_it_is_shared_when_the_draft_pool_is_allocated(self): + self.assertEqual( + _run(False), ["target_pool", "draft_pool", "token_map", "lm_head"] + ) + + def test_init_ends_with_the_early_share(self): + tree = ast.parse(textwrap.dedent(inspect.getsource(EagleDraftWorker.__init__))) + last = tree.body[0].body[-1] + self.assertEqual( + ast.unparse(last), "self._maybe_share_embed_before_pool_sizing()" + ) + + +if __name__ == "__main__": + unittest.main() From 1574eb89dc330d112e0539c16cffa12865d2988f Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Sun, 25 Oct 2026 21:07:06 -0700 Subject: [PATCH 2/8] test: consolidate megakernel DSA binding coverage --- .../nvidia/test_megakernel_binding.py | 102 ++++++++++++++ .../nvidia/test_megakernel_binding_dsa.py | 124 ------------------ 2 files changed, 102 insertions(+), 124 deletions(-) delete mode 100644 test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding_dsa.py diff --git a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py index c67be2cf7d1d..87c92e90945f 100644 --- a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py +++ b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py @@ -75,5 +75,107 @@ def test_a_replaced_source_fails_assert_live_by_name(self): binding.assert_live() +class DsaPool: + index_page_size = 64 + + def __init__(self): + self.keys = [torch.zeros(4, 1, 8) for _ in range(2)] + self.index_k = [torch.zeros(4, 12) for _ in range(2)] + + def get_key_buffer(self, layer_id): + return self.keys[layer_id] + + def get_index_k_with_scale_buffer(self, layer_id): + return self.index_k[layer_id] + + def set_mla_kv_buffer(self, layer, loc, k, k_rope): + return layer, loc, k, k_rope + + +def dsa_backend(pool): + return SimpleNamespace( + token_to_kv_pool=pool, + forward_metadata="metadata", + dsa_decode_impl="trtllm", + kv_lora_rank=512, + qk_nope_head_dim=192, + qk_rope_head_dim=64, + kv_cache_dim=656, + physical_page_size=64, + use_mha=False, + kv_cache_dtype=torch.float8_e4m3fn, + use_fused_topk=True, + dsa_topk_backend="topk backend", + dcp_size=1, + dsa_index_kpool=1, + hisparse_coordinator=None, + workspace_buffer=torch.zeros(16, dtype=torch.uint8), + _multi_ctas_kv_counter_buffer=torch.zeros(8, dtype=torch.uint8), + get_indexer_metadata=lambda layer_id, fb: ("indexer", layer_id, fb), + ) + + +class TestMegakernelBindingDsa(CustomTestCase): + def test_the_dsa_view_is_the_running_dsa_backend(self): + backend = dsa_backend(DsaPool()) + with patch(BACKEND, return_value=backend): + dsa = mb.MegakernelBinding(model=torch.nn.Module()).forward_view().dsa + same = { + "kv_lora_rank": "kv_lora_rank", + "qk_nope_head_dim": "qk_nope_head_dim", + "qk_rope_head_dim": "qk_rope_head_dim", + "kv_cache_dim": "kv_cache_dim", + "physical_page_size": "physical_page_size", + "decode_impl": "dsa_decode_impl", + "use_mha": "use_mha", + "kv_cache_dtype": "kv_cache_dtype", + "use_fused_topk": "use_fused_topk", + "topk_backend": "dsa_topk_backend", + "dcp_size": "dcp_size", + "index_kpool": "dsa_index_kpool", + "workspace_buffer": "workspace_buffer", + "multi_ctas_kv_counter_buffer": "_multi_ctas_kv_counter_buffer", + "indexer_metadata": "get_indexer_metadata", + } + for member, attr in same.items(): + self.assertIs(getattr(dsa, member), getattr(backend, attr), member) + self.assertFalse(dsa.hisparse) + backend.use_mha = True # read live: the backend sets it per forward + self.assertTrue(dsa.use_mha) + self.assertEqual(dsa.indexer_metadata(3, "fb"), ("indexer", 3, "fb")) + + def test_a_backend_without_dsa_has_no_dsa_view(self): + backend = SimpleNamespace(token_to_kv_pool=DsaPool(), forward_metadata=None) + with patch(BACKEND, return_value=backend): + binding = mb.MegakernelBinding(model=torch.nn.Module()) + self.assertIsNone(binding.forward_view().dsa) + binding.assert_live() + + def test_the_pool_view_offers_the_mla_and_index_k_buffers(self): + pool = DsaPool() + view = mb.KvPoolView(pool) + self.assertIs(view.get_key_buffer(1), pool.keys[1]) + self.assertIs(view.get_index_k_with_scale_buffer(0), pool.index_k[0]) + self.assertEqual(view.index_page_size, 64) + self.assertEqual(view.set_mla_kv_buffer(1, 2, 3, 4), (1, 2, 3, 4)) + # a pool without index-K pages + self.assertIsNone(mb.KvPoolView(SimpleNamespace()).index_page_size) + + def test_a_replaced_dsa_buffer_fails_assert_live(self): + for attr, name in ( + ("workspace_buffer", "dsa_workspace"), + ("_multi_ctas_kv_counter_buffer", "dsa_kv_counter"), + ): + with self.subTest(buffer=attr): + backend = dsa_backend(DsaPool()) + binding = mb.MegakernelBinding(model=torch.nn.Module()) + with patch(BACKEND, return_value=backend): + binding.forward_view() + binding.assert_live() + setattr(backend, attr, torch.zeros(32, dtype=torch.uint8)) + with self.assertRaisesRegex(RuntimeError, name): + binding.assert_live() + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding_dsa.py b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding_dsa.py deleted file mode 100644 index 200d37185cac..000000000000 --- a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding_dsa.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""CPU checks of the megakernel binding's DSA (sparse MLA) view: it is the running -DSA backend and its pool, other backends have none, and a replaced decode buffer is -caught.""" - -import unittest -from types import SimpleNamespace -from unittest.mock import patch - -import torch - -from sglang.srt.kernel_plugins.nvidia import binding as mb -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") - -BACKEND = "sglang.srt.model_executor.forward_context.get_attn_backend" - - -class DsaPool: - index_page_size = 64 - - def __init__(self): - self.keys = [torch.zeros(4, 1, 8) for _ in range(2)] - self.index_k = [torch.zeros(4, 12) for _ in range(2)] - - def get_key_buffer(self, layer_id): - return self.keys[layer_id] - - def get_index_k_with_scale_buffer(self, layer_id): - return self.index_k[layer_id] - - def set_mla_kv_buffer(self, layer, loc, k, k_rope): - return layer, loc, k, k_rope - - -def dsa_backend(pool): - return SimpleNamespace( - token_to_kv_pool=pool, - forward_metadata="metadata", - dsa_decode_impl="trtllm", - kv_lora_rank=512, - qk_nope_head_dim=192, - qk_rope_head_dim=64, - kv_cache_dim=656, - physical_page_size=64, - use_mha=False, - kv_cache_dtype=torch.float8_e4m3fn, - use_fused_topk=True, - dsa_topk_backend="topk backend", - dcp_size=1, - dsa_index_kpool=1, - hisparse_coordinator=None, - workspace_buffer=torch.zeros(16, dtype=torch.uint8), - _multi_ctas_kv_counter_buffer=torch.zeros(8, dtype=torch.uint8), - get_indexer_metadata=lambda layer_id, fb: ("indexer", layer_id, fb), - ) - - -class TestMegakernelBindingDsa(CustomTestCase): - def test_the_dsa_view_is_the_running_dsa_backend(self): - backend = dsa_backend(DsaPool()) - with patch(BACKEND, return_value=backend): - dsa = mb.MegakernelBinding(model=torch.nn.Module()).forward_view().dsa - same = { - "kv_lora_rank": "kv_lora_rank", - "qk_nope_head_dim": "qk_nope_head_dim", - "qk_rope_head_dim": "qk_rope_head_dim", - "kv_cache_dim": "kv_cache_dim", - "physical_page_size": "physical_page_size", - "decode_impl": "dsa_decode_impl", - "use_mha": "use_mha", - "kv_cache_dtype": "kv_cache_dtype", - "use_fused_topk": "use_fused_topk", - "topk_backend": "dsa_topk_backend", - "dcp_size": "dcp_size", - "index_kpool": "dsa_index_kpool", - "workspace_buffer": "workspace_buffer", - "multi_ctas_kv_counter_buffer": "_multi_ctas_kv_counter_buffer", - "indexer_metadata": "get_indexer_metadata", - } - for member, attr in same.items(): - self.assertIs(getattr(dsa, member), getattr(backend, attr), member) - self.assertFalse(dsa.hisparse) - backend.use_mha = True # read live: the backend sets it per forward - self.assertTrue(dsa.use_mha) - self.assertEqual(dsa.indexer_metadata(3, "fb"), ("indexer", 3, "fb")) - - def test_a_backend_without_dsa_has_no_dsa_view(self): - backend = SimpleNamespace(token_to_kv_pool=DsaPool(), forward_metadata=None) - with patch(BACKEND, return_value=backend): - binding = mb.MegakernelBinding(model=torch.nn.Module()) - self.assertIsNone(binding.forward_view().dsa) - binding.assert_live() - - def test_the_pool_view_offers_the_mla_and_index_k_buffers(self): - pool = DsaPool() - view = mb.KvPoolView(pool) - self.assertIs(view.get_key_buffer(1), pool.keys[1]) - self.assertIs(view.get_index_k_with_scale_buffer(0), pool.index_k[0]) - self.assertEqual(view.index_page_size, 64) - self.assertEqual(view.set_mla_kv_buffer(1, 2, 3, 4), (1, 2, 3, 4)) - # a pool without index-K pages - self.assertIsNone(mb.KvPoolView(SimpleNamespace()).index_page_size) - - def test_a_replaced_dsa_buffer_fails_assert_live(self): - for attr, name in ( - ("workspace_buffer", "dsa_workspace"), - ("_multi_ctas_kv_counter_buffer", "dsa_kv_counter"), - ): - with self.subTest(buffer=attr): - backend = dsa_backend(DsaPool()) - binding = mb.MegakernelBinding(model=torch.nn.Module()) - with patch(BACKEND, return_value=backend): - binding.forward_view() - binding.assert_live() - setattr(backend, attr, torch.zeros(32, dtype=torch.uint8)) - with self.assertRaisesRegex(RuntimeError, name): - binding.assert_live() - - -if __name__ == "__main__": - unittest.main() From ba86b681b51d7a35804b3240b808cb8e67b8560e Mon Sep 17 00:00:00 2001 From: Rohan Varma Date: Wed, 28 Oct 2026 11:03:02 -0400 Subject: [PATCH 3/8] [Test] [NVIDIA] megakernels: draft capture fake for megakernel API v2 sgl-project/sglang#45266 made the megakernel contract version 2: the binding reads draft_loop, draft_extend, begin_draft_round and graph_variants from every built object. The draft capture test's built object offers none of them, as main's FakeBuilt does. --- .../kernel_plugins/nvidia/test_megakernel_draft_capture.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_draft_capture.py b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_draft_capture.py index 41c66669d9c1..1b30eb94ba5d 100644 --- a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_draft_capture.py +++ b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_draft_capture.py @@ -39,7 +39,12 @@ class DraftBuilt: """Covers draft decode steps of up to max_bs rows: one row a request, so off the width grid.""" + # the EAGLE entries and graph variants it does not offer draft_sampler = None + draft_loop = None + draft_extend = None + begin_draft_round = None + graph_variants = None def __init__(self, widths, max_bs): self.widths, self.max_bs = widths, max_bs From 2758ef78829a21898970c698d11be43cd949ce2b Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Sun, 15 Nov 2026 21:25:57 -0800 Subject: [PATCH 4/8] [Test] spec: Stub the draft-probs vocab scatter in test_eagle_draft_embed_share The merge of main added `_init_draft_probs_vocab_scatter()` to `EagleDraftWorker.alloc_memory_pool`. That method reads `target_worker`, which the test's bare worker does not have. Stub it out like the other init calls, since the test only checks the order of the sharing calls. Co-Authored-By: Claude Opus 5.5 --- test/registered/unit/spec/test_eagle_draft_embed_share.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/registered/unit/spec/test_eagle_draft_embed_share.py b/test/registered/unit/spec/test_eagle_draft_embed_share.py index 966599b4afee..75f9b810d760 100644 --- a/test/registered/unit/spec/test_eagle_draft_embed_share.py +++ b/test/registered/unit/spec/test_eagle_draft_embed_share.py @@ -35,6 +35,9 @@ def _run(replicate: bool) -> list: patch.object( EagleDraftWorker, "init_lm_head", lambda self: calls.append("lm_head") ), + patch.object( + EagleDraftWorker, "_init_draft_probs_vocab_scatter", lambda self: None + ), patch( "sglang.srt.speculative.eagle_worker_v2.get_spec", return_value=SimpleNamespace(speculative_use_rejection_sampling=False), From 7c0ede9292c379079872b67cf20a953b8a77e5fb Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Sun, 15 Nov 2026 23:13:56 -0800 Subject: [PATCH 5/8] [Fix] megakernels: Fix two CI failures in the DSA view and draft capture tests - KvPoolView.set_mla_kv_buffer is now a method that forwards its arguments to the pool, not a property returning the pool's bound method. test_physical_write_loc checks every class's set_mla_kv_buffer as a pool write door, and a door must forward its loc untouched. Callers see the same behavior. - test_inkling_checkpoint_indices: give the fake draft model runner an empty capture_tail_hooks list. EAGLEDraftExtendCudaGraphRunner now runs those hooks after each captured draft-extend forward. Co-Authored-By: Claude Opus 5.5 --- python/sglang/srt/kernel_plugins/nvidia/binding.py | 6 +++--- .../ops/attention/test_inkling_checkpoint_indices.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/kernel_plugins/nvidia/binding.py b/python/sglang/srt/kernel_plugins/nvidia/binding.py index 84a332663618..2376becdb830 100644 --- a/python/sglang/srt/kernel_plugins/nvidia/binding.py +++ b/python/sglang/srt/kernel_plugins/nvidia/binding.py @@ -199,9 +199,9 @@ def index_page_size(self) -> Optional[int]: """Tokens per page of the index-K buffer; None for a pool without one.""" return getattr(self.pool, "index_page_size", None) - @property - def set_mla_kv_buffer(self) -> Callable[..., None]: - return self.pool.set_mla_kv_buffer + def set_mla_kv_buffer(self, *args: Any, **kwargs: Any) -> None: + # Forwards the loc argument untouched, as a pool write door must. + return self.pool.set_mla_kv_buffer(*args, **kwargs) class DsaView: diff --git a/test/registered/kernels/ops/attention/test_inkling_checkpoint_indices.py b/test/registered/kernels/ops/attention/test_inkling_checkpoint_indices.py index 9d77990eeeb4..8274158498f6 100644 --- a/test/registered/kernels/ops/attention/test_inkling_checkpoint_indices.py +++ b/test/registered/kernels/ops/attention/test_inkling_checkpoint_indices.py @@ -865,6 +865,7 @@ def _make_draft_checkpoint_runner(self, multilayer, static, tracking, fused=Fals spec_algorithm=SimpleNamespace(is_standalone=lambda: False), device_timer=None, canary_manager=None, + capture_tail_hooks=[], # The KV side is not under test: a static pool's runner_ids binds # the captured batch to its own write ids. runner_ids=RunnerIds( From 08289754492983bd7c119aebd060bcd6337845c0 Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Mon, 16 Nov 2026 00:30:07 -0800 Subject: [PATCH 6/8] [Test] attention kits: Give the EAGLE draft harnesses' mock runners capture tail hooks EAGLEDraftCudaGraphRunner and EAGLEDraftExtendCudaGraphRunner now run model_runner.capture_tail_hooks after each captured forward. The attention-unittest mock model runners skip ModelRunner.__init__, so they lack the attribute, and test_fa4.py's EAGLE draft runner cases failed with AttributeError. The draft worker harnesses now default it to an empty list. Co-Authored-By: Claude Opus 5.5 --- .../runner_modes/speculative_draft_extend_runner.py | 8 ++++++++ .../runner_modes/speculative_draft_runner.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py index d80e9fa61bf3..47f8385b32b8 100644 --- a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py +++ b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py @@ -433,6 +433,10 @@ def __init__( settings: EagleDraftRunnerSettings, ): self.model_runner = fixture.runner + # The draft graph runners run the model runner's capture tail hooks. + fixture.runner.capture_tail_hooks = getattr( + fixture.runner, "capture_tail_hooks", [] + ) self.target_worker = SimpleNamespace(model_runner=fixture.runner) self.draft_extend_attn_backend = draft_extend_attn_backend self.topk = settings.topk @@ -461,6 +465,10 @@ def __init__( settings: EagleDraftRunnerSettings, ): self.draft_runner = fixture.runner + # The draft graph runners run the model runner's capture tail hooks. + fixture.runner.capture_tail_hooks = getattr( + fixture.runner, "capture_tail_hooks", [] + ) self.target_worker = SimpleNamespace(model_runner=fixture.runner) self.draft_extend_attn_backend = draft_extend_attn_backend self.topk = settings.topk diff --git a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py index 62e7bb4e1e5f..7674a5260df4 100644 --- a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py +++ b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py @@ -170,6 +170,10 @@ def __init__( settings: EagleDraftRunnerSettings, ): self.model_runner = fixture.runner + # The draft graph runners run the model runner's capture tail hooks. + fixture.runner.capture_tail_hooks = getattr( + fixture.runner, "capture_tail_hooks", [] + ) self.draft_attn_backend = draft_attn_backend self.topk = settings.topk self.speculative_num_steps = settings.speculative_num_steps @@ -218,6 +222,10 @@ def __init__( settings: EagleDraftRunnerSettings, ): self.model_runner = fixture.runner + # The draft graph runners run the model runner's capture tail hooks. + fixture.runner.capture_tail_hooks = getattr( + fixture.runner, "capture_tail_hooks", [] + ) self.draft_attn_backend = draft_attn_backend self.topk = settings.topk self.speculative_num_steps = settings.speculative_num_steps From cc01bc7f2b8f3e83909c1d6c7b42aea9a1d8148c Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Mon, 16 Nov 2026 04:00:46 -0800 Subject: [PATCH 7/8] [Fix] megakernels: Read attn_dcp_size from the DSA backend in DsaView #48721 renamed the DSA backend's dcp_size attribute to attn_dcp_size, so DsaView.dcp_size raised AttributeError at the first CUDA graph capture of a megakernel decode on DSA models (reported for GLM-5.2). Update the unit test's fake backend to match. Co-Authored-By: Claude Opus 5.5 --- python/sglang/srt/kernel_plugins/nvidia/binding.py | 2 +- .../unit/kernel_plugins/nvidia/test_megakernel_binding.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/kernel_plugins/nvidia/binding.py b/python/sglang/srt/kernel_plugins/nvidia/binding.py index 2376becdb830..82402686e6ee 100644 --- a/python/sglang/srt/kernel_plugins/nvidia/binding.py +++ b/python/sglang/srt/kernel_plugins/nvidia/binding.py @@ -254,7 +254,7 @@ def topk_backend(self) -> Any: @property def dcp_size(self) -> int: - return self._backend.dcp_size + return self._backend.attn_dcp_size @property def index_kpool(self) -> int: diff --git a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py index 6818e9985458..55be40eff4ff 100644 --- a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py +++ b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py @@ -1008,7 +1008,7 @@ def dsa_backend(pool): kv_cache_dtype=torch.float8_e4m3fn, use_fused_topk=True, dsa_topk_backend="topk backend", - dcp_size=1, + attn_dcp_size=1, dsa_index_kpool=1, hisparse_coordinator=None, workspace_buffer=torch.zeros(16, dtype=torch.uint8), @@ -1033,7 +1033,7 @@ def test_the_dsa_view_is_the_running_dsa_backend(self): "kv_cache_dtype": "kv_cache_dtype", "use_fused_topk": "use_fused_topk", "topk_backend": "dsa_topk_backend", - "dcp_size": "dcp_size", + "dcp_size": "attn_dcp_size", "index_kpool": "dsa_index_kpool", "workspace_buffer": "workspace_buffer", "multi_ctas_kv_counter_buffer": "_multi_ctas_kv_counter_buffer", From 44b24efaddef686322d18aef3305fdb588d65569 Mon Sep 17 00:00:00 2001 From: shuw Date: Mon, 16 Nov 2026 13:17:25 -0600 Subject: [PATCH 8/8] [Test] spec: Check the early embed share call site, not its exact position test_init_ends_with_the_early_share required the share to be the last statement of EagleDraftWorker.__init__, so any unrelated line appended to __init__ failed it. The behavioral tests call the share directly (running __init__ needs a GPU), so a source check is still needed to catch the call being dropped. Assert instead that __init__ calls it exactly once, after self.draft_runner is set. Co-Authored-By: Claude Opus 5.5 --- .../unit/spec/test_eagle_draft_embed_share.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/test/registered/unit/spec/test_eagle_draft_embed_share.py b/test/registered/unit/spec/test_eagle_draft_embed_share.py index 75f9b810d760..c914049fb32b 100644 --- a/test/registered/unit/spec/test_eagle_draft_embed_share.py +++ b/test/registered/unit/spec/test_eagle_draft_embed_share.py @@ -61,12 +61,21 @@ def test_otherwise_it_is_shared_when_the_draft_pool_is_allocated(self): _run(False), ["target_pool", "draft_pool", "token_map", "lm_head"] ) - def test_init_ends_with_the_early_share(self): + def test_init_shares_once_after_the_draft_runner_exists(self): + # The tests above call the early share directly, since running __init__ needs + # a GPU and a model. This guards the call site: __init__ must still make the + # call, once, after the draft model is built. Its exact position is free. tree = ast.parse(textwrap.dedent(inspect.getsource(EagleDraftWorker.__init__))) - last = tree.body[0].body[-1] - self.assertEqual( - ast.unparse(last), "self._maybe_share_embed_before_pool_sizing()" - ) + body = [ast.unparse(stmt) for stmt in tree.body[0].body] + share = [ + i + for i, s in enumerate(body) + if s == "self._maybe_share_embed_before_pool_sizing()" + ] + runner = [i for i, s in enumerate(body) if s.startswith("self.draft_runner =")] + self.assertEqual(len(share), 1, "__init__ must call the early share once") + self.assertEqual(len(runner), 1, "__init__ must set self.draft_runner once") + self.assertGreater(share[0], runner[0]) if __name__ == "__main__":