diff --git a/python/sglang/kernels/plugin_kernels/nvidia/spec.py b/python/sglang/kernels/plugin_kernels/nvidia/spec.py index c9953bf8e80e..1624fb886eab 100644 --- a/python/sglang/kernels/plugin_kernels/nvidia/spec.py +++ b/python/sglang/kernels/plugin_kernels/nvidia/spec.py @@ -28,6 +28,13 @@ # the sglang.srt.plugins entry point that registers the specs ENTRY_POINT = "nvidia_megakernel" FLAG = "--enable-nvidia-megakernel" +# The EAGLE draft graph runners run a draft model's capture check (Step); a draft +# spec that serves steps can require it +EAGLE_DRAFT_CAPTURE_CHECK = True +# The variables of Requirements.env that the flag sets to the spec's value when they +# are unset, at argument resolution: SGLang reads each only when a model is built, in +# the model processes started after it. A value the user set is kept and checked. +FLAG_SETS_ENV = frozenset({"SGLANG_ENABLE_EMBED_REPLICATION"}) class Feature(enum.Enum): @@ -69,7 +76,8 @@ class Requirements(msgspec.Struct, frozen=True): server_args: Mapping[str, frozenset] = {} # an HF config field -> the value the kernels are built for hf_config: Mapping[str, Any] = {} - # an SGLang environment variable in envs -> its required value + # an SGLang environment variable in envs -> its required value (the flag sets one + # in FLAG_SETS_ENV that is unset) env: Mapping[str, Any] = {} @@ -105,6 +113,12 @@ class Step(msgspec.Struct, frozen=True): "target_verify" – speculative-decoding target verify; runs in a CUDA graph. "prefill" – context prefill; runs eagerly. With prefill_graph its transformer body may be captured (SupportsPrefillBody). + "draft_decode", "draft_extend" – a draft spec's EAGLE draft steps. + + A draft spec's steps may be off its width grid (an EAGLE draft decode step has + one row a request); the dispatch does not check rows against the widths. The + capture check of a draft model runs once, when the first of its EAGLE draft + graph runners has captured every batch size. """ mode: str diff --git a/python/sglang/srt/kernel_plugins/nvidia/binding.py b/python/sglang/srt/kernel_plugins/nvidia/binding.py index c511388fa9d7..82402686e6ee 100644 --- a/python/sglang/srt/kernel_plugins/nvidia/binding.py +++ b/python/sglang/srt/kernel_plugins/nvidia/binding.py @@ -4,11 +4,12 @@ It checks a plugin's spec against the resolved configuration at startup, builds it once the model is prepared, and installs the whole-step dispatch on the model. It is also the one place a megakernel reads SGLang's runtime state: the KV pool, the -attention metadata of the running forward and the Engram tables. A spec's build gets -a ``MegakernelBinding`` as ``BuildContext.binding``; a covered forward gets the -running forward's ``ForwardView`` as ``Step.view`` (a draft entry asks -``binding.forward_view()``). Everything else a megakernel touches is the model it -replaces, SGLang's kernel ops, and the reads listed below. +attention metadata of the running forward, the DSA (sparse MLA) backend's layout and +buffers and the Engram tables. A spec's build gets a ``MegakernelBinding`` as +``BuildContext.binding``; a covered forward gets the running forward's +``ForwardView`` as ``Step.view`` (a draft entry asks ``binding.forward_view()``). +Everything else a megakernel touches is the model it replaces, SGLang's kernel ops, +and the reads listed below. """ from __future__ import annotations @@ -29,6 +30,7 @@ ENTRIES, ENTRY_POINT, FLAG, + FLAG_SETS_ENV, GRAPH_VARIANT_RUNNERS, BuildContext, BuiltMegakernel, @@ -130,6 +132,18 @@ "capture-mode queries (get_is_capture_mode, is_in_breakable_cuda_graph)", "MegakernelBinding.engram_host_table", # the per-layer host table object ) +# Called by a megakernel on a DSA (sparse MLA) backend, tolerated until offered as +# data. +TOLERATED_DSA_CALLABLES = ( + "KvPoolView.set_mla_kv_buffer", # the MLA key write + "DsaView.indexer_metadata", # the indexer's metadata of the running forward +) +# Written into the DSA pool by a megakernel (the step's rows, at its out_cache_loc), +# tolerated until SGLang owns the writes. +TOLERATED_DSA_WRITES = ( + "KvPoolView.get_key_buffer", # the MLA key rows + "KvPoolView.get_index_k_with_scale_buffer", # the index-K rows and their scales +) class KvPoolView: @@ -176,6 +190,97 @@ def get_extra_key_bytes_per_token(self, layer_id: int) -> int: def set_swa_key_buffer_radix_fused_norm_rope(self) -> Callable[..., None]: return self.pool.set_swa_key_buffer_radix_fused_norm_rope + # MLA / DSA pools + def get_index_k_with_scale_buffer(self, layer_id: int) -> torch.Tensor: + return self.pool.get_index_k_with_scale_buffer(layer_id) + + @property + def index_page_size(self) -> Optional[int]: + """Tokens per page of the index-K buffer; None for a pool without one.""" + return getattr(self.pool, "index_page_size", None) + + def set_mla_kv_buffer(self, *args: Any, **kwargs: Any) -> None: + # Forwards the loc argument untouched, as a pool write door must. + return self.pool.set_mla_kv_buffer(*args, **kwargs) + + +class DsaView: + """The running DSA (sparse MLA) backend's layout and the buffers its decode + uses, under names a megakernel can rely on.""" + + def __init__(self, backend: Any): + self._backend = backend + + @property + def kv_lora_rank(self) -> int: + return self._backend.kv_lora_rank + + @property + def qk_nope_head_dim(self) -> int: + return self._backend.qk_nope_head_dim + + @property + def qk_rope_head_dim(self) -> int: + return self._backend.qk_rope_head_dim + + @property + def kv_cache_dim(self) -> int: + return self._backend.kv_cache_dim + + @property + def physical_page_size(self) -> int: + return self._backend.physical_page_size + + @property + def decode_impl(self) -> str: + return self._backend.dsa_decode_impl + + @property + def use_mha(self) -> bool: + """Whether the running forward takes the MHA path instead of MLA.""" + return self._backend.use_mha + + @property + def kv_cache_dtype(self) -> torch.dtype: + return self._backend.kv_cache_dtype + + @property + def use_fused_topk(self) -> bool: + return self._backend.use_fused_topk + + @property + def topk_backend(self) -> Any: + return self._backend.dsa_topk_backend + + @property + def dcp_size(self) -> int: + return self._backend.attn_dcp_size + + @property + def index_kpool(self) -> int: + return self._backend.dsa_index_kpool + + @property + def hisparse(self) -> bool: + return self._backend.hisparse_coordinator is not None + + @property + def workspace_buffer(self) -> Optional[torch.Tensor]: + """The decode kernels' workspace, or None when the backend has none.""" + return self._backend.workspace_buffer + + @property + def multi_ctas_kv_counter_buffer(self) -> Optional[torch.Tensor]: + """The TRT-LLM decode's multi-CTA KV counter, sized before the decode graphs + are captured; None when the backend has none.""" + return self._backend._multi_ctas_kv_counter_buffer + + @property + def indexer_metadata(self) -> Callable[[int, Any], Any]: + """The backend's indexer metadata of the running forward, by + (layer_id, forward_batch).""" + return self._backend.get_indexer_metadata + class KdaStateView(msgspec.Struct, frozen=True): """One KDA layer's SGLang-owned, non-ReplaySSM recurrent state.""" @@ -332,6 +437,11 @@ def low_ratio_sources(self) -> Callable[..., Any]: def core_compressor(self) -> Callable[..., Any]: return self._backend.forward_core_compressor + @property + def dsa(self) -> Optional[DsaView]: + """The DSA backend's view, or None when the running backend is not DSA.""" + return _dsa_view(self._backend) + class _SealedSource(msgspec.Struct, frozen=True): identity: tuple @@ -393,6 +503,13 @@ def forward_view(self, forward_batch: Optional[ForwardBatch] = None) -> ForwardV view = ForwardView( backend, self._seal_cache, hybrid=hybrid, legacy=not hybrid_family ) + dsa = None if hybrid_family else view.dsa + if dsa is not None: + for name, buffer in _dsa_buffers(dsa).items(): + if buffer is not None: + self._seal( + name, buffer, lambda name=name: self._current_dsa_buffer(name) + ) return view def prefill_view(self) -> ForwardView: @@ -950,12 +1067,33 @@ def _current_pool(self) -> Any: return get_attn_backend().token_to_kv_pool + def _current_dsa_buffer(self, name: str) -> Optional[torch.Tensor]: + from sglang.srt.model_executor.forward_context import get_attn_backend + + dsa = _dsa_view(get_attn_backend()) + return None if dsa is None else _dsa_buffers(dsa)[name] + def _history(self) -> Optional[torch.Tensor]: # only models with Engram layers have a hasher hasher = getattr(self._model, "engram_hasher", None) return None if hasher is None else hasher.history +def _dsa_view(backend: Any) -> Optional[DsaView]: + # only the DSA backend resolves a DSA decode implementation + if getattr(backend, "dsa_decode_impl", None) is None: + return None + return DsaView(backend) + + +def _dsa_buffers(dsa: DsaView) -> dict[str, Optional[torch.Tensor]]: + """The DSA backend's buffers a captured step reads by address, sealed by name.""" + return { + "dsa_workspace": dsa.workspace_buffer, + "dsa_kv_counter": dsa.multi_ctas_kv_counter_buffer, + } + + def _identity(value: Any) -> tuple: if isinstance(value, torch.Tensor): return ( @@ -1148,31 +1286,59 @@ def _dtype_reasons(spec: Optional[MegakernelSpec], model_config: Any) -> list[st return [f"dtype {dtype!r} is not one of {sorted(spec.requires.dtypes)}"] +def set_flag_env(spec: Optional[MegakernelSpec]) -> list[str]: + """Set each variable of FLAG_SETS_ENV that the spec requires and the user left + unset to the spec's value; return their names.""" + if spec is None: + return [] + names = [ + name + for name in spec.requires.env + if name in FLAG_SETS_ENV and not getattr(envs, name).is_set() + ] + for name in names: + getattr(envs, name).set(spec.requires.env[name]) + return names + + def handle_nvidia_megakernel(server_args: Any) -> None: """Resolution step: fail before the weights load when the flag cannot be served, - listing every reason.""" + listing every reason. The model processes, started after it, inherit what it sets + for the spec (FLAG_SETS_ENV).""" cfg = resolving_view(server_args) if not cfg.enable_nvidia_megakernel: return archs = model_config_of(server_args).hf_config.architectures if msa_serves(archs) and not cfg.enable_amd_monokernel: return # handle_monokernel checks the MSA plugin's spec + env_set = [] reasons = plugin_reasons() if not reasons: model_config = model_config_of(server_args) + hf_config, isa = model_config.hf_config, device_isa() + env_set = set_flag_env( + find_spec(architectures=hf_config.architectures, isa=isa)[0] + ) spec, reasons = megakernel_disable_reasons( - cfg=cfg, hf_config=model_config.hf_config, isa=device_isa() + cfg=cfg, hf_config=hf_config, isa=isa ) reasons += _dtype_reasons(spec, model_config) if cfg.enable_amd_monokernel: # both replace the model's forward, which would fail only after weights load reasons.append("--enable-amd-monokernel is also set; use one of the two") if reasons: + # a refused setup leaves the environment as it was + for name in env_set: + getattr(envs, name).clear() raise ValueError( f"{FLAG} cannot run this setup:\n - " + "\n - ".join(reasons) ) logger.warning("%s is experimental. Its kernels and coverage may change.", FLAG) logger.info("%s: %s on %s", FLAG, spec.name, device_isa()) + for name in env_set: + logger.info( + "%s: set %s=%s for %s", FLAG, name, getattr(envs, name).get(), spec.name + ) class _InstalledMegakernel(msgspec.Struct, frozen=True): diff --git a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py index e2168618065f..62ddcd59e216 100644 --- a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py @@ -420,6 +420,25 @@ def can_run_graph(self, forward_batch: ForwardBatch): # ----------------------------------------------------------------- # Capture (per-shape) # ----------------------------------------------------------------- + def _run_capture_tail_hooks(self, out, forward_batch, num_tokens: int) -> None: + """The model runner's capture tail hooks, after the draft forwards of a captured + batch size. The draft steps ran under the per-step attention backends, which + share the draft's KV pool (checked here when a hook exists); a hook that reads + the running backend sees the first.""" + hooks = self.model_runner.capture_tail_hooks + if not hooks: + return + backends = self.draft_attn_backend.attn_backends + pool = getattr(backends[0], "token_to_kv_pool", None) + if any(getattr(b, "token_to_kv_pool", None) is not pool for b in backends): + raise RuntimeError( + "the draft's per-step attention backends do not share one KV pool; " + "the capture tail hooks check the running backend's" + ) + with forward_context(ForwardContext(attn_backend=backends[0])): + for capture_hook in hooks: + capture_hook(self, out, forward_batch, num_tokens) + def capture_one_shape( self, size: int, @@ -562,6 +581,7 @@ def run_stock(): forward_batch.spec_info.hidden_states = hidden_states_backup forward_batch.spec_info.dsa_topk_indices = dsa_topk_indices_backup forward_batch.positions.sub_(self.eagle_worker.speculative_num_steps - 1) + self._run_capture_tail_hooks(ret, forward_batch, num_tokens) return ret def run_once(): diff --git a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py index e7c88bea73b6..c4ea66613f9c 100644 --- a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py @@ -416,6 +416,12 @@ def can_run_graph(self, forward_batch: ForwardBatch): return is_bs_supported + def _run_capture_tail_hooks(self, out, forward_batch, num_tokens: int) -> None: + """The model runner's capture tail hooks, after the draft-extend forward of a + captured batch size (under the runner's own attention backend).""" + for capture_hook in self.model_runner.capture_tail_hooks: + capture_hook(self, out, forward_batch, num_tokens) + def capture_one_shape( self, size: int, @@ -535,6 +541,7 @@ def run_stock(): forward_batch.out_cache_loc = output_cache_loc_backup forward_batch.spec_info.hidden_states = hidden_states_backup + self._run_capture_tail_hooks(ret, forward_batch, num_tokens) return ret def run_once(): diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 8e2ac75b407d..cb2d19970fa3 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -244,6 +244,21 @@ def __init__( self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device) + self.target_embed_and_head_shared = False + self._maybe_share_embed_before_pool_sizing() + + def _maybe_share_embed_before_pool_sizing(self): + """Under embedding replication the draft loads its own full embedding table on + every rank. Share the target's now, so that copy is freed before the scheduler + sizes the KV pool instead of after.""" + if envs.SGLANG_ENABLE_EMBED_REPLICATION.get(): + self._share_target_embed_and_head() + + def _share_target_embed_and_head(self): + self.init_token_map() + self.init_lm_head() + self.target_embed_and_head_shared = True + def alloc_memory_pool( self, memory_pool_config=None, @@ -258,8 +273,8 @@ def alloc_memory_pool( req_to_token_pool=req_to_token_pool, token_to_kv_pool_allocator=token_to_kv_pool_allocator, ) - self.init_token_map() - self.init_lm_head() + if not self.target_embed_and_head_shared: + self._share_target_embed_and_head() self._init_draft_probs_vocab_scatter() if get_spec().speculative_use_rejection_sampling: diff --git a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py index d80e9fa61bf3..47f8385b32b8 100644 --- a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py +++ b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py @@ -433,6 +433,10 @@ def __init__( settings: EagleDraftRunnerSettings, ): self.model_runner = fixture.runner + # The draft graph runners run the model runner's capture tail hooks. + fixture.runner.capture_tail_hooks = getattr( + fixture.runner, "capture_tail_hooks", [] + ) self.target_worker = SimpleNamespace(model_runner=fixture.runner) self.draft_extend_attn_backend = draft_extend_attn_backend self.topk = settings.topk @@ -461,6 +465,10 @@ def __init__( settings: EagleDraftRunnerSettings, ): self.draft_runner = fixture.runner + # The draft graph runners run the model runner's capture tail hooks. + fixture.runner.capture_tail_hooks = getattr( + fixture.runner, "capture_tail_hooks", [] + ) self.target_worker = SimpleNamespace(model_runner=fixture.runner) self.draft_extend_attn_backend = draft_extend_attn_backend self.topk = settings.topk diff --git a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py index 62e7bb4e1e5f..7674a5260df4 100644 --- a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py +++ b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py @@ -170,6 +170,10 @@ def __init__( settings: EagleDraftRunnerSettings, ): self.model_runner = fixture.runner + # The draft graph runners run the model runner's capture tail hooks. + fixture.runner.capture_tail_hooks = getattr( + fixture.runner, "capture_tail_hooks", [] + ) self.draft_attn_backend = draft_attn_backend self.topk = settings.topk self.speculative_num_steps = settings.speculative_num_steps @@ -218,6 +222,10 @@ def __init__( settings: EagleDraftRunnerSettings, ): self.model_runner = fixture.runner + # The draft graph runners run the model runner's capture tail hooks. + fixture.runner.capture_tail_hooks = getattr( + fixture.runner, "capture_tail_hooks", [] + ) self.draft_attn_backend = draft_attn_backend self.topk = settings.topk self.speculative_num_steps = settings.speculative_num_steps diff --git a/test/registered/kernels/ops/attention/test_inkling_checkpoint_indices.py b/test/registered/kernels/ops/attention/test_inkling_checkpoint_indices.py index 9d77990eeeb4..8274158498f6 100644 --- a/test/registered/kernels/ops/attention/test_inkling_checkpoint_indices.py +++ b/test/registered/kernels/ops/attention/test_inkling_checkpoint_indices.py @@ -865,6 +865,7 @@ def _make_draft_checkpoint_runner(self, multilayer, static, tracking, fused=Fals spec_algorithm=SimpleNamespace(is_standalone=lambda: False), device_timer=None, canary_manager=None, + capture_tail_hooks=[], # The KV side is not under test: a static pool's runner_ids binds # the captured batch to its own write ids. runner_ids=RunnerIds( diff --git a/test/registered/unit/kernel_plugins/nvidia/_fakes.py b/test/registered/unit/kernel_plugins/nvidia/_fakes.py index 298fbc1c1e6f..1bbab88268c4 100644 --- a/test/registered/unit/kernel_plugins/nvidia/_fakes.py +++ b/test/registered/unit/kernel_plugins/nvidia/_fakes.py @@ -67,6 +67,16 @@ def _build(model, hf_config, context): ) # an empty field accepts any value ANY_TP = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset())) +# specs that need the replicated input embedding, which the flag sets when it is unset +# (FLAG_SETS_ENV); the second also needs a variable the flag never sets +REPLICATION = "SGLANG_ENABLE_EMBED_REPLICATION" +REPLICATED = replace(SPEC, requires=replace(SPEC.requires, env={REPLICATION: True})) +REPLICATED_AND_BROADCAST = replace( + SPEC, + requires=replace( + SPEC.requires, env={REPLICATION: True, "SGLANG_DSA_TOPK_BROADCAST": True} + ), +) # a draft spec: builds no width, offers only the draft sampler DRAFT = replace( SPEC, diff --git a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py index cec5e0938c87..55be40eff4ff 100644 --- a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py +++ b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py @@ -977,5 +977,107 @@ def test_each_hybrid_source_family_is_sealed(self): binding.assert_live() +class DsaPool: + index_page_size = 64 + + def __init__(self): + self.keys = [torch.zeros(4, 1, 8) for _ in range(2)] + self.index_k = [torch.zeros(4, 12) for _ in range(2)] + + def get_key_buffer(self, layer_id): + return self.keys[layer_id] + + def get_index_k_with_scale_buffer(self, layer_id): + return self.index_k[layer_id] + + def set_mla_kv_buffer(self, layer, loc, k, k_rope): + return layer, loc, k, k_rope + + +def dsa_backend(pool): + return SimpleNamespace( + token_to_kv_pool=pool, + forward_metadata="metadata", + dsa_decode_impl="trtllm", + kv_lora_rank=512, + qk_nope_head_dim=192, + qk_rope_head_dim=64, + kv_cache_dim=656, + physical_page_size=64, + use_mha=False, + kv_cache_dtype=torch.float8_e4m3fn, + use_fused_topk=True, + dsa_topk_backend="topk backend", + attn_dcp_size=1, + dsa_index_kpool=1, + hisparse_coordinator=None, + workspace_buffer=torch.zeros(16, dtype=torch.uint8), + _multi_ctas_kv_counter_buffer=torch.zeros(8, dtype=torch.uint8), + get_indexer_metadata=lambda layer_id, fb: ("indexer", layer_id, fb), + ) + + +class TestMegakernelBindingDsa(CustomTestCase): + def test_the_dsa_view_is_the_running_dsa_backend(self): + backend = dsa_backend(DsaPool()) + with patch(BACKEND, return_value=backend): + dsa = mb.MegakernelBinding(model=torch.nn.Module()).forward_view().dsa + same = { + "kv_lora_rank": "kv_lora_rank", + "qk_nope_head_dim": "qk_nope_head_dim", + "qk_rope_head_dim": "qk_rope_head_dim", + "kv_cache_dim": "kv_cache_dim", + "physical_page_size": "physical_page_size", + "decode_impl": "dsa_decode_impl", + "use_mha": "use_mha", + "kv_cache_dtype": "kv_cache_dtype", + "use_fused_topk": "use_fused_topk", + "topk_backend": "dsa_topk_backend", + "dcp_size": "attn_dcp_size", + "index_kpool": "dsa_index_kpool", + "workspace_buffer": "workspace_buffer", + "multi_ctas_kv_counter_buffer": "_multi_ctas_kv_counter_buffer", + "indexer_metadata": "get_indexer_metadata", + } + for member, attr in same.items(): + self.assertIs(getattr(dsa, member), getattr(backend, attr), member) + self.assertFalse(dsa.hisparse) + backend.use_mha = True # read live: the backend sets it per forward + self.assertTrue(dsa.use_mha) + self.assertEqual(dsa.indexer_metadata(3, "fb"), ("indexer", 3, "fb")) + + def test_a_backend_without_dsa_has_no_dsa_view(self): + backend = SimpleNamespace(token_to_kv_pool=DsaPool(), forward_metadata=None) + with patch(BACKEND, return_value=backend): + binding = mb.MegakernelBinding(model=torch.nn.Module()) + self.assertIsNone(binding.forward_view().dsa) + binding.assert_live() + + def test_the_pool_view_offers_the_mla_and_index_k_buffers(self): + pool = DsaPool() + view = mb.KvPoolView(pool) + self.assertIs(view.get_key_buffer(1), pool.keys[1]) + self.assertIs(view.get_index_k_with_scale_buffer(0), pool.index_k[0]) + self.assertEqual(view.index_page_size, 64) + self.assertEqual(view.set_mla_kv_buffer(1, 2, 3, 4), (1, 2, 3, 4)) + # a pool without index-K pages + self.assertIsNone(mb.KvPoolView(SimpleNamespace()).index_page_size) + + def test_a_replaced_dsa_buffer_fails_assert_live(self): + for attr, name in ( + ("workspace_buffer", "dsa_workspace"), + ("_multi_ctas_kv_counter_buffer", "dsa_kv_counter"), + ): + with self.subTest(buffer=attr): + backend = dsa_backend(DsaPool()) + binding = mb.MegakernelBinding(model=torch.nn.Module()) + with patch(BACKEND, return_value=backend): + binding.forward_view() + binding.assert_live() + setattr(backend, attr, torch.zeros(32, dtype=torch.uint8)) + with self.assertRaisesRegex(RuntimeError, name): + binding.assert_live() + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_draft_capture.py b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_draft_capture.py new file mode 100644 index 000000000000..8f84f4b087bf --- /dev/null +++ b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_draft_capture.py @@ -0,0 +1,207 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU checks of the capture check of a draft spec, fired by the EAGLE draft graph +runners' capture hook call: after the draft forwards of each captured batch size, +with the runner's batch sizes, under the draft decode's first per-step backend.""" + +import ast +import inspect +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import torch +from msgspec.structs import replace + +from sglang.kernels.plugin_kernels.nvidia import spec as registry +from sglang.kernels.plugin_kernels.nvidia.spec import Step +from sglang.srt.kernel_plugins.nvidia import binding +from sglang.srt.kernel_plugins.nvidia.binding import MegakernelBinding +from sglang.srt.model_executor.forward_context import get_attn_backend +from sglang.srt.runtime_context import publish, reset_context +from sglang.srt.speculative.eagle_draft_cuda_graph_runner import ( + EAGLEDraftCudaGraphRunner, +) +from sglang.srt.speculative.eagle_draft_extend_cuda_graph_runner import ( + EAGLEDraftExtendCudaGraphRunner, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from _fakes import ARCH, ISA, SPEC, Lm, batch, server_args # noqa: E402 + +register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") + + +class DraftBuilt: + """Covers draft decode steps of up to max_bs rows: one row a request, so off the + width grid.""" + + # the EAGLE entries and graph variants it does not offer + draft_sampler = None + draft_loop = None + draft_extend = None + begin_draft_round = None + graph_variants = None + + def __init__(self, widths, max_bs): + self.widths, self.max_bs = widths, max_bs + + def summary(self): + return "fake draft" + + def plan(self, input_ids, forward_batch, *args, **kwargs): + rows = int(input_ids.shape[0]) + return Step(mode="draft_decode", rows=rows) if rows <= self.max_bs else None + + def forward(self, step, original, input_ids, positions, forward_batch, **kw): + return "megakernel" + + +def draft_spec(max_bs): + return replace( + SPEC, build=lambda model, hf_config, ctx: DraftBuilt(ctx.widths, max_bs) + ) + + +def draft_decode_runner(hooks, capture_bs=(), backends=None): + """An EAGLE draft decode runner with only what its capture hook call reads.""" + pool = object() + runner = EAGLEDraftCudaGraphRunner.__new__(EAGLEDraftCudaGraphRunner) + runner.capture_bs = list(capture_bs) + runner.model_runner = SimpleNamespace(capture_tail_hooks=hooks) + runner.draft_attn_backend = SimpleNamespace( + attn_backends=backends + or [SimpleNamespace(token_to_kv_pool=pool, step=i) for i in range(3)] + ) + return runner + + +def capture_draft_graphs(spec, capture_bs, steps=2): + """Install spec for a draft model, then capture its graphs as the EAGLE draft + decode runner does: the draft forwards of a batch size, then its capture hook + call.""" + lm, hooks = Lm(), [] + gpu = SimpleNamespace(multi_processor_count=160) + with ( + patch.object(binding, "find_spec", return_value=(spec, "")), + patch.object(binding, "device_isa", return_value=ISA), + patch.object(binding, "deadline_s", return_value=30.0), + patch.object(binding, "_gather_tp", side_effect=lambda x, group=None: [x]), + patch.object(binding, "_tokens_per_request", return_value=4), + patch.object(binding, "_decode_graphs_enabled", return_value=True), + patch.object(torch.cuda, "current_device", return_value=0), + patch.object(torch.cuda, "get_device_properties", return_value=gpu), + patch.object(torch.cuda, "mem_get_info", return_value=(0, 0)), + ): + binding.install_megakernel( + model=lm, + hf_config=SimpleNamespace(architectures=[ARCH]), + capture_tail_hooks=hooks, + is_draft=True, + ) + runner = draft_decode_runner(hooks, capture_bs) + with patch.object(torch.cuda, "is_current_stream_capturing", return_value=True): + for bs in capture_bs: + out = [lm(batch(bs).input_ids, None, batch(bs)) for _ in range(steps)] + runner._run_capture_tail_hooks(out, batch(bs), bs) + return hooks + + +class TestMegakernelDraftCapture(CustomTestCase): + def setUp(self): + registry_patch = patch.dict(registry._REGISTRY, clear=True) + registry_patch.start() + self.addCleanup(registry_patch.stop) + registry.register(architecture=ARCH, path="_fakes:SPEC") + publish(server_args(enable_nvidia_megakernel=True), role="test") + self.addCleanup(reset_context) + view_patch = patch.object( + MegakernelBinding, "forward_view", return_value=SimpleNamespace(hybrid=None) + ) + view_patch.start() + self.addCleanup(view_patch.stop) + + def test_a_draft_runner_capture_is_checked(self): + with ( + patch.object(MegakernelBinding, "assert_live") as assert_live, + self.assertLogs(binding.logger, "INFO") as logs, + ): + hooks = capture_draft_graphs(draft_spec(max_bs=2), capture_bs=[1, 2, 4]) + self.assertEqual(len(hooks), 1) + assert_live.assert_called_once_with() + self.assertIn( + "runs the decode graphs of batch sizes [1, 2] (2 of 3)", + "".join(logs.output), + ) + + def test_a_draft_runner_with_no_routed_graph_is_refused(self): + with self.assertRaisesRegex(ValueError, "none of the 2 captured decode graphs"): + capture_draft_graphs(draft_spec(max_bs=0), capture_bs=[1, 2]) + + +class TestEagleDraftRunnersFireTheHooks(CustomTestCase): + def test_the_draft_decode_runner_uses_its_first_step_backend(self): + seen = [] + runner = draft_decode_runner( + [lambda *args: seen.append((args, get_attn_backend()))] + ) + runner._run_capture_tail_hooks("out", "forward batch", 8) + first = runner.draft_attn_backend.attn_backends[0] + self.assertEqual(seen, [((runner, "out", "forward batch", 8), first)]) + + def test_step_backends_on_different_pools_are_refused(self): + backends = [SimpleNamespace(token_to_kv_pool=object()) for _ in range(2)] + runner = draft_decode_runner([lambda *args: None], backends=backends) + with self.assertRaisesRegex(RuntimeError, "do not share one KV pool"): + runner._run_capture_tail_hooks("out", "forward batch", 8) + + def test_without_hooks_nothing_is_read(self): + runner = draft_decode_runner([], backends=[None]) + runner._run_capture_tail_hooks("out", "forward batch", 8) + + def test_the_draft_extend_runner_fires_the_hooks(self): + seen = [] + runner = EAGLEDraftExtendCudaGraphRunner.__new__( + EAGLEDraftExtendCudaGraphRunner + ) + runner.model_runner = SimpleNamespace( + capture_tail_hooks=[lambda *args: seen.append(args)] + ) + runner._run_capture_tail_hooks("out", "forward batch", 16) + self.assertEqual(seen, [(runner, "out", "forward batch", 16)]) + + def test_the_captured_functions_call_the_hooks(self): + """capture_one_shape's stock draft forward ends with the hook call, with its + output, batch and token count.""" + for cls in (EAGLEDraftCudaGraphRunner, EAGLEDraftExtendCudaGraphRunner): + with self.subTest(runner=cls.__name__): + tree = ast.parse(inspect.getsource(inspect.getmodule(cls))) + owner = next( + n + for n in ast.walk(tree) + if isinstance(n, ast.ClassDef) and n.name == cls.__name__ + ) + capture = next( + n + for n in owner.body + if isinstance(n, ast.FunctionDef) and n.name == "capture_one_shape" + ) + run_stock = next( + n + for n in ast.walk(capture) + if isinstance(n, ast.FunctionDef) and n.name == "run_stock" + ) + calls = [ + [ast.unparse(a) for a in n.args] + for n in ast.walk(run_stock) + if isinstance(n, ast.Call) + and ast.unparse(n.func) == "self._run_capture_tail_hooks" + ] + self.assertEqual(calls, [["ret", "forward_batch", "num_tokens"]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_guards.py b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_guards.py index cce17e415524..7673752840a7 100644 --- a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_guards.py +++ b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_guards.py @@ -1,5 +1,6 @@ """CPU checks of --enable-nvidia-megakernel's startup resolution.""" +import os import sys import unittest from pathlib import Path @@ -7,6 +8,7 @@ from unittest.mock import patch from sglang.kernels.plugin_kernels.nvidia import spec as registry +from sglang.srt.environ import envs from sglang.srt.kernel_plugins.nvidia import binding from sglang.srt.kernel_plugins.nvidia.binding import ( megakernel_disable_reasons, @@ -15,7 +17,7 @@ from sglang.test.test_utils import CustomTestCase sys.path.insert(0, str(Path(__file__).resolve().parent)) -from _fakes import ARCH, ISA, server_args # noqa: E402 +from _fakes import ARCH, ISA, REPLICATION, server_args # noqa: E402 register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") @@ -52,5 +54,68 @@ def tp_reasons(path): self.assertEqual(tp_reasons("_fakes:ANY_TP"), []) +class TestFlagSetsEnv(CustomTestCase): + """With the flag, a variable of FLAG_SETS_ENV that the served spec requires and + the user left unset is set to the spec's value; a set value is kept.""" + + def setUp(self): + for patcher in ( + patch.dict(registry._REGISTRY, clear=True), + patch.dict(os.environ), + ): + patcher.start() + self.addCleanup(patcher.stop) + for name in (REPLICATION, "SGLANG_DSA_TOPK_BROADCAST"): + os.environ.pop(name, None) + + def resolve(self, path="_fakes:REPLICATED", **kw): + registry.register(architecture=ARCH, path=path) + hf_config = SimpleNamespace(architectures=[ARCH], model_type="fake") + model_config = SimpleNamespace(hf_config=hf_config, dtype="bfloat16") + with ( + patch.object(binding, "plugin_reasons", return_value=[]), + patch.object(binding, "device_isa", return_value=ISA), + patch.object(binding, "model_config_of", return_value=model_config), + ): + binding.handle_nvidia_megakernel( + server_args(**{"enable_nvidia_megakernel": True, **kw}) + ) + + def test_an_unset_variable_is_set_and_logged(self): + with self.assertLogs(binding.logger, "INFO") as logs: + self.resolve() + self.assertTrue(envs.SGLANG_ENABLE_EMBED_REPLICATION.get()) + self.assertIn(f"set {REPLICATION}=True for fake", "\n".join(logs.output)) + + def test_a_set_value_is_kept_and_checked(self): + os.environ[REPLICATION] = "1" + self.resolve() + self.assertEqual(os.environ[REPLICATION], "1") + os.environ[REPLICATION] = "0" + with self.assertRaisesRegex(ValueError, f"{REPLICATION} False is not True"): + self.resolve() + self.assertEqual(os.environ[REPLICATION], "0") + + def test_without_the_flag_nothing_is_set(self): + self.resolve(enable_nvidia_megakernel=False) + self.assertNotIn(REPLICATION, os.environ) + + def test_a_spec_that_does_not_require_it_leaves_it_unset(self): + self.resolve(path="_fakes:SPEC") + self.assertNotIn(REPLICATION, os.environ) + + def test_a_refused_setup_leaves_the_environment_as_it_was(self): + # SGLANG_DSA_TOPK_BROADCAST is not in FLAG_SETS_ENV: checked, never set + with self.assertRaisesRegex( + ValueError, "SGLANG_DSA_TOPK_BROADCAST False is not True" + ): + self.resolve(path="_fakes:REPLICATED_AND_BROADCAST") + self.assertNotIn(REPLICATION, os.environ) + self.assertNotIn("SGLANG_DSA_TOPK_BROADCAST", os.environ) + + def test_the_flag_sets_only_variables_read_when_a_model_is_built(self): + self.assertEqual(registry.FLAG_SETS_ENV, {REPLICATION}) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/spec/test_eagle_draft_embed_share.py b/test/registered/unit/spec/test_eagle_draft_embed_share.py new file mode 100644 index 000000000000..c914049fb32b --- /dev/null +++ b/test/registered/unit/spec/test_eagle_draft_embed_share.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU checks of when the EAGLE draft worker binds the target's embedding and lm_head: +under embedding replication before the scheduler sizes the KV pool (so the draft's own +full table is freed first), otherwise in ``alloc_memory_pool`` as before.""" + +import ast +import inspect +import textwrap +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from sglang.srt.environ import envs +from sglang.srt.speculative.eagle_worker_v2 import EagleDraftWorker +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _run(replicate: bool) -> list: + """The order of the draft worker's calls from the end of ``__init__`` through + ``alloc_memory_pool``.""" + calls = [] + worker = EagleDraftWorker.__new__(EagleDraftWorker) + worker.draft_worker = MagicMock() + worker.draft_worker.alloc_memory_pool.side_effect = lambda **_: calls.append( + "draft_pool" + ) + with ( + envs.SGLANG_ENABLE_EMBED_REPLICATION.override(replicate), + patch.object( + EagleDraftWorker, "init_token_map", lambda self: calls.append("token_map") + ), + patch.object( + EagleDraftWorker, "init_lm_head", lambda self: calls.append("lm_head") + ), + patch.object( + EagleDraftWorker, "_init_draft_probs_vocab_scatter", lambda self: None + ), + patch( + "sglang.srt.speculative.eagle_worker_v2.get_spec", + return_value=SimpleNamespace(speculative_use_rejection_sampling=False), + ), + ): + worker.target_embed_and_head_shared = False + worker._maybe_share_embed_before_pool_sizing() + calls.append("target_pool") + worker.alloc_memory_pool() + return calls + + +class TestEagleDraftEmbedShare(CustomTestCase): + def test_replicated_embedding_is_shared_before_the_pool_is_sized(self): + self.assertEqual( + _run(True), ["token_map", "lm_head", "target_pool", "draft_pool"] + ) + + def test_otherwise_it_is_shared_when_the_draft_pool_is_allocated(self): + self.assertEqual( + _run(False), ["target_pool", "draft_pool", "token_map", "lm_head"] + ) + + def test_init_shares_once_after_the_draft_runner_exists(self): + # The tests above call the early share directly, since running __init__ needs + # a GPU and a model. This guards the call site: __init__ must still make the + # call, once, after the draft model is built. Its exact position is free. + tree = ast.parse(textwrap.dedent(inspect.getsource(EagleDraftWorker.__init__))) + body = [ast.unparse(stmt) for stmt in tree.body[0].body] + share = [ + i + for i, s in enumerate(body) + if s == "self._maybe_share_embed_before_pool_sizing()" + ] + runner = [i for i, s in enumerate(body) if s.startswith("self.draft_runner =")] + self.assertEqual(len(share), 1, "__init__ must call the early share once") + self.assertEqual(len(runner), 1, "__init__ must set self.draft_runner once") + self.assertGreater(share[0], runner[0]) + + +if __name__ == "__main__": + unittest.main()