From 3dd1bdc5de8285fce42dc28163afc129821e6e46 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Sat, 17 Oct 2026 20:18:47 -0700 Subject: [PATCH 01/10] [Feat] [NVIDIA] server-args: Add --enable-nvidia-megakernel A launch flag for the NVIDIA megakernel plugin (the sgl-nvidia-mega package), in the exec argument group. SGLang only carries the flag and a startup check: the handle_nvidia_megakernel resolution step, in the argument-resolution pipeline, raises when the flag is set and the plugin's sglang.srt.plugins entry point (nvidia_megakernel) is not installed, with an install hint, or when SGLANG_PLUGINS is set and does not list it, since the plugin would then never load. Everything else, including the checks of the supported setup, stays in the plugin. Off by default; with it off nothing changes. --- .../advanced_features/server_arguments.mdx | 6 +++ python/sglang/srt/arg_groups/fields/exec_.py | 4 ++ python/sglang/srt/arg_groups/pipeline.py | 4 ++ .../sglang/srt/arg_groups/resolution_hooks.py | 1 + python/sglang/srt/plugins/__init__.py | 24 ++++++++++++ .../unit/plugins/test_load_plugins.py | 39 +++++++++++++++++++ 6 files changed, 78 insertions(+) diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index a09038aba7f1..4a9a2622fae5 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -1441,6 +1441,12 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec Choose the kernels for decode attention layers (have priority over --attention-backend). `None` triton, torch_native, flex_attention, dsa, nsa, dsv4, compressed, fa3, fa4, flashinfer, flashmla, trtllm_mla, cutedsl_mla, tokenspeed_mla, trtllm_mha, aiter, wave, intel_amx, ascend, intel_xpu + + + `--enable-nvidia-megakernel` + Enable the NVIDIA megakernel from the sgl-nvidia-mega package. Supported decode and verify steps use its fused kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature. + `False` + bool flag (set to enable) `--monokernel-backend` diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index 387fdc077b4a..466a587617ff 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -190,6 +190,10 @@ class ExecKernel(msgspec.Struct): Optional[bool], "Enable Lean (Work-Centric) Attention decode kernel for long-context serving. When None (default), uses auto-gate that activates Lean for long contexts and falls back to standard kernel for short contexts. Set to True to force enable, False to force disable.", ] = None + enable_nvidia_megakernel: A[ + bool, + "Enable the NVIDIA megakernel from the sgl-nvidia-mega package. Supported decode and verify steps use its fused kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature.", + ] = False monokernel_backend: A[ Optional[str], "Select a registered monokernel backend, e.g. amd from sgl-amd-mono. Supported decode and verify steps use persistent kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature.", diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py index d5b84fc2d4a1..c8e24cd85a4d 100644 --- a/python/sglang/srt/arg_groups/pipeline.py +++ b/python/sglang/srt/arg_groups/pipeline.py @@ -347,6 +347,10 @@ def run_resolution_pipeline(server_args: Any) -> None: run_hook(handle_monokernel, server_args) + from sglang.srt.plugins import handle_nvidia_megakernel + + run_hook(handle_nvidia_megakernel, server_args) + finalize_cuda_graph_prefill_max_context(server_args) # Capability adjustments can late-force a different attention backend or diff --git a/python/sglang/srt/arg_groups/resolution_hooks.py b/python/sglang/srt/arg_groups/resolution_hooks.py index 79c417428438..2349b25ce6cc 100644 --- a/python/sglang/srt/arg_groups/resolution_hooks.py +++ b/python/sglang/srt/arg_groups/resolution_hooks.py @@ -94,6 +94,7 @@ "handle_other_validations", "handle_model_capability_adjustments", "handle_monokernel", + "handle_nvidia_megakernel", "validate_deepep_v2_speculative_draft", "validate_deepep_v2_dispatch_token_budget", } diff --git a/python/sglang/srt/plugins/__init__.py b/python/sglang/srt/plugins/__init__.py index 00ae1acd1826..ece5c107ae34 100644 --- a/python/sglang/srt/plugins/__init__.py +++ b/python/sglang/srt/plugins/__init__.py @@ -28,6 +28,9 @@ PLATFORM_PLUGINS_GROUP = "sglang.srt.platforms" GENERAL_PLUGINS_GROUP = "sglang.srt.plugins" +# The general plugin that --enable-nvidia-megakernel needs (sgl-nvidia-mega) +NVIDIA_MEGAKERNEL_PLUGIN = "nvidia_megakernel" + # Guard against multiple loads in the same process _plugins_loaded = False @@ -100,6 +103,27 @@ def _get_excluded_dists() -> set[str]: return {ep.dist.name for ep in platform_eps if ep.dist and ep.name != selected} +def handle_nvidia_megakernel(server_args: Any) -> None: + """Fail at startup when --enable-nvidia-megakernel cannot load its plugin.""" + from sglang.srt.arg_groups.model_override_base import resolving_view + + if not resolving_view(server_args).enable_nvidia_megakernel: + return + name = NVIDIA_MEGAKERNEL_PLUGIN + if name not in {ep.name for ep in entry_points(group=GENERAL_PLUGINS_GROUP)}: + raise ValueError( + f"--enable-nvidia-megakernel needs the {name!r} plugin, which is not " + "installed. Install sgl-nvidia-mega " + "(https://github.com/sgl-project/sgl-nvidia-mega)." + ) + allowed = envs.SGLANG_PLUGINS.get() + if allowed and name not in {x.strip() for x in allowed.split(",")}: + raise ValueError( + f"--enable-nvidia-megakernel needs the {name!r} plugin, but " + f"SGLANG_PLUGINS={allowed!r} does not list it." + ) + + def load_plugins(): """ Load and execute all general plugins, then apply registered hooks. diff --git a/test/registered/unit/plugins/test_load_plugins.py b/test/registered/unit/plugins/test_load_plugins.py index 3b84f88934b0..837a8dd33935 100644 --- a/test/registered/unit/plugins/test_load_plugins.py +++ b/test/registered/unit/plugins/test_load_plugins.py @@ -181,6 +181,45 @@ def bad_plugin(): self.assertIsNone(_current_plugin_source.get()) +class TestNvidiaMegakernelFlag(CustomTestCase): + """--enable-nvidia-megakernel fails at argument resolution unless its plugin loads.""" + + def _check(self, enabled, eps, allowed=""): + from types import SimpleNamespace + + from sglang.srt.plugins import handle_nvidia_megakernel + + view = SimpleNamespace(enable_nvidia_megakernel=enabled) + with ( + patch( + "sglang.srt.arg_groups.model_override_base.resolving_view", + return_value=view, + ), + patch("sglang.srt.plugins.entry_points", return_value=eps) as found, + patch("sglang.srt.plugins.envs") as mock_envs, + ): + mock_envs.SGLANG_PLUGINS.get.return_value = allowed + handle_nvidia_megakernel(object()) + return found + + def test_flag_off_checks_nothing(self): + found = self._check(False, []) + found.assert_not_called() + + def test_flag_on_without_the_plugin_raises_with_an_install_hint(self): + with self.assertRaisesRegex(ValueError, "Install sgl-nvidia-mega"): + self._check(True, [_make_ep("other")]) + + def test_flag_on_with_sglang_plugins_excluding_it_raises(self): + with self.assertRaisesRegex(ValueError, "SGLANG_PLUGINS='other'"): + self._check(True, [_make_ep("nvidia_megakernel")], allowed="other") + + def test_flag_on_with_the_plugin_passes(self): + eps = [_make_ep("nvidia_megakernel")] + self._check(True, eps) + self._check(True, eps, allowed="other, nvidia_megakernel") + + if __name__ == "__main__": import unittest From 14f9fd27d4b2201a285e2dbb6963de174bd8e19a Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Sat, 17 Oct 2026 20:38:37 -0700 Subject: [PATCH 02/10] [Feat] [NVIDIA] megakernels: Add the spec contract and interface stubs The sglang.srt.megakernels package for --enable-nvidia-megakernel. spec.py holds the contract a plugin implements (MegakernelSpec, Requirements, Numerics, BuildContext, Step, the BuiltMegakernel protocol with an optional draft_sampler factory) and the lazy per-architecture registry the plugin's entry point fills. guards.py and runtime.py are interface stubs, filled in by the next commits. --- python/sglang/srt/megakernels/__init__.py | 7 + python/sglang/srt/megakernels/guards.py | 30 ++++ python/sglang/srt/megakernels/runtime.py | 38 ++++ python/sglang/srt/megakernels/spec.py | 200 ++++++++++++++++++++++ 4 files changed, 275 insertions(+) create mode 100644 python/sglang/srt/megakernels/__init__.py create mode 100644 python/sglang/srt/megakernels/guards.py create mode 100644 python/sglang/srt/megakernels/runtime.py create mode 100644 python/sglang/srt/megakernels/spec.py diff --git a/python/sglang/srt/megakernels/__init__.py b/python/sglang/srt/megakernels/__init__.py new file mode 100644 index 000000000000..c0ea4256c628 --- /dev/null +++ b/python/sglang/srt/megakernels/__init__.py @@ -0,0 +1,7 @@ +"""NVIDIA megakernels: whole-step dispatch for out-of-tree megakernel plugins. + +Plugins register specs through the sglang.srt.plugins entry point. With +--enable-nvidia-megakernel the runtime validates the configuration, builds the +spec after the model is prepared, and routes covered steps through it; other +steps use the model forward. +""" diff --git a/python/sglang/srt/megakernels/guards.py b/python/sglang/srt/megakernels/guards.py new file mode 100644 index 000000000000..67418b790e6c --- /dev/null +++ b/python/sglang/srt/megakernels/guards.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Startup checks for --enable-nvidia-megakernel on the resolved configuration.""" + +from __future__ import annotations + +from typing import Any, Optional + +from sglang.srt.megakernels.spec import MegakernelSpec + + +def device_isa() -> Optional[str]: + """This process's GPU as "sm_", or None without a visible GPU.""" + raise NotImplementedError # TODO(L98) + + +def deadline_s(watchdog_timeout: float) -> float: + """The deadline of every cross-GPU wait, below the watchdog timeout.""" + raise NotImplementedError # TODO(L98) + + +def megakernel_disable_reasons( + *, cfg: Any, hf_config: Any, isa: Optional[str] +) -> tuple[Optional[MegakernelSpec], list[str]]: + """The spec for this model and every reason it cannot serve this configuration.""" + raise NotImplementedError # TODO(L98) + + +def handle_nvidia_megakernel(server_args: Any) -> None: + """Resolution step: fail before the weights load when the flag cannot be served.""" + raise NotImplementedError # TODO(L98) diff --git a/python/sglang/srt/megakernels/runtime.py b/python/sglang/srt/megakernels/runtime.py new file mode 100644 index 000000000000..0a8f08f1a7fd --- /dev/null +++ b/python/sglang/srt/megakernels/runtime.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Build and whole-step dispatch for --enable-nvidia-megakernel.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Callable, Optional + +import torch + +from sglang.srt.megakernels.spec import BuiltMegakernel + +if TYPE_CHECKING: + from transformers import PretrainedConfig + + +def megakernel_enabled() -> bool: + """Whether this server runs with --enable-nvidia-megakernel.""" + raise NotImplementedError + + +def install_megakernel( + *, + model: torch.nn.Module, + hf_config: PretrainedConfig, + capture_tail_hooks: list[Callable[..., None]], +) -> None: + """Build the model's spec and install its forward; runs before KV cache sizing.""" + raise NotImplementedError + + +def built_megakernel(model: torch.nn.Module) -> Optional[BuiltMegakernel]: + """The object the spec built for this model, or None.""" + raise NotImplementedError + + +def megakernel_draft_sampler(draft_model: torch.nn.Module) -> Optional[Callable]: + """The built draft spec's draft_sampler factory, or None for SGLang's proposal.""" + raise NotImplementedError diff --git a/python/sglang/srt/megakernels/spec.py b/python/sglang/srt/megakernels/spec.py new file mode 100644 index 000000000000..6afc59df6c46 --- /dev/null +++ b/python/sglang/srt/megakernels/spec.py @@ -0,0 +1,200 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Megakernel spec contract and the lazy per-architecture registry.""" + +from __future__ import annotations + +import enum +import importlib +from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Protocol, Sequence + +import msgspec + +if TYPE_CHECKING: + import torch + +# bumped on any breaking change to the contracts in this module +MEGAKERNEL_API_VERSION = 1 + +# the sglang.srt.plugins entry point that registers the specs +ENTRY_POINT = "nvidia_megakernel" + + +class Feature(enum.Enum): + """SGLang features refused unless a spec lists them.""" + + PIPELINE_PARALLEL = "pipeline parallelism" + MULTI_NODE_TP = "multi-node tensor parallelism" + DP_ATTENTION = "DP attention" + EXPERT_PARALLEL = "expert parallelism" + EPLB = "EPLB" + CONTEXT_PARALLEL = "context parallelism" + SPECULATIVE_DECODING = "speculative decoding" + RAGGED_VERIFY = "uneven tokens per request in verify (SGLANG_RAGGED_VERIFY_MODE)" + TWO_BATCH_OVERLAP = "two-batch / single-batch overlap" + LORA = "LoRA" + PD_DISAGGREGATION = "PD disaggregation" + RETURN_ROUTED_EXPERTS = "--enable-return-routed-experts" + RETURN_INDEXER_TOPK = "--enable-return-indexer-topk" + KV_OFFLOAD = "KV offload" + HISPARSE = "hierarchical sparse attention" + MEMORY_RELEASE = "memory release" + CPU_OFFLOAD = "CPU offload" + TORCH_COMPILE = "torch-compiled decode" + DETERMINISTIC_INFERENCE = "deterministic inference" + PARTITIONED_GPU = "partitioned GPUs" + + +class Requirements(msgspec.Struct, frozen=True): + """Configuration a spec needs, checked on the resolved settings at startup; an + empty field accepts any value.""" + + isas: frozenset[str] # e.g. "sm_103" + tp_sizes: frozenset[int] + resident_blocks: int = 0 # SMs the persistent grids need + dtypes: frozenset[str] = frozenset() + # the decode CUDA graph backends the served steps run under + decode_graph_backends: frozenset[str] = frozenset({"full", "disabled"}) + # a server argument -> its accepted values, as resolved for this server + server_args: Mapping[str, frozenset] = {} + # an HF config field -> the value the kernels are built for + hf_config: Mapping[str, Any] = {} + # an SGLang environment variable in envs -> its required value + env: Mapping[str, Any] = {} + + +class NumericsKind(enum.Enum): + BITWISE = "bitwise" + TOLERANCE = "tolerance" + + +class Numerics(msgspec.Struct, frozen=True): + """Declared agreement with the standard path on the same GPU, logged at build.""" + + kind: NumericsKind + tolerances: Mapping[str, float] = {} + + def __post_init__(self): + if self.kind is NumericsKind.TOLERANCE and not self.tolerances: + raise ValueError("a TOLERANCE spec must state its tolerances") + + +class BuildContext(msgspec.Struct, frozen=True): + widths: tuple[int, ...] # step rows to build, ascending + tokens_per_request: int + deadline_s: float # every cross-GPU wait's deadline, below --watchdog-timeout + + +class Step(msgspec.Struct, frozen=True): + """A covered step, decided from host state alike on every rank.""" + + mode: str # "decode" or "target_verify" + rows: int + + +class BuiltMegakernel(Protocol): + """The object a spec's build returns. + + draft_sampler is None, or a factory the DSpark draft sampler calls once with + (*, gamma, max_bs, greedy_draft, noise, draft_tokens, step_logits, + step_temperatures). It returns bs -> None (SGLang's proposal at that batch size) + or step(hidden_states, anchor_tokens) -> (tokens, confidence_features), which + replaces the base logits and the Markov block. The sizes it serves must agree on + every rank (SGLang refuses otherwise), and a step must produce identical tokens + on every rank, since it replaces the proposal's rank-0 broadcast; write + draft_tokens[: bs * gamma] and the sampled rows' step_logits; and advance + noise.offset once per sampled block. + """ + + widths: tuple[int, ...] # the rows it built, possibly fewer than requested + draft_sampler: Optional[Callable[..., Callable[[int], Optional[Callable]]]] + + def summary(self) -> str: + """What was built, for the startup line.""" + + def plan( + self, input_ids: Any, forward_batch: Any, *args: Any, **kwargs: Any + ) -> Optional[Step]: + """The step to run, or None for the model's own forward; host-only, so a + captured graph replays the decision made at capture.""" + + def forward(self, step: Step, original: Callable, *args: Any, **kwargs: Any) -> Any: + """Run step in place of original (the model's bound forward), taking and + returning what it does; capturable in a CUDA graph.""" + + +class MegakernelSpec(msgspec.Struct, frozen=True): + """A model-specific plugin contract; build returns its BuiltMegakernel. + + Argument resolution checks the served model's spec in full. A spec found only + at load, e.g. for a draft model, is checked there against its ISA, tp_sizes, + decode_graph_backends, hf_config, env and model_disable_reasons. + """ + + name: str + requires: Requirements + features: frozenset[Feature] # the refused-by-default features it accepts + numerics: Numerics + max_rows: int # the widest step it covers + build: Callable[[torch.nn.Module, Any, BuildContext], BuiltMegakernel] + api_version: int # the MEGAKERNEL_API_VERSION this spec is written against + # (model, hf_config), after load: per-rank checks of the loaded model + model_disable_reasons: Optional[Callable[[torch.nn.Module, Any], list[str]]] = None + # (cfg, hf_config), before load: the checks requires cannot state + config_disable_reasons: Optional[Callable[[Any, Any], list[str]]] = None + + +# HF architecture -> the "module:attribute" path of its spec +_REGISTRY: dict[str, str] = {} + + +def register(*, architecture: str, path: str) -> None: + """Register a module:attribute spec path from the plugin entry point.""" + if _REGISTRY.get(architecture, path) != path: + raise ValueError( + f"a megakernel spec for {architecture} is already registered: " + f"{_REGISTRY[architecture]}" + ) + _REGISTRY[architecture] = path + + +def registered_architectures() -> list[str]: + """The architectures with a registered spec; empty when no plugin registered any.""" + return sorted(_REGISTRY) + + +def spec_registered(architectures: Optional[Sequence[str]]) -> bool: + """Whether any of the architectures has a registered spec, usable or not.""" + return any(arch in _REGISTRY for arch in architectures or ()) + + +def find_spec( + *, architectures: Optional[Sequence[str]], isa: Optional[str] +) -> tuple[Optional[MegakernelSpec], str]: + """The compatible spec and an empty reason, or None and a rejection reason.""" + why = [] + for arch in architectures or (): + path = _REGISTRY.get(arch) + if path is None: + continue + module, _, attr = path.partition(":") + # a broken plugin is a refusal every rank reports, not a crash on one + try: + spec = getattr(importlib.import_module(module), attr) + except Exception as e: + why.append(f"{path} does not import: {type(e).__name__}: {e}") + continue + if not isinstance(spec, MegakernelSpec): + why.append(f"{path} is not a MegakernelSpec") + elif spec.api_version != MEGAKERNEL_API_VERSION: + why.append( + f"{path} uses megakernel API version {spec.api_version}; this " + f"SGLang requires version {MEGAKERNEL_API_VERSION}" + ) + elif isa not in spec.requires.isas: + why.append(f"{path} supports {sorted(spec.requires.isas)}, not {isa}") + else: + return spec, "" + return None, "; ".join(why) or ( + f"no megakernel spec is registered for {list(architectures or ())} " + f"(registered: {registered_architectures()})" + ) From 2fdbcac3ef4bb3eab7720ddd771693da690aaa10 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Sat, 17 Oct 2026 20:47:11 -0700 Subject: [PATCH 03/10] [Feat] [NVIDIA] megakernels: Build and dispatch the spec after ModelRunner.initialize install_megakernel runs as the last step of ModelRunner.initialize, after every post-load change to the model and before the KV cache is sized. With --enable-nvidia-megakernel it finds the spec for the model's architecture on every TP rank, gathers every rank's refusal reasons and raises on all ranks if any rank has one, builds the spec at the step widths of whole requests up to its max_rows, and replaces the forward on that model instance only: a step the built object plans runs it, every other step the model's forward. Weight updates then refuse, one line logs what is active, and a capture check refuses a runner none of whose decode graphs run the megakernel. A model without a registered spec, such as a draft, keeps its forward; a spec that builds no width only offers its draft sampler (megakernel_draft_sampler). --- python/sglang/srt/megakernels/runtime.py | 296 ++++++++++++++++- python/sglang/srt/megakernels/spec.py | 1 + .../sglang/srt/model_executor/model_runner.py | 7 + .../megakernels/test_megakernel_runtime.py | 311 ++++++++++++++++++ 4 files changed, 609 insertions(+), 6 deletions(-) create mode 100644 test/registered/unit/megakernels/test_megakernel_runtime.py diff --git a/python/sglang/srt/megakernels/runtime.py b/python/sglang/srt/megakernels/runtime.py index 0a8f08f1a7fd..7df2b586a782 100644 --- a/python/sglang/srt/megakernels/runtime.py +++ b/python/sglang/srt/megakernels/runtime.py @@ -3,19 +3,52 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Callable, Optional +import functools +import logging +import time +from typing import TYPE_CHECKING, Any, Callable, Optional import torch +import torch.distributed as dist -from sglang.srt.megakernels.spec import BuiltMegakernel +from sglang.srt.environ import envs +from sglang.srt.megakernels.guards import deadline_s, device_isa +from sglang.srt.megakernels.spec import ( + FLAG, + BuildContext, + BuiltMegakernel, + Feature, + MegakernelSpec, + Step, + find_spec, + spec_registered, +) +from sglang.srt.model_executor.cuda_graph_config import Backend +from sglang.srt.runtime_context import ( + get_context, + get_device, + get_exec, + get_parallel, + get_spec, +) if TYPE_CHECKING: from transformers import PretrainedConfig + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + from sglang.srt.model_executor.runner.decode_cuda_graph_runner import ( + DecodeCudaGraphRunner, + ) + +logger = logging.getLogger(__name__) + def megakernel_enabled() -> bool: """Whether this server runs with --enable-nvidia-megakernel.""" - raise NotImplementedError + # the model loader also runs without a server's config (tools, unit tests) + if not get_context().is_config_namespace_published("exec"): + return False + return get_exec().kernel.enable_nvidia_megakernel def install_megakernel( @@ -25,14 +58,265 @@ def install_megakernel( capture_tail_hooks: list[Callable[..., None]], ) -> None: """Build the model's spec and install its forward; runs before KV cache sizing.""" - raise NotImplementedError + if not megakernel_enabled(): + return + spec = _find_spec_on_every_rank(hf_config=hf_config) + if spec is None: + return + # Resolution checked the target's spec on the server arguments; a draft's spec + # is checked only here, on what the runner reads (no server-argument checks). + reasons = _requirement_reasons(spec=spec, hf_config=hf_config) + reasons += _model_disable_reasons(spec=spec, model=model, hf_config=hf_config) + _refuse(_gather_reasons(reasons)) + + tokens = _tokens_per_request() + context = BuildContext( + widths=tuple(range(tokens, spec.max_rows + 1, tokens)), + tokens_per_request=tokens, + deadline_s=deadline_s(get_device().watchdog_timeout), + ) + start, free_before = time.perf_counter(), torch.cuda.mem_get_info()[0] + built = spec.build(model, hf_config, context) + # not setattr: a built object that is an nn.Module must not join the state dict + model.__dict__["_megakernel"] = built + # Different widths would leave ranks waiting in different collectives. + megakernel_agree(tuple(built.widths), what=f"widths {spec.name} built") + if spec.max_rows > 0 and not built.widths: + _refuse([f"{spec.name} built no width of {list(context.widths)}"]) + # Captured kernels keep state derived from the weights that updates cannot refresh. + model._derived_weight_cache_error = ( + f"Online weight updates are not supported with {FLAG} ({spec.name})" + ) + built_s = time.perf_counter() - start + used_mib = (free_before - torch.cuda.mem_get_info()[0]) / 2**20 + if not built.widths: + # a spec that covers no step, e.g. a draft spec offering draft_sampler only + logger.info( + "NVIDIA megakernel: %s: %s", + spec.name, + "draft sampler" + if built.draft_sampler is not None + else "no step and no draft sampler", + ) + return + logger.info( + "NVIDIA megakernel: active (%s: %s; widths %s; %s numerics; built in %.1f s, " + "%.1f MiB; wait deadline %.0f s)", + spec.name, + built.summary(), + list(built.widths), + spec.numerics.kind.value, + built_s, + used_mib, + context.deadline_s, + ) + dispatch = _Dispatch(built=built, original=model.forward) + model.forward = dispatch.wrapped() + if _decode_graphs_enabled(): + capture_tail_hooks.append(_CaptureCheck(spec=spec, dispatch=dispatch)) def built_megakernel(model: torch.nn.Module) -> Optional[BuiltMegakernel]: """The object the spec built for this model, or None.""" - raise NotImplementedError + return model.__dict__.get("_megakernel") + + +def megakernel_agree(value: Any, *, what: str, group: Any = None) -> None: + """Raise on every rank unless every rank of group (default: the TP group) holds + the same value; ranks that differ would wait in different collectives.""" + values = _gather_tp(value, group=group) + if any(v != value for v in values): + raise RuntimeError( + f"{FLAG}: the ranks' {what} differ: " + + "; ".join(f"rank {rank}: {v}" for rank, v in enumerate(values)) + ) def megakernel_draft_sampler(draft_model: torch.nn.Module) -> Optional[Callable]: """The built draft spec's draft_sampler factory, or None for SGLang's proposal.""" - raise NotImplementedError + built = built_megakernel(draft_model) + return None if built is None else built.draft_sampler + + +def _find_spec_on_every_rank( + *, hf_config: PretrainedConfig +) -> Optional[MegakernelSpec]: + """The spec every rank found, None when no rank has one registered; else raise.""" + archs = hf_config.architectures + spec, why = find_spec(architectures=archs, isa=device_isa()) + # Ranks branch together; one skipping the collectives below would hang its peers. + lookups = _gather_tp( + ( + spec_registered(archs), + why, + None if spec is None else (spec.name, spec.api_version), + ) + ) + if not any(is_registered for is_registered, _, _ in lookups): + # A draft model may have no registered spec. + logger.info("NVIDIA megakernel: %s; %s keeps its own forward", why, archs) + return None + _refuse( + [ + f"rank {rank}: {why}" + for rank, (_, why, identity) in enumerate(lookups) + if identity is None + ] + ) + if len({identity for *_, identity in lookups}) > 1: + raise ValueError( + f"{FLAG}: the ranks load different specs: " + + "; ".join(f"rank {r}: {ident}" for r, (*_, ident) in enumerate(lookups)) + ) + return spec + + +def _requirement_reasons( + *, spec: MegakernelSpec, hf_config: PretrainedConfig +) -> list[str]: + """The spec's requirements this runner can check without the server arguments.""" + req, tp_size = spec.requires, get_parallel().tp_size + reasons = [] + if tp_size not in req.tp_sizes: + reasons.append(f"tp_size {tp_size} is not one of {sorted(req.tp_sizes)}") + graph = get_exec().graph.cuda_graph_config + backend = Backend.DISABLED if graph is None else graph.decode.backend + if req.decode_graph_backends and backend not in req.decode_graph_backends: + reasons.append( + f"decode CUDA graphs {backend!r} are not one of " + f"{sorted(req.decode_graph_backends)}" + ) + exactly = [ + (f"config.{key}", getattr(hf_config, key, None), want) + for key, want in req.hf_config.items() + ] + [(name, getattr(envs, name).get(), want) for name, want in req.env.items()] + reasons += [ + f"{what} {v!r} is not {want!r}" for what, v, want in exactly if v != want + ] + return reasons + + +def _model_disable_reasons( + *, spec: MegakernelSpec, model: torch.nn.Module, hf_config: PretrainedConfig +) -> list[str]: + """This rank's reasons the loaded model cannot run the spec.""" + reasons = [] + if spec.model_disable_reasons is not None: + reasons += spec.model_disable_reasons(model, hf_config) + if "forward" in model.__dict__: + reasons.append("the model's forward is already replaced on this instance") + sms = torch.cuda.get_device_properties( + torch.cuda.current_device() + ).multi_processor_count + if sms < spec.requires.resident_blocks: + reasons.append( + f"{Feature.PARTITIONED_GPU.value}: {sms} SMs, the persistent grids need " + f"{spec.requires.resident_blocks}" + ) + return reasons + + +class _Dispatch: + """The installed forward: covered steps run the built megakernel.""" + + def __init__(self, *, built: BuiltMegakernel, original: Callable): + self.built, self.original = built, original + # batch size -> the step planned while capturing its decode graphs; plan reads + # only the mode and rows, so every graph variant of a batch size plans alike + self.captured: dict[int, Optional[Step]] = {} + + def wrapped(self) -> Callable: + # Graph runners and supports_pp inspect the original forward's signature. + @functools.wraps(self.original) + def forward(input_ids, positions, forward_batch, *args, **kwargs): + step = self.built.plan(input_ids, forward_batch, *args, **kwargs) + if torch.cuda.is_current_stream_capturing(): + self.captured[int(forward_batch.batch_size)] = step + if step is None: + return self.original( + input_ids, positions, forward_batch, *args, **kwargs + ) + return self.built.forward( + step, + self.original, + input_ids, + positions, + forward_batch, + *args, + **kwargs, + ) + + return forward + + +class _CaptureCheck: + """Once every decode graph is captured, log the ones that run the megakernel and + refuse a runner none of whose graphs do.""" + + def __init__(self, *, spec: MegakernelSpec, dispatch: _Dispatch): + self.spec, self.dispatch = spec, dispatch + self.is_done = False + + def __call__( + self, + runner: DecodeCudaGraphRunner, + out: Any, + forward_batch: ForwardBatch, + num_tokens: int, + ) -> None: + captured = self.dispatch.captured + if self.is_done or not set(runner.capture_bs) <= set(captured): + return + self.is_done = True + routed = sorted(bs for bs, step in captured.items() if step is not None) + if not routed: + raise ValueError( + f"{FLAG}: none of the {len(captured)} captured decode graphs " + f"(batch sizes {sorted(captured)}) runs {self.spec.name}" + ) + logger.info( + "NVIDIA megakernel: %s runs the decode graphs of batch sizes %s (%d of " + "%d); every other step runs the standard forward", + self.spec.name, + routed, + len(routed), + len(captured), + ) + + +def _tokens_per_request() -> int: + """Tokens per request of this server's decode-phase steps.""" + spec_config = get_spec() + if spec_config.speculative_algorithm is None: + return 1 + return int(spec_config.speculative_num_draft_tokens or 1) + + +def _decode_graphs_enabled() -> bool: + graph = get_exec().graph.cuda_graph_config + return graph is not None and graph.decode.backend != Backend.DISABLED + + +def _gather_tp(obj: Any, group: Any = None) -> list[Any]: + """Gather a value from every rank of group (default: TP) through its CPU group.""" + group = get_parallel().tp_group if group is None else group + if group.world_size == 1: + return [obj] + out: list[Any] = [None] * group.world_size + dist.all_gather_object(out, obj, group=group.cpu_group) + return out + + +def _gather_reasons(reasons: list[str]) -> list[str]: + return [ + f"rank {rank}: {reason}" + for rank, rank_reasons in enumerate(_gather_tp(reasons)) + for reason in rank_reasons + ] + + +def _refuse(reasons: list[str]) -> None: + if reasons: + raise ValueError( + f"{FLAG} cannot run this setup:\n - " + "\n - ".join(reasons) + ) diff --git a/python/sglang/srt/megakernels/spec.py b/python/sglang/srt/megakernels/spec.py index 6afc59df6c46..bd242a72fa27 100644 --- a/python/sglang/srt/megakernels/spec.py +++ b/python/sglang/srt/megakernels/spec.py @@ -17,6 +17,7 @@ # the sglang.srt.plugins entry point that registers the specs ENTRY_POINT = "nvidia_megakernel" +FLAG = "--enable-nvidia-megakernel" class Feature(enum.Enum): diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 6818ad777830..ab082dd2784c 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -92,6 +92,7 @@ from sglang.srt.lora.lora_manager import LoRAManager, init_lora_cuda_graph_moe_buffers from sglang.srt.lora.lora_registry import LoRARef from sglang.srt.managers.schedule_batch import sanity_check_mm_pad_shift_value +from sglang.srt.megakernels.runtime import install_megakernel from sglang.srt.mem_cache import kv_cache_dtype from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.kv_cache_configurator import ( @@ -726,6 +727,12 @@ def initialize(self): self.maybe_init_lora_manager() self.maybe_enable_batch_invariant_mode() self.configure_kv_cache_dtype() + # after every post-load change to the model, before the KV cache is sized + install_megakernel( + model=self.model, + hf_config=self.model_config.hf_config, + capture_tail_hooks=self.capture_tail_hooks, + ) def init_memory_saver_adapter(self): self.memory_saver_adapter = TorchMemorySaverAdapter.create( diff --git a/test/registered/unit/megakernels/test_megakernel_runtime.py b/test/registered/unit/megakernels/test_megakernel_runtime.py new file mode 100644 index 000000000000..8fca2463f844 --- /dev/null +++ b/test/registered/unit/megakernels/test_megakernel_runtime.py @@ -0,0 +1,311 @@ +"""CPU contracts of the megakernel registry, install, dispatch and capture check.""" + +import inspect +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch +from msgspec.structs import replace + +from sglang.srt.megakernels import runtime +from sglang.srt.megakernels import spec as registry +from sglang.srt.megakernels.guards import deadline_s +from sglang.srt.megakernels.spec import ( + MegakernelSpec, + Numerics, + NumericsKind, + Requirements, + Step, +) +from sglang.srt.model_executor.cuda_graph_config import ( + Backend, + Phase, + default_cuda_graph_config, + with_phase, +) +from sglang.srt.runtime_context import publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") + +ARCH = "FakeMegaForCausalLM" +ISA = "sm_103" + + +class FakeBuilt: + """Covers steps of 1-4 rows and records the steps it ran.""" + + draft_sampler = None + + def __init__(self, widths): + self.widths = tuple(w for w in widths if w <= 4) + self.steps = [] + + def summary(self): + return "fake" + + def plan(self, input_ids, forward_batch, *args, **kwargs): + rows = int(input_ids.shape[0]) + return Step(mode="decode", rows=rows) if rows in self.widths else None + + def forward(self, step, original, input_ids, positions, forward_batch, **kw): + self.steps.append(step.rows) + return "megakernel" + + +def _build(model, hf_config, context): + model.context = context + return FakeBuilt(context.widths) + + +SPEC = MegakernelSpec( + name="fake", + requires=Requirements(isas=frozenset({ISA}), tp_sizes=frozenset({4})), + features=frozenset(), + numerics=Numerics(kind=NumericsKind.BITWISE), + max_rows=8, + build=_build, + api_version=registry.MEGAKERNEL_API_VERSION, +) +OLD_SPEC = replace(SPEC, api_version=registry.MEGAKERNEL_API_VERSION - 1) +# a draft spec: builds no width, offers only the draft sampler +DRAFT_SPEC = replace( + SPEC, + max_rows=0, + build=lambda model, hf, ctx: SimpleNamespace( + widths=(), summary=lambda: "draft sampler", draft_sampler=print + ), +) + + +class Lm(torch.nn.Module): + def forward(self, input_ids, positions, forward_batch, input_embeds=None): + return "standard" + + +def batch(rows): + return SimpleNamespace(batch_size=rows, input_ids=torch.zeros(rows)) + + +def server_args(**kw): + defaults = { + "model_path": "dummy", + "enable_nvidia_megakernel": True, + "tp_size": 4, + "speculative_algorithm": "DSPARK", + "speculative_num_draft_tokens": 6, + "cuda_graph_config": default_cuda_graph_config(), + } + return ServerArgs(**{**defaults, **kw}) + + +def install( + *, archs=(ARCH,), gathered=None, tokens=2, graphs=True, hooks=None, lm=None +): + """install_megakernel on an Lm as rank 0 of the published server; gathered maps + a gathered value to what the other ranks report, default the same as this rank.""" + lm, hooks = lm or Lm(), [] if hooks is None else hooks + gather = gathered or (lambda x, group=None: [x]) + gpu = SimpleNamespace(multi_processor_count=160) + with ( + patch.object(runtime, "device_isa", return_value=ISA), + patch.object(runtime, "deadline_s", return_value=30.0), + patch.object(runtime, "_gather_tp", side_effect=gather), + patch.object(runtime, "_tokens_per_request", return_value=tokens), + patch.object(runtime, "_decode_graphs_enabled", return_value=graphs), + patch.object(torch.cuda, "current_device", return_value=0), + patch.object(torch.cuda, "get_device_properties", return_value=gpu), + patch.object(torch.cuda, "mem_get_info", return_value=(0, 0)), + ): + runtime.install_megakernel( + model=lm, + hf_config=SimpleNamespace(architectures=list(archs)), + capture_tail_hooks=hooks, + ) + return lm + + +class TestMegakernelRuntime(CustomTestCase): + def setUp(self): + registry_patch = patch.dict(registry._REGISTRY, clear=True) + registry_patch.start() + self.addCleanup(registry_patch.stop) + registry.register(architecture=ARCH, path=f"{__name__}:SPEC") + publish(server_args(), role="test") + self.addCleanup(reset_context) + + def _install_spec(self, spec, **kw): + with patch.object(runtime, "find_spec", return_value=(spec, "")): + return install(**kw) + + def test_the_runtime_reads_the_published_config(self): + self.assertTrue(runtime.megakernel_enabled()) + self.assertEqual(runtime._tokens_per_request(), 6) + self.assertTrue(runtime._decode_graphs_enabled()) + reset_context() + eager = with_phase( + default_cuda_graph_config(), Phase.DECODE, backend=Backend.DISABLED + ) + publish(server_args(watchdog_timeout=40, cuda_graph_config=eager), role="test") + self.assertFalse(runtime._decode_graphs_enabled()) + self.assertEqual(deadline_s(runtime.get_device().watchdog_timeout), 20.0) + reset_context() + publish(server_args(enable_nvidia_megakernel=False), role="test") + self.assertFalse(runtime.megakernel_enabled()) + + def test_a_spec_found_at_load_is_checked_on_what_the_runner_reads(self): + # a draft's spec is never checked at resolution + strict = replace( + SPEC, + requires=replace( + SPEC.requires, + tp_sizes=frozenset({8}), + decode_graph_backends=frozenset({"breakable"}), + hf_config={"model_type": "fake"}, + ), + ) + with self.assertRaises(ValueError) as refused: + self._install_spec(strict) + for want in ("tp_size 4", "decode CUDA graphs 'full'", "config.model_type"): + self.assertIn(want, str(refused.exception)) + + def test_each_load_refusal_is_raised(self): + def other_spec_on_rank_1(value, group=None): + is_lookup = isinstance(value, tuple) and isinstance(value[-1], tuple) + other = (value[0], "", ("other", 1)) if is_lookup else value + return [value, other] + + with self.assertRaisesRegex(ValueError, "load different specs"): + install(gathered=other_spec_on_rank_1) + partitioned = replace( + SPEC, requires=replace(SPEC.requires, resident_blocks=200) + ) + with self.assertRaisesRegex(ValueError, "160 SMs"): + self._install_spec(partitioned) + with self.assertRaisesRegex(ValueError, "already replaced"): + install(lm=install(tokens=1)) + # a target that covers no width must not quietly keep the standard forward + empty = replace(SPEC, build=lambda model, hf, ctx: FakeBuilt(())) + with self.assertRaisesRegex(ValueError, "built no width of"): + self._install_spec(empty) + + def test_find_spec_refuses_with_a_reason(self): + registry.register(architecture="Broken", path="no_such_module:SPEC") + registry.register(architecture="Old", path=f"{__name__}:OLD_SPEC") + self.assertIs(registry.find_spec(architectures=[ARCH], isa=ISA)[0], SPEC) + version = registry.MEGAKERNEL_API_VERSION + for archs, isa, why in ( + ([ARCH], "sm_90", "supports ['sm_103'], not sm_90"), + (["Other"], ISA, "no megakernel spec is registered for ['Other']"), + (["Broken"], ISA, "does not import: ModuleNotFoundError"), + (["Old"], ISA, f"{version - 1}; this SGLang requires version {version}"), + ): + with self.subTest(archs=archs, isa=isa): + spec, reason = registry.find_spec(architectures=archs, isa=isa) + self.assertIsNone(spec) + self.assertIn(why, reason) + + def test_an_architecture_has_one_spec(self): + # Repeated registration must not silently replace another plugin's spec. + registry.register(architecture=ARCH, path=f"{__name__}:SPEC") + with self.assertRaises(ValueError): + registry.register(architecture=ARCH, path="other:SPEC") + + def test_without_the_flag_nothing_is_read(self): + model = SimpleNamespace() # any attribute read would raise + with patch.object(runtime, "megakernel_enabled", return_value=False): + runtime.install_megakernel( + model=model, hf_config=None, capture_tail_hooks=[] + ) + self.assertEqual(vars(model), {}) + + def test_a_model_without_a_spec_on_every_rank_keeps_its_forward(self): + lm = install(archs=["DraftWithoutSpec"]) + self.assertNotIn("forward", lm.__dict__) + self.assertIsNone(runtime.built_megakernel(lm)) + + def test_every_rank_refuses_when_any_rank_cannot_serve(self): + # rank 1 lacks the spec: rank 0 must raise too, or it waits in the build + def no_spec_on_rank_1(value, group=None): + return [value, (value[0], "rank-1 reason", None)] + + with self.assertRaisesRegex(ValueError, "rank 1: rank-1 reason"): + install(gathered=no_spec_on_rank_1) + + def model_reason_on_rank_1(value, group=None): + return [value, ["no mHC pre-mix"]] if isinstance(value, list) else [value] + + with self.assertRaisesRegex(ValueError, "rank 1: no mHC pre-mix"): + install(gathered=model_reason_on_rank_1) + + def test_ranks_that_built_different_widths_are_refused(self): + def fewer_on_rank_1(value, group=None): + is_widths = isinstance(value, tuple) and all( + isinstance(w, int) for w in value + ) + return [value, value[:1]] if is_widths else [value] + + with self.assertRaisesRegex(RuntimeError, "widths fake built differ"): + install(gathered=fewer_on_rank_1) + + def test_widths_are_whole_requests_up_to_max_rows(self): + self.assertEqual(install(tokens=2).context.widths, (2, 4, 6, 8)) + self.assertEqual(install(tokens=3).context.widths, (3, 6)) + + def test_covered_steps_run_the_megakernel_and_others_the_forward(self): + lm = install(tokens=1) + built = runtime.built_megakernel(lm) + with patch.object( + torch.cuda, "is_current_stream_capturing", return_value=False + ): + self.assertEqual(lm(batch(4).input_ids, None, batch(4)), "megakernel") + self.assertEqual(lm(batch(6).input_ids, None, batch(6)), "standard") + self.assertEqual(built.steps, [4]) + # graph runners and supports_pp read the original forward's parameters + self.assertEqual(inspect.signature(lm.forward), inspect.signature(Lm().forward)) + self.assertIn("--enable-nvidia-megakernel", lm._derived_weight_cache_error) + + def test_a_draft_spec_installs_no_forward_and_offers_its_sampler(self): + registry.register(architecture="Draft", path=f"{__name__}:DRAFT_SPEC") + hooks = [] + lm = install(archs=["Draft"], hooks=hooks) + self.assertNotIn("forward", lm.__dict__) + self.assertEqual(hooks, []) + self.assertIs(runtime.megakernel_draft_sampler(lm), print) + self.assertIsNone(runtime.megakernel_draft_sampler(Lm())) + nothing = replace( + DRAFT_SPEC, + build=lambda model, hf, ctx: SimpleNamespace( + widths=(), summary=lambda: "", draft_sampler=None + ), + ) + with self.assertLogs(runtime.logger, "INFO") as logs: + self._install_spec(nothing) + self.assertIn("no step and no draft sampler", "".join(logs.output)) + + def test_capture_check_refuses_when_no_graph_runs_the_megakernel(self): + def capture(sizes, capture_bs): + hooks = [] + lm = install(tokens=1, hooks=hooks) + runner = SimpleNamespace(capture_bs=capture_bs) + with patch.object( + torch.cuda, "is_current_stream_capturing", return_value=True + ): + for bs in sizes: + lm(batch(bs).input_ids, None, batch(bs)) + hooks[0](runner, None, batch(bs), bs) + + capture([2, 8], capture_bs=[2, 8, 16]) # not all captured yet: no verdict + capture([2, 8], capture_bs=[2, 8]) + with self.assertRaisesRegex(ValueError, "none of the 2 captured decode graphs"): + capture([8, 16], capture_bs=[8, 16]) + hooks = [] + install(graphs=False, hooks=hooks) + self.assertEqual(hooks, []) # eager steps are routed by plan alone + + +if __name__ == "__main__": + unittest.main() From dd07279d72146f3e0f5a5c766c0275a09f10d7ab Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Sat, 17 Oct 2026 20:47:52 -0700 Subject: [PATCH 04/10] [Feat] [NVIDIA] megakernels: Check the setup at argument resolution guards.handle_nvidia_megakernel, the resolution step in the argument-resolution pipeline, now owns every startup check of --enable-nvidia-megakernel, before any weight loads: - the sglang.srt.plugins entry point is installed (with an install hint), and SGLANG_PLUGINS, when set, lists it; - the model has a spec for this GPU, and every reason it cannot serve the resolved setup is listed together: SGLang features in use that the spec does not accept (one probe each, on the resolved settings), tensor parallel size, dtype, decode CUDA graph backend, server arguments, HF config fields, environment variables, the watchdog's wait deadline, and the spec's own configuration checks. deadline_s and device_isa serve the runtime. The plugin-loader check from the flag commit moves here, so sglang.srt.plugins is unchanged. --- python/sglang/srt/arg_groups/pipeline.py | 7 +- python/sglang/srt/megakernels/guards.py | 182 ++++++++++++++- python/sglang/srt/plugins/__init__.py | 24 -- .../megakernels/test_megakernel_contract.py | 102 +++++++++ .../megakernels/test_megakernel_guards.py | 214 ++++++++++++++++++ .../unit/plugins/test_load_plugins.py | 39 ---- 6 files changed, 493 insertions(+), 75 deletions(-) create mode 100644 test/registered/unit/megakernels/test_megakernel_contract.py create mode 100644 test/registered/unit/megakernels/test_megakernel_guards.py diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py index c8e24cd85a4d..7cdab757d777 100644 --- a/python/sglang/srt/arg_groups/pipeline.py +++ b/python/sglang/srt/arg_groups/pipeline.py @@ -343,13 +343,12 @@ def run_resolution_pipeline(server_args: Any) -> None: run_hook(handle_model_capability_adjustments, server_args) # needs the resolved CUDA-graph, attention-backend and speculative settings + from sglang.srt.megakernels.guards import handle_nvidia_megakernel from sglang.srt.monokernels.guards import handle_monokernel - run_hook(handle_monokernel, server_args) - - from sglang.srt.plugins import handle_nvidia_megakernel - + # first, so a server with both kernel flags gets every reason in one error run_hook(handle_nvidia_megakernel, server_args) + run_hook(handle_monokernel, server_args) finalize_cuda_graph_prefill_max_context(server_args) diff --git a/python/sglang/srt/megakernels/guards.py b/python/sglang/srt/megakernels/guards.py index 67418b790e6c..47a3b71332a1 100644 --- a/python/sglang/srt/megakernels/guards.py +++ b/python/sglang/srt/megakernels/guards.py @@ -3,28 +3,194 @@ from __future__ import annotations +import logging +from importlib.metadata import entry_points from typing import Any, Optional -from sglang.srt.megakernels.spec import MegakernelSpec +import torch + +from sglang.srt.arg_groups.model_override_base import model_config_of, resolving_view +from sglang.srt.environ import envs +from sglang.srt.megakernels.spec import ( + ENTRY_POINT, + FLAG, + Feature, + MegakernelSpec, + find_spec, + registered_architectures, +) +from sglang.srt.plugins import GENERAL_PLUGINS_GROUP + +logger = logging.getLogger(__name__) + +PLUGIN_URL = "https://github.com/sgl-project/sgl-nvidia-mega" + +# Leave time for peers to abort before the process watchdog fires. +MAX_DEADLINE_S, MIN_DEADLINE_S = 60.0, 5.0 def device_isa() -> Optional[str]: """This process's GPU as "sm_", or None without a visible GPU.""" - raise NotImplementedError # TODO(L98) + if not torch.cuda.is_available(): + return None + props = torch.cuda.get_device_properties(torch.cuda.current_device()) + return f"sm_{props.major}{props.minor}" + + +def deadline_s(watchdog_timeout: Optional[float]) -> float: + """The deadline of every cross-GPU wait: half the watchdog, at most MAX_DEADLINE_S.""" + if not watchdog_timeout or watchdog_timeout <= 0: + return MAX_DEADLINE_S + return min(MAX_DEADLINE_S, 0.5 * float(watchdog_timeout)) + +def _ragged_verify(cfg: Any) -> bool: + mode = envs.SGLANG_RAGGED_VERIFY_MODE.get().lower() + return cfg.speculative_algorithm is not None and mode != "static" -def deadline_s(watchdog_timeout: float) -> float: - """The deadline of every cross-GPU wait, below the watchdog timeout.""" - raise NotImplementedError # TODO(L98) + +# One probe a feature, on the resolved settings; a spec accepts features by listing +# them. PARTITIONED_GPU needs the loaded device, so the runtime checks it. +FEATURE_PROBES = { + Feature.PIPELINE_PARALLEL: lambda c: c.pp_size > 1, + Feature.MULTI_NODE_TP: lambda c: c.nnodes > 1, + Feature.DP_ATTENTION: lambda c: ( + bool(c.enable_dp_attention) or (c.attn_dp_size or 1) > 1 + ), + Feature.EXPERT_PARALLEL: lambda c: (c.ep_size or 1) > 1, + Feature.EPLB: lambda c: bool(c.enable_eplb), + Feature.CONTEXT_PARALLEL: lambda c: ( + (c.attn_cp_size or 1) > 1 or (c.dcp_size or 1) > 1 or bool(c.enable_prefill_cp) + ), + Feature.SPECULATIVE_DECODING: lambda c: c.speculative_algorithm is not None, + Feature.RAGGED_VERIFY: _ragged_verify, + Feature.TWO_BATCH_OVERLAP: lambda c: bool( + c.enable_two_batch_overlap or c.enable_single_batch_overlap + ), + Feature.LORA: lambda c: bool(c.enable_lora or c.lora_paths), + Feature.PD_DISAGGREGATION: lambda c: (c.disaggregation_mode or "null") != "null", + Feature.RETURN_ROUTED_EXPERTS: lambda c: bool(c.enable_return_routed_experts), + Feature.RETURN_INDEXER_TOPK: lambda c: bool(c.enable_return_indexer_topk), + Feature.KV_OFFLOAD: lambda c: bool( + c.enable_hierarchical_cache + or c.disaggregation_decode_enable_offload_kvcache + or c.enable_lmcache + or c.enable_flexkv + ), + Feature.HISPARSE: lambda c: bool(c.enable_hisparse), + Feature.MEMORY_RELEASE: lambda c: bool( + c.enable_memory_saver or c.enable_weights_cpu_backup + ), + Feature.CPU_OFFLOAD: lambda c: ( + (c.cpu_offload_gb or 0) > 0 or (c.offload_group_size or 0) > 0 + ), + Feature.TORCH_COMPILE: lambda c: bool(c.enable_torch_compile), + Feature.DETERMINISTIC_INFERENCE: lambda c: bool( + c.enable_deterministic_inference or c.rl_on_policy_target + ), +} + + +def features_in_use(cfg: Any) -> set[Feature]: + return {feature for feature, probe in FEATURE_PROBES.items() if probe(cfg)} + + +def plugin_reasons() -> list[str]: + """Why the plugin that registers the specs will not load ([] when it will).""" + if ENTRY_POINT not in {ep.name for ep in entry_points(group=GENERAL_PLUGINS_GROUP)}: + return [ + f"the {ENTRY_POINT!r} plugin is not installed; install sgl-nvidia-mega " + f"({PLUGIN_URL})" + ] + allowed = envs.SGLANG_PLUGINS.get() + if allowed and ENTRY_POINT not in {x.strip() for x in allowed.split(",")}: + return [f"SGLANG_PLUGINS={allowed!r} does not list the {ENTRY_POINT!r} plugin"] + return [] def megakernel_disable_reasons( *, cfg: Any, hf_config: Any, isa: Optional[str] ) -> tuple[Optional[MegakernelSpec], list[str]]: """The spec for this model and every reason it cannot serve this configuration.""" - raise NotImplementedError # TODO(L98) + if isa is None: + return None, ["no GPU is visible"] + spec, why = find_spec(architectures=hf_config.architectures, isa=isa) + if spec is None and not registered_architectures(): + # the plugin loader logs and skips a plugin that fails, so nothing registered + why = ( + f"the {ENTRY_POINT!r} plugin is installed but registered no spec; see " + "'Failed to load plugin' or 'Failed to execute general plugin' above" + ) + if spec is None: + return None, [why] + req = spec.requires + reasons = [ + f"{f.value} is in use and {spec.name} does not accept it" + for f in sorted(features_in_use(cfg) - spec.features, key=lambda f: f.name) + ] + one_of = [ + ("tp_size", cfg.tp_size, req.tp_sizes), + ( + "decode CUDA graphs", + cfg.cuda_graph_config.decode.backend, + req.decode_graph_backends, + ), + ] + [(arg, getattr(cfg, arg), ok) for arg, ok in req.server_args.items()] + exactly = [ + # another model's config may not have the field at all + (f"config.{key}", getattr(hf_config, key, None), want) + for key, want in req.hf_config.items() + ] + [(name, getattr(envs, name).get(), want) for name, want in req.env.items()] + reasons += [ + f"{what} {value!r} is not one of {sorted(map(str, allowed))}" + for what, value, allowed in one_of + if allowed and value not in allowed + ] + reasons += [ + f"{what} {value!r} is not {want!r}" + for what, value, want in exactly + if value != want + ] + watchdog = cfg.watchdog_timeout + if watchdog and deadline_s(watchdog) < MIN_DEADLINE_S: + reasons.append( + f"--watchdog-timeout {watchdog:g} s leaves the waits a deadline under " + f"{MIN_DEADLINE_S:g} s" + ) + if spec.config_disable_reasons is not None: + reasons += spec.config_disable_reasons(cfg, hf_config) + return spec, reasons + + +def _dtype_reasons(spec: Optional[MegakernelSpec], model_config: Any) -> list[str]: + # the resolved dtype, which the HF config alone does not give + dtype = str(model_config.dtype).replace("torch.", "") + if spec is None or not spec.requires.dtypes or dtype in spec.requires.dtypes: + return [] + return [f"dtype {dtype!r} is not one of {sorted(spec.requires.dtypes)}"] def handle_nvidia_megakernel(server_args: Any) -> None: - """Resolution step: fail before the weights load when the flag cannot be served.""" - raise NotImplementedError # TODO(L98) + """Resolution step: fail before the weights load when the flag cannot be served, + listing every reason.""" + cfg = resolving_view(server_args) + if not cfg.enable_nvidia_megakernel: + return + reasons = plugin_reasons() + if not reasons: + model_config = model_config_of(server_args) + spec, reasons = megakernel_disable_reasons( + cfg=cfg, hf_config=model_config.hf_config, isa=device_isa() + ) + reasons += _dtype_reasons(spec, model_config) + if cfg.monokernel_backend is not None: + # both replace the model's forward, which would fail only after weights load + reasons.append( + f"--monokernel-backend {cfg.monokernel_backend} is also set; use one of the two" + ) + if reasons: + raise ValueError( + f"{FLAG} cannot run this setup:\n - " + "\n - ".join(reasons) + ) + logger.warning("%s is experimental. Its kernels and coverage may change.", FLAG) + logger.info("%s: %s on %s", FLAG, spec.name, device_isa()) diff --git a/python/sglang/srt/plugins/__init__.py b/python/sglang/srt/plugins/__init__.py index ece5c107ae34..00ae1acd1826 100644 --- a/python/sglang/srt/plugins/__init__.py +++ b/python/sglang/srt/plugins/__init__.py @@ -28,9 +28,6 @@ PLATFORM_PLUGINS_GROUP = "sglang.srt.platforms" GENERAL_PLUGINS_GROUP = "sglang.srt.plugins" -# The general plugin that --enable-nvidia-megakernel needs (sgl-nvidia-mega) -NVIDIA_MEGAKERNEL_PLUGIN = "nvidia_megakernel" - # Guard against multiple loads in the same process _plugins_loaded = False @@ -103,27 +100,6 @@ def _get_excluded_dists() -> set[str]: return {ep.dist.name for ep in platform_eps if ep.dist and ep.name != selected} -def handle_nvidia_megakernel(server_args: Any) -> None: - """Fail at startup when --enable-nvidia-megakernel cannot load its plugin.""" - from sglang.srt.arg_groups.model_override_base import resolving_view - - if not resolving_view(server_args).enable_nvidia_megakernel: - return - name = NVIDIA_MEGAKERNEL_PLUGIN - if name not in {ep.name for ep in entry_points(group=GENERAL_PLUGINS_GROUP)}: - raise ValueError( - f"--enable-nvidia-megakernel needs the {name!r} plugin, which is not " - "installed. Install sgl-nvidia-mega " - "(https://github.com/sgl-project/sgl-nvidia-mega)." - ) - allowed = envs.SGLANG_PLUGINS.get() - if allowed and name not in {x.strip() for x in allowed.split(",")}: - raise ValueError( - f"--enable-nvidia-megakernel needs the {name!r} plugin, but " - f"SGLANG_PLUGINS={allowed!r} does not list it." - ) - - def load_plugins(): """ Load and execute all general plugins, then apply registered hooks. diff --git a/test/registered/unit/megakernels/test_megakernel_contract.py b/test/registered/unit/megakernels/test_megakernel_contract.py new file mode 100644 index 000000000000..f3b16208d96a --- /dev/null +++ b/test/registered/unit/megakernels/test_megakernel_contract.py @@ -0,0 +1,102 @@ +"""A rename in SGLang that the megakernel framework reads fails here, not on a GPU.""" + +import ast +import unittest +from functools import lru_cache +from pathlib import Path + +from sglang.srt.megakernels import spec +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, stage="base-a", runner_config="cpu") + +ROOT = Path(spec.__file__).resolve().parents[2] # python/sglang + +# Names the framework reads that its CPU tests reach only through fakes: runner, +# batch and process-group attributes, string protocols, launch-only arguments. +READS = { + "srt/arg_groups/fields/*.py": ( + "cuda_graph_config", + "enable_nvidia_megakernel", + "speculative_algorithm", + "speculative_num_draft_tokens", + "watchdog_timeout", + ), + "srt/configs/model_config.py": ("dtype", "hf_config"), + "srt/distributed/parallel_state.py": ("world_size", "cpu_group"), + "srt/model_executor/forward_batch_info.py": ( + "forward_mode", + "batch_size", + "input_ids", + ), + "srt/model_executor/model_runner.py": ( + "capture_tail_hooks", + "decode_cuda_graph_runner", + ), + "srt/model_executor/model_runner_components/weight_updater.py": ( + '"_derived_weight_cache_error"', + ), + "srt/model_executor/runner/decode_cuda_graph_runner.py": ("capture_bs",), + "srt/runtime_context.py": ('"tp_group"',), + "srt/model_executor/cuda_graph_config.py": ('"full"', '"disabled"'), + "srt/plugins/__init__.py": ('"sglang.srt.plugins"', "GENERAL_PLUGINS_GROUP"), + "srt/environ.py": ("SGLANG_PLUGINS", "SGLANG_RAGGED_VERIFY_MODE"), +} + + +@lru_cache(maxsize=None) +def _provided(path: Path) -> frozenset: + """Names a file provides: definitions, attributes, imports and string literals.""" + names = set() + for node in ast.walk(ast.parse(path.read_text())): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(node.name) + elif isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)): + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + for target in targets: + for t in ast.walk(target): + if isinstance(t, ast.Name): + names.add(t.id) + elif isinstance(t, ast.Attribute): + names.add(t.attr) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + names.update((a.asname or a.name).split(".")[0] for a in node.names) + elif isinstance(node, ast.Constant) and isinstance(node.value, str): + names.add(f'"{node.value}"') + return frozenset(names) + + +class TestResolutionOrder(CustomTestCase): + def test_the_megakernel_step_runs_before_the_other_kernel_flag_step(self): + """With both kernel flags set, the megakernel step must raise first, listing the + conflict with every other reason, before the other step refuses on its own.""" + tree = ast.parse((ROOT / "srt/arg_groups/pipeline.py").read_text()) + line = { + node.args[0].id: node.lineno + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "run_hook" + and isinstance(node.args[0], ast.Name) + } + self.assertLess(line["handle_nvidia_megakernel"], line["handle_monokernel"]) + + +class TestSglangReads(CustomTestCase): + def test_sglang_provides_what_the_framework_reads(self): + for pattern, names in READS.items(): + with self.subTest(pattern=pattern): + files = sorted(ROOT.glob(pattern)) + self.assertTrue(files, f"no file matches {pattern}") + provided = frozenset().union(*map(_provided, files)) + missing = [name for name in names if name not in provided] + self.assertFalse( + missing, + f"SGLang no longer provides {missing} in {pattern}; repair the " + "megakernel framework and READS", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/megakernels/test_megakernel_guards.py b/test/registered/unit/megakernels/test_megakernel_guards.py new file mode 100644 index 000000000000..3d64a67f7c62 --- /dev/null +++ b/test/registered/unit/megakernels/test_megakernel_guards.py @@ -0,0 +1,214 @@ +"""CPU contracts of --enable-nvidia-megakernel's startup checks and wait deadlines.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from msgspec.structs import replace + +from sglang.srt.environ import envs +from sglang.srt.megakernels import guards +from sglang.srt.megakernels import spec as registry +from sglang.srt.megakernels.guards import ( + MAX_DEADLINE_S, + deadline_s, + features_in_use, + handle_nvidia_megakernel, + megakernel_disable_reasons, +) +from sglang.srt.megakernels.spec import ( + MEGAKERNEL_API_VERSION, + Feature, + MegakernelSpec, + Numerics, + NumericsKind, + Requirements, +) +from sglang.srt.model_executor.cuda_graph_config import ( + Backend, + Phase, + default_cuda_graph_config, + with_phase, +) +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, stage="base-a", runner_config="cpu") + +ARCH = "FakeMegaForCausalLM" + +SPEC = MegakernelSpec( + name="fake", + requires=Requirements( + isas=frozenset({"sm_103"}), + tp_sizes=frozenset({4}), + dtypes=frozenset({"bfloat16"}), + server_args={"kv_cache_dtype": frozenset({"fp8_e4m3"})}, + hf_config={"model_type": "fake"}, + env={"SGLANG_DSPARK_EMBED_IN_GRAPH": True}, + ), + features=frozenset({Feature.SPECULATIVE_DECODING}), + numerics=Numerics(kind=NumericsKind.BITWISE), + max_rows=48, + build=lambda model, hf_config, ctx: None, + api_version=MEGAKERNEL_API_VERSION, + config_disable_reasons=lambda cfg, hf: ( + ["context_length too long"] if cfg.context_length == 7 else [] + ), +) + + +def server_args(**kw): + """Arguments as the resolution step sees them, with the graph config resolved.""" + defaults = { + "model_path": "dummy", + "tp_size": 4, + "kv_cache_dtype": "fp8_e4m3", + "cuda_graph_config": default_cuda_graph_config(), + } + return ServerArgs(**{**defaults, **kw}) + + +def entry_point(name): + ep = MagicMock() + ep.name = name + return ep + + +def reasons_for(cfg, *, arch=ARCH, isa="sm_103", model_type="fake"): + hf_config = SimpleNamespace(architectures=[arch], model_type=model_type) + return megakernel_disable_reasons(cfg=cfg, hf_config=hf_config, isa=isa) + + +class TestMegakernelGuards(CustomTestCase): + def setUp(self): + registry_patch = patch.dict(registry._REGISTRY, clear=True) + registry_patch.start() + self.addCleanup(registry_patch.stop) + registry.register(architecture=ARCH, path=f"{__name__}:SPEC") + + def test_every_feature_has_a_probe_and_none_is_on_by_default(self): + # a feature without a probe is never refused; a probe stuck on refuses every server + post_load = {Feature.PARTITIONED_GPU} # the runtime checks it on the device + self.assertEqual(set(guards.FEATURE_PROBES), set(Feature) - post_load) + self.assertEqual(features_in_use(ServerArgs(model_path="dummy")), set()) + + def test_features_in_use_are_refused_unless_accepted(self): + _, reasons = reasons_for(server_args(pp_size=2, speculative_algorithm="DSPARK")) + self.assertEqual(len(reasons), 1, reasons) # SPEC accepts speculative decoding + self.assertIn(Feature.PIPELINE_PARALLEL.value, reasons[0]) + + def test_every_refusal_reason_is_listed(self): + with envs.SGLANG_DSPARK_EMBED_IN_GRAPH.override(False): + spec, reasons = reasons_for( + server_args( + tp_size=8, + kv_cache_dtype="auto", + enable_eplb=True, + context_length=7, + watchdog_timeout=6, + ), + model_type="other", + ) + self.assertIs(spec, SPEC) + for want in ( + "tp_size 8", + "kv_cache_dtype 'auto'", + "config.model_type 'other'", + "SGLANG_DSPARK_EMBED_IN_GRAPH False", + Feature.EPLB.value, + "context_length", + "--watchdog-timeout", + ): + self.assertTrue(any(want in r for r in reasons), (want, reasons)) + self.assertEqual(reasons_for(server_args())[1], []) + self.assertEqual(reasons_for(server_args(), isa=None)[1], ["no GPU is visible"]) + for arch, isa, why in ( + ("Other", "sm_103", "Other"), + (ARCH, "sm_100", "sm_100"), + ): + spec, reasons = reasons_for(server_args(), arch=arch, isa=isa) + self.assertIsNone(spec) + self.assertIn(why, reasons[0]) + + def test_a_plugin_that_registered_nothing_is_named(self): + """A plugin that fails to load is only logged, so an empty registry must not + read as an unsupported model.""" + registry._REGISTRY.clear() + spec, reasons = reasons_for(server_args()) + self.assertIsNone(spec) + self.assertIn("installed but registered no spec", reasons[0]) + + def test_the_decode_graph_backend_is_checked(self): + full_only = replace( + SPEC, requires=replace(SPEC.requires, decode_graph_backends={"full"}) + ) + graphs = with_phase( + default_cuda_graph_config(), Phase.DECODE, backend=Backend.BREAKABLE + ) + with patch.object(guards, "find_spec", return_value=(full_only, "")): + _, reasons = reasons_for(server_args(cuda_graph_config=graphs)) + self.assertTrue(any("decode CUDA graphs" in r for r in reasons), reasons) + + def test_the_flag_needs_its_plugin_to_load(self): + """Without its plugin, the flag must fail at startup, not serve the standard path.""" + cases = ( + ([], "", "install sgl-nvidia-mega"), + ([entry_point("nvidia_megakernel")], "other", "SGLANG_PLUGINS='other'"), + ) + for eps, allowed, want in cases: + with ( + self.subTest(want=want), + patch.object(guards, "entry_points", return_value=eps), + patch.object(guards.envs.SGLANG_PLUGINS, "get", return_value=allowed), + ): + self.assertTrue(any(want in r for r in guards.plugin_reasons())) + eps = [entry_point("nvidia_megakernel")] + for allowed in ("", "other, nvidia_megakernel"): + with ( + patch.object(guards, "entry_points", return_value=eps), + patch.object(guards.envs.SGLANG_PLUGINS, "get", return_value=allowed), + ): + self.assertEqual(guards.plugin_reasons(), []) + + def test_the_resolution_step_raises_every_reason_or_nothing(self): + with patch.object(guards, "plugin_reasons") as plugin: + handle_nvidia_megakernel(server_args()) # flag off: nothing is looked up + plugin.assert_not_called() + on = server_args(enable_nvidia_megakernel=True, tp_size=8) + with ( + patch.object(guards, "plugin_reasons", return_value=[]), + patch.object(guards, "device_isa", return_value="sm_103"), + patch.object(guards, "megakernel_disable_reasons") as reasons, + patch.object(guards, "model_config_of") as model_config, + ): + model_config.return_value = SimpleNamespace(hf_config=None, dtype="float16") + reasons.return_value = (SPEC, ["tp_size 8 is not one of ['4']"]) + # the resolved dtype is checked here, and listed with the other reasons + with self.assertRaisesRegex( + ValueError, "(?s)tp_size 8.*\n - dtype 'float16'" + ): + handle_nvidia_megakernel(on) + model_config.return_value = SimpleNamespace( + hf_config=None, dtype="bfloat16" + ) + reasons.return_value = (SPEC, []) + handle_nvidia_megakernel(on) + # both flags replace the model's forward: refused before any weight loads, + # listed with the spec's own reasons + both = server_args(enable_nvidia_megakernel=True, monokernel_backend="x") + reasons.return_value = (SPEC, ["tp_size 8 is not one of ['4']"]) + with self.assertRaisesRegex( + ValueError, "(?s)tp_size 8.*\n - --monokernel-backend x is also set" + ): + handle_nvidia_megakernel(both) + + def test_deadlines_stay_below_the_watchdog(self): + self.assertEqual(deadline_s(None), MAX_DEADLINE_S) + self.assertEqual(deadline_s(3600), MAX_DEADLINE_S) + self.assertEqual(deadline_s(30), 15.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/plugins/test_load_plugins.py b/test/registered/unit/plugins/test_load_plugins.py index 837a8dd33935..3b84f88934b0 100644 --- a/test/registered/unit/plugins/test_load_plugins.py +++ b/test/registered/unit/plugins/test_load_plugins.py @@ -181,45 +181,6 @@ def bad_plugin(): self.assertIsNone(_current_plugin_source.get()) -class TestNvidiaMegakernelFlag(CustomTestCase): - """--enable-nvidia-megakernel fails at argument resolution unless its plugin loads.""" - - def _check(self, enabled, eps, allowed=""): - from types import SimpleNamespace - - from sglang.srt.plugins import handle_nvidia_megakernel - - view = SimpleNamespace(enable_nvidia_megakernel=enabled) - with ( - patch( - "sglang.srt.arg_groups.model_override_base.resolving_view", - return_value=view, - ), - patch("sglang.srt.plugins.entry_points", return_value=eps) as found, - patch("sglang.srt.plugins.envs") as mock_envs, - ): - mock_envs.SGLANG_PLUGINS.get.return_value = allowed - handle_nvidia_megakernel(object()) - return found - - def test_flag_off_checks_nothing(self): - found = self._check(False, []) - found.assert_not_called() - - def test_flag_on_without_the_plugin_raises_with_an_install_hint(self): - with self.assertRaisesRegex(ValueError, "Install sgl-nvidia-mega"): - self._check(True, [_make_ep("other")]) - - def test_flag_on_with_sglang_plugins_excluding_it_raises(self): - with self.assertRaisesRegex(ValueError, "SGLANG_PLUGINS='other'"): - self._check(True, [_make_ep("nvidia_megakernel")], allowed="other") - - def test_flag_on_with_the_plugin_passes(self): - eps = [_make_ep("nvidia_megakernel")] - self._check(True, eps) - self._check(True, eps, allowed="other, nvidia_megakernel") - - if __name__ == "__main__": import unittest From 58fb997cb59df6d2f570bd7f583b26ff7a0a6697 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Sat, 17 Oct 2026 20:40:25 -0700 Subject: [PATCH 05/10] [Feat] [NVIDIA] dspark: Let the NVIDIA megakernel run the folded draft proposal maybe_build_draft_sampler asks the megakernel framework for the draft model's draft-sampler factory (megakernel_draft_sampler) and passes it to DsparkDraftSampler, which calls it once with its own buffers (draft tokens, step logits, step temperatures, the draft noise). For a batch size the factory serves, the returned step replaces compute_base_logits and draft_markov_block; every other size, and every server without the megakernel, runs today's proposal. A sampled draft's step logits are zeroed up front when the factory is set. --- .../dspark_components/dspark_draft_sampler.py | 68 ++++++-- .../unit/spec/test_dspark_draft_block.py | 153 ++++++++++++++++++ 2 files changed, 209 insertions(+), 12 deletions(-) diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py index 3797bbcae4da..b7e9de071d20 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py @@ -1,10 +1,12 @@ from __future__ import annotations import logging -from typing import Optional +from typing import Callable, Optional import torch +from sglang.srt.layers.dp_attention import get_dp_tp_group +from sglang.srt.megakernels.runtime import megakernel_agree, megakernel_draft_sampler from sglang.srt.speculative.dspark_components.dspark_draft import ( DraftNoise, DraftParams, @@ -32,6 +34,7 @@ def __init__( tp_sync: SpecTpSync, greedy_draft: bool, out=None, + megakernel_sampler: Optional[Callable] = None, ): self.model = model self.markov_head = model.markov_head @@ -73,6 +76,35 @@ def __init__( # [max_bs, gamma, vocab] step logits for the sampling accept, sized on # the first call; a greedy draft keeps none. self.step_logits: Optional[torch.Tensor] = None + self._megakernel_steps = None + if megakernel_sampler is not None: + self._megakernel_steps = self._build_megakernel_steps(megakernel_sampler) + + def _build_megakernel_steps(self, megakernel_sampler: Callable): + if not self.greedy_draft: + # The megakernel leaves temperature-0 rows' logits unwritten; a mixed accept + # still reads them. + self.step_logits = torch.zeros( + (self.max_bs, self.gamma, _step_logits_width(self.model)), + dtype=self.markov_head.step_logits_dtype(torch.bfloat16), + device=self.out.device, + ) + steps = megakernel_sampler( + gamma=self.gamma, + max_bs=self.max_bs, + greedy_draft=self.greedy_draft, + noise=self._noise, + draft_tokens=self.out, + step_logits=self.step_logits, + step_temperatures=self.params.step_temperatures, + ) + # A rank on SGLang's block would wait in its per-step token sync (this group), + # which the megakernel's step skips. + served = tuple(bs for bs in range(1, self.max_bs + 1) if steps(bs) is not None) + megakernel_agree( + served, what="draft sampler batch sizes", group=get_dp_tp_group() + ) + return steps def _step_logits(self, base_logits: torch.Tensor) -> torch.Tensor: if self.step_logits is None: @@ -100,10 +132,26 @@ def __call__(self, hidden_states, input_ids): bs=bs, gamma=self.gamma, ) - base_logits, confidence_features = self.model.compute_base_logits(model_hidden) - base_logits = base_logits.view(bs, self.gamma, -1) anchor = input_ids.view(bs, self.query_token_num)[:, 0] + step = None if self._megakernel_steps is None else self._megakernel_steps(bs) + if step is not None: + draft_tokens, confidence_features = step(model_hidden, anchor) + else: + draft_tokens, confidence_features = self._markov_block( + model_hidden, sample_hidden, anchor, bs + ) + if self.confidence_out is not None: + confidence, confidence_raw = self.model.compute_confidence( + anchor_tokens=anchor, + draft_tokens=draft_tokens, + confidence_features=confidence_features, + ) + self.confidence_out[:bs].copy_(confidence) + self.confidence_raw_out[:bs].copy_(confidence_raw) + def _markov_block(self, model_hidden, sample_hidden, anchor, bs: int): + base_logits, confidence_features = self.model.compute_base_logits(model_hidden) + base_logits = base_logits.view(bs, self.gamma, -1) draft_tokens, _ = draft_markov_block( markov_head=self.markov_head, base_logits=base_logits, @@ -118,14 +166,7 @@ def __call__(self, hidden_states, input_ids): greedy_draft=self.greedy_draft, ) self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1)) - if self.confidence_out is not None: - confidence, confidence_raw = self.model.compute_confidence( - anchor_tokens=anchor, - draft_tokens=draft_tokens, - confidence_features=confidence_features, - ) - self.confidence_out[:bs].copy_(confidence) - self.confidence_raw_out[:bs].copy_(confidence_raw) + return draft_tokens, confidence_features def _step_logits_width(model) -> int: @@ -157,10 +198,12 @@ def _eager(reason): return _eager("gamma<=0") if draft_model.markov_head is None: return _eager("no markov head") + megakernel_sampler = megakernel_draft_sampler(draft_model) if tp_rank == 0: logger.info( - "DSpark draft proposal (%s) folded into the draft cuda graph.", + "DSpark draft proposal (%s) folded into the draft cuda graph%s.", "greedy draft" if greedy_draft else "sampling", + "; the NVIDIA megakernel may run it" if megakernel_sampler else "", ) return DsparkDraftSampler( model=draft_model, @@ -171,4 +214,5 @@ def _eager(reason): tp_sync=tp_sync, greedy_draft=greedy_draft, out=out, + megakernel_sampler=megakernel_sampler, ) diff --git a/test/registered/unit/spec/test_dspark_draft_block.py b/test/registered/unit/spec/test_dspark_draft_block.py index d1b8abac20f0..bb3810159d53 100644 --- a/test/registered/unit/spec/test_dspark_draft_block.py +++ b/test/registered/unit/spec/test_dspark_draft_block.py @@ -1,5 +1,6 @@ import unittest from types import SimpleNamespace +from unittest.mock import Mock, patch import torch @@ -9,6 +10,7 @@ RNNHead, VanillaMarkov, ) +from sglang.srt.speculative.dspark_components import dspark_draft_sampler as ds from sglang.srt.speculative.dspark_components.dspark_draft import ( DraftNoise, sample_draft_block, @@ -124,5 +126,156 @@ def test_only_the_plain_w2_gemv_head_fuses(self): self.assertIs(head.greedy_w2_weight() is not None, fuses) +def _folded_draft_model(): + """A draft model for the folded proposal.""" + model = SimpleNamespace( + markov_head=VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK), + sample_from_anchor=True, + confidence_head=SimpleNamespace(proj=SimpleNamespace(weight=torch.zeros(1))), + lm_head=SimpleNamespace(org_vocab_size=VOCAB), + compute_base_logits=Mock( + side_effect=lambda h: (torch.zeros(h.shape[0], VOCAB), h + 1) + ), + compute_confidence=Mock( + side_effect=lambda **kw: (torch.zeros(BS, GAMMA), torch.zeros(BS, GAMMA)) + ), + ) + return model + + +def _build(model, *, greedy_draft, factory=None, agree=None): + """maybe_build_draft_sampler, with factory as the megakernel's draft sampler.""" + with ( + patch.object(ds, "megakernel_draft_sampler", return_value=factory), + patch.object(ds, "megakernel_agree", agree or Mock()), + patch.object(ds, "get_dp_tp_group", return_value="draft tp group"), + ): + return ds.maybe_build_draft_sampler( + draft_model=model, + gamma=GAMMA, + max_bs=2 * BS, + device="cpu", + tp_rank=1, + noise=DraftNoise(key=3, offset=torch.zeros(1, dtype=torch.int64)), + greedy_draft=greedy_draft, + tp_sync="sync", + ) + + +class TestMegakernelDraftSampler(CustomTestCase): + def test_a_plugin_step_replaces_only_the_base_logits_and_markov_block(self): + hidden = torch.randn(BS * GAMMA, HIDDEN) + input_ids = torch.arange(BS * GAMMA) + for greedy_draft in (False, True): + factory = Mock() + step = Mock(side_effect=lambda h, anchor: ("tokens", h * 2)) + factory.return_value = lambda bs: step if bs == BS else None + model = _folded_draft_model() + with ( + self.subTest(greedy_draft=greedy_draft), + patch.object(ds, "draft_markov_block", side_effect=AssertionError), + ): + sampler = _build(model, greedy_draft=greedy_draft, factory=factory) + sampler(hidden, input_ids) + model.compute_base_logits.assert_not_called() + # the plugin writes into SGLang's buffers; a temperature-0 row it skips + # must read as finite logits in a mixed accept + kwargs = factory.call_args.kwargs + self.assertIs(kwargs["draft_tokens"], sampler.out) + self.assertIs(kwargs["noise"], sampler._noise) + self.assertIs(kwargs["step_temperatures"], sampler.params.step_temperatures) + self.assertEqual(kwargs["greedy_draft"], greedy_draft) + if greedy_draft: + self.assertIsNone(kwargs["step_logits"]) + else: + self.assertIs(kwargs["step_logits"], sampler.step_logits) + self.assertEqual(sampler.step_logits.shape, (2 * BS, GAMMA, VOCAB)) + self.assertFalse(sampler.step_logits.any()) + self.assertTrue(torch.equal(step.call_args.args[1], input_ids[::GAMMA])) + confidence = model.compute_confidence.call_args.kwargs + self.assertEqual(confidence["draft_tokens"], "tokens") + self.assertTrue(torch.equal(confidence["confidence_features"], hidden * 2)) + + def test_without_a_plugin_step_the_stock_proposal_runs(self): + """Unset (no megakernel draft sampler), and a batch size the megakernel declines: + the stock base logits and Markov block run with today's arguments.""" + declined = lambda **kw: lambda bs: None # noqa: E731 + for factory in (None, declined): + model = _folded_draft_model() + tokens = torch.full((BS, GAMMA), 7) + with ( + patch.object( + ds, "draft_markov_block", return_value=(tokens, None) + ) as stock, + patch.object(torch.cuda, "is_current_stream_capturing", lambda: False), + ): + sampler = _build(model, greedy_draft=False, factory=factory) + if factory is None: + self.assertIsNone(sampler.step_logits) # sized on the first call + sampler(torch.ones(BS * GAMMA, HIDDEN), torch.arange(BS * GAMMA)) + kwargs = stock.call_args.kwargs + self.assertEqual( + kwargs.keys(), + { + "markov_head", + "base_logits", + "anchor_tokens", + "draft_hidden", + "step_temperatures", + "noise", + "tp_sync", + "logits_out", + "greedy_draft", + }, + ) + self.assertEqual(kwargs["base_logits"].shape, (BS, GAMMA, VOCAB)) + self.assertTrue( + torch.equal(kwargs["anchor_tokens"], torch.arange(0, 20, 5)) + ) + self.assertEqual( + kwargs["logits_out"].data_ptr(), sampler.step_logits.data_ptr() + ) + self.assertTrue(sampler.out[: BS * GAMMA].eq(7).all()) + features = model.compute_confidence.call_args.kwargs["confidence_features"] + self.assertTrue(features.eq(2).all()) + + def test_every_rank_serves_the_same_draft_batch_sizes(self): + """A rank whose megakernel declines a size runs SGLang's block, whose per-step + token sync its peers on the megakernel step never enter: the sizes must agree, + over the group that sync uses.""" + agree = Mock() + factory = Mock(return_value=lambda bs: "step" if bs in (1, BS) else None) + _build(_folded_draft_model(), greedy_draft=True, factory=factory, agree=agree) + agree.assert_called_once_with( + (1, BS), what="draft sampler batch sizes", group="draft tp group" + ) + + def test_ranks_serving_different_draft_batch_sizes_are_refused(self): + from sglang.srt.megakernels import runtime + + factory = Mock(return_value=lambda bs: "step" if bs == 1 else None) + group = SimpleNamespace(world_size=2, cpu_group=None) + + def gather(values, value, group): + values[:] = [value, (1, 2)] # the other rank also serves batch 2 + + with ( + patch.object(runtime.dist, "all_gather_object", gather), + patch.object(ds, "megakernel_draft_sampler", return_value=factory), + patch.object(ds, "get_dp_tp_group", return_value=group), + self.assertRaisesRegex(RuntimeError, r"rank 1: \(1, 2\)"), + ): + ds.maybe_build_draft_sampler( + draft_model=_folded_draft_model(), + gamma=GAMMA, + max_bs=2 * BS, + device="cpu", + tp_rank=0, + noise=DraftNoise(key=3, offset=torch.zeros(1, dtype=torch.int64)), + greedy_draft=True, + tp_sync="sync", + ) + + if __name__ == "__main__": unittest.main() From 312f1c670b87087e9290a48d2fa65220c2501d41 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Sun, 18 Oct 2026 18:08:22 -0700 Subject: [PATCH 06/10] [Feat] [NVIDIA] megakernels: hand a megakernel SGLang's runtime state as data, in one binding megakernels/binding.py is the one place a megakernel reads SGLang's runtime state. A spec's build gets a MegakernelBinding as BuildContext.binding; a covered forward gets the running forward's ForwardView as Step.view, and a draft entry asks binding.forward_view(). The view carries the attention backend's forward metadata, the KV pool under the pool's own names, and the backend's attention, KV-write, SWA-location and compressor/indexer calls; the binding carries the Engram history and host tables. Each source is sealed at its first read and checked once the decode graphs are captured. The callables, writes and reads a megakernel is still allowed beyond that data are listed in the same file. --- python/sglang/srt/megakernels/binding.py | 171 ++++++++++++++++++ python/sglang/srt/megakernels/runtime.py | 15 +- python/sglang/srt/megakernels/spec.py | 8 + .../megakernels/test_megakernel_binding.py | 120 ++++++++++++ .../megakernels/test_megakernel_runtime.py | 13 +- 5 files changed, 322 insertions(+), 5 deletions(-) create mode 100644 python/sglang/srt/megakernels/binding.py create mode 100644 test/registered/unit/megakernels/test_megakernel_binding.py diff --git a/python/sglang/srt/megakernels/binding.py b/python/sglang/srt/megakernels/binding.py new file mode 100644 index 000000000000..e329a2ddfd55 --- /dev/null +++ b/python/sglang/srt/megakernels/binding.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The data SGLang hands a megakernel's build, forward and draft entries. + +The one place a megakernel reads SGLang's runtime state: the KV pool, the attention +metadata of the running forward and the Engram tables. A spec's build gets a +``MegakernelBinding`` as ``BuildContext.binding``; a covered forward gets the +running forward's ``ForwardView`` as ``Step.view`` (a draft entry asks +``binding.forward_view()``). Everything else a megakernel touches is the model it +replaces, SGLang's kernel ops, and the reads listed below. +""" + +from __future__ import annotations + +from typing import Any, Callable, Optional + +import torch + +# Called by a megakernel, tolerated until offered as data (its contract test pins +# both lists). +TOLERATED_CALLABLES = ( + "ForwardView.attention", # the attention backend's own forward + "ForwardView.store_cache", # the KV write + "ForwardView.swa_out_cache_loc", + "ForwardView.low_ratio_sources", # compressor / indexer inputs + "ForwardView.core_compressor", + "KvPoolView.set_swa_key_buffer_radix_fused_norm_rope", # the fused SWA key write + "KvPoolView.pool", # passed back to SGLang's own draft KV store only + "sglang.kernels.ops", # SGLang's kernel ops + "the replaced model's modules, weights and quant methods", +) +# Written into SGLang's objects, tolerated until SGLang owns these buffers. +TOLERATED_WRITES = ( + "ForwardView.metadata.q_pad_buffer", + "ForwardView.metadata.low_ratio_req_indices", + "ForwardView.metadata.low_ratio_pos_i64", +) +# Read, tolerated: configuration, not runtime state. +TOLERATED_READS = ( + "runtime_context getters (get_parallel, get_exec, get_spec, get_platform)", + "capture-mode queries (get_is_capture_mode, is_in_breakable_cuda_graph)", + "MegakernelBinding.engram_host_table", # the per-layer host table object +) + + +class KvPoolView: + """The running forward's KV pool, under the pool's own names: its layout and + per-layer key buffers (a pool offers the ones of its kind).""" + + def __init__(self, pool: Any): + self.pool = pool + + @property + def uniform_fp8(self) -> bool: + return self.pool.uniform_fp8 + + @property + def request_window(self) -> Any: + return self.pool.request_window + + def get_swa_key_buffer_radix(self, layer_id: int) -> torch.Tensor: + return self.pool.get_swa_key_buffer_radix(layer_id) + + def get_swa_key_page_size(self) -> int: + return self.pool.get_swa_key_page_size() + + def get_swa_key_bytes_per_token(self) -> int: + return self.pool.get_swa_key_bytes_per_token() + + def get_extra_key_buffer(self, layer_id: int) -> Optional[torch.Tensor]: + return self.pool.get_extra_key_buffer(layer_id) + + def get_extra_key_page_size(self, layer_id: int) -> int: + return self.pool.get_extra_key_page_size(layer_id) + + def get_extra_key_bytes_per_token(self, layer_id: int) -> int: + return self.pool.get_extra_key_bytes_per_token(layer_id) + + @property + def set_swa_key_buffer_radix_fused_norm_rope(self) -> Callable[..., None]: + return self.pool.set_swa_key_buffer_radix_fused_norm_rope + + +class ForwardView: + """The running forward's attention data, read from its attention backend; built + at capture for a graph step, so its tensors are the ones the graph replays.""" + + def __init__(self, backend: Any): + self._backend = backend + self.metadata = backend.forward_metadata + self.kv_pool = KvPoolView(backend.token_to_kv_pool) + + @property + def trtllm_attn(self) -> bool: + return self._backend.trtllm_attn + + @property + def attention(self) -> Callable[..., Any]: + return self._backend.forward + + @property + def store_cache(self) -> Callable[..., None]: + return self._backend.store_cache + + @property + def swa_out_cache_loc(self) -> Callable[[Any], torch.Tensor]: + return self._backend.get_swa_out_cache_loc + + @property + def low_ratio_sources(self) -> Callable[..., Any]: + return self._backend.forward_low_ratio_sources + + @property + def core_compressor(self) -> Callable[..., Any]: + return self._backend.forward_core_compressor + + +class MegakernelBinding: + """Built at install, before the KV pools exist; read at build or forward. Each + source is sealed at the first read that finds it; ``assert_live`` (after the + decode graphs are captured) raises once one was reallocated.""" + + def __init__(self, *, model: torch.nn.Module): + self._model = model + self._sealed: dict[str, tuple] = {} + + def forward_view(self) -> ForwardView: + """The view of the forward running now (a target step or a draft step).""" + from sglang.srt.model_executor.forward_context import get_attn_backend + + view = ForwardView(get_attn_backend()) + self._seal("kv_pool", view.kv_pool.pool) + return view + + def engram_history(self) -> Optional[torch.Tensor]: + """The model's Engram hash history ``[slots + 1, width]`` int32, or None.""" + history = self._history() + if history is None or history.shape[1] == 0: + return None + self._seal("engram_history", history) + return history + + def engram_host_table(self, engram: torch.nn.Module) -> Any: + """The host table of one of the model's Engram layers, or None.""" + return engram.embed.host_table + + def assert_live(self) -> None: + current = {"kv_pool": self._current_pool, "engram_history": self._history} + for name, sealed in self._sealed.items(): + if _identity(current[name]()) != sealed: + raise RuntimeError( + f"megakernel binding: {name} was reallocated after its first read" + ) + + def _seal(self, name: str, value: Any) -> None: + self._sealed.setdefault(name, _identity(value)) + + def _current_pool(self) -> Any: + from sglang.srt.model_executor.forward_context import get_attn_backend + + return get_attn_backend().token_to_kv_pool + + def _history(self) -> Optional[torch.Tensor]: + # only models with Engram layers have a hasher + hasher = getattr(self._model, "engram_hasher", None) + return None if hasher is None else hasher.history + + +def _identity(value: Any) -> tuple: + if isinstance(value, torch.Tensor): + return (value.data_ptr(), tuple(value.shape), value.dtype) + return (id(value),) diff --git a/python/sglang/srt/megakernels/runtime.py b/python/sglang/srt/megakernels/runtime.py index 7df2b586a782..bde3c5cea3d0 100644 --- a/python/sglang/srt/megakernels/runtime.py +++ b/python/sglang/srt/megakernels/runtime.py @@ -8,10 +8,12 @@ import time from typing import TYPE_CHECKING, Any, Callable, Optional +import msgspec import torch import torch.distributed as dist from sglang.srt.environ import envs +from sglang.srt.megakernels.binding import MegakernelBinding from sglang.srt.megakernels.guards import deadline_s, device_isa from sglang.srt.megakernels.spec import ( FLAG, @@ -70,10 +72,12 @@ def install_megakernel( _refuse(_gather_reasons(reasons)) tokens = _tokens_per_request() + binding = MegakernelBinding(model=model) context = BuildContext( widths=tuple(range(tokens, spec.max_rows + 1, tokens)), tokens_per_request=tokens, deadline_s=deadline_s(get_device().watchdog_timeout), + binding=binding, ) start, free_before = time.perf_counter(), torch.cuda.mem_get_info()[0] built = spec.build(model, hf_config, context) @@ -110,7 +114,7 @@ def install_megakernel( used_mib, context.deadline_s, ) - dispatch = _Dispatch(built=built, original=model.forward) + dispatch = _Dispatch(built=built, original=model.forward, binding=binding) model.forward = dispatch.wrapped() if _decode_graphs_enabled(): capture_tail_hooks.append(_CaptureCheck(spec=spec, dispatch=dispatch)) @@ -219,8 +223,10 @@ def _model_disable_reasons( class _Dispatch: """The installed forward: covered steps run the built megakernel.""" - def __init__(self, *, built: BuiltMegakernel, original: Callable): - self.built, self.original = built, original + def __init__( + self, *, built: BuiltMegakernel, original: Callable, binding: MegakernelBinding + ): + self.built, self.original, self.binding = built, original, binding # batch size -> the step planned while capturing its decode graphs; plan reads # only the mode and rows, so every graph variant of a batch size plans alike self.captured: dict[int, Optional[Step]] = {} @@ -237,7 +243,7 @@ def forward(input_ids, positions, forward_batch, *args, **kwargs): input_ids, positions, forward_batch, *args, **kwargs ) return self.built.forward( - step, + msgspec.structs.replace(step, view=self.binding.forward_view()), self.original, input_ids, positions, @@ -268,6 +274,7 @@ def __call__( if self.is_done or not set(runner.capture_bs) <= set(captured): return self.is_done = True + self.dispatch.binding.assert_live() routed = sorted(bs for bs, step in captured.items() if step is not None) if not routed: raise ValueError( diff --git a/python/sglang/srt/megakernels/spec.py b/python/sglang/srt/megakernels/spec.py index bd242a72fa27..63749110eba1 100644 --- a/python/sglang/srt/megakernels/spec.py +++ b/python/sglang/srt/megakernels/spec.py @@ -83,6 +83,8 @@ class BuildContext(msgspec.Struct, frozen=True): widths: tuple[int, ...] # step rows to build, ascending tokens_per_request: int deadline_s: float # every cross-GPU wait's deadline, below --watchdog-timeout + # SGLang's runtime state for this model (megakernels/binding.py) + binding: Any = None class Step(msgspec.Struct, frozen=True): @@ -90,6 +92,8 @@ class Step(msgspec.Struct, frozen=True): mode: str # "decode" or "target_verify" rows: int + # the running forward's ForwardView, set by SGLang before forward runs the step + view: Any = None class BuiltMegakernel(Protocol): @@ -104,6 +108,10 @@ class BuiltMegakernel(Protocol): on every rank, since it replaces the proposal's rank-0 broadcast; write draft_tokens[: bs * gamma] and the sampled rows' step_logits; and advance noise.offset once per sampled block. + + SGLang's runtime state reaches a build as BuildContext.binding and a covered + step as Step.view (megakernels/binding.py, which also lists what else may be + called, written or read). """ widths: tuple[int, ...] # the rows it built, possibly fewer than requested diff --git a/test/registered/unit/megakernels/test_megakernel_binding.py b/test/registered/unit/megakernels/test_megakernel_binding.py new file mode 100644 index 000000000000..e3dc326c9066 --- /dev/null +++ b/test/registered/unit/megakernels/test_megakernel_binding.py @@ -0,0 +1,120 @@ +"""CPU contract of the megakernel binding: the running forward's attention data and +KV pool under SGLang's own names, read from the forward context; sources sealed at +their first read and checked after capture; the tolerated lists name real members.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.megakernels import binding as mb +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") + + +class Pool: + uniform_fp8 = False + request_window = None + + def __init__(self): + self.keys = [torch.zeros(4, 8) for _ in range(2)] + + def get_swa_key_buffer_radix(self, layer_id): + return self.keys[layer_id] + + def get_swa_key_page_size(self): + return 128 + + def get_swa_key_bytes_per_token(self): + return 576 + + def get_extra_key_buffer(self, layer_id): + return None + + def get_extra_key_page_size(self, layer_id): + return 64 + layer_id + + def get_extra_key_bytes_per_token(self, layer_id): + return 32 + layer_id + + def set_swa_key_buffer_radix_fused_norm_rope(self, **kw): + return kw + + +def backend(pool): + return SimpleNamespace( + token_to_kv_pool=pool, + forward_metadata=SimpleNamespace(core_attn_metadata="core"), + trtllm_attn=False, + forward=lambda *a, **k: "attention", + store_cache=lambda **k: None, + get_swa_out_cache_loc=lambda fb: "swa_loc", + forward_low_ratio_sources=lambda **k: "sources", + forward_core_compressor=lambda **k: "compressed", + ) + + +class TestMegakernelBinding(CustomTestCase): + def _view(self, binding, attn_backend): + with patch( + "sglang.srt.model_executor.forward_context.get_attn_backend", + return_value=attn_backend, + ): + return binding.forward_view() + + def test_the_view_is_the_running_forward_under_sglangs_names(self): + pool = Pool() + view = self._view(mb.MegakernelBinding(model=torch.nn.Module()), backend(pool)) + self.assertEqual(view.metadata.core_attn_metadata, "core") + self.assertIs(view.kv_pool.pool, pool) + self.assertIs(view.kv_pool.get_swa_key_buffer_radix(1), pool.keys[1]) + self.assertEqual(view.kv_pool.get_extra_key_page_size(1), 65) + self.assertEqual(view.swa_out_cache_loc(None), "swa_loc") + self.assertEqual(view.attention(), "attention") + + def test_a_replaced_pool_or_history_fails_assert_live_by_name(self): + model = torch.nn.Module() + model.engram_hasher = SimpleNamespace(history=torch.zeros(5, 3)) + binding = mb.MegakernelBinding(model=model) + attn_backend = backend(Pool()) + self._view(binding, attn_backend) + self.assertIs(binding.engram_history(), model.engram_hasher.history) + with patch( + "sglang.srt.model_executor.forward_context.get_attn_backend", + return_value=attn_backend, + ): + binding.assert_live() + model.engram_hasher.history = torch.zeros(9, 3) + with self.assertRaisesRegex(RuntimeError, "engram_history was reallocated"): + binding.assert_live() + binding = mb.MegakernelBinding(model=torch.nn.Module()) + self._view(binding, backend(Pool())) + with patch( + "sglang.srt.model_executor.forward_context.get_attn_backend", + return_value=backend(Pool()), + ): + with self.assertRaisesRegex(RuntimeError, "kv_pool was reallocated"): + binding.assert_live() + + def test_a_model_without_engram_reads_none_and_seals_nothing(self): + binding = mb.MegakernelBinding(model=torch.nn.Module()) + self.assertIsNone(binding.engram_history()) + self.assertEqual(binding._sealed, {}) + + def test_every_tolerated_member_exists(self): + """A plugin pins these names; renaming one here must fail first.""" + view = self._view( + mb.MegakernelBinding(model=torch.nn.Module()), backend(Pool()) + ) + owners = {"ForwardView": view, "KvPoolView": view.kv_pool} + for name in mb.TOLERATED_CALLABLES: + owner, _, attr = name.partition(".") + if owner in owners: + self.assertTrue(hasattr(owners[owner], attr), name) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/megakernels/test_megakernel_runtime.py b/test/registered/unit/megakernels/test_megakernel_runtime.py index 8fca2463f844..c28a43a6798b 100644 --- a/test/registered/unit/megakernels/test_megakernel_runtime.py +++ b/test/registered/unit/megakernels/test_megakernel_runtime.py @@ -10,6 +10,7 @@ from sglang.srt.megakernels import runtime from sglang.srt.megakernels import spec as registry +from sglang.srt.megakernels.binding import MegakernelBinding from sglang.srt.megakernels.guards import deadline_s from sglang.srt.megakernels.spec import ( MegakernelSpec, @@ -42,7 +43,7 @@ class FakeBuilt: def __init__(self, widths): self.widths = tuple(w for w in widths if w <= 4) - self.steps = [] + self.steps, self.views = [], [] def summary(self): return "fake" @@ -53,6 +54,7 @@ def plan(self, input_ids, forward_batch, *args, **kwargs): def forward(self, step, original, input_ids, positions, forward_batch, **kw): self.steps.append(step.rows) + self.views.append(step.view) return "megakernel" @@ -136,6 +138,12 @@ def setUp(self): registry.register(architecture=ARCH, path=f"{__name__}:SPEC") publish(server_args(), role="test") self.addCleanup(reset_context) + # the running forward's view reads its attention backend, which tests lack + view_patch = patch.object( + MegakernelBinding, "forward_view", return_value="view" + ) + view_patch.start() + self.addCleanup(view_patch.stop) def _install_spec(self, spec, **kw): with patch.object(runtime, "find_spec", return_value=(spec, "")): @@ -264,6 +272,9 @@ def test_covered_steps_run_the_megakernel_and_others_the_forward(self): self.assertEqual(lm(batch(4).input_ids, None, batch(4)), "megakernel") self.assertEqual(lm(batch(6).input_ids, None, batch(6)), "standard") self.assertEqual(built.steps, [4]) + # only a covered step gets the running forward's view, through Step.view + self.assertEqual(built.views, ["view"]) + self.assertIsInstance(lm.context.binding, MegakernelBinding) # graph runners and supports_pp read the original forward's parameters self.assertEqual(inspect.signature(lm.forward), inspect.signature(Lm().forward)) self.assertIn("--enable-nvidia-megakernel", lm._derived_weight_cache_error) From 06f3ea7aa067eeac726b5a01738c15abecd97587 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Sun, 18 Oct 2026 21:06:58 -0700 Subject: [PATCH 07/10] [Fix] [NVIDIA] megakernels: refuse a target without its spec in the workers; an empty TP set accepts any size - install_megakernel takes the model's role: a target model whose spec no rank found (the plugin failed to load in the workers after argument resolution found it) is refused on every rank, naming the cause, instead of running its own forward; a draft model without a spec still keeps its own forward. - The post-load requirement check treats an empty tp_sizes as any size, as argument resolution and the Requirements contract do. --- python/sglang/srt/megakernels/runtime.py | 26 +++++++++++----- .../sglang/srt/model_executor/model_runner.py | 1 + .../megakernels/test_megakernel_guards.py | 10 +++++++ .../megakernels/test_megakernel_runtime.py | 30 ++++++++++++++++--- 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/python/sglang/srt/megakernels/runtime.py b/python/sglang/srt/megakernels/runtime.py index bde3c5cea3d0..64a2369e2a6e 100644 --- a/python/sglang/srt/megakernels/runtime.py +++ b/python/sglang/srt/megakernels/runtime.py @@ -58,11 +58,12 @@ def install_megakernel( model: torch.nn.Module, hf_config: PretrainedConfig, capture_tail_hooks: list[Callable[..., None]], + is_draft: bool, ) -> None: """Build the model's spec and install its forward; runs before KV cache sizing.""" if not megakernel_enabled(): return - spec = _find_spec_on_every_rank(hf_config=hf_config) + spec = _find_spec_on_every_rank(hf_config=hf_config, is_draft=is_draft) if spec is None: return # Resolution checked the target's spec on the server arguments; a draft's spec @@ -143,9 +144,10 @@ def megakernel_draft_sampler(draft_model: torch.nn.Module) -> Optional[Callable] def _find_spec_on_every_rank( - *, hf_config: PretrainedConfig + *, hf_config: PretrainedConfig, is_draft: bool ) -> Optional[MegakernelSpec]: - """The spec every rank found, None when no rank has one registered; else raise.""" + """The spec every rank found; for a draft model, None when no rank has one + registered; else raise on every rank.""" archs = hf_config.architectures spec, why = find_spec(architectures=archs, isa=device_isa()) # Ranks branch together; one skipping the collectives below would hang its peers. @@ -157,9 +159,18 @@ def _find_spec_on_every_rank( ) ) if not any(is_registered for is_registered, _, _ in lookups): - # A draft model may have no registered spec. - logger.info("NVIDIA megakernel: %s; %s keeps its own forward", why, archs) - return None + if is_draft: + # A draft model may have no registered spec. + logger.info("NVIDIA megakernel: %s; %s keeps its own forward", why, archs) + return None + # resolution found the target's spec, so the plugin failed to load here + _refuse( + [ + f"{why} in the model workers, though argument resolution found it; " + "see 'Failed to load plugin' or 'Failed to execute general plugin' " + "above" + ] + ) _refuse( [ f"rank {rank}: {why}" @@ -181,7 +192,8 @@ def _requirement_reasons( """The spec's requirements this runner can check without the server arguments.""" req, tp_size = spec.requires, get_parallel().tp_size reasons = [] - if tp_size not in req.tp_sizes: + # an empty set accepts any TP size, as at argument resolution + if req.tp_sizes and tp_size not in req.tp_sizes: reasons.append(f"tp_size {tp_size} is not one of {sorted(req.tp_sizes)}") graph = get_exec().graph.cuda_graph_config backend = Backend.DISABLED if graph is None else graph.decode.backend diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index ab082dd2784c..86a6b329a17f 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -732,6 +732,7 @@ def initialize(self): model=self.model, hf_config=self.model_config.hf_config, capture_tail_hooks=self.capture_tail_hooks, + is_draft=self.is_draft_worker, ) def init_memory_saver_adapter(self): diff --git a/test/registered/unit/megakernels/test_megakernel_guards.py b/test/registered/unit/megakernels/test_megakernel_guards.py index 3d64a67f7c62..7a436dae3f47 100644 --- a/test/registered/unit/megakernels/test_megakernel_guards.py +++ b/test/registered/unit/megakernels/test_megakernel_guards.py @@ -58,6 +58,9 @@ ), ) +# an empty field accepts any value +EMPTY_TP_SPEC = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset())) + def server_args(**kw): """Arguments as the resolution step sees them, with the graph config resolved.""" @@ -132,6 +135,13 @@ def test_every_refusal_reason_is_listed(self): self.assertIsNone(spec) self.assertIn(why, reasons[0]) + def test_an_empty_tp_set_accepts_any_tp_size(self): + """Requirements: an empty field accepts any value (resolution check).""" + with patch.dict(registry._REGISTRY, clear=True): + registry.register(architecture=ARCH, path=f"{__name__}:EMPTY_TP_SPEC") + _, reasons = reasons_for(server_args(tp_size=8)) + self.assertFalse(any("tp_size" in r for r in reasons), reasons) + def test_a_plugin_that_registered_nothing_is_named(self): """A plugin that fails to load is only logged, so an empty registry must not read as an unsupported model.""" diff --git a/test/registered/unit/megakernels/test_megakernel_runtime.py b/test/registered/unit/megakernels/test_megakernel_runtime.py index c28a43a6798b..f2d125cb3dc0 100644 --- a/test/registered/unit/megakernels/test_megakernel_runtime.py +++ b/test/registered/unit/megakernels/test_megakernel_runtime.py @@ -105,7 +105,14 @@ def server_args(**kw): def install( - *, archs=(ARCH,), gathered=None, tokens=2, graphs=True, hooks=None, lm=None + *, + archs=(ARCH,), + gathered=None, + tokens=2, + graphs=True, + hooks=None, + lm=None, + is_draft=False, ): """install_megakernel on an Lm as rank 0 of the published server; gathered maps a gathered value to what the other ranks report, default the same as this rank.""" @@ -126,6 +133,7 @@ def install( model=lm, hf_config=SimpleNamespace(architectures=list(archs)), capture_tail_hooks=hooks, + is_draft=is_draft, ) return lm @@ -226,14 +234,28 @@ def test_without_the_flag_nothing_is_read(self): model = SimpleNamespace() # any attribute read would raise with patch.object(runtime, "megakernel_enabled", return_value=False): runtime.install_megakernel( - model=model, hf_config=None, capture_tail_hooks=[] + model=model, hf_config=None, capture_tail_hooks=[], is_draft=False ) self.assertEqual(vars(model), {}) - def test_a_model_without_a_spec_on_every_rank_keeps_its_forward(self): - lm = install(archs=["DraftWithoutSpec"]) + def test_a_draft_without_a_spec_on_every_rank_keeps_its_forward(self): + with self.assertLogs(runtime.logger, "INFO") as logs: + lm = install(archs=["DraftWithoutSpec"], is_draft=True) self.assertNotIn("forward", lm.__dict__) self.assertIsNone(runtime.built_megakernel(lm)) + self.assertIn("keeps its own forward", "".join(logs.output)) + + def test_a_target_without_a_spec_on_every_rank_is_refused(self): + """Resolution found the target's spec, but the plugin failed in every + worker: the flag must not silently run the model's own forward.""" + with self.assertRaisesRegex(ValueError, "in the model workers"): + install(archs=["TargetWithoutSpec"]) + + def test_an_empty_tp_set_accepts_any_tp_size(self): + """Requirements: an empty field accepts any value (post-load check).""" + spec = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset())) + reasons = runtime._requirement_reasons(spec=spec, hf_config=SimpleNamespace()) + self.assertFalse(any("tp_size" in r for r in reasons), reasons) def test_every_rank_refuses_when_any_rank_cannot_serve(self): # rank 1 lacks the spec: rank 0 must raise too, or it waits in the build From 7b7015e72e4ffd6641159de0d8f3f466313f4e8b Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Sun, 18 Oct 2026 21:56:33 -0700 Subject: [PATCH 08/10] [Test] [NVIDIA] megakernels: fewer, behaviour-level tests One test per behaviour the framework guarantees: the binding's view and tolerated names, a replaced source caught after capture; every refusal reason at resolution and the resolution step; install routing covered and uncovered steps; a target without a spec refused and the draft fallback; the post-load checks; ranks that disagree refused; the capture check; the draft sampler only on the batch sizes every rank serves. --- test/registered/unit/megakernels/_fakes.py | 108 +++++++ .../megakernels/test_megakernel_binding.py | 76 ++--- .../megakernels/test_megakernel_contract.py | 31 +- .../megakernels/test_megakernel_guards.py | 159 ++-------- .../megakernels/test_megakernel_runtime.py | 287 ++++-------------- .../unit/spec/test_dspark_draft_block.py | 134 ++------ 6 files changed, 244 insertions(+), 551 deletions(-) create mode 100644 test/registered/unit/megakernels/_fakes.py diff --git a/test/registered/unit/megakernels/_fakes.py b/test/registered/unit/megakernels/_fakes.py new file mode 100644 index 000000000000..0042d6b3897c --- /dev/null +++ b/test/registered/unit/megakernels/_fakes.py @@ -0,0 +1,108 @@ +"""Shared fakes for the megakernel unit tests: one spec, one model, server arguments.""" + +from types import SimpleNamespace + +import torch +from msgspec.structs import replace + +from sglang.srt.megakernels import spec as registry +from sglang.srt.megakernels.spec import ( + Feature, + MegakernelSpec, + Numerics, + NumericsKind, + Requirements, + Step, +) +from sglang.srt.model_executor.cuda_graph_config import default_cuda_graph_config +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci( + est_time=0, stage="base-a", runner_config="cpu", disabled="megakernel test fakes" +) + +ARCH, ISA = "FakeMegaForCausalLM", "sm_103" + + +class FakeBuilt: + """Covers steps of 1-4 rows and records the steps it ran.""" + + draft_sampler = None + + def __init__(self, widths): + self.widths = tuple(w for w in widths if w <= 4) + self.steps, self.views = [], [] + + def summary(self): + return "fake" + + def plan(self, input_ids, forward_batch, *args, **kwargs): + rows = int(input_ids.shape[0]) + return Step(mode="decode", rows=rows) if rows in self.widths else None + + def forward(self, step, original, input_ids, positions, forward_batch, **kw): + self.steps.append(step.rows) + self.views.append(step.view) + return "megakernel" + + +def _build(model, hf_config, context): + model.context = context + return FakeBuilt(context.widths) + + +# what the post-load check reads; argument resolution also checks STRICT's extras +SPEC = MegakernelSpec( + name="fake", + requires=Requirements(isas=frozenset({ISA}), tp_sizes=frozenset({4})), + features=frozenset({Feature.SPECULATIVE_DECODING}), + numerics=Numerics(kind=NumericsKind.BITWISE), + max_rows=8, + build=_build, + api_version=registry.MEGAKERNEL_API_VERSION, +) +STRICT = replace( + SPEC, + requires=replace( + SPEC.requires, + dtypes=frozenset({"bfloat16"}), + server_args={"kv_cache_dtype": frozenset({"fp8_e4m3"})}, + hf_config={"model_type": "fake"}, + env={"SGLANG_DSPARK_EMBED_IN_GRAPH": True}, + ), + config_disable_reasons=lambda cfg, hf: ( + ["context_length too long"] if cfg.context_length == 7 else [] + ), +) +ANY_TP = replace(STRICT, requires=replace(STRICT.requires, tp_sizes=frozenset())) +# a draft spec: builds no width, offers only the draft sampler +DRAFT = replace( + SPEC, + max_rows=0, + build=lambda model, hf, ctx: SimpleNamespace( + widths=(), summary=lambda: "draft sampler", draft_sampler=print + ), +) + + +class Lm(torch.nn.Module): + def forward(self, input_ids, positions, forward_batch, input_embeds=None): + return "standard" + + +def batch(rows): + return SimpleNamespace(batch_size=rows, input_ids=torch.zeros(rows)) + + +def server_args(**kw): + """Arguments as the resolution step sees them, with the graph config resolved.""" + defaults = { + "model_path": "dummy", + "tp_size": 4, + "kv_cache_dtype": "fp8_e4m3", + "speculative_algorithm": "DSPARK", + "speculative_num_draft_tokens": 6, + "cuda_graph_config": default_cuda_graph_config(), + } + return ServerArgs(**{**defaults, **kw}) diff --git a/test/registered/unit/megakernels/test_megakernel_binding.py b/test/registered/unit/megakernels/test_megakernel_binding.py index e3dc326c9066..72f7d7d51bed 100644 --- a/test/registered/unit/megakernels/test_megakernel_binding.py +++ b/test/registered/unit/megakernels/test_megakernel_binding.py @@ -1,6 +1,6 @@ """CPU contract of the megakernel binding: the running forward's attention data and -KV pool under SGLang's own names, read from the forward context; sources sealed at -their first read and checked after capture; the tolerated lists name real members.""" +KV pool under SGLang's own names, sources checked after capture, and the tolerated +lists naming real members.""" import unittest from types import SimpleNamespace @@ -14,6 +14,8 @@ register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") +BACKEND = "sglang.srt.model_executor.forward_context.get_attn_backend" + class Pool: uniform_fp8 = False @@ -25,21 +27,9 @@ def __init__(self): def get_swa_key_buffer_radix(self, layer_id): return self.keys[layer_id] - def get_swa_key_page_size(self): - return 128 - - def get_swa_key_bytes_per_token(self): - return 576 - - def get_extra_key_buffer(self, layer_id): - return None - def get_extra_key_page_size(self, layer_id): return 64 + layer_id - def get_extra_key_bytes_per_token(self, layer_id): - return 32 + layer_id - def set_swa_key_buffer_radix_fused_norm_rope(self, **kw): return kw @@ -58,63 +48,41 @@ def backend(pool): class TestMegakernelBinding(CustomTestCase): - def _view(self, binding, attn_backend): - with patch( - "sglang.srt.model_executor.forward_context.get_attn_backend", - return_value=attn_backend, - ): - return binding.forward_view() - - def test_the_view_is_the_running_forward_under_sglangs_names(self): + def test_the_view_is_the_running_forward_and_every_tolerated_member_exists(self): pool = Pool() - view = self._view(mb.MegakernelBinding(model=torch.nn.Module()), backend(pool)) + with patch(BACKEND, return_value=backend(pool)): + view = mb.MegakernelBinding(model=torch.nn.Module()).forward_view() self.assertEqual(view.metadata.core_attn_metadata, "core") self.assertIs(view.kv_pool.pool, pool) self.assertIs(view.kv_pool.get_swa_key_buffer_radix(1), pool.keys[1]) self.assertEqual(view.kv_pool.get_extra_key_page_size(1), 65) self.assertEqual(view.swa_out_cache_loc(None), "swa_loc") - self.assertEqual(view.attention(), "attention") + # a plugin pins these names: a rename here must fail first + owners = {"ForwardView": view, "KvPoolView": view.kv_pool} + for name in mb.TOLERATED_CALLABLES: + owner, _, attr = name.partition(".") + if owner in owners: + self.assertTrue(hasattr(owners[owner], attr), name) - def test_a_replaced_pool_or_history_fails_assert_live_by_name(self): + def test_a_replaced_source_fails_assert_live_by_name(self): model = torch.nn.Module() model.engram_hasher = SimpleNamespace(history=torch.zeros(5, 3)) binding = mb.MegakernelBinding(model=model) attn_backend = backend(Pool()) - self._view(binding, attn_backend) - self.assertIs(binding.engram_history(), model.engram_hasher.history) - with patch( - "sglang.srt.model_executor.forward_context.get_attn_backend", - return_value=attn_backend, - ): + with patch(BACKEND, return_value=attn_backend): + binding.forward_view() + self.assertIs(binding.engram_history(), model.engram_hasher.history) binding.assert_live() model.engram_hasher.history = torch.zeros(9, 3) - with self.assertRaisesRegex(RuntimeError, "engram_history was reallocated"): + with self.assertRaisesRegex(RuntimeError, "engram_history"): binding.assert_live() binding = mb.MegakernelBinding(model=torch.nn.Module()) - self._view(binding, backend(Pool())) - with patch( - "sglang.srt.model_executor.forward_context.get_attn_backend", - return_value=backend(Pool()), - ): - with self.assertRaisesRegex(RuntimeError, "kv_pool was reallocated"): + with patch(BACKEND, return_value=backend(Pool())): + binding.forward_view() + with patch(BACKEND, return_value=backend(Pool())): + with self.assertRaisesRegex(RuntimeError, "kv_pool"): binding.assert_live() - def test_a_model_without_engram_reads_none_and_seals_nothing(self): - binding = mb.MegakernelBinding(model=torch.nn.Module()) - self.assertIsNone(binding.engram_history()) - self.assertEqual(binding._sealed, {}) - - def test_every_tolerated_member_exists(self): - """A plugin pins these names; renaming one here must fail first.""" - view = self._view( - mb.MegakernelBinding(model=torch.nn.Module()), backend(Pool()) - ) - owners = {"ForwardView": view, "KvPoolView": view.kv_pool} - for name in mb.TOLERATED_CALLABLES: - owner, _, attr = name.partition(".") - if owner in owners: - self.assertTrue(hasattr(owners[owner], attr), name) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/megakernels/test_megakernel_contract.py b/test/registered/unit/megakernels/test_megakernel_contract.py index f3b16208d96a..956e07a42254 100644 --- a/test/registered/unit/megakernels/test_megakernel_contract.py +++ b/test/registered/unit/megakernels/test_megakernel_contract.py @@ -14,22 +14,11 @@ ROOT = Path(spec.__file__).resolve().parents[2] # python/sglang # Names the framework reads that its CPU tests reach only through fakes: runner, -# batch and process-group attributes, string protocols, launch-only arguments. +# batch and process-group attributes, string protocols. READS = { - "srt/arg_groups/fields/*.py": ( - "cuda_graph_config", - "enable_nvidia_megakernel", - "speculative_algorithm", - "speculative_num_draft_tokens", - "watchdog_timeout", - ), "srt/configs/model_config.py": ("dtype", "hf_config"), "srt/distributed/parallel_state.py": ("world_size", "cpu_group"), - "srt/model_executor/forward_batch_info.py": ( - "forward_mode", - "batch_size", - "input_ids", - ), + "srt/model_executor/forward_batch_info.py": ("batch_size", "input_ids"), "srt/model_executor/model_runner.py": ( "capture_tail_hooks", "decode_cuda_graph_runner", @@ -67,22 +56,6 @@ def _provided(path: Path) -> frozenset: return frozenset(names) -class TestResolutionOrder(CustomTestCase): - def test_the_megakernel_step_runs_before_the_other_kernel_flag_step(self): - """With both kernel flags set, the megakernel step must raise first, listing the - conflict with every other reason, before the other step refuses on its own.""" - tree = ast.parse((ROOT / "srt/arg_groups/pipeline.py").read_text()) - line = { - node.args[0].id: node.lineno - for node in ast.walk(tree) - if isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == "run_hook" - and isinstance(node.args[0], ast.Name) - } - self.assertLess(line["handle_nvidia_megakernel"], line["handle_monokernel"]) - - class TestSglangReads(CustomTestCase): def test_sglang_provides_what_the_framework_reads(self): for pattern, names in READS.items(): diff --git a/test/registered/unit/megakernels/test_megakernel_guards.py b/test/registered/unit/megakernels/test_megakernel_guards.py index 7a436dae3f47..ecfe61c452a2 100644 --- a/test/registered/unit/megakernels/test_megakernel_guards.py +++ b/test/registered/unit/megakernels/test_megakernel_guards.py @@ -1,10 +1,10 @@ """CPU contracts of --enable-nvidia-megakernel's startup checks and wait deadlines.""" +import sys import unittest +from pathlib import Path from types import SimpleNamespace -from unittest.mock import MagicMock, patch - -from msgspec.structs import replace +from unittest.mock import patch from sglang.srt.environ import envs from sglang.srt.megakernels import guards @@ -12,71 +12,17 @@ from sglang.srt.megakernels.guards import ( MAX_DEADLINE_S, deadline_s, - features_in_use, handle_nvidia_megakernel, megakernel_disable_reasons, ) -from sglang.srt.megakernels.spec import ( - MEGAKERNEL_API_VERSION, - Feature, - MegakernelSpec, - Numerics, - NumericsKind, - Requirements, -) -from sglang.srt.model_executor.cuda_graph_config import ( - Backend, - Phase, - default_cuda_graph_config, - with_phase, -) -from sglang.srt.server_args import ServerArgs +from sglang.srt.megakernels.spec import Feature from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase -register_cpu_ci(est_time=10, stage="base-a", runner_config="cpu") - -ARCH = "FakeMegaForCausalLM" - -SPEC = MegakernelSpec( - name="fake", - requires=Requirements( - isas=frozenset({"sm_103"}), - tp_sizes=frozenset({4}), - dtypes=frozenset({"bfloat16"}), - server_args={"kv_cache_dtype": frozenset({"fp8_e4m3"})}, - hf_config={"model_type": "fake"}, - env={"SGLANG_DSPARK_EMBED_IN_GRAPH": True}, - ), - features=frozenset({Feature.SPECULATIVE_DECODING}), - numerics=Numerics(kind=NumericsKind.BITWISE), - max_rows=48, - build=lambda model, hf_config, ctx: None, - api_version=MEGAKERNEL_API_VERSION, - config_disable_reasons=lambda cfg, hf: ( - ["context_length too long"] if cfg.context_length == 7 else [] - ), -) - -# an empty field accepts any value -EMPTY_TP_SPEC = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset())) - +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from _fakes import ARCH, STRICT, server_args # noqa: E402 -def server_args(**kw): - """Arguments as the resolution step sees them, with the graph config resolved.""" - defaults = { - "model_path": "dummy", - "tp_size": 4, - "kv_cache_dtype": "fp8_e4m3", - "cuda_graph_config": default_cuda_graph_config(), - } - return ServerArgs(**{**defaults, **kw}) - - -def entry_point(name): - ep = MagicMock() - ep.name = name - return ep +register_cpu_ci(est_time=10, stage="base-a", runner_config="cpu") def reasons_for(cfg, *, arch=ARCH, isa="sm_103", model_type="fake"): @@ -89,24 +35,14 @@ def setUp(self): registry_patch = patch.dict(registry._REGISTRY, clear=True) registry_patch.start() self.addCleanup(registry_patch.stop) - registry.register(architecture=ARCH, path=f"{__name__}:SPEC") - - def test_every_feature_has_a_probe_and_none_is_on_by_default(self): - # a feature without a probe is never refused; a probe stuck on refuses every server - post_load = {Feature.PARTITIONED_GPU} # the runtime checks it on the device - self.assertEqual(set(guards.FEATURE_PROBES), set(Feature) - post_load) - self.assertEqual(features_in_use(ServerArgs(model_path="dummy")), set()) - - def test_features_in_use_are_refused_unless_accepted(self): - _, reasons = reasons_for(server_args(pp_size=2, speculative_algorithm="DSPARK")) - self.assertEqual(len(reasons), 1, reasons) # SPEC accepts speculative decoding - self.assertIn(Feature.PIPELINE_PARALLEL.value, reasons[0]) + registry.register(architecture=ARCH, path="_fakes:STRICT") def test_every_refusal_reason_is_listed(self): with envs.SGLANG_DSPARK_EMBED_IN_GRAPH.override(False): spec, reasons = reasons_for( server_args( tp_size=8, + pp_size=2, kv_cache_dtype="auto", enable_eplb=True, context_length=7, @@ -114,18 +50,20 @@ def test_every_refusal_reason_is_listed(self): ), model_type="other", ) - self.assertIs(spec, SPEC) + self.assertIs(spec, STRICT) for want in ( "tp_size 8", "kv_cache_dtype 'auto'", "config.model_type 'other'", "SGLANG_DSPARK_EMBED_IN_GRAPH False", Feature.EPLB.value, + Feature.PIPELINE_PARALLEL.value, "context_length", "--watchdog-timeout", ): self.assertTrue(any(want in r for r in reasons), (want, reasons)) - self.assertEqual(reasons_for(server_args())[1], []) + with envs.SGLANG_DSPARK_EMBED_IN_GRAPH.override(True): + self.assertEqual(reasons_for(server_args())[1], []) self.assertEqual(reasons_for(server_args(), isa=None)[1], ["no GPU is visible"]) for arch, isa, why in ( ("Other", "sm_103", "Other"), @@ -134,53 +72,16 @@ def test_every_refusal_reason_is_listed(self): spec, reasons = reasons_for(server_args(), arch=arch, isa=isa) self.assertIsNone(spec) self.assertIn(why, reasons[0]) - - def test_an_empty_tp_set_accepts_any_tp_size(self): - """Requirements: an empty field accepts any value (resolution check).""" - with patch.dict(registry._REGISTRY, clear=True): - registry.register(architecture=ARCH, path=f"{__name__}:EMPTY_TP_SPEC") - _, reasons = reasons_for(server_args(tp_size=8)) - self.assertFalse(any("tp_size" in r for r in reasons), reasons) - - def test_a_plugin_that_registered_nothing_is_named(self): - """A plugin that fails to load is only logged, so an empty registry must not - read as an unsupported model.""" + # an empty field accepts any value registry._REGISTRY.clear() - spec, reasons = reasons_for(server_args()) - self.assertIsNone(spec) - self.assertIn("installed but registered no spec", reasons[0]) - - def test_the_decode_graph_backend_is_checked(self): - full_only = replace( - SPEC, requires=replace(SPEC.requires, decode_graph_backends={"full"}) - ) - graphs = with_phase( - default_cuda_graph_config(), Phase.DECODE, backend=Backend.BREAKABLE - ) - with patch.object(guards, "find_spec", return_value=(full_only, "")): - _, reasons = reasons_for(server_args(cuda_graph_config=graphs)) - self.assertTrue(any("decode CUDA graphs" in r for r in reasons), reasons) - - def test_the_flag_needs_its_plugin_to_load(self): - """Without its plugin, the flag must fail at startup, not serve the standard path.""" - cases = ( - ([], "", "install sgl-nvidia-mega"), - ([entry_point("nvidia_megakernel")], "other", "SGLANG_PLUGINS='other'"), - ) - for eps, allowed, want in cases: - with ( - self.subTest(want=want), - patch.object(guards, "entry_points", return_value=eps), - patch.object(guards.envs.SGLANG_PLUGINS, "get", return_value=allowed), - ): - self.assertTrue(any(want in r for r in guards.plugin_reasons())) - eps = [entry_point("nvidia_megakernel")] - for allowed in ("", "other, nvidia_megakernel"): - with ( - patch.object(guards, "entry_points", return_value=eps), - patch.object(guards.envs.SGLANG_PLUGINS, "get", return_value=allowed), - ): - self.assertEqual(guards.plugin_reasons(), []) + registry.register(architecture=ARCH, path="_fakes:ANY_TP") + tp8 = reasons_for(server_args(tp_size=8))[1] + self.assertFalse(any("tp_size" in r for r in tp8), tp8) + # a plugin that fails to load is only logged: name it, not the model + registry._REGISTRY.clear() + self.assertIn("registered no spec", reasons_for(server_args())[1][0]) + with patch.object(guards, "entry_points", return_value=[]): + self.assertIn("install sgl-nvidia-mega", guards.plugin_reasons()[0]) def test_the_resolution_step_raises_every_reason_or_nothing(self): with patch.object(guards, "plugin_reasons") as plugin: @@ -194,7 +95,7 @@ def test_the_resolution_step_raises_every_reason_or_nothing(self): patch.object(guards, "model_config_of") as model_config, ): model_config.return_value = SimpleNamespace(hf_config=None, dtype="float16") - reasons.return_value = (SPEC, ["tp_size 8 is not one of ['4']"]) + reasons.return_value = (STRICT, ["tp_size 8 is not one of ['4']"]) # the resolved dtype is checked here, and listed with the other reasons with self.assertRaisesRegex( ValueError, "(?s)tp_size 8.*\n - dtype 'float16'" @@ -203,20 +104,14 @@ def test_the_resolution_step_raises_every_reason_or_nothing(self): model_config.return_value = SimpleNamespace( hf_config=None, dtype="bfloat16" ) - reasons.return_value = (SPEC, []) + reasons.return_value = (STRICT, []) handle_nvidia_megakernel(on) - # both flags replace the model's forward: refused before any weight loads, - # listed with the spec's own reasons + # both flags replace the model's forward: refused before any weight loads both = server_args(enable_nvidia_megakernel=True, monokernel_backend="x") - reasons.return_value = (SPEC, ["tp_size 8 is not one of ['4']"]) - with self.assertRaisesRegex( - ValueError, "(?s)tp_size 8.*\n - --monokernel-backend x is also set" - ): + with self.assertRaisesRegex(ValueError, "--monokernel-backend x"): handle_nvidia_megakernel(both) - - def test_deadlines_stay_below_the_watchdog(self): + # every wait's deadline stays below the watchdog self.assertEqual(deadline_s(None), MAX_DEADLINE_S) - self.assertEqual(deadline_s(3600), MAX_DEADLINE_S) self.assertEqual(deadline_s(30), 15.0) diff --git a/test/registered/unit/megakernels/test_megakernel_runtime.py b/test/registered/unit/megakernels/test_megakernel_runtime.py index f2d125cb3dc0..dee13bf046f0 100644 --- a/test/registered/unit/megakernels/test_megakernel_runtime.py +++ b/test/registered/unit/megakernels/test_megakernel_runtime.py @@ -1,7 +1,9 @@ -"""CPU contracts of the megakernel registry, install, dispatch and capture check.""" +"""CPU contracts of the megakernel install, dispatch and capture check.""" import inspect +import sys import unittest +from pathlib import Path from types import SimpleNamespace from unittest.mock import patch @@ -11,97 +13,14 @@ from sglang.srt.megakernels import runtime from sglang.srt.megakernels import spec as registry from sglang.srt.megakernels.binding import MegakernelBinding -from sglang.srt.megakernels.guards import deadline_s -from sglang.srt.megakernels.spec import ( - MegakernelSpec, - Numerics, - NumericsKind, - Requirements, - Step, -) -from sglang.srt.model_executor.cuda_graph_config import ( - Backend, - Phase, - default_cuda_graph_config, - with_phase, -) from sglang.srt.runtime_context import publish, reset_context -from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase -register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") - -ARCH = "FakeMegaForCausalLM" -ISA = "sm_103" - - -class FakeBuilt: - """Covers steps of 1-4 rows and records the steps it ran.""" - - draft_sampler = None - - def __init__(self, widths): - self.widths = tuple(w for w in widths if w <= 4) - self.steps, self.views = [], [] - - def summary(self): - return "fake" - - def plan(self, input_ids, forward_batch, *args, **kwargs): - rows = int(input_ids.shape[0]) - return Step(mode="decode", rows=rows) if rows in self.widths else None - - def forward(self, step, original, input_ids, positions, forward_batch, **kw): - self.steps.append(step.rows) - self.views.append(step.view) - return "megakernel" - +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from _fakes import ARCH, ISA, SPEC, FakeBuilt, Lm, batch, server_args # noqa: E402 -def _build(model, hf_config, context): - model.context = context - return FakeBuilt(context.widths) - - -SPEC = MegakernelSpec( - name="fake", - requires=Requirements(isas=frozenset({ISA}), tp_sizes=frozenset({4})), - features=frozenset(), - numerics=Numerics(kind=NumericsKind.BITWISE), - max_rows=8, - build=_build, - api_version=registry.MEGAKERNEL_API_VERSION, -) -OLD_SPEC = replace(SPEC, api_version=registry.MEGAKERNEL_API_VERSION - 1) -# a draft spec: builds no width, offers only the draft sampler -DRAFT_SPEC = replace( - SPEC, - max_rows=0, - build=lambda model, hf, ctx: SimpleNamespace( - widths=(), summary=lambda: "draft sampler", draft_sampler=print - ), -) - - -class Lm(torch.nn.Module): - def forward(self, input_ids, positions, forward_batch, input_embeds=None): - return "standard" - - -def batch(rows): - return SimpleNamespace(batch_size=rows, input_ids=torch.zeros(rows)) - - -def server_args(**kw): - defaults = { - "model_path": "dummy", - "enable_nvidia_megakernel": True, - "tp_size": 4, - "speculative_algorithm": "DSPARK", - "speculative_num_draft_tokens": 6, - "cuda_graph_config": default_cuda_graph_config(), - } - return ServerArgs(**{**defaults, **kw}) +register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") def install( @@ -117,12 +36,13 @@ def install( """install_megakernel on an Lm as rank 0 of the published server; gathered maps a gathered value to what the other ranks report, default the same as this rank.""" lm, hooks = lm or Lm(), [] if hooks is None else hooks - gather = gathered or (lambda x, group=None: [x]) gpu = SimpleNamespace(multi_processor_count=160) with ( patch.object(runtime, "device_isa", return_value=ISA), patch.object(runtime, "deadline_s", return_value=30.0), - patch.object(runtime, "_gather_tp", side_effect=gather), + patch.object( + runtime, "_gather_tp", side_effect=gathered or (lambda x, group=None: [x]) + ), patch.object(runtime, "_tokens_per_request", return_value=tokens), patch.object(runtime, "_decode_graphs_enabled", return_value=graphs), patch.object(torch.cuda, "current_device", return_value=0), @@ -143,8 +63,8 @@ def setUp(self): registry_patch = patch.dict(registry._REGISTRY, clear=True) registry_patch.start() self.addCleanup(registry_patch.stop) - registry.register(architecture=ARCH, path=f"{__name__}:SPEC") - publish(server_args(), role="test") + registry.register(architecture=ARCH, path="_fakes:SPEC") + publish(server_args(enable_nvidia_megakernel=True), role="test") self.addCleanup(reset_context) # the running forward's view reads its attention backend, which tests lack view_patch = patch.object( @@ -157,135 +77,14 @@ def _install_spec(self, spec, **kw): with patch.object(runtime, "find_spec", return_value=(spec, "")): return install(**kw) - def test_the_runtime_reads_the_published_config(self): - self.assertTrue(runtime.megakernel_enabled()) - self.assertEqual(runtime._tokens_per_request(), 6) - self.assertTrue(runtime._decode_graphs_enabled()) - reset_context() - eager = with_phase( - default_cuda_graph_config(), Phase.DECODE, backend=Backend.DISABLED - ) - publish(server_args(watchdog_timeout=40, cuda_graph_config=eager), role="test") - self.assertFalse(runtime._decode_graphs_enabled()) - self.assertEqual(deadline_s(runtime.get_device().watchdog_timeout), 20.0) - reset_context() - publish(server_args(enable_nvidia_megakernel=False), role="test") - self.assertFalse(runtime.megakernel_enabled()) - - def test_a_spec_found_at_load_is_checked_on_what_the_runner_reads(self): - # a draft's spec is never checked at resolution - strict = replace( - SPEC, - requires=replace( - SPEC.requires, - tp_sizes=frozenset({8}), - decode_graph_backends=frozenset({"breakable"}), - hf_config={"model_type": "fake"}, - ), - ) - with self.assertRaises(ValueError) as refused: - self._install_spec(strict) - for want in ("tp_size 4", "decode CUDA graphs 'full'", "config.model_type"): - self.assertIn(want, str(refused.exception)) - - def test_each_load_refusal_is_raised(self): - def other_spec_on_rank_1(value, group=None): - is_lookup = isinstance(value, tuple) and isinstance(value[-1], tuple) - other = (value[0], "", ("other", 1)) if is_lookup else value - return [value, other] - - with self.assertRaisesRegex(ValueError, "load different specs"): - install(gathered=other_spec_on_rank_1) - partitioned = replace( - SPEC, requires=replace(SPEC.requires, resident_blocks=200) - ) - with self.assertRaisesRegex(ValueError, "160 SMs"): - self._install_spec(partitioned) - with self.assertRaisesRegex(ValueError, "already replaced"): - install(lm=install(tokens=1)) - # a target that covers no width must not quietly keep the standard forward - empty = replace(SPEC, build=lambda model, hf, ctx: FakeBuilt(())) - with self.assertRaisesRegex(ValueError, "built no width of"): - self._install_spec(empty) - - def test_find_spec_refuses_with_a_reason(self): - registry.register(architecture="Broken", path="no_such_module:SPEC") - registry.register(architecture="Old", path=f"{__name__}:OLD_SPEC") - self.assertIs(registry.find_spec(architectures=[ARCH], isa=ISA)[0], SPEC) - version = registry.MEGAKERNEL_API_VERSION - for archs, isa, why in ( - ([ARCH], "sm_90", "supports ['sm_103'], not sm_90"), - (["Other"], ISA, "no megakernel spec is registered for ['Other']"), - (["Broken"], ISA, "does not import: ModuleNotFoundError"), - (["Old"], ISA, f"{version - 1}; this SGLang requires version {version}"), - ): - with self.subTest(archs=archs, isa=isa): - spec, reason = registry.find_spec(architectures=archs, isa=isa) - self.assertIsNone(spec) - self.assertIn(why, reason) - - def test_an_architecture_has_one_spec(self): - # Repeated registration must not silently replace another plugin's spec. - registry.register(architecture=ARCH, path=f"{__name__}:SPEC") - with self.assertRaises(ValueError): - registry.register(architecture=ARCH, path="other:SPEC") - - def test_without_the_flag_nothing_is_read(self): - model = SimpleNamespace() # any attribute read would raise + def test_covered_steps_run_the_megakernel_and_others_the_forward(self): + model = SimpleNamespace() # without the flag, any attribute read would raise with patch.object(runtime, "megakernel_enabled", return_value=False): runtime.install_megakernel( model=model, hf_config=None, capture_tail_hooks=[], is_draft=False ) self.assertEqual(vars(model), {}) - - def test_a_draft_without_a_spec_on_every_rank_keeps_its_forward(self): - with self.assertLogs(runtime.logger, "INFO") as logs: - lm = install(archs=["DraftWithoutSpec"], is_draft=True) - self.assertNotIn("forward", lm.__dict__) - self.assertIsNone(runtime.built_megakernel(lm)) - self.assertIn("keeps its own forward", "".join(logs.output)) - - def test_a_target_without_a_spec_on_every_rank_is_refused(self): - """Resolution found the target's spec, but the plugin failed in every - worker: the flag must not silently run the model's own forward.""" - with self.assertRaisesRegex(ValueError, "in the model workers"): - install(archs=["TargetWithoutSpec"]) - - def test_an_empty_tp_set_accepts_any_tp_size(self): - """Requirements: an empty field accepts any value (post-load check).""" - spec = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset())) - reasons = runtime._requirement_reasons(spec=spec, hf_config=SimpleNamespace()) - self.assertFalse(any("tp_size" in r for r in reasons), reasons) - - def test_every_rank_refuses_when_any_rank_cannot_serve(self): - # rank 1 lacks the spec: rank 0 must raise too, or it waits in the build - def no_spec_on_rank_1(value, group=None): - return [value, (value[0], "rank-1 reason", None)] - - with self.assertRaisesRegex(ValueError, "rank 1: rank-1 reason"): - install(gathered=no_spec_on_rank_1) - - def model_reason_on_rank_1(value, group=None): - return [value, ["no mHC pre-mix"]] if isinstance(value, list) else [value] - - with self.assertRaisesRegex(ValueError, "rank 1: no mHC pre-mix"): - install(gathered=model_reason_on_rank_1) - - def test_ranks_that_built_different_widths_are_refused(self): - def fewer_on_rank_1(value, group=None): - is_widths = isinstance(value, tuple) and all( - isinstance(w, int) for w in value - ) - return [value, value[:1]] if is_widths else [value] - - with self.assertRaisesRegex(RuntimeError, "widths fake built differ"): - install(gathered=fewer_on_rank_1) - - def test_widths_are_whole_requests_up_to_max_rows(self): self.assertEqual(install(tokens=2).context.widths, (2, 4, 6, 8)) - self.assertEqual(install(tokens=3).context.widths, (3, 6)) - - def test_covered_steps_run_the_megakernel_and_others_the_forward(self): lm = install(tokens=1) built = runtime.built_megakernel(lm) with patch.object( @@ -293,31 +92,59 @@ def test_covered_steps_run_the_megakernel_and_others_the_forward(self): ): self.assertEqual(lm(batch(4).input_ids, None, batch(4)), "megakernel") self.assertEqual(lm(batch(6).input_ids, None, batch(6)), "standard") - self.assertEqual(built.steps, [4]) - # only a covered step gets the running forward's view, through Step.view - self.assertEqual(built.views, ["view"]) + # only a covered step runs the megakernel, with the running forward's view + self.assertEqual((built.steps, built.views), ([4], ["view"])) self.assertIsInstance(lm.context.binding, MegakernelBinding) # graph runners and supports_pp read the original forward's parameters self.assertEqual(inspect.signature(lm.forward), inspect.signature(Lm().forward)) self.assertIn("--enable-nvidia-megakernel", lm._derived_weight_cache_error) + with self.assertRaisesRegex(ValueError, "already replaced"): + install(lm=lm) - def test_a_draft_spec_installs_no_forward_and_offers_its_sampler(self): - registry.register(architecture="Draft", path=f"{__name__}:DRAFT_SPEC") + def test_a_target_without_a_spec_is_refused_and_a_draft_keeps_its_forward(self): + # resolution found the target's spec but the plugin failed in every worker: + # the flag must not silently run the model's own forward + with self.assertRaisesRegex(ValueError, "in the model workers"): + install(archs=["TargetWithoutSpec"]) + with self.assertLogs(runtime.logger, "INFO") as logs: + lm = install(archs=["DraftWithoutSpec"], is_draft=True) + self.assertNotIn("forward", lm.__dict__) + self.assertIn("keeps its own forward", "".join(logs.output)) + registry.register(architecture="Draft", path="_fakes:DRAFT") hooks = [] - lm = install(archs=["Draft"], hooks=hooks) + lm = install(archs=["Draft"], hooks=hooks, is_draft=True) self.assertNotIn("forward", lm.__dict__) self.assertEqual(hooks, []) self.assertIs(runtime.megakernel_draft_sampler(lm), print) - self.assertIsNone(runtime.megakernel_draft_sampler(Lm())) - nothing = replace( - DRAFT_SPEC, - build=lambda model, hf, ctx: SimpleNamespace( - widths=(), summary=lambda: "", draft_sampler=None - ), - ) - with self.assertLogs(runtime.logger, "INFO") as logs: - self._install_spec(nothing) - self.assertIn("no step and no draft sampler", "".join(logs.output)) + + def test_a_spec_is_checked_on_what_the_runner_reads(self): + tp8 = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset({8}))) + with self.assertRaisesRegex(ValueError, "tp_size 4"): + self._install_spec(tp8) + # an empty field accepts any value + any_tp = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset())) + self.assertIsNotNone(runtime.built_megakernel(self._install_spec(any_tp))) + # a target that covers no width must not quietly keep the standard forward + empty = replace(SPEC, build=lambda model, hf, ctx: FakeBuilt(())) + with self.assertRaisesRegex(ValueError, "built no width of"): + self._install_spec(empty) + + def test_ranks_that_disagree_are_refused_on_every_rank(self): + """A rank that differs would wait in a different collective.""" + + def no_spec_on_rank_1(value, group=None): + return [value, (value[0], "rank-1 reason", None)] + + def fewer_widths_on_rank_1(value, group=None): + widths = isinstance(value, tuple) and all(isinstance(w, int) for w in value) + return [value, value[:1]] if widths else [value] + + for gathered, error, want in ( + (no_spec_on_rank_1, ValueError, "rank 1: rank-1 reason"), + (fewer_widths_on_rank_1, RuntimeError, "widths fake built differ"), + ): + with self.subTest(want=want), self.assertRaisesRegex(error, want): + install(gathered=gathered) def test_capture_check_refuses_when_no_graph_runs_the_megakernel(self): def capture(sizes, capture_bs): diff --git a/test/registered/unit/spec/test_dspark_draft_block.py b/test/registered/unit/spec/test_dspark_draft_block.py index bb3810159d53..5fb6ab186a32 100644 --- a/test/registered/unit/spec/test_dspark_draft_block.py +++ b/test/registered/unit/spec/test_dspark_draft_block.py @@ -163,118 +163,40 @@ def _build(model, *, greedy_draft, factory=None, agree=None): class TestMegakernelDraftSampler(CustomTestCase): - def test_a_plugin_step_replaces_only_the_base_logits_and_markov_block(self): - hidden = torch.randn(BS * GAMMA, HIDDEN) - input_ids = torch.arange(BS * GAMMA) - for greedy_draft in (False, True): - factory = Mock() - step = Mock(side_effect=lambda h, anchor: ("tokens", h * 2)) - factory.return_value = lambda bs: step if bs == BS else None - model = _folded_draft_model() - with ( - self.subTest(greedy_draft=greedy_draft), - patch.object(ds, "draft_markov_block", side_effect=AssertionError), - ): - sampler = _build(model, greedy_draft=greedy_draft, factory=factory) - sampler(hidden, input_ids) - model.compute_base_logits.assert_not_called() - # the plugin writes into SGLang's buffers; a temperature-0 row it skips - # must read as finite logits in a mixed accept - kwargs = factory.call_args.kwargs - self.assertIs(kwargs["draft_tokens"], sampler.out) - self.assertIs(kwargs["noise"], sampler._noise) - self.assertIs(kwargs["step_temperatures"], sampler.params.step_temperatures) - self.assertEqual(kwargs["greedy_draft"], greedy_draft) - if greedy_draft: - self.assertIsNone(kwargs["step_logits"]) - else: - self.assertIs(kwargs["step_logits"], sampler.step_logits) - self.assertEqual(sampler.step_logits.shape, (2 * BS, GAMMA, VOCAB)) - self.assertFalse(sampler.step_logits.any()) - self.assertTrue(torch.equal(step.call_args.args[1], input_ids[::GAMMA])) - confidence = model.compute_confidence.call_args.kwargs - self.assertEqual(confidence["draft_tokens"], "tokens") - self.assertTrue(torch.equal(confidence["confidence_features"], hidden * 2)) - - def test_without_a_plugin_step_the_stock_proposal_runs(self): - """Unset (no megakernel draft sampler), and a batch size the megakernel declines: - the stock base logits and Markov block run with today's arguments.""" - declined = lambda **kw: lambda bs: None # noqa: E731 - for factory in (None, declined): - model = _folded_draft_model() - tokens = torch.full((BS, GAMMA), 7) - with ( - patch.object( - ds, "draft_markov_block", return_value=(tokens, None) - ) as stock, - patch.object(torch.cuda, "is_current_stream_capturing", lambda: False), - ): - sampler = _build(model, greedy_draft=False, factory=factory) - if factory is None: - self.assertIsNone(sampler.step_logits) # sized on the first call - sampler(torch.ones(BS * GAMMA, HIDDEN), torch.arange(BS * GAMMA)) - kwargs = stock.call_args.kwargs - self.assertEqual( - kwargs.keys(), - { - "markov_head", - "base_logits", - "anchor_tokens", - "draft_hidden", - "step_temperatures", - "noise", - "tp_sync", - "logits_out", - "greedy_draft", - }, - ) - self.assertEqual(kwargs["base_logits"].shape, (BS, GAMMA, VOCAB)) - self.assertTrue( - torch.equal(kwargs["anchor_tokens"], torch.arange(0, 20, 5)) - ) - self.assertEqual( - kwargs["logits_out"].data_ptr(), sampler.step_logits.data_ptr() - ) - self.assertTrue(sampler.out[: BS * GAMMA].eq(7).all()) - features = model.compute_confidence.call_args.kwargs["confidence_features"] - self.assertTrue(features.eq(2).all()) - - def test_every_rank_serves_the_same_draft_batch_sizes(self): - """A rank whose megakernel declines a size runs SGLang's block, whose per-step - token sync its peers on the megakernel step never enter: the sizes must agree, - over the group that sync uses.""" - agree = Mock() - factory = Mock(return_value=lambda bs: "step" if bs in (1, BS) else None) - _build(_folded_draft_model(), greedy_draft=True, factory=factory, agree=agree) + def test_the_megakernel_step_runs_only_on_the_batch_sizes_every_rank_serves(self): + """A served batch size runs the megakernel's step in place of the base + logits and the Markov block, writing SGLang's buffers; every other size runs + SGLang's block. The served sizes are agreed over the token-sync group, since + a rank on SGLang's block waits in a sync its peers never enter.""" + hidden, input_ids = torch.randn(BS * GAMMA, HIDDEN), torch.arange(BS * GAMMA) + step = Mock(side_effect=lambda h, anchor: ("tokens", h * 2)) + factory = Mock(return_value=lambda bs: step if bs == BS else None) + agree, model = Mock(), _folded_draft_model() + with patch.object(ds, "draft_markov_block", side_effect=AssertionError): + sampler = _build(model, greedy_draft=False, factory=factory, agree=agree) + sampler(hidden, input_ids) agree.assert_called_once_with( - (1, BS), what="draft sampler batch sizes", group="draft tp group" + (BS,), what="draft sampler batch sizes", group="draft tp group" ) + model.compute_base_logits.assert_not_called() + self.assertIs(factory.call_args.kwargs["draft_tokens"], sampler.out) + self.assertIs(factory.call_args.kwargs["step_logits"], sampler.step_logits) + self.assertTrue(torch.equal(step.call_args.args[1], input_ids[::GAMMA])) - def test_ranks_serving_different_draft_batch_sizes_are_refused(self): - from sglang.srt.megakernels import runtime - - factory = Mock(return_value=lambda bs: "step" if bs == 1 else None) - group = SimpleNamespace(world_size=2, cpu_group=None) - - def gather(values, value, group): - values[:] = [value, (1, 2)] # the other rank also serves batch 2 - + declined = Mock(return_value=lambda bs: None) + tokens = torch.full((BS, GAMMA), 7) with ( - patch.object(runtime.dist, "all_gather_object", gather), - patch.object(ds, "megakernel_draft_sampler", return_value=factory), - patch.object(ds, "get_dp_tp_group", return_value=group), - self.assertRaisesRegex(RuntimeError, r"rank 1: \(1, 2\)"), + patch.object( + ds, "draft_markov_block", return_value=(tokens, None) + ) as stock, + patch.object(torch.cuda, "is_current_stream_capturing", lambda: False), ): - ds.maybe_build_draft_sampler( - draft_model=_folded_draft_model(), - gamma=GAMMA, - max_bs=2 * BS, - device="cpu", - tp_rank=0, - noise=DraftNoise(key=3, offset=torch.zeros(1, dtype=torch.int64)), - greedy_draft=True, - tp_sync="sync", + sampler = _build( + _folded_draft_model(), greedy_draft=False, factory=declined ) + sampler(torch.ones(BS * GAMMA, HIDDEN), input_ids) + stock.assert_called_once() + self.assertTrue(sampler.out[: BS * GAMMA].eq(7).all()) if __name__ == "__main__": From 5ba9bb2dc23ff27696aa209e932f53f40ba76c4a Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Mon, 19 Oct 2026 00:26:35 -0700 Subject: [PATCH 09/10] [Feat] [NVIDIA] docker: install the pinned sgl-nvidia-mega plugin in the CUDA images Install sgl-nvidia-mega, the plugin behind --enable-nvidia-megakernel, in the framework_final stage of docker/Dockerfile and docker/Dockerfile.cu134, as docker/rocm.Dockerfile does for sgl-amd-mono: after SGLang, clone it, check out the pinned SGL_NVIDIA_MEGA_COMMIT (sgl-nvidia-mega main, d47deb3), install it editable with --no-deps, and assert that its nvidia_megakernel entry point is registered. The runtime stage copies site-packages and /sgl-workspace from framework_final, so it ships the plugin too. Scope: the plugin's SM103 kernels compile at runtime, so the step skips with a message when the image's CUDA_VERSION is below 12.9 (both recipes build CUDA 13.x today). It is not limited by architecture: GB300 is arm64, B300 is x86_64. The plugin does nothing unless a server passes the flag. Opt out with --build-arg ENABLE_SGL_NVIDIA_MEGA=0. --- docker/Dockerfile | 20 ++++++++++++++++++++ docker/Dockerfile.cu134 | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index aeb6d29dfe1b..cfee6ba207c2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -626,6 +626,26 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --reinstall /tmp/gateway_wheels/*.whl \ && rm -rf /tmp/gateway_wheels +# sgl-nvidia-mega: the NVIDIA megakernel plugin, inert unless a server passes +# --enable-nvidia-megakernel. Installed with CUDA >= 12.9 only (its kernels are +# SM103, compiled at runtime); disable with --build-arg ENABLE_SGL_NVIDIA_MEGA=0. +ARG ENABLE_SGL_NVIDIA_MEGA=1 +ARG SGL_NVIDIA_MEGA_REPO="https://github.com/sgl-project/sgl-nvidia-mega.git" +ARG SGL_NVIDIA_MEGA_COMMIT="d47deb3390e14741c9420ac702954e7611a14da1" +RUN --mount=type=cache,target=/root/.cache/uv \ + if [ "${ENABLE_SGL_NVIDIA_MEGA}" != "1" ]; then \ + echo "[sgl-nvidia-mega] skip (ENABLE_SGL_NVIDIA_MEGA=${ENABLE_SGL_NVIDIA_MEGA})"; \ + elif ! printf '%s\n' 12.9 "${CUDA_VERSION}" | sort -V -C; then \ + echo "[sgl-nvidia-mega] skip (CUDA_VERSION=${CUDA_VERSION} < 12.9)"; \ + else \ + git clone "${SGL_NVIDIA_MEGA_REPO}" /sgl-workspace/sgl-nvidia-mega \ + && cd /sgl-workspace/sgl-nvidia-mega \ + && git checkout -f "${SGL_NVIDIA_MEGA_COMMIT}" \ + && uv pip install --no-deps -e . \ + && cd / \ + && python3 -c "from importlib.metadata import entry_points; assert any(e.name == 'nvidia_megakernel' for e in entry_points(group='sglang.srt.plugins')), 'sgl-nvidia-mega plugin not registered'"; \ + fi + # Set workspace directory WORKDIR /sgl-workspace/sglang diff --git a/docker/Dockerfile.cu134 b/docker/Dockerfile.cu134 index 9a3c4f09f494..54f3921c42dc 100644 --- a/docker/Dockerfile.cu134 +++ b/docker/Dockerfile.cu134 @@ -877,6 +877,26 @@ RUN --mount=type=cache,target=/root/.cache/pip \ python3 -m pip install --force-reinstall /tmp/gateway_wheels/*.whl \ && rm -rf /tmp/gateway_wheels +# sgl-nvidia-mega: the NVIDIA megakernel plugin, inert unless a server passes +# --enable-nvidia-megakernel. Installed with CUDA >= 12.9 only (its kernels are +# SM103, compiled at runtime); disable with --build-arg ENABLE_SGL_NVIDIA_MEGA=0. +ARG ENABLE_SGL_NVIDIA_MEGA=1 +ARG SGL_NVIDIA_MEGA_REPO="https://github.com/sgl-project/sgl-nvidia-mega.git" +ARG SGL_NVIDIA_MEGA_COMMIT="d47deb3390e14741c9420ac702954e7611a14da1" +RUN --mount=type=cache,target=/root/.cache/pip \ + if [ "${ENABLE_SGL_NVIDIA_MEGA}" != "1" ]; then \ + echo "[sgl-nvidia-mega] skip (ENABLE_SGL_NVIDIA_MEGA=${ENABLE_SGL_NVIDIA_MEGA})"; \ + elif ! printf '%s\n' 12.9 "${CUDA_VERSION}" | sort -V -C; then \ + echo "[sgl-nvidia-mega] skip (CUDA_VERSION=${CUDA_VERSION} < 12.9)"; \ + else \ + git clone "${SGL_NVIDIA_MEGA_REPO}" /sgl-workspace/sgl-nvidia-mega \ + && cd /sgl-workspace/sgl-nvidia-mega \ + && git checkout -f "${SGL_NVIDIA_MEGA_COMMIT}" \ + && python3 -m pip install --no-deps -e . \ + && cd / \ + && python3 -c "from importlib.metadata import entry_points; assert any(e.name == 'nvidia_megakernel' for e in entry_points(group='sglang.srt.plugins')), 'sgl-nvidia-mega plugin not registered'"; \ + fi + # Set workspace directory WORKDIR /sgl-workspace/sglang From db395737a74d8deca48e0de0f9a4cf8ac09c7528 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Mon, 19 Oct 2026 11:17:20 -0700 Subject: [PATCH 10/10] [Test] [NVIDIA] megakernels: keep one behavioural test per failure mode Drop the AST name scan of the SGLang members the framework reads and the binding's tolerated-name enumeration; neither shows the runtime object provides the interface. Keep one test each for the disabled flag, covered-step vs fallback routing, a missing or unusable target spec vs an optional draft spec (including an empty tp_sizes at resolution and after load), rank disagreement, and no captured graph using the megakernel; the binding keeps the view-is-the-running-backend and sealed-source checks. --- test/registered/unit/megakernels/_fakes.py | 17 +-- .../megakernels/test_megakernel_binding.py | 15 +-- .../megakernels/test_megakernel_contract.py | 75 ----------- .../megakernels/test_megakernel_guards.py | 117 ++++-------------- .../megakernels/test_megakernel_runtime.py | 66 +++------- 5 files changed, 50 insertions(+), 240 deletions(-) delete mode 100644 test/registered/unit/megakernels/test_megakernel_contract.py diff --git a/test/registered/unit/megakernels/_fakes.py b/test/registered/unit/megakernels/_fakes.py index 0042d6b3897c..31308d8e7b88 100644 --- a/test/registered/unit/megakernels/_fakes.py +++ b/test/registered/unit/megakernels/_fakes.py @@ -52,7 +52,6 @@ def _build(model, hf_config, context): return FakeBuilt(context.widths) -# what the post-load check reads; argument resolution also checks STRICT's extras SPEC = MegakernelSpec( name="fake", requires=Requirements(isas=frozenset({ISA}), tp_sizes=frozenset({4})), @@ -62,20 +61,8 @@ def _build(model, hf_config, context): build=_build, api_version=registry.MEGAKERNEL_API_VERSION, ) -STRICT = replace( - SPEC, - requires=replace( - SPEC.requires, - dtypes=frozenset({"bfloat16"}), - server_args={"kv_cache_dtype": frozenset({"fp8_e4m3"})}, - hf_config={"model_type": "fake"}, - env={"SGLANG_DSPARK_EMBED_IN_GRAPH": True}, - ), - config_disable_reasons=lambda cfg, hf: ( - ["context_length too long"] if cfg.context_length == 7 else [] - ), -) -ANY_TP = replace(STRICT, requires=replace(STRICT.requires, tp_sizes=frozenset())) +# an empty field accepts any value +ANY_TP = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset())) # a draft spec: builds no width, offers only the draft sampler DRAFT = replace( SPEC, diff --git a/test/registered/unit/megakernels/test_megakernel_binding.py b/test/registered/unit/megakernels/test_megakernel_binding.py index 72f7d7d51bed..f7fc62e43682 100644 --- a/test/registered/unit/megakernels/test_megakernel_binding.py +++ b/test/registered/unit/megakernels/test_megakernel_binding.py @@ -1,6 +1,5 @@ -"""CPU contract of the megakernel binding: the running forward's attention data and -KV pool under SGLang's own names, sources checked after capture, and the tolerated -lists naming real members.""" +"""CPU checks of the megakernel binding: the view is the running forward's attention +backend and KV pool, and a source replaced after capture is caught.""" import unittest from types import SimpleNamespace @@ -48,21 +47,13 @@ def backend(pool): class TestMegakernelBinding(CustomTestCase): - def test_the_view_is_the_running_forward_and_every_tolerated_member_exists(self): + def test_the_view_is_the_running_forwards_backend_and_pool(self): pool = Pool() with patch(BACKEND, return_value=backend(pool)): view = mb.MegakernelBinding(model=torch.nn.Module()).forward_view() self.assertEqual(view.metadata.core_attn_metadata, "core") self.assertIs(view.kv_pool.pool, pool) self.assertIs(view.kv_pool.get_swa_key_buffer_radix(1), pool.keys[1]) - self.assertEqual(view.kv_pool.get_extra_key_page_size(1), 65) - self.assertEqual(view.swa_out_cache_loc(None), "swa_loc") - # a plugin pins these names: a rename here must fail first - owners = {"ForwardView": view, "KvPoolView": view.kv_pool} - for name in mb.TOLERATED_CALLABLES: - owner, _, attr = name.partition(".") - if owner in owners: - self.assertTrue(hasattr(owners[owner], attr), name) def test_a_replaced_source_fails_assert_live_by_name(self): model = torch.nn.Module() diff --git a/test/registered/unit/megakernels/test_megakernel_contract.py b/test/registered/unit/megakernels/test_megakernel_contract.py deleted file mode 100644 index 956e07a42254..000000000000 --- a/test/registered/unit/megakernels/test_megakernel_contract.py +++ /dev/null @@ -1,75 +0,0 @@ -"""A rename in SGLang that the megakernel framework reads fails here, not on a GPU.""" - -import ast -import unittest -from functools import lru_cache -from pathlib import Path - -from sglang.srt.megakernels import spec -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=10, stage="base-a", runner_config="cpu") - -ROOT = Path(spec.__file__).resolve().parents[2] # python/sglang - -# Names the framework reads that its CPU tests reach only through fakes: runner, -# batch and process-group attributes, string protocols. -READS = { - "srt/configs/model_config.py": ("dtype", "hf_config"), - "srt/distributed/parallel_state.py": ("world_size", "cpu_group"), - "srt/model_executor/forward_batch_info.py": ("batch_size", "input_ids"), - "srt/model_executor/model_runner.py": ( - "capture_tail_hooks", - "decode_cuda_graph_runner", - ), - "srt/model_executor/model_runner_components/weight_updater.py": ( - '"_derived_weight_cache_error"', - ), - "srt/model_executor/runner/decode_cuda_graph_runner.py": ("capture_bs",), - "srt/runtime_context.py": ('"tp_group"',), - "srt/model_executor/cuda_graph_config.py": ('"full"', '"disabled"'), - "srt/plugins/__init__.py": ('"sglang.srt.plugins"', "GENERAL_PLUGINS_GROUP"), - "srt/environ.py": ("SGLANG_PLUGINS", "SGLANG_RAGGED_VERIFY_MODE"), -} - - -@lru_cache(maxsize=None) -def _provided(path: Path) -> frozenset: - """Names a file provides: definitions, attributes, imports and string literals.""" - names = set() - for node in ast.walk(ast.parse(path.read_text())): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names.add(node.name) - elif isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - for target in targets: - for t in ast.walk(target): - if isinstance(t, ast.Name): - names.add(t.id) - elif isinstance(t, ast.Attribute): - names.add(t.attr) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - names.update((a.asname or a.name).split(".")[0] for a in node.names) - elif isinstance(node, ast.Constant) and isinstance(node.value, str): - names.add(f'"{node.value}"') - return frozenset(names) - - -class TestSglangReads(CustomTestCase): - def test_sglang_provides_what_the_framework_reads(self): - for pattern, names in READS.items(): - with self.subTest(pattern=pattern): - files = sorted(ROOT.glob(pattern)) - self.assertTrue(files, f"no file matches {pattern}") - provided = frozenset().union(*map(_provided, files)) - missing = [name for name in names if name not in provided] - self.assertFalse( - missing, - f"SGLang no longer provides {missing} in {pattern}; repair the " - "megakernel framework and READS", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/megakernels/test_megakernel_guards.py b/test/registered/unit/megakernels/test_megakernel_guards.py index ecfe61c452a2..746f9e07831d 100644 --- a/test/registered/unit/megakernels/test_megakernel_guards.py +++ b/test/registered/unit/megakernels/test_megakernel_guards.py @@ -1,4 +1,4 @@ -"""CPU contracts of --enable-nvidia-megakernel's startup checks and wait deadlines.""" +"""CPU checks of --enable-nvidia-megakernel's startup resolution.""" import sys import unittest @@ -6,28 +6,16 @@ from types import SimpleNamespace from unittest.mock import patch -from sglang.srt.environ import envs -from sglang.srt.megakernels import guards +from sglang.srt.megakernels import guards, runtime from sglang.srt.megakernels import spec as registry -from sglang.srt.megakernels.guards import ( - MAX_DEADLINE_S, - deadline_s, - handle_nvidia_megakernel, - megakernel_disable_reasons, -) -from sglang.srt.megakernels.spec import Feature +from sglang.srt.megakernels.guards import megakernel_disable_reasons from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase sys.path.insert(0, str(Path(__file__).resolve().parent)) -from _fakes import ARCH, STRICT, server_args # noqa: E402 +from _fakes import ARCH, ISA, server_args # noqa: E402 -register_cpu_ci(est_time=10, stage="base-a", runner_config="cpu") - - -def reasons_for(cfg, *, arch=ARCH, isa="sm_103", model_type="fake"): - hf_config = SimpleNamespace(architectures=[arch], model_type=model_type) - return megakernel_disable_reasons(cfg=cfg, hf_config=hf_config, isa=isa) +register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") class TestMegakernelGuards(CustomTestCase): @@ -35,84 +23,31 @@ def setUp(self): registry_patch = patch.dict(registry._REGISTRY, clear=True) registry_patch.start() self.addCleanup(registry_patch.stop) - registry.register(architecture=ARCH, path="_fakes:STRICT") - def test_every_refusal_reason_is_listed(self): - with envs.SGLANG_DSPARK_EMBED_IN_GRAPH.override(False): - spec, reasons = reasons_for( - server_args( - tp_size=8, - pp_size=2, - kv_cache_dtype="auto", - enable_eplb=True, - context_length=7, - watchdog_timeout=6, - ), - model_type="other", + def test_a_disabled_flag_looks_nothing_up_and_keeps_the_forward(self): + with patch.object(guards, "plugin_reasons") as plugin: + guards.handle_nvidia_megakernel(server_args()) + plugin.assert_not_called() + model = SimpleNamespace() # any attribute read would raise + with patch.object(runtime, "megakernel_enabled", return_value=False): + runtime.install_megakernel( + model=model, hf_config=None, capture_tail_hooks=[], is_draft=False ) - self.assertIs(spec, STRICT) - for want in ( - "tp_size 8", - "kv_cache_dtype 'auto'", - "config.model_type 'other'", - "SGLANG_DSPARK_EMBED_IN_GRAPH False", - Feature.EPLB.value, - Feature.PIPELINE_PARALLEL.value, - "context_length", - "--watchdog-timeout", - ): - self.assertTrue(any(want in r for r in reasons), (want, reasons)) - with envs.SGLANG_DSPARK_EMBED_IN_GRAPH.override(True): - self.assertEqual(reasons_for(server_args())[1], []) - self.assertEqual(reasons_for(server_args(), isa=None)[1], ["no GPU is visible"]) - for arch, isa, why in ( - ("Other", "sm_103", "Other"), - (ARCH, "sm_100", "sm_100"), - ): - spec, reasons = reasons_for(server_args(), arch=arch, isa=isa) - self.assertIsNone(spec) - self.assertIn(why, reasons[0]) - # an empty field accepts any value - registry._REGISTRY.clear() - registry.register(architecture=ARCH, path="_fakes:ANY_TP") - tp8 = reasons_for(server_args(tp_size=8))[1] - self.assertFalse(any("tp_size" in r for r in tp8), tp8) - # a plugin that fails to load is only logged: name it, not the model - registry._REGISTRY.clear() - self.assertIn("registered no spec", reasons_for(server_args())[1][0]) - with patch.object(guards, "entry_points", return_value=[]): - self.assertIn("install sgl-nvidia-mega", guards.plugin_reasons()[0]) + self.assertEqual(vars(model), {}) - def test_the_resolution_step_raises_every_reason_or_nothing(self): - with patch.object(guards, "plugin_reasons") as plugin: - handle_nvidia_megakernel(server_args()) # flag off: nothing is looked up - plugin.assert_not_called() - on = server_args(enable_nvidia_megakernel=True, tp_size=8) - with ( - patch.object(guards, "plugin_reasons", return_value=[]), - patch.object(guards, "device_isa", return_value="sm_103"), - patch.object(guards, "megakernel_disable_reasons") as reasons, - patch.object(guards, "model_config_of") as model_config, - ): - model_config.return_value = SimpleNamespace(hf_config=None, dtype="float16") - reasons.return_value = (STRICT, ["tp_size 8 is not one of ['4']"]) - # the resolved dtype is checked here, and listed with the other reasons - with self.assertRaisesRegex( - ValueError, "(?s)tp_size 8.*\n - dtype 'float16'" - ): - handle_nvidia_megakernel(on) - model_config.return_value = SimpleNamespace( - hf_config=None, dtype="bfloat16" + def test_an_unmet_tp_size_is_refused_and_an_empty_field_accepts_any(self): + hf_config = SimpleNamespace(architectures=[ARCH], model_type="fake") + + def tp_reasons(path): + registry._REGISTRY.clear() + registry.register(architecture=ARCH, path=path) + _, reasons = megakernel_disable_reasons( + cfg=server_args(tp_size=8), hf_config=hf_config, isa=ISA ) - reasons.return_value = (STRICT, []) - handle_nvidia_megakernel(on) - # both flags replace the model's forward: refused before any weight loads - both = server_args(enable_nvidia_megakernel=True, monokernel_backend="x") - with self.assertRaisesRegex(ValueError, "--monokernel-backend x"): - handle_nvidia_megakernel(both) - # every wait's deadline stays below the watchdog - self.assertEqual(deadline_s(None), MAX_DEADLINE_S) - self.assertEqual(deadline_s(30), 15.0) + return [r for r in reasons if "tp_size" in r] + + self.assertTrue(tp_reasons("_fakes:SPEC")) + self.assertEqual(tp_reasons("_fakes:ANY_TP"), []) if __name__ == "__main__": diff --git a/test/registered/unit/megakernels/test_megakernel_runtime.py b/test/registered/unit/megakernels/test_megakernel_runtime.py index dee13bf046f0..84da1bd6cae2 100644 --- a/test/registered/unit/megakernels/test_megakernel_runtime.py +++ b/test/registered/unit/megakernels/test_megakernel_runtime.py @@ -1,6 +1,5 @@ -"""CPU contracts of the megakernel install, dispatch and capture check.""" +"""CPU checks of the megakernel install, dispatch and capture check.""" -import inspect import sys import unittest from pathlib import Path @@ -18,7 +17,7 @@ from sglang.test.test_utils import CustomTestCase sys.path.insert(0, str(Path(__file__).resolve().parent)) -from _fakes import ARCH, ISA, SPEC, FakeBuilt, Lm, batch, server_args # noqa: E402 +from _fakes import ANY_TP, ARCH, ISA, SPEC, Lm, batch, server_args # noqa: E402 register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") @@ -28,14 +27,12 @@ def install( archs=(ARCH,), gathered=None, tokens=2, - graphs=True, hooks=None, - lm=None, is_draft=False, ): """install_megakernel on an Lm as rank 0 of the published server; gathered maps a gathered value to what the other ranks report, default the same as this rank.""" - lm, hooks = lm or Lm(), [] if hooks is None else hooks + lm, hooks = Lm(), [] if hooks is None else hooks gpu = SimpleNamespace(multi_processor_count=160) with ( patch.object(runtime, "device_isa", return_value=ISA), @@ -44,7 +41,7 @@ def install( runtime, "_gather_tp", side_effect=gathered or (lambda x, group=None: [x]) ), patch.object(runtime, "_tokens_per_request", return_value=tokens), - patch.object(runtime, "_decode_graphs_enabled", return_value=graphs), + patch.object(runtime, "_decode_graphs_enabled", return_value=True), patch.object(torch.cuda, "current_device", return_value=0), patch.object(torch.cuda, "get_device_properties", return_value=gpu), patch.object(torch.cuda, "mem_get_info", return_value=(0, 0)), @@ -78,13 +75,6 @@ def _install_spec(self, spec, **kw): return install(**kw) def test_covered_steps_run_the_megakernel_and_others_the_forward(self): - model = SimpleNamespace() # without the flag, any attribute read would raise - with patch.object(runtime, "megakernel_enabled", return_value=False): - runtime.install_megakernel( - model=model, hf_config=None, capture_tail_hooks=[], is_draft=False - ) - self.assertEqual(vars(model), {}) - self.assertEqual(install(tokens=2).context.widths, (2, 4, 6, 8)) lm = install(tokens=1) built = runtime.built_megakernel(lm) with patch.object( @@ -94,41 +84,28 @@ def test_covered_steps_run_the_megakernel_and_others_the_forward(self): self.assertEqual(lm(batch(6).input_ids, None, batch(6)), "standard") # only a covered step runs the megakernel, with the running forward's view self.assertEqual((built.steps, built.views), ([4], ["view"])) - self.assertIsInstance(lm.context.binding, MegakernelBinding) - # graph runners and supports_pp read the original forward's parameters - self.assertEqual(inspect.signature(lm.forward), inspect.signature(Lm().forward)) - self.assertIn("--enable-nvidia-megakernel", lm._derived_weight_cache_error) - with self.assertRaisesRegex(ValueError, "already replaced"): - install(lm=lm) - - def test_a_target_without_a_spec_is_refused_and_a_draft_keeps_its_forward(self): - # resolution found the target's spec but the plugin failed in every worker: + + def test_a_target_without_a_usable_spec_is_refused_and_a_draft_keeps_its_forward( + self, + ): # the flag must not silently run the model's own forward with self.assertRaisesRegex(ValueError, "in the model workers"): install(archs=["TargetWithoutSpec"]) + tp8 = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset({8}))) + with self.assertRaisesRegex(ValueError, "tp_size 4"): + self._install_spec(tp8) + # an empty field accepts any value + self.assertIsNotNone(runtime.built_megakernel(self._install_spec(ANY_TP))) with self.assertLogs(runtime.logger, "INFO") as logs: lm = install(archs=["DraftWithoutSpec"], is_draft=True) self.assertNotIn("forward", lm.__dict__) self.assertIn("keeps its own forward", "".join(logs.output)) registry.register(architecture="Draft", path="_fakes:DRAFT") - hooks = [] - lm = install(archs=["Draft"], hooks=hooks, is_draft=True) + # a draft spec offers only its draft sampler + lm = install(archs=["Draft"], is_draft=True) self.assertNotIn("forward", lm.__dict__) - self.assertEqual(hooks, []) self.assertIs(runtime.megakernel_draft_sampler(lm), print) - def test_a_spec_is_checked_on_what_the_runner_reads(self): - tp8 = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset({8}))) - with self.assertRaisesRegex(ValueError, "tp_size 4"): - self._install_spec(tp8) - # an empty field accepts any value - any_tp = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset())) - self.assertIsNotNone(runtime.built_megakernel(self._install_spec(any_tp))) - # a target that covers no width must not quietly keep the standard forward - empty = replace(SPEC, build=lambda model, hf, ctx: FakeBuilt(())) - with self.assertRaisesRegex(ValueError, "built no width of"): - self._install_spec(empty) - def test_ranks_that_disagree_are_refused_on_every_rank(self): """A rank that differs would wait in a different collective.""" @@ -139,12 +116,10 @@ def fewer_widths_on_rank_1(value, group=None): widths = isinstance(value, tuple) and all(isinstance(w, int) for w in value) return [value, value[:1]] if widths else [value] - for gathered, error, want in ( - (no_spec_on_rank_1, ValueError, "rank 1: rank-1 reason"), - (fewer_widths_on_rank_1, RuntimeError, "widths fake built differ"), - ): - with self.subTest(want=want), self.assertRaisesRegex(error, want): - install(gathered=gathered) + with self.assertRaisesRegex(ValueError, "rank 1: rank-1 reason"): + install(gathered=no_spec_on_rank_1) + with self.assertRaisesRegex(RuntimeError, "widths fake built differ"): + install(gathered=fewer_widths_on_rank_1) def test_capture_check_refuses_when_no_graph_runs_the_megakernel(self): def capture(sizes, capture_bs): @@ -162,9 +137,6 @@ def capture(sizes, capture_bs): capture([2, 8], capture_bs=[2, 8]) with self.assertRaisesRegex(ValueError, "none of the 2 captured decode graphs"): capture([8, 16], capture_bs=[8, 16]) - hooks = [] - install(graphs=False, hooks=hooks) - self.assertEqual(hooks, []) # eager steps are routed by plan alone if __name__ == "__main__":