diff --git a/docker/Dockerfile b/docker/Dockerfile
index aeb6d29dfe1b..cfee6ba207c2 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -626,6 +626,26 @@ RUN --mount=type=cache,target=/root/.cache/uv \
uv pip install --reinstall /tmp/gateway_wheels/*.whl \
&& rm -rf /tmp/gateway_wheels
+# sgl-nvidia-mega: the NVIDIA megakernel plugin, inert unless a server passes
+# --enable-nvidia-megakernel. Installed with CUDA >= 12.9 only (its kernels are
+# SM103, compiled at runtime); disable with --build-arg ENABLE_SGL_NVIDIA_MEGA=0.
+ARG ENABLE_SGL_NVIDIA_MEGA=1
+ARG SGL_NVIDIA_MEGA_REPO="https://github.com/sgl-project/sgl-nvidia-mega.git"
+ARG SGL_NVIDIA_MEGA_COMMIT="d47deb3390e14741c9420ac702954e7611a14da1"
+RUN --mount=type=cache,target=/root/.cache/uv \
+ if [ "${ENABLE_SGL_NVIDIA_MEGA}" != "1" ]; then \
+ echo "[sgl-nvidia-mega] skip (ENABLE_SGL_NVIDIA_MEGA=${ENABLE_SGL_NVIDIA_MEGA})"; \
+ elif ! printf '%s\n' 12.9 "${CUDA_VERSION}" | sort -V -C; then \
+ echo "[sgl-nvidia-mega] skip (CUDA_VERSION=${CUDA_VERSION} < 12.9)"; \
+ else \
+ git clone "${SGL_NVIDIA_MEGA_REPO}" /sgl-workspace/sgl-nvidia-mega \
+ && cd /sgl-workspace/sgl-nvidia-mega \
+ && git checkout -f "${SGL_NVIDIA_MEGA_COMMIT}" \
+ && uv pip install --no-deps -e . \
+ && cd / \
+ && python3 -c "from importlib.metadata import entry_points; assert any(e.name == 'nvidia_megakernel' for e in entry_points(group='sglang.srt.plugins')), 'sgl-nvidia-mega plugin not registered'"; \
+ fi
+
# Set workspace directory
WORKDIR /sgl-workspace/sglang
diff --git a/docker/Dockerfile.cu134 b/docker/Dockerfile.cu134
index 9a3c4f09f494..54f3921c42dc 100644
--- a/docker/Dockerfile.cu134
+++ b/docker/Dockerfile.cu134
@@ -877,6 +877,26 @@ RUN --mount=type=cache,target=/root/.cache/pip \
python3 -m pip install --force-reinstall /tmp/gateway_wheels/*.whl \
&& rm -rf /tmp/gateway_wheels
+# sgl-nvidia-mega: the NVIDIA megakernel plugin, inert unless a server passes
+# --enable-nvidia-megakernel. Installed with CUDA >= 12.9 only (its kernels are
+# SM103, compiled at runtime); disable with --build-arg ENABLE_SGL_NVIDIA_MEGA=0.
+ARG ENABLE_SGL_NVIDIA_MEGA=1
+ARG SGL_NVIDIA_MEGA_REPO="https://github.com/sgl-project/sgl-nvidia-mega.git"
+ARG SGL_NVIDIA_MEGA_COMMIT="d47deb3390e14741c9420ac702954e7611a14da1"
+RUN --mount=type=cache,target=/root/.cache/pip \
+ if [ "${ENABLE_SGL_NVIDIA_MEGA}" != "1" ]; then \
+ echo "[sgl-nvidia-mega] skip (ENABLE_SGL_NVIDIA_MEGA=${ENABLE_SGL_NVIDIA_MEGA})"; \
+ elif ! printf '%s\n' 12.9 "${CUDA_VERSION}" | sort -V -C; then \
+ echo "[sgl-nvidia-mega] skip (CUDA_VERSION=${CUDA_VERSION} < 12.9)"; \
+ else \
+ git clone "${SGL_NVIDIA_MEGA_REPO}" /sgl-workspace/sgl-nvidia-mega \
+ && cd /sgl-workspace/sgl-nvidia-mega \
+ && git checkout -f "${SGL_NVIDIA_MEGA_COMMIT}" \
+ && python3 -m pip install --no-deps -e . \
+ && cd / \
+ && python3 -c "from importlib.metadata import entry_points; assert any(e.name == 'nvidia_megakernel' for e in entry_points(group='sglang.srt.plugins')), 'sgl-nvidia-mega plugin not registered'"; \
+ fi
+
# Set workspace directory
WORKDIR /sgl-workspace/sglang
diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx
index a09038aba7f1..4a9a2622fae5 100644
--- a/docs/docs/advanced_features/server_arguments.mdx
+++ b/docs/docs/advanced_features/server_arguments.mdx
@@ -1441,6 +1441,12 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec
Choose the kernels for decode attention layers (have priority over --attention-backend). |
`None` |
triton, torch_native, flex_attention, dsa, nsa, dsv4, compressed, fa3, fa4, flashinfer, flashmla, trtllm_mla, cutedsl_mla, tokenspeed_mla, trtllm_mha, aiter, wave, intel_amx, ascend, intel_xpu |
+
+
+ | `--enable-nvidia-megakernel` |
+ Enable the NVIDIA megakernel from the sgl-nvidia-mega package. Supported decode and verify steps use its fused kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature. |
+ `False` |
+ bool flag (set to enable) |
| `--monokernel-backend` |
diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py
index 387fdc077b4a..466a587617ff 100644
--- a/python/sglang/srt/arg_groups/fields/exec_.py
+++ b/python/sglang/srt/arg_groups/fields/exec_.py
@@ -190,6 +190,10 @@ class ExecKernel(msgspec.Struct):
Optional[bool],
"Enable Lean (Work-Centric) Attention decode kernel for long-context serving. When None (default), uses auto-gate that activates Lean for long contexts and falls back to standard kernel for short contexts. Set to True to force enable, False to force disable.",
] = None
+ enable_nvidia_megakernel: A[
+ bool,
+ "Enable the NVIDIA megakernel from the sgl-nvidia-mega package. Supported decode and verify steps use its fused kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature.",
+ ] = False
monokernel_backend: A[
Optional[str],
"Select a registered monokernel backend, e.g. amd from sgl-amd-mono. Supported decode and verify steps use persistent kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature.",
diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py
index d5b84fc2d4a1..7cdab757d777 100644
--- a/python/sglang/srt/arg_groups/pipeline.py
+++ b/python/sglang/srt/arg_groups/pipeline.py
@@ -343,8 +343,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(handle_model_capability_adjustments, server_args)
# needs the resolved CUDA-graph, attention-backend and speculative settings
+ from sglang.srt.megakernels.guards import handle_nvidia_megakernel
from sglang.srt.monokernels.guards import handle_monokernel
+ # first, so a server with both kernel flags gets every reason in one error
+ run_hook(handle_nvidia_megakernel, server_args)
run_hook(handle_monokernel, server_args)
finalize_cuda_graph_prefill_max_context(server_args)
diff --git a/python/sglang/srt/arg_groups/resolution_hooks.py b/python/sglang/srt/arg_groups/resolution_hooks.py
index 79c417428438..2349b25ce6cc 100644
--- a/python/sglang/srt/arg_groups/resolution_hooks.py
+++ b/python/sglang/srt/arg_groups/resolution_hooks.py
@@ -94,6 +94,7 @@
"handle_other_validations",
"handle_model_capability_adjustments",
"handle_monokernel",
+ "handle_nvidia_megakernel",
"validate_deepep_v2_speculative_draft",
"validate_deepep_v2_dispatch_token_budget",
}
diff --git a/python/sglang/srt/megakernels/__init__.py b/python/sglang/srt/megakernels/__init__.py
new file mode 100644
index 000000000000..c0ea4256c628
--- /dev/null
+++ b/python/sglang/srt/megakernels/__init__.py
@@ -0,0 +1,7 @@
+"""NVIDIA megakernels: whole-step dispatch for out-of-tree megakernel plugins.
+
+Plugins register specs through the sglang.srt.plugins entry point. With
+--enable-nvidia-megakernel the runtime validates the configuration, builds the
+spec after the model is prepared, and routes covered steps through it; other
+steps use the model forward.
+"""
diff --git a/python/sglang/srt/megakernels/binding.py b/python/sglang/srt/megakernels/binding.py
new file mode 100644
index 000000000000..e329a2ddfd55
--- /dev/null
+++ b/python/sglang/srt/megakernels/binding.py
@@ -0,0 +1,171 @@
+# SPDX-License-Identifier: Apache-2.0
+"""The data SGLang hands a megakernel's build, forward and draft entries.
+
+The one place a megakernel reads SGLang's runtime state: the KV pool, the attention
+metadata of the running forward and the Engram tables. A spec's build gets a
+``MegakernelBinding`` as ``BuildContext.binding``; a covered forward gets the
+running forward's ``ForwardView`` as ``Step.view`` (a draft entry asks
+``binding.forward_view()``). Everything else a megakernel touches is the model it
+replaces, SGLang's kernel ops, and the reads listed below.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Optional
+
+import torch
+
+# Called by a megakernel, tolerated until offered as data (its contract test pins
+# both lists).
+TOLERATED_CALLABLES = (
+ "ForwardView.attention", # the attention backend's own forward
+ "ForwardView.store_cache", # the KV write
+ "ForwardView.swa_out_cache_loc",
+ "ForwardView.low_ratio_sources", # compressor / indexer inputs
+ "ForwardView.core_compressor",
+ "KvPoolView.set_swa_key_buffer_radix_fused_norm_rope", # the fused SWA key write
+ "KvPoolView.pool", # passed back to SGLang's own draft KV store only
+ "sglang.kernels.ops", # SGLang's kernel ops
+ "the replaced model's modules, weights and quant methods",
+)
+# Written into SGLang's objects, tolerated until SGLang owns these buffers.
+TOLERATED_WRITES = (
+ "ForwardView.metadata.q_pad_buffer",
+ "ForwardView.metadata.low_ratio_req_indices",
+ "ForwardView.metadata.low_ratio_pos_i64",
+)
+# Read, tolerated: configuration, not runtime state.
+TOLERATED_READS = (
+ "runtime_context getters (get_parallel, get_exec, get_spec, get_platform)",
+ "capture-mode queries (get_is_capture_mode, is_in_breakable_cuda_graph)",
+ "MegakernelBinding.engram_host_table", # the per-layer host table object
+)
+
+
+class KvPoolView:
+ """The running forward's KV pool, under the pool's own names: its layout and
+ per-layer key buffers (a pool offers the ones of its kind)."""
+
+ def __init__(self, pool: Any):
+ self.pool = pool
+
+ @property
+ def uniform_fp8(self) -> bool:
+ return self.pool.uniform_fp8
+
+ @property
+ def request_window(self) -> Any:
+ return self.pool.request_window
+
+ def get_swa_key_buffer_radix(self, layer_id: int) -> torch.Tensor:
+ return self.pool.get_swa_key_buffer_radix(layer_id)
+
+ def get_swa_key_page_size(self) -> int:
+ return self.pool.get_swa_key_page_size()
+
+ def get_swa_key_bytes_per_token(self) -> int:
+ return self.pool.get_swa_key_bytes_per_token()
+
+ def get_extra_key_buffer(self, layer_id: int) -> Optional[torch.Tensor]:
+ return self.pool.get_extra_key_buffer(layer_id)
+
+ def get_extra_key_page_size(self, layer_id: int) -> int:
+ return self.pool.get_extra_key_page_size(layer_id)
+
+ def get_extra_key_bytes_per_token(self, layer_id: int) -> int:
+ return self.pool.get_extra_key_bytes_per_token(layer_id)
+
+ @property
+ def set_swa_key_buffer_radix_fused_norm_rope(self) -> Callable[..., None]:
+ return self.pool.set_swa_key_buffer_radix_fused_norm_rope
+
+
+class ForwardView:
+ """The running forward's attention data, read from its attention backend; built
+ at capture for a graph step, so its tensors are the ones the graph replays."""
+
+ def __init__(self, backend: Any):
+ self._backend = backend
+ self.metadata = backend.forward_metadata
+ self.kv_pool = KvPoolView(backend.token_to_kv_pool)
+
+ @property
+ def trtllm_attn(self) -> bool:
+ return self._backend.trtllm_attn
+
+ @property
+ def attention(self) -> Callable[..., Any]:
+ return self._backend.forward
+
+ @property
+ def store_cache(self) -> Callable[..., None]:
+ return self._backend.store_cache
+
+ @property
+ def swa_out_cache_loc(self) -> Callable[[Any], torch.Tensor]:
+ return self._backend.get_swa_out_cache_loc
+
+ @property
+ def low_ratio_sources(self) -> Callable[..., Any]:
+ return self._backend.forward_low_ratio_sources
+
+ @property
+ def core_compressor(self) -> Callable[..., Any]:
+ return self._backend.forward_core_compressor
+
+
+class MegakernelBinding:
+ """Built at install, before the KV pools exist; read at build or forward. Each
+ source is sealed at the first read that finds it; ``assert_live`` (after the
+ decode graphs are captured) raises once one was reallocated."""
+
+ def __init__(self, *, model: torch.nn.Module):
+ self._model = model
+ self._sealed: dict[str, tuple] = {}
+
+ def forward_view(self) -> ForwardView:
+ """The view of the forward running now (a target step or a draft step)."""
+ from sglang.srt.model_executor.forward_context import get_attn_backend
+
+ view = ForwardView(get_attn_backend())
+ self._seal("kv_pool", view.kv_pool.pool)
+ return view
+
+ def engram_history(self) -> Optional[torch.Tensor]:
+ """The model's Engram hash history ``[slots + 1, width]`` int32, or None."""
+ history = self._history()
+ if history is None or history.shape[1] == 0:
+ return None
+ self._seal("engram_history", history)
+ return history
+
+ def engram_host_table(self, engram: torch.nn.Module) -> Any:
+ """The host table of one of the model's Engram layers, or None."""
+ return engram.embed.host_table
+
+ def assert_live(self) -> None:
+ current = {"kv_pool": self._current_pool, "engram_history": self._history}
+ for name, sealed in self._sealed.items():
+ if _identity(current[name]()) != sealed:
+ raise RuntimeError(
+ f"megakernel binding: {name} was reallocated after its first read"
+ )
+
+ def _seal(self, name: str, value: Any) -> None:
+ self._sealed.setdefault(name, _identity(value))
+
+ def _current_pool(self) -> Any:
+ from sglang.srt.model_executor.forward_context import get_attn_backend
+
+ return get_attn_backend().token_to_kv_pool
+
+ def _history(self) -> Optional[torch.Tensor]:
+ # only models with Engram layers have a hasher
+ hasher = getattr(self._model, "engram_hasher", None)
+ return None if hasher is None else hasher.history
+
+
+def _identity(value: Any) -> tuple:
+ if isinstance(value, torch.Tensor):
+ return (value.data_ptr(), tuple(value.shape), value.dtype)
+ return (id(value),)
diff --git a/python/sglang/srt/megakernels/guards.py b/python/sglang/srt/megakernels/guards.py
new file mode 100644
index 000000000000..47a3b71332a1
--- /dev/null
+++ b/python/sglang/srt/megakernels/guards.py
@@ -0,0 +1,196 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Startup checks for --enable-nvidia-megakernel on the resolved configuration."""
+
+from __future__ import annotations
+
+import logging
+from importlib.metadata import entry_points
+from typing import Any, Optional
+
+import torch
+
+from sglang.srt.arg_groups.model_override_base import model_config_of, resolving_view
+from sglang.srt.environ import envs
+from sglang.srt.megakernels.spec import (
+ ENTRY_POINT,
+ FLAG,
+ Feature,
+ MegakernelSpec,
+ find_spec,
+ registered_architectures,
+)
+from sglang.srt.plugins import GENERAL_PLUGINS_GROUP
+
+logger = logging.getLogger(__name__)
+
+PLUGIN_URL = "https://github.com/sgl-project/sgl-nvidia-mega"
+
+# Leave time for peers to abort before the process watchdog fires.
+MAX_DEADLINE_S, MIN_DEADLINE_S = 60.0, 5.0
+
+
+def device_isa() -> Optional[str]:
+ """This process's GPU as "sm_", or None without a visible GPU."""
+ if not torch.cuda.is_available():
+ return None
+ props = torch.cuda.get_device_properties(torch.cuda.current_device())
+ return f"sm_{props.major}{props.minor}"
+
+
+def deadline_s(watchdog_timeout: Optional[float]) -> float:
+ """The deadline of every cross-GPU wait: half the watchdog, at most MAX_DEADLINE_S."""
+ if not watchdog_timeout or watchdog_timeout <= 0:
+ return MAX_DEADLINE_S
+ return min(MAX_DEADLINE_S, 0.5 * float(watchdog_timeout))
+
+
+def _ragged_verify(cfg: Any) -> bool:
+ mode = envs.SGLANG_RAGGED_VERIFY_MODE.get().lower()
+ return cfg.speculative_algorithm is not None and mode != "static"
+
+
+# One probe a feature, on the resolved settings; a spec accepts features by listing
+# them. PARTITIONED_GPU needs the loaded device, so the runtime checks it.
+FEATURE_PROBES = {
+ Feature.PIPELINE_PARALLEL: lambda c: c.pp_size > 1,
+ Feature.MULTI_NODE_TP: lambda c: c.nnodes > 1,
+ Feature.DP_ATTENTION: lambda c: (
+ bool(c.enable_dp_attention) or (c.attn_dp_size or 1) > 1
+ ),
+ Feature.EXPERT_PARALLEL: lambda c: (c.ep_size or 1) > 1,
+ Feature.EPLB: lambda c: bool(c.enable_eplb),
+ Feature.CONTEXT_PARALLEL: lambda c: (
+ (c.attn_cp_size or 1) > 1 or (c.dcp_size or 1) > 1 or bool(c.enable_prefill_cp)
+ ),
+ Feature.SPECULATIVE_DECODING: lambda c: c.speculative_algorithm is not None,
+ Feature.RAGGED_VERIFY: _ragged_verify,
+ Feature.TWO_BATCH_OVERLAP: lambda c: bool(
+ c.enable_two_batch_overlap or c.enable_single_batch_overlap
+ ),
+ Feature.LORA: lambda c: bool(c.enable_lora or c.lora_paths),
+ Feature.PD_DISAGGREGATION: lambda c: (c.disaggregation_mode or "null") != "null",
+ Feature.RETURN_ROUTED_EXPERTS: lambda c: bool(c.enable_return_routed_experts),
+ Feature.RETURN_INDEXER_TOPK: lambda c: bool(c.enable_return_indexer_topk),
+ Feature.KV_OFFLOAD: lambda c: bool(
+ c.enable_hierarchical_cache
+ or c.disaggregation_decode_enable_offload_kvcache
+ or c.enable_lmcache
+ or c.enable_flexkv
+ ),
+ Feature.HISPARSE: lambda c: bool(c.enable_hisparse),
+ Feature.MEMORY_RELEASE: lambda c: bool(
+ c.enable_memory_saver or c.enable_weights_cpu_backup
+ ),
+ Feature.CPU_OFFLOAD: lambda c: (
+ (c.cpu_offload_gb or 0) > 0 or (c.offload_group_size or 0) > 0
+ ),
+ Feature.TORCH_COMPILE: lambda c: bool(c.enable_torch_compile),
+ Feature.DETERMINISTIC_INFERENCE: lambda c: bool(
+ c.enable_deterministic_inference or c.rl_on_policy_target
+ ),
+}
+
+
+def features_in_use(cfg: Any) -> set[Feature]:
+ return {feature for feature, probe in FEATURE_PROBES.items() if probe(cfg)}
+
+
+def plugin_reasons() -> list[str]:
+ """Why the plugin that registers the specs will not load ([] when it will)."""
+ if ENTRY_POINT not in {ep.name for ep in entry_points(group=GENERAL_PLUGINS_GROUP)}:
+ return [
+ f"the {ENTRY_POINT!r} plugin is not installed; install sgl-nvidia-mega "
+ f"({PLUGIN_URL})"
+ ]
+ allowed = envs.SGLANG_PLUGINS.get()
+ if allowed and ENTRY_POINT not in {x.strip() for x in allowed.split(",")}:
+ return [f"SGLANG_PLUGINS={allowed!r} does not list the {ENTRY_POINT!r} plugin"]
+ return []
+
+
+def megakernel_disable_reasons(
+ *, cfg: Any, hf_config: Any, isa: Optional[str]
+) -> tuple[Optional[MegakernelSpec], list[str]]:
+ """The spec for this model and every reason it cannot serve this configuration."""
+ if isa is None:
+ return None, ["no GPU is visible"]
+ spec, why = find_spec(architectures=hf_config.architectures, isa=isa)
+ if spec is None and not registered_architectures():
+ # the plugin loader logs and skips a plugin that fails, so nothing registered
+ why = (
+ f"the {ENTRY_POINT!r} plugin is installed but registered no spec; see "
+ "'Failed to load plugin' or 'Failed to execute general plugin' above"
+ )
+ if spec is None:
+ return None, [why]
+ req = spec.requires
+ reasons = [
+ f"{f.value} is in use and {spec.name} does not accept it"
+ for f in sorted(features_in_use(cfg) - spec.features, key=lambda f: f.name)
+ ]
+ one_of = [
+ ("tp_size", cfg.tp_size, req.tp_sizes),
+ (
+ "decode CUDA graphs",
+ cfg.cuda_graph_config.decode.backend,
+ req.decode_graph_backends,
+ ),
+ ] + [(arg, getattr(cfg, arg), ok) for arg, ok in req.server_args.items()]
+ exactly = [
+ # another model's config may not have the field at all
+ (f"config.{key}", getattr(hf_config, key, None), want)
+ for key, want in req.hf_config.items()
+ ] + [(name, getattr(envs, name).get(), want) for name, want in req.env.items()]
+ reasons += [
+ f"{what} {value!r} is not one of {sorted(map(str, allowed))}"
+ for what, value, allowed in one_of
+ if allowed and value not in allowed
+ ]
+ reasons += [
+ f"{what} {value!r} is not {want!r}"
+ for what, value, want in exactly
+ if value != want
+ ]
+ watchdog = cfg.watchdog_timeout
+ if watchdog and deadline_s(watchdog) < MIN_DEADLINE_S:
+ reasons.append(
+ f"--watchdog-timeout {watchdog:g} s leaves the waits a deadline under "
+ f"{MIN_DEADLINE_S:g} s"
+ )
+ if spec.config_disable_reasons is not None:
+ reasons += spec.config_disable_reasons(cfg, hf_config)
+ return spec, reasons
+
+
+def _dtype_reasons(spec: Optional[MegakernelSpec], model_config: Any) -> list[str]:
+ # the resolved dtype, which the HF config alone does not give
+ dtype = str(model_config.dtype).replace("torch.", "")
+ if spec is None or not spec.requires.dtypes or dtype in spec.requires.dtypes:
+ return []
+ return [f"dtype {dtype!r} is not one of {sorted(spec.requires.dtypes)}"]
+
+
+def handle_nvidia_megakernel(server_args: Any) -> None:
+ """Resolution step: fail before the weights load when the flag cannot be served,
+ listing every reason."""
+ cfg = resolving_view(server_args)
+ if not cfg.enable_nvidia_megakernel:
+ return
+ reasons = plugin_reasons()
+ if not reasons:
+ model_config = model_config_of(server_args)
+ spec, reasons = megakernel_disable_reasons(
+ cfg=cfg, hf_config=model_config.hf_config, isa=device_isa()
+ )
+ reasons += _dtype_reasons(spec, model_config)
+ if cfg.monokernel_backend is not None:
+ # both replace the model's forward, which would fail only after weights load
+ reasons.append(
+ f"--monokernel-backend {cfg.monokernel_backend} is also set; use one of the two"
+ )
+ if reasons:
+ raise ValueError(
+ f"{FLAG} cannot run this setup:\n - " + "\n - ".join(reasons)
+ )
+ logger.warning("%s is experimental. Its kernels and coverage may change.", FLAG)
+ logger.info("%s: %s on %s", FLAG, spec.name, device_isa())
diff --git a/python/sglang/srt/megakernels/runtime.py b/python/sglang/srt/megakernels/runtime.py
new file mode 100644
index 000000000000..64a2369e2a6e
--- /dev/null
+++ b/python/sglang/srt/megakernels/runtime.py
@@ -0,0 +1,341 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Build and whole-step dispatch for --enable-nvidia-megakernel."""
+
+from __future__ import annotations
+
+import functools
+import logging
+import time
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+import msgspec
+import torch
+import torch.distributed as dist
+
+from sglang.srt.environ import envs
+from sglang.srt.megakernels.binding import MegakernelBinding
+from sglang.srt.megakernels.guards import deadline_s, device_isa
+from sglang.srt.megakernels.spec import (
+ FLAG,
+ BuildContext,
+ BuiltMegakernel,
+ Feature,
+ MegakernelSpec,
+ Step,
+ find_spec,
+ spec_registered,
+)
+from sglang.srt.model_executor.cuda_graph_config import Backend
+from sglang.srt.runtime_context import (
+ get_context,
+ get_device,
+ get_exec,
+ get_parallel,
+ get_spec,
+)
+
+if TYPE_CHECKING:
+ from transformers import PretrainedConfig
+
+ from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+ from sglang.srt.model_executor.runner.decode_cuda_graph_runner import (
+ DecodeCudaGraphRunner,
+ )
+
+logger = logging.getLogger(__name__)
+
+
+def megakernel_enabled() -> bool:
+ """Whether this server runs with --enable-nvidia-megakernel."""
+ # the model loader also runs without a server's config (tools, unit tests)
+ if not get_context().is_config_namespace_published("exec"):
+ return False
+ return get_exec().kernel.enable_nvidia_megakernel
+
+
+def install_megakernel(
+ *,
+ model: torch.nn.Module,
+ hf_config: PretrainedConfig,
+ capture_tail_hooks: list[Callable[..., None]],
+ is_draft: bool,
+) -> None:
+ """Build the model's spec and install its forward; runs before KV cache sizing."""
+ if not megakernel_enabled():
+ return
+ spec = _find_spec_on_every_rank(hf_config=hf_config, is_draft=is_draft)
+ if spec is None:
+ return
+ # Resolution checked the target's spec on the server arguments; a draft's spec
+ # is checked only here, on what the runner reads (no server-argument checks).
+ reasons = _requirement_reasons(spec=spec, hf_config=hf_config)
+ reasons += _model_disable_reasons(spec=spec, model=model, hf_config=hf_config)
+ _refuse(_gather_reasons(reasons))
+
+ tokens = _tokens_per_request()
+ binding = MegakernelBinding(model=model)
+ context = BuildContext(
+ widths=tuple(range(tokens, spec.max_rows + 1, tokens)),
+ tokens_per_request=tokens,
+ deadline_s=deadline_s(get_device().watchdog_timeout),
+ binding=binding,
+ )
+ start, free_before = time.perf_counter(), torch.cuda.mem_get_info()[0]
+ built = spec.build(model, hf_config, context)
+ # not setattr: a built object that is an nn.Module must not join the state dict
+ model.__dict__["_megakernel"] = built
+ # Different widths would leave ranks waiting in different collectives.
+ megakernel_agree(tuple(built.widths), what=f"widths {spec.name} built")
+ if spec.max_rows > 0 and not built.widths:
+ _refuse([f"{spec.name} built no width of {list(context.widths)}"])
+ # Captured kernels keep state derived from the weights that updates cannot refresh.
+ model._derived_weight_cache_error = (
+ f"Online weight updates are not supported with {FLAG} ({spec.name})"
+ )
+ built_s = time.perf_counter() - start
+ used_mib = (free_before - torch.cuda.mem_get_info()[0]) / 2**20
+ if not built.widths:
+ # a spec that covers no step, e.g. a draft spec offering draft_sampler only
+ logger.info(
+ "NVIDIA megakernel: %s: %s",
+ spec.name,
+ "draft sampler"
+ if built.draft_sampler is not None
+ else "no step and no draft sampler",
+ )
+ return
+ logger.info(
+ "NVIDIA megakernel: active (%s: %s; widths %s; %s numerics; built in %.1f s, "
+ "%.1f MiB; wait deadline %.0f s)",
+ spec.name,
+ built.summary(),
+ list(built.widths),
+ spec.numerics.kind.value,
+ built_s,
+ used_mib,
+ context.deadline_s,
+ )
+ dispatch = _Dispatch(built=built, original=model.forward, binding=binding)
+ model.forward = dispatch.wrapped()
+ if _decode_graphs_enabled():
+ capture_tail_hooks.append(_CaptureCheck(spec=spec, dispatch=dispatch))
+
+
+def built_megakernel(model: torch.nn.Module) -> Optional[BuiltMegakernel]:
+ """The object the spec built for this model, or None."""
+ return model.__dict__.get("_megakernel")
+
+
+def megakernel_agree(value: Any, *, what: str, group: Any = None) -> None:
+ """Raise on every rank unless every rank of group (default: the TP group) holds
+ the same value; ranks that differ would wait in different collectives."""
+ values = _gather_tp(value, group=group)
+ if any(v != value for v in values):
+ raise RuntimeError(
+ f"{FLAG}: the ranks' {what} differ: "
+ + "; ".join(f"rank {rank}: {v}" for rank, v in enumerate(values))
+ )
+
+
+def megakernel_draft_sampler(draft_model: torch.nn.Module) -> Optional[Callable]:
+ """The built draft spec's draft_sampler factory, or None for SGLang's proposal."""
+ built = built_megakernel(draft_model)
+ return None if built is None else built.draft_sampler
+
+
+def _find_spec_on_every_rank(
+ *, hf_config: PretrainedConfig, is_draft: bool
+) -> Optional[MegakernelSpec]:
+ """The spec every rank found; for a draft model, None when no rank has one
+ registered; else raise on every rank."""
+ archs = hf_config.architectures
+ spec, why = find_spec(architectures=archs, isa=device_isa())
+ # Ranks branch together; one skipping the collectives below would hang its peers.
+ lookups = _gather_tp(
+ (
+ spec_registered(archs),
+ why,
+ None if spec is None else (spec.name, spec.api_version),
+ )
+ )
+ if not any(is_registered for is_registered, _, _ in lookups):
+ if is_draft:
+ # A draft model may have no registered spec.
+ logger.info("NVIDIA megakernel: %s; %s keeps its own forward", why, archs)
+ return None
+ # resolution found the target's spec, so the plugin failed to load here
+ _refuse(
+ [
+ f"{why} in the model workers, though argument resolution found it; "
+ "see 'Failed to load plugin' or 'Failed to execute general plugin' "
+ "above"
+ ]
+ )
+ _refuse(
+ [
+ f"rank {rank}: {why}"
+ for rank, (_, why, identity) in enumerate(lookups)
+ if identity is None
+ ]
+ )
+ if len({identity for *_, identity in lookups}) > 1:
+ raise ValueError(
+ f"{FLAG}: the ranks load different specs: "
+ + "; ".join(f"rank {r}: {ident}" for r, (*_, ident) in enumerate(lookups))
+ )
+ return spec
+
+
+def _requirement_reasons(
+ *, spec: MegakernelSpec, hf_config: PretrainedConfig
+) -> list[str]:
+ """The spec's requirements this runner can check without the server arguments."""
+ req, tp_size = spec.requires, get_parallel().tp_size
+ reasons = []
+ # an empty set accepts any TP size, as at argument resolution
+ if req.tp_sizes and tp_size not in req.tp_sizes:
+ reasons.append(f"tp_size {tp_size} is not one of {sorted(req.tp_sizes)}")
+ graph = get_exec().graph.cuda_graph_config
+ backend = Backend.DISABLED if graph is None else graph.decode.backend
+ if req.decode_graph_backends and backend not in req.decode_graph_backends:
+ reasons.append(
+ f"decode CUDA graphs {backend!r} are not one of "
+ f"{sorted(req.decode_graph_backends)}"
+ )
+ exactly = [
+ (f"config.{key}", getattr(hf_config, key, None), want)
+ for key, want in req.hf_config.items()
+ ] + [(name, getattr(envs, name).get(), want) for name, want in req.env.items()]
+ reasons += [
+ f"{what} {v!r} is not {want!r}" for what, v, want in exactly if v != want
+ ]
+ return reasons
+
+
+def _model_disable_reasons(
+ *, spec: MegakernelSpec, model: torch.nn.Module, hf_config: PretrainedConfig
+) -> list[str]:
+ """This rank's reasons the loaded model cannot run the spec."""
+ reasons = []
+ if spec.model_disable_reasons is not None:
+ reasons += spec.model_disable_reasons(model, hf_config)
+ if "forward" in model.__dict__:
+ reasons.append("the model's forward is already replaced on this instance")
+ sms = torch.cuda.get_device_properties(
+ torch.cuda.current_device()
+ ).multi_processor_count
+ if sms < spec.requires.resident_blocks:
+ reasons.append(
+ f"{Feature.PARTITIONED_GPU.value}: {sms} SMs, the persistent grids need "
+ f"{spec.requires.resident_blocks}"
+ )
+ return reasons
+
+
+class _Dispatch:
+ """The installed forward: covered steps run the built megakernel."""
+
+ def __init__(
+ self, *, built: BuiltMegakernel, original: Callable, binding: MegakernelBinding
+ ):
+ self.built, self.original, self.binding = built, original, binding
+ # batch size -> the step planned while capturing its decode graphs; plan reads
+ # only the mode and rows, so every graph variant of a batch size plans alike
+ self.captured: dict[int, Optional[Step]] = {}
+
+ def wrapped(self) -> Callable:
+ # Graph runners and supports_pp inspect the original forward's signature.
+ @functools.wraps(self.original)
+ def forward(input_ids, positions, forward_batch, *args, **kwargs):
+ step = self.built.plan(input_ids, forward_batch, *args, **kwargs)
+ if torch.cuda.is_current_stream_capturing():
+ self.captured[int(forward_batch.batch_size)] = step
+ if step is None:
+ return self.original(
+ input_ids, positions, forward_batch, *args, **kwargs
+ )
+ return self.built.forward(
+ msgspec.structs.replace(step, view=self.binding.forward_view()),
+ self.original,
+ input_ids,
+ positions,
+ forward_batch,
+ *args,
+ **kwargs,
+ )
+
+ return forward
+
+
+class _CaptureCheck:
+ """Once every decode graph is captured, log the ones that run the megakernel and
+ refuse a runner none of whose graphs do."""
+
+ def __init__(self, *, spec: MegakernelSpec, dispatch: _Dispatch):
+ self.spec, self.dispatch = spec, dispatch
+ self.is_done = False
+
+ def __call__(
+ self,
+ runner: DecodeCudaGraphRunner,
+ out: Any,
+ forward_batch: ForwardBatch,
+ num_tokens: int,
+ ) -> None:
+ captured = self.dispatch.captured
+ if self.is_done or not set(runner.capture_bs) <= set(captured):
+ return
+ self.is_done = True
+ self.dispatch.binding.assert_live()
+ routed = sorted(bs for bs, step in captured.items() if step is not None)
+ if not routed:
+ raise ValueError(
+ f"{FLAG}: none of the {len(captured)} captured decode graphs "
+ f"(batch sizes {sorted(captured)}) runs {self.spec.name}"
+ )
+ logger.info(
+ "NVIDIA megakernel: %s runs the decode graphs of batch sizes %s (%d of "
+ "%d); every other step runs the standard forward",
+ self.spec.name,
+ routed,
+ len(routed),
+ len(captured),
+ )
+
+
+def _tokens_per_request() -> int:
+ """Tokens per request of this server's decode-phase steps."""
+ spec_config = get_spec()
+ if spec_config.speculative_algorithm is None:
+ return 1
+ return int(spec_config.speculative_num_draft_tokens or 1)
+
+
+def _decode_graphs_enabled() -> bool:
+ graph = get_exec().graph.cuda_graph_config
+ return graph is not None and graph.decode.backend != Backend.DISABLED
+
+
+def _gather_tp(obj: Any, group: Any = None) -> list[Any]:
+ """Gather a value from every rank of group (default: TP) through its CPU group."""
+ group = get_parallel().tp_group if group is None else group
+ if group.world_size == 1:
+ return [obj]
+ out: list[Any] = [None] * group.world_size
+ dist.all_gather_object(out, obj, group=group.cpu_group)
+ return out
+
+
+def _gather_reasons(reasons: list[str]) -> list[str]:
+ return [
+ f"rank {rank}: {reason}"
+ for rank, rank_reasons in enumerate(_gather_tp(reasons))
+ for reason in rank_reasons
+ ]
+
+
+def _refuse(reasons: list[str]) -> None:
+ if reasons:
+ raise ValueError(
+ f"{FLAG} cannot run this setup:\n - " + "\n - ".join(reasons)
+ )
diff --git a/python/sglang/srt/megakernels/spec.py b/python/sglang/srt/megakernels/spec.py
new file mode 100644
index 000000000000..63749110eba1
--- /dev/null
+++ b/python/sglang/srt/megakernels/spec.py
@@ -0,0 +1,209 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Megakernel spec contract and the lazy per-architecture registry."""
+
+from __future__ import annotations
+
+import enum
+import importlib
+from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Protocol, Sequence
+
+import msgspec
+
+if TYPE_CHECKING:
+ import torch
+
+# bumped on any breaking change to the contracts in this module
+MEGAKERNEL_API_VERSION = 1
+
+# the sglang.srt.plugins entry point that registers the specs
+ENTRY_POINT = "nvidia_megakernel"
+FLAG = "--enable-nvidia-megakernel"
+
+
+class Feature(enum.Enum):
+ """SGLang features refused unless a spec lists them."""
+
+ PIPELINE_PARALLEL = "pipeline parallelism"
+ MULTI_NODE_TP = "multi-node tensor parallelism"
+ DP_ATTENTION = "DP attention"
+ EXPERT_PARALLEL = "expert parallelism"
+ EPLB = "EPLB"
+ CONTEXT_PARALLEL = "context parallelism"
+ SPECULATIVE_DECODING = "speculative decoding"
+ RAGGED_VERIFY = "uneven tokens per request in verify (SGLANG_RAGGED_VERIFY_MODE)"
+ TWO_BATCH_OVERLAP = "two-batch / single-batch overlap"
+ LORA = "LoRA"
+ PD_DISAGGREGATION = "PD disaggregation"
+ RETURN_ROUTED_EXPERTS = "--enable-return-routed-experts"
+ RETURN_INDEXER_TOPK = "--enable-return-indexer-topk"
+ KV_OFFLOAD = "KV offload"
+ HISPARSE = "hierarchical sparse attention"
+ MEMORY_RELEASE = "memory release"
+ CPU_OFFLOAD = "CPU offload"
+ TORCH_COMPILE = "torch-compiled decode"
+ DETERMINISTIC_INFERENCE = "deterministic inference"
+ PARTITIONED_GPU = "partitioned GPUs"
+
+
+class Requirements(msgspec.Struct, frozen=True):
+ """Configuration a spec needs, checked on the resolved settings at startup; an
+ empty field accepts any value."""
+
+ isas: frozenset[str] # e.g. "sm_103"
+ tp_sizes: frozenset[int]
+ resident_blocks: int = 0 # SMs the persistent grids need
+ dtypes: frozenset[str] = frozenset()
+ # the decode CUDA graph backends the served steps run under
+ decode_graph_backends: frozenset[str] = frozenset({"full", "disabled"})
+ # a server argument -> its accepted values, as resolved for this server
+ server_args: Mapping[str, frozenset] = {}
+ # an HF config field -> the value the kernels are built for
+ hf_config: Mapping[str, Any] = {}
+ # an SGLang environment variable in envs -> its required value
+ env: Mapping[str, Any] = {}
+
+
+class NumericsKind(enum.Enum):
+ BITWISE = "bitwise"
+ TOLERANCE = "tolerance"
+
+
+class Numerics(msgspec.Struct, frozen=True):
+ """Declared agreement with the standard path on the same GPU, logged at build."""
+
+ kind: NumericsKind
+ tolerances: Mapping[str, float] = {}
+
+ def __post_init__(self):
+ if self.kind is NumericsKind.TOLERANCE and not self.tolerances:
+ raise ValueError("a TOLERANCE spec must state its tolerances")
+
+
+class BuildContext(msgspec.Struct, frozen=True):
+ widths: tuple[int, ...] # step rows to build, ascending
+ tokens_per_request: int
+ deadline_s: float # every cross-GPU wait's deadline, below --watchdog-timeout
+ # SGLang's runtime state for this model (megakernels/binding.py)
+ binding: Any = None
+
+
+class Step(msgspec.Struct, frozen=True):
+ """A covered step, decided from host state alike on every rank."""
+
+ mode: str # "decode" or "target_verify"
+ rows: int
+ # the running forward's ForwardView, set by SGLang before forward runs the step
+ view: Any = None
+
+
+class BuiltMegakernel(Protocol):
+ """The object a spec's build returns.
+
+ draft_sampler is None, or a factory the DSpark draft sampler calls once with
+ (*, gamma, max_bs, greedy_draft, noise, draft_tokens, step_logits,
+ step_temperatures). It returns bs -> None (SGLang's proposal at that batch size)
+ or step(hidden_states, anchor_tokens) -> (tokens, confidence_features), which
+ replaces the base logits and the Markov block. The sizes it serves must agree on
+ every rank (SGLang refuses otherwise), and a step must produce identical tokens
+ on every rank, since it replaces the proposal's rank-0 broadcast; write
+ draft_tokens[: bs * gamma] and the sampled rows' step_logits; and advance
+ noise.offset once per sampled block.
+
+ SGLang's runtime state reaches a build as BuildContext.binding and a covered
+ step as Step.view (megakernels/binding.py, which also lists what else may be
+ called, written or read).
+ """
+
+ widths: tuple[int, ...] # the rows it built, possibly fewer than requested
+ draft_sampler: Optional[Callable[..., Callable[[int], Optional[Callable]]]]
+
+ def summary(self) -> str:
+ """What was built, for the startup line."""
+
+ def plan(
+ self, input_ids: Any, forward_batch: Any, *args: Any, **kwargs: Any
+ ) -> Optional[Step]:
+ """The step to run, or None for the model's own forward; host-only, so a
+ captured graph replays the decision made at capture."""
+
+ def forward(self, step: Step, original: Callable, *args: Any, **kwargs: Any) -> Any:
+ """Run step in place of original (the model's bound forward), taking and
+ returning what it does; capturable in a CUDA graph."""
+
+
+class MegakernelSpec(msgspec.Struct, frozen=True):
+ """A model-specific plugin contract; build returns its BuiltMegakernel.
+
+ Argument resolution checks the served model's spec in full. A spec found only
+ at load, e.g. for a draft model, is checked there against its ISA, tp_sizes,
+ decode_graph_backends, hf_config, env and model_disable_reasons.
+ """
+
+ name: str
+ requires: Requirements
+ features: frozenset[Feature] # the refused-by-default features it accepts
+ numerics: Numerics
+ max_rows: int # the widest step it covers
+ build: Callable[[torch.nn.Module, Any, BuildContext], BuiltMegakernel]
+ api_version: int # the MEGAKERNEL_API_VERSION this spec is written against
+ # (model, hf_config), after load: per-rank checks of the loaded model
+ model_disable_reasons: Optional[Callable[[torch.nn.Module, Any], list[str]]] = None
+ # (cfg, hf_config), before load: the checks requires cannot state
+ config_disable_reasons: Optional[Callable[[Any, Any], list[str]]] = None
+
+
+# HF architecture -> the "module:attribute" path of its spec
+_REGISTRY: dict[str, str] = {}
+
+
+def register(*, architecture: str, path: str) -> None:
+ """Register a module:attribute spec path from the plugin entry point."""
+ if _REGISTRY.get(architecture, path) != path:
+ raise ValueError(
+ f"a megakernel spec for {architecture} is already registered: "
+ f"{_REGISTRY[architecture]}"
+ )
+ _REGISTRY[architecture] = path
+
+
+def registered_architectures() -> list[str]:
+ """The architectures with a registered spec; empty when no plugin registered any."""
+ return sorted(_REGISTRY)
+
+
+def spec_registered(architectures: Optional[Sequence[str]]) -> bool:
+ """Whether any of the architectures has a registered spec, usable or not."""
+ return any(arch in _REGISTRY for arch in architectures or ())
+
+
+def find_spec(
+ *, architectures: Optional[Sequence[str]], isa: Optional[str]
+) -> tuple[Optional[MegakernelSpec], str]:
+ """The compatible spec and an empty reason, or None and a rejection reason."""
+ why = []
+ for arch in architectures or ():
+ path = _REGISTRY.get(arch)
+ if path is None:
+ continue
+ module, _, attr = path.partition(":")
+ # a broken plugin is a refusal every rank reports, not a crash on one
+ try:
+ spec = getattr(importlib.import_module(module), attr)
+ except Exception as e:
+ why.append(f"{path} does not import: {type(e).__name__}: {e}")
+ continue
+ if not isinstance(spec, MegakernelSpec):
+ why.append(f"{path} is not a MegakernelSpec")
+ elif spec.api_version != MEGAKERNEL_API_VERSION:
+ why.append(
+ f"{path} uses megakernel API version {spec.api_version}; this "
+ f"SGLang requires version {MEGAKERNEL_API_VERSION}"
+ )
+ elif isa not in spec.requires.isas:
+ why.append(f"{path} supports {sorted(spec.requires.isas)}, not {isa}")
+ else:
+ return spec, ""
+ return None, "; ".join(why) or (
+ f"no megakernel spec is registered for {list(architectures or ())} "
+ f"(registered: {registered_architectures()})"
+ )
diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py
index 6818ad777830..86a6b329a17f 100644
--- a/python/sglang/srt/model_executor/model_runner.py
+++ b/python/sglang/srt/model_executor/model_runner.py
@@ -92,6 +92,7 @@
from sglang.srt.lora.lora_manager import LoRAManager, init_lora_cuda_graph_moe_buffers
from sglang.srt.lora.lora_registry import LoRARef
from sglang.srt.managers.schedule_batch import sanity_check_mm_pad_shift_value
+from sglang.srt.megakernels.runtime import install_megakernel
from sglang.srt.mem_cache import kv_cache_dtype
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.kv_cache_configurator import (
@@ -726,6 +727,13 @@ def initialize(self):
self.maybe_init_lora_manager()
self.maybe_enable_batch_invariant_mode()
self.configure_kv_cache_dtype()
+ # after every post-load change to the model, before the KV cache is sized
+ install_megakernel(
+ model=self.model,
+ hf_config=self.model_config.hf_config,
+ capture_tail_hooks=self.capture_tail_hooks,
+ is_draft=self.is_draft_worker,
+ )
def init_memory_saver_adapter(self):
self.memory_saver_adapter = TorchMemorySaverAdapter.create(
diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py
index 3797bbcae4da..b7e9de071d20 100644
--- a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py
+++ b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py
@@ -1,10 +1,12 @@
from __future__ import annotations
import logging
-from typing import Optional
+from typing import Callable, Optional
import torch
+from sglang.srt.layers.dp_attention import get_dp_tp_group
+from sglang.srt.megakernels.runtime import megakernel_agree, megakernel_draft_sampler
from sglang.srt.speculative.dspark_components.dspark_draft import (
DraftNoise,
DraftParams,
@@ -32,6 +34,7 @@ def __init__(
tp_sync: SpecTpSync,
greedy_draft: bool,
out=None,
+ megakernel_sampler: Optional[Callable] = None,
):
self.model = model
self.markov_head = model.markov_head
@@ -73,6 +76,35 @@ def __init__(
# [max_bs, gamma, vocab] step logits for the sampling accept, sized on
# the first call; a greedy draft keeps none.
self.step_logits: Optional[torch.Tensor] = None
+ self._megakernel_steps = None
+ if megakernel_sampler is not None:
+ self._megakernel_steps = self._build_megakernel_steps(megakernel_sampler)
+
+ def _build_megakernel_steps(self, megakernel_sampler: Callable):
+ if not self.greedy_draft:
+ # The megakernel leaves temperature-0 rows' logits unwritten; a mixed accept
+ # still reads them.
+ self.step_logits = torch.zeros(
+ (self.max_bs, self.gamma, _step_logits_width(self.model)),
+ dtype=self.markov_head.step_logits_dtype(torch.bfloat16),
+ device=self.out.device,
+ )
+ steps = megakernel_sampler(
+ gamma=self.gamma,
+ max_bs=self.max_bs,
+ greedy_draft=self.greedy_draft,
+ noise=self._noise,
+ draft_tokens=self.out,
+ step_logits=self.step_logits,
+ step_temperatures=self.params.step_temperatures,
+ )
+ # A rank on SGLang's block would wait in its per-step token sync (this group),
+ # which the megakernel's step skips.
+ served = tuple(bs for bs in range(1, self.max_bs + 1) if steps(bs) is not None)
+ megakernel_agree(
+ served, what="draft sampler batch sizes", group=get_dp_tp_group()
+ )
+ return steps
def _step_logits(self, base_logits: torch.Tensor) -> torch.Tensor:
if self.step_logits is None:
@@ -100,10 +132,26 @@ def __call__(self, hidden_states, input_ids):
bs=bs,
gamma=self.gamma,
)
- base_logits, confidence_features = self.model.compute_base_logits(model_hidden)
- base_logits = base_logits.view(bs, self.gamma, -1)
anchor = input_ids.view(bs, self.query_token_num)[:, 0]
+ step = None if self._megakernel_steps is None else self._megakernel_steps(bs)
+ if step is not None:
+ draft_tokens, confidence_features = step(model_hidden, anchor)
+ else:
+ draft_tokens, confidence_features = self._markov_block(
+ model_hidden, sample_hidden, anchor, bs
+ )
+ if self.confidence_out is not None:
+ confidence, confidence_raw = self.model.compute_confidence(
+ anchor_tokens=anchor,
+ draft_tokens=draft_tokens,
+ confidence_features=confidence_features,
+ )
+ self.confidence_out[:bs].copy_(confidence)
+ self.confidence_raw_out[:bs].copy_(confidence_raw)
+ def _markov_block(self, model_hidden, sample_hidden, anchor, bs: int):
+ base_logits, confidence_features = self.model.compute_base_logits(model_hidden)
+ base_logits = base_logits.view(bs, self.gamma, -1)
draft_tokens, _ = draft_markov_block(
markov_head=self.markov_head,
base_logits=base_logits,
@@ -118,14 +166,7 @@ def __call__(self, hidden_states, input_ids):
greedy_draft=self.greedy_draft,
)
self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1))
- if self.confidence_out is not None:
- confidence, confidence_raw = self.model.compute_confidence(
- anchor_tokens=anchor,
- draft_tokens=draft_tokens,
- confidence_features=confidence_features,
- )
- self.confidence_out[:bs].copy_(confidence)
- self.confidence_raw_out[:bs].copy_(confidence_raw)
+ return draft_tokens, confidence_features
def _step_logits_width(model) -> int:
@@ -157,10 +198,12 @@ def _eager(reason):
return _eager("gamma<=0")
if draft_model.markov_head is None:
return _eager("no markov head")
+ megakernel_sampler = megakernel_draft_sampler(draft_model)
if tp_rank == 0:
logger.info(
- "DSpark draft proposal (%s) folded into the draft cuda graph.",
+ "DSpark draft proposal (%s) folded into the draft cuda graph%s.",
"greedy draft" if greedy_draft else "sampling",
+ "; the NVIDIA megakernel may run it" if megakernel_sampler else "",
)
return DsparkDraftSampler(
model=draft_model,
@@ -171,4 +214,5 @@ def _eager(reason):
tp_sync=tp_sync,
greedy_draft=greedy_draft,
out=out,
+ megakernel_sampler=megakernel_sampler,
)
diff --git a/test/registered/unit/megakernels/_fakes.py b/test/registered/unit/megakernels/_fakes.py
new file mode 100644
index 000000000000..31308d8e7b88
--- /dev/null
+++ b/test/registered/unit/megakernels/_fakes.py
@@ -0,0 +1,95 @@
+"""Shared fakes for the megakernel unit tests: one spec, one model, server arguments."""
+
+from types import SimpleNamespace
+
+import torch
+from msgspec.structs import replace
+
+from sglang.srt.megakernels import spec as registry
+from sglang.srt.megakernels.spec import (
+ Feature,
+ MegakernelSpec,
+ Numerics,
+ NumericsKind,
+ Requirements,
+ Step,
+)
+from sglang.srt.model_executor.cuda_graph_config import default_cuda_graph_config
+from sglang.srt.server_args import ServerArgs
+from sglang.test.ci.ci_register import register_cpu_ci
+
+register_cpu_ci(
+ est_time=0, stage="base-a", runner_config="cpu", disabled="megakernel test fakes"
+)
+
+ARCH, ISA = "FakeMegaForCausalLM", "sm_103"
+
+
+class FakeBuilt:
+ """Covers steps of 1-4 rows and records the steps it ran."""
+
+ draft_sampler = None
+
+ def __init__(self, widths):
+ self.widths = tuple(w for w in widths if w <= 4)
+ self.steps, self.views = [], []
+
+ def summary(self):
+ return "fake"
+
+ def plan(self, input_ids, forward_batch, *args, **kwargs):
+ rows = int(input_ids.shape[0])
+ return Step(mode="decode", rows=rows) if rows in self.widths else None
+
+ def forward(self, step, original, input_ids, positions, forward_batch, **kw):
+ self.steps.append(step.rows)
+ self.views.append(step.view)
+ return "megakernel"
+
+
+def _build(model, hf_config, context):
+ model.context = context
+ return FakeBuilt(context.widths)
+
+
+SPEC = MegakernelSpec(
+ name="fake",
+ requires=Requirements(isas=frozenset({ISA}), tp_sizes=frozenset({4})),
+ features=frozenset({Feature.SPECULATIVE_DECODING}),
+ numerics=Numerics(kind=NumericsKind.BITWISE),
+ max_rows=8,
+ build=_build,
+ api_version=registry.MEGAKERNEL_API_VERSION,
+)
+# an empty field accepts any value
+ANY_TP = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset()))
+# a draft spec: builds no width, offers only the draft sampler
+DRAFT = replace(
+ SPEC,
+ max_rows=0,
+ build=lambda model, hf, ctx: SimpleNamespace(
+ widths=(), summary=lambda: "draft sampler", draft_sampler=print
+ ),
+)
+
+
+class Lm(torch.nn.Module):
+ def forward(self, input_ids, positions, forward_batch, input_embeds=None):
+ return "standard"
+
+
+def batch(rows):
+ return SimpleNamespace(batch_size=rows, input_ids=torch.zeros(rows))
+
+
+def server_args(**kw):
+ """Arguments as the resolution step sees them, with the graph config resolved."""
+ defaults = {
+ "model_path": "dummy",
+ "tp_size": 4,
+ "kv_cache_dtype": "fp8_e4m3",
+ "speculative_algorithm": "DSPARK",
+ "speculative_num_draft_tokens": 6,
+ "cuda_graph_config": default_cuda_graph_config(),
+ }
+ return ServerArgs(**{**defaults, **kw})
diff --git a/test/registered/unit/megakernels/test_megakernel_binding.py b/test/registered/unit/megakernels/test_megakernel_binding.py
new file mode 100644
index 000000000000..f7fc62e43682
--- /dev/null
+++ b/test/registered/unit/megakernels/test_megakernel_binding.py
@@ -0,0 +1,79 @@
+"""CPU checks of the megakernel binding: the view is the running forward's attention
+backend and KV pool, and a source replaced after capture is caught."""
+
+import unittest
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import torch
+
+from sglang.srt.megakernels import binding as mb
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu")
+
+BACKEND = "sglang.srt.model_executor.forward_context.get_attn_backend"
+
+
+class Pool:
+ uniform_fp8 = False
+ request_window = None
+
+ def __init__(self):
+ self.keys = [torch.zeros(4, 8) for _ in range(2)]
+
+ def get_swa_key_buffer_radix(self, layer_id):
+ return self.keys[layer_id]
+
+ def get_extra_key_page_size(self, layer_id):
+ return 64 + layer_id
+
+ def set_swa_key_buffer_radix_fused_norm_rope(self, **kw):
+ return kw
+
+
+def backend(pool):
+ return SimpleNamespace(
+ token_to_kv_pool=pool,
+ forward_metadata=SimpleNamespace(core_attn_metadata="core"),
+ trtllm_attn=False,
+ forward=lambda *a, **k: "attention",
+ store_cache=lambda **k: None,
+ get_swa_out_cache_loc=lambda fb: "swa_loc",
+ forward_low_ratio_sources=lambda **k: "sources",
+ forward_core_compressor=lambda **k: "compressed",
+ )
+
+
+class TestMegakernelBinding(CustomTestCase):
+ def test_the_view_is_the_running_forwards_backend_and_pool(self):
+ pool = Pool()
+ with patch(BACKEND, return_value=backend(pool)):
+ view = mb.MegakernelBinding(model=torch.nn.Module()).forward_view()
+ self.assertEqual(view.metadata.core_attn_metadata, "core")
+ self.assertIs(view.kv_pool.pool, pool)
+ self.assertIs(view.kv_pool.get_swa_key_buffer_radix(1), pool.keys[1])
+
+ def test_a_replaced_source_fails_assert_live_by_name(self):
+ model = torch.nn.Module()
+ model.engram_hasher = SimpleNamespace(history=torch.zeros(5, 3))
+ binding = mb.MegakernelBinding(model=model)
+ attn_backend = backend(Pool())
+ with patch(BACKEND, return_value=attn_backend):
+ binding.forward_view()
+ self.assertIs(binding.engram_history(), model.engram_hasher.history)
+ binding.assert_live()
+ model.engram_hasher.history = torch.zeros(9, 3)
+ with self.assertRaisesRegex(RuntimeError, "engram_history"):
+ binding.assert_live()
+ binding = mb.MegakernelBinding(model=torch.nn.Module())
+ with patch(BACKEND, return_value=backend(Pool())):
+ binding.forward_view()
+ with patch(BACKEND, return_value=backend(Pool())):
+ with self.assertRaisesRegex(RuntimeError, "kv_pool"):
+ binding.assert_live()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/megakernels/test_megakernel_guards.py b/test/registered/unit/megakernels/test_megakernel_guards.py
new file mode 100644
index 000000000000..746f9e07831d
--- /dev/null
+++ b/test/registered/unit/megakernels/test_megakernel_guards.py
@@ -0,0 +1,54 @@
+"""CPU checks of --enable-nvidia-megakernel's startup resolution."""
+
+import sys
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from sglang.srt.megakernels import guards, runtime
+from sglang.srt.megakernels import spec as registry
+from sglang.srt.megakernels.guards import megakernel_disable_reasons
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _fakes import ARCH, ISA, server_args # noqa: E402
+
+register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu")
+
+
+class TestMegakernelGuards(CustomTestCase):
+ def setUp(self):
+ registry_patch = patch.dict(registry._REGISTRY, clear=True)
+ registry_patch.start()
+ self.addCleanup(registry_patch.stop)
+
+ def test_a_disabled_flag_looks_nothing_up_and_keeps_the_forward(self):
+ with patch.object(guards, "plugin_reasons") as plugin:
+ guards.handle_nvidia_megakernel(server_args())
+ plugin.assert_not_called()
+ model = SimpleNamespace() # any attribute read would raise
+ with patch.object(runtime, "megakernel_enabled", return_value=False):
+ runtime.install_megakernel(
+ model=model, hf_config=None, capture_tail_hooks=[], is_draft=False
+ )
+ self.assertEqual(vars(model), {})
+
+ def test_an_unmet_tp_size_is_refused_and_an_empty_field_accepts_any(self):
+ hf_config = SimpleNamespace(architectures=[ARCH], model_type="fake")
+
+ def tp_reasons(path):
+ registry._REGISTRY.clear()
+ registry.register(architecture=ARCH, path=path)
+ _, reasons = megakernel_disable_reasons(
+ cfg=server_args(tp_size=8), hf_config=hf_config, isa=ISA
+ )
+ return [r for r in reasons if "tp_size" in r]
+
+ self.assertTrue(tp_reasons("_fakes:SPEC"))
+ self.assertEqual(tp_reasons("_fakes:ANY_TP"), [])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/megakernels/test_megakernel_runtime.py b/test/registered/unit/megakernels/test_megakernel_runtime.py
new file mode 100644
index 000000000000..84da1bd6cae2
--- /dev/null
+++ b/test/registered/unit/megakernels/test_megakernel_runtime.py
@@ -0,0 +1,143 @@
+"""CPU checks of the megakernel install, dispatch and capture check."""
+
+import sys
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import torch
+from msgspec.structs import replace
+
+from sglang.srt.megakernels import runtime
+from sglang.srt.megakernels import spec as registry
+from sglang.srt.megakernels.binding import MegakernelBinding
+from sglang.srt.runtime_context import publish, reset_context
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _fakes import ANY_TP, ARCH, ISA, SPEC, Lm, batch, server_args # noqa: E402
+
+register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu")
+
+
+def install(
+ *,
+ archs=(ARCH,),
+ gathered=None,
+ tokens=2,
+ hooks=None,
+ is_draft=False,
+):
+ """install_megakernel on an Lm as rank 0 of the published server; gathered maps
+ a gathered value to what the other ranks report, default the same as this rank."""
+ lm, hooks = Lm(), [] if hooks is None else hooks
+ gpu = SimpleNamespace(multi_processor_count=160)
+ with (
+ patch.object(runtime, "device_isa", return_value=ISA),
+ patch.object(runtime, "deadline_s", return_value=30.0),
+ patch.object(
+ runtime, "_gather_tp", side_effect=gathered or (lambda x, group=None: [x])
+ ),
+ patch.object(runtime, "_tokens_per_request", return_value=tokens),
+ patch.object(runtime, "_decode_graphs_enabled", return_value=True),
+ patch.object(torch.cuda, "current_device", return_value=0),
+ patch.object(torch.cuda, "get_device_properties", return_value=gpu),
+ patch.object(torch.cuda, "mem_get_info", return_value=(0, 0)),
+ ):
+ runtime.install_megakernel(
+ model=lm,
+ hf_config=SimpleNamespace(architectures=list(archs)),
+ capture_tail_hooks=hooks,
+ is_draft=is_draft,
+ )
+ return lm
+
+
+class TestMegakernelRuntime(CustomTestCase):
+ def setUp(self):
+ registry_patch = patch.dict(registry._REGISTRY, clear=True)
+ registry_patch.start()
+ self.addCleanup(registry_patch.stop)
+ registry.register(architecture=ARCH, path="_fakes:SPEC")
+ publish(server_args(enable_nvidia_megakernel=True), role="test")
+ self.addCleanup(reset_context)
+ # the running forward's view reads its attention backend, which tests lack
+ view_patch = patch.object(
+ MegakernelBinding, "forward_view", return_value="view"
+ )
+ view_patch.start()
+ self.addCleanup(view_patch.stop)
+
+ def _install_spec(self, spec, **kw):
+ with patch.object(runtime, "find_spec", return_value=(spec, "")):
+ return install(**kw)
+
+ def test_covered_steps_run_the_megakernel_and_others_the_forward(self):
+ lm = install(tokens=1)
+ built = runtime.built_megakernel(lm)
+ with patch.object(
+ torch.cuda, "is_current_stream_capturing", return_value=False
+ ):
+ self.assertEqual(lm(batch(4).input_ids, None, batch(4)), "megakernel")
+ self.assertEqual(lm(batch(6).input_ids, None, batch(6)), "standard")
+ # only a covered step runs the megakernel, with the running forward's view
+ self.assertEqual((built.steps, built.views), ([4], ["view"]))
+
+ def test_a_target_without_a_usable_spec_is_refused_and_a_draft_keeps_its_forward(
+ self,
+ ):
+ # the flag must not silently run the model's own forward
+ with self.assertRaisesRegex(ValueError, "in the model workers"):
+ install(archs=["TargetWithoutSpec"])
+ tp8 = replace(SPEC, requires=replace(SPEC.requires, tp_sizes=frozenset({8})))
+ with self.assertRaisesRegex(ValueError, "tp_size 4"):
+ self._install_spec(tp8)
+ # an empty field accepts any value
+ self.assertIsNotNone(runtime.built_megakernel(self._install_spec(ANY_TP)))
+ with self.assertLogs(runtime.logger, "INFO") as logs:
+ lm = install(archs=["DraftWithoutSpec"], is_draft=True)
+ self.assertNotIn("forward", lm.__dict__)
+ self.assertIn("keeps its own forward", "".join(logs.output))
+ registry.register(architecture="Draft", path="_fakes:DRAFT")
+ # a draft spec offers only its draft sampler
+ lm = install(archs=["Draft"], is_draft=True)
+ self.assertNotIn("forward", lm.__dict__)
+ self.assertIs(runtime.megakernel_draft_sampler(lm), print)
+
+ def test_ranks_that_disagree_are_refused_on_every_rank(self):
+ """A rank that differs would wait in a different collective."""
+
+ def no_spec_on_rank_1(value, group=None):
+ return [value, (value[0], "rank-1 reason", None)]
+
+ def fewer_widths_on_rank_1(value, group=None):
+ widths = isinstance(value, tuple) and all(isinstance(w, int) for w in value)
+ return [value, value[:1]] if widths else [value]
+
+ with self.assertRaisesRegex(ValueError, "rank 1: rank-1 reason"):
+ install(gathered=no_spec_on_rank_1)
+ with self.assertRaisesRegex(RuntimeError, "widths fake built differ"):
+ install(gathered=fewer_widths_on_rank_1)
+
+ def test_capture_check_refuses_when_no_graph_runs_the_megakernel(self):
+ def capture(sizes, capture_bs):
+ hooks = []
+ lm = install(tokens=1, hooks=hooks)
+ runner = SimpleNamespace(capture_bs=capture_bs)
+ with patch.object(
+ torch.cuda, "is_current_stream_capturing", return_value=True
+ ):
+ for bs in sizes:
+ lm(batch(bs).input_ids, None, batch(bs))
+ hooks[0](runner, None, batch(bs), bs)
+
+ capture([2, 8], capture_bs=[2, 8, 16]) # not all captured yet: no verdict
+ capture([2, 8], capture_bs=[2, 8])
+ with self.assertRaisesRegex(ValueError, "none of the 2 captured decode graphs"):
+ capture([8, 16], capture_bs=[8, 16])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/spec/test_dspark_draft_block.py b/test/registered/unit/spec/test_dspark_draft_block.py
index d1b8abac20f0..5fb6ab186a32 100644
--- a/test/registered/unit/spec/test_dspark_draft_block.py
+++ b/test/registered/unit/spec/test_dspark_draft_block.py
@@ -1,5 +1,6 @@
import unittest
from types import SimpleNamespace
+from unittest.mock import Mock, patch
import torch
@@ -9,6 +10,7 @@
RNNHead,
VanillaMarkov,
)
+from sglang.srt.speculative.dspark_components import dspark_draft_sampler as ds
from sglang.srt.speculative.dspark_components.dspark_draft import (
DraftNoise,
sample_draft_block,
@@ -124,5 +126,78 @@ def test_only_the_plain_w2_gemv_head_fuses(self):
self.assertIs(head.greedy_w2_weight() is not None, fuses)
+def _folded_draft_model():
+ """A draft model for the folded proposal."""
+ model = SimpleNamespace(
+ markov_head=VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK),
+ sample_from_anchor=True,
+ confidence_head=SimpleNamespace(proj=SimpleNamespace(weight=torch.zeros(1))),
+ lm_head=SimpleNamespace(org_vocab_size=VOCAB),
+ compute_base_logits=Mock(
+ side_effect=lambda h: (torch.zeros(h.shape[0], VOCAB), h + 1)
+ ),
+ compute_confidence=Mock(
+ side_effect=lambda **kw: (torch.zeros(BS, GAMMA), torch.zeros(BS, GAMMA))
+ ),
+ )
+ return model
+
+
+def _build(model, *, greedy_draft, factory=None, agree=None):
+ """maybe_build_draft_sampler, with factory as the megakernel's draft sampler."""
+ with (
+ patch.object(ds, "megakernel_draft_sampler", return_value=factory),
+ patch.object(ds, "megakernel_agree", agree or Mock()),
+ patch.object(ds, "get_dp_tp_group", return_value="draft tp group"),
+ ):
+ return ds.maybe_build_draft_sampler(
+ draft_model=model,
+ gamma=GAMMA,
+ max_bs=2 * BS,
+ device="cpu",
+ tp_rank=1,
+ noise=DraftNoise(key=3, offset=torch.zeros(1, dtype=torch.int64)),
+ greedy_draft=greedy_draft,
+ tp_sync="sync",
+ )
+
+
+class TestMegakernelDraftSampler(CustomTestCase):
+ def test_the_megakernel_step_runs_only_on_the_batch_sizes_every_rank_serves(self):
+ """A served batch size runs the megakernel's step in place of the base
+ logits and the Markov block, writing SGLang's buffers; every other size runs
+ SGLang's block. The served sizes are agreed over the token-sync group, since
+ a rank on SGLang's block waits in a sync its peers never enter."""
+ hidden, input_ids = torch.randn(BS * GAMMA, HIDDEN), torch.arange(BS * GAMMA)
+ step = Mock(side_effect=lambda h, anchor: ("tokens", h * 2))
+ factory = Mock(return_value=lambda bs: step if bs == BS else None)
+ agree, model = Mock(), _folded_draft_model()
+ with patch.object(ds, "draft_markov_block", side_effect=AssertionError):
+ sampler = _build(model, greedy_draft=False, factory=factory, agree=agree)
+ sampler(hidden, input_ids)
+ agree.assert_called_once_with(
+ (BS,), what="draft sampler batch sizes", group="draft tp group"
+ )
+ model.compute_base_logits.assert_not_called()
+ self.assertIs(factory.call_args.kwargs["draft_tokens"], sampler.out)
+ self.assertIs(factory.call_args.kwargs["step_logits"], sampler.step_logits)
+ self.assertTrue(torch.equal(step.call_args.args[1], input_ids[::GAMMA]))
+
+ declined = Mock(return_value=lambda bs: None)
+ tokens = torch.full((BS, GAMMA), 7)
+ with (
+ patch.object(
+ ds, "draft_markov_block", return_value=(tokens, None)
+ ) as stock,
+ patch.object(torch.cuda, "is_current_stream_capturing", lambda: False),
+ ):
+ sampler = _build(
+ _folded_draft_model(), greedy_draft=False, factory=declined
+ )
+ sampler(torch.ones(BS * GAMMA, HIDDEN), input_ids)
+ stock.assert_called_once()
+ self.assertTrue(sampler.out[: BS * GAMMA].eq(7).all())
+
+
if __name__ == "__main__":
unittest.main()