Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion python/sglang/kernels/plugin_kernels/nvidia/spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@
# the sglang.srt.plugins entry point that registers the specs
ENTRY_POINT = "nvidia_megakernel"
FLAG = "--enable-nvidia-megakernel"
# The EAGLE draft graph runners run a draft model's capture check (Step); a draft
# spec that serves steps can require it
EAGLE_DRAFT_CAPTURE_CHECK = True
# The variables of Requirements.env that the flag sets to the spec's value when they
# are unset, at argument resolution: SGLang reads each only when a model is built, in
# the model processes started after it. A value the user set is kept and checked.
FLAG_SETS_ENV = frozenset({"SGLANG_ENABLE_EMBED_REPLICATION"})


class Feature(enum.Enum):
Expand Down Expand Up @@ -69,7 +76,8 @@ class Requirements(msgspec.Struct, frozen=True):
server_args: Mapping[str, frozenset] = {}
# an HF config field -> the value the kernels are built for
hf_config: Mapping[str, Any] = {}
# an SGLang environment variable in envs -> its required value
# an SGLang environment variable in envs -> its required value (the flag sets one
# in FLAG_SETS_ENV that is unset)
env: Mapping[str, Any] = {}


Expand Down Expand Up @@ -105,6 +113,12 @@ class Step(msgspec.Struct, frozen=True):
"target_verify" – speculative-decoding target verify; runs in a CUDA graph.
"prefill" – context prefill; runs eagerly. With prefill_graph its
transformer body may be captured (SupportsPrefillBody).
"draft_decode", "draft_extend" – a draft spec's EAGLE draft steps.

A draft spec's steps may be off its width grid (an EAGLE draft decode step has
one row a request); the dispatch does not check rows against the widths. The
capture check of a draft model runs once, when the first of its EAGLE draft
graph runners has captured every batch size.
"""

mode: str
Expand Down
180 changes: 173 additions & 7 deletions python/sglang/srt/kernel_plugins/nvidia/binding.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@
It checks a plugin's spec against the resolved configuration at startup, builds it
once the model is prepared, and installs the whole-step dispatch on the model. It is
also the one place a megakernel reads SGLang's runtime state: the KV pool, the
attention metadata of the running forward and the Engram tables. A spec's build gets
a ``MegakernelBinding`` as ``BuildContext.binding``; a covered forward gets the
running forward's ``ForwardView`` as ``Step.view`` (a draft entry asks
``binding.forward_view()``). Everything else a megakernel touches is the model it
replaces, SGLang's kernel ops, and the reads listed below.
attention metadata of the running forward, the DSA (sparse MLA) backend's layout and
buffers and the Engram tables. A spec's build gets a ``MegakernelBinding`` as
``BuildContext.binding``; a covered forward gets the running forward's
``ForwardView`` as ``Step.view`` (a draft entry asks ``binding.forward_view()``).
Everything else a megakernel touches is the model it replaces, SGLang's kernel ops,
and the reads listed below.
"""

from __future__ import annotations
Expand All @@ -29,6 +30,7 @@
ENTRIES,
ENTRY_POINT,
FLAG,
FLAG_SETS_ENV,
GRAPH_VARIANT_RUNNERS,
BuildContext,
BuiltMegakernel,
Expand Down Expand Up @@ -130,6 +132,18 @@
"capture-mode queries (get_is_capture_mode, is_in_breakable_cuda_graph)",
"MegakernelBinding.engram_host_table", # the per-layer host table object
)
# Called by a megakernel on a DSA (sparse MLA) backend, tolerated until offered as
# data.
TOLERATED_DSA_CALLABLES = (
"KvPoolView.set_mla_kv_buffer", # the MLA key write
"DsaView.indexer_metadata", # the indexer's metadata of the running forward
)
# Written into the DSA pool by a megakernel (the step's rows, at its out_cache_loc),
# tolerated until SGLang owns the writes.
TOLERATED_DSA_WRITES = (
"KvPoolView.get_key_buffer", # the MLA key rows
"KvPoolView.get_index_k_with_scale_buffer", # the index-K rows and their scales
)


class KvPoolView:
Expand Down Expand Up @@ -176,6 +190,97 @@ def get_extra_key_bytes_per_token(self, layer_id: int) -> int:
def set_swa_key_buffer_radix_fused_norm_rope(self) -> Callable[..., None]:
return self.pool.set_swa_key_buffer_radix_fused_norm_rope

# MLA / DSA pools
def get_index_k_with_scale_buffer(self, layer_id: int) -> torch.Tensor:
return self.pool.get_index_k_with_scale_buffer(layer_id)

@property
def index_page_size(self) -> Optional[int]:
"""Tokens per page of the index-K buffer; None for a pool without one."""
return getattr(self.pool, "index_page_size", None)

def set_mla_kv_buffer(self, *args: Any, **kwargs: Any) -> None:
# Forwards the loc argument untouched, as a pool write door must.
return self.pool.set_mla_kv_buffer(*args, **kwargs)


class DsaView:
"""The running DSA (sparse MLA) backend's layout and the buffers its decode
uses, under names a megakernel can rely on."""

def __init__(self, backend: Any):
self._backend = backend

@property
def kv_lora_rank(self) -> int:
return self._backend.kv_lora_rank

@property
def qk_nope_head_dim(self) -> int:
return self._backend.qk_nope_head_dim

@property
def qk_rope_head_dim(self) -> int:
return self._backend.qk_rope_head_dim

@property
def kv_cache_dim(self) -> int:
return self._backend.kv_cache_dim

@property
def physical_page_size(self) -> int:
return self._backend.physical_page_size

@property
def decode_impl(self) -> str:
return self._backend.dsa_decode_impl

@property
def use_mha(self) -> bool:
"""Whether the running forward takes the MHA path instead of MLA."""
return self._backend.use_mha

@property
def kv_cache_dtype(self) -> torch.dtype:
return self._backend.kv_cache_dtype

@property
def use_fused_topk(self) -> bool:
return self._backend.use_fused_topk

@property
def topk_backend(self) -> Any:
return self._backend.dsa_topk_backend

@property
def dcp_size(self) -> int:
return self._backend.attn_dcp_size

@property
def index_kpool(self) -> int:
return self._backend.dsa_index_kpool

@property
def hisparse(self) -> bool:
return self._backend.hisparse_coordinator is not None

@property
def workspace_buffer(self) -> Optional[torch.Tensor]:
"""The decode kernels' workspace, or None when the backend has none."""
return self._backend.workspace_buffer

@property
def multi_ctas_kv_counter_buffer(self) -> Optional[torch.Tensor]:
"""The TRT-LLM decode's multi-CTA KV counter, sized before the decode graphs
are captured; None when the backend has none."""
return self._backend._multi_ctas_kv_counter_buffer

@property
def indexer_metadata(self) -> Callable[[int, Any], Any]:
"""The backend's indexer metadata of the running forward, by
(layer_id, forward_batch)."""
return self._backend.get_indexer_metadata


class KdaStateView(msgspec.Struct, frozen=True):
"""One KDA layer's SGLang-owned, non-ReplaySSM recurrent state."""
Expand Down Expand Up @@ -332,6 +437,11 @@ def low_ratio_sources(self) -> Callable[..., Any]:
def core_compressor(self) -> Callable[..., Any]:
return self._backend.forward_core_compressor

@property
def dsa(self) -> Optional[DsaView]:
"""The DSA backend's view, or None when the running backend is not DSA."""
return _dsa_view(self._backend)


class _SealedSource(msgspec.Struct, frozen=True):
identity: tuple
Expand Down Expand Up @@ -393,6 +503,13 @@ def forward_view(self, forward_batch: Optional[ForwardBatch] = None) -> ForwardV
view = ForwardView(
backend, self._seal_cache, hybrid=hybrid, legacy=not hybrid_family
)
dsa = None if hybrid_family else view.dsa

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[by Claude Code] Severity: functional | Confidence: High

Issue

The DSA buffers are sealed by name only (dsa_workspace, dsa_kv_counter). An EAGLE draft builds one DeepseekSparseAttnBackend per draft step (draft_utils.py), and each backend allocates its own _multi_ctas_kv_counter_buffer in __init__. So when the draft binding's forward_view reaches step 1, it sees a different counter under the same name, and _seal raises dsa_kv_counter was reallocated after its first read. The head still has this path: only the dcp_size fix landed.

Example

GLM-5.2 NVFP4 with EAGLE and --enable-nvidia-megakernel fails at draft CUDA graph capture, as reported in #46210 (comment).

Fix

Key the DSA seal by backend, for example (id(backend), name) with a getter for that backend. Or apply the tested minimal fix from #46210 (comment) and skip the DSA seal when install_megakernel runs with is_draft=True. Add a CPU test with two backends that own different counters.

if dsa is not None:
for name, buffer in _dsa_buffers(dsa).items():
if buffer is not None:
self._seal(
name, buffer, lambda name=name: self._current_dsa_buffer(name)
)
return view

def prefill_view(self) -> ForwardView:
Expand Down Expand Up @@ -950,12 +1067,33 @@ def _current_pool(self) -> Any:

return get_attn_backend().token_to_kv_pool

def _current_dsa_buffer(self, name: str) -> Optional[torch.Tensor]:
from sglang.srt.model_executor.forward_context import get_attn_backend

dsa = _dsa_view(get_attn_backend())
return None if dsa is None else _dsa_buffers(dsa)[name]

def _history(self) -> Optional[torch.Tensor]:
# only models with Engram layers have a hasher
hasher = getattr(self._model, "engram_hasher", None)
return None if hasher is None else hasher.history


def _dsa_view(backend: Any) -> Optional[DsaView]:
# only the DSA backend resolves a DSA decode implementation
if getattr(backend, "dsa_decode_impl", None) is None:
return None
return DsaView(backend)


def _dsa_buffers(dsa: DsaView) -> dict[str, Optional[torch.Tensor]]:
"""The DSA backend's buffers a captured step reads by address, sealed by name."""
return {
"dsa_workspace": dsa.workspace_buffer,
"dsa_kv_counter": dsa.multi_ctas_kv_counter_buffer,
}


def _identity(value: Any) -> tuple:
if isinstance(value, torch.Tensor):
return (
Expand Down Expand Up @@ -1148,31 +1286,59 @@ def _dtype_reasons(spec: Optional[MegakernelSpec], model_config: Any) -> list[st
return [f"dtype {dtype!r} is not one of {sorted(spec.requires.dtypes)}"]


def set_flag_env(spec: Optional[MegakernelSpec]) -> list[str]:
"""Set each variable of FLAG_SETS_ENV that the spec requires and the user left
unset to the spec's value; return their names."""
if spec is None:
return []
names = [
name
for name in spec.requires.env
if name in FLAG_SETS_ENV and not getattr(envs, name).is_set()
]
for name in names:
getattr(envs, name).set(spec.requires.env[name])
return names


def handle_nvidia_megakernel(server_args: Any) -> None:
"""Resolution step: fail before the weights load when the flag cannot be served,
listing every reason."""
listing every reason. The model processes, started after it, inherit what it sets
for the spec (FLAG_SETS_ENV)."""
cfg = resolving_view(server_args)
if not cfg.enable_nvidia_megakernel:
return
archs = model_config_of(server_args).hf_config.architectures
if msa_serves(archs) and not cfg.enable_amd_monokernel:
return # handle_monokernel checks the MSA plugin's spec
env_set = []
reasons = plugin_reasons()
if not reasons:
model_config = model_config_of(server_args)
hf_config, isa = model_config.hf_config, device_isa()
env_set = set_flag_env(
find_spec(architectures=hf_config.architectures, isa=isa)[0]
)
spec, reasons = megakernel_disable_reasons(
cfg=cfg, hf_config=model_config.hf_config, isa=device_isa()
cfg=cfg, hf_config=hf_config, isa=isa
)
reasons += _dtype_reasons(spec, model_config)
if cfg.enable_amd_monokernel:
# both replace the model's forward, which would fail only after weights load
reasons.append("--enable-amd-monokernel is also set; use one of the two")
if reasons:
# a refused setup leaves the environment as it was
for name in env_set:
getattr(envs, name).clear()
raise ValueError(
f"{FLAG} cannot run this setup:\n - " + "\n - ".join(reasons)
)
logger.warning("%s is experimental. Its kernels and coverage may change.", FLAG)
logger.info("%s: %s on %s", FLAG, spec.name, device_isa())
for name in env_set:
logger.info(
"%s: set %s=%s for %s", FLAG, name, getattr(envs, name).get(), spec.name
)


class _InstalledMegakernel(msgspec.Struct, frozen=True):
Expand Down
20 changes: 20 additions & 0 deletions python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,25 @@ def can_run_graph(self, forward_batch: ForwardBatch):
# -----------------------------------------------------------------
# Capture (per-shape)
# -----------------------------------------------------------------
def _run_capture_tail_hooks(self, out, forward_batch, num_tokens: int) -> None:
"""The model runner's capture tail hooks, after the draft forwards of a captured
batch size. The draft steps ran under the per-step attention backends, which
share the draft's KV pool (checked here when a hook exists); a hook that reads
the running backend sees the first."""
hooks = self.model_runner.capture_tail_hooks
if not hooks:
return
backends = self.draft_attn_backend.attn_backends
pool = getattr(backends[0], "token_to_kv_pool", None)
if any(getattr(b, "token_to_kv_pool", None) is not pool for b in backends):
raise RuntimeError(
"the draft's per-step attention backends do not share one KV pool; "
"the capture tail hooks check the running backend's"
)
with forward_context(ForwardContext(attn_backend=backends[0])):
for capture_hook in hooks:
capture_hook(self, out, forward_batch, num_tokens)

def capture_one_shape(
self,
size: int,
Expand Down Expand Up @@ -562,6 +581,7 @@ def run_stock():
forward_batch.spec_info.hidden_states = hidden_states_backup
forward_batch.spec_info.dsa_topk_indices = dsa_topk_indices_backup
forward_batch.positions.sub_(self.eagle_worker.speculative_num_steps - 1)
self._run_capture_tail_hooks(ret, forward_batch, num_tokens)
return ret

def run_once():
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,12 @@ def can_run_graph(self, forward_batch: ForwardBatch):

return is_bs_supported

def _run_capture_tail_hooks(self, out, forward_batch, num_tokens: int) -> None:
"""The model runner's capture tail hooks, after the draft-extend forward of a
captured batch size (under the runner's own attention backend)."""
for capture_hook in self.model_runner.capture_tail_hooks:
capture_hook(self, out, forward_batch, num_tokens)

def capture_one_shape(
self,
size: int,
Expand Down Expand Up @@ -535,6 +541,7 @@ def run_stock():

forward_batch.out_cache_loc = output_cache_loc_backup
forward_batch.spec_info.hidden_states = hidden_states_backup
self._run_capture_tail_hooks(ret, forward_batch, num_tokens)
return ret

def run_once():
Expand Down
19 changes: 17 additions & 2 deletions python/sglang/srt/speculative/eagle_worker_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,21 @@ def __init__(

self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device)

self.target_embed_and_head_shared = False
self._maybe_share_embed_before_pool_sizing()

def _maybe_share_embed_before_pool_sizing(self):
"""Under embedding replication the draft loads its own full embedding table on
every rank. Share the target's now, so that copy is freed before the scheduler
sizes the KV pool instead of after."""
if envs.SGLANG_ENABLE_EMBED_REPLICATION.get():
self._share_target_embed_and_head()

def _share_target_embed_and_head(self):
self.init_token_map()
self.init_lm_head()
self.target_embed_and_head_shared = True

def alloc_memory_pool(
self,
memory_pool_config=None,
Expand All @@ -258,8 +273,8 @@ def alloc_memory_pool(
req_to_token_pool=req_to_token_pool,
token_to_kv_pool_allocator=token_to_kv_pool_allocator,
)
self.init_token_map()
self.init_lm_head()
if not self.target_embed_and_head_shared:
self._share_target_embed_and_head()
self._init_draft_probs_vocab_scatter()

if get_spec().speculative_use_rejection_sampling:
Expand Down
Loading
Loading