Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -626,6 +626,26 @@ RUN --mount=type=cache,target=/root/.cache/uv \
uv pip install --reinstall /tmp/gateway_wheels/*.whl \
&& rm -rf /tmp/gateway_wheels

# sgl-nvidia-mega: the NVIDIA megakernel plugin, inert unless a server passes
# --enable-nvidia-megakernel. Installed with CUDA >= 12.9 only (its kernels are
# SM103, compiled at runtime); disable with --build-arg ENABLE_SGL_NVIDIA_MEGA=0.
ARG ENABLE_SGL_NVIDIA_MEGA=1
ARG SGL_NVIDIA_MEGA_REPO="https://github.com/sgl-project/sgl-nvidia-mega.git"
ARG SGL_NVIDIA_MEGA_COMMIT="d47deb3390e14741c9420ac702954e7611a14da1"
RUN --mount=type=cache,target=/root/.cache/uv \
if [ "${ENABLE_SGL_NVIDIA_MEGA}" != "1" ]; then \
echo "[sgl-nvidia-mega] skip (ENABLE_SGL_NVIDIA_MEGA=${ENABLE_SGL_NVIDIA_MEGA})"; \
elif ! printf '%s\n' 12.9 "${CUDA_VERSION}" | sort -V -C; then \
echo "[sgl-nvidia-mega] skip (CUDA_VERSION=${CUDA_VERSION} < 12.9)"; \
else \
git clone "${SGL_NVIDIA_MEGA_REPO}" /sgl-workspace/sgl-nvidia-mega \
&& cd /sgl-workspace/sgl-nvidia-mega \
&& git checkout -f "${SGL_NVIDIA_MEGA_COMMIT}" \
&& uv pip install --no-deps -e . \
&& cd / \
&& python3 -c "from importlib.metadata import entry_points; assert any(e.name == 'nvidia_megakernel' for e in entry_points(group='sglang.srt.plugins')), 'sgl-nvidia-mega plugin not registered'"; \
fi

# Set workspace directory
WORKDIR /sgl-workspace/sglang

Expand Down
20 changes: 20 additions & 0 deletions docker/Dockerfile.cu134
Original file line number Diff line number Diff line change
Expand Up @@ -877,6 +877,26 @@ RUN --mount=type=cache,target=/root/.cache/pip \
python3 -m pip install --force-reinstall /tmp/gateway_wheels/*.whl \
&& rm -rf /tmp/gateway_wheels

# sgl-nvidia-mega: the NVIDIA megakernel plugin, inert unless a server passes
# --enable-nvidia-megakernel. Installed with CUDA >= 12.9 only (its kernels are
# SM103, compiled at runtime); disable with --build-arg ENABLE_SGL_NVIDIA_MEGA=0.
ARG ENABLE_SGL_NVIDIA_MEGA=1
ARG SGL_NVIDIA_MEGA_REPO="https://github.com/sgl-project/sgl-nvidia-mega.git"
ARG SGL_NVIDIA_MEGA_COMMIT="d47deb3390e14741c9420ac702954e7611a14da1"
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "${ENABLE_SGL_NVIDIA_MEGA}" != "1" ]; then \
echo "[sgl-nvidia-mega] skip (ENABLE_SGL_NVIDIA_MEGA=${ENABLE_SGL_NVIDIA_MEGA})"; \
elif ! printf '%s\n' 12.9 "${CUDA_VERSION}" | sort -V -C; then \
echo "[sgl-nvidia-mega] skip (CUDA_VERSION=${CUDA_VERSION} < 12.9)"; \
else \
git clone "${SGL_NVIDIA_MEGA_REPO}" /sgl-workspace/sgl-nvidia-mega \
&& cd /sgl-workspace/sgl-nvidia-mega \
&& git checkout -f "${SGL_NVIDIA_MEGA_COMMIT}" \
&& python3 -m pip install --no-deps -e . \
&& cd / \
&& python3 -c "from importlib.metadata import entry_points; assert any(e.name == 'nvidia_megakernel' for e in entry_points(group='sglang.srt.plugins')), 'sgl-nvidia-mega plugin not registered'"; \
fi

# Set workspace directory
WORKDIR /sgl-workspace/sglang

Expand Down
6 changes: 6 additions & 0 deletions docs/docs/advanced_features/server_arguments.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1441,6 +1441,12 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Choose the kernels for decode attention layers (have priority over --attention-backend).</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`None`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>triton</code>, <code>torch_native</code>, <code>flex_attention</code>, <code>dsa</code>, <code>nsa</code>, <code>dsv4</code>, <code>compressed</code>, <code>fa3</code>, <code>fa4</code>, <code>flashinfer</code>, <code>flashmla</code>, <code>trtllm_mla</code>, <code>cutedsl_mla</code>, <code>tokenspeed_mla</code>, <code>trtllm_mha</code>, <code>aiter</code>, <code>wave</code>, <code>intel_amx</code>, <code>ascend</code>, <code>intel_xpu</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--enable-nvidia-megakernel`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Enable the NVIDIA megakernel from the sgl-nvidia-mega package. Supported decode and verify steps use its fused kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`False`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>bool flag (set to enable)</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--monokernel-backend`</td>
Expand Down
4 changes: 4 additions & 0 deletions python/sglang/srt/arg_groups/fields/exec_.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,10 @@ class ExecKernel(msgspec.Struct):
Optional[bool],
"Enable Lean (Work-Centric) Attention decode kernel for long-context serving. When None (default), uses auto-gate that activates Lean for long contexts and falls back to standard kernel for short contexts. Set to True to force enable, False to force disable.",
] = None
enable_nvidia_megakernel: A[
bool,
"Enable the NVIDIA megakernel from the sgl-nvidia-mega package. Supported decode and verify steps use its fused kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature.",
] = False
monokernel_backend: A[
Optional[str],
"Select a registered monokernel backend, e.g. amd from sgl-amd-mono. Supported decode and verify steps use persistent kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature.",
Expand Down
3 changes: 3 additions & 0 deletions python/sglang/srt/arg_groups/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,8 +343,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(handle_model_capability_adjustments, server_args)

# needs the resolved CUDA-graph, attention-backend and speculative settings
from sglang.srt.megakernels.guards import handle_nvidia_megakernel
from sglang.srt.monokernels.guards import handle_monokernel

# first, so a server with both kernel flags gets every reason in one error
run_hook(handle_nvidia_megakernel, server_args)
run_hook(handle_monokernel, server_args)

finalize_cuda_graph_prefill_max_context(server_args)
Expand Down
1 change: 1 addition & 0 deletions python/sglang/srt/arg_groups/resolution_hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@
"handle_other_validations",
"handle_model_capability_adjustments",
"handle_monokernel",
"handle_nvidia_megakernel",
"validate_deepep_v2_speculative_draft",
"validate_deepep_v2_dispatch_token_budget",
}
Expand Down
7 changes: 7 additions & 0 deletions python/sglang/srt/megakernels/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
"""NVIDIA megakernels: whole-step dispatch for out-of-tree megakernel plugins.

Plugins register specs through the sglang.srt.plugins entry point. With
--enable-nvidia-megakernel the runtime validates the configuration, builds the
spec after the model is prepared, and routes covered steps through it; other
steps use the model forward.
"""
171 changes: 171 additions & 0 deletions python/sglang/srt/megakernels/binding.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
# SPDX-License-Identifier: Apache-2.0
"""The data SGLang hands a megakernel's build, forward and draft entries.

The one place a megakernel reads SGLang's runtime state: the KV pool, the attention
metadata of the running forward and the Engram tables. A spec's build gets a
``MegakernelBinding`` as ``BuildContext.binding``; a covered forward gets the
running forward's ``ForwardView`` as ``Step.view`` (a draft entry asks
``binding.forward_view()``). Everything else a megakernel touches is the model it
replaces, SGLang's kernel ops, and the reads listed below.
"""

from __future__ import annotations

from typing import Any, Callable, Optional

import torch

# Called by a megakernel, tolerated until offered as data (its contract test pins
# both lists).
TOLERATED_CALLABLES = (
"ForwardView.attention", # the attention backend's own forward
"ForwardView.store_cache", # the KV write
"ForwardView.swa_out_cache_loc",
"ForwardView.low_ratio_sources", # compressor / indexer inputs
"ForwardView.core_compressor",
"KvPoolView.set_swa_key_buffer_radix_fused_norm_rope", # the fused SWA key write
"KvPoolView.pool", # passed back to SGLang's own draft KV store only
"sglang.kernels.ops", # SGLang's kernel ops
"the replaced model's modules, weights and quant methods",
)
# Written into SGLang's objects, tolerated until SGLang owns these buffers.
TOLERATED_WRITES = (
"ForwardView.metadata.q_pad_buffer",
"ForwardView.metadata.low_ratio_req_indices",
"ForwardView.metadata.low_ratio_pos_i64",
)
# Read, tolerated: configuration, not runtime state.
TOLERATED_READS = (
"runtime_context getters (get_parallel, get_exec, get_spec, get_platform)",
"capture-mode queries (get_is_capture_mode, is_in_breakable_cuda_graph)",
"MegakernelBinding.engram_host_table", # the per-layer host table object
)


class KvPoolView:
"""The running forward's KV pool, under the pool's own names: its layout and
per-layer key buffers (a pool offers the ones of its kind)."""

def __init__(self, pool: Any):
self.pool = pool

@property
def uniform_fp8(self) -> bool:
return self.pool.uniform_fp8

@property
def request_window(self) -> Any:
return self.pool.request_window

def get_swa_key_buffer_radix(self, layer_id: int) -> torch.Tensor:
return self.pool.get_swa_key_buffer_radix(layer_id)

def get_swa_key_page_size(self) -> int:
return self.pool.get_swa_key_page_size()

def get_swa_key_bytes_per_token(self) -> int:
return self.pool.get_swa_key_bytes_per_token()

def get_extra_key_buffer(self, layer_id: int) -> Optional[torch.Tensor]:
return self.pool.get_extra_key_buffer(layer_id)

def get_extra_key_page_size(self, layer_id: int) -> int:
return self.pool.get_extra_key_page_size(layer_id)

def get_extra_key_bytes_per_token(self, layer_id: int) -> int:
return self.pool.get_extra_key_bytes_per_token(layer_id)

@property
def set_swa_key_buffer_radix_fused_norm_rope(self) -> Callable[..., None]:
return self.pool.set_swa_key_buffer_radix_fused_norm_rope


class ForwardView:
"""The running forward's attention data, read from its attention backend; built
at capture for a graph step, so its tensors are the ones the graph replays."""

def __init__(self, backend: Any):
self._backend = backend
self.metadata = backend.forward_metadata
self.kv_pool = KvPoolView(backend.token_to_kv_pool)

@property
def trtllm_attn(self) -> bool:
return self._backend.trtllm_attn

@property
def attention(self) -> Callable[..., Any]:
return self._backend.forward

@property
def store_cache(self) -> Callable[..., None]:
return self._backend.store_cache

@property
def swa_out_cache_loc(self) -> Callable[[Any], torch.Tensor]:
return self._backend.get_swa_out_cache_loc

@property
def low_ratio_sources(self) -> Callable[..., Any]:
return self._backend.forward_low_ratio_sources

@property
def core_compressor(self) -> Callable[..., Any]:
return self._backend.forward_core_compressor


class MegakernelBinding:
"""Built at install, before the KV pools exist; read at build or forward. Each
source is sealed at the first read that finds it; ``assert_live`` (after the
decode graphs are captured) raises once one was reallocated."""

def __init__(self, *, model: torch.nn.Module):
self._model = model
self._sealed: dict[str, tuple] = {}

def forward_view(self) -> ForwardView:
"""The view of the forward running now (a target step or a draft step)."""
from sglang.srt.model_executor.forward_context import get_attn_backend

view = ForwardView(get_attn_backend())
self._seal("kv_pool", view.kv_pool.pool)
return view

def engram_history(self) -> Optional[torch.Tensor]:
"""The model's Engram hash history ``[slots + 1, width]`` int32, or None."""
history = self._history()
if history is None or history.shape[1] == 0:
return None
self._seal("engram_history", history)
return history

def engram_host_table(self, engram: torch.nn.Module) -> Any:
"""The host table of one of the model's Engram layers, or None."""
return engram.embed.host_table

def assert_live(self) -> None:
current = {"kv_pool": self._current_pool, "engram_history": self._history}
for name, sealed in self._sealed.items():
if _identity(current[name]()) != sealed:
raise RuntimeError(
f"megakernel binding: {name} was reallocated after its first read"
)

def _seal(self, name: str, value: Any) -> None:
self._sealed.setdefault(name, _identity(value))

def _current_pool(self) -> Any:
from sglang.srt.model_executor.forward_context import get_attn_backend

return get_attn_backend().token_to_kv_pool

def _history(self) -> Optional[torch.Tensor]:
# only models with Engram layers have a hasher
hasher = getattr(self._model, "engram_hasher", None)
return None if hasher is None else hasher.history


def _identity(value: Any) -> tuple:
if isinstance(value, torch.Tensor):
return (value.data_ptr(), tuple(value.shape), value.dtype)
return (id(value),)
Loading
Loading