From d0f43aedb0f90fa72b498ac4b8338b6228de4c24 Mon Sep 17 00:00:00 2001 From: "Shu Wang." Date: Tue, 20 Oct 2026 23:21:00 +0000 Subject: [PATCH 1/8] [Feat] [GB300/Rubin] glm_dsa, spec_step: GLM-5 DSA fused verify and NextN draft; EAGLE v2 step CUDA graph - glm_dsa: target verify (4-16 rows) and NextN draft/draft-extend megakernel specs for GlmMoeDsaForCausalLM on SM103 and SM107 (TP8, NVFP4 MoE). - spec_step: one CUDA graph per EAGLE v2 topk-1 decode step plus bitwise- identical spec fusions, installed via HookRegistry behind env switches. --- .github/CODEOWNERS | 2 + README.md | 19 + pyproject.toml | 1 + sgl_nvidia_mega/glm_dsa/__init__.py | 9 + sgl_nvidia_mega/glm_dsa/attention.py | 729 ++++++ sgl_nvidia_mega/glm_dsa/draft.py | 543 +++++ sgl_nvidia_mega/glm_dsa/draft_attention.py | 209 ++ sgl_nvidia_mega/glm_dsa/draft_spec.py | 126 + sgl_nvidia_mega/glm_dsa/eligibility.py | 399 ++++ sgl_nvidia_mega/glm_dsa/hops.py | 359 +++ sgl_nvidia_mega/glm_dsa/kernels/__init__.py | 2 + .../glm_dsa/kernels/cute/__init__.py | 33 + .../glm_dsa/kernels/cute/attn/__init__.py | 3 + .../glm_dsa/kernels/cute/attn/api.py | 157 ++ .../kernels/cute/attn/indexer_q_proj.py | 384 +++ .../glm_dsa/kernels/cute/attn/mla_prologue.py | 1630 +++++++++++++ .../glm_dsa/kernels/cute/attn/mla_uv_proj.py | 394 ++++ .../kernels/cute/attn/qkv_a_proj_small_m.py | 434 ++++ .../kernels/cute/attn/sparse_mla_decode.py | 666 ++++++ .../glm_dsa/kernels/cute/draft/__init__.py | 4 + .../glm_dsa/kernels/cute/draft/_draft_ptx.py | 63 + .../glm_dsa/kernels/cute/draft/api.py | 69 + .../kernels/cute/draft/distributed_argmax.py | 386 +++ .../kernels/cute/draft/mtp_eh_proj_shard.py | 176 ++ .../kernels/cute/draft/mtp_embed_eh_norm.py | 184 ++ .../glm_dsa/kernels/cute/hop/__init__.py | 6 + .../glm_dsa/kernels/cute/hop/_ptx_local.py | 260 ++ .../kernels/cute/hop/allreduce_norm.py | 186 ++ .../glm_dsa/kernels/cute/hop/api.py | 573 +++++ .../glm_dsa/kernels/cute/hop/constants.py | 69 + .../glm_dsa/kernels/cute/hop/moe_fold.py | 810 +++++++ .../glm_dsa/kernels/cute/hop/o_proj.py | 1157 +++++++++ .../glm_dsa/kernels/cute/hop/transport.py | 798 +++++++ .../glm_dsa/kernels/cute/launch.py | 103 + .../glm_dsa/kernels/cute/moe/__init__.py | 3 + .../glm_dsa/kernels/cute/moe/_core_ptx.py | 256 ++ .../glm_dsa/kernels/cute/moe/api.py | 330 +++ .../glm_dsa/kernels/cute/moe/constants.py | 13 + .../glm_dsa/kernels/cute/moe/engine.py | 2083 +++++++++++++++++ .../glm_dsa/kernels/cute/moe/engine16.py | 1573 +++++++++++++ .../glm_dsa/kernels/cute/moe/finalize.py | 133 ++ .../glm_dsa/kernels/cute/moe/routing.py | 127 + .../glm_dsa/kernels/cute/moe/schedule.py | 150 ++ .../glm_dsa/kernels/cute/moe/weights.py | 221 ++ .../glm_dsa/kernels/cute/packing.py | 140 ++ .../kernels/cute/primitives/__init__.py | 3 + .../kernels/cute/primitives/ptx_core.py | 845 +++++++ .../kernels/cute/primitives/ptx_sys.py | 401 ++++ .../glm_dsa/kernels/cute/primitives/ptx_tc.py | 417 ++++ sgl_nvidia_mega/glm_dsa/moe.py | 308 +++ sgl_nvidia_mega/glm_dsa/providers.py | 595 +++++ sgl_nvidia_mega/glm_dsa/runtime.py | 316 +++ sgl_nvidia_mega/glm_dsa/spec.py | 184 ++ sgl_nvidia_mega/glm_dsa/switches.py | 49 + sgl_nvidia_mega/plugin.py | 9 + sgl_nvidia_mega/spec_step/__init__.py | 12 + .../spec_step/child_graph_capture.py | 177 ++ .../include/sgl_nvidia_mega/spec_sampling.cuh | 512 ++++ sgl_nvidia_mega/spec_step/csrc/sampling.cuh | 230 ++ sgl_nvidia_mega/spec_step/hooks.py | 422 ++++ sgl_nvidia_mega/spec_step/lifted.py | 720 ++++++ sgl_nvidia_mega/spec_step/ops.py | 267 +++ sgl_nvidia_mega/spec_step/runner.py | 1146 +++++++++ sgl_nvidia_mega/spec_step/switches.py | 47 + 64 files changed, 22632 insertions(+) create mode 100644 sgl_nvidia_mega/glm_dsa/__init__.py create mode 100644 sgl_nvidia_mega/glm_dsa/attention.py create mode 100644 sgl_nvidia_mega/glm_dsa/draft.py create mode 100644 sgl_nvidia_mega/glm_dsa/draft_attention.py create mode 100644 sgl_nvidia_mega/glm_dsa/draft_spec.py create mode 100644 sgl_nvidia_mega/glm_dsa/eligibility.py create mode 100644 sgl_nvidia_mega/glm_dsa/hops.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/__init__.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/__init__.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/attn/__init__.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/attn/api.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/attn/indexer_q_proj.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/attn/mla_prologue.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/attn/mla_uv_proj.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/attn/qkv_a_proj_small_m.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/attn/sparse_mla_decode.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/draft/__init__.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/draft/_draft_ptx.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/draft/api.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/draft/distributed_argmax.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/draft/mtp_eh_proj_shard.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/draft/mtp_embed_eh_norm.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/hop/__init__.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/hop/_ptx_local.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/hop/allreduce_norm.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/hop/api.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/hop/constants.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/hop/moe_fold.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/hop/o_proj.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/hop/transport.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/launch.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/moe/__init__.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/moe/_core_ptx.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/moe/api.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/moe/constants.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/moe/engine.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/moe/engine16.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/moe/finalize.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/moe/routing.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/moe/schedule.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/moe/weights.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/packing.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/__init__.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/ptx_core.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/ptx_sys.py create mode 100644 sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/ptx_tc.py create mode 100644 sgl_nvidia_mega/glm_dsa/moe.py create mode 100644 sgl_nvidia_mega/glm_dsa/providers.py create mode 100644 sgl_nvidia_mega/glm_dsa/runtime.py create mode 100644 sgl_nvidia_mega/glm_dsa/spec.py create mode 100644 sgl_nvidia_mega/glm_dsa/switches.py create mode 100644 sgl_nvidia_mega/spec_step/__init__.py create mode 100644 sgl_nvidia_mega/spec_step/child_graph_capture.py create mode 100644 sgl_nvidia_mega/spec_step/csrc/include/sgl_nvidia_mega/spec_sampling.cuh create mode 100644 sgl_nvidia_mega/spec_step/csrc/sampling.cuh create mode 100644 sgl_nvidia_mega/spec_step/hooks.py create mode 100644 sgl_nvidia_mega/spec_step/lifted.py create mode 100644 sgl_nvidia_mega/spec_step/ops.py create mode 100644 sgl_nvidia_mega/spec_step/runner.py create mode 100644 sgl_nvidia_mega/spec_step/switches.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f6fd23b3..e38abd59 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,3 +4,5 @@ # Per model: one owner for its step code, kernels and tests. Each new model's PR # adds its line. **/*deepseek_v41* @YAMY1234 +**/*glm_dsa* @wenscarl +**/*spec_step* @wenscarl diff --git a/README.md b/README.md index 92a8e4ed..40b1d75f 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,28 @@ The package loads as an SGLang [general plugin](https://github.com/sgl-project/s | Model | GPU | Tensor parallel | Coverage | | --- | --- | --- | --- | | DeepSeek-V4.1-Flash | GB300 (SM103) | 4 | DSpark target verify steps of 1 to 384 rows (batch 1 to 64 at 6 rows a request) | +| GLM-5 DSA (NVFP4) | GB300 (SM103), Rubin (SM107) | 8 | EAGLE topk-1 target verify steps of 4 to 16 rows (batch 1 to 4 at 4 rows a request), NextN draft and draft-extend steps of 1 to 16 rows | Larger verify steps, prefill and extend run SGLang's standard path. The DSpark draft step runs on the plugin's draft spec. +GLM-5 DSA needs `--kv-cache-dtype fp8_e4m3 --page-size 64 --attention-backend dsa --dsa-decode-backend trtllm --moe-runner-backend flashinfer_trtllm --speculative-algorithm EAGLE --speculative-eagle-topk 1 --speculative-num-steps 3 --speculative-num-draft-tokens 4` and `SGLANG_ENABLE_EMBED_REPLICATION=1`. Without NVLink multicast (no all-reduce fusion), set `SGLANG_NVIDIA_MEGAKERNEL_GLM_ALLOW_NO_AR_FUSION=1`. + +### Speculative step graph (any EAGLE v2 model) + +Off by default; each switch installs its SGLang hooks only when set to `1`. + +| Switch | Effect | +| --- | --- | +| `SGLANG_NVIDIA_MEGAKERNEL_SPEC_STEP_GRAPH` | One CUDA graph per topk-1 decode step (draft, target verify, draft extend and the work between them), batch size 1 to 8 | +| `SGLANG_SPEC_STEP_GRAPH_FUSED_INPUT_COPY` | The step graph's input copies in one launch | +| `SGLANG_SPEC_PACKED_VERIFY_BROADCAST` | One TP broadcast of the verify result instead of three | +| `SGLANG_SPEC_LEAN_SAMPLED_VERIFY` | Chain sampled verify without the expanded temperatures and zero-filled draft probs | +| `SGLANG_SPEC_CHUNKED_VERIFY_SAMPLING` | With the lean verify: the sampling passes split over vocabulary chunks | +| `SGLANG_SPEC_FUSED_DRAFT_EXTEND_TAIL` | The topk-1 draft-extend argmax, probs and DSA seed gather in two launches | +| `SGLANG_SPEC_FUSED_RELAY` | The relay stash in one scatter, row views for a single request | + +Every fusion's outputs are bitwise identical to the path it replaces. + ### What a covered step runs The step is the plugin's own code over the weights SGLang loaded (`deepseek_v41/runtime`): the embedding and the Engram hashes, then per layer the mHC boundary (`boundary/`), the attention (`attention/`), the MoE (`moe/`) and Engram (`engram/`), then the head and the logits (`logits/`). Each package has a stock implementation that reproduces SGLang's decode step bit for bit, and puts this package's kernels in where the slot plan (`plan.py`) gives them a width: diff --git a/pyproject.toml b/pyproject.toml index b09c6de9..d1edb6fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,3 +24,4 @@ include = ["sgl_nvidia_mega*"] # the kernels' CUDA sources and launch tables ship with the package [tool.setuptools.package-data] "*" = ["*.cu", "*.cuh", "*.json"] +"sgl_nvidia_mega.spec_step" = ["csrc/*.cuh", "csrc/include/sgl_nvidia_mega/*.cuh"] diff --git a/sgl_nvidia_mega/glm_dsa/__init__.py b/sgl_nvidia_mega/glm_dsa/__init__.py new file mode 100644 index 00000000..e5a28762 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/__init__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""GLM-5 DSA (``GlmMoeDsaForCausalLM``, NVFP4, TP8) megakernel on SM103 / SM107: the +fused MTP target-verify step of 1-4 requests and the fused NextN draft layer. + +Read ``spec`` -> ``eligibility`` -> ``runtime`` -> ``providers``; the draft is +``draft_spec`` -> ``draft``. The CuTe DSL kernels are under ``kernels/cute`` and import +no inference engine. +""" diff --git a/sgl_nvidia_mega/glm_dsa/attention.py b/sgl_nvidia_mega/glm_dsa/attention.py new file mode 100644 index 00000000..81cb4c3a --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/attention.py @@ -0,0 +1,729 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""CuTe DSL kernels in the verify attention of the opt-in fused GLM-5 DSA path. + +``FusedAttention`` serves the ``providers.AttentionDecode`` steps of one verify +row count: ``qkv_a`` on ``qkv_a_proj_small_m`` at up to 8 rows (on layers with +their own indexer together with the indexer's key / head-gate projection, left in +``step.index_kw``; elsewhere the stock fused-A GEMM, and on indexer layers the +stock key / head-gate projection right after it, which the prologue consumes); +``prologue`` on ``mla_prologue`` at SGLang's rounding points (q_a / kv_a norms, +``q_b_proj``, W_UK absorption, RoPE + FP8 query left in ``step.q_fp8``, the MLA +and index-K cache rows), so ``index_k``, ``q_absorb`` and ``rope_cache`` pass +through; ``index_q`` on ``indexer_q_proj``; ``sparse_mla`` on ``sparse_mla_decode`` +over the fused top-k page table (16 warps per CTA up to 4 rows, 8 above); +``uv_proj`` on ``mla_uv_proj``. The indexer's logits and top-k stay stock. +With SGLANG_NVIDIA_MEGAKERNEL_GLM_SHARED_QKV_A_SMALL_M (default on) the layers +without their own top-k also run ``qkv_a`` on the small-M kernel, and with +SGLANG_NVIDIA_MEGAKERNEL_GLM_DNORM (default on) on its epilogue-scaled form, which +consumes the fold hop's deferred input norm. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +import numpy as np +import torch + +from sgl_nvidia_mega.glm_dsa import switches +from sgl_nvidia_mega.glm_dsa.eligibility import HIDDEN +from sgl_nvidia_mega.glm_dsa.providers import ( + AttentionDecode, + IndexQ, + PrologueOut, + StepContext, + compile_cached, + cu_stream, +) + +# One tensor-parallel rank (TP 8). +HEADS, Q_LORA, KV_LORA, ROPE, NOPE, VALUE_DIM = 8, 2048, 512, 64, 192, 256 +INDEX_HEADS, INDEX_DIM, TOPK, INDEX_PAGE = 32, 128, 2048, 64 +KV_B_ROWS = HEADS * (NOPE + VALUE_DIM) +Q_B_ROWS = HEADS * (NOPE + ROPE) +MQA_DIM = KV_LORA + ROPE +QKV_A_ROWS = Q_LORA + KV_LORA + ROPE +INDEX_KW = INDEX_DIM + INDEX_HEADS +LOG2E = 1.4426950408889634 + +logger = logging.getLogger(__name__) + + +def _prologue_numerics(): + """The prologue's numeric options: this engine's stock prologue arithmetic.""" + from sgl_nvidia_mega.glm_dsa.kernels.cute.attn.mla_prologue import PrologueNumerics + + return PrologueNumerics( + norm_order="strided_8x256", + norm_weight_plus_zero=True, + kv_norm_bf16=True, + skip_slot0=True, + index_mean_sum="sequential", + index_rope_odd="round_x1_cos", + index_k_bf16=True, + index_scale="fp32_amax", + q_rope="one_product_rounded", + ) + + +# the RoPE arithmetic of this engine's stock indexer query path +INDEX_Q_ROPE_FORM = "mul_sub" + +# L2 prefetch hints (evict_last; no output depends on them) of the value +# up-projection while it waits for the attention core: on the target's layers this +# layer's o_proj weight, which the o_proj hop reads next (of each 48-row slab, the +# rows one hop CTA reads, its first 122,880 B: 15 MiB over the 128 slabs), then the +# first 8 MiB of the next layer's q_a / kv_a weight. +NEXT_QKV_A_PREFETCH_BYTES = 8 << 20 +O_PROJ_STAGE_SLAB_BYTES = 48 * 2048 * 2 +O_PROJ_STAGE_PREFIX_BYTES = 122_880 +EVICT_LAST = 0 + +Slot = Tuple[int, int, int, int, int] +NO_SLOT: Slot = (0, 0, 0, 0, 0) + + +def kernel_rows(rows: int) -> Optional[int]: + """The row capacity (8 or 16) of the kernel forms that serve ``rows``.""" + return 8 if 1 <= rows <= 8 else 16 if rows <= 16 else None + + +def sparse_warps(rows: int) -> int: + """Warps per CTA of the sparse MLA form: one CTA per SM up to 4 rows, two above.""" + return 16 if rows <= 4 else 8 + + +def _f32(x: float) -> float: + return float(np.float32(x)) + + +def _i64(*values) -> list: + from cutlass import Int64 + + return [Int64(v) for v in values] + + +def _addr(t: Optional[torch.Tensor]) -> Tuple[int, int]: + """(address, row stride) of ``t``; (0, 0) for None.""" + return (0, 0) if t is None else (t.data_ptr(), t.stride(0)) + + +def _fail(what: str, cls=RuntimeError) -> Exception: + return cls(f"fused verify attention: {what}") + + +def _prefetch_slot(w, nbytes: int, stride: int = 0, prefix: int = 0) -> Slot: + """An evict_last L2 prefetch slot over the first ``nbytes`` of ``w`` (with + ``stride``: the first ``prefix`` bytes of every ``stride`` bytes), or no slot + unless ``w`` is a contiguous, 128-byte aligned CUDA tensor.""" + if w is None or not w.is_cuda or not w.is_contiguous() or w.data_ptr() % 128: + return NO_SLOT + size = min(nbytes, w.numel() * w.element_size()) + return (w.data_ptr(), size, stride, prefix, EVICT_LAST) + + +def _o_proj_stage_slot(w: torch.Tensor) -> Slot: + """The strided staging slot over this layer's o_proj weight, or no slot unless + it is the expected BF16 [hidden, heads * value_dim] tensor.""" + if w.dtype != torch.bfloat16 or tuple(w.shape) != (HIDDEN, HEADS * VALUE_DIM): + return NO_SLOT + return _prefetch_slot( + w, + w.numel() * w.element_size(), + O_PROJ_STAGE_SLAB_BYTES, + O_PROJ_STAGE_PREFIX_BYTES, + ) + + +def _slot_args(slot: Slot) -> tuple: + from cutlass import Int32 + + return *_i64(*slot[:4]), Int32(slot[4]) + + +def _bf16_weight(linear, shape, what: str) -> torch.Tensor: + weight = linear.weight + if weight.dtype != torch.bfloat16 or tuple(weight.shape) != shape: + raise ValueError( + f"fused verify attention needs a bf16 {what} weight of shape {shape} per " + f"rank, got {weight.dtype} {tuple(weight.shape)}" + ) + return weight + + +def _check_rows(t: torch.Tensor, rows: int, cols: int, dtype, what: str) -> None: + """A [rows, cols] tensor with unit column stride and 16-byte aligned rows.""" + if ( + t.dtype != dtype + or t.dim() != 2 + or tuple(t.shape) != (rows, cols) + or t.stride(1) != 1 + or t.data_ptr() % 16 + or (t.stride(0) * t.element_size()) % 16 + ): + raise _fail(f"unexpected {what} {tuple(t.shape)} / {t.stride()} / {t.dtype}") + + +def _check_index(t: torch.Tensor, rows: int, what: str) -> None: + if t.dtype != torch.int64 or t.dim() != 1 or t.numel() < rows or t.stride(0) != 1: + raise _fail(f"unexpected {what} {tuple(t.shape)} {t.dtype}") + + +@dataclass +class LayerArgs: + """Per-layer constants of the attention kernels: device addresses of tensors + owned by the ``DerivedWeights`` passed to ``AttentionGlue`` (kept alive there), + and the model-owned norm weights / RoPE caches (live as the model).""" + + w_uv: int + stage0: Slot + w_qb: int + w_uk: int + q_norm_w: torch.Tensor + kv_norm_w: torch.Tensor + q_eps: float + kv_eps: float + rope: torch.Tensor + # layers with their own indexer + w_qkva: int = 0 + w_iq: int = 0 + ik_w: Optional[torch.Tensor] = None + ik_b: Optional[torch.Tensor] = None + ik_eps: float = 0.0 + index_rope: Optional[torch.Tensor] = None + w_scale: float = 0.0 + stage1: Slot = NO_SLOT # the value up-projection's second staging slot + # layers without their own indexer: the packed q_a / kv_a weight of the small-M + # projection (0: not served) and the input RMSNorm epsilon (deferred input norm) + w_qkva_s: int = 0 + in_eps: float = 0.0 + + +class FusedAttention(AttentionDecode): + """Verify attention of one row count on the attention-family kernels.""" + + def __init__(self, glue: AttentionGlue, rows: int) -> None: + self.glue, fn = glue, glue.fn + self.max_rows = cap = kernel_rows(rows) + self.uv = fn[f"uv.{cap}"] + self.sparse = fn[f"sparse.{sparse_warps(rows)}"] + self.pro = fn[f"prologue.{cap}"] + # the target's layers that reuse the previous top-k: the end-trigger prologue + self.pro_end = fn.get(f"prologue_end.{cap}") + self.iq = fn[f"index_q.{cap}"] + # the small-M q_a / kv_a forms (8 rows only), if built: + self.qa = fn.get(f"qkv_a.{cap}") # indexer layers + self.qa_s = fn.get(f"qkv_a_shared.{cap}") # shared-top-k layers (opt-in) + self.qa_s_dn = fn.get(f"qkv_a_shared_dn.{cap}") # + deferred input norm + # (DNORM; the composition wires its producer when ``takes_deferred_norm``) + self.takes_deferred_norm = self.qa_s_dn is not None + + def runs_indexer(self, attn, step: StepContext) -> bool: + """Whether this layer's indexer runs in this forward: the layers with their + own top-k (the MTP draft's step group overrides it).""" + return not attn.skip_topk + + def qkv_a(self, attn, x, step: StepContext) -> torch.Tensor: + dn, step.dnorm = step.dnorm, None + if not self.runs_indexer(attn, step): + la = self.glue.layers.get(attn.layer_id) + if dn is not None: + # x is bf16(residual * gamma): only the epilogue-scaled projection + # of this very layer may consume it + if dn[0] != attn.layer_id or self.qa_s_dn is None or not la.w_qkva_s: + raise _fail( + f"layer {attn.layer_id} got the deferred input norm of " + f"layer {dn[0]}" + ) + inv_rms = (dn[1], la.in_eps) + return self._small_m(self.qa_s_dn, la.w_qkva_s, x, inv_rms)[0] + if self.qa_s is not None and la is not None and la.w_qkva_s: + return self._small_m(self.qa_s, la.w_qkva_s, x)[0] + # layers that reuse the previous indexer keep the stock fused-A GEMM + return super().qkv_a(attn, x, step) + if dn is not None: + raise _fail(f"indexer layer {attn.layer_id} got a deferred input norm") + if self.qa is None: + qkv = super().qkv_a(attn, x, step) + step.index_kw, _ = attn.indexer.wk_weights_proj(x) + return qkv + w = self.glue.layers[attn.layer_id].w_qkva + qkv, step.index_kw = self._small_m(self.qa, w, x, index_kw=True) + return qkv + + def _small_m(self, fn, w: int, x, dn=None, index_kw=False): + """``(qkv, kw)``: q_a / kv_a of ``x`` over the packed weight at ``w``, and + with ``index_kw`` the indexer key / head-gate projection (else None); with + ``dn`` (block sums, eps) ``x`` is ``bf16(residual * gamma)`` and the + epilogue applies the row's inverse RMS (deferred input norm).""" + from cutlass import Float32, Int32 + + rows = x.shape[0] + _check_rows(x, rows, HIDDEN, torch.bfloat16, "attention input") + qkv = torch.empty((rows, QKV_A_ROWS), dtype=torch.bfloat16, device=x.device) + kw = None + if index_kw: + kw = torch.empty((rows, INDEX_KW), dtype=torch.bfloat16, device=x.device) + tail = () + if dn is not None: + _check_rows(dn[0], rows, 8, torch.float32, "deferred-norm block sums") + tail = (*_i64(dn[0].data_ptr()), Float32(dn[1])) # bsum, eps + fn( + *_i64(w), # weights + *_i64(*_addr(x)), # x, x_stride + Int32(rows), + *_i64(*_addr(qkv)), # out_main, out_main_stride + *_i64(*_addr(kw)), # out_aux, out_aux_stride (0, 0: none) + *tail, + cu_stream(x.device), + ) + return qkv, kw + + def prologue( + self, attn, qkv_latent: torch.Tensor, step: StepContext + ) -> PrologueOut: + from cutlass import Float32, Int32 + + from sglang.srt.model_executor.forward_context import get_attn_backend + + rows = qkv_latent.shape[0] + _check_rows(qkv_latent, rows, QKV_A_ROWS, torch.bfloat16, "q_a / kv_a latent") + full = self.runs_indexer(attn, step) + dev = qkv_latent.device + la = self.glue.layers[attn.layer_id] + loc = step.forward_batch.out_cache_loc + pos = step.positions + _check_index(loc, rows, "cache slots") + _check_index(pos, rows, "positions") + pool = get_attn_backend().token_to_kv_pool + mla = pool.get_key_buffer(attn.layer_id) + if ( + mla.dim() != 3 + or tuple(mla.shape[1:]) != (1, MQA_DIM) + or mla.element_size() != 1 + ): + raise _fail(f"MLA cache {tuple(mla.shape)}") + mqa = torch.empty((rows, HEADS, MQA_DIM), dtype=torch.float8_e4m3fn, device=dev) + # the indexer operands, on the layers that run the indexer only + kw = qc = None + idx = idx_stride = ik_w = ik_b = irope = irope_stride = 0 + if full: + kw = step.index_kw + if kw is None: + raise _fail("no indexer key projection") + _check_rows(kw, rows, INDEX_KW, torch.bfloat16, "indexer key projection") + if hasattr(pool, "_is_layer_owned") and not pool._is_layer_owned( + attn.layer_id + ): + raise _fail("index-K rows of another owner") + # the stock index-K store's host-side bookkeeping, then its rows + if hasattr(pool, "invalidate_index_buffer_for_layer"): + pool.invalidate_index_buffer_for_layer(attn.layer_id) + buf = pool.get_index_k_with_scale_buffer(layer_id=attn.layer_id) + idx, idx_stride = buf.data_ptr(), buf.stride(0) * buf.element_size() + qc = torch.empty((rows, Q_LORA), dtype=torch.bfloat16, device=dev) + ik_w, ik_b = la.ik_w.data_ptr(), la.ik_b.data_ptr() + irope, irope_stride = _addr(la.index_rope) + base, stride = qkv_latent.data_ptr(), qkv_latent.stride(0) + ones = self.glue.ones.data_ptr() + pro = self.pro if full or self.pro_end is None else self.pro_end + pro( + *_i64(base, stride), # q_c, q_c_stride + *_i64(base + Q_LORA * 2, stride), # kv_c, kv_c_stride + *_i64(base + (Q_LORA + KV_LORA) * 2, stride), # k_pe, k_pe_stride + *_i64(*_addr(kw)), # idx_k, idx_k_stride (0, 0: no indexer) + *_i64(pos.data_ptr(), loc.data_ptr()), # positions, slot_mapping + *_i64(loc.data_ptr() if full else 0), # idx_slot_mapping + Int32(rows), + Int32(int(full)), # has_indexer + *_i64(la.w_qb, la.w_uk), + *_i64(la.q_norm_w.data_ptr(), la.kv_norm_w.data_ptr()), + Float32(la.q_eps), + Float32(la.kv_eps), + *_i64(*_addr(la.rope)), # rope_cache, rope_stride + *_i64(ones, ones), # q_scale, k_scale + *_i64(ik_w, ik_b), # index-K norm weight, bias + Float32(la.ik_eps), + *_i64(irope, irope_stride), # index_rope_cache, index_rope_stride + *_i64(*_addr(mqa)), # mqa_q, mqa_stride + *_i64(mla.data_ptr(), *mla.stride()[:2], 1), # mla_cache, strides, size + *_i64(idx, idx_stride, INDEX_PAGE), # idx_cache, block stride, size + *_i64(0, 0), # topk, topk_stride: unused + Int32(0), # topk_width: unused + *_i64(*_addr(qc)), # qc_out, qc_out_stride + cu_stream(dev), + ) + step.q_fp8 = mqa + return PrologueOut(qc, None, None, None, None) + + def _done(self, *args) -> None: + """A step the prologue did: the index-K rows, the W_UK absorption. + ``q_absorb`` returns None: the prologue left the absorbed FP8 query in + ``step.q_fp8``, which ``rope_cache`` returns.""" + + index_k = q_absorb = _done + + def index_q(self, attn, x, q_lora: torch.Tensor, step: StepContext) -> IndexQ: + from cutlass import Float32, Int32 + + rows = q_lora.shape[0] + _check_rows(q_lora, rows, Q_LORA, torch.bfloat16, "indexer query latent") + _, weights_raw = self._index_kw(attn.indexer, x, step) + la = self.glue.layers[attn.layer_id] + pos = step.positions + _check_index(pos, rows, "positions") + dev = q_lora.device + q = torch.empty( + (rows, INDEX_HEADS, INDEX_DIM), dtype=torch.float8_e4m3fn, device=dev + ) + w = torch.empty((rows, INDEX_HEADS), dtype=torch.float32, device=dev) + self.iq( + *_i64(la.w_iq), # weights + *_i64(*_addr(q_lora)), # x, x_stride + *_i64(pos.data_ptr()), # positions + *_i64(*_addr(la.index_rope)), # rope_cache, rope_stride + *_i64(*_addr(weights_raw)), # index_weights, index_weights_stride + Float32(la.w_scale), + Int32(rows), + *_i64(*_addr(q)), # q_fp8, q_stride + *_i64(*_addr(w)), # w_out, w_out_stride + cu_stream(dev), + ) + return IndexQ(q, w.unsqueeze(-1)) + + def rope_cache(self, attn, q_nope_out, q_pe, k_nope, k_pe, step: StepContext): + if step.q_fp8 is None: + raise _fail("the prologue left no FP8 query") + return step.q_fp8 + + def sparse_mla(self, attn, q_fp8: torch.Tensor, topk_indices, step: StepContext): + from cutlass import Float32, Int32 + + from sglang.srt.model_executor.forward_context import get_attn_backend + + backend = get_attn_backend() + layer = attn.attn_mqa + rows = q_fp8.shape[0] + if topk_indices is None: + raise _fail("no top-k cache rows") + # as the stock path: the fused top-k page table holds physical cache rows + ids = backend._get_fused_topk_page_table( + backend._pad_topk_indices(topk_indices, rows) + ) + if ( + ids.dtype != torch.int32 + or tuple(ids.shape) != (rows, TOPK) + or not ids.is_contiguous() + or q_fp8.dtype != torch.float8_e4m3fn + or tuple(q_fp8.shape) != (rows, HEADS, MQA_DIM) + or not q_fp8.is_contiguous() + ): + raise _fail( + "unexpected sparse MLA inputs " + f"{tuple(q_fp8.shape)} {q_fp8.dtype} / {tuple(ids.shape)} {ids.dtype}" + ) + kv = backend.token_to_kv_pool.get_key_buffer(layer.layer_id) + if kv.shape[-1] != MQA_DIM or not kv.is_contiguous() or kv.data_ptr() % 16: + raise _fail(f"MLA cache {tuple(kv.shape)}") + k_scale = getattr(layer, "k_scale_float", None) + bmm1_scale = 1.0 * (1.0 if k_scale is None else float(k_scale)) * layer.scaling + out = torch.empty( + (rows, HEADS, KV_LORA), dtype=torch.bfloat16, device=q_fp8.device + ) + self.sparse( + *_i64(q_fp8.data_ptr(), kv.data_ptr()), # q, kv + *_i64(ids.data_ptr(), out.data_ptr()), # ids, out + *_i64(kv.numel() // MQA_DIM), # kv_rows + Int32(rows), + Float32(_f32(bmm1_scale * LOG2E)), # scale_log2 + Float32(1.0), # bmm2_scale + # unused: the ids are physical cache rows (no block-table translation) + *_i64(0, 0, 0), # request_of_row, block_table, block_table_stride + Int32(0), # max_blocks + Int32(0), # block_shift + *_i64(0, 0), # block_rows, phys_out + cu_stream(q_fp8.device), + ) + return out + + def uv_proj(self, attn, attn_output: torch.Tensor) -> torch.Tensor: + from cutlass import Int32 + + rows = attn_output.shape[0] + if ( + attn_output.dtype != torch.bfloat16 + or attn_output.dim() != 3 + or tuple(attn_output.shape[1:]) != (HEADS, KV_LORA) + or attn_output.stride(2) != 1 + or not 1 <= rows <= self.max_rows + or attn_output.data_ptr() % 16 + or attn_output.stride(0) % 8 + or attn_output.stride(1) % 8 + ): + raise _fail( + "unexpected value up-projection input " + f"{tuple(attn_output.shape)} / {attn_output.stride()} / " + f"{attn_output.dtype}" + ) + out = torch.empty( + (rows, HEADS * VALUE_DIM), dtype=torch.bfloat16, device=attn_output.device + ) + la = self.glue.layers[attn.layer_id] + self.uv( + *_i64(la.w_uv), # weights + *_i64(attn_output.data_ptr(), *attn_output.stride()[:2]), # attn, strides + *_i64(*_addr(out)), # out, out_row_stride + Int32(rows), + *_slot_args(la.stage0), + *_slot_args(la.stage1), + cu_stream(attn_output.device), + ) + return out + + +def _rope_cache_fp32(cache: torch.Tensor, what: str) -> bool: + """Whether a [positions, cos 32 | sin 32] cache is FP32 (else BF16).""" + if ( + cache.dtype not in (torch.float32, torch.bfloat16) + or cache.dim() != 2 + or cache.shape[1] != ROPE + or cache.stride(1) != 1 + ): + raise _fail( + f"unsupported {what} {tuple(cache.shape)} {cache.dtype}", ValueError + ) + return cache.dtype == torch.float32 + + +class AttentionGlue: + """Derived attention weights, compiled kernels and per-row-count steps.""" + + def __init__(self, model, derived, layers=None) -> None: + from sgl_nvidia_mega.glm_dsa.kernels.cute.packing import ( + pack_indexer_q, + pack_q_b, + pack_qkv_a, + pack_uv_proj, + pack_w_uk_t, + ) + from sglang.srt.layers.layernorm import LayerNorm + + # the target model's glue (the MTP draft passes its own layers) + self.target = layers is None + layers = list(model.model.layers if layers is None else layers) + self.decoder_layers = layers + dev = layers[0].self_attn.kv_b_proj.weight.device + self.grid = torch.cuda.get_device_properties(dev).multi_processor_count + self.ones = torch.ones(1, dtype=torch.float32, device=dev) + self.layers: Dict[int, LayerArgs] = {} + # the compiled kernel forms by ``attn.api.compile_specs`` name: + # "uv|prologue|index_q.{8,16}" (row capacity), "prologue_end.{8,16}" (target + # only), "sparse.{16,8}" (warps per CTA: 16 up to 4 rows, 8 above) and + # "qkv_a[_shared[_dn]].8" + self.fn: Dict[str, object] = {} + rope_f32, index_rope_f32 = set(), set() + + def fused_a(attn) -> torch.Tensor: + qkv_a = attn.fused_qkv_a_proj_with_mqa + return _bf16_weight(qkv_a, (QKV_A_ROWS, HIDDEN), "fused q_a / kv_a") + + # the target's layers without their own indexer, on the small-M projection + # (opt-in): one packed copy of the fused q_a / kv_a weight per layer + shared_qa: Dict[int, torch.Tensor] = {} + if self.target and switches.shared_qkv_a_small_m(): + for i, layer in enumerate(layers): + attn = layer.self_attn + if getattr(attn, "indexer", None) is None and getattr( + attn, "skip_topk", False + ): + packed = pack_qkv_a(fused_a(attn), self.grid) + shared_qa[i] = derived.add("attention.qkv_a_shared", packed) + self.shared_qkv_a_layers = len(shared_qa) + for i, layer in enumerate(layers): + attn = layer.self_attn + kv_b = _bf16_weight(attn.kv_b_proj, (KV_B_ROWS, KV_LORA), "kv_b_proj") + q_b = _bf16_weight(attn.q_b_proj, (Q_B_ROWS, Q_LORA), "q_b_proj") + qkv_a = fused_a(attn) + nxt = None + if i + 1 < len(layers): + # the tensor the next layer's projection reads + nxt = shared_qa.get( + i + 1, layers[i + 1].self_attn.fused_qkv_a_proj_with_mqa.weight + ) + rope = attn.rotary_emb.cos_sin_cache + rope_f32.add(_rope_cache_fp32(rope, "RoPE cache")) + args = LayerArgs( + w_uv=derived.add("attention.uv_proj", pack_uv_proj(kv_b)).data_ptr(), + stage0=_prefetch_slot(nxt, NEXT_QKV_A_PREFETCH_BYTES), + w_qb=derived.add("attention.q_b", pack_q_b(q_b)).data_ptr(), + w_uk=derived.add("attention.w_uk_t", pack_w_uk_t(kv_b)).data_ptr(), + q_norm_w=attn.q_a_layernorm.weight, + kv_norm_w=attn.kv_a_layernorm.weight, + q_eps=float(attn.q_a_layernorm.variance_epsilon), + kv_eps=float(attn.kv_a_layernorm.variance_epsilon), + rope=rope, + ) + self.layers[attn.layer_id] = args + if i in shared_qa: + args.w_qkva_s = shared_qa[i].data_ptr() + args.in_eps = _f32(layer.input_layernorm.variance_epsilon) + if self.target: + # slot 0 stages this layer's o_proj weight, slot 1 the next q_a / kv_a + wo = _o_proj_stage_slot(attn.o_proj.weight) + if wo != NO_SLOT: + args.stage0, args.stage1 = wo, args.stage0 + # every layer with an indexer (the draft's may run it); the target's + # layers that reuse the previous top-k have none + ix = getattr(attn, "indexer", None) + if ix is None: + continue + if ( + not isinstance(ix.k_norm, LayerNorm) + or ix.k_norm.weight.dtype != torch.float32 + or ix.k_norm.bias is None + ): + raise ValueError( + "fused verify attention needs the fp32 LayerNorm index-K norm" + ) + if ix.n_heads != INDEX_HEADS or ix.head_dim != INDEX_DIM: + raise _fail("unexpected indexer shape", ValueError) + wq_b = _bf16_weight( + ix.wq_b, (INDEX_HEADS * INDEX_DIM, Q_LORA), "indexer wq_b" + ) + wkw = _bf16_weight( + ix.wk_weights_proj, (INDEX_KW, HIDDEN), "indexer key / head gate" + ) + args.w_iq = derived.add( + "attention.index_q", pack_indexer_q(wq_b) + ).data_ptr() + args.w_qkva = derived.add( + "attention.qkv_a_index", + pack_qkv_a(torch.cat([qkv_a, wkw], dim=0), self.grid), + ).data_ptr() + args.ik_w, args.ik_b = ix.k_norm.weight, ix.k_norm.bias + args.ik_eps = float(ix.k_norm.variance_epsilon) + args.index_rope = ix.rotary_emb.cos_sin_cache + index_rope_f32.add(_rope_cache_fp32(args.index_rope, "index RoPE cache")) + args.w_scale = _f32(ix.softmax_scale * ix.n_heads**-0.5) + if len(rope_f32) != 1 or len(index_rope_f32) > 1: + raise _fail("RoPE cache dtypes differ by layer", ValueError) + self.rope_f32 = rope_f32.pop() + self.index_rope_f32 = index_rope_f32.pop() if index_rope_f32 else True + + def compile(self, rows: Tuple[int, ...]) -> None: + import cutlass.cute as cute + + from sgl_nvidia_mega.glm_dsa.kernels.cute.attn import api + + shared = QKV_A_ROWS if self.shared_qkv_a_layers else None + specs = api.compile_specs( + sorted({kernel_rows(m) for m in rows} - {None}), + numerics=_prologue_numerics(), + index_q_rope_form=INDEX_Q_ROPE_FORM, + rope_cache_fp32=self.rope_f32, + index_rope_cache_fp32=self.index_rope_f32, + warps=sorted({sparse_warps(m) for m in rows if kernel_rows(m)}), + qkv_a=(QKV_A_ROWS + INDEX_KW, QKV_A_ROWS), + grid=self.grid, + prologue_end=self.target, + qkv_a_shared=shared, + qkv_a_shared_dn=bool(shared) and switches.dnorm(), + ) + for name, (kernel, key, args) in specs.items(): + fn = compile_cached( + kernel, key, lambda k=kernel, a=args: cute.compile(k, *a) + ) + self.fn[name] = _allow_cluster16(fn) if name.startswith("sparse.") else fn + if name == "qkv_a_shared.8": + logger.info( + "GLM DSA fused decode: q_a / kv_a of %d layers without their own " + "indexer on the small-M projection (packed copies %.1f MiB)", + self.shared_qkv_a_layers, + self.shared_qkv_a_layers * QKV_A_ROWS * HIDDEN * 2 / 2**20, + ) + elif name == "qkv_a_shared_dn.8": + logger.info( + "GLM DSA fused decode: deferred input norm: epilogue-scaled q_a / " + "kv_a projection built" + ) + + def step_reasons(self, model, forward_batch) -> List[str]: + """Host-only facts the attention kernels need beyond the stock copies + (read at the first capture of each row count; any reason on any rank + keeps that row count on the stock forward).""" + from sglang.kernels.ops.kvcache.dsa_fused_store_index_cache import ( + can_use_dsa_fused_store, + ) + from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz + from sglang.srt.model_executor.forward_context import get_attn_backend + + pool = get_attn_backend().token_to_kv_pool + reasons: List[str] = [] + page = getattr(pool, "index_page_size", None) + if page != INDEX_PAGE: + reasons.append(f"index-K page size {page} is not {INDEX_PAGE}") + elif is_fp8_fnuz() or not can_use_dsa_fused_store( + torch.bfloat16, torch.int64, page + ): + reasons.append("the fused index-K store is not available") + for layer in self.decoder_layers: + attn = layer.self_attn + lid = attn.layer_id + mla = pool.get_key_buffer(lid) + if ( + mla.dim() != 3 + or tuple(mla.shape[1:]) != (1, MQA_DIM) + or mla.element_size() != 1 + or not mla.is_contiguous() + ): + reasons.append(f"layer {lid}: MLA cache {tuple(mla.shape)} {mla.dtype}") + if getattr(attn, "indexer", None) is None: + continue + if hasattr(pool, "_is_layer_owned") and not pool._is_layer_owned(lid): + reasons.append(f"layer {lid}: index-K rows owned elsewhere") + idx = pool.get_index_k_with_scale_buffer(layer_id=lid) + if ( + idx.dtype != torch.uint8 + or idx.dim() != 2 + or idx.shape[1] != INDEX_PAGE * (INDEX_DIM + 4) + or idx.stride(1) != 1 + ): + reasons.append(f"layer {lid}: index-K cache {tuple(idx.shape)}") + if len(reasons) > 8: + break + return reasons + + def for_rows(self, rows: int) -> Optional[AttentionDecode]: + if f"uv.{kernel_rows(rows)}" not in self.fn: + return None + return FusedAttention(self, rows) + + +def _allow_cluster16(compiled): + """The sparse MLA kernel's 16-CTA cluster is a non-portable cluster size: + allow it on every kernel of the compiled module before its first launch.""" + import cuda.bindings.driver as cuda + + lib = getattr(compiled, "library", None) + if lib is None and hasattr(compiled, "to"): + compiled.to(torch.cuda.current_device()) + lib = getattr(compiled, "library", None) + if lib is None: + raise _fail("no module handle for the cluster size") + _, n = cuda.cuLibraryGetKernelCount(lib) + _, kernels = cuda.cuLibraryEnumerateKernels(n, lib) + dev = torch.cuda.current_device() + attr = cuda.CUfunction_attribute.CU_FUNC_ATTRIBUTE_NON_PORTABLE_CLUSTER_SIZE_ALLOWED + for k in kernels: + res = cuda.cuKernelSetAttribute(attr, 1, k, dev) + err = res[0] if isinstance(res, tuple) else res + if err != cuda.CUresult.CUDA_SUCCESS: + raise _fail(f"cluster size attribute: {err}") + return compiled + + +# the family protocol (providers.build_providers); the MTP draft passes ``layers`` +build = AttentionGlue diff --git a/sgl_nvidia_mega/glm_dsa/draft.py b/sgl_nvidia_mega/glm_dsa/draft.py new file mode 100644 index 00000000..9a24d0dd --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/draft.py @@ -0,0 +1,543 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""CuTe DSL kernels for the GLM-5 DSA NextN draft layer (SM103 / SM107, TP8, MTP topk 1). + +Built by the draft spec (``draft_spec``) unless ``SGLANG_NVIDIA_MEGAKERNEL_GLM_FUSED_DRAFT=0``. Inside the +draft-decode and draft-extend CUDA graphs: + +* the embedding gather and the two eh RMSNorms run as one kernel on the replicated embedding table (needs + ``SGLANG_ENABLE_EMBED_REPLICATION=1``), bitwise equal to ``fused_eh_norm(F.embedding(...))``; +* ``eh_proj`` runs as a column-sharded GEMV (each rank computes its ``hidden / tp`` output columns) followed by an + all-gather, instead of the replicated GEMM on every rank; when the target's fused hops are served + (the target spec), the gather is the hop family's plain all-reduce of a zero-padded row on + the target's mailbox (each column has one non-zero contributor; a -0 column comes out +0); +* the draft top-1 runs as a distributed argmax over each rank's logit shard (one cooperative kernel that exchanges a + (value, index) candidate per row through a symmetric-memory mailbox) instead of the logits all-gather; the stock + ``draft_topk1_postprocess`` then reads a one-hot logits row, so the token it picks is unchanged. + +Every eager step and every unsupported batch runs the stock code. +""" + +from __future__ import annotations + +import logging +from functools import partial +from typing import TYPE_CHECKING, Optional + +import torch +import torch.distributed as dist + +from sglang.srt.distributed import tensor_model_parallel_all_gather +from sglang.srt.distributed.parallel_state import get_tp_group +from sgl_nvidia_mega.glm_dsa import ( + draft_attention, + eligibility, + runtime, +) +from sgl_nvidia_mega.glm_dsa.eligibility import HIDDEN, TP_SIZE +from sgl_nvidia_mega.glm_dsa.providers import compile_cached, cu_stream + +if TYPE_CHECKING: + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + +logger = logging.getLogger(__name__) + +# draft decode: one row per request; draft extend: 4 per request +EH_ROWS = (1, 2, 3, 4, 8, 12, 16) +# the draft-extend graphs select one row per request before the lm_head, so both consumers see B <= 4 rows +TOP1_DECODE_ROWS = 8 +TOP1_EXTEND_ROWS = 8 +# a peer that never publishes makes the waiting ranks trap after this bound (longer than any first-use +# JIT skew between ranks; far below the collective watchdog), after writing the error record +TOP1_TIMEOUT_NS = 30_000_000_000 +# call-site ids of the draft's waits in the error record (the target's hops use layer * 4 + kind) +SITE_TOP1_DECODE, SITE_TOP1_EXTEND, SITE_EH_GATHER = 1001, 1002, 1003 +SITE_NAMES = { + SITE_TOP1_DECODE: "draft top-1 (decode)", + SITE_TOP1_EXTEND: "draft top-1 (extend)", + SITE_EH_GATHER: "draft eh gather", +} +# rows per hop launch (the target's mailbox instance A); 12 / 16 rows run as two launches +HOP_ROWS = 8 +# value table of this engine's eh-norm arithmetic (fused_eh_norm as SGLang JIT-compiles it) +EH_NORM_OPTIONS = dict( + sum_rule="lane_fma_cta_tree", + mean_rule="div_rn", + rsqrt_ftz=False, + apply_rule="x_r_w", +) +# draft decode: draft_topk1_postprocess maps NaN to the fp32 constant -1e30 before its argmax; +# draft extend: torch.argmax (NaN is the maximum, the first NaN wins) +NAN_REPLACEMENT_BITS = 0xF149F2CA +ONE_HOT_FILL = -torch.finfo(torch.float32).max + + +class FusedDraft: + """Kernels, derived weights and top-1 mailboxes of the NextN draft layer.""" + + def __init__(self) -> None: + self.armed = False + self.eh_weight: Optional[torch.Tensor] = None + self._eh = {} + self._front = None + self._top1 = {} + self.error_record = None + self._hops = None # the target's hop kernels (mailboxes, record) when served + self._pad: Optional[torch.Tensor] = None + self._nextn = None + self._attn_family = None # the draft layer's attention (draft_attention) + self._attn_agreed = {} # (mode, rows) -> every rank serves the kernels there + + def arm(self, nextn) -> None: + """After the target's embedding / lm_head are shared: derive, compile, rendezvous.""" + from sgl_nvidia_mega.glm_dsa.kernels.cute import cutlass_supported + + ok, why = cutlass_supported() + if not ok: + logger.warning( + "GLM-DSA fused draft: %s; the draft runs the stock path.", why + ) + return + from sgl_nvidia_mega.glm_dsa.kernels.cute.draft.api import compile_specs + + tp = get_tp_group() + self.rank, self.world = tp.rank_in_group, tp.world_size + self.n_local = HIDDEN // self.world + self.col_off = self.rank * self.n_local + w = nextn.model.eh_proj.weight + if w.dtype != torch.bfloat16 or tuple(w.shape) != (HIDDEN, 2 * HIDDEN): + raise ValueError( + f"GLM-DSA fused draft: eh_proj {tuple(w.shape)} {w.dtype} is not the GLM-5 shape" + ) + self.eh_weight = w[self.col_off : self.col_off + self.n_local].contiguous() + specs = compile_specs( + eh_rows=EH_ROWS, + top1_rows=(TOP1_DECODE_ROWS, TOP1_EXTEND_ROWS), + eh_norm=EH_NORM_OPTIONS, + ) + for rows in EH_ROWS: + spec = specs[f"eh_proj_shard_{rows}"] + self._eh[rows] = (spec[0], _compile(*spec)) + self._front = _compile(*specs["embed_eh_norm"]) + dev = w.device + self._nextn = nextn + self._arm_hops(dev) + self._arm_attention(nextn) + if self.error_record is None: + from sgl_nvidia_mega.glm_dsa.kernels.cute.hop.transport import ErrorRecord + + self.error_record = ErrorRecord(dev) + if runtime.TARGET is not None: + # the target's reporter logs the draft top-1's record too + runtime.TARGET.errors.add(self.failure) + for name, rows in (("decode", TOP1_DECODE_ROWS), ("extend", TOP1_EXTEND_ROWS)): + spec = specs[f"distributed_argmax_{rows}"] + mailbox = _symm_mailbox(spec[0].mailbox_words(self.world), tp, dev) + ctrl = torch.zeros(spec[0].CTRL_WORDS, dtype=torch.int32, device=dev) + self._top1[name] = (spec[0], _compile(*spec), mailbox, ctrl) + torch.cuda.synchronize(dev) + dist.barrier(group=tp.cpu_group) + self.armed = True + logger.info( + "GLM-DSA fused draft armed: eh_proj shard rows %s, top-1 rows <= %d / %d; derived eh_proj shard %.1f MB; " + "hops on the target's mailbox: %s; attention kernels: %s.", + EH_ROWS, + TOP1_DECODE_ROWS, + TOP1_EXTEND_ROWS, + self.eh_weight.numel() * 2 / 1e6, + "eh gather" if self._hops is not None else "no", + "MLA prologue + extend q_a / kv_a GEMM" if self._attn_family else "no", + ) + + def _arm_attention(self, nextn) -> None: + """Build and compile the draft layer's attention kernels when every rank can (else the + draft layer's attention stays stock on every rank).""" + family, why = None, None + try: + family = draft_attention.DraftAttentionGlue(nextn) + family.compile(EH_ROWS) + except ValueError as e: + family, why = None, str(e) + whys = runtime.tp_gather([] if why is None else [why]) + if any(whys): + if self.rank == 0: + logger.warning( + "GLM-DSA fused draft attention stays stock on every rank: %s", + runtime.per_rank(whys), + ) + return + self._attn_family = family + + def _arm_hops(self, dev) -> None: + """Compile the eh gather when the target serves its hops (the same decision on every rank: + the target's ranks agreed on its hop family).""" + family = _target_hop_family() + if family is None: + return + from sgl_nvidia_mega.glm_dsa.kernels.cute.hop import api as hop_api + + hops = family.kernels + specs = {s.name: s for s in hop_api.unit_specs(plain_allreduce=True)} + spec = specs["allreduce"] + if "allreduce" not in hops.compiled: + # the eh gather is the family's plain all-reduce unit (its own cache key) + hops.compiled["allreduce"] = _compile(spec.obj, spec.constexprs, spec.args) + # static: only this rank's columns are ever written, the others stay +0 + self._pad = torch.zeros(max(EH_ROWS), HIDDEN, dtype=torch.bfloat16, device=dev) + self._hops = hops + # the draft's waits write the target's record: one record, one reporter per process + self.error_record = hops.err + family.register_sites(SITE_NAMES) + + def failure(self) -> Optional[dict]: + """The timeout record of the draft's waits (host memory: readable after the CUDA + context is lost), or None.""" + rec = self.error_record.read() if self.error_record else None + if rec is not None and rec["site"] in SITE_NAMES: + rec = {**rec, "wait": SITE_NAMES[rec["site"]]} + return rec + + def engaged(self) -> bool: + return self.armed and torch.cuda.is_current_stream_capturing() + + def front(self, ids, table, prev, enorm_w, hnorm_w, eps) -> Optional[torch.Tensor]: + """[N, 2H] = cat(enorm(table[ids]), hnorm(prev)) as fused_eh_norm(F.embedding(...)), or None.""" + from cutlass import Float32, Int32, Int64 + + n = ids.shape[0] + if ( + not self.engaged() + or n == 0 + or table.dtype != torch.bfloat16 + or table.shape[1] != HIDDEN + ): + return None + out = torch.empty(n, 2 * HIDDEN, dtype=torch.bfloat16, device=ids.device) + self._front( + Int64(ids.data_ptr()), + Int64(table.data_ptr()), + Int64(table.stride(0)), + Int64(table.shape[0]), + Int64(prev.data_ptr()), + Int64(prev.stride(0)), + Int64(enorm_w.data_ptr()), + Int64(hnorm_w.data_ptr()), + Int64(out.data_ptr()), + Int64(out.stride(0)), + Float32(eps), + Int32(n), + cu_stream(), + ) + return out + + def eh_proj(self, eh_input: torch.Tensor) -> Optional[torch.Tensor]: + """Column-sharded eh_proj + all-gather, or None (stock path). With the target's + hops: this rank's columns of the static zero row, then the hop family's plain + all-reduce, at most 8 rows per launch.""" + from cutlass import Int32, Int64 + + rows = eh_input.shape[0] + if not self.engaged() or rows not in self._eh or not eh_input.is_contiguous(): + return None + k, fn = self._eh[rows] + if self._hops is not None: + part, ld, col = self._pad[:rows], HIDDEN, self.col_off + else: + part = torch.empty( + rows, self.n_local, dtype=eh_input.dtype, device=eh_input.device + ) + ld, col = part.stride(0), 0 + fn( + Int64(self.eh_weight.data_ptr()), + Int64(eh_input.data_ptr()), + Int64(part.data_ptr()), + Int32(self.n_local), + Int32(ld), + Int32(col), + Int32(k.grid(self.n_local)), + cu_stream(), + ) + if self._hops is None: + return tensor_model_parallel_all_gather(part, dim=-1) + out = torch.empty_like(part) + for lo in range(0, rows, HOP_ROWS): + hi = min(rows, lo + HOP_ROWS) + self._hops.allreduce(part[lo:hi], out[lo:hi], site=SITE_EH_GATHER) + return out + + def top1_one_hot( + self, logits_shard: torch.Tensor, n_valid: int, vocab: int, extend: bool + ) -> Optional[torch.Tensor]: + """A [rows, vocab] fp32 one-hot of the global top-1 (stock NaN rule), or None.""" + from cutlass import Int32, Int64, Uint32, Uint64 + + rows, vloc = logits_shard.shape + k, fn, mailbox, ctrl = self._top1["extend" if extend else "decode"] + if ( + not self.engaged() + or rows > k.max_rows + or logits_shard.dtype != torch.bfloat16 + ): + return None + k.check_shapes(rows, vloc, n_valid, self.world, self.rank) + ids = torch.empty(rows, dtype=torch.int64, device=logits_shard.device) + vals = torch.empty(rows, dtype=torch.float32, device=logits_shard.device) + fn( + Int64(logits_shard.data_ptr()), + Int64(mailbox.data_ptr()), + Int64(mailbox.peers.data_ptr()), + Int64(ctrl.data_ptr()), + Int64(ids.data_ptr()), + Int64(vals.data_ptr()), + Int32(rows), + Int32(vloc), + Int32(n_valid), + Int32(self.world), + Int32(self.rank), + Uint64(TOP1_TIMEOUT_NS), + Int32(0 if extend else 1), + Uint32(NAN_REPLACEMENT_BITS), + Int64(self.error_record.device_ptr if self.error_record else 0), + Int32(SITE_TOP1_EXTEND if extend else SITE_TOP1_DECODE), + cu_stream(), + ) + one_hot = torch.full( + (rows, vocab), ONE_HOT_FILL, dtype=torch.float32, device=logits_shard.device + ) + one_hot.scatter_(1, ids[:, None], 0.0) + return one_hot + + def attention_for(self, rows: int, forward_batch: ForwardBatch): + """The draft layer's attention step group served at ``rows`` in this forward mode (a + ``draft_attention.DraftAttention``), or None: the stock decoder layer runs. At the first + capture of each (mode, rows), every rank reports the host-only facts the kernels need; any + reason on any rank keeps that draft graph stock on every rank (a host-only gather on the + CPU group, legal inside the capture).""" + family = self._attn_family + if family is None: + return None + mode = "decode" if forward_batch.forward_mode.is_decode() else "extend" + agreed = self._attn_agreed.get((mode, rows)) + if agreed is None: + gathered = runtime.tp_gather( + list(family.step_reasons(self._nextn, forward_batch)) + ) + agreed = self._attn_agreed[(mode, rows)] = not any(gathered) + if self.rank == 0 and agreed: + logger.info( + "GLM-DSA fused draft attention on: draft %s graph, %d rows", + mode, + rows, + ) + elif self.rank == 0: + logger.warning( + "GLM-DSA fused draft attention stays stock: draft %s graph, %d rows: %s", + mode, + rows, + runtime.per_rank(gathered), + ) + return family.for_rows(rows) if agreed else None + + def run(self, model, input_ids, positions, forward_batch: ForwardBatch): + """The served draft forward of an engaged step.""" + attention = self.attention_for(int(positions.shape[0]), forward_batch) + return self.forward_steps(model, input_ids, positions, forward_batch, attention) + + def forward_steps(self, model, input_ids, positions, forward_batch, attention=None): + """``DeepseekModelNextN.forward`` (CUDA path) with the fused front and the sharded eh_proj (each + stock unless engaged), and the decoder layer's attention through ``attention`` (a step group; None: + the stock decoder layer).""" + from sglang.kernels.ops.layernorm.fused_eh_norm import fused_eh_norm + from sglang.srt.eplb.expert_distribution import ( + get_global_expert_distribution_recorder, + ) + from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState + from sglang.srt.layers.layer_boundary.residual import batch as residual_batch + from sglang.srt.utils import BumpAllocator + + # the zero fill stays the plain launch right before the front kernel (its ids / rows are read after its + # grid-dependency wait, which orders it only after the immediately preceding grid) + zero_allocator = BumpAllocator( + buffer_size=2, dtype=torch.float32, device=input_ids.device + ) + hidden_states = None + if input_ids.shape[0] > 0: + prev = forward_batch.spec_info.hidden_states + eh_input = None + # the fused front needs plain token ids (no multimodal embeddings) + if ( + forward_batch.mm_input_embeds is None + and not forward_batch.contains_mm_inputs() + and not model.embed_tokens.enable_tp + ): + eh_input = self.front( + input_ids, + model.embed_tokens.weight, + prev, + model.enorm.weight, + model.hnorm.weight, + model.enorm.variance_epsilon, + ) + if eh_input is None: + embeds = model.embed_input_ids(input_ids, forward_batch) + eh_input = fused_eh_norm( + embeds, + prev, + model.enorm.weight, + model.hnorm.weight, + model.enorm.variance_epsilon, + ) + hidden_states = self.eh_proj(eh_input) + if hidden_states is None: + hidden_states = model.eh_proj(eh_input) + else: + hidden_states = model.embed_input_ids(input_ids, forward_batch) + + residual_batch.start(forward_batch) + index_topk_share = IndexTopKShareState.from_mtp_carry(forward_batch) + with get_global_expert_distribution_recorder().disable_this_region(): + decoder = model.decoder + if attention is not None: + decoder = partial( + draft_attention.decoder_layer_forward, model.decoder, attention + ) + hidden_states, topk_indices = decoder( + positions, + hidden_states, + forward_batch, + zero_allocator, + prev_topk_indices=index_topk_share.topk_indices, + ) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) + if not forward_batch.forward_mode.is_idle(): + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, model.shared_head.norm + ) + index_topk_share.update(topk_indices) + index_topk_share.publish() + return hidden_states + + +def make_draft_logits_processor(config, fused_draft: FusedDraft): + """The stock logits processor whose TP logits path serves the draft top-1 from the distributed argmax.""" + from sglang.srt.layers.logits_processor import LogitsProcessor + + class GlmDsaDraftLogitsProcessor(LogitsProcessor): + def _get_logits( + self, + hidden_states, + lm_head, + logits_metadata, + embedding_bias=None, + use_logits_buffer=True, + ): + fd = fused_draft + mode = logits_metadata.forward_mode + if ( + not fd.engaged() + or embedding_bias is not None + or self.logit_scale is not None + or self.final_logit_softcapping + or self.do_tensor_parallel_all_gather_dp_attn + or not self.do_tensor_parallel_all_gather + or self.use_attn_tp_group + or not (mode.is_decode() or mode.is_draft_extend_v2()) + ): + return super()._get_logits( + hidden_states, + lm_head, + logits_metadata, + embedding_bias, + use_logits_buffer, + ) + si = lm_head.shard_indices + shard = self._compute_lm_head(hidden_states, lm_head, embedding_bias) + one_hot = None + if ( + si.num_added_elements == 0 + and si.org_vocab_start_index == fd.rank * shard.shape[1] + ): + one_hot = fd.top1_one_hot( + shard, + si.num_org_elements, + self.vocab_size, + extend=mode.is_draft_extend_v2(), + ) + # same ranks, same decision: everyone takes the stock gather + if one_hot is None: + one_hot = self._logits_gatherer(shard) + return self._copy_logits_to_buffer( + one_hot, logits_metadata, use_buffer=use_logits_buffer + ) + + return GlmDsaDraftLogitsProcessor(config) + + +def _compile(kernel, constexprs, args): + """``cute.compile(kernel, *args)`` through the fused path's compile cache.""" + import cutlass.cute as cute + + return compile_cached(kernel, constexprs, lambda: cute.compile(kernel, *args)) + + +def _target_hop_family(): + """The prepared target's hop family when its fused hops are served on every + rank, else None.""" + rt = runtime.TARGET + family = rt.families.get("hops") if rt is not None else None + return family if getattr(family, "kernels", None) is not None else None + + +def _symm_mailbox(words: int, tp, device) -> torch.Tensor: + import torch.distributed._symmetric_memory as symm_mem + + t = symm_mem.empty((words,), dtype=torch.int64, device=device) + t.fill_(-1) + handle = symm_mem.rendezvous(t, tp.device_group.group_name) + t.peers = torch.tensor( + [int(p) for p in handle.buffer_ptrs], dtype=torch.int64, device=device + ) + t.handle = handle + return t + + +def check_draft_config(model) -> None: + """Startup checks of the fused draft (an unsupported setting raises ``ValueError``).""" + from sglang.srt.environ import envs + from sglang.srt.runtime_context import get_parallel, get_spec + + problems = [] + + def need(ok: bool, what: str) -> None: + if not ok: + problems.append(what) + + cfg, spec, par = model.config, get_spec(), get_parallel() + need( + torch.cuda.is_available() + and torch.cuda.get_device_capability() in eligibility._SM_ACCEPTED, + "an SM103 or SM107 GPU", + ) + need(getattr(cfg, "hidden_size", None) == HIDDEN, f"hidden size {HIDDEN} (GLM-5)") + need(par.tp_size == TP_SIZE, f"tensor parallel size {TP_SIZE}") + need(getattr(par, "pp_size", 1) == 1, "pipeline parallel size 1") + need(not getattr(par, "enable_dp_attention", False), "no DP attention") + need(not getattr(par, "enable_dp_lm_head", False), "no DP lm_head") + need(getattr(spec, "speculative_eagle_topk", 1) == 1, "MTP / EAGLE with topk 1") + need( + not getattr(spec, "speculative_use_rejection_sampling", False), + "greedy draft acceptance (no draft probs)", + ) + need( + envs.SGLANG_ENABLE_EMBED_REPLICATION.get(), + "SGLANG_ENABLE_EMBED_REPLICATION=1 (replicated draft table)", + ) + if problems: + raise ValueError( + "the GLM-5 DSA fused draft needs: " + "; ".join(problems) + "." + ) + + +def create_fused_draft(model) -> FusedDraft: + check_draft_config(model) + return FusedDraft() diff --git a/sgl_nvidia_mega/glm_dsa/draft_attention.py b/sgl_nvidia_mega/glm_dsa/draft_attention.py new file mode 100644 index 00000000..79213cc7 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/draft_attention.py @@ -0,0 +1,209 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""Attention of the GLM-5 DSA NextN draft layer in steps, and the draft layer's +forward with its attention in steps. + +``DraftAttention`` is the verify step group (``providers.AttentionDecode``): a +draft forward takes its branches in DRAFT_EXTEND_V2, and in its DECODE passes the +sparse MLA of the DSA backend's ``forward_decode`` (``sparse_mla(decode=True)``). +The stock DECODE prologue also overlaps ``q_b_proj`` with the indexer on a side +stream; the served draft runs the fused prologue instead, and the stock bodies +compute the same values on one stream. ``decoder_layer_forward`` is +``DeepseekV2DecoderLayer.forward`` (main 44b44774) with the attention called +through a step group; on the stock bodies it is bitwise equal to the stock draft +layer (the draft replica test checks it in both modes). + +``DraftFusedAttention`` serves the attention family's MLA prologue in every draft +pass and its q_a / kv_a GEMM (with the indexer key / head-gate projection) in the +draft extend at up to 8 rows; the indexer query, the indexer, the sparse MLA and +the value up-projection stay stock. The draft layer has its own weights and KV +pool, hence its own ``AttentionGlue``; compiled forms are shared with the target +through the compile cache. +""" + +from __future__ import annotations + +from contextlib import nullcontext +from dataclasses import dataclass +from typing import List, Optional + +import torch + +from sgl_nvidia_mega.glm_dsa.attention import ( + AttentionGlue, + FusedAttention, + kernel_rows, +) +from sgl_nvidia_mega.glm_dsa.providers import ( + AttentionDecode, + StepContext, +) +from sgl_nvidia_mega.glm_dsa.runtime import DerivedWeights + + +@dataclass +class DraftStepContext(StepContext): + """``StepContext`` of one draft forward.""" + + # Whether this forward's draft layer runs its own indexer (the NextN layer + # does whenever no top-k carry is passed in: should_run_indexer). + run_indexer: Optional[bool] = None + + +class DraftAttention(AttentionDecode): + """The verify attention steps, plus the DECODE-mode sparse MLA of a draft + forward. Kernel overrides of the draft layer subclass this group.""" + + def forward(self, layer, x, prev_topk_indices, step: DraftStepContext): + step.run_indexer = layer.self_attn.should_run_indexer(prev_topk_indices) + return super().forward(layer, x, prev_topk_indices, step) + + def check_mode(self, attn, mode) -> None: + # Stock tests is_decode_or_idle(); IDLE batches need DP attention, which + # the fused draft refuses, so is_decode() selects the same branches. + if mode.is_decode(): + _check_decode_branches(attn) + else: + super().check_mode(attn, mode) + + def sparse_mla(self, attn, q_fp8, topk_indices, step: DraftStepContext): + decode = step.forward_batch.forward_mode.is_decode() + return super().sparse_mla(attn, q_fp8, topk_indices, step, decode) + + +def _decode_problems(attn=None) -> List[str]: + """The configurations the DECODE copy (and the draft prologue) do not cover.""" + from sglang.srt.model_executor.forward_context import get_attn_backend + from sglang.srt.model_executor.runner_utils.capture_mode import ( + get_capture_attention_variant, + ) + + backend = get_attn_backend() + variant = get_capture_attention_variant() + problems = [] if variant is None else [f"capture attention variant {variant!r}"] + if getattr(backend, "dcp_size", 1) > 1: + problems.append("decode context parallelism") + if not getattr(backend, "use_fused_topk", False): + problems.append("no fused top-k page table") + if attn is not None and (attn.q_lora_rank is None or not attn.use_dsa): + problems.append("not a DSA attention with q_lora") + return problems + + +def _check_decode_branches(attn) -> None: + """Any other configuration is a programming error (the fused draft engages + only on this one).""" + problems = _decode_problems(attn) + if problems: + raise RuntimeError( + "DraftAttention DECODE steps do not copy this configuration: " + + "; ".join(problems) + ) + + +class DraftFusedAttention(FusedAttention, DraftAttention): + """Draft attention with the MLA prologue on the family's kernel.""" + + # The draft rule: these steps stay stock in the draft layer. + index_q = AttentionDecode.index_q + sparse_mla = DraftAttention.sparse_mla + uv_proj = AttentionDecode.uv_proj + + def __init__(self, glue: AttentionGlue, rows: int) -> None: + super().__init__(glue, rows) + # qkv_a (per forward): the family's GEMM in the draft extend, else the + # stock fused-A GEMM (+ the indexer key / head-gate projection) + self._qa_extend, self.qa = self.qa, None + + def runs_indexer(self, attn, step) -> bool: + # The draft layer runs its own indexer in the forwards without a top-k + # carry (DraftAttention.forward), although it is marked skip_topk. + return bool(step.run_indexer) + + def qkv_a(self, attn, x, step) -> torch.Tensor: + decode = step.forward_batch.forward_mode.is_decode() + self.qa = None if decode else self._qa_extend + return super().qkv_a(attn, x, step) + + +class DraftAttentionGlue(AttentionGlue): + """The draft layer's packed attention weights (owned here), compiled forms + and step groups per row count.""" + + def __init__(self, nextn) -> None: + self.derived = DerivedWeights() + super().__init__(nextn, self.derived, layers=[nextn.model.decoder]) + + def step_reasons(self, nextn, forward_batch) -> List[str]: + """Host-only facts the draft prologue needs (read at the first capture of + each draft graph; any reason on any rank keeps it stock).""" + return super().step_reasons(nextn, forward_batch) + _decode_problems() + + def for_rows(self, rows: int) -> Optional[DraftAttention]: + if f"prologue.{kernel_rows(rows)}" not in self.fn: + return None + return DraftFusedAttention(self, rows) + + +def decoder_layer_forward( + layer, + attention: AttentionDecode, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch, + zero_allocator, + prev_topk_indices: Optional[torch.Tensor] = None, +): + """``DeepseekV2DecoderLayer.forward`` (main 44b44774) for the draft layer, + with ``self.self_attn(...)`` replaced by the step group's ``forward`` and + ``o_proj`` (``forward_absorb_core`` ends with ``self.o_proj``). Returns + ``(hidden_states, topk_indices)`` as the stock layer does.""" + from sglang.srt.layers.layer_boundary import get_attn_tp_context + from sglang.srt.layers.layer_boundary.residual import access as residual_access + from sglang.srt.models.deepseek_v2 import DeepseekV2MoE + + hidden_states_orig = residual_access.buffer(hidden_states) + # The NextN forward passes no captured outputs, GEMM-output allocator or + # next full-attention layer (its prefetch is then a no-op). + hidden_states = layer.attn_boundary.prepare( + hidden_states, + forward_batch, + capture_gathered=None, + quant_format=layer._resolve_gfx95_quant_format(), + ) + step = DraftStepContext( + forward_batch=forward_batch, + positions=positions, + zero_allocator=zero_allocator, + ) + # The step group enters the attention's decode-TP context itself and clears + # the attention inputs; o_proj then runs in a second entry of that context. + # Stock runs both in one entry and clears after o_proj. Equivalent here: the + # context only acts with decode context parallelism (refused), and o_proj + # reads no attention input. + o_input, topk_indices = attention.forward( + layer, hidden_states, prev_topk_indices, step + ) + with layer.self_attn.maybe_use_decode_attn_tp(forward_batch): + hidden_states, _ = layer.self_attn.o_proj(o_input) + get_attn_tp_context().clear_attn_inputs() + + hidden_states = layer.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = layer.ffn_boundary.prepare(hidden_states, forward_batch) + + if ( + isinstance(layer.mlp, DeepseekV2MoE) + and not layer.mlp.experts.moe_runner_config.inplace + and not torch.compiler.is_compiling() + and isinstance(hidden_states_orig, torch.Tensor) + ): + from sglang.srt.layers.moe.moe_runner.base import moe_output_buffer_ctx + + mlp_ctx = moe_output_buffer_ctx(hidden_states_orig) + else: + mlp_ctx = nullcontext() + + with layer.ffn_boundary.exit(forward_batch) as ffn_exit, mlp_ctx: + hidden_states = layer.mlp(hidden_states, forward_batch, None) + hidden_states = ffn_exit.finish(hidden_states) + return hidden_states, topk_indices diff --git a/sgl_nvidia_mega/glm_dsa/draft_spec.py b/sgl_nvidia_mega/glm_dsa/draft_spec.py new file mode 100644 index 00000000..cab031fb --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/draft_spec.py @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The GLM-5 DSA NextN draft's spec: the fused draft layer (``draft.FusedDraft``) in +the draft-decode and draft-extend CUDA graphs. + +The draft kernels are compiled and their mailboxes rendezvous once the target's +embedding and LM head are shared with the draft (``set_embed_and_head``, after the +draft's build). ``SGLANG_NVIDIA_MEGAKERNEL_GLM_FUSED_DRAFT=0`` keeps SGLang's draft. +""" + +from __future__ import annotations + +from sglang.srt.megakernels.spec import ( + Feature, + MegakernelSpec, + Numerics, + NumericsKind, + Requirements, +) + +from sgl_nvidia_mega.glm_dsa import spec as target +from sgl_nvidia_mega.glm_dsa import switches + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/megakernels/spec.py": ( + "Feature", + "MegakernelSpec", + "Numerics", + "NumericsKind", + "Requirements", + "Step", + ), + "srt/models/deepseek_v2.py": ("set_embed_and_head", "logits_processor", "lm_head"), +} + +# draft decode: one row per request; draft extend: 4 rows per request +DRAFT_ROWS = (1, 2, 3, 4, 8, 12, 16) + +# The draft top-1 is the stock argmax of the same logits (the distributed argmax +# keeps its NaN and tie rules); the embedding front is bitwise fused_eh_norm. +NUMERICS = Numerics( + kind=NumericsKind.TOLERANCE, + tolerances={"draft.top1.max_abs": 0.0, "draft.embed_eh_norm.max_abs": 0.0}, +) + + +class _BuiltDraft: + """The installed draft forward: armed steps run the fused draft layer.""" + + draft_sampler = None + widths = DRAFT_ROWS + + def __init__(self, model): + from sgl_nvidia_mega.glm_dsa import draft + + self.model = model + self.fused = draft.create_fused_draft(model) + model.logits_processor = draft.make_draft_logits_processor( + model.config, self.fused + ) + share = model.set_embed_and_head + + def set_embed_and_head(embed, head): + share(embed, head) + self.fused.arm(model) + + # an instance attribute: the class and every other instance keep theirs + model.set_embed_and_head = set_embed_and_head + + def summary(self) -> str: + return "GLM-5 NextN draft: fused embedding front, eh_proj shard, distributed top-1" + + def plan(self, input_ids, forward_batch, *args, **kwargs): + from sglang.srt.megakernels.spec import Step + + pp_proxy = kwargs.get("pp_proxy_tensors", args[0] if args else None) + if pp_proxy is not None or not self.fused.engaged(): + return None + mode = "decode" if forward_batch.forward_mode.is_decode() else "draft_extend" + return Step(mode=mode, rows=int(input_ids.shape[0])) + + def forward(self, step, original, input_ids, positions, forward_batch, *a, **kw): + hidden_states = self.fused.run(self.model.model, input_ids, positions, forward_batch) + return self.model.logits_processor( + input_ids, hidden_states, self.model.lm_head, forward_batch + ) + + +class _StockDraft: + """SGLANG_NVIDIA_MEGAKERNEL_GLM_FUSED_DRAFT=0: no step, SGLang's draft forward.""" + + draft_sampler = None + widths = () + + def summary(self) -> str: + return "GLM-5 NextN draft: SGLang's forward" + + def plan(self, *args, **kwargs): + return None + + def forward(self, step, original, *args, **kwargs): + raise AssertionError("the stock draft covers no step") + + +def _build(model, hf_config, ctx): + return _BuiltDraft(model) if switches.fused_draft() else _StockDraft() + + +SPEC = MegakernelSpec( + name="GLM-5 DSA NextN draft", + # the target's device, TP size, dtype and decode graphs: the draft shares its GPUs + requires=Requirements( + isas=target.REQUIREMENTS.isas, + tp_sizes=target.REQUIREMENTS.tp_sizes, + dtypes=target.REQUIREMENTS.dtypes, + decode_graph_backends=target.REQUIREMENTS.decode_graph_backends, + env=target.REQUIREMENTS.env, + ), + features=frozenset({Feature.SPECULATIVE_DECODING, Feature.MULTI_NODE_TP}), + numerics=NUMERICS, + # 0 keeps SGLang's draft forward (no step covered, nothing installed) + max_rows=max(DRAFT_ROWS) if switches.fused_draft() else 0, + build=_build, + api_version=target.API_VERSION, +) diff --git a/sgl_nvidia_mega/glm_dsa/eligibility.py b/sgl_nvidia_mega/glm_dsa/eligibility.py new file mode 100644 index 00000000..ed4a61d3 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/eligibility.py @@ -0,0 +1,399 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""When the fused GLM-5 DSA decode / verify path engages. + +* Tier 1, at construction (``check_construction``): server arguments and + checkpoint shapes; on opt-in an unsupported setting raises ``ValueError`` + naming every reason. +* Tier 2, after loading (``check_loaded``): facts of the built, processed modules. +* Tier 3, per forward (``step_in_graph_runner``): host-only facts of the batch, + never device data, so a captured graph and its replays agree. At the first + capture of each row count ``attention_step_reasons`` checks that the stock + attention takes the branches the attention steps copy (agreed across ranks). +""" + +from __future__ import annotations + +from typing import List + +import torch + +# Verify rows per forward: 1-4 requests x 4 tokens (3 draft tokens + 1). +VERIFY_TOKENS_PER_REQUEST = 4 +SUPPORTED_ROWS = (4, 8, 12, 16) +TP_SIZE = 8 +HIDDEN = 6144 +_SM_ACCEPTED = ((10, 3), (10, 7)) +# what the refusals name as the switch that asked for the path +SWITCH = "--enable-nvidia-megakernel (GLM-5 DSA)" + +# GLM-5 DSA checkpoint shapes the path is built for. +_EXPECTED_SHAPES = { + "hidden_size": HIDDEN, + "num_attention_heads": 64, + "q_lora_rank": 2048, + "kv_lora_rank": 512, + "qk_rope_head_dim": 64, + "qk_nope_head_dim": 192, + "v_head_dim": 256, + "index_n_heads": 32, + "index_head_dim": 128, + "index_topk": 2048, + "n_routed_experts": 256, + "num_experts_per_tok": 8, + "n_shared_experts": 1, + "moe_intermediate_size": 2048, + "first_k_dense_replace": 3, + "num_hidden_layers": 78, +} + +# Server arguments the path is validated at: name -> accepted values (the last +# one is the one to pass; None = unset). +_VALUES = { + "tp_size": (TP_SIZE,), + **dict.fromkeys( + ("pp_size", "dp_size", "ep_size", "dcp_size", "attn_cp_size", "moe_dp_size"), + (None, 1), + ), + "moe_a2a_backend": (None, "none"), + "kv_cache_dtype": ("fp8_e4m3",), + "page_size": (64,), + "attention_backend": ("dsa",), + "decode_attention_backend": (None, "dsa"), + "dsa_decode_backend": ("trtllm",), + "moe_runner_backend": ("flashinfer_trtllm",), + "init_expert_location": (None, "trivial"), + "expert_distribution_recorder_mode": (None,), +} +_EAGLE_VALUES = { + "speculative_eagle_topk": (1,), + "speculative_num_draft_tokens": (VERIFY_TOKENS_PER_REQUEST,), + "speculative_num_steps": (VERIFY_TOKENS_PER_REQUEST - 1,), +} +# Server arguments that must be off: features, layouts and communication +# variants that change the verify step's stock op sequence. +_OFF = tuple( + """enable_dp_attention enable_prefill_cp enable_cp_decode_attn_tp + enable_kv_cache_sharding enable_layernorm_sp enable_attn_tp_input_scattered + enable_dense_mlp_attn_tp enable_shared_experts_attn_tp enable_quant_communications + enable_fused_moe_sum_all_reduce enable_dp_lm_head enable_hisparse + enable_dsa_cache_layer_split enforce_shared_experts_fusion enable_eplb + ep_num_redundant_experts enable_return_routed_experts enable_return_indexer_topk + enable_lora lora_paths enable_deterministic_inference enable_two_batch_overlap + enable_single_batch_overlap enable_torch_compile disable_cuda_graph + disable_decode_cuda_graph""".split() +) +_EAGLE_OFF = ("speculative_adaptive", "enable_multi_layer_eagle") +# Reasons the generated "requires --x " / "--x is not supported" would +# misstate (no such flag, or a hint lost); said once when two flags share one. +_WHY = { + "expert_distribution_recorder_mode": ( + "the expert distribution recorder is not supported" + ), + "enforce_shared_experts_fusion": ( + "requires shared-experts fusion off (--disable-shared-experts-fusion)" + ), + **dict.fromkeys( + ("disable_cuda_graph", "disable_decode_cuda_graph"), + "requires decode CUDA graphs (the path runs only inside captured full graphs)", + ), +} + + +def _mismatches(server_args, values, off) -> List[str]: + def flag(name): + return "--" + name.replace("_", "-") + + out = [] + for name, ok in values.items(): + got = getattr(server_args, name, None) + if got not in ok: + out.append(_WHY.get(name, f"requires {flag(name)} {ok[-1]} (got {got!r})")) + out += [ + _WHY.get(n, f"{flag(n)} is not supported") + for n in off + if getattr(server_args, n, None) + ] + return list(dict.fromkeys(out)) + + +def _sm_reason(cap) -> List[str]: + if cap is None and torch.cuda.is_available(): + cap = torch.cuda.get_device_capability() + if tuple(cap or ()) in _SM_ACCEPTED: + return [] + return [ + f"requires an SM103 or SM107 GPU (found compute capability {cap}); " + "other architectures are not validated" + ] + + +def _ar_fusion_ok(applies, rows: int) -> bool: + from sgl_nvidia_mega.glm_dsa import switches + + return applies(rows) or switches.allow_no_ar_fusion() + + +def _refuse(switch: str, what: str, reasons: List[str]) -> None: + if reasons: + raise ValueError( + f"{switch} is set but {what} is not supported:\n - " + + "\n - ".join(reasons) + ) + + +def construction_reasons(config, server_args, *, device_capability=None) -> List[str]: + """Every reason the configuration is not supported (empty if supported).""" + from sglang.srt.environ import envs + + reasons = _sm_reason(device_capability) + if getattr(config, "model_type", None) != "glm_moe_dsa": + reasons.append( + f"model_type {getattr(config, 'model_type', None)!r} is not glm_moe_dsa" + ) + for key, want in _EXPECTED_SHAPES.items(): + if getattr(config, key, None) != want: + reasons.append( + f"config.{key} = {getattr(config, key, None)!r}, expected {want!r}" + ) + reasons += _mismatches(server_args, _VALUES, _OFF) + # Speculative decoding: EAGLE / MTP chain, 3 draft tokens (NEXTN resolves to EAGLE). + algo = getattr(server_args, "speculative_algorithm", None) + if algo is None: + reasons.append( + "requires speculative decoding (EAGLE / NEXTN with the MTP layer); " + "plain decode is not supported" + ) + elif str(algo).upper() != "EAGLE": + reasons.append( + f"speculative algorithm {algo!r} is not supported (EAGLE / NEXTN only)" + ) + else: + reasons += _mismatches(server_args, _EAGLE_VALUES, _EAGLE_OFF) + # The attention steps implement the bf16 absorbed query and the fused DSA + # top-k page table (providers.AttentionDecode). + if envs.SGLANG_EXPERIMENTAL_LORA_OPTI.get(): + reasons.append("SGLANG_EXPERIMENTAL_LORA_OPTI is not supported") + if envs.SGLANG_ENABLE_DSA_Q8KV8_BORN_FP8_Q.get(): + reasons.append("SGLANG_ENABLE_DSA_Q8KV8_BORN_FP8_Q is not supported") + if not envs.SGLANG_DSA_FUSE_TOPK.get(): + reasons.append("requires SGLANG_DSA_FUSE_TOPK (the fused DSA top-k page table)") + # The path runs only inside captured full decode graphs. + cfg = getattr(server_args, "cuda_graph_config", None) or {} + entry = cfg.get("decode") if isinstance(cfg, dict) else getattr(cfg, "decode", None) + backend = ( + entry.get("backend") + if isinstance(entry, dict) + else getattr(entry, "backend", None) + ) + if backend not in (None, "full"): + reasons.append(f"requires the full decode CUDA graph backend (got {backend!r})") + return reasons + + +def check_construction(model) -> None: + """Tier 1. Raise ``ValueError`` listing every unsupported setting.""" + from sglang.srt.runtime_context import get_server_args + + reasons = construction_reasons(model.config, get_server_args()) + if getattr(model, "num_fused_shared_experts", 0) != 0: + reasons.append( + "requires shared-experts fusion off (num_fused_shared_experts == 0; " + "pass --disable-shared-experts-fusion)" + ) + _refuse(SWITCH, "the configuration", reasons) + + +def check_loaded(model) -> None: + """Tier 2. Raise ``ValueError`` listing every unsupported fact of the loaded model.""" + from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion + from sglang.srt.layers.layer_boundary.residual.add_norm import ( + flashinfer_ar_fusion_applies, + ) + from sglang.srt.layers.moe.utils import post_experts_output_is_complete + from sglang.srt.models.deepseek_v2 import DeepseekV2MLP, DeepseekV2MoE + + reasons: List[str] = [] + mm = model.model + if getattr(model, "capture_aux_hidden_states", False): + reasons.append( + "auxiliary hidden-state capture (EAGLE3 / DFLASH) is not supported" + ) + if mm.start_layer != 0 or mm.end_layer != model.config.num_hidden_layers: + reasons.append("pipeline-parallel layer ranges are not supported") + for i in range(mm.start_layer, mm.end_layer): + attn, mlp = mm.layers[i].self_attn, mm.layers[i].mlp + if getattr(attn, "use_deep_gemm_bmm", False): + reasons.append(f"layer {i}: DeepGEMM MLA bmm is not supported") + w_vc = getattr(attn, "w_vc", None) + if w_vc is None or w_vc.dtype != torch.bfloat16: + reasons.append( + f"layer {i}: W_UV must be bf16 (got {getattr(w_vc, 'dtype', None)})" + ) + if getattr(attn, "learnable_sink_param", None) is not None: + reasons.append(f"layer {i}: attention sinks are not supported") + if getattr(attn, "dcp_size", 1) != 1: + reasons.append(f"layer {i}: decode context parallelism is not supported") + if i < model.config.first_k_dense_replace: + if not isinstance(mlp, DeepseekV2MLP): + reasons.append(f"layer {i}: expected a dense MLP layer") + continue + if not isinstance(mlp, DeepseekV2MoE): + reasons.append(f"layer {i}: expected a MoE layer") + if getattr(mlp, "num_fused_shared_experts", 0) != 0: + reasons.append( + f"layer {i}: shared experts are fused into the routed experts" + ) + if getattr(mlp, "shared_experts", None) is None: + reasons.append(f"layer {i}: no shared-expert module") + method = type(getattr(mlp.experts, "quant_method", None)).__name__ + if method != "ModelOptNvFp4FusedMoEMethod": + reasons.append(f"layer {i}: routed experts must be NVFP4 (got {method})") + if not isinstance(mm.layers[mm.end_layer - 1].mlp, DeepseekV2MoE): + reasons.append("the last layer must be a MoE layer") + sliced = [ + i + for i in range(mm.start_layer, mm.end_layer) + if getattr(mm.layers[i].attn_boundary, "input_on_attn_tp_slices", False) + ] + if sliced: + reasons.append( + f"layer {sliced[0]}: attention input on attention-TP slices is not supported" + ) + if getattr(mm, "gemm_output_zero_allocator_size", 0): + reasons.append("the GEMM output zero allocator is not supported") + # The residual steps are the fused all-reduce + residual + RMSNorm the layer + # boundaries use at verify sizes. + if uses_cutedsl_ar_fusion(): + reasons.append("the CuTe DSL all-reduce fusion backend is not supported") + elif not _ar_fusion_ok(flashinfer_ar_fusion_applies, max(SUPPORTED_ROWS)): + reasons.append( + "requires the FlashInfer all-reduce + RMSNorm fusion at verify sizes " + "(--flashinfer-allreduce-fusion-backend not disabled; without NVLink " + "multicast set SGLANG_NVIDIA_MEGAKERNEL_GLM_ALLOW_NO_AR_FUSION=1)" + ) + if getattr(mm, "llama_4_scaling_config", None) is not None: + reasons.append("llama-4 attention scaling is not supported") + # The fused steps perform the MLP output's tensor-parallel sum, as the stock + # residual step does; a MoE combine that already completes it is out. + if post_experts_output_is_complete(is_tp_path=True): + reasons.append( + "the MoE combine already completes the tensor-parallel sum " + "(DWDP, FP4 all-gather dispatch or an all-to-all backend)" + ) + _refuse(SWITCH, "the loaded model", reasons) + + +def absorbed_mla(model, forward_batch) -> bool: + """Whether stock attention runs this batch on the absorbed MLA path (the one + the attention steps implement).""" + from sglang.srt.models.deepseek_common.attention_forward_methods.forward_methods import ( + AttnForwardMethod, + ) + + attn = model.model.layers[model.model.start_layer].self_attn + return attn.dispatch_attn_forward_method(forward_batch) == AttnForwardMethod.MLA + + +def attention_step_reasons(model, forward_batch) -> List[str]: + """Why stock verify attention would leave the branches ``providers.AttentionDecode`` + copies (empty if it would not); read at the first capture of each row count, + once the backend exists.""" + from sglang.srt.layers.attention.dsa_backend import DeepseekSparseAttnBackend + from sglang.srt.model_executor.forward_context import get_attn_backend + from sglang.srt.models.deepseek_common.attention_forward_methods.forward_methods import ( + AttnForwardMethod, + ) + + backend = get_attn_backend() + if not isinstance(backend, DeepseekSparseAttnBackend): + return [f"attention backend {type(backend).__name__} is not the DSA backend"] + reasons: List[str] = [] + if backend.dsa_decode_impl != "trtllm" or backend.use_mha: + reasons.append( + f"verify attention is not the TRT-LLM sparse MLA kernel " + f"(impl {backend.dsa_decode_impl!r}, use_mha {backend.use_mha})" + ) + if backend.kv_cache_dtype != torch.float8_e4m3fn: + reasons.append(f"KV cache dtype {backend.kv_cache_dtype} is not fp8_e4m3") + if not backend.use_fused_topk: + reasons.append("the fused DSA top-k page table is off") + elif not ( + backend.dsa_topk_backend.is_sgl_kernel() + or backend.dsa_topk_backend.is_flashinfer() + ): + reasons.append(f"DSA top-k backend {backend.dsa_topk_backend} is not supported") + if backend.dcp_size != 1 or backend.dsa_index_kpool > 1: + reasons.append( + "decode context parallelism / index K-pool layouts are not supported" + ) + if backend.hisparse_coordinator is not None: + reasons.append("HiSparse is not supported") + if forward_batch.attn_cp_metadata is not None: + reasons.append("attention context parallelism is not supported") + + mm = model.model + for i in range(mm.start_layer, mm.end_layer): + attn = mm.layers[i].self_attn + indexer = getattr(attn, "indexer", None) + why = [] + if attn.dispatch_attn_forward_method(forward_batch) != AttnForwardMethod.MLA: + why.append("not absorbed MLA") + if attn.q_lora_rank is None or attn.rotary_emb is None: + why.append("no q_lora / RoPE") + if not attn._fuse_rope_for_trtllm_mla(forward_batch): + why.append("RoPE is not fused into the FP8 query quantization") + if attn._can_fuse_bmm_into_attention(forward_batch): + why.append("the W_UK bmm is fused into attention") + if getattr(attn, "w_kc", None) is None or attn.w_kc.dtype != torch.bfloat16: + why.append("W_UK is not bf16") + if getattr(attn, "_kimi_split_gguf_kv_b", False) or hasattr( + attn, "prepare_attention_output_gate" + ): + why.append("unsupported attention variant") + if attn.attn_mqa.is_cross_attention: + why.append("cross attention") + if attn.skip_topk is None or (not attn.skip_topk and indexer is None): + why.append("no DSA indexer") + elif not attn.skip_topk: + if not indexer.use_dsa_indexer_fusion: + why.append("the indexer projections are not fused") + if ( + indexer.use_gfx950_fused_indexer + or indexer.dsa_enable_prefill_cp + or indexer._aiter_fused_fp8_active(forward_batch) + ): + why.append("unsupported indexer path") + if why: + reasons.append(f"layer {i}: " + ", ".join(why)) + if len(reasons) > 8: + break + return reasons + + +def step_in_graph_runner( + model, rows: int, forward_batch, input_embeds=None, pp_proxy_tensors=None +) -> bool: + """Every Tier 3 condition but the capture itself (``runtime.engages``): an + eligible verify forward inside the decode graph runner's full-graph capture + procedure (its eager warmups included).""" + from sglang.srt.layers.layer_boundary.residual.add_norm import ( + flashinfer_ar_fusion_applies, + ) + from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import ( + is_in_breakable_cuda_graph, + ) + from sglang.srt.model_executor.runner_utils.capture_mode import get_is_capture_mode + + return ( + forward_batch.forward_mode.is_target_verify() + and rows in SUPPORTED_ROWS + and input_embeds is None + and pp_proxy_tensors is None + and not getattr(forward_batch, "can_run_tbo", False) + and not getattr(model, "capture_aux_hidden_states", False) + and _ar_fusion_ok(flashinfer_ar_fusion_applies, rows) + and absorbed_mla(model, forward_batch) + and get_is_capture_mode() + and not is_in_breakable_cuda_graph() + ) diff --git a/sgl_nvidia_mega/glm_dsa/hops.py b/sgl_nvidia_mega/glm_dsa/hops.py new file mode 100644 index 00000000..d36ecaa6 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/hops.py @@ -0,0 +1,359 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""CuTe DSL all-reduce hops of the GLM-5.3 fused verify forward. + +Every boundary at which the stock forward sums a tensor-parallel partial (the +``o_proj`` output, the MLP / MoE output) is served by one hop: a kernel +publishes this rank's partial to every rank's symmetric-memory mailbox and +reduces the eight partials in rank order, adds the residual and applies the +RMSNorm. The ``o_proj`` hop also computes the ``o_proj`` GEMV; on MoE layers it +computes the router logits during the all-reduce and routes the rows (top-8), +and the MLP-output hop of a MoE layer folds the MoE engine's per-expert outputs +(finalize) into the all-reduce. + +Forms per verify row count (a function of the row count and the layer index +only, so every rank runs the same hop sequence on its mailboxes): + +* MoE family served, rows <= 8: the routing ``o_proj`` hop and the fold hop on + MoE layers; dense layers use the plain hops at rows <= 4, stock above. +* MoE family served, rows 9..16: the 16-row routing hop and fold on their own + mailbox; dense layers stock. +* MoE family stock at this row count (declined at startup), rows <= 8 ("far + form"): the plain hops on every boundary. Otherwise stock. + +The kernels run only inside captured verify graphs; the mailboxes are created +and initialized before the first capture. +""" + +from __future__ import annotations + +import logging +from typing import Any, Optional + +import torch +import torch.distributed as dist + +from sgl_nvidia_mega.glm_dsa import switches +from sgl_nvidia_mega.glm_dsa.eligibility import HIDDEN, TP_SIZE +from sgl_nvidia_mega.glm_dsa.providers import ( + HopSteps, + StepContext, + compile_cached, +) + +logger = logging.getLogger(__name__) + +O_PROJ_IN = 2048 +TOP_K = 8 # routed experts per row +SMALL_ROWS = 8 # mailbox instance A (instance B: 9..16 rows) +DENSE_MAX_ROWS = 4 # plain hops on the dense layers beside the fold; stock above +# Error sites: layer * 4 + {0: o_proj, 1: MLP output, 2: final}; sites from +# EXTERNAL_SITES on are other kernels' (named with ``HopFamily.register_sites``). +HOPS_PER_LAYER = 4 +EXTERNAL_SITES = 1000 + + +class HopFamily: + """Derived weights, compiled units and mailboxes of the hop kernels.""" + + def __init__(self, model: Any, derived: Any, api: Any) -> None: + from sglang.srt.distributed.parallel_state import get_tp_group + + inner, config = model.model, model.config + span = range(inner.start_layer, inner.end_layer) + self.api, self.derived = api, derived + self.layers = [inner.layers[i] for i in span] + self.first, self.last = inner.start_layer, inner.end_layer - 1 + self.eps = float(config.rms_norm_eps) + self.is_moe = [bool(layer.is_layer_sparse) for layer in self.layers] + self.rscale = float(getattr(config, "routed_scaling_factor", 1.0)) + self.device = torch.device("cuda", torch.cuda.current_device()) + self.group = get_tp_group() + self.rank = self.group.rank_in_group + self.w_o = [layer.self_attn.o_proj.weight for layer in self.layers] + self.router: list = [] + self.kernels: Any = None + self.site_names: dict = {} + # DNORM (both switches): the layers without their own top-k whose input + # hop is the MoE fold may defer their attention input norm. + dnorm = switches.dnorm() + self.dnorm_layers = frozenset( + layer.layer_id + for i, layer in enumerate(self.layers) + if dnorm + and i > 0 + and self.is_moe[i - 1] + and getattr(layer.self_attn, "skip_topk", False) + ) + + def check(self) -> str: + """Empty if this model and environment fit the kernels, else the reason.""" + if self.group.world_size != TP_SIZE: + size = self.group.world_size + return f"tensor-parallel size {size} (the hops need {TP_SIZE})" + for w in self.w_o: + if w.dtype != torch.bfloat16 or tuple(w.shape) != (HIDDEN, O_PROJ_IN): + return "o_proj weight is not a bf16 [6144, 2048] shard" + if not w.is_contiguous(): + return "o_proj weight is not contiguous" + for layer in self.layers: + for norm in (layer.input_layernorm, layer.post_attention_layernorm): + if float(norm.variance_epsilon) != self.eps: + return "the RMSNorm epsilons differ" + for layer, moe in zip(self.layers, self.is_moe): + bias = layer.mlp.gate.e_score_correction_bias if moe else None + if moe and (bias is None or bias.dtype != torch.float32): + return "the router has no fp32 score-correction bias" + import torch.distributed._symmetric_memory as symm_mem + + get_backend = getattr(symm_mem, "get_backend", None) + backend = get_backend(self.device) if get_backend is not None else None + if backend not in (None, "CUDA"): + return f"symmetric-memory backend {backend!r} (the hops need 'CUDA')" + if not self.api.fenced_trigger_supported(): + return "the CuTe DSL toolchain does not provide the o_proj hop's start edge" + return "" + + def compile(self, rows) -> None: + """Compile the units served at ``rows`` and create the mailboxes. + + Collective over the tensor-parallel group: every rank agrees (CPU + all-reduce) on whether the hops are served at all and on the split + fold's row limit, so the hop sequence is the same on every rank. + """ + reason = self.check() + small = any(m <= SMALL_ROWS for m in rows) + big = any(m > SMALL_ROWS for m in rows) + names = set() + if small: + names |= {"allreduce_norm", "o_proj_produce", "o_proj_consume"} + names |= {"o_proj_produce_router", "o_proj_consume_router"} + names |= {"moe_fold_split", "moe_fold_plain"} + if self.dnorm_layers: + names |= {"moe_fold_split_dn", "moe_fold_plain_dn"} + if big: + names |= {"o_proj_produce_router16", "o_proj_consume_router16"} + names |= {"moe_fold_block_sums16", "rmsnorm_from_block_sums16"} + compiled, probe = {}, None + if not reason: + import cutlass.cute as cute + + for spec in (s for s in self.api.unit_specs() if s.name in names): + compiled[spec.name] = compile_cached( + spec.obj, + spec.constexprs, + lambda spec=spec: cute.compile(spec.obj, *spec.args), + ) + if spec.name == "moe_fold_split" and not hasattr( + compiled[spec.name], "library" + ): + # The occupancy query needs the driver library, which a + # cache-loaded unit may not keep: query a fresh compile. + probe = cute.compile(spec.obj, *spec.args) + ok = torch.tensor([0 if reason else 1], dtype=torch.int32) + dist.all_reduce(ok, op=dist.ReduceOp.MIN, group=self.group.cpu_group) + if int(ok.item()) == 0: + logger.warning( + "GLM decode hops fall back to the stock all-reduce: %s", + reason or "another rank declined", + ) + return + # Per MoE layer, the router folded into the o_proj hop: W_router scaled by + # the post-attention norm weight, times this rank's o_proj shard, plus this + # rank's slice of the scaled router, as bf16 hi / lo (derived copies). + for layer, moe in zip(self.layers, self.is_moe): + hoist = None + if moe: + hoist = self.api.router_hoist_weights( + layer.mlp.gate.weight, + layer.post_attention_layernorm.weight, + layer.self_attn.o_proj.weight, + self.rank, + ) + for t in hoist: + self.derived.add("hops.router_hoist", t) + self.router.append(hoist) + kernels = self.api.HopKernels.build( + compiled, + self.group.device_group.group_name, + self.device, + self.eps, + self.rscale, + self.device.index, + instance_b=big, + router=True, + probe=probe, + ) + split = torch.tensor([kernels.split_max_rows], dtype=torch.int32) + dist.all_reduce(split, op=dist.ReduceOp.MIN, group=self.group.cpu_group) + kernels.split_max_rows = int(split.item()) + self.kernels = kernels + + def register_sites(self, names: dict) -> None: + """Name other kernels' sites (>= ``EXTERNAL_SITES``) in this error record.""" + for site in names: + if int(site) < EXTERNAL_SITES: + raise ValueError(f"error site {site} is in the hop family's range") + self.site_names.update({int(s): str(n) for s, n in names.items()}) + + def error_report(self) -> Optional[dict]: + """The decoded host-mapped error record of a timed-out hop wait, or None + (host memory only: readable after the CUDA context is lost).""" + rec = self.kernels.err.read() if self.kernels is not None else None + if rec is None: + return None + site = int(rec["site"]) + if site >= EXTERNAL_SITES: + return {**rec, "layer": None, "hop": self.site_names.get(site, "external")} + layer, kind = divmod(site, HOPS_PER_LAYER) + hop = ("o_proj", "mlp_output", "final")[kind] if kind < 3 else str(kind) + return {**rec, "layer": layer, "hop": hop} + + def for_rows(self, rows: int) -> Optional[HopSteps]: + """The hop step group at ``rows`` (None: stock): the fold form, whose + ``fallback`` (the far form up to ``SMALL_ROWS`` rows, else stock) the + composition serves where the MoE family is stock at ``rows``.""" + if self.kernels is None: + return None + far = _FarHops(self) if rows <= SMALL_ROWS else None + return _FoldHops(self, rows, fallback=far) + + def site(self, layer_id: int, kind: int) -> int: + return layer_id * HOPS_PER_LAYER + kind + + def o_proj(self, layer, o_input, residual, step: Optional[StepContext] = None): + """The ``o_proj`` hop; with ``step`` (MoE layers) the routing variant, whose + expert ids / weights ([rows, 8] int32 / fp32) go to ``step.routing``.""" + i, site = layer.layer_id - self.first, self.site(layer.layer_id, 0) + out_res, out_norm = torch.empty_like(residual), torch.empty_like(residual) + tail = (residual, layer.post_attention_layernorm.weight, out_res, out_norm) + if step is None: + self.kernels.o_proj(o_input, self.w_o[i], *tail, site=site) + return out_norm, out_res + rows, dev = residual.shape[0], residual.device + ids = torch.empty((rows, TOP_K), dtype=torch.int32, device=dev) + wts = torch.empty((rows, TOP_K), dtype=torch.float32, device=dev) + bias = layer.mlp.gate.e_score_correction_bias + self.kernels.o_proj_router( + o_input, self.w_o[i], *self.router[i], bias, *tail, ids, wts, site=site + ) + step.routing = (ids, wts) + return out_norm, out_res + + def allreduce_norm(self, partial, residual, gamma, site: int): + out_res, out_norm = torch.empty_like(residual), torch.empty_like(residual) + self.kernels.allreduce_norm( + partial, residual, gamma, out_res, out_norm, site=site + ) + return out_norm, out_res + + def fold(self, rec, residual, gamma, site: int, dn: bool = False): + """The fold hop of the MoE records: ``(normed, residual out)``; with ``dn`` + (<= 8 rows) the norm is deferred: ``(u = bf16(x * gamma), residual out, + block sums [rows, 8] fp32)``.""" + rows = residual.shape[0] + out_res, out = torch.empty_like(residual), torch.empty_like(residual) + bsum = None + if dn or rows > SMALL_ROWS: + bsum = torch.empty((rows, 8), dtype=torch.float32, device=residual.device) + args = (rec.pq, rec.shpartials, rec.ids_fin, rec.w_fin, rec.g2s, residual) + if dn: + self.kernels.moe_fold_dn(*args, gamma, out_res, out, bsum, site=site) + return out, out_res, bsum + self.kernels.moe_fold(*args, gamma, out_res, out, bsum=bsum, site=site) + return out, out_res + + +class _FarHops(HopSteps): + """Rows <= 8 with the stock MLP / MoE: every boundary is a hop.""" + + def __init__(self, family: HopFamily) -> None: + self.f = family + + def pre_attention(self, layer, mlp_out, residual, step: StepContext): + if residual is None: # first layer: the embedding + return super().pre_attention(layer, mlp_out, residual, step) + site = self.f.site(layer.layer_id, 1) + gamma = layer.input_layernorm.weight + return self.f.allreduce_norm(mlp_out, residual, gamma, site) + + def post_attention(self, layer, o_input, residual, step: StepContext): + return self.f.o_proj(layer, o_input, residual) + + def final(self, model, mlp_out, residual, step: StepContext) -> torch.Tensor: + site = self.f.site(self.f.last, 2) + gamma = model.model.norm.weight + return self.f.allreduce_norm(mlp_out, residual, gamma, site)[0] + + +class _FoldHops(HopSteps): + """The MoE family is served: routing ``o_proj`` hops and folds on MoE + layers, plain hops on dense layers up to ``DENSE_MAX_ROWS`` rows. Requires + the MoE family at the same row count; ``fallback`` is served where it is + stock.""" + + requires = ("mlp",) + + def __init__( + self, family: HopFamily, rows: int, fallback: Optional[HopSteps] + ) -> None: + self.f, self.rows, self.fallback = family, rows, fallback + self.dense = rows <= DENSE_MAX_ROWS + self.dnorm_layers = frozenset() + + def defer_input_norm(self) -> None: + """Serve DNORM on the family's DNORM layers; called by the composition + when this row count's attention consumes it.""" + if self.rows <= SMALL_ROWS: + self.dnorm_layers = self.f.dnorm_layers + logger.info( + "GLM DSA fused decode: deferred input norm served on %d layers at %d " + "rows", + len(self.dnorm_layers), + self.rows, + ) + + def pre_attention(self, layer, mlp_out, residual, step: StepContext): + if residual is None: + return super().pre_attention(layer, mlp_out, residual, step) + site = self.f.site(layer.layer_id, 1) + gamma = layer.input_layernorm.weight + if not isinstance(mlp_out, torch.Tensor): + if layer.layer_id in self.dnorm_layers: + u, out_res, bsum = self.f.fold(mlp_out, residual, gamma, site, dn=True) + step.dnorm = (layer.layer_id, bsum) + return u, out_res + return self.f.fold(mlp_out, residual, gamma, site) + if self.dense: + return self.f.allreduce_norm(mlp_out, residual, gamma, site) + return super().pre_attention(layer, mlp_out, residual, step) + + def post_attention(self, layer, o_input, residual, step: StepContext): + if self.f.is_moe[layer.layer_id - self.f.first]: + return self.f.o_proj(layer, o_input, residual, step) + if self.dense: + return self.f.o_proj(layer, o_input, residual) + return super().post_attention(layer, o_input, residual, step) + + def final(self, model, mlp_out, residual, step: StepContext) -> torch.Tensor: + site = self.f.site(self.f.last, 2) + gamma = model.model.norm.weight + if not isinstance(mlp_out, torch.Tensor): + return self.f.fold(mlp_out, residual, gamma, site)[0] + if self.dense: + return self.f.allreduce_norm(mlp_out, residual, gamma, site)[0] + return super().final(model, mlp_out, residual, step) + + +def build(model: Any, derived: Any) -> Optional[HopFamily]: + """The hop family of ``model``, or None to keep every boundary stock.""" + # Defensive: hop kernel modules that fail to import keep every boundary on the + # stock all-reduce (the attention / MoE kernels do not need them). + try: + from sgl_nvidia_mega.glm_dsa.kernels.cute.hop import api + except ImportError as e: + logger.warning( + "GLM decode hops unavailable (%s); using the stock all-reduce.", e + ) + return None + return HopFamily(model, derived, api) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/__init__.py b/sgl_nvidia_mega/glm_dsa/kernels/__init__.py new file mode 100644 index 00000000..0ce09150 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/__init__.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/__init__.py new file mode 100644 index 00000000..4b2425cf --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/__init__.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""CuTe DSL kernels for GLM-5.x DSA decode and verify at 1-16 rows per step. + +Kernels only: each engine's integration code decides eligibility, owns every +buffer and calls the family ``api`` modules; nothing here reads the environment +or imports an inference engine. ``cutlass_supported()`` is the version gate the +engines call before engaging any kernel: on a cutlass-dsl version this package +was not validated on, the engine declines to its stock path with one warning (it +never raises, also not on explicit opt-in). +""" + +from __future__ import annotations + +__all__ = ["VALIDATED_CUTLASS_DSL", "cutlass_supported"] + +# cutlass-dsl versions the bodies, gates and SASS evidence were taken on. +VALIDATED_CUTLASS_DSL: tuple[str, ...] = ("4.8.0",) + + +def cutlass_supported() -> tuple[bool, str]: + """(ok, reason): False when cutlass is missing or not a validated version.""" + try: + import cutlass + except Exception as e: # noqa: BLE001 + return False, f"cutlass-dsl not importable ({type(e).__name__})" + ver = str(getattr(cutlass, "__version__", "")) + if ver not in VALIDATED_CUTLASS_DSL: + validated = ", ".join(VALIDATED_CUTLASS_DSL) + return False, ( + f"cutlass-dsl {ver or ''} not validated (validated: {validated})" + ) + return True, "" diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/__init__.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/__init__.py new file mode 100644 index 00000000..b4896b01 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/api.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/api.py new file mode 100644 index 00000000..73cdd2d7 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/api.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Compile units of the attention family, with one source of example arguments. + +``compile_specs`` returns ``{name: (kernel, constexprs, example_args)}`` for every +attention kernel form the decode / verify glue compiles: the kernel object, the +constexpr options that identify the compiled form (the compile-cache key), and typed +example arguments for ``cute.compile`` (zero pointers, unit sizes, a ``CUstream``). +The engine glue compiles exactly these units, and tests compile and reload them, so +the two never hold separate argument tuples. + +Names: ``uv.``, ``prologue.``, ``index_q.`` (``cap`` = max rows of the +form, 8 or 16), ``sparse.`` (8 or 16 warps) and ``qkv_a.8`` (only when +``qkv_a`` is given; the skinny projection has an 8-row form only). +""" + +import dataclasses +from collections.abc import Iterable + + +def example_args() -> dict[str, tuple]: + """Typed example arguments per kernel kind (values are irrelevant to the + compiled code: pointers 0, sizes 1, scalars 1e-5).""" + import cuda.bindings.driver as cuda + from cutlass import Float32, Int32, Int64 + + z, f, s = Int64(0), Float32(1e-5), cuda.CUstream(0) + one = Int32(1) + return { + "uv": (z,) * 6 + (one,) + (z,) * 4 + (Int32(0),) + (z,) * 4 + (Int32(0), s), + "sparse": (z,) * 5 + (one, f, f, z, z, z, Int32(0), Int32(6), z, z, s), + "index_q": (z,) * 8 + (f, one, z, z, z, z, s), + "qkv_a": (z, z, z, one, z, z, z, z, s), + "qkv_a_dn": (z, z, z, one, z, z, z, z, z, f, s), + "prologue": (z,) * 11 + + (one, one) + + (z,) * 4 + + (f, f) + + (z,) * 6 + + (f,) + + (z,) * 13 + + (Int32(0), z, z, s), + } + + +def compile_specs( + caps: Iterable[int] = (8, 16), + *, + numerics, + index_q_rope_form: str, + rope_cache_fp32: bool = True, + index_rope_cache_fp32: bool = True, + index_scale_ue8m0x4: bool = False, + warps: Iterable[int] = (8, 16), + qkv_a: tuple[int, int] | None = None, + grid: int | None = None, + prologue_end: bool = False, + qkv_a_shared: int | None = None, + qkv_a_shared_dn: bool = False, +) -> dict[str, tuple]: + """The attention units for row capacities ``caps`` and sparse-MLA warp counts + ``warps``. ``numerics`` is the engine's ``PrologueNumerics`` value table and + ``index_q_rope_form`` its index-query RoPE layout; ``index_scale_ue8m0x4`` selects + the prologue's index-K scale layout of four UE8M0 bytes. + ``qkv_a = (n_rows, n_main)`` adds the skinny q_a / kv_a (+ indexer key) projection + at 8 rows on ``grid`` CTAs (default: the current device's multiprocessor count). + ``prologue_end`` adds ``prologue_end.``: the prologue that triggers its + dependents after its last output store. ``qkv_a_shared = n_rows`` adds + ``qkv_a_shared.8``: the skinny projection of the q_a / kv_a rows alone (the layers + without their own indexer), at 8 rows on ``grid`` CTAs; ``qkv_a_shared_dn`` adds + its epilogue-scaled form ``qkv_a_shared_dn.8`` (deferred input norm).""" + from .indexer_q_proj import IndexerQProj + from .mla_prologue import MlaPrologue + from .mla_uv_proj import MlaUvProj + from .qkv_a_proj_small_m import QkvAProjSmallM, QkvAProjSmallMDn + from .sparse_mla_decode import SparseMlaDecode + + args = example_args() + specs: dict[str, tuple] = {} + caps = sorted(set(caps)) + for cap in caps: + specs[f"uv.{cap}"] = (MlaUvProj(cap), {"max_rows": cap}, args["uv"]) + key = { + "max_rows": cap, + **dataclasses.asdict(numerics), + "rope_cache_fp32": rope_cache_fp32, + "index_rope_cache_fp32": index_rope_cache_fp32, + } + if index_scale_ue8m0x4: + key["index_scale_ue8m0x4"] = True + specs[f"prologue.{cap}"] = ( + MlaPrologue( + cap, + numerics, + rope_cache_fp32, + index_rope_cache_fp32, + index_scale_ue8m0x4, + ), + key, + args["prologue"], + ) + if prologue_end: + specs[f"prologue_end.{cap}"] = ( + MlaPrologue( + cap, + numerics, + rope_cache_fp32, + index_rope_cache_fp32, + index_scale_ue8m0x4, + trigger_at_end=True, + ), + {**key, "trigger_at_end": True}, + args["prologue"], + ) + key = { + "max_rows": cap, + "rope_form": index_q_rope_form, + "rope_cache_fp32": index_rope_cache_fp32, + } + specs[f"index_q.{cap}"] = ( + IndexerQProj(cap, index_q_rope_form, index_rope_cache_fp32), + key, + args["index_q"], + ) + for w in sorted(set(warps)): + key = {"warps": w, "logical_ids": False, "early_trigger": True} + specs[f"sparse.{w}"] = (SparseMlaDecode(w, False, True), key, args["sparse"]) + if (qkv_a is not None or qkv_a_shared is not None) and 8 in caps: + if grid is None: + import torch + + grid = torch.cuda.get_device_properties( + torch.accelerator.current_device_index() + ).multi_processor_count + if qkv_a is not None and 8 in caps: + n_rows, n_main = qkv_a + kernel = QkvAProjSmallM(n_rows, n_main, grid) + key = {"n_rows": n_rows, "n_main": n_main, "grid": grid, "slots": kernel.slots} + specs["qkv_a.8"] = (kernel, key, args["qkv_a"]) + if qkv_a_shared is not None and 8 in caps: + kernel = QkvAProjSmallM(qkv_a_shared, qkv_a_shared, grid) + key = { + "n_rows": qkv_a_shared, + "n_main": qkv_a_shared, + "grid": grid, + "slots": kernel.slots, + } + specs["qkv_a_shared.8"] = (kernel, key, args["qkv_a"]) + if qkv_a_shared_dn: + kernel = QkvAProjSmallMDn(qkv_a_shared, qkv_a_shared, grid) + specs["qkv_a_shared_dn.8"] = ( + kernel, + {**key, "epilogue_scale": True}, + args["qkv_a_dn"], + ) + return specs diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/indexer_q_proj.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/indexer_q_proj.py new file mode 100644 index 00000000..0a08c236 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/indexer_q_proj.py @@ -0,0 +1,384 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Indexer query projection with RoPE, power-of-two scaling and FP8 quantization (decode +/ verify rows). + +For the 32 indexer heads (head dim 128) and each row ``m < rows``: + + q = bf16(q_c_normed[m] @ wq_b^T) [32, 128] (FP32 accumulate, + one BF16 rounding) q[:, :64] = interleaved RoPE of q[:, :64] at positions[m] amax_h + = max(1e-4, max |q[h, :]|); scale_h = 2^ceil(log2(amax_h / 448)) (exponent-bit + rounding) q_fp8[m, h] = e4m3_satfinite(q[h] / scale_h); w_out[m, h] = + (index_weights[m, h] * scale_h) * w_scale + +Work split: 32 clusters (one per head) of 4 CTAs (256 threads); CTA ``c`` of head ``h`` +owns the 32 output dims ``[32 c, 32 c + 32)`` over the full K = 2048 (CTAs 0 and 1 hold +the RoPE dims). Each warp loads its 16 KiB of the CTA's packed weight slice with one +bulk copy before the PDL wait and sums its 256-wide K range with ``mma.sync`` m16n8k16 +(rows on A); the 8 warp partials are added in warp order from 0.0. The per-row amax is +reduced within the CTA by shuffles and across the 4 CTAs through distributed shared +memory. + +Option fields (compile time): ``max_rows`` (8 or 16), ``rope_form`` (the RoPE +arithmetic: ``"mul_sub"``: x0 c - x1 s and x1 c + x0 s with separately rounded products; +``"fma_x0_first"``: fma(x0, c, -(x1 s)) and fma(x1, c, x0 s); ``"fma_x1_first"``: +fma(-x1, s, x0 c) and fma(x0, s, x1 c)), ``rope_cache_fp32`` (the cos / sin cache is +FP32, else BF16; layout [positions, 64] = cos | sin). + +Ordering: dependents are triggered at entry; only the constant weights are read before +``griddepcontrol.wait``; ``fence.proxy.async.global`` follows the wait and every +thread's last output store. The cluster amax exchange publishes with +``barrier.cluster.arrive`` (release) / ``wait`` (acquire); a second arrive after the +remote reads and the final wait keep every CTA's shared memory alive while its peers +read it. +""" + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass import Float32, Int32, Int64, Uint32 + +from ..primitives import ptx_core as ptx + +HEADS = 32 +HEAD_DIM = 128 +K = 2048 +CLUSTER = 4 +ROWS_PER_CTA = HEAD_DIM // CLUSTER # 32 output dims per CTA +WARPS = 8 +THREADS = WARPS * 32 +STEPS_PER_WARP = K // 32 // WARPS # 8 k-steps of 32 +WARP_BYTES = STEPS_PER_WARP * ROWS_PER_CTA * 64 # 16 KiB +CTA_WEIGHT_BYTES = WARPS * WARP_BYTES # 128 KiB +GRID = HEADS * CLUSTER +ROPE_FORMS = ("mul_sub", "fma_x0_first", "fma_x1_first") +AMAX_FLOOR = 1e-4 +INV_448 = 1.0 / 448.0 + + +class IndexerQProj: + """See the module docstring. Call arguments (device addresses; strides in elements): + ``weights`` (packed by ``packing.pack_indexer_q``), ``x`` ([rows, 2048] BF16) and + its row stride, ``positions`` (int64 [rows]), ``rope_cache`` and its row stride, + ``index_weights`` ([rows, 32] BF16) and its row stride, ``w_scale``, ``rows``, + ``q_fp8`` ([rows, 32, 128] E4M3, head stride 128) and its row stride, ``w_out`` + ([rows, 32] FP32) + and its row stride, then the stream.""" + + def __init__( + self, max_rows: int, rope_form: str = "mul_sub", rope_cache_fp32: bool = True + ): + if max_rows not in (8, 16): + raise ValueError(f"max_rows must be 8 or 16, got {max_rows}") + if rope_form not in ROPE_FORMS: + raise ValueError( + f"rope_form must be one of {ROPE_FORMS}, got {rope_form!r}" + ) + self.max_rows = max_rows + self.rope_form = rope_form + self.rope_cache_fp32 = rope_cache_fp32 + + @cute.jit + def __call__( + self, + weights: Int64, + x: Int64, + x_stride: Int64, + positions: Int64, + rope_cache: Int64, + rope_stride: Int64, + index_weights: Int64, + index_weights_stride: Int64, + w_scale: Float32, + rows: Int32, + q_fp8: Int64, + q_stride: Int64, + w_out: Int64, + w_out_stride: Int64, + stream: cuda.CUstream, + ): + self.kernel( + weights, + x, + x_stride, + positions, + rope_cache, + rope_stride, + index_weights, + index_weights_stride, + w_scale, + rows, + q_fp8, + q_stride, + w_out, + w_out_stride, + ).launch( + grid=[GRID, 1, 1], + block=[THREADS, 1, 1], + cluster=[CLUSTER, 1, 1], + stream=stream, + use_pdl=True, + max_number_threads=[THREADS, 1, 1], + min_blocks_per_mp=1, + ) + + @cute.kernel + def kernel( + self, + weights: Int64, + x: Int64, + x_stride: Int64, + positions: Int64, + rope_cache: Int64, + rope_stride: Int64, + index_weights: Int64, + index_weights_stride: Int64, + w_scale: Float32, + rows: Int32, + q_fp8: Int64, + q_stride: Int64, + w_out: Int64, + w_out_stride: Int64, + ): + max_rows = self.max_rows + tid_, _, _ = cute.arch.thread_idx() + cta_, _, _ = cute.arch.block_idx() + tid = Int32(tid_) + cta = Int32(cta_) + smem = cutlass.utils.SmemAllocator() + wbuf = smem.allocate_tensor( + cutlass.Uint8, cute.make_layout(CTA_WEIGHT_BYTES), byte_alignment=1024 + ) + red_buf = smem.allocate_tensor( + Float32, + cute.make_layout(WARPS * max_rows * ROWS_PER_CTA), + byte_alignment=16, + ) + amax_buf = smem.allocate_tensor( + Float32, cute.make_layout(max_rows + 16), byte_alignment=16 + ) + bars = smem.allocate_array(Int64, WARPS) + s_w = ptx.shared_address(wbuf.iterator) + s_red = ptx.shared_address(red_buf.iterator) + s_amax = ptx.shared_address(amax_buf.iterator) + s_bar = ptx.shared_address(bars) + + ptx.pdl_launch_dependents() + lane = tid % 32 + warp = tid // 32 + g = lane // 4 + t = lane % 4 + head = cta // CLUSTER + part = cta % CLUSTER + ring = s_w + warp * WARP_BYTES + bar = s_bar + warp * 8 + # constant weights: this warp's 16 KiB, one bulk copy (evict-first L2 hint) + if lane == 0: + ptx.mbarrier_init(bar, 1) + ptx.fence_mbarrier_init_release_cluster() + ptx.mbarrier_arrive_expect_tx(bar, WARP_BYTES) + src = weights + Int64(cta) * CTA_WEIGHT_BYTES + Int64(warp) * WARP_BYTES + ptx.bulk_copy_g2s(ring, src, WARP_BYTES, bar, ptx.l2_policy("evict_first")) + cute.arch.sync_warp() + + self.wait_inputs() + + # A fragments: rows g (and g + 8), latent [256 warp + 32 i + 8 t, +8) + zero4 = (Uint32(0), Uint32(0), Uint32(0), Uint32(0)) + row_lo = ( + x + (Int64(g) * x_stride + Int64(warp * STEPS_PER_WARP * 32 + 8 * t)) * 2 + ) + row_hi = row_lo + Int64(8) * x_stride * 2 + lo = [] + hi = [] + for i in cutlass.range_constexpr(STEPS_PER_WARP): + lo.append(self.load_row(row_lo + i * 64, g < rows)) + if cutlass.const_expr(max_rows == 16): + hi.append(self.load_row(row_hi + i * 64, g + 8 < rows)) + else: + hi.append(zero4) + + # epilogue mapping: (row em, dim pair ep) + ep = lane % 16 + if cutlass.const_expr(max_rows == 16): + em = 2 * warp + lane // 16 + active = em < rows + else: + em = warp + active = (em < rows) & (lane < 16) + rc = Float32(0.0) + rs = Float32(0.0) + wv = Float32(0.0) + if active: + pos = ptx.ld_global_s64(positions + Int64(em) * 8) + if part < 2: + rc = self.rope_value(rope_cache, rope_stride, pos, 16 * part + ep) + rs = self.rope_value(rope_cache, rope_stride, pos, 32 + 16 * part + ep) + if (part == 0) & (ep == 0): + at = ( + index_weights + (Int64(em) * index_weights_stride + Int64(head)) * 2 + ) + wv = ptx.bf16_to_f32(ptx.ld_global_u16(at)) + + self.wait_weights(bar) + + zero = Float32(0.0) + acc = [(zero, zero, zero, zero) for _ in range(4)] + for i in cutlass.range_constexpr(STEPS_PER_WARP): + a = lo[i] + a8 = hi[i] + blk = ring + i * (ROWS_PER_CTA * 64) + for j in cutlass.range_constexpr(4): + b = ptx.ld_shared_v4(blk + (8 * j + g) * 64 + t * 16) + c = ptx.mma_m16n8k16_bf16( + (a[0], a8[0], a[1], a8[1]), (b[0], b[1]), acc[j] + ) + acc[j] = ptx.mma_m16n8k16_bf16( + (a[2], a8[2], a[3], a8[3]), (b[2], b[3]), c + ) + if g < rows: + for j in cutlass.range_constexpr(4): + at = s_red + ((warp * max_rows + g) * ROWS_PER_CTA + 8 * j + 2 * t) * 4 + ptx.st_shared_v2_f32(at, acc[j][0], acc[j][1]) + if cutlass.const_expr(max_rows == 16): # noqa: SIM102 + if g + 8 < rows: + for j in cutlass.range_constexpr(4): + at = ( + s_red + + ((warp * max_rows + g + 8) * ROWS_PER_CTA + 8 * j + 2 * t) * 4 + ) + ptx.st_shared_v2_f32(at, acc[j][2], acc[j][3]) + self.sync_partials() + + # fixed-order sum, BF16 rounding, RoPE, local amax + v0 = Float32(0.0) + v1 = Float32(0.0) + am = Float32(AMAX_FLOOR) + if active: + s0 = Float32(0.0) + s1 = Float32(0.0) + for w in cutlass.range_constexpr(WARPS): + q0, q1 = ptx.ld_shared_v2_f32( + s_red + ((w * max_rows + em) * ROWS_PER_CTA + 2 * ep) * 4 + ) + s0 = ptx.add_rn(s0, q0) + s1 = ptx.add_rn(s1, q1) + v0 = ptx.bf16_to_f32(ptx.cvt_rn_bf16_f32(s0)) + v1 = ptx.bf16_to_f32(ptx.cvt_rn_bf16_f32(s1)) + if part < 2: + o0, o1 = self.rope(v0, v1, rc, rs) + v0 = o0 + v1 = o1 + am = ptx.max_f32( + ptx.max_f32(ptx.abs_f32(v0), ptx.abs_f32(v1)), Float32(AMAX_FLOOR) + ) + offsets = (16, 8, 4, 2, 1) if max_rows == 8 else (8, 4, 2, 1) + for k in cutlass.range_constexpr(len(offsets)): + am = ptx.max_f32(am, cute.arch.shuffle_sync_bfly(am, offsets[k])) + if cutlass.const_expr(max_rows == 8): # noqa: SIM108 + publisher = lane == 0 + else: + publisher = lane % 16 == 0 + if publisher & (em < rows): + ptx.st_shared_f32(s_amax + em * 4, am) + + self.publish_amax() + amax_g = Float32(0.0) + if active: + for src in cutlass.range_constexpr(CLUSTER): + v = ptx.ld_shared_cluster_f32( + ptx.mapa_shared_cluster(s_amax + em * 4, src) + ) + if cutlass.const_expr(src == 0): + amax_g = ptx.max_f32(v, amax_g) + else: + amax_g = ptx.max_f32(amax_g, v) + self.done_reading() + if active: + raw = ptx.mul_rn(amax_g, Float32(INV_448)) + eb = (ptx.f32_bits(raw) + Uint32(0x007FFFFF)) & Uint32(0x7F800000) + scale = ptx.bits_f32(eb) + inv = ptx.bits_f32(Uint32(0x7F000000) - eb) + b0 = Uint32( + ptx.cvt_rn_satfinite_e4m3x2_f32(ptx.mul_rn(v0, inv), Float32(0.0)) + ) & Uint32(0xFF) + b1 = Uint32( + ptx.cvt_rn_satfinite_e4m3x2_f32(ptx.mul_rn(v1, inv), Float32(0.0)) + ) & Uint32(0xFF) + dst = ( + q_fp8 + + Int64(em) * q_stride + + Int64(head * HEAD_DIM + 32 * part + 2 * ep) + ) + ptx.st_global_b16(dst, cutlass.Uint16(b0 | (b1 << Uint32(8)))) + if (part == 0) & (ep == 0): + wo = ptx.mul_rn(ptx.mul_rn(wv, scale), w_scale) + ptx.st_global_f32( + w_out + (Int64(em) * w_out_stride + Int64(head)) * 4, wo + ) + self.release_outputs() + self.exit_wait() + + # ------------------------------------------------------------------ steps + @cute.jit + def load_row(self, addr: Int64, valid): + w0 = Uint32(0) + w1 = Uint32(0) + w2 = Uint32(0) + w3 = Uint32(0) + if valid: + w0, w1, w2, w3 = ptx.ld_global_v4(addr) + return w0, w1, w2, w3 + + @cute.jit + def rope_value(self, cache: Int64, stride: Int64, pos: Int64, col: Int32): + idx = pos * stride + Int64(col) + if cutlass.const_expr(self.rope_cache_fp32): + v = ptx.ld_global_f32(cache + idx * 4) + else: + v = ptx.bf16_to_f32(ptx.ld_global_u16(cache + idx * 2)) + return v + + @cute.jit + def rope(self, x0: Float32, x1: Float32, c: Float32, s: Float32): + if cutlass.const_expr(self.rope_form == "mul_sub"): + o0 = ptx.sub_rn(ptx.mul_rn(x0, c), ptx.mul_rn(x1, s)) + o1 = ptx.add_rn(ptx.mul_rn(x1, c), ptx.mul_rn(x0, s)) + elif cutlass.const_expr(self.rope_form == "fma_x0_first"): + o0 = ptx.fma_rn(x0, c, ptx.neg_f32(ptx.mul_rn(x1, s))) + o1 = ptx.fma_rn(x1, c, ptx.mul_rn(x0, s)) + else: + o0 = ptx.fma_rn(ptx.neg_f32(x1), s, ptx.mul_rn(x0, c)) + o1 = ptx.fma_rn(x0, s, ptx.mul_rn(x1, c)) + return o0, o1 + + @cute.jit + def wait_inputs(self): + ptx.pdl_wait() + ptx.fence_proxy_async_global() + + @cute.jit + def wait_weights(self, bar: Int32): + ptx.mbarrier_wait_parity(bar, 0) + + @cute.jit + def sync_partials(self): + ptx.sync_threads() + + @cute.jit + def publish_amax(self): + ptx.cluster_arrive_release() + ptx.cluster_wait_acquire() + + @cute.jit + def done_reading(self): + ptx.cluster_arrive_release() + + @cute.jit + def release_outputs(self): + ptx.fence_proxy_async_global() + + @cute.jit + def exit_wait(self): + ptx.cluster_wait_acquire() diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/mla_prologue.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/mla_prologue.py new file mode 100644 index 00000000..da04f43a --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/mla_prologue.py @@ -0,0 +1,1630 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""MLA attention prologue for decode / verify rows (rows <= 8): everything between the +fused q_a / kv_a projection and the sparse attention core, in one cluster launch, with +an engine's own rounding points. + +For each row ``m < rows`` (8 local heads, q_lora 2048, kv_lora 512, nope 192, rope 64): + + q_a = bf16(RMSNorm(q_c[m]) * q_norm_w) (optionally written to qc_out) + q = bf16(q_a @ W_qb^T) [8, 256] = nope 192 | rope 64 + mqa_q[m, h, :512] = e4m3(bf16(q_nope[h] @ W_UK[h]^T) / q_scale) + mqa_q[m, h, 512:] = e4m3(RoPE(q_rope[h]) / q_scale) + MLA row of slot[m] = e4m3(RMSNorm(kv_c[m]) * kv_norm_w / k_scale) + | e4m3(RoPE(k_pe[m]) / k_scale) + full (indexer) layers: index-K row = e4m3(RoPE(LayerNorm(idx_k[m])) / s) with + s from max(amax, 1e-4) / 448 (stored with the row); top-k buffer rows set to -1. + +Work split: 9 clusters of 8 CTAs (256 threads). Clusters 0..7 = heads: CTA ``r`` owns +q_b rows ``[32 r, 32 r + 32)`` of its head and absorbed columns ``[64 r, 64 r + 64)``; +its packed q_b (128 KiB) and W_UK^T (24 KiB) slices are bulk-copied before the PDL wait. +Ranks 0..5 publish their q_nope rows in shared memory; after a cluster barrier every +rank gathers the head's 192 nope values over distributed shared memory. Cluster 8: CTA 0 +writes the MLA rows, CTA 1 the index-K rows, CTAs 2..7 the top-k fill. + +Option fields (compile time): ``max_rows`` (8 or 16), ``numerics`` +(``PrologueNumerics``: the sum orders, rounding points and scale rule, each named by +the arithmetic it selects), ``rope_cache_fp32`` and ``index_rope_cache_fp32`` (cos / +sin caches FP32, else BF16; layout [positions, 64] = cos | sin), +``index_scale_ue8m0x4`` (with the power-of-two index-K scale: store it as four equal +UE8M0 bytes instead of an FP32). The layer kind (indexer or not) and ``qc_out`` (null += not written) are runtime. + +Ordering: dependents are triggered at entry (``trigger_at_end``: after each CTA's last +output store and fence instead, so no prologue CTA is still resident when the dependents' +CTAs are placed; outputs are the same); only constant weights / norm parameters / +scales are read before ``griddepcontrol.wait``, and every CTA executes the wait on every +path; ``fence.proxy.async.global`` follows the wait and each thread's last output store. +The q_nope hand-off publishes with ``barrier.cluster.arrive`` (release) / ``wait`` +(acquire); a second arrive after the remote reads and the final wait keep every CTA's +shared memory alive while its peers read it. +""" + +from dataclasses import dataclass + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass import Float32, Int32, Int64, Uint32 + +from ..primitives import ptx_core as ptx + +HEADS = 8 +Q_LORA = 2048 +KV_LORA = 512 +ROPE = 64 +NOPE = 192 +Q_HEAD = NOPE + ROPE # 256 +CLUSTER = 8 +THREADS = 256 +MQA_DIM = KV_LORA + ROPE # 576 bytes per head per row +INDEX_DIM = 128 +GRID = (HEADS + 1) * CLUSTER +NORM_ORDERS = ("strided_8x256", "lane_contiguous") +INDEX_MEAN_SUMS = ("sequential", "pairwise") +INDEX_ROPE_ODD = ("round_x1_cos", "round_x0_sin") +INDEX_SCALES = ("fp32_amax", "ue8m0") +Q_ROPES = ("one_product_rounded", "both_products_rounded") + + +@dataclass(frozen=True) +class PrologueNumerics: + """The prologue's numeric options, each named by the arithmetic it selects. + + ``norm_order`` (q_a and kv_a RMSNorm sums of squares): ``"strided_8x256"``: one + warp per row; lane l adds the separately rounded squares of elements 8 l + 256 b + + v (q_a: b, v = 0..7 in (b, v) order; kv_a: b = 0, 1) to 0, then a butterfly xor 1, + 2, 4, 8, 16. ``"lane_contiguous"``: each thread accumulates the squares of its + contiguous elements with FMA from 0 (kv_a: lane l owns 16 l .. 16 l + 15), then a + butterfly xor 16, 8, 4, 2, 1; the 8 warp sums of a q_a row are added in warp order. + ``norm_weight_plus_zero``: the norm weights enter as ``w + 0``. + ``kv_norm_bf16``: the kv_a norm output is rounded to BF16 before the FP8 + quantization. ``skip_slot0``: cache slot 0 (a padding slot) is never written; else + only negative slots are skipped. + ``index_mean_sum`` (index-K LayerNorm, a lane's 4 elements): ``"sequential"``: + (((0 + x0) + x1) + x2) + x3; ``"pairwise"``: (x0 + x1) + (x2 + x3). + ``index_rope_odd`` (index-K RoPE, odd dims e0 sin + e1 cos): which product is + rounded before the fused other one. ``index_k_bf16``: the RoPE output is rounded + to BF16 before the scale. ``index_scale``: ``"fp32_amax"``: s = max(amax, 1e-4) / + 448, values x (1 / s) clipped to +-448; ``"ue8m0"``: s rounded up to a power of + two, values x / s. + ``q_rope`` (q_pe RoPE): ``"one_product_rounded"``: x0 cos - x1 sin with x1 sin + rounded and x1 cos + x0 sin with x0 sin rounded, the other product fused; + ``"both_products_rounded"``: both products rounded, then the difference / sum. + """ + + norm_order: str + norm_weight_plus_zero: bool + kv_norm_bf16: bool + skip_slot0: bool + index_mean_sum: str + index_rope_odd: str + index_k_bf16: bool + index_scale: str + q_rope: str + + def __post_init__(self): + for value, allowed in ( + (self.norm_order, NORM_ORDERS), + (self.index_mean_sum, INDEX_MEAN_SUMS), + (self.index_rope_odd, INDEX_ROPE_ODD), + (self.index_scale, INDEX_SCALES), + (self.q_rope, Q_ROPES), + ): + if value not in allowed: + raise ValueError(f"{value!r} is not one of {allowed}") + + +# shared memory of a head CTA (bytes) +WQB_BYTES = 2 * (Q_LORA // 32) * 1024 # 2 row tiles x 64 k-steps x 1 KiB +WUK_BYTES = 4 * (NOPE // 32) * 1024 # 4 row tiles x 6 k-steps x 1 KiB +XS_STRIDE = Q_LORA * 2 + 64 +QN_STRIDE = NOPE * 2 + 64 + + +def layout(max_rows): + off = {"wqb": 0, "wuk": WQB_BYTES} + off["xs"] = off["wuk"] + WUK_BYTES + if max_rows == 8: + off["pub"] = off["xs"] + max_rows * XS_STRIDE + off["qn"] = off["pub"] + max_rows * 32 * 2 + off["red"] = off["qn"] + max_rows * QN_STRIDE + off["ssq"] = off["red"] + 8 * 2 * 32 * 16 + else: + # 16 rows: the partials and the gathered q_nope rows alias the token rows, + # which are dead once the q_b products are done (a CTA barrier separates them) + off["red"] = off["xs"] + off["qn"] = off["red"] + 8 * 2 * 2 * 32 * 16 + off["pub"] = off["xs"] + max_rows * XS_STRIDE + off["ssq"] = off["pub"] + max_rows * 32 * 2 + off["mbar"] = off["ssq"] + 8 * max_rows * 4 + off["total"] = off["mbar"] + 16 + return off + + +class MlaPrologue: + """See the module docstring. Call arguments (device addresses; row strides in + elements; cache strides in bytes): ``q_c``, ``kv_c``, ``k_pe``, ``idx_k`` with their + row strides (``idx_k`` unused on shared layers), ``positions``, ``slot_mapping``, + ``idx_slot_mapping`` (int64), ``rows``, ``has_indexer``, packed ``w_qb`` / ``w_uk`` + (``packing.pack_q_b`` / ``pack_w_uk_t``), ``q_norm_w`` / ``kv_norm_w`` (BF16), + ``q_eps`` / ``kv_eps``, ``rope_cache`` and its row stride, ``q_scale`` / ``k_scale`` + (FP32 device scalars), ``ik_w`` / ``ik_b`` (FP32 [128]), ``ik_eps``, + ``index_rope_cache`` and its row stride, ``mqa_q`` and its row stride, the MLA cache + (base, block stride, entry stride, block size), the index-K cache (base, block + stride, block size), ``topk`` (int32) with its row stride and + width, ``qc_out`` (or 0) and its row stride, then the stream.""" + + def __init__( + self, + max_rows: int, + numerics: PrologueNumerics, + rope_cache_fp32: bool = True, + index_rope_cache_fp32: bool = True, + index_scale_ue8m0x4: bool = False, + trigger_at_end: bool = False, + ): + if max_rows not in (8, 16): + raise ValueError(f"max_rows must be 8 or 16, got {max_rows}") + self.max_rows = max_rows + self.num = numerics + self.rope_cache_fp32 = rope_cache_fp32 + self.index_rope_cache_fp32 = index_rope_cache_fp32 + if index_scale_ue8m0x4 and numerics.index_scale != "ue8m0": + raise ValueError( + "index_scale_ue8m0x4 applies to the power-of-two scale only" + ) + self.index_scale_ue8m0x4 = index_scale_ue8m0x4 + self.trigger_at_end = trigger_at_end + self.off = layout(max_rows) + + @cute.jit + def __call__( + self, + q_c: Int64, + q_c_stride: Int64, + kv_c: Int64, + kv_c_stride: Int64, + k_pe: Int64, + k_pe_stride: Int64, + idx_k: Int64, + idx_k_stride: Int64, + positions: Int64, + slot_mapping: Int64, + idx_slot_mapping: Int64, + rows: Int32, + has_indexer: Int32, + w_qb: Int64, + w_uk: Int64, + q_norm_w: Int64, + kv_norm_w: Int64, + q_eps: Float32, + kv_eps: Float32, + rope_cache: Int64, + rope_stride: Int64, + q_scale: Int64, + k_scale: Int64, + ik_w: Int64, + ik_b: Int64, + ik_eps: Float32, + index_rope_cache: Int64, + index_rope_stride: Int64, + mqa_q: Int64, + mqa_stride: Int64, + mla_cache: Int64, + mla_block_stride: Int64, + mla_entry_stride: Int64, + mla_block_size: Int64, + idx_cache: Int64, + idx_block_stride: Int64, + idx_block_size: Int64, + topk: Int64, + topk_stride: Int64, + topk_width: Int32, + qc_out: Int64, + qc_out_stride: Int64, + stream: cuda.CUstream, + ): + self.kernel( + q_c, + q_c_stride, + kv_c, + kv_c_stride, + k_pe, + k_pe_stride, + idx_k, + idx_k_stride, + positions, + slot_mapping, + idx_slot_mapping, + rows, + has_indexer, + w_qb, + w_uk, + q_norm_w, + kv_norm_w, + q_eps, + kv_eps, + rope_cache, + rope_stride, + q_scale, + k_scale, + ik_w, + ik_b, + ik_eps, + index_rope_cache, + index_rope_stride, + mqa_q, + mqa_stride, + mla_cache, + mla_block_stride, + mla_entry_stride, + mla_block_size, + idx_cache, + idx_block_stride, + idx_block_size, + topk, + topk_stride, + topk_width, + qc_out, + qc_out_stride, + ).launch( + grid=[GRID, 1, 1], + block=[THREADS, 1, 1], + cluster=[CLUSTER, 1, 1], + stream=stream, + use_pdl=True, + max_number_threads=[THREADS, 1, 1], + min_blocks_per_mp=1, + ) + + @cute.kernel + def kernel( + self, + q_c: Int64, + q_c_stride: Int64, + kv_c: Int64, + kv_c_stride: Int64, + k_pe: Int64, + k_pe_stride: Int64, + idx_k: Int64, + idx_k_stride: Int64, + positions: Int64, + slot_mapping: Int64, + idx_slot_mapping: Int64, + rows: Int32, + has_indexer: Int32, + w_qb: Int64, + w_uk: Int64, + q_norm_w: Int64, + kv_norm_w: Int64, + q_eps: Float32, + kv_eps: Float32, + rope_cache: Int64, + rope_stride: Int64, + q_scale: Int64, + k_scale: Int64, + ik_w: Int64, + ik_b: Int64, + ik_eps: Float32, + index_rope_cache: Int64, + index_rope_stride: Int64, + mqa_q: Int64, + mqa_stride: Int64, + mla_cache: Int64, + mla_block_stride: Int64, + mla_entry_stride: Int64, + mla_block_size: Int64, + idx_cache: Int64, + idx_block_stride: Int64, + idx_block_size: Int64, + topk: Int64, + topk_stride: Int64, + topk_width: Int32, + qc_out: Int64, + qc_out_stride: Int64, + ): + tid_, _, _ = cute.arch.thread_idx() + cta_, _, _ = cute.arch.block_idx() + tid = Int32(tid_) + cta = Int32(cta_) + smem = cutlass.utils.SmemAllocator() + buf = smem.allocate_tensor( + cutlass.Uint8, cute.make_layout(self.off["total"]), byte_alignment=1024 + ) + base = ptx.shared_address(buf.iterator) + + if cutlass.const_expr(not self.trigger_at_end): + ptx.pdl_launch_dependents() + cluster_id = cta // CLUSTER + r = cta % CLUSTER + if cluster_id < HEADS: + if cutlass.const_expr(self.max_rows == 16): + self.head_cta16( + base, + cluster_id, + r, + tid, + q_c, + q_c_stride, + positions, + rows, + w_qb, + w_uk, + q_norm_w, + q_eps, + rope_cache, + rope_stride, + q_scale, + mqa_q, + mqa_stride, + qc_out, + qc_out_stride, + ) + else: + self.head_cta( + base, + cluster_id, + r, + tid, + q_c, + q_c_stride, + positions, + rows, + w_qb, + w_uk, + q_norm_w, + q_eps, + rope_cache, + rope_stride, + q_scale, + mqa_q, + mqa_stride, + qc_out, + qc_out_stride, + ) + else: + self.kv_cta( + r, + tid, + kv_c, + kv_c_stride, + k_pe, + k_pe_stride, + idx_k, + idx_k_stride, + positions, + slot_mapping, + idx_slot_mapping, + rows, + has_indexer, + kv_norm_w, + kv_eps, + rope_cache, + rope_stride, + k_scale, + ik_w, + ik_b, + ik_eps, + index_rope_cache, + index_rope_stride, + mla_cache, + mla_block_stride, + mla_entry_stride, + mla_block_size, + idx_cache, + idx_block_stride, + idx_block_size, + topk, + topk_stride, + topk_width, + ) + self.release_outputs() + if cutlass.const_expr(self.trigger_at_end): + ptx.pdl_launch_dependents() + + # ------------------------------------------------------------------ head clusters + @cute.jit + def head_cta( + self, + base: Int32, + h: Int32, + r: Int32, + tid: Int32, + q_c: Int64, + q_c_stride: Int64, + positions: Int64, + rows: Int32, + w_qb: Int64, + w_uk: Int64, + q_norm_w: Int64, + q_eps: Float32, + rope_cache: Int64, + rope_stride: Int64, + q_scale: Int64, + mqa_q: Int64, + mqa_stride: Int64, + qc_out: Int64, + qc_out_stride: Int64, + ): + off = self.off + max_rows = self.max_rows + lane = tid % 32 + warp = tid // 32 + bar = base + off["mbar"] + s_xs = base + off["xs"] + s_red = base + off["red"] + s_ssq = base + off["ssq"] + s_pub = base + off["pub"] + s_qn = base + off["qn"] + # constant weights: q_b slice (two 64 KiB halves) + W_UK^T slice (24 KiB), + # before the PDL wait + if tid == 0: + ptx.mbarrier_init(bar, 1) + ptx.fence_mbarrier_init_release_cluster() + ptx.fence_proxy_async_shared_cta() + ptx.mbarrier_arrive_expect_tx(bar, WQB_BYTES + WUK_BYTES) + src_qb = w_qb + Int64(h * 16 + 2 * r) * ((Q_LORA // 32) * 1024) + src_uk = w_uk + Int64(h * 32 + 4 * r) * ((NOPE // 32) * 1024) + for part in cutlass.range_constexpr(2): + ptx.bulk_copy_g2s( + base + off["wqb"] + part * (WQB_BYTES // 2), + src_qb + part * (WQB_BYTES // 2), + WQB_BYTES // 2, + bar, + ) + ptx.bulk_copy_g2s(base + off["wuk"], src_uk, WUK_BYTES, bar) + qw = ptx.ld_global_nc_v4(q_norm_w + Int64(tid) * 16) + inv_qs = ptx.rcp_rn(ptx.ld_global_nc_f32(q_scale)) + ptx.sync_threads() + + self.wait_q_inputs() + + # every predecessor-data load first (their latencies overlap) + qv = [] + for n in cutlass.range_constexpr(max_rows): + qv.append( + self.load16( + q_c + (Int64(n) * q_c_stride + Int64(tid * 8)) * 2, n < rows + ) + ) + rope_c = Float32(0.0) + rope_s = Float32(0.0) + if (r >= NOPE // 32) & (tid < 32 * rows): + pos = ptx.ld_global_s64(positions + Int64(tid // 32) * 8) + pr = ((r - NOPE // 32) * 32 + lane) // 2 + rope_c = self.rope_value( + rope_cache, rope_stride, pos, pr, self.rope_cache_fp32 + ) + rope_s = self.rope_value( + rope_cache, rope_stride, pos, 32 + pr, self.rope_cache_fp32 + ) + if cutlass.const_expr(self.num.norm_order == "strided_8x256"): + self.q_norm_sums_strided(s_xs, s_ssq, qv, warp, lane, tid, rows) + else: + self.q_norm_sums_contiguous(s_ssq, qv, warp, lane, rows) + ptx.sync_threads() + wv = [] + for e in cutlass.range_constexpr(4): + lo, hi = ptx.bf16x2_to_f32x2(qw[e]) + if cutlass.const_expr(self.num.norm_weight_plus_zero): + lo = ptx.add_rn(lo, Float32(0.0)) + hi = ptx.add_rn(hi, Float32(0.0)) + wv.append(lo) + wv.append(hi) + for n in cutlass.range_constexpr(max_rows): + if n < rows: + if cutlass.const_expr(self.num.norm_order == "strided_8x256"): + s = ptx.ld_shared_f32(s_ssq + n * 4) + else: + s = Float32(0.0) + for w in cutlass.range_constexpr(8): + s = ptx.add_rn( + s, ptx.ld_shared_f32(s_ssq + (w * max_rows + n) * 4) + ) + rr = ptx.rsqrt_approx(ptx.fma_rn(s, Float32(1.0 / Q_LORA), q_eps)) + words = [] + for e in cutlass.range_constexpr(4): + x0, x1 = ptx.bf16x2_to_f32x2(qv[n][e]) + words.append( + ptx.cvt_rn_bf16x2_f32( + ptx.mul_rn(ptx.mul_rn(x0, rr), wv[2 * e]), + ptx.mul_rn(ptx.mul_rn(x1, rr), wv[2 * e + 1]), + ) + ) + packed = (words[0], words[1], words[2], words[3]) + ptx.st_shared_v4(s_xs + n * XS_STRIDE + tid * 16, packed) + if (qc_out != Int64(0)) & (h == 0) & (r == 0): + ptx.st_global_v4( + qc_out + (Int64(n) * qc_out_stride + Int64(tid * 8)) * 2, packed + ) + ptx.sync_threads() + + self.wait_weights(bar) + + g = lane // 4 + t = lane % 4 + # q_b MMA (tokens on N = 8): warp w covers k-steps [8 w, 8 w + 8) for both + # 16-row tiles + zero = Float32(0.0) + c = [(zero, zero, zero, zero), (zero, zero, zero, zero)] + for i in cutlass.range_constexpr(8): + s_ = warp * 8 + i + bq = self.lds16(s_xs + g * XS_STRIDE + (s_ * 32 + 8 * t) * 2, g < rows) + for j in cutlass.range_constexpr(2): + blk = base + off["wqb"] + (j * (Q_LORA // 32) + s_) * 1024 + lo = ptx.ld_shared_v4(blk + lane * 16) + hi = ptx.ld_shared_v4(blk + 512 + lane * 16) + c1 = ptx.mma_m16n8k16_bf16( + (lo[0], hi[0], lo[1], hi[1]), (bq[0], bq[1]), c[j] + ) + c[j] = ptx.mma_m16n8k16_bf16( + (lo[2], hi[2], lo[3], hi[3]), (bq[2], bq[3]), c1 + ) + for j in cutlass.range_constexpr(2): + ptx.st_shared_v4( + s_red + ((warp * 2 + j) * 32 + lane) * 16, + ( + ptx.f32_bits(c[j][0]), + ptx.f32_bits(c[j][1]), + ptx.f32_bits(c[j][2]), + ptx.f32_bits(c[j][3]), + ), + ) + self.sync_partials() + if tid < 32 * rows: + rho = tid % 32 + n = tid // 32 + jt = rho // 16 + rr_ = rho % 16 + ln = (rr_ % 8) * 4 + n // 2 + reg = (rr_ // 8) * 2 + n % 2 + v = Float32(0.0) + for w in cutlass.range_constexpr(8): + v = ptx.add_rn( + v, + ptx.ld_shared_f32(s_red + ((w * 2 + jt) * 32 + ln) * 16 + reg * 4), + ) + qb = ptx.cvt_rn_bf16_f32(v) + if r < NOPE // 32: + ptx.st_shared_b16(s_pub + (n * 32 + rho) * 2, qb) + else: + d = (r - NOPE // 32) * 32 + rho + own = ptx.bf16_to_f32(qb) + other = cute.arch.shuffle_sync_bfly(own, 1) + o = Float32(0.0) + if d % 2 == 0: + o = self.q_rope_even(own, other, rope_c, rope_s) + else: + o = self.q_rope_odd(other, own, rope_c, rope_s) + o = ptx.mul_rn(o, inv_qs) + dst = mqa_q + Int64(n) * mqa_stride + Int64(h * MQA_DIM + KV_LORA + d) + ptx.st_global_u8( + dst, + cutlass.Uint16( + Uint32(ptx.cvt_rn_satfinite_e4m3x2_f32(o, Float32(0.0))) + & Uint32(0xFF) + ), + ) + # q_nope hand-off: ranks 0..5 published 32 rows each + self.handoff(base, tid, rows) + self.done_reading() + ptx.sync_threads() + # W_UK absorb: rows l in [64 r, 64 r + 64) (4 tiles), K = 192 (6 k-steps); warp + # -> (tile w % 4, half w // 4) + tile = warp % 4 + half = warp // 4 + cw = (zero, zero, zero, zero) + for i in cutlass.range_constexpr(3): + s_ = half * 3 + i + bq = self.lds16(s_qn + g * QN_STRIDE + (s_ * 32 + 8 * t) * 2, g < rows) + blk = base + off["wuk"] + (tile * (NOPE // 32) + s_) * 1024 + lo = ptx.ld_shared_v4(blk + lane * 16) + hi = ptx.ld_shared_v4(blk + 512 + lane * 16) + c1 = ptx.mma_m16n8k16_bf16((lo[0], hi[0], lo[1], hi[1]), (bq[0], bq[1]), cw) + cw = ptx.mma_m16n8k16_bf16((lo[2], hi[2], lo[3], hi[3]), (bq[2], bq[3]), c1) + ptx.st_shared_v4( + s_red + (warp * 32 + lane) * 16, + ( + ptx.f32_bits(cw[0]), + ptx.f32_bits(cw[1]), + ptx.f32_bits(cw[2]), + ptx.f32_bits(cw[3]), + ), + ) + self.sync_partials() + for k in cutlass.range_constexpr((64 * max_rows + THREADS - 1) // THREADS): + idx = tid + k * THREADS + if idx < 64 * rows: + rho = idx % 64 + n = idx // 64 + tl = rho // 16 + rr_ = rho % 16 + ln = (rr_ % 8) * 4 + n // 2 + reg = (rr_ // 8) * 2 + n % 2 + v = ptx.add_rn( + ptx.ld_shared_f32(s_red + (tl * 32 + ln) * 16 + reg * 4), + ptx.ld_shared_f32(s_red + ((tl + 4) * 32 + ln) * 16 + reg * 4), + ) + qb = ptx.bf16_to_f32(ptx.cvt_rn_bf16_f32(v)) + dst = mqa_q + Int64(n) * mqa_stride + Int64(h * MQA_DIM + r * 64 + rho) + ptx.st_global_u8( + dst, + cutlass.Uint16( + Uint32( + ptx.cvt_rn_satfinite_e4m3x2_f32( + ptx.mul_rn(qb, inv_qs), Float32(0.0) + ) + ) + & Uint32(0xFF) + ), + ) + self.release_outputs() + self.exit_wait() + + @cute.jit + def head_cta16( + self, + base: Int32, + h: Int32, + r: Int32, + tid: Int32, + q_c: Int64, + q_c_stride: Int64, + positions: Int64, + rows: Int32, + w_qb: Int64, + w_uk: Int64, + q_norm_w: Int64, + q_eps: Float32, + rope_cache: Int64, + rope_stride: Int64, + q_scale: Int64, + mqa_q: Int64, + mqa_stride: Int64, + qc_out: Int64, + qc_out_stride: Int64, + ): + """The head CTA at up to 16 rows. Per row, every operation and its order are + the 8-row form's (rows are independent columns of the tensor-core products), + so each row equals that row of an 8-row call. Tokens use two 8-wide N tiles; + the raw and normalized rows live in the XS slots (re-read, not registers); the + partials and the gathered q_nope rows reuse the XS region after the q_b + products (a CTA barrier separates them).""" + off = self.off + max_rows = self.max_rows + lane = tid % 32 + warp = tid // 32 + bar = base + off["mbar"] + s_xs = base + off["xs"] + s_red = base + off["red"] + s_ssq = base + off["ssq"] + s_pub = base + off["pub"] + s_qn = base + off["qn"] + strided = self.num.norm_order == "strided_8x256" + if tid == 0: + ptx.mbarrier_init(bar, 1) + ptx.fence_mbarrier_init_release_cluster() + ptx.fence_proxy_async_shared_cta() + ptx.mbarrier_arrive_expect_tx(bar, WQB_BYTES + WUK_BYTES) + src_qb = w_qb + Int64(h * 16 + 2 * r) * ((Q_LORA // 32) * 1024) + src_uk = w_uk + Int64(h * 32 + 4 * r) * ((NOPE // 32) * 1024) + for part in cutlass.range_constexpr(2): + ptx.bulk_copy_g2s( + base + off["wqb"] + part * (WQB_BYTES // 2), + src_qb + part * (WQB_BYTES // 2), + WQB_BYTES // 2, + bar, + ) + ptx.bulk_copy_g2s(base + off["wuk"], src_uk, WUK_BYTES, bar) + qw = ptx.ld_global_nc_v4(q_norm_w + Int64(tid) * 16) + inv_qs = ptx.rcp_rn(ptx.ld_global_nc_f32(q_scale)) + ptx.sync_threads() + + self.wait_q_inputs() + + qv = [] + for n in cutlass.range_constexpr(max_rows): + qv.append( + self.load16( + q_c + (Int64(n) * q_c_stride + Int64(tid * 8)) * 2, n < rows + ) + ) + # RoPE CTAs: two (token, pair) slots per thread (token tid / 32 and + 8) + rope_c = [Float32(0.0), Float32(0.0)] + rope_s = [Float32(0.0), Float32(0.0)] + for it in cutlass.range_constexpr(2): + rope_c[it], rope_s[it] = self.rope_pair( + (r >= NOPE // 32) & (tid + THREADS * it < 32 * rows), + positions, + (tid + THREADS * it) // 32, + ((r - NOPE // 32) * 32 + lane) // 2, + rope_cache, + rope_stride, + ) + for n in cutlass.range_constexpr(max_rows): + if n < rows: + ptx.st_shared_v4(s_xs + n * XS_STRIDE + tid * 16, qv[n]) + if cutlass.const_expr(strided): + ptx.sync_threads() + for grp in cutlass.range_constexpr(2): + n = warp + 8 * grp + if n < rows: + s = Float32(0.0) + row = s_xs + n * XS_STRIDE + for b in cutlass.range_constexpr(8): + u = ptx.ld_shared_v4(row + (lane + 32 * b) * 16) + for e in cutlass.range_constexpr(4): + lo, hi = ptx.bf16x2_to_f32x2(u[e]) + s = ptx.add_rn(s, ptx.mul_rn(lo, lo)) + s = ptx.add_rn(s, ptx.mul_rn(hi, hi)) + for k in cutlass.range_constexpr(5): + s = ptx.add_rn(s, cute.arch.shuffle_sync_bfly(s, 1 << k)) + if lane == 0: + ptx.st_shared_f32(s_ssq + n * 4, s) + else: + self.q_norm_sums_contiguous(s_ssq, qv, warp, lane, rows) + ptx.sync_threads() + wv = [] + for e in cutlass.range_constexpr(4): + lo, hi = ptx.bf16x2_to_f32x2(qw[e]) + if cutlass.const_expr(self.num.norm_weight_plus_zero): + lo = ptx.add_rn(lo, Float32(0.0)) + hi = ptx.add_rn(hi, Float32(0.0)) + wv.append(lo) + wv.append(hi) + for n in cutlass.range_constexpr(max_rows): + if n < rows: + if cutlass.const_expr(strided): + s = ptx.ld_shared_f32(s_ssq + n * 4) + else: + s = Float32(0.0) + for w in cutlass.range_constexpr(8): + s = ptx.add_rn( + s, ptx.ld_shared_f32(s_ssq + (w * max_rows + n) * 4) + ) + rr = ptx.rsqrt_approx(ptx.fma_rn(s, Float32(1.0 / Q_LORA), q_eps)) + raw = ptx.ld_shared_v4(s_xs + n * XS_STRIDE + tid * 16) + words = [] + for e in cutlass.range_constexpr(4): + x0, x1 = ptx.bf16x2_to_f32x2(raw[e]) + words.append( + ptx.cvt_rn_bf16x2_f32( + ptx.mul_rn(ptx.mul_rn(x0, rr), wv[2 * e]), + ptx.mul_rn(ptx.mul_rn(x1, rr), wv[2 * e + 1]), + ) + ) + packed = (words[0], words[1], words[2], words[3]) + ptx.st_shared_v4(s_xs + n * XS_STRIDE + tid * 16, packed) + if (qc_out != Int64(0)) & (h == 0) & (r == 0): + ptx.st_global_v4( + qc_out + (Int64(n) * qc_out_stride + Int64(tid * 8)) * 2, packed + ) + ptx.sync_threads() + + self.wait_weights(bar) + + g = lane // 4 + t = lane % 4 + zero = Float32(0.0) + z4 = (zero, zero, zero, zero) + c = [z4, z4, z4, z4] # [j * 2 + token tile] + for i in cutlass.range_constexpr(8): + s_ = warp * 8 + i + b0 = self.lds16(s_xs + g * XS_STRIDE + (s_ * 32 + 8 * t) * 2, g < rows) + b1 = self.lds16( + s_xs + (g + 8) * XS_STRIDE + (s_ * 32 + 8 * t) * 2, g + 8 < rows + ) + for j in cutlass.range_constexpr(2): + blk = base + off["wqb"] + (j * (Q_LORA // 32) + s_) * 1024 + lo = ptx.ld_shared_v4(blk + lane * 16) + hi = ptx.ld_shared_v4(blk + 512 + lane * 16) + for tt in cutlass.range_constexpr(2): + bq = b0 if tt == 0 else b1 + c1 = ptx.mma_m16n8k16_bf16( + (lo[0], hi[0], lo[1], hi[1]), (bq[0], bq[1]), c[j * 2 + tt] + ) + c[j * 2 + tt] = ptx.mma_m16n8k16_bf16( + (lo[2], hi[2], lo[3], hi[3]), (bq[2], bq[3]), c1 + ) + # the partials reuse the token rows: every warp must be done reading them + self.rows_dead() + for j in cutlass.range_constexpr(2): + for tt in cutlass.range_constexpr(2): + cc = c[j * 2 + tt] + ptx.st_shared_v4( + s_red + (((warp * 2 + j) * 2 + tt) * 32 + lane) * 16, + ( + ptx.f32_bits(cc[0]), + ptx.f32_bits(cc[1]), + ptx.f32_bits(cc[2]), + ptx.f32_bits(cc[3]), + ), + ) + self.sync_partials() + for it in cutlass.range_constexpr(2): + idx = tid + THREADS * it + if idx < 32 * rows: + rho = idx % 32 + n = idx // 32 + tt = n // 8 + nn = n % 8 + jt = rho // 16 + rr_ = rho % 16 + ln = (rr_ % 8) * 4 + nn // 2 + reg = (rr_ // 8) * 2 + nn % 2 + v = Float32(0.0) + for w in cutlass.range_constexpr(8): + v = ptx.add_rn( + v, + ptx.ld_shared_f32( + s_red + (((w * 2 + jt) * 2 + tt) * 32 + ln) * 16 + reg * 4 + ), + ) + qb = ptx.cvt_rn_bf16_f32(v) + if r < NOPE // 32: + ptx.st_shared_b16(s_pub + (n * 32 + rho) * 2, qb) + else: + d = (r - NOPE // 32) * 32 + rho + own = ptx.bf16_to_f32(qb) + other = cute.arch.shuffle_sync_bfly(own, 1) + o = Float32(0.0) + if d % 2 == 0: + o = self.q_rope_even(own, other, rope_c[it], rope_s[it]) + else: + o = self.q_rope_odd(other, own, rope_c[it], rope_s[it]) + o = ptx.mul_rn(o, inv_qs) + dst = ( + mqa_q + Int64(n) * mqa_stride + Int64(h * MQA_DIM + KV_LORA + d) + ) + ptx.st_global_u8( + dst, + cutlass.Uint16( + Uint32(ptx.cvt_rn_satfinite_e4m3x2_f32(o, Float32(0.0))) + & Uint32(0xFF) + ), + ) + self.publish() + for it in cutlass.range_constexpr(2): + self.gather(base, tid + THREADS * it, rows) + self.done_reading() + ptx.sync_threads() + tile = warp % 4 + half = warp // 4 + cw = [z4, z4] + for i in cutlass.range_constexpr(3): + s_ = half * 3 + i + b0 = self.lds16(s_qn + g * QN_STRIDE + (s_ * 32 + 8 * t) * 2, g < rows) + b1 = self.lds16( + s_qn + (g + 8) * QN_STRIDE + (s_ * 32 + 8 * t) * 2, g + 8 < rows + ) + blk = base + off["wuk"] + (tile * (NOPE // 32) + s_) * 1024 + lo = ptx.ld_shared_v4(blk + lane * 16) + hi = ptx.ld_shared_v4(blk + 512 + lane * 16) + for tt in cutlass.range_constexpr(2): + bq = b0 if tt == 0 else b1 + c1 = ptx.mma_m16n8k16_bf16( + (lo[0], hi[0], lo[1], hi[1]), (bq[0], bq[1]), cw[tt] + ) + cw[tt] = ptx.mma_m16n8k16_bf16( + (lo[2], hi[2], lo[3], hi[3]), (bq[2], bq[3]), c1 + ) + for tt in cutlass.range_constexpr(2): + ptx.st_shared_v4( + s_red + ((warp * 2 + tt) * 32 + lane) * 16, + ( + ptx.f32_bits(cw[tt][0]), + ptx.f32_bits(cw[tt][1]), + ptx.f32_bits(cw[tt][2]), + ptx.f32_bits(cw[tt][3]), + ), + ) + self.sync_partials() + for k in cutlass.range_constexpr((64 * max_rows + THREADS - 1) // THREADS): + idx = tid + k * THREADS + if idx < 64 * rows: + rho = idx % 64 + n = idx // 64 + tt = n // 8 + nn = n % 8 + tl = rho // 16 + rr_ = rho % 16 + ln = (rr_ % 8) * 4 + nn // 2 + reg = (rr_ // 8) * 2 + nn % 2 + v = ptx.add_rn( + ptx.ld_shared_f32(s_red + ((tl * 2 + tt) * 32 + ln) * 16 + reg * 4), + ptx.ld_shared_f32( + s_red + (((tl + 4) * 2 + tt) * 32 + ln) * 16 + reg * 4 + ), + ) + qb = ptx.bf16_to_f32(ptx.cvt_rn_bf16_f32(v)) + dst = mqa_q + Int64(n) * mqa_stride + Int64(h * MQA_DIM + r * 64 + rho) + ptx.st_global_u8( + dst, + cutlass.Uint16( + Uint32( + ptx.cvt_rn_satfinite_e4m3x2_f32( + ptx.mul_rn(qb, inv_qs), Float32(0.0) + ) + ) + & Uint32(0xFF) + ), + ) + self.release_outputs() + self.exit_wait() + + # ------------------------------------------------------------------ kv cluster + @cute.jit + def kv_cta( + self, + r: Int32, + tid: Int32, + kv_c: Int64, + kv_c_stride: Int64, + k_pe: Int64, + k_pe_stride: Int64, + idx_k: Int64, + idx_k_stride: Int64, + positions: Int64, + slot_mapping: Int64, + idx_slot_mapping: Int64, + rows: Int32, + has_indexer: Int32, + kv_norm_w: Int64, + kv_eps: Float32, + rope_cache: Int64, + rope_stride: Int64, + k_scale: Int64, + ik_w: Int64, + ik_b: Int64, + ik_eps: Float32, + index_rope_cache: Int64, + index_rope_stride: Int64, + mla_cache: Int64, + mla_block_stride: Int64, + mla_entry_stride: Int64, + mla_block_size: Int64, + idx_cache: Int64, + idx_block_stride: Int64, + idx_block_size: Int64, + topk: Int64, + topk_stride: Int64, + topk_width: Int32, + ): + lane = tid % 32 + warp = tid // 32 + if r == 0: + self.mla_rows( + lane, + warp, + kv_c, + kv_c_stride, + k_pe, + k_pe_stride, + positions, + slot_mapping, + rows, + kv_norm_w, + kv_eps, + rope_cache, + rope_stride, + k_scale, + mla_cache, + mla_block_stride, + mla_entry_stride, + mla_block_size, + ) + else: + if has_indexer == 0: + # shared layers: no work, but every path of the grid executes the PDL + # wait + ptx.pdl_wait() + else: + if r == 1: + self.index_k_rows( + lane, + warp, + idx_k, + idx_k_stride, + positions, + slot_mapping, + idx_slot_mapping, + rows, + ik_w, + ik_b, + ik_eps, + index_rope_cache, + index_rope_stride, + idx_cache, + idx_block_stride, + idx_block_size, + ) + else: + self.wait_inputs() + per_row = topk_width // 4 + end = rows * per_row + neg = Uint32(0xFFFFFFFF) + for idx in cutlass.range( + (r - 2) * THREADS + tid, end, (CLUSTER - 2) * THREADS, unroll=1 + ): + n = idx // per_row + cc = idx % per_row + ptx.st_global_v4( + topk + (Int64(n) * topk_stride + Int64(cc * 4)) * 4, + (neg, neg, neg, neg), + ) + + @cute.jit + def mla_rows( + self, + lane: Int32, + warp: Int32, + kv_c: Int64, + kv_c_stride: Int64, + k_pe: Int64, + k_pe_stride: Int64, + positions: Int64, + slot_mapping: Int64, + rows: Int32, + kv_norm_w: Int64, + kv_eps: Float32, + rope_cache: Int64, + rope_stride: Int64, + k_scale: Int64, + mla_cache: Int64, + mla_block_stride: Int64, + mla_entry_stride: Int64, + mla_block_size: Int64, + ): + """The kv RMSNorm + BF16 + per-tensor FP8 of the latent and the k_pe RoPE: one + MLA row per token (warp n = row n). Lane l owns elements 8 l + [0, 8) and + 256 + 8 l + [0, 8). Slot <= 0 is not written (slot 0 is the padding slot).""" + strided = self.num.norm_order == "strided_8x256" + # strided: lane l owns elements 8 l + [0, 8) and 256 + 8 l + [0, 8); + # contiguous: 16 l + [0, 16) + e0 = lane * 8 if strided else lane * 16 + e1 = KV_LORA // 2 + lane * 8 if strided else lane * 16 + 8 + w0 = ptx.ld_global_nc_v4(kv_norm_w + Int64(e0) * 2) + w1 = ptx.ld_global_nc_v4(kv_norm_w + Int64(e1) * 2) + ksc = ptx.ld_global_nc_f32(k_scale) + self.wait_kv_inputs() + for grp in cutlass.range_constexpr(self.max_rows // 8): + self.mla_rows_row( + warp + grp * 8, + lane, + kv_c, + kv_c_stride, + k_pe, + k_pe_stride, + positions, + slot_mapping, + rows, + kv_eps, + rope_cache, + rope_stride, + mla_cache, + mla_block_stride, + mla_entry_stride, + mla_block_size, + e0, + e1, + w0, + w1, + ksc, + ) + + @cute.jit + def mla_rows_row( + self, + n: Int32, + lane: Int32, + kv_c: Int64, + kv_c_stride: Int64, + k_pe: Int64, + k_pe_stride: Int64, + positions: Int64, + slot_mapping: Int64, + rows: Int32, + kv_eps: Float32, + rope_cache: Int64, + rope_stride: Int64, + mla_cache: Int64, + mla_block_stride: Int64, + mla_entry_stride: Int64, + mla_block_size: Int64, + e0: Int32, + e1: Int32, + w0, + w1, + ksc: Float32, + ): + strided = self.num.norm_order == "strided_8x256" + if n < rows: + slot = ptx.ld_global_s64(slot_mapping + Int64(n) * 8) + row = kv_c + Int64(n) * kv_c_stride * 2 + a = ptx.ld_global_v4(row + Int64(e0) * 2) + b = ptx.ld_global_v4(row + Int64(e1) * 2) + pos = ptx.ld_global_s64(positions + Int64(n) * 8) + kp = ptx.ld_global_u32( + k_pe + (Int64(n) * k_pe_stride + Int64(2 * lane)) * 2 + ) + c = self.rope_value( + rope_cache, rope_stride, pos, lane, self.rope_cache_fp32 + ) + sn = self.rope_value( + rope_cache, rope_stride, pos, 32 + lane, self.rope_cache_fp32 + ) + # negative slots are never written, slot 0 (padding) not under skip_slot0 + if slot > (Int64(0) if self.num.skip_slot0 else Int64(-1)): + xs = (a[0], a[1], a[2], a[3], b[0], b[1], b[2], b[3]) + ws = (w0[0], w0[1], w0[2], w0[3], w1[0], w1[1], w1[2], w1[3]) + x = [] + for e in cutlass.range_constexpr(8): + lo, hi = ptx.bf16x2_to_f32x2(xs[e]) + x.append(lo) + x.append(hi) + s = Float32(0.0) + for i in cutlass.range_constexpr(16): + if cutlass.const_expr(strided): + s = ptx.add_rn(s, ptx.mul_rn(x[i], x[i])) + else: + s = ptx.fma_rn(x[i], x[i], s) + for k in cutlass.range_constexpr(5): + o = (1 << k) if strided else (16 >> k) + s = ptx.add_rn(s, cute.arch.shuffle_sync_bfly(s, o)) + rr = ptx.rsqrt_approx(ptx.fma_rn(s, Float32(1.0 / KV_LORA), kv_eps)) + entry = ( + mla_cache + + (slot // mla_block_size) * mla_block_stride + + (slot % mla_block_size) * mla_entry_stride + ) + q = [] + for e in cutlass.range_constexpr(4): + word = Uint32(0) + for k in cutlass.range_constexpr(4): + i = e * 4 + k + wlo, whi = ptx.bf16x2_to_f32x2(ws[i // 2]) + wi = whi if i % 2 == 1 else wlo + xr = ptx.mul_rn(x[i], rr) + if cutlass.const_expr(self.num.norm_weight_plus_zero): + wi = ptx.add_rn(wi, Float32(0.0)) + y = ptx.mul_rn(xr, wi) + if cutlass.const_expr(self.num.kv_norm_bf16): + # the norm output materialized in BF16 + y = ptx.bf16_to_f32(ptx.cvt_rn_bf16_f32(y)) + byte = Uint32( + ptx.cvt_rn_satfinite_e4m3x2_f32( + ptx.div_full(y, ksc), Float32(0.0) + ) + ) & Uint32(0xFF) + word = word | (byte << Uint32(8 * k)) + q.append(word) + if cutlass.const_expr(strided): + ptx.st_global_v2(entry + Int64(e0), q[0], q[1]) + ptx.st_global_v2(entry + Int64(e1), q[2], q[3]) + else: + ptx.st_global_v4(entry + Int64(e0), (q[0], q[1], q[2], q[3])) + x1, x2 = ptx.bf16x2_to_f32x2(kp) + r1 = ptx.fma_rn(x1, c, ptx.neg_f32(ptx.mul_rn(x2, sn))) + r2 = ptx.fma_rn(x2, c, ptx.mul_rn(x1, sn)) + b1 = Uint32( + ptx.cvt_rn_satfinite_e4m3x2_f32(ptx.div_full(r1, ksc), Float32(0.0)) + ) & Uint32(0xFF) + b2 = Uint32( + ptx.cvt_rn_satfinite_e4m3x2_f32(ptx.div_full(r2, ksc), Float32(0.0)) + ) & Uint32(0xFF) + ptx.st_global_b16( + entry + Int64(KV_LORA + 2 * lane), + cutlass.Uint16(b1 | (b2 << Uint32(8))), + ) + + @cute.jit + def index_k_rows( + self, + lane: Int32, + warp: Int32, + idx_k: Int64, + idx_k_stride: Int64, + positions: Int64, + slot_mapping: Int64, + idx_slot_mapping: Int64, + rows: Int32, + ik_w: Int64, + ik_b: Int64, + ik_eps: Float32, + rope_cache: Int64, + rope_stride: Int64, + idx_cache: Int64, + idx_block_stride: Int64, + idx_block_size: Int64, + ): + """Index-K: LayerNorm(128) + interleaved RoPE (dims < 64) + BF16 + FP8 with an + FP32 scale, one row per token + (warp n = row n, lane l = dims 4 l .. 4 l + 3). Rows whose MLA or index slot is + negative are not written.""" + lw = ptx.ld_global_nc_v4(ik_w + Int64(lane) * 16) + lb = ptx.ld_global_nc_v4(ik_b + Int64(lane) * 16) + self.wait_inputs() + for grp in cutlass.range_constexpr(self.max_rows // 8): + self.index_k_rows_row( + warp + grp * 8, + lane, + idx_k, + idx_k_stride, + positions, + slot_mapping, + idx_slot_mapping, + rows, + ik_eps, + rope_cache, + rope_stride, + idx_cache, + idx_block_stride, + idx_block_size, + lw, + lb, + ) + + @cute.jit + def index_k_rows_row( + self, + n: Int32, + lane: Int32, + idx_k: Int64, + idx_k_stride: Int64, + positions: Int64, + slot_mapping: Int64, + idx_slot_mapping: Int64, + rows: Int32, + ik_eps: Float32, + rope_cache: Int64, + rope_stride: Int64, + idx_cache: Int64, + idx_block_stride: Int64, + idx_block_size: Int64, + lw, + lb, + ): + if n < rows: + mla_slot = ptx.ld_global_s64(slot_mapping + Int64(n) * 8) + slot = ptx.ld_global_s64(idx_slot_mapping + Int64(n) * 8) + ipos = ptx.ld_global_s64(positions + Int64(n) * 8) + kv = ptx.ld_global_v2( + idx_k + (Int64(n) * idx_k_stride + Int64(4 * lane)) * 2 + ) + x0, x1 = ptx.bf16x2_to_f32x2(kv[0]) + x2, x3 = ptx.bf16x2_to_f32x2(kv[1]) + x = (x0, x1, x2, x3) + num = self.num + if cutlass.const_expr(num.index_mean_sum == "sequential"): + sm = ptx.add_rn( + ptx.add_rn(ptx.add_rn(ptx.add_rn(Float32(0.0), x0), x1), x2), x3 + ) + else: + sm = ptx.add_rn(ptx.add_rn(x0, x1), ptx.add_rn(x2, x3)) + for k in cutlass.range_constexpr(5): + sm = ptx.add_rn(sm, cute.arch.shuffle_sync_bfly(sm, 16 >> k)) + d = [] + vs = Float32(0.0) + for e in cutlass.range_constexpr(4): + # x - mean with the mean's multiply folded in (one rounding) + d.append(ptx.fma_rn(sm, Float32(-1.0 / INDEX_DIM), x[e])) + vs = ptx.fma_rn(d[e], d[e], vs) + for k in cutlass.range_constexpr(5): + vs = ptx.add_rn(vs, cute.arch.shuffle_sync_bfly(vs, 16 >> k)) + rstd = ptx.rsqrt_approx(ptx.fma_rn(vs, Float32(1.0 / INDEX_DIM), ik_eps)) + nm = [] + for e in cutlass.range_constexpr(4): + nm.append( + ptx.fma_rn( + ptx.mul_rn(d[e], rstd), ptx.bits_f32(lw[e]), ptx.bits_f32(lb[e]) + ) + ) + res = self.index_rope( + lane < 16, + lane, + nm[0], + nm[1], + nm[2], + nm[3], + rope_cache, + rope_stride, + ipos, + ) + rb = [] + for e in cutlass.range_constexpr(4): + if cutlass.const_expr(num.index_k_bf16): + rb.append(ptx.bf16_to_f32(ptx.cvt_rn_bf16_f32(res[e]))) + else: + rb.append(res[e]) + am = ptx.max_f32( + ptx.max_f32(ptx.abs_f32(rb[0]), ptx.abs_f32(rb[1])), + ptx.max_f32(ptx.abs_f32(rb[2]), ptx.abs_f32(rb[3])), + ) + for k in cutlass.range_constexpr(5): + am = ptx.max_f32(am, cute.arch.shuffle_sync_bfly(am, 16 >> k)) + sc = ptx.div_rn(ptx.max_f32(Float32(1e-4), am), Float32(448.0)) + inv_sc = Float32(1.0) + if cutlass.const_expr(num.index_scale == "fp32_amax"): + inv_sc = ptx.div_rn(Float32(1.0), sc) + else: + # power-of-two scale 2^ceil(log2(s)), values divided by it + sc = cute.math.exp2(cute.math.ceil(cute.math.log2(sc))) + if (mla_slot >= Int64(0)) & (slot >= Int64(0)): + blk = slot // idx_block_size + ofs = slot % idx_block_size + row_base = idx_cache + blk * idx_block_stride + word = Uint32(0) + for e in cutlass.range_constexpr(4): + if cutlass.const_expr(num.index_scale == "fp32_amax"): + # x (1 / s), clipped to +-448 + v = ptx.max_f32( + ptx.min_f32(ptx.mul_rn(rb[e], inv_sc), Float32(448.0)), + Float32(-448.0), + ) + else: + v = ptx.div_rn(rb[e], sc) + byte = Uint32( + ptx.cvt_rn_satfinite_e4m3x2_f32(v, Float32(0.0)) + ) & Uint32(0xFF) + word = word | (byte << Uint32(8 * e)) + ptx.st_global_u32(row_base + ofs * INDEX_DIM + Int64(4 * lane), word) + if lane == 0: + at = row_base + idx_block_size * INDEX_DIM + ofs * 4 + if cutlass.const_expr(self.index_scale_ue8m0x4): + # the power-of-two scale as four equal UE8M0 bytes + e8 = (ptx.f32_bits(sc) >> Uint32(23)) & Uint32(0xFF) + ptx.st_global_u32(at, e8 * Uint32(0x01010101)) + else: + ptx.st_global_f32(at, sc) + + # ------------------------------------------------------------------ steps + @cute.jit + def q_norm_sums_strided( + self, + s_xs: Int32, + s_ssq: Int32, + qv, + warp: Int32, + lane: Int32, + tid: Int32, + rows: Int32, + ): + """q_a sums of squares, strided order: the raw rows go to the XS slots; warp n + sums row n, lane l adding the rounded squares of elements 8 l + 256 b + v + (b, v = 0..7) to 0 in (b, v) order, then a butterfly xor 1, 2, 4, 8, 16.""" + for n in cutlass.range_constexpr(self.max_rows): + if n < rows: + ptx.st_shared_v4(s_xs + n * XS_STRIDE + tid * 16, qv[n]) + ptx.sync_threads() + if warp < rows: + s = Float32(0.0) + row = s_xs + warp * XS_STRIDE + for b in cutlass.range_constexpr(8): + u = ptx.ld_shared_v4(row + (lane + 32 * b) * 16) + for e in cutlass.range_constexpr(4): + lo, hi = ptx.bf16x2_to_f32x2(u[e]) + s = ptx.add_rn(s, ptx.mul_rn(lo, lo)) + s = ptx.add_rn(s, ptx.mul_rn(hi, hi)) + for k in cutlass.range_constexpr(5): + s = ptx.add_rn(s, cute.arch.shuffle_sync_bfly(s, 1 << k)) + if lane == 0: + ptx.st_shared_f32(s_ssq + warp * 4, s) + + @cute.jit + def q_norm_sums_contiguous( + self, s_ssq: Int32, qv, warp: Int32, lane: Int32, rows: Int32 + ): + """q_a sums of squares, contiguous order: each thread fuses the squares of its 8 + elements into 0 in order, a butterfly xor 16 .. 1 sums the warp, and the 8 warp + sums of row n are added in warp order when the row is normalized.""" + for n in cutlass.range_constexpr(self.max_rows): + if n < rows: + s = Float32(0.0) + for e in cutlass.range_constexpr(4): + lo, hi = ptx.bf16x2_to_f32x2(qv[n][e]) + s = ptx.fma_rn(lo, lo, s) + s = ptx.fma_rn(hi, hi, s) + for k in cutlass.range_constexpr(5): + s = ptx.add_rn(s, cute.arch.shuffle_sync_bfly(s, 16 >> k)) + if lane == 0: + ptx.st_shared_f32(s_ssq + (warp * self.max_rows + n) * 4, s) + + @cute.jit + def load16(self, addr: Int64, valid): + w0 = Uint32(0) + w1 = Uint32(0) + w2 = Uint32(0) + w3 = Uint32(0) + if valid: + w0, w1, w2, w3 = ptx.ld_global_v4(addr) + return w0, w1, w2, w3 + + @cute.jit + def lds16(self, addr: Int32, valid): + w0 = Uint32(0) + w1 = Uint32(0) + w2 = Uint32(0) + w3 = Uint32(0) + if valid: + w0, w1, w2, w3 = ptx.ld_shared_v4(addr) + return w0, w1, w2, w3 + + @cute.jit + def rope_value( + self, + cache: Int64, + stride: Int64, + pos: Int64, + col: Int32, + fp32: cutlass.Constexpr[bool], + ): + idx = pos * stride + Int64(col) + if cutlass.const_expr(fp32): + v = ptx.ld_global_f32(cache + idx * 4) + else: + v = ptx.bf16_to_f32(ptx.ld_global_u16(cache + idx * 2)) + return v + + @cute.jit + def index_rope( + self, + valid, + lane: Int32, + n0: Float32, + n1: Float32, + n2: Float32, + n3: Float32, + cache: Int64, + stride: Int64, + pos: Int64, + ): + """Interleaved RoPE of this lane's dims 4 l .. 4 l + 3 when they are < 64 (pairs + (4 l, 4 l + 1), (4 l + 2, 4 l + 3); cos / sin index = dim / 2). Even: e0 cos - + e1 sin with e1 sin rounded. Odd: e0 sin + e1 cos with e1 cos rounded + (``round_x1_cos``) or e1 cos + e0 sin with e0 sin rounded (``round_x0_sin``).""" + o0 = n0 + o1 = n1 + o2 = n2 + o3 = n3 + if valid: + c0 = self.rope_value( + cache, stride, pos, 2 * lane, self.index_rope_cache_fp32 + ) + s0 = self.rope_value( + cache, stride, pos, 32 + 2 * lane, self.index_rope_cache_fp32 + ) + c1 = self.rope_value( + cache, stride, pos, 2 * lane + 1, self.index_rope_cache_fp32 + ) + s1 = self.rope_value( + cache, stride, pos, 33 + 2 * lane, self.index_rope_cache_fp32 + ) + o0 = ptx.fma_rn(n0, c0, ptx.neg_f32(ptx.mul_rn(n1, s0))) + o2 = ptx.fma_rn(n2, c1, ptx.neg_f32(ptx.mul_rn(n3, s1))) + if cutlass.const_expr(self.num.index_rope_odd == "round_x1_cos"): + o1 = ptx.fma_rn(n0, s0, ptx.mul_rn(n1, c0)) + o3 = ptx.fma_rn(n2, s1, ptx.mul_rn(n3, c1)) + else: + o1 = ptx.fma_rn(n1, c0, ptx.mul_rn(n0, s0)) + o3 = ptx.fma_rn(n3, c1, ptx.mul_rn(n2, s1)) + return o0, o1, o2, o3 + + @cute.jit + def q_rope_even(self, x0: Float32, x1: Float32, c: Float32, s: Float32) -> Float32: + """q_pe RoPE, even dim: x0 cos - x1 sin (one product rounded: x1 sin rounded, + the rest fused; else both products rounded, then the difference).""" + o = Float32(0.0) + if cutlass.const_expr(self.num.q_rope == "one_product_rounded"): + o = self.rope_even(x0, x1, c, s) + else: + o = ptx.sub_rn(ptx.mul_rn(x0, c), ptx.mul_rn(x1, s)) + return o + + @cute.jit + def q_rope_odd(self, x0: Float32, x1: Float32, c: Float32, s: Float32) -> Float32: + """q_pe RoPE, odd dim: x1 cos + x0 sin (one product rounded: x0 sin rounded, the + rest fused; else both products rounded, then the sum).""" + o = Float32(0.0) + if cutlass.const_expr(self.num.q_rope == "one_product_rounded"): + o = self.rope_odd(x0, x1, c, s) + else: + o = ptx.add_rn(ptx.mul_rn(x1, c), ptx.mul_rn(x0, s)) + return o + + @cute.jit + def rope_even(self, x0: Float32, x1: Float32, c: Float32, s: Float32) -> Float32: + """x0 cos - x1 sin, the product x1 sin rounded, the other fused.""" + return ptx.fma_rn(x0, c, ptx.neg_f32(ptx.mul_rn(x1, s))) + + @cute.jit + def rope_odd(self, x0: Float32, x1: Float32, c: Float32, s: Float32) -> Float32: + """x1 cos + x0 sin, the product x0 sin rounded, the other fused.""" + return ptx.fma_rn(x1, c, ptx.mul_rn(x0, s)) + + @cute.jit + def gather(self, base: Int32, tid: Int32, rows: Int32): + """The head's 192 nope values per row from ranks 0..5 (distributed shared + memory) into QN.""" + if tid < (NOPE // 32) * rows * 4: + src = tid // (rows * 4) + rem = tid % (rows * 4) + n = rem // 4 + ch = rem % 4 + raddr = ptx.mapa_shared_cluster( + base + self.off["pub"] + n * 64 + ch * 16, src + ) + v = ptx.ld_shared_cluster_v4(raddr) + ptx.st_shared_v4( + base + self.off["qn"] + n * QN_STRIDE + src * 64 + ch * 16, v + ) + + @cute.jit + def rope_pair( + self, + valid, + positions: Int64, + token: Int32, + pair: Int32, + cache: Int64, + stride: Int64, + ): + """The cos / sin of one (token, pair) q_pe RoPE slot, zero when unused.""" + c = Float32(0.0) + s = Float32(0.0) + if valid: + pos = ptx.ld_global_s64(positions + Int64(token) * 8) + c = self.rope_value(cache, stride, pos, pair, self.rope_cache_fp32) + s = self.rope_value(cache, stride, pos, 32 + pair, self.rope_cache_fp32) + return c, s + + @cute.jit + def rows_dead(self): + """Every warp's reads of the token rows -> the partials that reuse them.""" + ptx.sync_threads() + + @cute.jit + def handoff(self, base: Int32, tid: Int32, rows: Int32): + self.publish() + self.gather(base, tid, rows) + + @cute.jit + def wait_inputs(self): + ptx.pdl_wait() + ptx.fence_proxy_async_global() + + @cute.jit + def wait_q_inputs(self): + ptx.pdl_wait() + ptx.fence_proxy_async_global() + + @cute.jit + def wait_kv_inputs(self): + ptx.pdl_wait() + ptx.fence_proxy_async_global() + + @cute.jit + def wait_weights(self, bar: Int32): + ptx.mbarrier_wait_parity(bar, 0) + + @cute.jit + def sync_partials(self): + ptx.sync_threads() + + @cute.jit + def publish(self): + ptx.cluster_arrive_release() + ptx.cluster_wait_acquire() + + @cute.jit + def done_reading(self): + ptx.cluster_arrive_release() + + @cute.jit + def release_outputs(self): + ptx.fence_proxy_async_global() + + @cute.jit + def exit_wait(self): + ptx.cluster_wait_acquire() diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/mla_uv_proj.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/mla_uv_proj.py new file mode 100644 index 00000000..734ef71f --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/mla_uv_proj.py @@ -0,0 +1,394 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""MLA value up-projection for decode / verify rows, with L2 staging of static tensors. + +For the 8 attention heads of one tensor-parallel rank: + + out[m, 256 h + v] = sum_l attn[m, h, l] * W_UV[h][v, l] (m < rows) + +with BF16 inputs, FP32 accumulation and one BF16 rounding of each output. ``W_UV[h]`` +is rows ``[448 h + 192, 448 h + 448)`` of ``kv_b_proj.weight`` (see +``packing.pack_uv_proj``). It replaces the batched GEMM ``bmm(attn[8, M, 512], +W_UV[8, 512, 256])`` at up to ``max_rows`` rows (8 or 16). + +Work split: 128 CTAs of 256 threads. CTA ``(h, j)`` owns value dims ``[16 j, 16 j + +16)`` of head ``h`` over the whole latent (K = 512); warp ``w`` owns latent columns +``[64 w, 64 w + 64)`` and sums them with ``mma.sync`` m16n8k16 (A = attention rows, B = +weights). The 8 warp partials are added in warp order 0..7 from 0.0, then rounded to +BF16 once, so the result of a row does not depend on the other rows or on ``max_rows``. + +Ordering (PTX ISA memory consistency model): + * The kernel triggers its dependents at entry; nothing it writes is read by a + dependent before that dependent's own ``griddepcontrol.wait``. + * Before ``griddepcontrol.wait`` it reads only constant data: its weight tile (one + bulk copy completing on an mbarrier) and the L2 prefetches below. + * After the wait, ``fence.proxy.async.global`` orders the attention rows (which a + predecessor may have written through the async proxy) before the generic loads. + * Partials go through shared memory and ``bar.sync``. + * Every thread runs ``fence.proxy.async.global`` after its last output store, for a + consumer that reads the output through the async proxy. + +L2 staging: before the wait, the CTAs issue bulk L2 prefetches of up to two static +tensors (for example weights read later in the same step). Each slot stages either +its first ``bytes`` bytes (``stride == 0``) or, for every ``stride``-byte slab, the +first ``prefix`` bytes of that slab, with eviction priority ``policy`` (0 last, +1 normal, 2 first). The work items (4 KiB chunks) of slot 0 then slot 1 form one +list; item ``i`` is issued by CTA ``i % 128``. Prefetches are hints: no output +depends on them. +""" + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass import Float32, Int32, Int64, Uint8 + +from ..primitives import ptx_core as ptx + +HEADS = 8 +LATENT = 512 +VALUE_DIM = 256 +VALUES_PER_CTA = 16 +CTAS_PER_HEAD = VALUE_DIM // VALUES_PER_CTA +GRID = HEADS * CTAS_PER_HEAD +THREADS = 256 +WARPS = THREADS // 32 +STEPS_PER_WARP = LATENT // 32 // WARPS +N_TILES = VALUES_PER_CTA // 8 +WEIGHT_TILE_BYTES = (LATENT // 32) * VALUES_PER_CTA * 64 +STAGE_CHUNK = 4096 +MAX_ROWS = (8, 16) +POLICIES = ("evict_last", "evict_normal", "evict_first") +assert max(MAX_ROWS) * VALUES_PER_CTA <= THREADS + + +class MlaUvProj: + """``out[rows, 2048] = attn[rows, 8, 512] x W_UV`` for ``rows <= max_rows``. + + Arguments of a call (all addresses are device addresses; strides in elements): + ``weights`` (packed by ``packing.pack_uv_proj``), ``attn`` (BF16, unit stride + in the latent, rows and heads 16-byte aligned) with its row and head strides, + ``out`` (BF16, unit stride in the value dim) with its row stride, ``rows``, then + two staging slots ``(address or 0, bytes, stride, prefix, policy)``, then the + stream. + """ + + def __init__(self, max_rows: int): + if max_rows not in MAX_ROWS: + raise ValueError(f"max_rows must be one of {MAX_ROWS}, got {max_rows}") + self.max_rows = max_rows + + @cute.jit + def __call__( + self, + weights: Int64, + attn: Int64, + attn_row_stride: Int64, + attn_head_stride: Int64, + out: Int64, + out_row_stride: Int64, + rows: Int32, + stage0: Int64, + stage0_bytes: Int64, + stage0_stride: Int64, + stage0_prefix: Int64, + stage0_policy: Int32, + stage1: Int64, + stage1_bytes: Int64, + stage1_stride: Int64, + stage1_prefix: Int64, + stage1_policy: Int32, + stream: cuda.CUstream, + ): + self.kernel( + weights, + attn, + attn_row_stride, + attn_head_stride, + out, + out_row_stride, + rows, + stage0, + stage0_bytes, + stage0_stride, + stage0_prefix, + stage0_policy, + stage1, + stage1_bytes, + stage1_stride, + stage1_prefix, + stage1_policy, + ).launch( + grid=[GRID, 1, 1], + block=[THREADS, 1, 1], + stream=stream, + use_pdl=True, + max_number_threads=[THREADS, 1, 1], + min_blocks_per_mp=1, + ) + + @cute.kernel + def kernel( + self, + weights: Int64, + attn: Int64, + attn_row_stride: Int64, + attn_head_stride: Int64, + out: Int64, + out_row_stride: Int64, + rows: Int32, + stage0: Int64, + stage0_bytes: Int64, + stage0_stride: Int64, + stage0_prefix: Int64, + stage0_policy: Int32, + stage1: Int64, + stage1_bytes: Int64, + stage1_stride: Int64, + stage1_prefix: Int64, + stage1_policy: Int32, + ): + max_rows = self.max_rows + tid, _, _ = cute.arch.thread_idx() + cta, _, _ = cute.arch.block_idx() + smem = cutlass.utils.SmemAllocator() + tile = smem.allocate_tensor( + Uint8, cute.make_layout(WEIGHT_TILE_BYTES), byte_alignment=1024 + ) + partial = smem.allocate_tensor( + Float32, + cute.make_layout(WARPS * max_rows * VALUES_PER_CTA), + byte_alignment=16, + ) + mbar = smem.allocate_array(Int64, 1) + tile_s = ptx.shared_address(tile.iterator) + partial_s = ptx.shared_address(partial.iterator) + bar = ptx.shared_address(mbar) + + ptx.pdl_launch_dependents() + tid = Int32(tid) + cta = Int32(cta) + lane = tid % 32 + warp = tid // 32 + g = lane // 4 + t = lane % 4 + head = cta // CTAS_PER_HEAD + part = cta % CTAS_PER_HEAD + + # Constant weight tile: one bulk copy, issued before the PDL wait. + if tid == 0: + ptx.mbarrier_init(bar, 1) + ptx.fence_mbarrier_init_release_cluster() + ptx.mbarrier_arrive_expect_tx(bar, WEIGHT_TILE_BYTES) + src = weights + Int64(cta) * WEIGHT_TILE_BYTES + ptx.bulk_copy_g2s(tile_s, src, WEIGHT_TILE_BYTES, bar) + ptx.sync_threads() + + self.stage_l2( + cta, + tid, + stage0, + stage0_bytes, + stage0_stride, + stage0_prefix, + stage0_policy, + stage1, + stage1_bytes, + stage1_stride, + stage1_prefix, + stage1_policy, + ) + + self.wait_inputs() + + # A fragments: lane (g, t) of warp w holds latent [64 w + 32 i + 8 t, +8) of + # rows g and (16-row form) g + 8. + lo = cute.make_rmem_tensor((STEPS_PER_WARP, 4), cutlass.Uint32) + lo.fill(0) + hi = cute.make_rmem_tensor((STEPS_PER_WARP, 4), cutlass.Uint32) + hi.fill(0) + row_lo = ( + attn + + ( + Int64(g) * attn_row_stride + + Int64(head) * attn_head_stride + + Int64(warp * (STEPS_PER_WARP * 32) + 8 * t) + ) + * 2 + ) + row_hi = row_lo + Int64(8) * attn_row_stride * 2 + for i in cutlass.range_constexpr(STEPS_PER_WARP): + if g < rows: + v = ptx.ld_global_v4(row_lo + i * 64) + for k in cutlass.range_constexpr(4): + lo[i, k] = v[k] + if cutlass.const_expr(max_rows == 16): # noqa: SIM102 + if g + 8 < rows: + v = ptx.ld_global_v4(row_hi + i * 64) + for k in cutlass.range_constexpr(4): + hi[i, k] = v[k] + + self.wait_weights(bar) + + acc = cute.make_rmem_tensor((N_TILES, 4), Float32) + acc.fill(0.0) + for i in cutlass.range_constexpr(STEPS_PER_WARP): + block = tile_s + (warp * STEPS_PER_WARP + i) * (VALUES_PER_CTA * 64) + for n in cutlass.range_constexpr(N_TILES): + b = ptx.ld_shared_v4(block + (8 * n + g) * 64 + t * 16) + c = (acc[n, 0], acc[n, 1], acc[n, 2], acc[n, 3]) + c = ptx.mma_m16n8k16_bf16( + (lo[i, 0], hi[i, 0], lo[i, 1], hi[i, 1]), (b[0], b[1]), c + ) + c = ptx.mma_m16n8k16_bf16( + (lo[i, 2], hi[i, 2], lo[i, 3], hi[i, 3]), (b[2], b[3]), c + ) + for k in cutlass.range_constexpr(4): + acc[n, k] = c[k] + + # Warp partials -> shared memory -> fixed-order sum -> BF16. + if g < rows: + for n in cutlass.range_constexpr(N_TILES): + at = ( + partial_s + + ((warp * max_rows + g) * VALUES_PER_CTA + 8 * n + 2 * t) * 4 + ) + ptx.st_shared_v2_f32(at, acc[n, 0], acc[n, 1]) + if cutlass.const_expr(max_rows == 16): # noqa: SIM102 + if g + 8 < rows: + for n in cutlass.range_constexpr(N_TILES): + at = ( + partial_s + + ((warp * max_rows + g + 8) * VALUES_PER_CTA + 8 * n + 2 * t) + * 4 + ) + ptx.st_shared_v2_f32(at, acc[n, 2], acc[n, 3]) + self.sync_partials() + + # One output element per thread (rows * 16 <= 256 threads): thread tid owns row + # tid / 16, value tid % 16 of this CTA's slice. + if tid < rows * VALUES_PER_CTA: + r = tid // VALUES_PER_CTA + d = tid % VALUES_PER_CTA + s = Float32(0.0) + for w in cutlass.range_constexpr(WARPS): + at = partial_s + ((w * max_rows + r) * VALUES_PER_CTA + d) * 4 + s = ptx.add_rn(s, ptx.ld_shared_f32(at)) + col = head * VALUE_DIM + part * VALUES_PER_CTA + d + dst = out + (Int64(r) * out_row_stride + Int64(col)) * 2 + ptx.st_global_b16(dst, ptx.cvt_rn_bf16_f32(s)) + self.release_outputs() + + # Synchronization points, one method each. + + @cute.jit + def wait_inputs(self): + """Wait for the predecessor grid; its async-proxy writes before our loads.""" + ptx.pdl_wait() + ptx.fence_proxy_async_global() + + @cute.jit + def wait_weights(self, bar: Int32): + """Wait for the weight tile's bulk copy (phase 0 of ``bar``).""" + ptx.mbarrier_wait_parity(bar, 0) + + @cute.jit + def sync_partials(self): + """Warp partials in shared memory are visible to every thread.""" + ptx.sync_threads() + + @cute.jit + def release_outputs(self): + """Every thread, after its last output store: order the generic stores before + a consumer's async-proxy reads.""" + ptx.fence_proxy_async_global() + + @cute.jit + def issue_prefetch(self, addr: Int64, nbytes: Int32, policy: cutlass.Uint64): + ptx.bulk_prefetch_l2(addr, nbytes, policy) + + @cute.jit + def stage_l2( + self, + cta: Int32, + tid: Int32, + stage0: Int64, + stage0_bytes: Int64, + stage0_stride: Int64, + stage0_prefix: Int64, + stage0_policy: Int32, + stage1: Int64, + stage1_bytes: Int64, + stage1_stride: Int64, + stage1_prefix: Int64, + stage1_policy: Int32, + ): + """Issue this thread's share of the L2 prefetches (see the module docstring).""" + count0, per0, prefix0 = self.slot_items( + stage0, stage0_bytes, stage0_stride, stage0_prefix + ) + count1, per1, prefix1 = self.slot_items( + stage1, stage1_bytes, stage1_stride, stage1_prefix + ) + total = count0 + count1 + if total > Int64(0): + policy0 = self.policy(stage0_policy) + policy1 = self.policy(stage1_policy) + item = Int64(tid) * GRID + Int64(cta) + while item < total: + base = stage0 + size = stage0_bytes + stride = stage0_stride + per = per0 + prefix = prefix0 + policy = policy0 + i = item + if item >= count0: + base = stage1 + size = stage1_bytes + stride = stage1_stride + per = per1 + prefix = prefix1 + policy = policy1 + i = item - count0 + slab = i // per + chunk = (i % per) * STAGE_CHUNK + offset = slab * stride + chunk + remaining = prefix - chunk + if size - offset < remaining: + remaining = size - offset + if remaining > Int64(0): + nbytes = Int64(STAGE_CHUNK) + if remaining < nbytes: + nbytes = remaining + nbytes32 = Int32(nbytes) & Int32(-16) + if nbytes32 != Int32(0): + self.issue_prefetch(base + offset, nbytes32, policy) + item += Int64(GRID * THREADS) + + @cute.jit + def slot_items(self, base: Int64, size: Int64, stride: Int64, prefix: Int64): + """(items, items per slab, bytes staged per slab) of one staging slot.""" + count = Int64(0) + per = Int64(1) + staged = Int64(0) + if base != Int64(0): + staged = size + slabs = Int64(1) + if stride > Int64(0): + staged = stride + if prefix < stride: + staged = prefix + slabs = (size + stride - 1) // stride + per = (staged + STAGE_CHUNK - 1) // STAGE_CHUNK + count = slabs * per + return count, per, staged + + @cute.jit + def policy(self, code: Int32): + result = ptx.l2_policy(POLICIES[0]) + if code == Int32(1): + result = ptx.l2_policy(POLICIES[1]) + elif code == Int32(2): + result = ptx.l2_policy(POLICIES[2]) + return result diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/qkv_a_proj_small_m.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/qkv_a_proj_small_m.py new file mode 100644 index 00000000..332117a7 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/qkv_a_proj_small_m.py @@ -0,0 +1,434 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Row-owned skinny GEMM for the attention input projections at decode / verify row +counts (rows <= 8). + + out[m, n] = bf16( sum_k x[m, k] * W[n, k] ), x: [rows, 6144] BF16, W: [N, 6144] + BF16, FP32 accumulate + +Rows ``n < n_main`` go to ``out_main[:, n]``, the rest to ``out_aux[:, n - n_main]`` +(one launch serves the fused q_a / kv_a projection and the indexer key / weights +projection of a layer). + +Work split: CTA ``c`` of ``grid`` CTAs owns the even-aligned output rows ``[2 floor(G2 c +/ grid), 2 floor(G2 (c + 1) / grid))`` with ``G2 = N / 2`` and the full K: no cross-CTA +reduction. Its packed weight block (``packing.pack_qkv_a``, which depends on ``grid``) +is contiguous: ``[k-step 192][row R][32]``. Warp ``w`` owns k-steps ``[24 w, 24 w + +24)`` of every row and streams them through a ring of ``slots`` stages (4 k-steps each, +one mbarrier per slot); the first ``slots`` stages are issued before the PDL wait +(constant weights only). x stays in registers (24 x 16 B per lane, loaded right after +the wait). ``mma.sync`` m16n8k16 with the rows of x on A (rows ``g < rows``; rows ``g + +8`` zero) and 8 weight rows per n8 tile on B (the tile count of the largest CTA; rows a +CTA does not own are neither loaded nor stored); one 16-byte shared load per tile and +k-step feeds two MMAs. The 8 warp partials are +added in warp order from 0.0, rounded to BF16 once, and stored. + +Option fields (compile time): ``n_rows`` (N), ``n_main``, ``grid`` (the launch's CTA +count, normally the SM count; the packing must use the same value), ``slots`` (ring +depth, ``plan_slots``). + +Ordering: dependents are triggered at entry; only the constant weights are read before +``griddepcontrol.wait``; ``fence.proxy.async.global`` follows the wait and each storing +thread's output store. A ring slot is re-filled only after the warp's reads of it +(``__syncwarp``) and ``fence.proxy.async.shared::cta``. +""" + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass import Float32, Int32, Int64, Uint32 + +from ..primitives import ptx_core as ptx + +K = 6144 +STEPS = K // 32 # 192 k-steps of 32 +WARPS = 8 +THREADS = WARPS * 32 +STEPS_PER_WARP = STEPS // WARPS # 24 +STEPS_PER_STAGE = 4 +STAGES = STEPS_PER_WARP // STEPS_PER_STAGE # 6 +MAX_ROWS = 8 +MAX_TILES = 4 # n8 tiles per CTA (<= 32 output rows) +SMEM_BUDGET = 227 * 1024 + + +def rows_of(n_rows: int, grid: int, c: int) -> int: + groups = n_rows // 2 + return 2 * (groups * (c + 1) // grid - groups * c // grid) + + +def plan_slots(n_rows: int, grid: int, smem_budget: int = SMEM_BUDGET) -> int: + """Ring depth for the shared-memory budget (0 if even one stage per warp does not + fit).""" + rmax = max(rows_of(n_rows, grid, c) for c in range(grid)) + stride = STEPS_PER_STAGE * rmax * 64 + budget = min(smem_budget, 227 * 1024 - 1024) + slots = min(STAGES, budget // (WARPS * stride)) + if slots * stride < MAX_TILES * 8 * 8 * 4: + slots = 0 + return slots + + +class QkvAProjSmallM: + """See the module docstring. Call arguments (device addresses; strides in elements): + ``weights`` (packed), ``x`` and its row stride, ``rows`` (1..8), ``out_main`` and + its row stride, ``out_aux`` and its row stride (unused + when ``n_main == n_rows``), then the stream.""" + + def __init__(self, n_rows: int, n_main: int, grid: int, slots: int = 0): + if n_rows < 2 or n_rows % 2 or n_main < 2 or n_main % 2 or n_main > n_rows: + raise ValueError("n_rows and n_main must be even, 2 <= n_main <= n_rows") + if not 1 <= grid <= 512: + raise ValueError("grid must be in [1, 512]") + rs = [rows_of(n_rows, grid, c) for c in range(grid)] + if min(rs) < 2 or max(rs) > 8 * MAX_TILES: + raise ValueError( + f"rows per CTA must be in [2, 32], got {min(rs)}..{max(rs)}: " + "choose grid >= N / 32" + ) + self.n_rows, self.n_main, self.grid = n_rows, n_main, grid + self.slots = slots or plan_slots(n_rows, grid) + if self.slots < 1: + raise ValueError( + "shared memory budget too small for one ring stage per warp" + ) + self.stage_stride = STEPS_PER_STAGE * max(rs) * 64 + # n8 tiles per CTA: the most any CTA owns; a CTA with fewer rows skips the + # shared loads of its missing rows (their MMAs see zeros, their outputs are not + # stored) + self.tiles = max((r + 7) // 8 for r in rs) + + @cute.jit + def __call__( + self, + weights: Int64, + x: Int64, + x_stride: Int64, + rows: Int32, + out_main: Int64, + out_main_stride: Int64, + out_aux: Int64, + out_aux_stride: Int64, + stream: cuda.CUstream, + ): + self.kernel( + weights, + x, + x_stride, + rows, + out_main, + out_main_stride, + out_aux, + out_aux_stride, + ).launch( + grid=[self.grid, 1, 1], + block=[THREADS, 1, 1], + stream=stream, + use_pdl=True, + max_number_threads=[THREADS, 1, 1], + min_blocks_per_mp=1, + ) + + @cute.kernel + def kernel( + self, + weights: Int64, + x: Int64, + x_stride: Int64, + rows: Int32, + out_main: Int64, + out_main_stride: Int64, + out_aux: Int64, + out_aux_stride: Int64, + ): + self.body( + weights, + x, + x_stride, + rows, + out_main, + out_main_stride, + out_aux, + out_aux_stride, + ) + + @cute.jit + def body( + self, + weights, + x, + x_stride, + rows, + out_main, + out_main_stride, + out_aux, + out_aux_stride, + dn_bsum=None, + dn_eps=None, + ): + """The kernel's body; ``dn_bsum`` selects the epilogue-scaled form + (``QkvAProjSmallMDn``).""" + slots = self.slots + stride = self.stage_stride + tid_, _, _ = cute.arch.thread_idx() + cta_, _, _ = cute.arch.block_idx() + tid = Int32(tid_) + cta = Int64(cta_) + smem = cutlass.utils.SmemAllocator() + rings = smem.allocate_tensor( + cutlass.Uint8, cute.make_layout(WARPS * slots * stride), byte_alignment=1024 + ) + bars = smem.allocate_array(Int64, WARPS * STAGES) + s_ring0 = ptx.shared_address(rings.iterator) + s_bar0 = ptx.shared_address(bars) + + ptx.pdl_launch_dependents() + lane = tid % 32 + warp = tid // 32 + g = lane // 4 + t = lane % 4 + groups = Int64(self.n_rows // 2) + g0 = groups * cta // Int64(self.grid) + g1 = groups * (cta + Int64(1)) // Int64(self.grid) + r0 = Int32(g0 * 2) + nr = Int32((g1 - g0) * 2) # this CTA's output rows R + stage_bytes = STEPS_PER_STAGE * nr * 64 + wsrc = ( + weights + + Int64(r0) * (K * 2) + + Int64(warp * STEPS_PER_WARP) * Int64(nr) * 64 + ) + ring = s_ring0 + warp * (slots * stride) + bar0 = s_bar0 + warp * (STAGES * 8) + policy = ptx.l2_policy("evict_first") + + # constant weights: the first `slots` stages of this warp, issued before the PDL + # wait + if lane == 0: + for k in cutlass.range_constexpr(slots): + ptx.mbarrier_init(bar0 + k * 8, 1) + ptx.fence_mbarrier_init_release_cluster() + for q in cutlass.range_constexpr(min(slots, STAGES)): + self.issue(ring, bar0, wsrc, stage_bytes, q, policy) + cute.arch.sync_warp() + + self.wait_inputs() + + # epilogue-scaled form: this thread's output row's scale, from the block sums + # the predecessor wrote (loaded after the wait, used after the GEMM) + inv = Float32(1.0) + if cutlass.const_expr(dn_bsum is not None): + mi = (tid // 8) % 8 + tot = Float32(0.0) + if mi < rows: + for cc in cutlass.range_constexpr(8): + tot = ptx.add_rn( + tot, + ptx.ld_global_f32(dn_bsum + (Int64(mi) * 8 + Int64(cc)) * 4), + ) + inv = ptx.rsqrt_approx( + ptx.fma_rn(tot, ptx.rcp_rn(Float32(float(K))), dn_eps) + ) + + # x fragments for this warp's 24 k-steps (rows g < rows): coherent loads back to + # back + xrow = x + (Int64(g) * x_stride + Int64(warp * STEPS_PER_WARP * 32 + 8 * t)) * 2 + xr = [] + for i in cutlass.range_constexpr(STEPS_PER_WARP): + xr.append(self.load_row(xrow + i * 64, g < rows)) + + zero = Float32(0.0) + acc = [(zero, zero, zero, zero) for _ in range(self.tiles)] + z = Uint32(0) + for q in cutlass.range_constexpr(STAGES): + slot = q % slots + self.wait_stage(bar0 + slot * 8, (q // slots) & 1) + st = ring + slot * stride + for ii in cutlass.range_constexpr(STEPS_PER_STAGE): + a = xr[q * STEPS_PER_STAGE + ii] + blk = st + ii * nr * 64 + for j in cutlass.range_constexpr(self.tiles): + b0 = Uint32(0) + b1 = Uint32(0) + b2 = Uint32(0) + b3 = Uint32(0) + if 8 * j + g < nr: + b0, b1, b2, b3 = ptx.ld_shared_v4( + blk + (8 * j + g) * 64 + t * 16 + ) + c = ptx.mma_m16n8k16_bf16((a[0], z, a[1], z), (b0, b1), acc[j]) + acc[j] = ptx.mma_m16n8k16_bf16((a[2], z, a[3], z), (b2, b3), c) + if cutlass.const_expr(q + slots < STAGES): + self.refill_visible() + if lane == 0: + ptx.fence_proxy_async_shared_cta() + self.issue(ring, bar0, wsrc, stage_bytes, q + slots, policy) + + # cross-warp reduction in warp order: each warp's partials go to its own (fully + # consumed) ring + if g < rows: + for j in cutlass.range_constexpr(self.tiles): + ptx.st_shared_v2_f32( + ring + ((j * 8 + g) * 8 + 2 * t) * 4, acc[j][0], acc[j][1] + ) + self.sync_partials() + j = tid // 64 + m = (tid // 8) % 8 + n = tid % 8 + row = 8 * j + n + if (row < nr) & (m < rows): + v = Float32(0.0) + for w in cutlass.range_constexpr(WARPS): + v = ptx.add_rn( + v, + ptx.ld_shared_f32( + s_ring0 + w * (slots * stride) + ((j * 8 + m) * 8 + n) * 4 + ), + ) + if cutlass.const_expr(dn_bsum is not None): + v = ptx.mul_rn(v, inv) + o = ptx.cvt_rn_bf16_f32(v) + gr = r0 + row + if gr < self.n_main: + ptx.st_global_b16( + out_main + (Int64(m) * out_main_stride + Int64(gr)) * 2, o + ) + else: + ptx.st_global_b16( + out_aux + (Int64(m) * out_aux_stride + Int64(gr - self.n_main)) * 2, + o, + ) + self.release_outputs() + + # ------------------------------------------------------------------ steps + @cute.jit + def issue( + self, + ring: Int32, + bar0: Int32, + wsrc: Int64, + stage_bytes: Int32, + q: cutlass.Constexpr[int], + policy, + ): + """Lane 0: stage q of this warp into slot q % slots.""" + bar = bar0 + (q % self.slots) * 8 + ptx.mbarrier_arrive_expect_tx(bar, stage_bytes) + ptx.bulk_copy_g2s( + ring + (q % self.slots) * self.stage_stride, + wsrc + Int64(q) * Int64(stage_bytes), + stage_bytes, + bar, + policy, + ) + + @cute.jit + def load_row(self, addr: Int64, valid): + w0 = Uint32(0) + w1 = Uint32(0) + w2 = Uint32(0) + w3 = Uint32(0) + if valid: + w0, w1, w2, w3 = ptx.ld_global_v4(addr) + return w0, w1, w2, w3 + + @cute.jit + def wait_inputs(self): + ptx.pdl_wait() + ptx.fence_proxy_async_global() + + @cute.jit + def wait_stage(self, bar: Int32, parity: cutlass.Constexpr[int]): + ptx.mbarrier_wait_parity(bar, parity) + + @cute.jit + def refill_visible(self): + """The warp's generic reads of a slot -> the async-proxy re-fill of that slot + (with the proxy fence).""" + cute.arch.sync_warp() + + @cute.jit + def sync_partials(self): + ptx.sync_threads() + + @cute.jit + def release_outputs(self): + ptx.fence_proxy_async_global() + + +class QkvAProjSmallMDn(QkvAProjSmallM): + """``QkvAProjSmallM`` with the input's RMSNorm scale applied in the epilogue: the + input is ``u = bf16(x * gamma)`` (un-normalized) and every output row is + + out[m, n] = bf16( inv_m * sum_k u[m, k] * W[n, k] ), + inv_m = rsqrt(fma(sum_c bsum[m][c], rcp(6144), eps)), + + the fp32 accumulator scaled before its single bf16 rounding, so the output is + rounded at the scale of the normed input. ``bsum`` ``[rows, 8]`` fp32 are the + block sums of squares of ``x`` that the producer hop wrote (summed from 0 in + ``c`` order, the norm's own arithmetic); they are read after ``griddepcontrol.wait``. + Call arguments: those of ``QkvAProjSmallM``, then ``bsum`` and ``eps``, then the + stream.""" + + @cute.jit + def __call__( + self, + weights: Int64, + x: Int64, + x_stride: Int64, + rows: Int32, + out_main: Int64, + out_main_stride: Int64, + out_aux: Int64, + out_aux_stride: Int64, + bsum: Int64, + eps: Float32, + stream: cuda.CUstream, + ): + self.kernel_dn( + weights, + x, + x_stride, + rows, + out_main, + out_main_stride, + out_aux, + out_aux_stride, + bsum, + eps, + ).launch( + grid=[self.grid, 1, 1], + block=[THREADS, 1, 1], + stream=stream, + use_pdl=True, + max_number_threads=[THREADS, 1, 1], + min_blocks_per_mp=1, + ) + + @cute.kernel + def kernel_dn( + self, + weights: Int64, + x: Int64, + x_stride: Int64, + rows: Int32, + out_main: Int64, + out_main_stride: Int64, + out_aux: Int64, + out_aux_stride: Int64, + bsum: Int64, + eps: Float32, + ): + self.body( + weights, + x, + x_stride, + rows, + out_main, + out_main_stride, + out_aux, + out_aux_stride, + bsum, + eps, + ) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/sparse_mla_decode.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/sparse_mla_decode.py new file mode 100644 index 00000000..8170ede3 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/attn/sparse_mla_decode.py @@ -0,0 +1,666 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Sparse MLA decode / verify over the indexer's top-2048 tokens (FP8 MQA query and +cache). + +For one tensor-parallel rank (8 heads), per query row ``r`` and head ``h``: + + s_t = bmm1_scale * q[r, h] . kv[t] (576-wide FP8 dot products, FP32 + accumulate) out[r, h] = bmm2_scale * sum_t softmax(s)_t * kv[t][:512] (BF16) + +over the (up to) 2048 cache rows ``t`` selected for the row (ids < 0 are skipped). The +inputs are the stock decode call's: ``q`` [M, 8, 576] E4M3, the latent cache as flat +576-byte E4M3 rows, ``ids`` [M, 2048] int32 (physical cache rows, or logical token +positions mapped here through the request's block table). + +Structure: one 16-CTA cluster per query row; CTA ``c`` takes tokens ``[128 c, 128 c + +128)`` of the row's list. Its first eight warps each gather 16 cache rows (one bulk copy +per row, completing on the warp's mbarrier; unselected rows are zero-filled), compute +``K Q^T`` with ``mma.sync`` m16n8k32 E4M3 (tokens on M, heads on N) and the CTA's +softmax statistics; every warp then computes ``V^T P^T`` for its slice of the 512 output +dims, with P held as two E4M3 terms (``hi`` and the residual ``lo``, scaled by 2^8). The +16 partial results of a row are merged in a fixed order: each CTA pushes its partial +outputs and (max, sum) pairs into the owning CTA's shared memory with ``st.async``, +completing on the owner's mbarrier, and the owner rescales and sums them. The result is +deterministic and does not depend on the other rows. + +Ordering (PTX ISA memory consistency model): + * The cluster co-start: every CTA initializes its mbarriers and arms its combine + barrier, + then arrives (relaxed) on the cluster barrier, and waits on it before its first + ``st.async`` into a peer. + * Everything that reads predecessor data (q, ids, block table, cache rows) follows + ``griddepcontrol.wait`` and a ``fence.proxy.async.global``. + * Gathers complete on per-warp mbarriers (``expect_tx`` with release, waits with + acquire); + the combine's ``st.async`` complete on the owner's mbarrier at cluster scope. + * A CTA leaves only after the split exit cluster barrier, so no ``st.async`` is in + flight + toward an exited CTA; every thread runs ``fence.proxy.async.global`` after its + stores. + * With ``early_trigger`` the dependents are triggered right after the wait (for a + successor that waits on this grid's completion before reading its output). +""" + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass import Float32, Int32, Int64, Uint32 + +from ..primitives import ptx_core as ptx + +HEADS = 8 +QK_DIM = 576 +V_DIM = 512 +TOPK = 2048 +TOKENS_PER_CTA = 128 +CLUSTER = TOPK // TOKENS_PER_CTA # 16 CTAs per query row +QK_WARPS = 8 +TOKENS_PER_WARP = TOKENS_PER_CTA // QK_WARPS # 16 +ROW_BYTES = QK_DIM + 16 # shared-memory row pitch of a gathered cache row +QK_STEPS = QK_DIM // 32 +DIMS_PER_CTA = V_DIM // CLUSTER # 32 output dims merged by each CTA +P_PITCH = TOKENS_PER_CTA + 16 # bytes per head row of P (E4M3) +NEG_INF = float("-inf") + +# Shared memory (bytes) +OFF_KV = 0 +OFF_Q = OFF_KV + TOKENS_PER_CTA * ROW_BYTES +OFF_P = OFF_Q + HEADS * ROW_BYTES +P_BYTES = 2 * HEADS * P_PITCH +OFF_RED = OFF_P + P_BYTES # float [2][8 warps][8 heads]: max, then sum +OFF_RECV = OFF_RED + 2 * QK_WARPS * HEADS * 4 # float [16 sources][32 dims][8 heads] +OFF_STATS = ( + OFF_RECV + CLUSTER * DIMS_PER_CTA * HEADS * 4 +) # float [16 sources][2][8 heads] +OFF_BAR = OFF_STATS + CLUSTER * 2 * HEADS * 4 # u64 [8 gathers, q, combine] +SMEM_BYTES = OFF_BAR + (QK_WARPS + 2) * 8 +RECV_BYTES = CLUSTER * (DIMS_PER_CTA * HEADS + 2 * HEADS) * 4 +BAR_Q = QK_WARPS +BAR_COMBINE = QK_WARPS + 1 + + +class SparseMlaDecode: + """Sparse MLA decode over ``rows`` query rows. + + ``warps``: 16 (one CTA per SM; for up to 4 rows) or 8 (two CTAs per SM; more rows + per wave). ``logical_ids``: ``ids`` hold token positions, mapped through + ``block_table[request_of_row[r]]`` (``-1`` when negative or past the table); the + mapped ids are also written to ``phys_out`` when it is non-zero. ``early_trigger``: + trigger the dependents right after the PDL wait. + + Call arguments (device addresses, element strides): ``q``, ``kv``, ``ids``, ``out`` + ([rows, 8, 512] BF16), ``kv_rows`` (number of addressable cache rows), ``rows``, + ``scale_log2`` (bmm1_scale * log2(e)), ``bmm2_scale``, then for logical ids + ``request_of_row``, ``block_table``, ``block_table_stride``, ``max_blocks``, + ``block_shift`` (log2 of the block size), ``block_rows`` (cache rows per block), + ``phys_out`` (0: none), then the stream. + """ + + def __init__( + self, warps: int = 16, logical_ids: bool = False, early_trigger: bool = True + ): + if warps not in (8, 16): + raise ValueError(f"warps must be 8 or 16, got {warps}") + self.warps = warps + self.min_blocks = 1 if warps == 16 else 2 + self.logical_ids = logical_ids + self.early_trigger = early_trigger + + @cute.jit + def __call__( + self, + q: Int64, + kv: Int64, + ids: Int64, + out: Int64, + kv_rows: Int64, + rows: Int32, + scale_log2: Float32, + bmm2_scale: Float32, + request_of_row: Int64, + block_table: Int64, + block_table_stride: Int64, + max_blocks: Int32, + block_shift: Int32, + block_rows: Int64, + phys_out: Int64, + stream: cuda.CUstream, + ): + self.kernel( + q, + kv, + ids, + out, + kv_rows, + scale_log2, + bmm2_scale, + request_of_row, + block_table, + block_table_stride, + max_blocks, + block_shift, + block_rows, + phys_out, + ).launch( + grid=[CLUSTER, rows, 1], + block=[self.warps * 32, 1, 1], + cluster=[CLUSTER, 1, 1], + stream=stream, + use_pdl=True, + max_number_threads=[self.warps * 32, 1, 1], + min_blocks_per_mp=self.min_blocks, + ) + + @cute.kernel + def kernel( + self, + q: Int64, + kv: Int64, + ids: Int64, + out: Int64, + kv_rows: Int64, + scale_log2: Float32, + bmm2_scale: Float32, + request_of_row: Int64, + block_table: Int64, + block_table_stride: Int64, + max_blocks: Int32, + block_shift: Int32, + block_rows: Int64, + phys_out: Int64, + ): + warps = self.warps + dims_per_warp = V_DIM // warps + pairs = dims_per_warp // 32 + tid_, _, _ = cute.arch.thread_idx() + crank_, r_, _ = cute.arch.block_idx() + tid = Int32(tid_) + crank = Int32(crank_) + r = Int32(r_) + warp = tid // 32 + lane = tid % 32 + g = lane // 4 + t4 = lane % 4 + + smem = cutlass.utils.SmemAllocator() + buf = smem.allocate_tensor( + cutlass.Uint8, cute.make_layout(SMEM_BYTES), byte_alignment=128 + ) + base = ptx.shared_address(buf.iterator) + s_kv = base + OFF_KV + s_q = base + OFF_Q + s_ph = base + OFF_P + s_pl = base + OFF_P + HEADS * P_PITCH + s_red = base + OFF_RED + s_recv = base + OFF_RECV + s_stats = base + OFF_STATS + s_bar = base + OFF_BAR + + if tid == 0: + for i in cutlass.range_constexpr(QK_WARPS + 2): + ptx.mbarrier_init(s_bar + i * 8, 1) + ptx.fence_mbarrier_init_release_cluster() + # owner side of the combine: expect every peer's partial before the co-start + ptx.mbarrier_arrive_expect_tx( + s_bar + BAR_COMBINE * 8, RECV_BYTES, ".release.cluster" + ) + ptx.fence_proxy_async_shared_cta() + ptx.sync_threads() + self.cluster_costart_arrive() + + self.wait_inputs() + + # q rows (8 x 576 B), warp 0 + if warp == 0: + if lane == 0: + ptx.mbarrier_arrive_expect_tx( + s_bar + BAR_Q * 8, HEADS * QK_DIM, ".release.cta" + ) + cute.arch.sync_warp() + if lane < HEADS: + src = q + (Int64(r) * HEADS + Int64(lane)) * QK_DIM + ptx.bulk_copy_g2s( + s_q + lane * ROW_BYTES, src, QK_DIM, s_bar + BAR_Q * 8 + ) + + # gather: warp w < 8 takes list positions j = 128 c + 16 w + lane (lanes < 16) + vmask = Uint32(0) + s0 = Float32(0.0) + s1 = Float32(0.0) + s2 = Float32(0.0) + s3 = Float32(0.0) + if warp < QK_WARPS: + j = crank * TOKENS_PER_CTA + warp * TOKENS_PER_WARP + lane + tok_id = Int32(-1) + if lane < TOKENS_PER_WARP: + tok_id = ptx.ld_global_s32(ids + (Int64(r) * TOPK + Int64(j)) * 4) + if cutlass.const_expr(self.logical_ids): + tok_id = self.map_logical( + tok_id, + r, + j, + lane, + request_of_row, + block_table, + block_table_stride, + max_blocks, + block_shift, + block_rows, + phys_out, + ) + ok = (lane < TOKENS_PER_WARP) & (tok_id >= 0) & (Int64(tok_id) < kv_rows) + vmask = Uint32(cute.arch.vote_ballot_sync(ok)) + self.gather(s_kv, s_bar, kv, tok_id, ok, vmask, warp, lane) + + # S^T = K Q^T for this warp's 16 tokens (two accumulator chains: even / odd + # k-steps) + self.wait_q(s_bar + BAR_Q * 8) + qa = s_q + (lane % 8) * ROW_BYTES + ((lane // 8) % 2) * 16 + qf = [ptx.ldmatrix_x2(qa + ks * 32) for ks in range(QK_STEPS)] + self.wait_gather(s_bar + warp * 8) + row = warp * TOKENS_PER_WARP + (lane % 8) + ((lane // 8) % 2) * 8 + ka = s_kv + row * ROW_BYTES + (lane // 16) * 16 + zero = Float32(0.0) + even = (zero, zero, zero, zero) + odd = (zero, zero, zero, zero) + for ks in cutlass.range_constexpr(QK_STEPS): + a = ptx.ldmatrix_x4(ka + ks * 32) + if cutlass.const_expr(ks % 2 == 1): + odd = ptx.mma_m16n8k32_e4m3(a, qf[ks], odd) + else: + even = ptx.mma_m16n8k32_e4m3(a, qf[ks], even) + s0 = ptx.add_rn(odd[0], even[0]) + s1 = ptx.add_rn(odd[1], even[1]) + s2 = ptx.add_rn(odd[2], even[2]) + s3 = ptx.add_rn(odd[3], even[3]) + + # softmax statistics (base 2). s0: (token g, head 2 t4), s1: (g, 2 t4 + 1), + # s2: (g + 8, 2 t4), s3: (g + 8, 2 t4 + 1) + v0 = (vmask >> Uint32(g)) & Uint32(1) + v1 = (vmask >> Uint32(g + 8)) & Uint32(1) + ninf = Float32(NEG_INF) + t0 = ptx.mul_rn(scale_log2, s0) + t1 = ptx.mul_rn(scale_log2, s1) + t2 = ptx.mul_rn(scale_log2, s2) + t3 = ptx.mul_rn(scale_log2, s3) + s0 = ninf + s1 = ninf + s2 = ninf + s3 = ninf + if v0 != Uint32(0): + s0 = t0 + s1 = t1 + if v1 != Uint32(0): + s2 = t2 + s3 = t3 + mx0 = self.nan_max(ninf, self.nan_max(s0, s2)) + mx1 = self.nan_max(ninf, self.nan_max(s1, s3)) + for o in cutlass.range_constexpr(2, 5): + mx0 = self.nan_max(mx0, cute.arch.shuffle_sync_bfly(mx0, 1 << o)) + mx1 = self.nan_max(mx1, cute.arch.shuffle_sync_bfly(mx1, 1 << o)) + if g == 0: + ptx.st_shared_v2_f32(s_red + (warp * HEADS + 2 * t4) * 4, mx0, mx1) + ptx.sync_threads() + + if warp < QK_WARPS: + ninf = Float32(NEG_INF) + m0 = ninf + m1 = ninf + for w in cutlass.range_constexpr(QK_WARPS): + a0, a1 = ptx.ld_shared_v2_f32(s_red + (w * HEADS + 2 * t4) * 4) + m0 = self.nan_max(m0, a0) + m1 = self.nan_max(m1, a1) + # m = -inf: every token of this CTA's slice is masked (p = 0) + ms0 = m0 + ms1 = m1 + if m0 == ninf: + ms0 = Float32(0.0) + if m1 == ninf: + ms1 = Float32(0.0) + eight = Float32(8.0) + p00 = ptx.ex2_approx(ptx.add_rn(ptx.sub_rn(s0, ms0), eight)) + p01 = ptx.ex2_approx(ptx.add_rn(ptx.sub_rn(s1, ms1), eight)) + p10 = ptx.ex2_approx(ptx.add_rn(ptx.sub_rn(s2, ms0), eight)) + p11 = ptx.ex2_approx(ptx.add_rn(ptx.sub_rn(s3, ms1), eight)) + # P scaled by 2^8, split into an E4M3 term and an E4M3 residual; l sums both + # terms + h0 = ptx.cvt_rn_satfinite_e4m3x2_f32(p00, p01) + h1 = ptx.cvt_rn_satfinite_e4m3x2_f32(p10, p11) + fh0x, fh0y = ptx.e4m3x2_to_f32x2(h0) + fh1x, fh1y = ptx.e4m3x2_to_f32x2(h1) + lo0 = ptx.cvt_rn_satfinite_e4m3x2_f32( + ptx.sub_rn(p00, fh0x), ptx.sub_rn(p01, fh0y) + ) + lo1 = ptx.cvt_rn_satfinite_e4m3x2_f32( + ptx.sub_rn(p10, fh1x), ptx.sub_rn(p11, fh1y) + ) + fl0x, fl0y = ptx.e4m3x2_to_f32x2(lo0) + fl1x, fl1y = ptx.e4m3x2_to_f32x2(lo1) + l0 = ptx.add_rn( + ptx.add_rn(ptx.add_rn(fh0x, fl0x), ptx.add_rn(fh1x, fl1x)), Float32(0.0) + ) + l1 = ptx.add_rn( + ptx.add_rn(ptx.add_rn(fh0y, fl0y), ptx.add_rn(fh1y, fl1y)), Float32(0.0) + ) + tok = warp * TOKENS_PER_WARP + g + at = (2 * t4) * P_PITCH + tok + ptx.st_shared_u8(s_ph + at, h0 & 0xFF) + ptx.st_shared_u8(s_ph + at + P_PITCH, h0 >> 8) + ptx.st_shared_u8(s_ph + at + 8, h1 & 0xFF) + ptx.st_shared_u8(s_ph + at + P_PITCH + 8, h1 >> 8) + ptx.st_shared_u8(s_pl + at, lo0 & 0xFF) + ptx.st_shared_u8(s_pl + at + P_PITCH, lo0 >> 8) + ptx.st_shared_u8(s_pl + at + 8, lo1 & 0xFF) + ptx.st_shared_u8(s_pl + at + P_PITCH + 8, lo1 >> 8) + for o in cutlass.range_constexpr(2, 5): + l0 = ptx.add_rn(l0, cute.arch.shuffle_sync_bfly(l0, 1 << o)) + l1 = ptx.add_rn(l1, cute.arch.shuffle_sync_bfly(l1, 1 << o)) + if g == 0: + ptx.st_shared_v2_f32( + s_red + (QK_WARPS * HEADS + warp * HEADS + 2 * t4) * 4, l0, l1 + ) + + # every warp reads every gathered row below + self.wait_all_gathers(s_bar) + ptx.sync_threads() + + # O^T = V^T P^T: warp owns dims [dims_per_warp * warp, +dims_per_warp) as 32-dim + # tile pairs + pah = s_ph + (lane % 8) * P_PITCH + ((lane // 8) % 2) * 16 + pal = s_pl + (lane % 8) * P_PITCH + ((lane // 8) % 2) * 16 + i8 = lane % 8 + vset = (lane // 8) % 2 + doff = (lane // 16) * 16 + trow = (i8 // 2) * 4 + vset * 2 + (i8 % 2) + zero = Float32(0.0) + if cutlass.const_expr(self.warps == 16): + # 16 warps (up to 4 rows): accumulators in registers with static indices and + # the 32-token k-step loop rolled two at a time, which keeps the code small; + # the same MMAs in the same order as the unrolled form, so the same result + acc_t = cute.make_rmem_tensor((pairs, 2, 4), Float32) + for pr in cutlass.range_constexpr(pairs): + for eo in cutlass.range_constexpr(2): + for k in cutlass.range_constexpr(4): + acc_t[pr, eo, k] = zero + for ks in cutlass.range(TOKENS_PER_CTA // 32, unroll=2): + bh = ptx.ldmatrix_x2(pah + ks * 32) + bl = ptx.ldmatrix_x2(pal + ks * 32) + for pr in cutlass.range_constexpr(pairs): + d0 = warp * dims_per_warp + pr * 32 + ra = ptx.ldmatrix_x4( + s_kv + (ks * 32 + trow) * ROW_BYTES + d0 + doff, trans=True + ) + rb = ptx.ldmatrix_x4( + s_kv + (ks * 32 + 16 + trow) * ROW_BYTES + d0 + doff, + trans=True, + ) + ev = ( + ptx.prmt(ra[0], ra[1], 0x6420), + ptx.prmt(ra[2], ra[3], 0x6420), + ptx.prmt(rb[0], rb[1], 0x6420), + ptx.prmt(rb[2], rb[3], 0x6420), + ) + od = ( + ptx.prmt(ra[0], ra[1], 0x7531), + ptx.prmt(ra[2], ra[3], 0x7531), + ptx.prmt(rb[0], rb[1], 0x7531), + ptx.prmt(rb[2], rb[3], 0x7531), + ) + c0 = tuple(acc_t[pr, 0, k] for k in range(4)) + c1 = tuple(acc_t[pr, 1, k] for k in range(4)) + c0 = ptx.mma_m16n8k32_e4m3(ev, bh, c0) + c1 = ptx.mma_m16n8k32_e4m3(od, bh, c1) + c0 = ptx.mma_m16n8k32_e4m3(ev, bl, c0) + c1 = ptx.mma_m16n8k32_e4m3(od, bl, c1) + for k in cutlass.range_constexpr(4): + acc_t[pr, 0, k] = c0[k] + acc_t[pr, 1, k] = c1[k] + acc = [ + [tuple(acc_t[pr, eo, k] for k in range(4)) for eo in range(2)] + for pr in range(pairs) + ] + else: + acc = [ + [(zero, zero, zero, zero), (zero, zero, zero, zero)] + for _ in range(pairs) + ] + for ks in cutlass.range_constexpr(TOKENS_PER_CTA // 32): + bh = ptx.ldmatrix_x2(pah + ks * 32) + bl = ptx.ldmatrix_x2(pal + ks * 32) + for pr in cutlass.range_constexpr(pairs): + d0 = warp * dims_per_warp + pr * 32 + ra = ptx.ldmatrix_x4( + s_kv + (ks * 32 + trow) * ROW_BYTES + d0 + doff, trans=True + ) + rb = ptx.ldmatrix_x4( + s_kv + (ks * 32 + 16 + trow) * ROW_BYTES + d0 + doff, + trans=True, + ) + ev = ( + ptx.prmt(ra[0], ra[1], 0x6420), + ptx.prmt(ra[2], ra[3], 0x6420), + ptx.prmt(rb[0], rb[1], 0x6420), + ptx.prmt(rb[2], rb[3], 0x6420), + ) + od = ( + ptx.prmt(ra[0], ra[1], 0x7531), + ptx.prmt(ra[2], ra[3], 0x7531), + ptx.prmt(rb[0], rb[1], 0x7531), + ptx.prmt(rb[2], rb[3], 0x7531), + ) + acc[pr][0] = ptx.mma_m16n8k32_e4m3(ev, bh, acc[pr][0]) + acc[pr][1] = ptx.mma_m16n8k32_e4m3(od, bh, acc[pr][1]) + acc[pr][0] = ptx.mma_m16n8k32_e4m3(ev, bl, acc[pr][0]) + acc[pr][1] = ptx.mma_m16n8k32_e4m3(od, bl, acc[pr][1]) + + # combine: push this CTA's partials into the owners' shared memory (st.async) + self.cluster_costart_wait() + for pr in cutlass.range_constexpr(pairs): + dbase = warp * dims_per_warp + pr * 32 + dst = dbase // DIMS_PER_CTA + rrecv = ptx.mapa_shared_cluster( + s_recv + crank * (DIMS_PER_CTA * HEADS * 4), dst + ) + rbar = ptx.mapa_shared_cluster(s_bar + BAR_COMBINE * 8, dst) + for eo in cutlass.range_constexpr(2): + ld = dbase % DIMS_PER_CTA + 2 * g + eo + c = acc[pr][eo] + ptx.st_async_v2_f32(rrecv + (ld * HEADS + 2 * t4) * 4, c[0], c[1], rbar) + ptx.st_async_v2_f32( + rrecv + ((ld + 16) * HEADS + 2 * t4) * 4, c[2], c[3], rbar + ) + if tid < CLUSTER * 2 * HEADS: + dst = tid // (2 * HEADS) + which = (tid // HEADS) % 2 + h = tid % HEADS + v = Float32(NEG_INF) + if which == 0: + for w in cutlass.range_constexpr(QK_WARPS): + v = self.nan_max(v, ptx.ld_shared_f32(s_red + (w * HEADS + h) * 4)) + else: + v = ptx.add_rn( + ptx.ld_shared_f32(s_red + (QK_WARPS * HEADS + h) * 4), Float32(0.0) + ) + for w in cutlass.range_constexpr(1, QK_WARPS): + v = ptx.add_rn( + v, + ptx.ld_shared_f32( + s_red + (QK_WARPS * HEADS + w * HEADS + h) * 4 + ), + ) + remote = ptx.mapa_shared_cluster( + s_stats + ((crank * 2 + which) * HEADS + h) * 4, dst + ) + ptx.st_async_f32( + remote, v, ptx.mapa_shared_cluster(s_bar + BAR_COMBINE * 8, dst) + ) + self.wait_combine(s_bar + BAR_COMBINE * 8) + self.exit_arrive() + + # owner: rescale and sum the 16 partials of its 32 dims (one output element per + # thread) + if tid < HEADS * DIMS_PER_CTA: + ld = tid // HEADS + h = tid % HEADS + ninf = Float32(NEG_INF) + mi = [ + ptx.ld_shared_f32(s_stats + (i * 2 * HEADS + h) * 4) + for i in range(CLUSTER) + ] + mx = ninf + for i in cutlass.range_constexpr(CLUSTER): + mx = self.nan_max(mx, mi[i]) + lsum = Float32(0.0) + a = Float32(0.0) + if mx != ninf: + for i in cutlass.range_constexpr(CLUSTER): + w = ptx.ex2_approx(ptx.sub_rn(mi[i], mx)) + lsum = ptx.fma_rn( + w, + ptx.ld_shared_f32(s_stats + ((i * 2 + 1) * HEADS + h) * 4), + lsum, + ) + a = ptx.fma_rn( + w, + ptx.ld_shared_f32( + s_recv + ((i * DIMS_PER_CTA + ld) * HEADS + h) * 4 + ), + a, + ) + res = Float32(0.0) + if lsum != Float32(0.0): + res = ptx.mul_rn(a, ptx.div_rn(bmm2_scale, lsum)) + dst_addr = ( + out + + ( + (Int64(r) * HEADS + Int64(h)) * V_DIM + + Int64(crank * DIMS_PER_CTA + ld) + ) + * 2 + ) + ptx.st_global_b16(dst_addr, ptx.cvt_rn_bf16_f32(res)) + self.release_outputs() + self.exit_wait() + + # ------------------------------------------------------------------ steps + @cute.jit + def map_logical( + self, + tok_id: Int32, + r: Int32, + j: Int32, + lane: Int32, + request_of_row: Int64, + block_table: Int64, + block_table_stride: Int64, + max_blocks: Int32, + block_shift: Int32, + block_rows: Int64, + phys_out: Int64, + ): + """Logical token position -> physical cache row (-1: none), + recorded on request.""" + phys = Int32(-1) + if tok_id >= 0: + blk = Int32(Uint32(tok_id) >> Uint32(block_shift)) + if blk < max_blocks: + req = ptx.ld_global_s32(request_of_row + Int64(r) * 4) + at = ( + block_table + + (Int64(block_table_stride) * Int64(req) + Int64(blk)) * 4 + ) + page = ptx.ld_global_s32(at) + mask = ~(Int32(-1) << block_shift) + phys = page * Int32(block_rows) + (tok_id & mask) + if phys_out != Int64(0): # noqa: SIM102 + if lane < TOKENS_PER_WARP: + ptx.st_global_u32(phys_out + (Int64(r) * TOPK + Int64(j)) * 4, phys) + return phys + + @cute.jit + def gather( + self, + s_kv: Int32, + s_bar: Int32, + kv: Int64, + tok_id: Int32, + ok, + vmask: Uint32, + warp: Int32, + lane: Int32, + ): + """This warp's 16 cache rows into shared memory (bulk copies on its mbarrier; + zero rows when + not selected).""" + bar = s_bar + warp * 8 + if lane == 0: + ptx.mbarrier_arrive_expect_tx( + bar, Int32(cute.arch.popc(vmask)) * QK_DIM, ".release.cta" + ) + cute.arch.sync_warp() + dst = s_kv + (warp * TOKENS_PER_WARP + lane) * ROW_BYTES + if ok: + ptx.bulk_copy_g2s(dst, kv + Int64(tok_id) * QK_DIM, QK_DIM, bar) + else: + if lane < TOKENS_PER_WARP: + zero = (Uint32(0), Uint32(0), Uint32(0), Uint32(0)) + for c in cutlass.range_constexpr(QK_DIM // 16): + ptx.st_shared_v4(dst + c * 16, zero) + self.zero_fill_visible() + + @cute.jit + def nan_max(self, a: Float32, b: Float32) -> Float32: + """NaN-propagating max (max.NaN.f32): a NaN score keeps its softmax NaN, as an + fp64 reference does.""" + return ptx.max_nan(a, b) + + @cute.jit + def wait_inputs(self): + ptx.pdl_wait() + if cutlass.const_expr(self.early_trigger): + ptx.pdl_launch_dependents() + ptx.fence_proxy_async_global() + + @cute.jit + def zero_fill_visible(self): + """One lane's zero-fill stores -> every lane's ldmatrix of that row.""" + cute.arch.sync_warp() + + @cute.jit + def wait_q(self, bar: Int32): + ptx.mbarrier_wait_parity(bar, 0, ".acquire.cta") + + @cute.jit + def wait_gather(self, bar: Int32): + ptx.mbarrier_wait_parity(bar, 0, ".acquire.cta") + + @cute.jit + def wait_all_gathers(self, s_bar: Int32): + for w in cutlass.range_constexpr(QK_WARPS): + ptx.mbarrier_wait_parity(s_bar + w * 8, 0, ".acquire.cta") + + @cute.jit + def cluster_costart_arrive(self): + ptx.cluster_arrive_relaxed() + + @cute.jit + def cluster_costart_wait(self): + ptx.cluster_wait_acquire() + + @cute.jit + def wait_combine(self, bar: Int32): + ptx.mbarrier_wait_parity(bar, 0, ".acquire.cluster") + + @cute.jit + def exit_arrive(self): + ptx.cluster_arrive_relaxed() + + @cute.jit + def release_outputs(self): + ptx.fence_proxy_async_global() + + @cute.jit + def exit_wait(self): + ptx.cluster_wait_acquire() diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/__init__.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/__init__.py new file mode 100644 index 00000000..baa76290 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Draft-layer (multi-token prediction) kernels; import the modules directly.""" diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/_draft_ptx.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/_draft_ptx.py new file mode 100644 index 00000000..480d8cdf --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/_draft_ptx.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""DSL-level helpers of the draft kernels: static shared memory, bit casts and the +bf16 / NaN bit tests. Every PTX instruction the kernels emit comes from +``primitives.ptx_core`` / ``primitives.ptx_sys``. +""" + +import cutlass.cute as cute +from cutlass import Float32, Uint32 +from cutlass._mlir.dialects import llvm +from cutlass.cutlass_dsl import dsl_user_op + +# ---- shared memory + + +def static_smem(dtype, shape, stride=None): + """A statically allocated shared-memory tensor (16-byte aligned) of ``shape`` + (row-major unless ``stride``). + """ + size = 1 + for d in shape if isinstance(shape, tuple) else (shape,): + size *= d + ptr = cute.arch.alloc_smem(dtype, size, 16) + return cute.make_tensor(ptr, cute.make_layout(shape, stride=stride)) + + +# ---- bit casts (pure) + + +@dsl_user_op +def f32_from_bits(bits, *, loc=None, ip=None) -> Float32: + return Float32( + llvm.bitcast( + Float32.mlir_type, Uint32(bits).ir_value(loc=loc, ip=ip), loc=loc, ip=ip + ) + ) + + +@dsl_user_op +def bits_from_f32(value, *, loc=None, ip=None) -> Uint32: + return Uint32( + llvm.bitcast( + Uint32.mlir_type, Float32(value).ir_value(loc=loc, ip=ip), loc=loc, ip=ip + ) + ) + + +def bf16_low(word): + """FP32 value of the low BF16 half of a 32-bit word (exact).""" + return f32_from_bits(Uint32(word) << Uint32(16)) + + +def bf16_high(word): + """FP32 value of the high BF16 half of a 32-bit word (exact).""" + return f32_from_bits(Uint32(word) & Uint32(0xFFFF0000)) + + +def is_nan(value): + """True iff ``value`` is a NaN, from its bits (never folded by floating-point + assumptions). + """ + return (bits_from_f32(value) & Uint32(0x7FFFFFFF)) > Uint32(0x7F800000) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/api.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/api.py new file mode 100644 index 00000000..d90be4e4 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/api.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""Compile units of the draft kernels: for each unit, the kernel object, its compile-time constants (the compile-cache +key next to the class name) and example launch arguments. These are the compile units the draft glue and the export / +reload test of the persistent compile cache use. + +Each caller asks only for the units it ships (the eh shard GEMV rows, the top-1, and the embedding + eh norm front with +its value table). +""" + +import cuda.bindings.driver as cuda +from cutlass import Float32, Int32, Int64, Uint32, Uint64 + +HIDDEN = 6144 + + +def compile_specs( + *, + eh_rows=(), + top1_rows=(), + eh_norm=None, + stream=None, +): + """``{name: (kernel, constexprs, example_args)}`` for the requested units. + + ``eh_rows``: row counts of ``EhProjShard``; ``top1_rows``: ``max_rows`` of + ``DistributedArgmax``; ``eh_norm``: the ``EmbedEhNorm`` value table (None: no + front). The example arguments only fix the argument types (the stream value is + not part of the compiled function). + """ + s = stream if stream is not None else cuda.CUstream(0) + z = Int64(0) + out = {} + if eh_rows: + from .mtp_eh_proj_shard import EhProjShard + + for rows in eh_rows: + out[f"eh_proj_shard_{rows}"] = ( + EhProjShard(rows), + {"rows": rows}, + (z, z, z, Int32(1), Int32(1), Int32(1), Int32(1), s), + ) + if top1_rows: + from .distributed_argmax import DistributedArgmax + + for max_rows in top1_rows: + out[f"distributed_argmax_{max_rows}"] = ( + DistributedArgmax(max_rows), + {"max_rows": max_rows}, + ( + *(z,) * 6, + *(Int32(1),) * 5, + Uint64(1), + Int32(0), + Uint32(0), + z, + Int32(0), + s, + ), + ) + if eh_norm is not None: + from .mtp_embed_eh_norm import EmbedEhNorm + + out["embed_eh_norm"] = ( + EmbedEhNorm(HIDDEN, **eh_norm), + {"hidden": HIDDEN, **eh_norm}, + (*(z,) * 10, Float32(1e-6), Int32(1), s), + ) + return out diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/distributed_argmax.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/distributed_argmax.py new file mode 100644 index 00000000..cf3bfd0c --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/distributed_argmax.py @@ -0,0 +1,386 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Distributed first-maximum argmax over a vocab-parallel logits shard. + +Every rank of a tensor-parallel group holds ``logits[M, V_local]`` (bf16, the rank's +contiguous slice of the vocab). One launch per rank returns, on every rank, the global +token id and the fp32 value of the first maximum of each row of the gathered ``[M, W * +V_local]`` logits, i.e. ``torch.argmax`` over the gathered rows: + +* NaN ranks above every number and the first NaN wins (``nan_to_neg = 0``), or every NaN + is first replaced by a given finite value (``nan_to_neg = 1``, the rule of + ``torch.nan_to_num(nan=-1e30)`` on a bf16 tensor); +* ties (including -0 vs +0) go to the lowest global index; -0 is reported as +0; +* columns at or beyond ``n_valid`` (vocab padding) are treated as -inf. + +Protocol (one grid of ``max_rows`` CTAs per rank, CTA ``m`` serves row ``m``): + +* each CTA reduces its row slice to one candidate and publishes it as a 64-bit packet + ``value_bits << 32 | (epoch & 0x3FFF) << 18 | global_index`` with one relaxed + system-scope store into its slot of every peer's mailbox (unicast); idle rows (``m >= + M``) still publish, so no slot holds a packet older than two epochs and the 14-bit tag + cannot alias; +* the mailbox is ``[2][W][max_rows]`` u64 in symmetric memory, double-buffered by epoch + parity; one thread per CTA executes ``fence.sc.sys`` after the grid-dependency wait + and before the CTA's packet stores, which orders a writer's reuse of a parity slot + after the reader's previous read of it; +* each active CTA polls its W slots with relaxed system-scope loads until every slot + carries the current epoch's tag, bounded by a timeout: on expiry the sticky error word + is set, the host-mapped error record (if given) names this rank, the late peer, the + row count and the epoch, and the launch traps (fail-stop); +* the per-CTA epoch lives in a control word on the local GPU, read after the wait and + advanced at exit; there is no early ``launch_dependents``, so dependents start after + this grid completes; +* the grid is launched cooperatively, so all CTAs of a rank are co-resident; no CTA + waits for another CTA of its own grid. +""" + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass import Boolean, Float32, Int32, Int64, Uint32, Uint64 + +from ..primitives import ptx_core as core +from ..primitives import ptx_sys as sys_ops +from . import _draft_ptx as ptx + +_NEG_INF_BITS = 0xFF800000 +_CANONICAL_NAN_BITS = 0x7FC00000 +_INT32_MAX = 0x7FFFFFFF + + +class DistributedArgmax: + """One instance per row bucket (``max_rows`` in {4, 8, 16}); each instance has its + own mailbox and epochs. + """ + + THREADS = 1024 + LOADER_THREADS = 992 # warps 0..30 load and reduce; warp 31 issues the fence + FENCE_THREAD = 992 + VECTORS_PER_THREAD = 3 # 16-byte vectors per loader thread: V_local <= 3 * 8 * 992 + MAX_RANKS = 8 + # [0] sticky error word, [1] timeouts, [2..4] reserved, [16 + m] per-CTA epochs + CTRL_WORDS = 64 + EPOCH_WORD = 16 + TAG_MASK = 0x3FFF + INDEX_MASK = 0x3FFFF + TIMEOUT_NS = 10_000_000_000 + ERR_TIMEOUT = 1 + ERR_BAD_INDEX = 4 + # host-mapped error record (the hop transport's layout): {magic, kind, site, + # rank | peer << 8 | rows << 16, epoch} + ERR_RECORD_MAGIC = 0x48455252 + ERR_RECORD_KIND = 4 # top-1 poll + + def __init__(self, max_rows: int): + if max_rows not in (4, 8, 16): + raise ValueError(f"max_rows must be 4, 8 or 16, got {max_rows}") + self.max_rows = max_rows + + # ---- host contract + + @classmethod + def max_local_vocab(cls) -> int: + return cls.VECTORS_PER_THREAD * 8 * cls.LOADER_THREADS + + def mailbox_words(self, world: int) -> int: + return 2 * world * self.max_rows + + def check_shapes( + self, rows: int, local_vocab: int, n_valid: int, world: int, rank: int + ) -> None: + if not 1 <= world <= self.MAX_RANKS or not 0 <= rank < world: + raise ValueError(f"world {world} / rank {rank} out of range") + if not 1 <= rows <= self.max_rows: + raise ValueError(f"rows {rows} not in [1, {self.max_rows}]") + if local_vocab < 8 or local_vocab % 8 or local_vocab > self.max_local_vocab(): + raise ValueError( + f"local vocab {local_vocab}: a multiple of 8 in " + f"[8, {self.max_local_vocab()}]" + ) + if world * local_vocab >= 1 << 18: + raise ValueError("world * local vocab must be < 2^18 (18-bit global index)") + if not 1 <= n_valid <= local_vocab: + raise ValueError(f"n_valid {n_valid} not in [1, {local_vocab}]") + + # ---- launch + + @cute.jit + def __call__( + self, + logits: Int64, + mailbox: Int64, + peers: Int64, + ctrl: Int64, + out_ids: Int64, + out_vals: Int64, + rows: Int32, + local_vocab: Int32, + n_valid: Int32, + world: Int32, + rank: Int32, + timeout_ns: Uint64, + nan_to_neg: Int32, + nan_replacement_bits: Uint32, + err_host: Int64, + site: Int32, + stream: cuda.CUstream, + ): + """``err_host``: device address of a host-mapped error record (0: none); + ``site``: the caller's id for this call site, written into the record.""" + self.kernel( + logits, + mailbox, + peers, + ctrl, + out_ids, + out_vals, + rows, + local_vocab, + n_valid, + world, + rank, + timeout_ns, + nan_to_neg, + nan_replacement_bits, + err_host, + site, + ).launch( + grid=[self.max_rows, 1, 1], + block=[self.THREADS, 1, 1], + stream=stream, + use_pdl=True, + cooperative=True, + ) + + # ---- decision points + + def better(self, v, i, best, best_i): + """True iff (v, i) precedes (best, best_i) in argmax order: NaN first, then + larger, then lower index. + """ + v_nan = ptx.is_nan(v) + v_num = ptx.bits_from_f32(v) & Uint32(0x7FFFFFFF) <= Uint32(0x7F800000) + b_num = ptx.bits_from_f32(best) & Uint32(0x7FFFFFFF) <= Uint32(0x7F800000) + lower = i < best_i + numeric = (v > best) | ((v == best) & lower) + return (v_nan & (b_num | lower)) | (v_num & b_num & numeric) + + @cute.jit + def order_publish(self): + """Orders this rank's earlier reads of the parity slots before this CTA's packet + stores. + """ + sys_ops.fence_sc_sys() + + def accept(self, tag, epoch): + """True iff a polled packet's tag belongs to this call.""" + return tag == (epoch & Uint32(self.TAG_MASK)) + + @cute.jit + def on_timeout(self, ctrl: Int64, err_host: Int64, site, rank, peer, rows, epoch): + """Sticky error word and count, the host-mapped record, then ``trap``.""" + sys_ops.atom_or_b32(ctrl, Uint32(self.ERR_TIMEOUT)) + sys_ops.atom_add_u32(ctrl + Int64(4), Uint32(1)) + if err_host != Int64(0): + meta = ( + Uint32(rank) + | (Uint32(peer) << Uint32(8)) + | (Uint32(rows) << Uint32(16)) + ) + sys_ops.st_relaxed_sys_v4_u32( + err_host, + Uint32(self.ERR_RECORD_MAGIC), + Uint32(self.ERR_RECORD_KIND), + Uint32(site), + meta, + ) + sys_ops.st_relaxed_sys_u32(err_host + Int64(16), epoch) + sys_ops.fence_sc_sys() + sys_ops.trap() + + # ---- helpers + + def canonical_bits(self, v): + bits = ptx.bits_from_f32(v) + bits = Uint32( + cutlass.select_((bits & Uint32(0x7FFFFFFF)) == Uint32(0), Uint32(0), bits) + ) + return Uint32(cutlass.select_(ptx.is_nan(v), Uint32(_CANONICAL_NAN_BITS), bits)) + + # ---- kernel + + @cute.kernel + def kernel( + self, + logits: Int64, + mailbox: Int64, + peers: Int64, + ctrl: Int64, + out_ids: Int64, + out_vals: Int64, + rows: Int32, + local_vocab: Int32, + n_valid: Int32, + world: Int32, + rank: Int32, + timeout_ns: Uint64, + nan_to_neg: Int32, + nan_replacement_bits: Uint32, + err_host: Int64, + site: Int32, + ): + t, _, _ = cute.arch.thread_idx() + m, _, _ = cute.arch.block_idx() + tid = Int32(t) + row = Int32(m) + + core.pdl_wait() + if tid == self.FENCE_THREAD: + self.order_publish() + epoch_addr = ctrl + Int64(self.EPOCH_WORD + row) * Int64(4) + epoch = sys_ops.ld_volatile_u32(epoch_addr) + poisoned = sys_ops.ld_volatile_u32(ctrl) != Uint32(0) + + warp_vals = ptx.static_smem(Float32, 32) + warp_ids = ptx.static_smem(Int32, 32) + rank_vals = ptx.static_smem(Float32, self.MAX_RANKS) + rank_ids = ptx.static_smem(Int32, self.MAX_RANKS) + rank_bad = ptx.static_smem(Int32, self.MAX_RANKS) + + active = row < rows + best = ptx.f32_from_bits(Uint32(_NEG_INF_BITS)) + best_i = Int32(_INT32_MAX) + replacement = ptx.f32_from_bits(nan_replacement_bits) + neg_inf = ptx.f32_from_bits(Uint32(_NEG_INF_BITS)) + + # ---- local first maximum over this thread's 16-byte vectors + if active & (tid < self.LOADER_THREADS): + row_addr = logits + Int64(row) * Int64(local_vocab) * Int64(2) + n_vec = local_vocab // 8 + words = [] + for k in cutlass.range_constexpr(self.VECTORS_PER_THREAD): + v = tid + k * self.LOADER_THREADS + words.append( + core.ld_global_v4_if(row_addr + Int64(v) * Int64(16), v < n_vec) + ) + for k in cutlass.range_constexpr(self.VECTORS_PER_THREAD): + v = tid + k * self.LOADER_THREADS + in_row = v < n_vec + for e in cutlass.range_constexpr(8): + w = words[k][e // 2] + f = ptx.bf16_low(w) if e % 2 == 0 else ptx.bf16_high(w) + i = v * 8 + e + f = Float32( + cutlass.select_( + (nan_to_neg != Int32(0)) & ptx.is_nan(f), replacement, f + ) + ) + f = Float32(cutlass.select_(i >= n_valid, neg_inf, f)) + take = in_row & self.better(f, i, best, best_i) + best = Float32(cutlass.select_(take, f, best)) + best_i = Int32(cutlass.select_(take, i, best_i)) + + # ---- warp reduction (fixed butterfly order) + for s in cutlass.range_constexpr(5): + off = 16 >> s + other = cute.arch.shuffle_sync_bfly(best, off) + other_i = cute.arch.shuffle_sync_bfly(best_i, off) + take = self.better(other, other_i, best, best_i) + best = Float32(cutlass.select_(take, other, best)) + best_i = Int32(cutlass.select_(take, other_i, best_i)) + if tid % 32 == 0: + warp_vals[tid // 32] = best + warp_ids[tid // 32] = best_i + peer_base = Uint64(0) + if tid < world: + peer_base = sys_ops.ld_u64(peers + Int64(tid) * Int64(8)) + cute.arch.sync_threads() + + buf_off = Int64(epoch & Uint32(1)) * Int64(world) * Int64(self.max_rows) + tag = Uint64(epoch & Uint32(self.TAG_MASK)) << Uint64(18) + + # ---- publish: lane 0 of warp 0 builds the packet, lanes 0..W-1 store it to + # their peer + if tid < 32: + packet = Uint64(0) + if tid == 0: + if active: + for w in cutlass.range_constexpr(1, 32): + wv = warp_vals[w] + wi = warp_ids[w] + take = self.better(wv, wi, best, best_i) + best = Float32(cutlass.select_(take, wv, best)) + best_i = Int32(cutlass.select_(take, wi, best_i)) + gidx = Uint32(rank) * Uint32(local_vocab) + Uint32(best_i) + if (best_i < Int32(0)) | (best_i >= local_vocab): + sys_ops.atom_or_b32(ctrl, Uint32(self.ERR_BAD_INDEX)) + gidx = Uint32(0) + packet = ( + (Uint64(self.canonical_bits(best)) << Uint64(32)) + | tag + | Uint64(gidx & Uint32(self.INDEX_MASK)) + ) + else: + packet = (Uint64(_NEG_INF_BITS) << Uint64(32)) | tag + lo = cute.arch.shuffle_sync(Uint32(packet & Uint64(0xFFFFFFFF)), 0) + hi = cute.arch.shuffle_sync(Uint32(packet >> Uint64(32)), 0) + packet = (Uint64(hi) << Uint64(32)) | Uint64(lo) + if tid < world: + slot = buf_off + Int64(rank) * Int64(self.max_rows) + Int64(row) + sys_ops.st_relaxed_sys_u64(Int64(peer_base) + slot * Int64(8), packet) + + # ---- gather: lane s polls the slot rank s wrote into this rank's mailbox + if active & (tid < world): + slot_addr = mailbox + ( + buf_off + Int64(tid) * Int64(self.max_rows) + Int64(row) + ) * Int64(8) + got = Uint64(0) + timed_out = Int32(0) + if poisoned: + timed_out = Int32(1) + else: + got = sys_ops.ld_relaxed_sys_u64(slot_addr) + if not self.accept( + Uint32((got >> Uint64(18)) & Uint64(self.TAG_MASK)), epoch + ): + start = sys_ops.globaltimer() + done = Boolean(False) + while not done: + got = sys_ops.ld_relaxed_sys_u64(slot_addr) + if self.accept( + Uint32((got >> Uint64(18)) & Uint64(self.TAG_MASK)), epoch + ): + done = Boolean(True) + else: + if sys_ops.globaltimer() - start > timeout_ns: + self.on_timeout( + ctrl, err_host, site, rank, tid, rows, epoch + ) + timed_out = Int32(1) + done = Boolean(True) + rank_vals[tid] = ptx.f32_from_bits(Uint32(got >> Uint64(32))) + rank_ids[tid] = Int32(got & Uint64(self.INDEX_MASK)) + rank_bad[tid] = timed_out + cute.arch.sync_threads() + + # ---- combine the W candidates in rank order + if active & (tid == 0): + val = rank_vals[0] + idx = rank_ids[0] + bad = rank_bad[0] + for s in cutlass.range(1, world, unroll=1): + bad = bad | rank_bad[s] + sv = rank_vals[s] + si = rank_ids[s] + take = self.better(sv, si, val, idx) + val = Float32(cutlass.select_(take, sv, val)) + idx = Int32(cutlass.select_(take, si, idx)) + if bad != Int32(0): + idx = Int32(0) + val = ptx.f32_from_bits(Uint32(_CANONICAL_NAN_BITS)) + core.st_global_u64(out_ids + Int64(row) * Int64(8), Uint64(Int64(idx))) + core.st_global_f32(out_vals + Int64(row) * Int64(4), val) + if tid == 0: + core.st_global_u32(epoch_addr, epoch + Uint32(1)) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/mtp_eh_proj_shard.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/mtp_eh_proj_shard.py new file mode 100644 index 00000000..7a3e9da1 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/mtp_eh_proj_shard.py @@ -0,0 +1,176 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Column-sharded GEMV for the draft layer's input projection (``eh_proj``). + +``out[m, col_off + n] = bf16(sum_k x[m, k] * w[n, k])`` for this rank's ``n_local`` rows +of the projection weight, written straight into the rank's column slice of a ``[M, +out_ld]`` bf16 buffer (no copy kernel), so a stock all-gather or all-reduce can combine +the slices. + +* One CTA computes ``cols`` output columns over the full K; warp ``w`` owns column ``w + // ksplit`` and K-part ``w % ksplit``; every lane streams ``vpl`` 16-byte weight + vectors and keeps them in registers. +* Per row: an fp32 FMA chain in k order inside each lane, a fixed butterfly across the + warp, then the K-parts summed in part order from +0, and one round to bf16. The + summation order is fixed by the configuration, so the result is deterministic and each + row is independent of the other rows. +* The weight is static (never written by a preceding grid), so its stream is issued + before the grid-dependency wait (coherent loads with no L1 allocation); the + activations are read only after it. Dependents may launch at entry: they wait for this + grid's completion before reading the output. +""" + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass import Float32, Int32, Int64, Uint32 + +from ..primitives import ptx_core as core +from . import _draft_ptx as ptx + +# (cols, ksplit, vectors per lane) for K = 12288: rows 1-2 use 4 x 2 x 24, rows >= 3 use +# 1 x 8 x 6. +_CONFIGS = {"wide": (4, 2, 24), "deep": (1, 8, 6)} + + +def plan_config(rows: int) -> str: + return "wide" if rows <= 2 else "deep" + + +class EhProjShard: + """One compiled instance per row count ``rows`` (1..16).""" + + def __init__(self, rows: int, k: int = 12288, config: str | None = None): + if not 1 <= rows <= 16: + raise ValueError(f"rows must be in [1, 16], got {rows}") + self.rows = rows + self.k = k + self.config = plan_config(rows) if config is None else config + self.cols, self.ksplit, self.vpl = _CONFIGS[self.config] + if k != self.ksplit * 32 * self.vpl * 8: + raise ValueError( + f"k must be {self.ksplit * 32 * self.vpl * 8} for config {self.config}" + ) + self.threads = self.cols * self.ksplit * 32 + + def grid(self, n_local: int) -> int: + # every CTA owns `cols` existing rows, so no load needs a guard (a guarded + # weight stream lets the compiler sink it below the grid-dependency wait) + if n_local % self.cols: + raise ValueError(f"n_local {n_local} must be a multiple of {self.cols}") + return n_local // self.cols + + @cute.jit + def __call__( + self, + weight: Int64, + x: Int64, + out: Int64, + n_local: Int32, + out_ld: Int32, + col_off: Int32, + grid: Int32, + stream: cuda.CUstream, + ): + self.kernel(weight, x, out, n_local, out_ld, col_off).launch( + grid=[grid, 1, 1], block=[self.threads, 1, 1], stream=stream, use_pdl=True + ) + + def weight_stream(self, w_row, lane): + """This lane's ``vpl`` 16-byte weight vectors (a trace-time loop: fully + unrolled). Coherent loads with no L1 + allocation: the compiler keeps them ahead of the grid-dependency wait (a + non-coherent ``.nc`` stream may be sunk below it, which serializes the weight + stream behind the preceding grid). + """ + return [ + core.ld_global_v4_no_allocate( + w_row + (Int64(i * 32) + Int64(lane)) * Int64(16) + ) + for i in range(self.vpl) + ] + + @cute.jit + def wait_for_inputs(self): + core.pdl_wait() + + @cute.kernel + def kernel( + self, + weight: Int64, + x: Int64, + out: Int64, + n_local: Int32, + out_ld: Int32, + col_off: Int32, + ): + t, _, _ = cute.arch.thread_idx() + b, _, _ = cute.arch.block_idx() + tid = Int32(t) + core.pdl_launch_dependents() + warp = tid // 32 + lane = tid % 32 + c = warp // self.ksplit + kp = warp % self.ksplit + n = Int32(b) * self.cols + c + kbase = kp * (self.k // self.ksplit) + + # static weight stream: issued before the wait, held in registers + w_row = weight + (Int64(n) * Int64(self.k) + Int64(kbase)) * Int64(2) + wv = self.weight_stream(w_row, lane) + + self.wait_for_inputs() + + acc = [Float32(0.0)] * self.rows + for i in cutlass.range_constexpr(self.vpl): + wf = [] + for j in cutlass.range_constexpr(4): + wf.append(ptx.bf16_low(wv[i][j])) + wf.append(ptx.bf16_high(wv[i][j])) + for m in cutlass.range_constexpr(self.rows): + xa = x + ( + Int64(m * self.k) + + Int64(kbase) + + (Int64(i * 32) + Int64(lane)) * Int64(8) + ) * Int64(2) + xv = core.ld_global_v4(xa) + a = acc[m] + for j in cutlass.range_constexpr(4): + a = core.fma_rn(wf[2 * j], ptx.bf16_low(xv[j]), a) + a = core.fma_rn(wf[2 * j + 1], ptx.bf16_high(xv[j]), a) + acc[m] = a + + for m in cutlass.range_constexpr(self.rows): + a = acc[m] + for s in cutlass.range_constexpr(5): + a = core.add_rn(a, cute.arch.shuffle_sync_bfly(a, 16 >> s)) + acc[m] = a + + part = ptx.static_smem( + Float32, + (self.cols, self.ksplit, self.rows), + stride=(self.ksplit * self.rows, self.rows, 1), + ) + if lane == 0: + # one 4-byte store per row partial (the stores are not merged into vectors) + pbase = Uint32(part.iterator.toint()) + Uint32( + (c * self.ksplit + kp) * self.rows * 4 + ) + for m in cutlass.range_constexpr(self.rows): + core.st_shared_f32(pbase + Uint32(m * 4), acc[m]) + cute.arch.sync_threads() + + if tid < self.cols * self.rows: + cc = tid // self.rows + mm = tid % self.rows + nn = Int32(b) * self.cols + cc + # always true (n_local % cols == 0); a defensive bound + if nn < n_local: + s = Float32(0.0) + for k in cutlass.range_constexpr(self.ksplit): + s = core.add_rn(s, part[cc, k, mm]) + dst = out + ( + Int64(mm) * Int64(out_ld) + Int64(col_off) + Int64(nn) + ) * Int64(2) + core.st_global_b16(dst, core.cvt_rn_bf16_f32(s)) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/mtp_embed_eh_norm.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/mtp_embed_eh_norm.py new file mode 100644 index 00000000..b8364de4 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/draft/mtp_embed_eh_norm.py @@ -0,0 +1,184 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Draft-layer input front: embedding gather + two RMSNorms + concat, one CTA per token. + +``out[i, :H] = rmsnorm(table[ids[i]]) * enorm_w`` and ``out[i, H:] = rmsnorm(prev[i]) * +hnorm_w`` in bf16, for a replicated ``[vocab, H]`` embedding table. The arithmetic of +each norm is described by option fields (no named configurations live here; each caller +passes the values that reproduce its own reference): + +* ``sum_rule``: how the squares are summed. ``"lane_fma_cta_tree"``: each thread holds 8 + consecutive elements and accumulates ``fma(x, x, s)`` from +0 in element order, then a + warp butterfly (xor 16, 8, 4, 2, 1; ``s + other``), then warp 0 reduces the per-warp + sums (lanes past the warp count contribute +0) with the same butterfly. +* ``mean_rule``: ``"div_rn"``: ``mean = div.rn(sum, H)``, then ``mean + eps``. +* ``rsqrt_ftz``: whether the reciprocal square root flushes subnormals + (``rsqrt.approx[.ftz].f32``). +* ``apply_rule``: ``"x_r_w"``: ``bf16_rn((x * r) * w)`` with two round-to-nearest + products. + +An id outside ``[0, vocab)`` traps. The ids, the rows and the previous hidden states are +read after the grid-dependency wait; dependents are released after the last store. +""" + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass import Float32, Int32, Int64 + +from ..primitives import ptx_core as core +from ..primitives import ptx_sys as sys_ops +from . import _draft_ptx as ptx + +SUM_RULES = ("lane_fma_cta_tree",) +MEAN_RULES = ("div_rn",) +APPLY_RULES = ("x_r_w",) + + +class EmbedEhNorm: + """One compiled instance per hidden size and option set; any number of tokens (one + CTA each). + """ + + def __init__( + self, + hidden: int = 6144, + *, + sum_rule: str, + mean_rule: str, + rsqrt_ftz: bool, + apply_rule: str, + ): + if ( + sum_rule not in SUM_RULES + or mean_rule not in MEAN_RULES + or apply_rule not in APPLY_RULES + ): + raise ValueError( + "unsupported norm options " + f"{sum_rule!r} / {mean_rule!r} / {apply_rule!r}" + ) + if hidden % 256 or not 256 < hidden <= 8192: + raise ValueError(f"hidden size {hidden}: a multiple of 256 in (256, 8192]") + self.hidden = hidden + self.threads = hidden // 8 + self.warps = self.threads // 32 + self.rsqrt_ftz = rsqrt_ftz + + @cute.jit + def __call__( + self, + ids: Int64, + table: Int64, + table_stride: Int64, + vocab: Int64, + prev: Int64, + prev_stride: Int64, + enorm_w: Int64, + hnorm_w: Int64, + out: Int64, + out_stride: Int64, + eps: Float32, + tokens: Int32, + stream: cuda.CUstream, + ): + self.kernel( + ids, + table, + table_stride, + vocab, + prev, + prev_stride, + enorm_w, + hnorm_w, + out, + out_stride, + eps, + ).launch( + grid=[tokens, 1, 1], block=[self.threads, 1, 1], stream=stream, use_pdl=True + ) + + @cute.jit + def wait_for_inputs(self): + core.pdl_wait() + + @cute.jit + def inv_rms(self, words, smem, tid: Int32, eps: Float32): + # per-lane chain over the 8 elements, in element order, from +0 + s = Float32(0.0) + for j in cutlass.range_constexpr(4): + lo = ptx.bf16_low(words[j]) + hi = ptx.bf16_high(words[j]) + s = core.fma_rn(lo, lo, s) + s = core.fma_rn(hi, hi, s) + for k in cutlass.range_constexpr(5): + s = core.add_rn(s, cute.arch.shuffle_sync_bfly(s, 16 >> k)) + warp = tid // 32 + lane = tid % 32 + # every lane writes its warp's (identical) sum, as the stock CTA norm does + smem[warp] = s + cute.arch.sync_threads() + if warp == 0: + t = Float32(0.0) + if lane < self.warps: + t = smem[lane] + for k in cutlass.range_constexpr(5): + t = core.add_rn(t, cute.arch.shuffle_sync_bfly(t, 16 >> k)) + mean = core.div_rn(t, Float32(float(self.hidden))) + smem[32] = core.rsqrt_approx(core.add_rn(mean, eps), self.rsqrt_ftz) + cute.arch.sync_threads() + return smem[32] + + @cute.jit + def apply(self, words, wwords, r): + res = [] + for j in cutlass.range_constexpr(4): + a = core.mul_rn( + core.mul_rn(ptx.bf16_low(words[j]), r), ptx.bf16_low(wwords[j]) + ) + b = core.mul_rn( + core.mul_rn(ptx.bf16_high(words[j]), r), ptx.bf16_high(wwords[j]) + ) + res.append(core.cvt_rn_bf16x2_f32(a, b)) + return res + + @cute.kernel + def kernel( + self, + ids: Int64, + table: Int64, + table_stride: Int64, + vocab: Int64, + prev: Int64, + prev_stride: Int64, + enorm_w: Int64, + hnorm_w: Int64, + out: Int64, + out_stride: Int64, + eps: Float32, + ): + t, _, _ = cute.arch.thread_idx() + b, _, _ = cute.arch.block_idx() + tid = Int32(t) + tok = Int64(b) + smem = ptx.static_smem(Float32, 33) + col = Int64(tid) * Int64(16) + + self.wait_for_inputs() + tid_ = sys_ops.ld_u64_generic(ids + tok * Int64(8)) + idv = Int64(tid_) + sys_ops.trap_if((idv < Int64(0)) | (idv >= vocab)) + e = core.ld_global_v4(table + idv * table_stride * Int64(2) + col) + ew = core.ld_global_v4(enorm_w + col) + r = self.inv_rms(e, smem, tid, eps) + o = self.apply(e, ew, r) + orow = out + tok * out_stride * Int64(2) + core.st_global_v4(orow + col, o) + + h = core.ld_global_v4(prev + tok * prev_stride * Int64(2) + col) + hw = core.ld_global_v4(hnorm_w + col) + r2 = self.inv_rms(h, smem, tid, eps) + o2 = self.apply(h, hw, r2) + core.st_global_v4(orow + Int64(self.hidden * 2) + col, o2) + core.pdl_launch_dependents() diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/__init__.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/__init__.py new file mode 100644 index 00000000..a858d109 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Fused all-reduce hops of the decode / verify step: o_proj + all-reduce + residual + +RMSNorm (optionally with the router), the MLP-output all-reduce + residual + RMSNorm, +and the MoE finalize fold. Engines use ``hop.api``.""" diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/_ptx_local.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/_ptx_local.py new file mode 100644 index 00000000..32108c54 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/_ptx_local.py @@ -0,0 +1,260 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Bit views, shared-memory, cluster and routing helpers used only by the hop kernels. + +The rounded arithmetic (``add.rn`` / ``mul.rn`` / ``fma.rn`` / ``rcp.rn`` / +``rsqrt.approx`` / ``cvt.rn.bf16`` / ``cvt.rn.bf16x2``) comes from +``primitives/ptx_core.py``. The helpers kept here either have no equivalent there or +differ from it in a way that changes the generated code (bit views by ``llvm.bitcast`` +instead of ``mov.b32``, shared-memory loads without a ``memory`` clobber, the cluster +mbarrier forms), so moving them is a separate, code-changing step. + +Arithmetic helpers are pure (no side effects, no clobber) so they do not serialize +reduction loops. Shared-memory synchronization helpers are side-effecting with a +``~{memory}`` clobber. Barriers and warp collectives use the DSL's NVVM operations +(convergent), never raw asm. +""" + +import cutlass +import cutlass.cute as cute +from cutlass import Float32, Int32, Uint32 +from cutlass._mlir.dialects import llvm, nvvm +from cutlass.cutlass_dsl import dsl_user_op + +_MEM = "~{memory}" + + +def _emit(code, constraints, operands, results, side_effects, loc, ip): + if not results: + ir_type = None + elif len(results) == 1: + ir_type = results[0].mlir_type + else: + ir_type = llvm.StructType.get_literal([r.mlir_type for r in results]) + value = llvm.inline_asm( + ir_type, + [v.ir_value(loc=loc, ip=ip) for v in operands], + code, + constraints, + has_side_effects=side_effects, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + if not results: + return None + if len(results) == 1: + return results[0](value) + return tuple( + r(llvm.extractvalue(r.mlir_type, value, [i], loc=loc, ip=ip)) + for i, r in enumerate(results) + ) + + +# ---------------------------------------------------------------- bit views +def bits_f32(word) -> Float32: + return Float32(llvm.bitcast(Float32.mlir_type, Uint32(word).ir_value())) + + +def f32_bits(value) -> Uint32: + return Uint32(llvm.bitcast(Uint32.mlir_type, Float32(value).ir_value())) + + +def bf16_lo(word) -> Float32: + """fp32 value of the low bf16 half of a 32-bit word.""" + return bits_f32(Uint32(word) << 16) + + +def bf16_hi(word) -> Float32: + return bits_f32(Uint32(word) & Uint32(0xFFFF0000)) + + +@cute.jit +def no_neg_zero(word: Uint32) -> Uint32: + """``word`` with each negative-zero bf16 half made +0.""" + w = Uint32(word) + if (w & Uint32(0xFFFF)) == Uint32(0x8000): + w = w & Uint32(0xFFFF0000) + if (w >> 16) == Uint32(0x8000): + w = w & Uint32(0x0000FFFF) + return w + + +# ------------------------------------------------------------ mma (sm80+) +@dsl_user_op +def mma_bf16_m16n8k16(c, a, b, *, loc=None, ip=None): + """``mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32``; returns 4 fp32.""" + code = ( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {$0,$1,$2,$3}, " + "{$4,$5,$6,$7}, {$8,$9}, {$10,$11,$12,$13};" + ) + ops = tuple(Uint32(v) for v in (*a, *b)) + tuple(Float32(v) for v in c) + cons = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f" + return _emit(code, cons, ops, (Float32,) * 4, False, loc, ip) + + +# ------------------------------------------------------- mbarrier / DSMEM +@dsl_user_op +def mbar_init(addr, count, *, loc=None, ip=None): + code = "mbarrier.init.shared::cta.b64 [$0], $1;" + _emit(code, "r,r," + _MEM, (Uint32(addr), Uint32(count)), (), True, loc, ip) + + +@dsl_user_op +def fence_mbarrier_init_release_cluster(*, loc=None, ip=None): + _emit("fence.mbarrier_init.release.cluster;", _MEM, (), (), True, loc, ip) + + +@dsl_user_op +def mbar_arrive_expect_tx_release_cluster(addr, tx, *, loc=None, ip=None): + """Cluster-scope release arrive (the peers' ``st.async`` complete_tx on this barrier + are cluster-scope operations).""" + code = "mbarrier.arrive.expect_tx.release.cluster.shared::cta.b64 _, [$0], $1;" + _emit(code, "r,r," + _MEM, (Uint32(addr), Uint32(tx)), (), True, loc, ip) + + +@dsl_user_op +def mbar_try_wait_parity_acquire_cluster(addr, parity, *, loc=None, ip=None) -> Uint32: + code = ( + "{ .reg .pred p; mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 " + "p, [$1], $2; selp.u32 $0, 1, 0, p; }" + ) + ops = (Uint32(addr), Uint32(parity)) + return _emit(code, "=r,r,r," + _MEM, ops, (Uint32,), True, loc, ip) + + +@cute.jit +def mbar_wait_cluster(addr: Uint32, parity: Uint32): + ok = mbar_try_wait_parity_acquire_cluster(addr, parity) + while ok == Uint32(0): + ok = mbar_try_wait_parity_acquire_cluster(addr, parity) + + +@dsl_user_op +def mapa_u32(addr, rank, *, loc=None, ip=None) -> Uint32: + code = "mapa.shared::cluster.u32 $0, $1, $2;" + ops = (Uint32(addr), Uint32(rank)) + return _emit(code, "=r,r,r", ops, (Uint32,), True, loc, ip) + + +@dsl_user_op +def st_async_b32(raddr, value, rmbar, *, loc=None, ip=None): + code = "st.async.shared::cluster.mbarrier::complete_tx::bytes.b32 [$0], $1, [$2];" + ops = (Uint32(raddr), Uint32(value), Uint32(rmbar)) + _emit(code, "r,r,r," + _MEM, ops, (), True, loc, ip) + + +# ------------------------------------------------------------------ barriers +def bar_sync(barrier_id: int, count: int): + """Named CTA barrier (``bar.sync id, count``) through the convergent NVVM op.""" + cute.arch.barrier(barrier_id=barrier_id, number_of_threads=count) + + +@dsl_user_op +def cluster_arrive_relaxed(*, loc=None, ip=None): + """``barrier.cluster.arrive.relaxed.aligned`` (convergent NVVM op).""" + nvvm.cluster_arrive_relaxed(aligned=True, loc=loc, ip=ip) + + +@dsl_user_op +def cluster_wait(*, loc=None, ip=None): + """``barrier.cluster.wait.aligned`` (convergent NVVM op).""" + nvvm.cluster_wait(aligned=True, loc=loc, ip=ip) + + +def shfl_xor(value, offset: int): + return cute.arch.shuffle_sync_bfly(value, offset=offset) + + +def shfl_up(value, offset: int): + return cute.arch.shuffle_sync_up(value, offset=offset) + + +def thread_index() -> Int32: + tid, _, _ = cute.arch.thread_idx() + return Int32(tid) + + +def block_index() -> Int32: + bid, _, _ = cute.arch.block_idx() + return Int32(bid) + + +def grid_size() -> Int32: + g, _, _ = cute.arch.grid_dim() + return Int32(g) + + +def const(x): + return cutlass.const_expr(x) + + +# ------------------------------------------------------------- shared memory +@dsl_user_op +def st_shared_v4_u32(addr, w0, w1, w2, w3, *, loc=None, ip=None): + """16-byte shared store (side-effecting: keeps its place relative to barriers).""" + code = "st.shared.v4.u32 [$0], {$1,$2,$3,$4};" + ops = (Uint32(addr), Uint32(w0), Uint32(w1), Uint32(w2), Uint32(w3)) + _emit(code, "r,r,r,r,r", ops, (), True, loc, ip) + + +@dsl_user_op +def ld_shared_f32(addr, *, loc=None, ip=None) -> Float32: + """One 32-bit shared-memory load (not merged with its neighbours into a vector).""" + code = "ld.shared.f32 $0, [$1];" + return _emit(code, "=f,r", (Uint32(addr),), (Float32,), True, loc, ip) + + +@dsl_user_op +def ld_shared_v4_u32(addr, *, loc=None, ip=None): + code = "ld.shared.v4.u32 {$0,$1,$2,$3}, [$4];" + return _emit(code, "=r,=r,=r,=r,r", (Uint32(addr),), (Uint32,) * 4, True, loc, ip) + + +# ------------------------------------------------- routing scalars (stock order) +# The stock top-k routing (sigmoid scores + bias, top-8, scaled normalized weights) is +# compiled with fast math; these reproduce its exact instruction forms. +@dsl_user_op +def routing_sigmoid(x, *, loc=None, ip=None) -> Float32: + """``fma.rn.ftz(tanh.approx(mul.ftz(x, 0.5)), 0.5, 0.5)``.""" + code = ( + "{ .reg .f32 h, t; mul.ftz.f32 h, $1, 0f3F000000; tanh.approx.f32 t, h; " + "fma.rn.ftz.f32 $0, t, 0f3F000000, 0f3F000000; }" + ) + return _emit(code, "=f,f", (Float32(x),), (Float32,), False, loc, ip) + + +@dsl_user_op +def add_ftz(a, b, *, loc=None, ip=None) -> Float32: + ops = (Float32(a), Float32(b)) + return _emit("add.ftz.f32 $0, $1, $2;", "=f,f,f", ops, (Float32,), False, loc, ip) + + +@dsl_user_op +def routing_weight(scale, s, total, *, loc=None, ip=None) -> Float32: + """``bf16(div.approx.ftz(mul.ftz(scale, s), add.ftz(total, 1e-20)))`` as fp32.""" + code = ( + "{ .reg .f32 n, d, q; .reg .b16 h; mul.ftz.f32 n, $1, $2; " + "add.ftz.f32 d, $3, 0f1E3CE508; " + "div.approx.ftz.f32 q, n, d; cvt.rn.bf16.f32 h, q; mov.b32 $0, {0, h}; }" + ) + ops = (Float32(scale), Float32(s), Float32(total)) + return _emit(code, "=f,f,f,f", ops, (Float32,), False, loc, ip) + + +def umax(a, b): + return cutlass.max(Uint32(a), Uint32(b)) + + +def umin(a, b): + return cutlass.min(Uint32(a), Uint32(b)) + + +def redux_max_u32(v): + return cute.arch.warp_redux_sync(Uint32(v), "umax") + + +def redux_min_u32(v): + return cute.arch.warp_redux_sync(Uint32(v), "umin") diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/allreduce_norm.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/allreduce_norm.py new file mode 100644 index 00000000..ab1a2cc2 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/allreduce_norm.py @@ -0,0 +1,186 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""All-reduce of a bf16 ``[rows, HIDDEN]`` partial over the tensor-parallel group, fused +with the residual add and RMSNorm (``residual`` / ``norm`` variants). + +Grid: ``rows`` clusters of 8 consumer CTAs (one per row) plus one acknowledgement +cluster; 544 threads (512 polling + one epoch warp); cooperative (every CTA spins on +records written by CTAs of the same grid on other ranks, so all CTAs must be +co-resident); programmatic dependent launch: ``griddepcontrol.wait`` then +``launch_dependents`` at entry. Each CTA publishes the 256 records of its slice of the +row to every rank and consumes the same slice. +""" + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import cutlass.utils +from cutlass import Float32, Int32, Int64, Uint32 + +from ..primitives import ptx_sys as sys_ops +from . import _ptx_local as P +from .constants import BF16_NAN, CONSUMER_THREADS, HIDDEN, POLL_THREADS, TAG_MASK +from .transport import ConsSmem, Consumer, Mbox + + +class AllreduceNormHop(Consumer): + """``out_res = residual + allreduce(partial)`` (or the plain all-reduce without a + residual), ``out_norm = rmsnorm(out_res) * gamma`` when ``norm``.""" + + def __init__(self, *, residual: bool = True, norm: bool = True, max_rows: int = 8): + if residual and not norm: + raise ValueError("residual without RMSNorm is not provided") + if max_rows != 8: + raise ValueError("this hop is provided for up to 8 rows") + self.residual, self.norm, self.max_rows = residual, norm, max_rows + + @cute.jit + def __call__( + self, + partial: Int64, + residual: Int64, + gamma: Int64, + out_res: Int64, + out_norm: Int64, + eps: Float32, + rows: Int32, + p0: Int64, + p1: Int64, + p2: Int64, + p3: Int64, + p4: Int64, + p5: Int64, + p6: Int64, + p7: Int64, + rank: Int32, + err_host: Int64, + site: Int32, + stream: cuda.CUstream, + ): + """``p0 .. p7``: mailbox bases rotated so that ``p0`` is this rank's own.""" + self.kernel( + partial, + residual, + gamma, + out_res, + out_norm, + eps, + rows, + p0, + p1, + p2, + p3, + p4, + p5, + p6, + p7, + rank, + err_host, + site, + ).launch( + grid=[rows * 8 + 8, 1, 1], + block=[CONSUMER_THREADS, 1, 1], + max_number_threads=[CONSUMER_THREADS, 1, 1], + cluster=[8, 1, 1], + cooperative=True, + use_pdl=True, + stream=stream, + ) + + @cute.kernel + def kernel( + self, + partial: Int64, + residual: Int64, + gamma: Int64, + out_res: Int64, + out_norm: Int64, + eps: Float32, + rows: Int32, + p0: Int64, + p1: Int64, + p2: Int64, + p3: Int64, + p4: Int64, + p5: Int64, + p6: Int64, + p7: Int64, + rank: Int32, + err_host: Int64, + site: Int32, + ): + mb = Mbox((p0, p1, p2, p3, p4, p5, p6, p7), rank, self.max_rows, err_host, site) + t = P.thread_index() + bx = P.block_index() + smem = cutlass.utils.SmemAllocator() + cs = ConsSmem.allocate(smem, self.max_rows) + nmain = rows * 8 + if cutlass.const_expr(self.norm): # noqa: SIM102 + if bx < nmain: + self.cons_prologue(cs) + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + sys_ops.fence_proxy_async_global() # inputs may come from an async-proxy writer + if bx >= nmain: + e = self.epoch_begin(mb, cs.es, False) + if t == 0: + self.epoch_arrive(mb, cs.es, rows, P.grid_size()) + if bx == nmain: + self.ack_cta(mb, e) + sys_ops.exit_thread() + m = bx // 8 + c = bx % 8 + h0 = c * 768 + 6 * t + w = cute.make_rmem_tensor((3,), Uint32) + ru = cute.make_rmem_tensor((3,), Uint32) + gu = cute.make_rmem_tensor((3,), Uint32) + w.fill(0) + ru.fill(0) + gu.fill(0) + # every input load is issued before the epoch / acknowledgement check + if t < 128: + off = Int64(2) * (Int64(m) * HIDDEN + Int64(h0)) + if cutlass.const_expr(self.residual): + for k in cutlass.range_constexpr(3): + ru[k] = sys_ops.ld_cg_u32(residual + off + Int64(4 * k)) + if cutlass.const_expr(self.norm): + for k in cutlass.range_constexpr(3): + gu[k] = sys_ops.ld_cg_u32( + gamma + Int64(2) * Int64(h0) + Int64(4 * k) + ) + for k in cutlass.range_constexpr(3): + w[k] = sys_ops.ld_cg_u32(partial + off + Int64(4 * k)) + self.epoch_begin(mb, cs.es, True) + pz = cs.es.pz[0] + em = cs.es.erow[m] # row epoch: buffer slot and tag of row m + buf = self.buffer_of(em) + tag = em & Uint32(TAG_MASK) + if t < 128: + for k in cutlass.range_constexpr(3): + if pz != 0: + w[k] = Uint32(BF16_NAN | (BF16_NAN << 16)) + else: + w[k] = P.no_neg_zero(w[k]) + r0, r1 = self.pack_records(w[0], w[1], w[2], tag) + self.delay(mb, 1) + self.publish_records(mb, buf, m, c * 256 + 2 * t, r0, r1) + if t >= POLL_THREADS: # the epoch warp arrives, then leaves the CTA + if t == POLL_THREADS: + self.epoch_arrive(mb, cs.es, rows, P.grid_size()) + sys_ops.exit_thread() + self.consume_cta( + mb, + cs, + m, + c, + em, + pz, + ru, + gu, + out_res, + out_norm, + eps, + self.residual, + self.norm, + ) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/api.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/api.py new file mode 100644 index 00000000..35bb0233 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/api.py @@ -0,0 +1,573 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Hop-family API used by each engine's glue. + +* ``unit_specs()``: every compile unit (kernel class, Constexpr configuration, example + arguments, launch shape) with a cache key that names the class and every Constexpr + value; the glue compiles them through its own compile cache. +* ``fenced_trigger_supported()``: the load-time check of the o_proj hop's start edge + (PTX ISA 9.7.15.14 needs PTX ISA 9.2 and a PTX compiler of release 13.2 or newer); the + glue declines to the existing path when it fails. +* ``router_hoist_weights()``: the derived router weights of the router-hoisted o_proj + hop. +* ``HopKernels``: the compiled units with the mailboxes of one tensor-parallel group, + called with torch tensors; it chooses the fold form for a row count from the + cooperative occupancy. +""" + +import tempfile +from dataclasses import dataclass, field +from typing import Any + +import cutlass +import cutlass.cute as cute +import torch +from cuda.bindings import driver as cu +from cutlass import Float32, Int32, Int64 + +from .. import launch as L +from .allreduce_norm import AllreduceNormHop +from .constants import CONSUMER_THREADS, ELEMS_PER_CTA, HIDDEN, WORLD +from .moe_fold import ( + MoeFoldArBsum, + MoeFoldArNorm, + MoeFoldArNormDn, + RmsnormFromBlockSums, +) +from .o_proj import ROUTER_K, OProjArConsume, OProjArProduce, router_partial_bytes +from .transport import ErrorRecord, HopMailbox + +MIN_PTX_ISA = (9, 2) +# The split fold serves 1..4 rows (the plain fold above): the rows it is gated at. +SPLIT_MAX_ROWS = 4 +MIN_PTX_COMPILER = (13, 2) + + +@dataclass(frozen=True) +class UnitSpec: + name: str + obj: object + args: tuple + + @property + def constexprs(self) -> dict: + """Every Constexpr configuration value of the unit (the compile-cache key + besides the class).""" + return { + k: v + for k, v in sorted(vars(self.obj).items()) + if isinstance(v, (bool, int, str)) + } + + @property + def key(self) -> str: + """Compile-cache key part: qualified class name + every Constexpr configuration + value.""" + cls = type(self.obj) + return f"{cls.__module__}.{cls.__qualname__}{sorted(self.constexprs.items())}" + + +def _z(): + return Int64(0) + + +def _args_ar(): + return ( + *[_z()] * 5, + Float32(1e-5), + Int32(1), + *[_z()] * 8, + Int32(0), + _z(), + Int32(0), + cu.CUstream(0), + ) + + +def _args_produce(): + return ( + *[_z()] * 5, + Int32(1), + *[_z()] * 16, + Int32(0), + _z(), + Int32(0), + cu.CUstream(0), + ) + + +def _args_consume(): + return ( + *[_z()] * 4, + Float32(1e-5), + Int32(1), + _z(), + _z(), + Float32(1.0), + _z(), + _z(), + *[_z()] * 8, + Int32(0), + _z(), + Int32(0), + cu.CUstream(0), + ) + + +def _args_fold(): + return ( + *[_z()] * 9, + Float32(1e-5), + Int32(1), + *[_z()] * 8, + Int32(0), + _z(), + Int32(0), + cu.CUstream(0), + ) + + +def _args_fold_dn(): + return ( + *[_z()] * 10, + Int32(1), + *[_z()] * 8, + Int32(0), + _z(), + Int32(0), + cu.CUstream(0), + ) + + +def _args_bsum(): + return ( + *[_z()] * 8, + Int32(1), + *[_z()] * 8, + Int32(0), + _z(), + Int32(0), + cu.CUstream(0), + ) + + +def _args_apply(): + return (*[_z()] * 4, Float32(1e-5), Int32(1), cu.CUstream(0)) + + +def unit_specs(*, plain_allreduce: bool = False): + """The compile units of the forward hops (instance A: up to 8 rows; instance B: + 9..16).""" + specs = [ + UnitSpec( + "allreduce_norm", AllreduceNormHop(residual=True, norm=True), _args_ar() + ), + UnitSpec("o_proj_produce", OProjArProduce(), _args_produce()), + UnitSpec("o_proj_consume", OProjArConsume(), _args_consume()), + UnitSpec("o_proj_produce_router", OProjArProduce(router=True), _args_produce()), + UnitSpec("o_proj_consume_router", OProjArConsume(router=True), _args_consume()), + UnitSpec("moe_fold_split", MoeFoldArNorm(split=True), _args_fold()), + UnitSpec("moe_fold_plain", MoeFoldArNorm(split=False), _args_fold()), + UnitSpec( + "o_proj_produce_router16", + OProjArProduce(router=True, max_rows=16), + _args_produce(), + ), + UnitSpec( + "o_proj_consume_router16", + OProjArConsume(router=True, max_rows=16), + _args_consume(), + ), + UnitSpec("moe_fold_block_sums16", MoeFoldArBsum(), _args_bsum()), + UnitSpec("moe_fold_split_dn", MoeFoldArNormDn(split=True), _args_fold_dn()), + UnitSpec("moe_fold_plain_dn", MoeFoldArNormDn(split=False), _args_fold_dn()), + UnitSpec("rmsnorm_from_block_sums16", RmsnormFromBlockSums(), _args_apply()), + ] + if plain_allreduce: + specs.append( + UnitSpec( + "allreduce", AllreduceNormHop(residual=False, norm=False), _args_ar() + ) + ) + return specs + + +def _version(text, prefix): + """(major, minor) of the first ``X.Y`` in ``text``, else None.""" + i = text.find(prefix) + if i < 0: + return None + head = text[i + len(prefix) :].lstrip().split(None, 1) + parts = head[0].split(".") if head else [] + digits = ["".join(c for c in p if c.isdigit()) for p in parts[:2]] + if len(digits) < 2 or not all(digits): + return None + return int(digits[0]), int(digits[1]) + + +def fenced_trigger_supported() -> bool: + """True iff the pinned DSL toolchain emits PTX ISA >= 9.2 and its PTX compiler is + release >= 13.2: the condition under which a release fence before + ``launch_dependents`` orders the dependent grid's start (PTX ISA 9.7.15.14). Checked + on a compile of the o_proj producer whose PTX and cubin are kept in a temporary + directory.""" + ver = cutlass.CUDA_VERSION + if (ver.major, ver.minor) < MIN_PTX_COMPILER: + return False + with tempfile.TemporaryDirectory() as d: + f = cute.compile( + OProjArProduce(), + *_args_produce(), + options=f"--keep-ptx --keep-cubin --dump-dir={d}", + ) + ptx = f.__ptx__ or "" + cubin = f.__cubin__ or b"" + if isinstance(cubin, str): + cubin = cubin.encode("latin-1") + isa = _version(ptx, ".version") + comp = _version(cubin.decode("latin-1"), "Cuda compilation tools, release") + return ( + isa is not None + and comp is not None + and isa >= MIN_PTX_ISA + and comp >= MIN_PTX_COMPILER + ) + + +def router_hoist_weights( + w_router: torch.Tensor, gamma: torch.Tensor, w_o: torch.Tensor, rank: int +): + """``[Q_r | W'_slice]`` (256 x 2816) as fp32 split into bf16 hi + lo, with ``W' = + W_router * diag(gamma)`` (exact in fp64), ``Q_r = W' W_o^(r)`` (fp64 product rounded + to fp32) and ``W'_slice = W'[:, 768 r : 768 (r + 1)]``. ``w_o`` is this rank's + o_proj shard [6144, 2048].""" + slice_w = HIDDEN // WORLD + wp = w_router.double() * gamma.double()[None, :] + q = wp @ w_o.double() + full = torch.cat([q, wp[:, slice_w * rank : slice_w * (rank + 1)]], dim=1).float() + hi = full.to(torch.bfloat16) + lo = (full - hi.float()).to(torch.bfloat16) + assert hi.shape[1] == ROUTER_K + return hi.contiguous(), lo.contiguous() + + +@dataclass +class HopKernels: + """Compiled hop units + the mailboxes of one group. ``compiled`` maps unit names to + compiled functions (from ``unit_specs``); ``mbox_a`` / ``mbox_b`` are the + up-to-8-row and 9..16-row mailbox instances; ``rp_a`` / ``rp_b`` their + router-partial mailboxes.""" + + compiled: dict + mbox_a: HopMailbox + mbox_b: HopMailbox | None + rp_a: Any + rp_b: Any + err: ErrorRecord + eps: float + rscale: float + split_max_rows: int = 0 + _peers: dict = field(default_factory=dict) + + @classmethod + def build( + cls, + compiled, + group_name, + device, + eps, + rscale, + device_index, + *, + instance_b=True, + router=True, + probe=None, + ): + """Collective (every rank of the group): mailbox instance A (up to 8 rows), + instance B (9..16 rows) when ``instance_b``, their router-partial mailboxes when + ``router``, and the split fold's row limit from the occupancy query when the + split fold is compiled. ``probe``: a compiled split fold with its driver library + (for the occupancy query) when the one in ``compiled`` came from a cache that + does not keep it.""" + rp_a = _router_mailbox(group_name, 8, device) if router else None + rp_b = ( + _router_mailbox(group_name, 16, device) if router and instance_b else None + ) + # its init quiesces every rank + mbox_a = HopMailbox.create(group_name, 8, device) + mbox_b = HopMailbox.create(group_name, 16, device) if instance_b else None + self = cls( + compiled, mbox_a, mbox_b, rp_a, rp_b, ErrorRecord(device), eps, rscale + ) + if "moe_fold_split" in compiled: + self.split_max_rows = self._split_rows( + device_index, probe or compiled["moe_fold_split"] + ) + return self + + def reset(self): + """Collective. Re-initialize the mailboxes and zero the router-partial words + (their tags restart below the new row epochs, so no stale term can match).""" + for mb, rp in ((self.mbox_a, self.rp_a), (self.mbox_b, self.rp_b)): + if mb is not None: + mb.init( + -1, + before_last_barrier=(lambda rp=rp: rp.zero()) + if rp is not None + else None, + ) + + def _split_rows(self, device_index, unit): + """Largest row count, at most ``SPLIT_MAX_ROWS`` (the rows the split fold is + served and tested at), whose split fold (2 rows + 1 clusters of 8 CTAs) is + co-resident.""" + h = L.kernel_handle(unit, device_index, _fold_smem()) + best = 0 + for rows in range(1, SPLIT_MAX_ROWS + 1): + if ( + L.fits_cooperative(h, rows * 16 + 8, CONSUMER_THREADS) + and L.max_active_clusters(h, rows * 16 + 8, CONSUMER_THREADS, 8) + >= 2 * rows + 1 + ): + best = rows + return best + + def _rot(self, peers, rank): + return [Int64(peers[(rank + i) % WORLD]) for i in range(WORLD)] + + def _tail(self, mb, site): + return ( + Int32(mb.rank), + Int64(self.err.device_ptr), + Int32(site), + cu.CUstream(torch.cuda.current_stream().cuda_stream), + ) + + def mailbox_args(self, site=0): + """Trailing launch arguments of a kernel that runs on mailbox instance A with + this family's error record: the eight peer mailboxes in rotated order (this + rank's first), the rank, the record's device pointer, the error site and the + current stream. For kernels outside this module that share the instance (the + draft model's hops), which must keep the instance's hop sequence the same on + every rank.""" + mb = self.mbox_a + return (*self._rot(mb.peers, mb.rank), *self._tail(mb, site)) + + def allreduce_norm(self, partial, residual, gamma, out_res, out_norm, site=0): + mb = self.mbox_a + self.compiled["allreduce_norm"]( + _p(partial), + _p(residual), + _p(gamma), + _p(out_res), + _p(out_norm), + Float32(self.eps), + Int32(partial.shape[0]), + *self._rot(mb.peers, mb.rank), + *self._tail(mb, site), + ) + + def allreduce(self, partial, out, site=0): + """Plain all-reduce of ``partial`` [rows <= 8, 6144] bf16 into ``out`` (the + rank-order fp32 sum, one bf16 rounding; no residual, no norm), on mailbox + instance A. Needs the plain unit (``unit_specs(plain_allreduce=True)``).""" + if partial.shape[0] > 8: + raise ValueError("the plain all-reduce serves at most 8 rows per call") + mb = self.mbox_a + z = Int64(0) + self.compiled["allreduce"]( + _p(partial), + z, + z, + _p(out), + z, + Float32(self.eps), + Int32(partial.shape[0]), + *self._rot(mb.peers, mb.rank), + *self._tail(mb, site), + ) + + def o_proj(self, x, w_o, residual, gamma, out_res, out_norm, site=0): + mb = self.mbox_a + rows = Int32(x.shape[0]) + pr = self._rot(mb.peers, mb.rank) + z = Int64(0) + self.compiled["o_proj_produce"]( + _p(x), _p(w_o), z, z, z, rows, *pr, *[z] * WORLD, *self._tail(mb, site) + ) + self.compiled["o_proj_consume"]( + _p(residual), + _p(gamma), + _p(out_res), + _p(out_norm), + Float32(self.eps), + rows, + z, + z, + Float32(self.rscale), + z, + z, + *pr, + *self._tail(mb, site), + ) + + def o_proj_router( + self, x, w_o, qh, ql, bias, residual, gamma, out_res, out_norm, ids, wts, site=0 + ): + rows = x.shape[0] + big = rows > 8 + mb, rp = (self.mbox_b, self.rp_b) if big else (self.mbox_a, self.rp_a) + assert mb is not None, "hop mailbox instance B (9..16 rows) was not built" + sfx = "16" if big else "" + pr = self._rot(mb.peers, mb.rank) + self.compiled["o_proj_produce_router" + sfx]( + _p(x), + _p(w_o), + _p(qh), + _p(ql), + _p(residual), + Int32(rows), + *pr, + *self._rot(rp.peers, mb.rank), + *self._tail(mb, site), + ) + self.compiled["o_proj_consume_router" + sfx]( + _p(residual), + _p(gamma), + _p(out_res), + _p(out_norm), + Float32(self.eps), + Int32(rows), + Int64(rp.local), + _p(bias), + Float32(self.rscale), + _p(ids), + _p(wts), + *pr, + *self._tail(mb, site), + ) + + def moe_fold( + self, + pq, + shp, + ids, + wfin, + g2s, + residual, + gamma, + out_res, + out_norm, + bsum=None, + site=0, + ): + """Fold + all-reduce + residual + RMSNorm. 9..16 rows: block-sum fold, then the + norm apply (``bsum`` [rows, 8] fp32 scratch).""" + rows = int(out_res.shape[0]) + if rows > 8: + mb = self.mbox_b + assert mb is not None, "hop mailbox instance B (9..16 rows) was not built" + self.compiled["moe_fold_block_sums16"]( + _p(pq), + _p(shp), + _p(ids), + _p(wfin), + _p(g2s), + _p(residual), + _p(out_res), + _p(bsum), + Int32(rows), + *self._rot(mb.peers, mb.rank), + *self._tail(mb, site), + ) + self.compiled["rmsnorm_from_block_sums16"]( + _p(out_res), + _p(bsum), + _p(gamma), + _p(out_norm), + Float32(self.eps), + Int32(rows), + cu.CUstream(torch.cuda.current_stream().cuda_stream), + ) + return + mb = self.mbox_a + unit = "moe_fold_split" if rows <= self.split_max_rows else "moe_fold_plain" + self.compiled[unit]( + _p(pq), + _p(shp), + _p(ids), + _p(wfin), + _p(g2s), + _p(residual), + _p(gamma), + _p(out_res), + _p(out_norm), + Float32(self.eps), + Int32(rows), + *self._rot(mb.peers, mb.rank), + *self._tail(mb, site), + ) + + def moe_fold_dn( + self, pq, shp, ids, wfin, g2s, residual, gamma, out_res, out_u, bsum, site=0 + ): + """Fold + all-reduce + residual with the RMSNorm deferred (up to 8 rows): + ``out_u = bf16(x * gamma)`` and the block sums ``bsum`` [rows, 8] fp32 of the + norm (``MoeFoldArNormDn``).""" + rows = int(out_res.shape[0]) + assert rows <= 8, "the deferred-norm fold serves up to 8 rows" + mb = self.mbox_a + unit = ( + "moe_fold_split_dn" if rows <= self.split_max_rows else "moe_fold_plain_dn" + ) + self.compiled[unit]( + _p(pq), + _p(shp), + _p(ids), + _p(wfin), + _p(g2s), + _p(residual), + _p(gamma), + _p(out_res), + _p(out_u), + _p(bsum), + Int32(rows), + *self._rot(mb.peers, mb.rank), + *self._tail(mb, site), + ) + + +def _p(t): + return Int64(t.data_ptr() if t is not None else 0) + + +def _fold_smem(): + """Dynamic shared memory of the fold units (consumer block + y[8][768] + sh[768] + fp32).""" + return 21664 + 4 * (8 * ELEMS_PER_CTA + ELEMS_PER_CTA) + + +@dataclass +class _RouterMailbox: + storage: Any + view: Any + local: int + peers: list + + def zero(self): + self.view.zero_() + + +def _router_mailbox(group_name, max_rows, device): + import torch.distributed._symmetric_memory as symm_mem + + n = router_partial_bytes(max_rows) + st = symm_mem.empty(n + 4096, dtype=torch.uint8, device=device) + h = symm_mem.rendezvous(st, group_name) + peers = [int(p) + (-int(p)) % 4096 for p in h.buffer_ptrs] + local = peers[int(h.rank)] + off = local - st.data_ptr() + view = st[off : off + n] + view.zero_() + return _RouterMailbox(st, view, local, peers) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/constants.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/constants.py new file mode 100644 index 00000000..01054d01 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/constants.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Fixed sizes and the mailbox layout of the tensor-parallel hop kernels. + +The hops all-reduce rows of ``HIDDEN`` bf16 values across ``WORLD`` ranks. A row is sent +as ``NREC`` 64-bit records ``{3 x bf16, tag16}``; the tag is the row's epoch modulo +2^16, so a consumer accepts a record only if it was written in the epoch it is reading. +Buffers rotate through ``NBUF`` slots; a producer reuses a slot only after acquiring an +acknowledgement released by every consumer rank (every ``KACK`` epochs). Every value +here is the served value; none is a tuning knob. +""" + +HIDDEN = 6144 # model hidden size (all-reduced row width) +O_PROJ_IN = 2048 # o_proj input width per rank +WORLD = 8 # tensor-parallel ranks +TOPK = 8 # routed experts per token +NREC = HIDDEN // 3 # 64-bit records per row +CLUSTER = 8 # consumer CTAs per row (one cluster) +REC_PER_CTA = NREC // CLUSTER # 256 records per consumer CTA +PAIRS = REC_PER_CTA // 2 # 128 record pairs (16 bytes) per consumer CTA +ELEMS_PER_CTA = HIDDEN // CLUSTER # 768 +SOURCE_GROUPS = 4 # consumer thread groups, two source ranks each +POLL_THREADS = PAIRS * SOURCE_GROUPS # 512 +EPOCH_THREAD = POLL_THREADS # lane 0 of the extra epoch warp +CONSUMER_THREADS = POLL_THREADS + 32 # 544 + +NBUF = 16 # buffer slots in rotation (a power of two: the u32 epoch wrap is seamless) +KACK = 4 # acknowledgement period in epochs (a power of two) +TAG_BITS = 16 +TAG_MASK = (1 << TAG_BITS) - 1 +assert NBUF >= 2 * KACK + 1 and NBUF & (NBUF - 1) == 0 and KACK & (KACK - 1) == 0 + +CTRL_BYTES = 4096 +# control block, u32 word index (128-byte lines): ack[src] at 32 * src (written by rank +# src), E at 256, the per-row epochs from 264 (same line as E), the sticky error word at +# 512 and the arrival counter at 768 (local only). +W_ACK = 0 +W_E = 256 +W_EROW = 264 +W_ERR = 512 +W_ARRIVE = 768 + +POLL_TIMER_STRIDE = 16 # %globaltimer is read once every 16 polls +# bound of every cross-GPU wait; then the error record and trap +TIMEOUT_NS = 20_000_000_000 +POISONED_TIMEOUT_NS = 2_000_000 # bound once this rank's error word is set +BF16_NAN = 0x7FC0 + +# error record kinds (host-mapped record, word 0) +ERR_RECORD_POLL = 1 +ERR_ACK_WAIT = 2 +ERR_ROUTER_POLL = 3 +ERR_TOP1_POLL = 4 # the draft family's distributed top-1 (same record, its own kind) +ERR_KIND_NAMES = { + ERR_RECORD_POLL: "record poll", + ERR_ACK_WAIT: "ack wait", + ERR_ROUTER_POLL: "router poll", + ERR_TOP1_POLL: "top-1 poll", +} + + +def record_bytes(max_rows: int) -> int: + """Bytes of the record area of a mailbox instance for ``max_rows`` rows.""" + return NBUF * WORLD * max_rows * NREC * 8 + + +def mailbox_bytes(max_rows: int) -> int: + return record_bytes(max_rows) + CTRL_BYTES diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/moe_fold.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/moe_fold.py new file mode 100644 index 00000000..86cbb027 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/moe_fold.py @@ -0,0 +1,810 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""MoE finalize folded into the next all-reduce hop (+ residual + RMSNorm). + +The MoE engine leaves its routed outputs unreduced (finalize deferred): +``pq[4][rows][8][H]`` fp32 quarter partials per (row, slot), ``shp[4][rows][H]`` fp32 +shared-expert quarters, and ``ids`` / ``w`` / ``g2s`` ``[rows][8]`` (``ids < 0``: +dropped slot). This hop computes, per element, exactly the finalize of the engine and +publishes the result as its partial: + + y_k = bf16(((q0 + q1) + q2) + q3) * g2s_k) (0 for a dropped slot) + s = fma(w_k, y_k, s) over k = 0..7 ascending (dropped slots skipped) + out = bf16(bf16(s) + bf16(((sh0 + sh1) + sh2) + sh3)) + +then all-reduces it like ``allreduce_norm_hop`` (residual + RMSNorm). Part 1 (the 512 +polling threads, before the epoch barrier) loads every input of its (slot, 16-byte +chunk) pairs before using any and writes ``y`` to shared memory; part 2 (the 128 +publishers, after the barrier) runs the ordered fma. The epoch / acknowledgement control +loads are issued by the epoch warp so they overlap the fp32 loads. + +``split`` form: the grid adds one helper CTA per (row, cluster rank) that folds and +publishes the second half of the slice's columns (the main CTA the first half), so each +CTA streams half the fp32 inputs; the main CTA still consumes the whole slice. Every CTA +must be co-resident (the main CTAs wait on records written by helper CTAs of the same +grid), so the split form is used only where the cooperative occupancy allows 2 * rows + +1 clusters. +""" + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import cutlass.utils +from cutlass import Float32, Int32, Int64, Uint32 + +from ..primitives import ptx_core as C +from ..primitives import ptx_sys as sys_ops +from . import _ptx_local as P +from .constants import ( + BF16_NAN, + CONSUMER_THREADS, + ELEMS_PER_CTA, + HIDDEN, + POLL_THREADS, + TAG_MASK, + TOPK, +) +from .transport import ConsSmem, Consumer, Mbox + +CHUNKS = ELEMS_PER_CTA // 4 # 16-byte fp32 chunks of one CTA's 768 elements (192) + + +class MoeFoldArNorm(Consumer): + """Fold of the MoE finalize + all-reduce + residual + RMSNorm for up to ``max_rows`` + rows.""" + + def __init__(self, *, split: bool, max_rows: int = 8): + if max_rows != 8: + raise ValueError("this fold is provided for up to 8 rows") + self.split, self.max_rows = split, max_rows + + @cute.jit + def __call__( + self, + pq: Int64, + shp: Int64, + ids: Int64, + wfin: Int64, + g2s: Int64, + residual: Int64, + gamma: Int64, + out_res: Int64, + out_norm: Int64, + eps: Float32, + rows: Int32, + p0: Int64, + p1: Int64, + p2: Int64, + p3: Int64, + p4: Int64, + p5: Int64, + p6: Int64, + p7: Int64, + rank: Int32, + err_host: Int64, + site: Int32, + stream: cuda.CUstream, + ): + nct = rows * 8 + 8 + if cutlass.const_expr(self.split): + nct = rows * 16 + 8 + self.kernel( + pq, + shp, + ids, + wfin, + g2s, + residual, + gamma, + out_res, + out_norm, + eps, + rows, + p0, + p1, + p2, + p3, + p4, + p5, + p6, + p7, + rank, + err_host, + site, + ).launch( + grid=[nct, 1, 1], + block=[CONSUMER_THREADS, 1, 1], + max_number_threads=[CONSUMER_THREADS, 1, 1], + cluster=[8, 1, 1], + cooperative=True, + use_pdl=True, + stream=stream, + ) + + def slot_order(self, kk): + """Slot of the ``kk``-th term of the ordered fma (ascending).""" + return kk + + @cute.kernel + def kernel( + self, + pq: Int64, + shp: Int64, + ids: Int64, + wfin: Int64, + g2s: Int64, + residual: Int64, + gamma: Int64, + out_res: Int64, + out_norm: Int64, + eps: Float32, + rows: Int32, + p0: Int64, + p1: Int64, + p2: Int64, + p3: Int64, + p4: Int64, + p5: Int64, + p6: Int64, + p7: Int64, + rank: Int32, + err_host: Int64, + site: Int32, + ): + self.fold_body( + pq, + shp, + ids, + wfin, + g2s, + residual, + gamma, + out_res, + out_norm, + eps, + rows, + (p0, p1, p2, p3, p4, p5, p6, p7), + rank, + err_host, + site, + ) + + @cute.jit + def fold_body( + self, + pq, + shp, + ids, + wfin, + g2s, + residual, + gamma, + out_res, + out_norm, + eps, + rows, + peers, + rank, + err_host, + site, + dn_bsum=None, + ): + """The fold kernel's body; ``dn_bsum`` selects the deferred-norm consumer + epilogue (``Consumer.consume_cta``).""" + mb = Mbox(peers, rank, self.max_rows, err_host, site) + t = P.thread_index() + bx = P.block_index() + smem = cutlass.utils.SmemAllocator() + cs = ConsSmem.allocate(smem, self.max_rows) + fy = smem.allocate_tensor( + Float32, + cute.make_layout((TOPK, ELEMS_PER_CTA), stride=(ELEMS_PER_CTA, 1)), + 16, + ) + fsh = smem.allocate_tensor(Float32, cute.make_layout(ELEMS_PER_CTA), 16) + nmain = rows * 8 + is_ack = (bx >= nmain) and (bx < nmain + 8) + is_helper = Int32(0) + if cutlass.const_expr(self.split): # noqa: SIM102 + if bx >= nmain + 8: + is_helper = Int32(1) + if bx < nmain: + self.cons_prologue(cs) + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + # the engine's outputs may come from an async-proxy writer + sys_ops.fence_proxy_async_global() + if is_ack: + e = self.epoch_begin(mb, cs.es, False) + if t == 0: + self.epoch_arrive(mb, cs.es, rows, P.grid_size()) + if bx == nmain: + self.ack_cta(mb, e) + sys_ops.exit_thread() + bxm = bx + if is_helper != 0: + bxm = bx - nmain - 8 + m = bxm // 8 + c = bxm % 8 + h0 = c * 768 + 6 * t + ru = cute.make_rmem_tensor((3,), Uint32) + gu = cute.make_rmem_tensor((3,), Uint32) + ru.fill(0) + gu.fill(0) + if t < 128 and is_helper == 0: + off = Int64(2) * (Int64(m) * HIDDEN + Int64(h0)) + for k in cutlass.range_constexpr(3): + ru[k] = sys_ops.ld_cg_u32(residual + off + Int64(4 * k)) + for k in cutlass.range_constexpr(3): + gu[k] = sys_ops.ld_cg_u32(gamma + Int64(2) * Int64(h0) + Int64(4 * k)) + fik = cute.make_rmem_tensor((TOPK,), Int32) + fwk = cute.make_rmem_tensor((TOPK,), Float32) + fik.fill(0) + fwk.fill(0.0) + col = c * ELEMS_PER_CTA + if cutlass.const_expr(self.split): + self.fold_part1_half(pq, shp, ids, g2s, fy, fsh, rows, m, col, is_helper) + npub = 64 + else: + self.fold_part1(pq, shp, ids, g2s, fy, fsh, rows, m, col) + npub = 128 + if t < npub: + for k in cutlass.range_constexpr(TOPK): + fik[k] = sys_ops.ld_cg_s32(ids + Int64(4 * (m * TOPK + k))) + fwk[k] = P.bits_f32(sys_ops.ld_cg_u32(wfin + Int64(4 * (m * TOPK + k)))) + # the epoch warp reads the controls + self.epoch_begin(mb, cs.es, True, POLL_THREADS) + pz = cs.es.pz[0] + em = cs.es.erow[m] + buf = self.buffer_of(em) + tag = em & Uint32(TAG_MASK) + if t < npub: + tp = t + if cutlass.const_expr(self.split): + tp = t + 64 * is_helper + w = self.fold_part2(fy, fsh, fik, fwk, tp) + for k in cutlass.range_constexpr(3): + if pz != 0: + w[k] = Uint32(BF16_NAN | (BF16_NAN << 16)) + else: + w[k] = P.no_neg_zero(w[k]) + r0, r1 = self.pack_records(w[0], w[1], w[2], tag) + self.delay(mb, 1) + self.publish_records(mb, buf, m, c * 256 + 2 * tp, r0, r1) + if cutlass.const_expr(self.split): # noqa: SIM102 + if is_helper != 0: # helper: publish only + if t == POLL_THREADS: + self.epoch_arrive(mb, cs.es, rows, P.grid_size()) + sys_ops.exit_thread() + if t >= POLL_THREADS: # the epoch warp arrives, then leaves the CTA + if t == POLL_THREADS: + self.epoch_arrive(mb, cs.es, rows, P.grid_size()) + sys_ops.exit_thread() + self.consume_cta( + mb, cs, m, c, em, pz, ru, gu, out_res, out_norm, eps, True, True, dn_bsum + ) + + @cute.jit + def fold_part1( + self, pq: Int64, shp: Int64, ids: Int64, g2s: Int64, fy, fsh, rows, m, col + ): + """Part 1, plain form: thread t handles (slot, chunk) pairs p = t + 512 i, i < + 3; the last 192 polling threads also sum the shared-expert quarters. Every load + before any use.""" + t = P.thread_index() + if t < POLL_THREADS: + v = cute.make_rmem_tensor((3, 4, 4), Float32) + ik = cute.make_rmem_tensor((3,), Int32) + gk = cute.make_rmem_tensor((3,), Float32) + for i in cutlass.range_constexpr(3): + p = t + POLL_THREADS * i + k = p // CHUNKS + ch = p - k * CHUNKS + ik[i] = sys_ops.ld_cg_s32(ids + Int64(4 * (m * TOPK + k))) + gk[i] = P.bits_f32(sys_ops.ld_cg_u32(g2s + Int64(4 * (m * TOPK + k)))) + for q in cutlass.range_constexpr(4): + a = pq + Int64(4) * ( + ((Int64(q) * Int64(rows) + Int64(m)) * TOPK + Int64(k)) * HIDDEN + + Int64(col + 4 * ch) + ) + x0, x1, x2, x3 = sys_ops.ld_cg_v4_f32(a) + v[i, q, 0] = x0 + v[i, q, 1] = x1 + v[i, q, 2] = x2 + v[i, q, 3] = x3 + sv = cute.make_rmem_tensor((4, 4), Float32) + sv.fill(0.0) + if t >= POLL_THREADS - CHUNKS: + ch = t - (POLL_THREADS - CHUNKS) + for q in cutlass.range_constexpr(4): + a = shp + Int64(4) * ( + (Int64(q) * Int64(rows) + Int64(m)) * HIDDEN + + Int64(col + 4 * ch) + ) + x0, x1, x2, x3 = sys_ops.ld_cg_v4_f32(a) + sv[q, 0] = x0 + sv[q, 1] = x1 + sv[q, 2] = x2 + sv[q, 3] = x3 + for i in cutlass.range_constexpr(3): + p = t + POLL_THREADS * i + k = p // CHUNKS + ch = p - k * CHUNKS + self.store_y(fy, v, i, ik[i], gk[i], k, ch) + if t >= POLL_THREADS - CHUNKS: + self.store_sh(fsh, sv, t - (POLL_THREADS - CHUNKS)) + + @cute.jit + def fold_part1_half( + self, pq: Int64, shp: Int64, ids: Int64, g2s: Int64, fy, fsh, rows, m, col, half + ): + """Part 1, split form: half ``half`` of the slice (chunks [96 half, 96 half + + 96)); pairs p = t + 512 i, i < 2, p < 768; the last 96 polling threads sum the + shared quarters.""" + t = P.thread_index() + hch: cutlass.Constexpr = CHUNKS // 2 + if t < POLL_THREADS: + v = cute.make_rmem_tensor((2, 4, 4), Float32) + ik = cute.make_rmem_tensor((2,), Int32) + gk = cute.make_rmem_tensor((2,), Float32) + v.fill(0.0) + ik.fill(0) + gk.fill(0.0) + for i in cutlass.range_constexpr(2): + p = t + POLL_THREADS * i + if p < TOPK * hch: + k = p // hch + ch = hch * half + (p - k * hch) + ik[i] = sys_ops.ld_cg_s32(ids + Int64(4 * (m * TOPK + k))) + gk[i] = P.bits_f32( + sys_ops.ld_cg_u32(g2s + Int64(4 * (m * TOPK + k))) + ) + for q in cutlass.range_constexpr(4): + a = pq + Int64(4) * ( + ((Int64(q) * Int64(rows) + Int64(m)) * TOPK + Int64(k)) + * HIDDEN + + Int64(col + 4 * ch) + ) + x0, x1, x2, x3 = sys_ops.ld_cg_v4_f32(a) + v[i, q, 0] = x0 + v[i, q, 1] = x1 + v[i, q, 2] = x2 + v[i, q, 3] = x3 + sv = cute.make_rmem_tensor((4, 4), Float32) + sv.fill(0.0) + if t >= POLL_THREADS - hch: + ch = hch * half + (t - (POLL_THREADS - hch)) + for q in cutlass.range_constexpr(4): + a = shp + Int64(4) * ( + (Int64(q) * Int64(rows) + Int64(m)) * HIDDEN + + Int64(col + 4 * ch) + ) + x0, x1, x2, x3 = sys_ops.ld_cg_v4_f32(a) + sv[q, 0] = x0 + sv[q, 1] = x1 + sv[q, 2] = x2 + sv[q, 3] = x3 + for i in cutlass.range_constexpr(2): + p = t + POLL_THREADS * i + if p < TOPK * hch: + k = p // hch + ch = hch * half + (p - k * hch) + self.store_y(fy, v, i, ik[i], gk[i], k, ch) + if t >= POLL_THREADS - hch: + self.store_sh(fsh, sv, hch * half + (t - (POLL_THREADS - hch))) + + @cute.jit + def store_y(self, fy, v, i, ik, gk, k, ch): + """``y = bf16((((q0 + q1) + q2) + q3) * g)`` per lane (0 for a dropped slot) + into ``fy[k]``.""" + px = C.mul_rn( + C.add_rn( + C.add_rn(C.add_rn(v[i, 0, 0], v[i, 1, 0]), v[i, 2, 0]), v[i, 3, 0] + ), + gk, + ) + py = C.mul_rn( + C.add_rn( + C.add_rn(C.add_rn(v[i, 0, 1], v[i, 1, 1]), v[i, 2, 1]), v[i, 3, 1] + ), + gk, + ) + pz = C.mul_rn( + C.add_rn( + C.add_rn(C.add_rn(v[i, 0, 2], v[i, 1, 2]), v[i, 2, 2]), v[i, 3, 2] + ), + gk, + ) + pw = C.mul_rn( + C.add_rn( + C.add_rn(C.add_rn(v[i, 0, 3], v[i, 1, 3]), v[i, 2, 3]), v[i, 3, 3] + ), + gk, + ) + w0 = C.cvt_rn_bf16x2_f32(px, py) + w1 = C.cvt_rn_bf16x2_f32(pz, pw) + y0 = P.bf16_lo(w0) + y1 = P.bf16_hi(w0) + y2 = P.bf16_lo(w1) + y3 = P.bf16_hi(w1) + if ik < 0: + y0 = Float32(0.0) + y1 = Float32(0.0) + y2 = Float32(0.0) + y3 = Float32(0.0) + P.st_shared_v4_u32( + Uint32((fy.iterator + (k * ELEMS_PER_CTA + 4 * ch)).toint()), + P.f32_bits(y0), + P.f32_bits(y1), + P.f32_bits(y2), + P.f32_bits(y3), + ) + + @cute.jit + def store_sh(self, fsh, sv, ch): + """Shared-expert chunk ``((sh0 + sh1) + sh2) + sh3`` (fp32, before its bf16 + rounding).""" + s0 = C.add_rn(C.add_rn(C.add_rn(sv[0, 0], sv[1, 0]), sv[2, 0]), sv[3, 0]) + s1 = C.add_rn(C.add_rn(C.add_rn(sv[0, 1], sv[1, 1]), sv[2, 1]), sv[3, 1]) + s2 = C.add_rn(C.add_rn(C.add_rn(sv[0, 2], sv[1, 2]), sv[2, 2]), sv[3, 2]) + s3 = C.add_rn(C.add_rn(C.add_rn(sv[0, 3], sv[1, 3]), sv[2, 3]), sv[3, 3]) + P.st_shared_v4_u32( + Uint32((fsh.iterator + 4 * ch).toint()), + P.f32_bits(s0), + P.f32_bits(s1), + P.f32_bits(s2), + P.f32_bits(s3), + ) + + @cute.jit + def fold_part2(self, fy, fsh, fik, fwk, tp): + """Part 2 for publisher slot ``tp`` (6 elements): the ordered fma over the + slots, then ``bf16(bf16(routed) + bf16(shared))``; returns three bf16x2 + words.""" + sa = cute.make_rmem_tensor((6,), Float32) + sh = cute.make_rmem_tensor((6,), Float32) + for x in cutlass.range_constexpr(6): + acc = Float32(0.0) + for kk in cutlass.range_constexpr(TOPK): + k: cutlass.Constexpr = self.slot_order(kk) + if fik[k] >= 0: + acc = C.fma_rn(fwk[k], fy[k, 6 * tp + x], acc) + sa[x] = acc + sh[x] = fsh[6 * tp + x] + w = cute.make_rmem_tensor((3,), Uint32) + for k in cutlass.range_constexpr(3): + ra = C.cvt_rn_bf16x2_f32(sa[2 * k], sa[2 * k + 1]) + rs = C.cvt_rn_bf16x2_f32(sh[2 * k], sh[2 * k + 1]) + o0 = C.add_rn(P.bf16_lo(ra), P.bf16_lo(rs)) + o1 = C.add_rn(P.bf16_hi(ra), P.bf16_hi(rs)) + w[k] = C.cvt_rn_bf16x2_f32(o0, o1) + return w + + +class MoeFoldArNormDn(MoeFoldArNorm): + """The fold for up to 8 rows with the attention input's RMSNorm deferred to the + consumer of its projection: fold + all-reduce + residual as ``MoeFoldArNorm``, then + ``u = bf16(x * (0 + gamma))`` in place of the normed output and each CTA's block sum + of squares to ``bsum[rows][8]`` (the norm's own value and order); the cluster does + not exchange the block sums. ``inv = rsqrt(fma(sum_c bsum[m][c], rcp(HIDDEN), eps))`` + (sum from 0 in ``c`` order) gives the normed row as ``(x * inv) * gamma``.""" + + @cute.jit + def __call__( + self, + pq: Int64, + shp: Int64, + ids: Int64, + wfin: Int64, + g2s: Int64, + residual: Int64, + gamma: Int64, + out_res: Int64, + out_u: Int64, + bsum: Int64, + rows: Int32, + p0: Int64, + p1: Int64, + p2: Int64, + p3: Int64, + p4: Int64, + p5: Int64, + p6: Int64, + p7: Int64, + rank: Int32, + err_host: Int64, + site: Int32, + stream: cuda.CUstream, + ): + nct = rows * 8 + 8 + if cutlass.const_expr(self.split): + nct = rows * 16 + 8 + self.kernel_dn( + pq, + shp, + ids, + wfin, + g2s, + residual, + gamma, + out_res, + out_u, + bsum, + rows, + p0, + p1, + p2, + p3, + p4, + p5, + p6, + p7, + rank, + err_host, + site, + ).launch( + grid=[nct, 1, 1], + block=[CONSUMER_THREADS, 1, 1], + max_number_threads=[CONSUMER_THREADS, 1, 1], + cluster=[8, 1, 1], + cooperative=True, + use_pdl=True, + stream=stream, + ) + + @cute.kernel + def kernel_dn( + self, + pq: Int64, + shp: Int64, + ids: Int64, + wfin: Int64, + g2s: Int64, + residual: Int64, + gamma: Int64, + out_res: Int64, + out_u: Int64, + bsum: Int64, + rows: Int32, + p0: Int64, + p1: Int64, + p2: Int64, + p3: Int64, + p4: Int64, + p5: Int64, + p6: Int64, + p7: Int64, + rank: Int32, + err_host: Int64, + site: Int32, + ): + self.fold_body( + pq, + shp, + ids, + wfin, + g2s, + residual, + gamma, + out_res, + out_u, + Float32(0.0), + rows, + (p0, p1, p2, p3, p4, p5, p6, p7), + rank, + err_host, + site, + bsum, + ) + + +class MoeFoldArBsum(MoeFoldArNorm): + """The fold for 9..16 rows with the RMSNorm deferred: fold + all-reduce + residual, + then each CTA's block sum of squares to ``bsum[rows][8]``; ``RmsnormFromBlockSums`` + applies the norm. Together they equal the normed fold bitwise. With no cluster + exchange there is no cluster launch: one cooperative grid of ``rows * 8`` CTAs plus + one acknowledgement CTA.""" + + def __init__(self, *, max_rows: int = 16): + if max_rows != 16: + raise ValueError("the block-sum fold is provided for up to 16 rows") + self.split, self.max_rows = False, max_rows + + @cute.jit + def __call__( + self, + pq: Int64, + shp: Int64, + ids: Int64, + wfin: Int64, + g2s: Int64, + residual: Int64, + out_res: Int64, + bsum: Int64, + rows: Int32, + p0: Int64, + p1: Int64, + p2: Int64, + p3: Int64, + p4: Int64, + p5: Int64, + p6: Int64, + p7: Int64, + rank: Int32, + err_host: Int64, + site: Int32, + stream: cuda.CUstream, + ): + self.kernel_bsum( + pq, + shp, + ids, + wfin, + g2s, + residual, + out_res, + bsum, + rows, + p0, + p1, + p2, + p3, + p4, + p5, + p6, + p7, + rank, + err_host, + site, + ).launch( + grid=[rows * 8 + 1, 1, 1], + block=[CONSUMER_THREADS, 1, 1], + max_number_threads=[CONSUMER_THREADS, 1, 1], + cooperative=True, + use_pdl=True, + stream=stream, + ) + + @cute.kernel + def kernel_bsum( + self, + pq: Int64, + shp: Int64, + ids: Int64, + wfin: Int64, + g2s: Int64, + residual: Int64, + out_res: Int64, + bsum: Int64, + rows: Int32, + p0: Int64, + p1: Int64, + p2: Int64, + p3: Int64, + p4: Int64, + p5: Int64, + p6: Int64, + p7: Int64, + rank: Int32, + err_host: Int64, + site: Int32, + ): + mb = Mbox((p0, p1, p2, p3, p4, p5, p6, p7), rank, self.max_rows, err_host, site) + t = P.thread_index() + bx = P.block_index() + smem = cutlass.utils.SmemAllocator() + cs = ConsSmem.allocate(smem, self.max_rows) + fy = smem.allocate_tensor( + Float32, + cute.make_layout((TOPK, ELEMS_PER_CTA), stride=(ELEMS_PER_CTA, 1)), + 16, + ) + fsh = smem.allocate_tensor(Float32, cute.make_layout(ELEMS_PER_CTA), 16) + nmain = rows * 8 + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + sys_ops.fence_proxy_async_global() + if bx >= nmain: + e = self.epoch_begin(mb, cs.es, False) + if t == 0: + self.epoch_arrive(mb, cs.es, rows, P.grid_size()) + if bx == nmain: + self.ack_cta(mb, e) + sys_ops.exit_thread() + m = bx // 8 + c = bx % 8 + h0 = c * 768 + 6 * t + ru = cute.make_rmem_tensor((3,), Uint32) + ru.fill(0) + if t < 128: + off = Int64(2) * (Int64(m) * HIDDEN + Int64(h0)) + for k in cutlass.range_constexpr(3): + ru[k] = sys_ops.ld_cg_u32(residual + off + Int64(4 * k)) + fik = cute.make_rmem_tensor((TOPK,), Int32) + fwk = cute.make_rmem_tensor((TOPK,), Float32) + fik.fill(0) + fwk.fill(0.0) + self.fold_part1(pq, shp, ids, g2s, fy, fsh, rows, m, c * ELEMS_PER_CTA) + if t < 128: + for k in cutlass.range_constexpr(TOPK): + fik[k] = sys_ops.ld_cg_s32(ids + Int64(4 * (m * TOPK + k))) + fwk[k] = P.bits_f32(sys_ops.ld_cg_u32(wfin + Int64(4 * (m * TOPK + k)))) + self.epoch_begin(mb, cs.es, True, POLL_THREADS) + pz = cs.es.pz[0] + em = cs.es.erow[m] + tag = em & Uint32(TAG_MASK) + if t < 128: + w = self.fold_part2(fy, fsh, fik, fwk, t) + for k in cutlass.range_constexpr(3): + if pz != 0: + w[k] = Uint32(BF16_NAN | (BF16_NAN << 16)) + else: + w[k] = P.no_neg_zero(w[k]) + r0, r1 = self.pack_records(w[0], w[1], w[2], tag) + self.delay(mb, 1) + self.publish_records(mb, self.buffer_of(em), m, c * 256 + 2 * t, r0, r1) + if t >= POLL_THREADS: + if t == POLL_THREADS: + self.epoch_arrive(mb, cs.es, rows, P.grid_size()) + sys_ops.exit_thread() + self.consume_cta_bsum(mb, cs, m, c, em, pz, ru, out_res, bsum) + + +class RmsnormFromBlockSums(Consumer): + """``y[m] = bf16((x[m] * inv_m) * (0 + gamma))`` with ``inv_m = rsqrt(fma(sum_c + bsum[m][c], rcp(HIDDEN), eps))``, the sum from 0 in ``c`` order: exactly the normed + fold's norm. One CTA of 768 threads per row; programmatic dependent of the block-sum + fold. ``x`` and ``bsum`` come from that fold, which may still run while this grid is + resident, so they are read with coherent loads; ``gamma`` is static.""" + + def __init__(self, *, max_rows: int = 16): + self.max_rows = max_rows + + @cute.jit + def __call__( + self, + x: Int64, + bsum: Int64, + gamma: Int64, + y: Int64, + eps: Float32, + rows: Int32, + stream: cuda.CUstream, + ): + self.kernel(x, bsum, gamma, y, eps).launch( + grid=[rows, 1, 1], + block=[768, 1, 1], + max_number_threads=[768, 1, 1], + use_pdl=True, + stream=stream, + ) + + @cute.kernel + def kernel(self, x: Int64, bsum: Int64, gamma: Int64, y: Int64, eps: Float32): + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + sys_ops.fence_proxy_async_global() + m = P.block_index() + t = P.thread_index() + tot = Float32(0.0) + for cc in cutlass.range_constexpr(8): + tot = C.add_rn( + tot, + P.bits_f32( + sys_ops.ld_u32(bsum + Int64(4) * (Int64(m) * 8 + Int64(cc))) + ), + ) + inv = C.rsqrt_approx(C.fma_rn(tot, C.rcp_rn(Float32(6144.0)), eps)) + xo = x + Int64(2) * (Int64(m) * HIDDEN + Int64(8 * t)) + go = gamma + Int64(2) * Int64(8 * t) + yo = y + Int64(2) * (Int64(m) * HIDDEN + Int64(8 * t)) + for k in cutlass.range_constexpr(4): + xv = sys_ops.ld_u32(xo + Int64(4 * k)) + gv = sys_ops.ld_nc_u32(go + Int64(4 * k)) # static weight + g0 = C.add_rn(Float32(0.0), P.bf16_lo(gv)) + g1 = C.add_rn(Float32(0.0), P.bf16_hi(gv)) + y0 = C.cvt_rn_bf16_f32(C.mul_rn(C.mul_rn(P.bf16_lo(xv), inv), g0)) + y1 = C.cvt_rn_bf16_f32(C.mul_rn(C.mul_rn(P.bf16_hi(xv), inv), g1)) + sys_ops.st_global_u32(yo + Int64(4 * k), Uint32(y0) | (Uint32(y1) << 16)) + sys_ops.fence_proxy_async_global() diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/o_proj.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/o_proj.py new file mode 100644 index 00000000..761e2e33 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/o_proj.py @@ -0,0 +1,1157 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""o_proj all-reduce hop: a producer grid computes this rank's o_proj partial and +publishes it as mailbox records; a consumer grid (its programmatic dependent) +all-reduces the records with the residual add and RMSNorm. + +Producer: one synchronization CTA plus 128 GEMV CTAs of 384 threads. GEMV CTA ``c`` owns +output rows ``[48c, 48c + 48)`` (12 warps = 3 row tiles x 4 K quarters); its weight +slice is loaded into registers before ``griddepcontrol.wait`` (static weights only), +then the activations are staged, ``mma.sync.m16n8k16`` accumulates per K quarter (tokens +on n8), and the four quarters are summed in a fixed order. + +Router (``router=True``, MoE layers): the router logits are computed through the o_proj +hop. With ``W' = W_router * diag(gamma)`` and ``Q_r = W' W_o^(r)`` (fp32, split into +bf16 hi + lo), ``logits[m] = rstd_m * sum_r (Q_r a_r + W'[:, slice r] residual_m[slice +r])``. Sixteen more producer CTAs compute this rank's term for 16 experts each and +publish ``{fp32, tag32}`` words to every rank's router-partial mailbox; the consumer's +cluster rank 0 of each row gathers the eight terms in rank order, multiplies by the +row's RMSNorm rstd and routes the row (sigmoid scores + bias, top-8, normalized scaled +weights) in one warp. + +Start edge (PTX ISA 9.7.15.14): the synchronization CTA executes ``griddepcontrol.wait; +fence.acq_rel.gpu; griddepcontrol.launch_dependents``, so the consumer grid's start is +ordered after every grid this producer depends on, and the consumer reads its residual, +gamma and epochs without waiting at entry. The consumer's last arriving CTA waits for +the producer grid (``griddepcontrol.wait``) before advancing the epoch (a slow producer +CTA may still be reading it), and thread 0 waits at exit, so the consumer completes +after the producer. +""" + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import cutlass.utils +from cutlass import Float32, Int32, Int64, Uint32, Uint64 + +from ..primitives import ptx_core as C +from ..primitives import ptx_sys as sys_ops +from . import _ptx_local as P +from .constants import ( + BF16_NAN, + CONSUMER_THREADS, + ERR_ROUTER_POLL, + HIDDEN, + NBUF, + O_PROJ_IN, + POLL_THREADS, + POLL_TIMER_STRIDE, + TAG_MASK, + TOPK, + WORLD, +) +from .transport import ConsSmem, Consumer, EpochSmem, Mbox, Transport + +GEMV_ROWS = 48 # output rows per GEMV CTA +GEMV_CTAS = HIDDEN // GEMV_ROWS # 128 +GEMV_THREADS = 384 # 12 warps: 3 row tiles x 4 K quarters +GEMV_STEPS = 16 # 32-wide K steps per quarter (512 / 32) +XS_STRIDE = O_PROJ_IN + 32 # staged activation row stride (bf16) +EXPERTS = 256 +ROUTER_CTAS = 16 # 16 experts (one m16 tile) per router CTA +ROUTER_K = O_PROJ_IN + HIDDEN // WORLD # 2816 = [o_proj input | residual slice] +ROUTER_STEPS = 8 # max 32-wide K steps per warp (warps 0..3: 8, 4..11: 7) +RXS_STRIDE = ROUTER_K + 32 +# the model's routed scaling factor (the glue passes the checkpoint value) +ROUTED_SCALE = 2.5 + + +def router_partial_bytes(max_rows: int) -> int: + """Router-partial mailbox ``[NBUF][8 src][max_rows][256]`` of ``{fp32, tag32}`` + words.""" + return NBUF * WORLD * max_rows * EXPERTS * 8 + + +def _view(base, offset, dtype, shape, stride=None): + ptr = cute.recast_ptr(base + offset, dtype=dtype) + return cute.make_tensor(ptr, cute.make_layout(shape, stride=stride)) + + +def _epoch_view(base, offset, max_rows): + return EpochSmem( + _view(base, offset, Uint32, 1), + _view(base, offset + 4, Int32, 1), + _view(base, offset + 8, Uint32, max_rows), + ) + + +class OProjArProduce(Transport): + """Producer of an o_proj hop for up to ``max_rows`` rows (``router``: + the router + terms).""" + + sync_ctas = 1 # the start-edge synchronization CTA + + def __init__(self, *, router: bool = False, max_rows: int = 8): + if max_rows not in (8, 16) or (max_rows == 16 and not router): + raise ValueError( + "provided for up to 8 rows, or up to 16 rows with the router" + ) + self.router, self.max_rows = router, max_rows + # rows 9..16: a second n8 token tile per mma (same A fragments, same k order), + # and the + # activations staged by 16-byte cp.async copies (one wait before the epoch + # barrier) + self.two_tiles = max_rows == 16 + self.async_stage = max_rows == 16 + + def smem_bytes(self): + gemv = ( + self.max_rows * XS_STRIDE * 2 + + 4 * 3 * 16 * self.max_rows * 4 + + 8 + + 4 * self.max_rows + ) + if not self.router: + return gemv + rt = ( + self.max_rows * RXS_STRIDE * 2 + + 12 * 16 * self.max_rows * 4 + + 8 + + 4 * self.max_rows + ) + return max(gemv, rt) + + @cute.jit + def stage16(self, smem_addr: Uint32, src: Int64): + """Stage 16 bytes of activations into shared memory.""" + if cutlass.const_expr(self.async_stage): + sys_ops.cp_async_cg_16(smem_addr, src) + else: + v = sys_ops.ld_cg_v4_u32(src) + P.st_shared_v4_u32(smem_addr, v[0], v[1], v[2], v[3]) + + @cute.jit + def stage_done(self): + if cutlass.const_expr(self.async_stage): + # this thread's copies landed; the epoch barrier publishes them + sys_ops.cp_async_wait_all() + + @cute.jit + def __call__( + self, + x: Int64, + w: Int64, + qh: Int64, + ql: Int64, + residual: Int64, + rows: Int32, + p0: Int64, + p1: Int64, + p2: Int64, + p3: Int64, + p4: Int64, + p5: Int64, + p6: Int64, + p7: Int64, + q0: Int64, + q1: Int64, + q2: Int64, + q3: Int64, + q4: Int64, + q5: Int64, + q6: Int64, + q7: Int64, + rank: Int32, + err_host: Int64, + site: Int32, + stream: cuda.CUstream, + ): + """``x`` [rows, 2048] bf16, ``w`` [6144, 2048] bf16 (this rank's o_proj shard). + Router: ``qh`` / ``ql`` [256, 2816] bf16, ``residual`` [rows, 6144] bf16, ``q0 + .. q7`` the router-partial mailbox bases rotated like ``p0 .. p7`` (unused + without the router).""" + nct = ( + GEMV_CTAS + + self.sync_ctas + + (ROUTER_CTAS if cutlass.const_expr(self.router) else 0) + ) + self.kernel( + x, + w, + qh, + ql, + residual, + rows, + p0, + p1, + p2, + p3, + p4, + p5, + p6, + p7, + q0, + q1, + q2, + q3, + q4, + q5, + q6, + q7, + rank, + err_host, + site, + ).launch( + grid=[nct, 1, 1], + block=[GEMV_THREADS, 1, 1], + max_number_threads=[GEMV_THREADS, 1, 1], + min_blocks_per_mp=1, + use_pdl=True, + stream=stream, + ) + + @cute.jit + def sync_cta(self): + """Start edge: every earlier grid (covered by this thread's wait) happens before + the dependent grid's start (release fence, then the trigger; PTX ISA + 9.7.15.14).""" + if P.thread_index() == 0: + cute.arch.griddepcontrol_wait() + sys_ops.fence_acq_rel_gpu() + cute.arch.griddepcontrol_launch_dependents() + + @cute.kernel + def kernel( + self, + x: Int64, + w: Int64, + qh: Int64, + ql: Int64, + residual: Int64, + rows: Int32, + p0: Int64, + p1: Int64, + p2: Int64, + p3: Int64, + p4: Int64, + p5: Int64, + p6: Int64, + p7: Int64, + q0: Int64, + q1: Int64, + q2: Int64, + q3: Int64, + q4: Int64, + q5: Int64, + q6: Int64, + q7: Int64, + rank: Int32, + err_host: Int64, + site: Int32, + ): + mb = Mbox((p0, p1, p2, p3, p4, p5, p6, p7), rank, self.max_rows, err_host, site) + bx = P.block_index() + smem = cutlass.utils.SmemAllocator() + raw = smem.allocate_tensor( + cutlass.Uint8, cute.make_layout(self.smem_bytes()), 16 + ) + base = raw.iterator + if cutlass.const_expr(self.sync_ctas > 0): # noqa: SIM102 + if bx < self.sync_ctas: + self.sync_cta() + sys_ops.exit_thread() + cta = bx - self.sync_ctas + pol = sys_ops.create_policy_evict_first() + if cutlass.const_expr(self.router): # noqa: SIM102 + if cta >= GEMV_CTAS: + self.router_cta( + mb, + base, + cta - GEMV_CTAS, + x, + qh, + ql, + residual, + rows, + (q0, q1, q2, q3, q4, q5, q6, q7), + pol, + ) + sys_ops.exit_thread() + self.gemv_cta(mb, base, cta, x, w, rows, pol) + + @cute.jit + def pdl_edge(self, rows: Int32): + """``griddepcontrol.wait`` then ``launch_dependents`` of a GEMV / router CTA, + inside a branch on a runtime value (``rows`` >= 1 in every launch): with the + wait unconditional, ptxas moves every ``ld.global.nc`` weight load issued above + it to below it, and the weight stream would no longer overlap the predecessor + grid. The branch keeps the loads above the wait (the block structure the + reference kernel has).""" + if rows > Int32(0): + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + + @cute.jit + def gemv_cta( + self, mb: Mbox, base, cta, x: Int64, w: Int64, rows: Int32, pol: Uint64 + ): + t = P.thread_index() + xs_bytes: cutlass.Constexpr = self.max_rows * XS_STRIDE * 2 + mx: cutlass.Constexpr = self.max_rows + red = _view( + base, xs_bytes, Float32, (4, 3, 16, mx), (3 * 16 * mx, 16 * mx, mx, 1) + ) + es = _epoch_view(base, xs_bytes + 4 * 3 * 16 * mx * 4, self.max_rows) + warp = t // 32 + lane = t % 32 + g = lane // 4 + q = lane % 4 + tile = warp % 3 + kq = warp // 3 + row0 = cta * GEMV_ROWS + tile * 16 + wa = w + Int64(2) * (Int64(row0 + g) * O_PROJ_IN + Int64(kq * 512 + 8 * q)) + wb = wa + Int64(2 * 8 * O_PROJ_IN) + a = cute.make_rmem_tensor((GEMV_STEPS, 4), Uint32) + b = cute.make_rmem_tensor((GEMV_STEPS, 4), Uint32) + # static weights: issued before the wait + for st in cutlass.range_constexpr(GEMV_STEPS): + v = sys_ops.ld_nc_hint_v4_u32(wa + Int64(64 * st), pol) + u = sys_ops.ld_nc_hint_v4_u32(wb + Int64(64 * st), pol) + for k in cutlass.range_constexpr(4): + a[st, k] = v[k] + b[st, k] = u[k] + self.pdl_edge(rows) + sys_ops.fence_proxy_async_global() # x may come from an async-proxy writer + xs_base = Uint32(base.toint()) + i = t + nchunk = rows * (O_PROJ_IN // 8) + while i < nchunk: + mm = i // (O_PROJ_IN // 8) + kc = i % (O_PROJ_IN // 8) + self.stage16( + xs_base + Uint32(2 * (mm * XS_STRIDE + kc * 8)), + x + Int64(2) * (Int64(mm) * O_PROJ_IN + Int64(kc * 8)), + ) + i = i + GEMV_THREADS + self.stage_done() + if cutlass.const_expr(not self.router): # noqa: SIM102 + if t < 8: + # a slow GEMV CTA between its trigger and its read of E + self.delay(mb, 6) + self.epoch_begin(mb, es, True) # its CTA barrier also covers the staging + pz = es.pz[0] + c0 = Float32(0.0) + c1 = Float32(0.0) + c2 = Float32(0.0) + c3 = Float32(0.0) + d0 = Float32(0.0) + d1 = Float32(0.0) + d2 = Float32(0.0) + d3 = Float32(0.0) + tok = g < rows + tok2 = g + 8 < rows + two = rows > 8 + xr = xs_base + Uint32(2 * (g * XS_STRIDE + kq * 512 + 8 * q)) + xr2 = xr + Uint32(2 * 8 * XS_STRIDE) + for st in cutlass.range_constexpr(GEMV_STEPS): + x0 = Uint32(0) + x1 = Uint32(0) + x2 = Uint32(0) + x3 = Uint32(0) + if tok: + x0, x1, x2, x3 = P.ld_shared_v4_u32(xr + Uint32(2 * 32 * st)) + c0, c1, c2, c3 = P.mma_bf16_m16n8k16( + (c0, c1, c2, c3), (a[st, 0], b[st, 0], a[st, 1], b[st, 1]), (x0, x1) + ) + c0, c1, c2, c3 = P.mma_bf16_m16n8k16( + (c0, c1, c2, c3), (a[st, 2], b[st, 2], a[st, 3], b[st, 3]), (x2, x3) + ) + if cutlass.const_expr(self.two_tiles): # noqa: SIM102 + if two: + y0 = Uint32(0) + y1 = Uint32(0) + y2 = Uint32(0) + y3 = Uint32(0) + if tok2: + y0, y1, y2, y3 = P.ld_shared_v4_u32(xr2 + Uint32(2 * 32 * st)) + d0, d1, d2, d3 = P.mma_bf16_m16n8k16( + (d0, d1, d2, d3), + (a[st, 0], b[st, 0], a[st, 1], b[st, 1]), + (y0, y1), + ) + d0, d1, d2, d3 = P.mma_bf16_m16n8k16( + (d0, d1, d2, d3), + (a[st, 2], b[st, 2], a[st, 3], b[st, 3]), + (y2, y3), + ) + red[kq, tile, g, 2 * q] = c0 + red[kq, tile, g, 2 * q + 1] = c1 + red[kq, tile, g + 8, 2 * q] = c2 + red[kq, tile, g + 8, 2 * q + 1] = c3 + if cutlass.const_expr(self.two_tiles): # noqa: SIM102 + if two: + red[kq, tile, g, 8 + 2 * q] = d0 + red[kq, tile, g, 8 + 2 * q + 1] = d1 + red[kq, tile, g + 8, 8 + 2 * q] = d2 + red[kq, tile, g + 8, 8 + 2 * q + 1] = d3 + cute.arch.sync_threads() + # 8 threads per row, 6 output rows (2 records) each + if t < rows * (GEMV_ROWS // 6): + m = t // (GEMV_ROWS // 6) + k6 = t % (GEMV_ROWS // 6) + wv = cute.make_rmem_tensor((3,), Uint32) + if pz != 0: # poisoned: NaN records, the partial sums are not read + for p in cutlass.range_constexpr(3): + wv[p] = Uint32(BF16_NAN | (BF16_NAN << 16)) + else: + for p in cutlass.range_constexpr(3): + h0 = self.quarter_sum(red, 6 * k6 + 2 * p, m) + h1 = self.quarter_sum(red, 6 * k6 + 2 * p + 1, m) + wv[p] = P.no_neg_zero(self.round_pair(h0, h1)) + em = es.erow[m] + r0, r1 = self.pack_records(wv[0], wv[1], wv[2], em & Uint32(TAG_MASK)) + if cutlass.const_expr(not self.router): + self.delay(mb, 1) + self.publish_records( + mb, self.buffer_of(em), m, (cta * GEMV_ROWS + 6 * k6) // 3, r0, r1 + ) + + def round_pair(self, h0, h1): + if cutlass.const_expr(self.router): + return C.cvt_rn_bf16x2_f32(h0, h1) + return Uint32(C.cvt_rn_bf16_f32(h0)) | (Uint32(C.cvt_rn_bf16_f32(h1)) << 16) + + @cute.jit + def quarter_sum(self, red, row, m): + """Fixed-order K-quarter sum ``((q0 + q1) + q2) + q3`` of output row ``row``, + token ``m``.""" + tl = row // 16 + rr = row % 16 + v = red[0, tl, rr, m] + v = C.add_rn(v, red[1, tl, rr, m]) + v = C.add_rn(v, red[2, tl, rr, m]) + v = C.add_rn(v, red[3, tl, rr, m]) + return v + + @cute.jit + def router_cta( + self, + mb: Mbox, + base, + h, + x: Int64, + qh: Int64, + ql: Int64, + residual: Int64, + rows: Int32, + rp, + pol, + ): + """Router CTA ``h``: rows ``[16h, 16h + 16)`` of ``[Q_r | W'_slice]`` (hi and + lo) in registers before the wait; the rank's term for 16 experts of every row, + published as ``{fp32, tag32 = E_row[m]}`` words to every rank's router-partial + mailbox.""" + t = P.thread_index() + xs_bytes: cutlass.Constexpr = self.max_rows * RXS_STRIDE * 2 + mx: cutlass.Constexpr = self.max_rows + red = _view(base, xs_bytes, Float32, (12, 16, mx), (16 * mx, mx, 1)) + es = _epoch_view(base, xs_bytes + 12 * 16 * mx * 4, self.max_rows) + warp = t // 32 + lane = t % 32 + g = lane // 4 + q = lane % 4 + s0 = 8 * warp + ns = 8 + if warp >= 4: + s0 = 32 + 7 * (warp - 4) + ns = 7 + ra = Int64(16 * h + g) * ROUTER_K + Int64(8 * q) + rb = ra + Int64(8 * ROUTER_K) + ah = cute.make_rmem_tensor((ROUTER_STEPS, 4), Uint32) + bh = cute.make_rmem_tensor((ROUTER_STEPS, 4), Uint32) + al = cute.make_rmem_tensor((ROUTER_STEPS, 4), Uint32) + bl = cute.make_rmem_tensor((ROUTER_STEPS, 4), Uint32) + ah.fill(0) + bh.fill(0) + al.fill(0) + bl.fill(0) + for i in cutlass.range_constexpr(ROUTER_STEPS): + if i < ns: + k0 = Int64(32 * (s0 + i)) + v0 = sys_ops.ld_nc_hint_v4_u32(qh + Int64(2) * (ra + k0), pol) + v1 = sys_ops.ld_nc_hint_v4_u32(qh + Int64(2) * (rb + k0), pol) + v2 = sys_ops.ld_nc_hint_v4_u32(ql + Int64(2) * (ra + k0), pol) + v3 = sys_ops.ld_nc_hint_v4_u32(ql + Int64(2) * (rb + k0), pol) + for k in cutlass.range_constexpr(4): + ah[i, k] = v0[k] + bh[i, k] = v1[k] + al[i, k] = v2[k] + bl[i, k] = v3[k] + self.pdl_edge(rows) + sys_ops.fence_proxy_async_global() + xs_base = Uint32(base.toint()) + i = t + nchunk = rows * (ROUTER_K // 8) + while i < nchunk: # B' = [x | residual slice of this rank] + mm = i // (ROUTER_K // 8) + kc = i % (ROUTER_K // 8) + src = x + Int64(2) * (Int64(mm) * O_PROJ_IN + Int64(kc * 8)) + if kc >= O_PROJ_IN // 8: + src = residual + Int64(2) * ( + Int64(mm) * HIDDEN + + Int64(mb.rank) * (HIDDEN // WORLD) + + Int64((kc - O_PROJ_IN // 8) * 8) + ) + self.stage16(xs_base + Uint32(2 * (mm * RXS_STRIDE + kc * 8)), src) + i = i + GEMV_THREADS + self.stage_done() + self.epoch_begin(mb, es, True) + hi0 = Float32(0.0) + hi1 = Float32(0.0) + hi2 = Float32(0.0) + hi3 = Float32(0.0) + lo0 = Float32(0.0) + lo1 = Float32(0.0) + lo2 = Float32(0.0) + lo3 = Float32(0.0) + e0 = Float32(0.0) + e1 = Float32(0.0) + e2 = Float32(0.0) + e3 = Float32(0.0) + f0 = Float32(0.0) + f1 = Float32(0.0) + f2 = Float32(0.0) + f3 = Float32(0.0) + tok = g < rows + tok2 = g + 8 < rows + two = rows > 8 + for i in cutlass.range_constexpr(ROUTER_STEPS): + if i < ns: + k0 = 32 * (s0 + i) + 8 * q + x0 = Uint32(0) + x1 = Uint32(0) + x2 = Uint32(0) + x3 = Uint32(0) + if tok: + x0, x1, x2, x3 = P.ld_shared_v4_u32( + xs_base + Uint32(2 * (g * RXS_STRIDE + k0)) + ) + hi0, hi1, hi2, hi3 = P.mma_bf16_m16n8k16( + (hi0, hi1, hi2, hi3), + (ah[i, 0], bh[i, 0], ah[i, 1], bh[i, 1]), + (x0, x1), + ) + hi0, hi1, hi2, hi3 = P.mma_bf16_m16n8k16( + (hi0, hi1, hi2, hi3), + (ah[i, 2], bh[i, 2], ah[i, 3], bh[i, 3]), + (x2, x3), + ) + lo0, lo1, lo2, lo3 = P.mma_bf16_m16n8k16( + (lo0, lo1, lo2, lo3), + (al[i, 0], bl[i, 0], al[i, 1], bl[i, 1]), + (x0, x1), + ) + lo0, lo1, lo2, lo3 = P.mma_bf16_m16n8k16( + (lo0, lo1, lo2, lo3), + (al[i, 2], bl[i, 2], al[i, 3], bl[i, 3]), + (x2, x3), + ) + if cutlass.const_expr(self.two_tiles): # noqa: SIM102 + if two: + y0 = Uint32(0) + y1 = Uint32(0) + y2 = Uint32(0) + y3 = Uint32(0) + if tok2: + y0, y1, y2, y3 = P.ld_shared_v4_u32( + xs_base + Uint32(2 * ((g + 8) * RXS_STRIDE + k0)) + ) + e0, e1, e2, e3 = P.mma_bf16_m16n8k16( + (e0, e1, e2, e3), + (ah[i, 0], bh[i, 0], ah[i, 1], bh[i, 1]), + (y0, y1), + ) + e0, e1, e2, e3 = P.mma_bf16_m16n8k16( + (e0, e1, e2, e3), + (ah[i, 2], bh[i, 2], ah[i, 3], bh[i, 3]), + (y2, y3), + ) + f0, f1, f2, f3 = P.mma_bf16_m16n8k16( + (f0, f1, f2, f3), + (al[i, 0], bl[i, 0], al[i, 1], bl[i, 1]), + (y0, y1), + ) + f0, f1, f2, f3 = P.mma_bf16_m16n8k16( + (f0, f1, f2, f3), + (al[i, 2], bl[i, 2], al[i, 3], bl[i, 3]), + (y2, y3), + ) + red[warp, g, 2 * q] = C.add_rn(hi0, lo0) + red[warp, g, 2 * q + 1] = C.add_rn(hi1, lo1) + red[warp, g + 8, 2 * q] = C.add_rn(hi2, lo2) + red[warp, g + 8, 2 * q + 1] = C.add_rn(hi3, lo3) + if cutlass.const_expr(self.two_tiles): # noqa: SIM102 + if two: + red[warp, g, 8 + 2 * q] = C.add_rn(e0, f0) + red[warp, g, 8 + 2 * q + 1] = C.add_rn(e1, f1) + red[warp, g + 8, 8 + 2 * q] = C.add_rn(e2, f2) + red[warp, g + 8, 8 + 2 * q + 1] = C.add_rn(e3, f3) + cute.arch.sync_threads() + if t < 16 * rows: + ex = t // rows + m = t % rows + v = red[0, ex, m] + for wi in cutlass.range_constexpr(1, 12): + v = C.add_rn(v, red[wi, ex, m]) + if es.pz[0] != 0: + v = P.bits_f32(Uint32(0x7FC00000)) + em = es.erow[m] + word = Uint64(P.f32_bits(v)) | (Uint64(em) << 32) + off = ( + ( + (Int64(self.buffer_of(em)) * WORLD + Int64(mb.rank)) * self.max_rows + + Int64(m) + ) + * EXPERTS + + Int64(16 * h + ex) + ) * 8 + for d in cutlass.range_constexpr(WORLD): + sys_ops.st_relaxed_sys_u64(rp[d] + off, word) + + +class OProjArConsume(Consumer): + """Consumer of an o_proj hop: all-reduce + residual + RMSNorm for up to ``max_rows`` + rows (``router``: + the router logits from the producer's terms and the per-row + top-8 routing). Not cooperative: it spins only on earlier grids and on remote + records; a cluster's shared-memory exchange is co-scheduled by the hardware.""" + + wait_prod = True # start edge: the last arriver waits for the producer grid + + def __init__( + self, + *, + router: bool = False, + residual: bool = True, + norm: bool = True, + max_rows: int = 8, + ): + if residual and not norm: + raise ValueError("residual without RMSNorm is not provided") + if router and not (residual and norm): + raise ValueError("the router consumer is provided with residual + RMSNorm") + if max_rows not in (8, 16) or (max_rows == 16 and not router): + raise ValueError( + "provided for up to 8 rows, or up to 16 rows with the router" + ) + self.router, self.residual, self.norm, self.max_rows = ( + router, + residual, + norm, + max_rows, + ) + + @cute.jit + def __call__( + self, + residual: Int64, + gamma: Int64, + out_res: Int64, + out_norm: Int64, + eps: Float32, + rows: Int32, + rp_local: Int64, + bias: Int64, + rscale: Float32, + ids: Int64, + wts: Int64, + p0: Int64, + p1: Int64, + p2: Int64, + p3: Int64, + p4: Int64, + p5: Int64, + p6: Int64, + p7: Int64, + rank: Int32, + err_host: Int64, + site: Int32, + stream: cuda.CUstream, + ): + """Router: ``rp_local`` this rank's router-partial mailbox, ``bias`` [256] fp32 + score bias, ``rscale`` the routed scaling factor, ``ids`` [rows, 8] int32 / + ``wts`` [rows, 8] fp32 outputs (unused without the router).""" + self.kernel( + residual, + gamma, + out_res, + out_norm, + eps, + rows, + rp_local, + bias, + rscale, + ids, + wts, + p0, + p1, + p2, + p3, + p4, + p5, + p6, + p7, + rank, + err_host, + site, + ).launch( + grid=[rows * 8 + 8, 1, 1], + block=[CONSUMER_THREADS, 1, 1], + max_number_threads=[CONSUMER_THREADS, 1, 1], + cluster=[8, 1, 1], + use_pdl=True, + stream=stream, + ) + + @cute.jit + def entry(self): + """Start edge: no wait at entry (ordered by the producer's fenced trigger).""" + cute.arch.griddepcontrol_launch_dependents() + + @cute.jit + def exit_wait(self): + cute.arch.griddepcontrol_wait() + + @cute.kernel + def kernel( + self, + residual: Int64, + gamma: Int64, + out_res: Int64, + out_norm: Int64, + eps: Float32, + rows: Int32, + rp_local: Int64, + bias: Int64, + rscale: Float32, + ids: Int64, + wts: Int64, + p0: Int64, + p1: Int64, + p2: Int64, + p3: Int64, + p4: Int64, + p5: Int64, + p6: Int64, + p7: Int64, + rank: Int32, + err_host: Int64, + site: Int32, + ): + mb = Mbox((p0, p1, p2, p3, p4, p5, p6, p7), rank, self.max_rows, err_host, site) + t = P.thread_index() + bx = P.block_index() + smem = cutlass.utils.SmemAllocator() + cs = ConsSmem.allocate(smem, self.max_rows) + lg = smem.allocate_tensor(Float32, cute.make_layout(EXPERTS), 16) + scr = smem.allocate_tensor(Float32, cute.make_layout(EXPERTS), 16) + # (footprint parity with the reference layout) + smem.allocate_tensor(Float32, cute.make_layout(1), 4) + nmain = rows * 8 + if cutlass.const_expr(self.norm): # noqa: SIM102 + if bx < nmain: + self.cons_prologue(cs) + self.entry() + # the residual may come from an async-proxy writer + sys_ops.fence_proxy_async_global() + if bx >= nmain: + e = self.epoch_begin(mb, cs.es, False) + if t == 0: + if bx == nmain: + self.ack_cta(mb, e) + self.epoch_arrive(mb, cs.es, rows, P.grid_size(), self.wait_prod) + if cutlass.const_expr(self.wait_prod): + self.exit_wait() + sys_ops.exit_thread() + m = bx // 8 + c = bx % 8 + h0 = c * 768 + 6 * t + bsv = cute.make_rmem_tensor((8,), Float32) + bsv.fill(0.0) + if cutlass.const_expr(self.router): # noqa: SIM102 + # routing warp: the static score bias into registers now + if c == 0 and t < 32: + v = sys_ops.ld_nc_v4_f32(bias + Int64(4 * 8 * t)) + u = sys_ops.ld_nc_v4_f32(bias + Int64(4 * 8 * t + 16)) + for k in cutlass.range_constexpr(4): + bsv[k] = v[k] + bsv[4 + k] = u[k] + ru = cute.make_rmem_tensor((3,), Uint32) + gu = cute.make_rmem_tensor((3,), Uint32) + ru.fill(0) + gu.fill(0) + if t < 128: + if cutlass.const_expr(self.residual): + off = Int64(2) * (Int64(m) * HIDDEN + Int64(h0)) + for k in cutlass.range_constexpr(3): + ru[k] = sys_ops.ld_cg_u32(residual + off + Int64(4 * k)) + if cutlass.const_expr(self.norm): + for k in cutlass.range_constexpr(3): + gu[k] = sys_ops.ld_cg_u32( + gamma + Int64(2) * Int64(h0) + Int64(4 * k) + ) + self.epoch_begin(mb, cs.es, False) + if t >= POLL_THREADS: # the epoch warp arrives, then leaves the CTA + if t == POLL_THREADS: + self.epoch_arrive(mb, cs.es, rows, P.grid_size(), self.wait_prod) + sys_ops.exit_thread() + if cutlass.const_expr(self.router): + self.consume_cta_router( + mb, + cs, + lg, + scr, + m, + c, + cs.es.erow[m], + cs.es.pz[0], + ru, + gu, + bsv, + out_res, + out_norm, + eps, + rp_local, + rscale, + ids, + wts, + ) + else: + self.consume_cta( + mb, + cs, + m, + c, + cs.es.erow[m], + cs.es.pz[0], + ru, + gu, + out_res, + out_norm, + eps, + self.residual, + self.norm, + ) + if cutlass.const_expr(self.wait_prod): # noqa: SIM102 + if t == 0: + self.exit_wait() + + # ------------------------------------------------------------ router consumer + @cute.jit + def consume_cta_router( + self, + mb: Mbox, + cs: ConsSmem, + lg, + scr, + m, + c, + e, + poisoned, + ru, + gu, + bsv, + out_res: Int64, + out_norm: Int64, + eps: Float32, + rp_local: Int64, + rscale: Float32, + ids: Int64, + wts: Int64, + ): + """The consumer epilogue with the shared-memory sum of squares, plus (cluster + rank 0) the router-term gather of threads 128..383 (expert ``t - 128``) and the + routing warp.""" + t = P.thread_index() + f = t % 128 + grp = t // 128 + buf = self.buffer_of(e) + tag = Uint32(e) & Uint32(TAG_MASK) + acc = cute.make_rmem_tensor((6,), Float32) + x0 = cute.make_rmem_tensor((6,), Float32) + x1 = cute.make_rmem_tensor((6,), Float32) + acc.fill(0.0) + self.poll_records(mb, buf, m, c * 256 + 2 * f, 2 * grp, tag, poisoned, x0, x1) + if grp > 0: + for k in cutlass.range_constexpr(6): + cs.raw[grp - 1, f, k] = x0[k] + cs.raw[grp - 1, f, 6 + k] = x1[k] + else: + for k in cutlass.range_constexpr(6): + acc[k] = C.add_rn(C.add_rn(Float32(0.0), x0[k]), x1[k]) + # router terms of (m, expert t - 128): the eight source words are issued now + rtr = (c == 0) and (t >= 128) and (t < 128 + EXPERTS) + ex = t - 128 + pw = cute.make_rmem_tensor((WORLD,), Uint64) + pw.fill(0) + if rtr: + for s in cutlass.range_constexpr(WORLD): + pw[s] = sys_ops.ld_relaxed_sys_u64( + rp_local + self.rp_off(buf, s, m, ex) + ) + P.bar_sync(1, 512) + hsum = Float32(0.0) + hto = Int32(0) + if rtr: # tag check + rank-order sum while group 0 finalizes the all-reduce + hsum, hto = self.router_gather(mb, pw, rp_local, buf, m, ex, e) + r = cute.make_rmem_tensor((6,), Float32) + r.fill(0.0) + h0 = c * 768 + 6 * f + if grp == 0: + for q in cutlass.range_constexpr(1, 4): + for k in cutlass.range_constexpr(6): + acc[k] = C.add_rn( + C.add_rn(acc[k], cs.raw[q - 1, f, k]), cs.raw[q - 1, f, 6 + k] + ) + rb = cute.make_rmem_tensor((3,), Uint32) + for k in cutlass.range_constexpr(3): + ab = P.no_neg_zero(C.cvt_rn_bf16x2_f32(acc[2 * k], acc[2 * k + 1])) + rb[k] = C.cvt_rn_bf16x2_f32( + C.add_rn(P.bf16_lo(ab), P.bf16_lo(ru[k])), + C.add_rn(P.bf16_hi(ab), P.bf16_hi(ru[k])), + ) + row = out_res + Int64(2) * (Int64(m) * HIDDEN + Int64(h0)) + for k in cutlass.range_constexpr(3): + sys_ops.st_global_u32(row + Int64(4 * k), rb[k]) + for k in cutlass.range_constexpr(3): + r[2 * k] = P.bf16_lo(rb[k]) + r[2 * k + 1] = P.bf16_hi(rb[k]) + for k in cutlass.range_constexpr(6): + cs.rs[6 * f + k] = r[k] + lane = t % 32 + warp = t // 32 + P.bar_sync(1, 512) + # 96 chunks of 8 elements: from 0 in element order, bf16-rounded squares + if t < 96: + ts = Float32(0.0) + for i in cutlass.range_constexpr(4): + a0 = cs.rs[8 * t + 2 * i] + a1 = cs.rs[8 * t + 2 * i + 1] + wq = C.cvt_rn_bf16x2_f32(C.mul_rn(a0, a0), C.mul_rn(a1, a1)) + ts = C.add_rn(C.add_rn(ts, P.bf16_lo(wq)), P.bf16_hi(wq)) + for i in cutlass.range_constexpr(5): + o: cutlass.Constexpr = 16 >> i + ts = C.add_rn(ts, P.shfl_xor(ts, o)) + if lane == 0: + cs.wsum[warp] = ts + P.bar_sync(1, 512) + bs = Float32(0.0) + if lane < 3: + bs = cs.wsum[lane] + for i in cutlass.range_constexpr(5): + o2: cutlass.Constexpr = 16 >> i + bs = C.add_rn(bs, P.shfl_xor(bs, o2)) + crank = Int32(cute.arch.block_idx_in_cluster()) + P.cluster_wait() + if t == 0: + la = Uint32((cs.csum.iterator + crank).toint()) + lm = cs.mbar_addr() + for cc in cutlass.range_constexpr(8): + P.st_async_b32(P.mapa_u32(la, cc), P.f32_bits(bs), P.mapa_u32(lm, cc)) + P.mbar_wait_cluster(cs.mbar_addr(), Uint32(0)) + P.cluster_arrive_relaxed() + tot = Float32(0.0) + for cc in cutlass.range_constexpr(8): + tot = C.add_rn(tot, cs.csum[cc]) + inv = C.rsqrt_approx(C.fma_rn(tot, C.rcp_rn(Float32(6144.0)), eps)) + if rtr: + if hto != 0: + self.fail(mb, ERR_ROUTER_POLL, 1, 0, m, e) + lg[ex] = C.mul_rn(hsum, inv) + if grp == 0: + row = out_norm + Int64(2) * (Int64(m) * HIDDEN + Int64(h0)) + for k in cutlass.range_constexpr(3): + g0 = C.add_rn(Float32(0.0), P.bf16_lo(gu[k])) + g1 = C.add_rn(Float32(0.0), P.bf16_hi(gu[k])) + y = C.cvt_rn_bf16x2_f32( + C.mul_rn(C.mul_rn(r[2 * k], inv), g0), + C.mul_rn(C.mul_rn(r[2 * k + 1], inv), g1), + ) + sys_ops.st_global_u32(row + Int64(4 * k), y) + sys_ops.fence_proxy_async_global() + if c == 0 and t < 128 + EXPERTS: + P.bar_sync(4, 128 + EXPERTS) # logits in shared memory, norm_out stored + if warp == 0: + self.route_row(lg, scr, bsv, lane, m, poisoned, rscale, ids, wts) + elif grp == 0: + P.bar_sync(2, 128) + P.cluster_wait() # exit barrier, wait + + def rp_off(self, buf, src, m, ex): + return ( + ((Int64(buf) * WORLD + Int64(src)) * self.max_rows + Int64(m)) * EXPERTS + + Int64(ex) + ) * 8 + + @cute.jit + def router_gather(self, mb: Mbox, pw, rp_local: Int64, buf, m, ex, e): + """Poll the eight words until each carries ``tag32 = e``; their fp32 halves + summed in rank order. Returns (sum, timed out).""" + t0 = Uint64(0) + it = Uint32(0) + hto = Int32(0) + for s in cutlass.range_constexpr(WORLD): + while (Uint32(pw[s] >> 32) != Uint32(e)) and (hto == 0): + pw[s] = sys_ops.ld_relaxed_sys_u64( + rp_local + self.rp_off(buf, s, m, ex) + ) + it = it + Uint32(1) + if (it & Uint32(POLL_TIMER_STRIDE - 1)) == Uint32(0): + now = sys_ops.globaltimer() + if t0 == Uint64(0): + t0 = now + elif now - t0 > Uint64(self.timeout_ns): + hto = Int32(1) + hsum = P.bits_f32(Uint32(pw[0] & Uint64(0xFFFFFFFF))) + for s in cutlass.range_constexpr(1, WORLD): + hsum = C.add_rn(hsum, P.bits_f32(Uint32(pw[s] & Uint64(0xFFFFFFFF)))) + return hsum, hto + + @cute.jit + def route_row( + self, lg, scr, bsv, lane, m, poisoned, rscale: Float32, ids: Int64, wts: Int64 + ): + """One warp routes row ``m``: score = sigmoid(logit); key = score + bias (NaN -> + -inf, -0 -> +0) mapped order-preserving to u32; top-8 by (key desc, expert id + asc); weights ``bf16(rscale * s / (sum of the 8 s + 1e-20))``.""" + sc = cute.make_rmem_tensor((8,), Float32) + key = cute.make_rmem_tensor((8,), Uint32) + for jj in cutlass.range_constexpr(8): + sc[jj] = P.routing_sigmoid(lg[8 * lane + jj]) + u = P.f32_bits(P.add_ftz(sc[jj], bsv[jj])) + if (u & Uint32(0x7FFFFFFF)) > Uint32(0x7F800000): + u = Uint32(0xFF800000) # NaN -> -inf + if (u & Uint32(0x7FFFFFFF)) == Uint32(0): + u = Uint32(0) # -0 -> +0 + if (u & Uint32(0x80000000)) != Uint32(0): + key[jj] = ~u + else: + key[jj] = u | Uint32(0x80000000) + myid, ok = self.top8_fast(key, lane) + if ok == 0: + myid = self.top8_merge(key, lane) + for jj in cutlass.range_constexpr(8): + scr[8 * lane + jj] = sc[jj] + cute.arch.sync_warp() + myw = Float32(0.0) + if lane < TOPK: + myw = scr[myid] + wsum = myw + for i in cutlass.range_constexpr(3): + o: cutlass.Constexpr = 4 >> i + wsum = P.add_ftz(wsum, P.shfl_xor(wsum, o)) + if lane < TOPK: + wn = P.routing_weight(rscale, myw, wsum) + if poisoned != 0: + wn = P.bits_f32(Uint32(0x7FFFFFFF)) + sys_ops.st_global_u32(ids + Int64(4 * (m * TOPK + lane)), Uint32(myid)) + sys_ops.st_global_u32(wts + Int64(4 * (m * TOPK + lane)), P.f32_bits(wn)) + cute.arch.sync_warp() + + @cute.jit + def top8_fast(self, key, lane): + """Composite ``c = (key & ~0xff) | (255 - id)``: one unsigned max orders by (key + >> 8 desc, id asc). Per-lane sort, then 9 rounds of one warp max over the lane + heads. Exact iff the top-9 composites have pairwise distinct ``key >> 8`` + between neighbours (else the caller runs the merge). Returns (id of rank + ``lane`` for lanes < 8, exact).""" + s = cute.make_rmem_tensor((8,), Uint32) + for j in cutlass.range_constexpr(8): + s[j] = (key[j] & Uint32(0xFFFFFF00)) | Uint32(255 - 8 * lane - j) + for a, b in ( + (0, 1), + (2, 3), + (4, 5), + (6, 7), + (0, 2), + (1, 3), + (4, 6), + (5, 7), + (1, 2), + (5, 6), + (0, 4), + (1, 5), + (2, 6), + (3, 7), + (2, 4), + (3, 5), + (1, 2), + (3, 4), + (5, 6), + ): + hi = P.umax(s[a], s[b]) + lo = P.umin(s[a], s[b]) + s[a] = hi + s[b] = lo + prev = Uint32(0) + mine = Uint32(0) + ok = Int32(1) + for rr in cutlass.range_constexpr(TOPK + 1): + mx = P.redux_max_u32(s[0]) + if cutlass.const_expr(rr > 0): # noqa: SIM102 + if (prev >> 8) == (mx >> 8): + ok = Int32(0) + prev = mx + if cutlass.const_expr(rr < TOPK): + if lane == rr: + mine = mx + if s[0] == mx: + for i in cutlass.range_constexpr(7): + s[i] = s[i + 1] + s[7] = Uint32(0) + return Int32(255) - Int32(mine & Uint32(0xFF)), ok + + @cute.jit + def top8_merge(self, key, lane): + """Exact fallback: each lane sorts its 8 (key, j) pairs (stable insertion + network), then 8 rounds of a 32-way merge: max key, then min expert id among + equal keys.""" + sk = cute.make_rmem_tensor((8,), Uint32) + sj = cute.make_rmem_tensor((8,), Int32) + for jj in cutlass.range_constexpr(8): + sk[jj] = key[jj] + sj[jj] = Int32(jj) + for i1 in cutlass.range_constexpr(1, 8): + for i2 in cutlass.range_constexpr(i1, 0, -1): + ka = sk[i2 - 1] + kb = sk[i2] + ja = sj[i2 - 1] + jb = sj[i2] + if sk[i2] > sk[i2 - 1]: + ka = sk[i2] + kb = sk[i2 - 1] + ja = sj[i2] + jb = sj[i2 - 1] + sk[i2 - 1] = ka + sk[i2] = kb + sj[i2 - 1] = ja + sj[i2] = jb + left = Int32(8) + myid = Int32(0) + for rr in cutlass.range_constexpr(TOPK): + hk = Uint32(0) + if left > 0: + hk = sk[0] + kmax = P.redux_max_u32(hk) + cand = Uint32(0xFFFFFFFF) + if left > 0 and hk == kmax: + cand = Uint32(8 * lane) + Uint32(sj[0]) + win = Int32(P.redux_min_u32(cand)) + if lane == (win >> 3): + for i in cutlass.range_constexpr(7): + sk[i] = sk[i + 1] + sj[i] = sj[i + 1] + left = left - 1 + if lane == rr: + myid = win + return myid diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/transport.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/transport.py new file mode 100644 index 00000000..8bed3ec9 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/transport.py @@ -0,0 +1,798 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Epoch-tagged mailbox transport of the tensor-parallel hop kernels. + +Protocol (one-shot push all-reduce, unicast only, no re-arm): + +* Records. A row of ``HIDDEN`` bf16 values is ``NREC`` 64-bit records ``{bf16[3j], + bf16[3j + 1], bf16[3j + 2], tag}``. A producer writes each record with one + ``st.relaxed.sys`` (element of a ``.v2.b64`` vector) into every rank's copy; a + consumer reads its local copy with ``ld.relaxed.sys`` and accepts a record only if its + tag equals the current row epoch modulo 2^16. Each 64-bit element of a vector access + is a single-copy-atomic scalar access (PTX ISA, vector data types), so an accepted + record is exactly the one written in that epoch. +* Epochs. A per-rank global epoch ``E`` advances once per hop; per-row epochs + ``E_row[m]`` advance only in hops with more than ``m`` rows and select the buffer slot + (``E_row[m] mod NBUF``) and the tag, so tags never alias whatever the row count does. + Every hop kernel reads them after ``griddepcontrol.wait``; the last CTA to arrive + (``atom.acq_rel.gpu`` counter) stores the new row epochs (``st.relaxed.gpu``) and then + ``E + 1`` (``st.release.gpu``). +* Buffer reuse. A producer may write slot ``e mod NBUF`` only after acquiring + (``ld.acquire.sys``, then a CTA barrier) an acknowledgement ``ack[d] >= e - NBUF`` + from every rank ``d``. Every ``KACK``-th epoch one thread of an extra CTA releases + ``fence.acq_rel.sys`` + ``st.relaxed.sys ack[rank] = e - 1`` to every rank. +* Bounded waits. Every cross-GPU wait reads ``%globaltimer`` once per 16 polls; past its + bound it sets the local sticky error word, writes an error record to host-mapped + memory and executes ``trap``. If the error word is already set when a hop starts, the + hop publishes NaN and uses a short bound (poisoned lock step), so a failure stops + every rank instead of hanging. + +The device functions take an ``Mbox`` view: the eight peer bases rotated so that index 0 +is this rank's own mailbox (stores are staggered starting from it), this rank, and the +record-area size of the instance. +""" + +from dataclasses import dataclass +from typing import Any + +import cutlass +import cutlass.cute as cute +import torch +import torch.distributed as dist +from cutlass import Float32, Int32, Int64, Uint32, Uint64 + +from ..primitives import ptx_core as C +from ..primitives import ptx_sys as sys_ops +from . import _ptx_local as P +from .constants import ( + CTRL_BYTES, + ERR_ACK_WAIT, + ERR_KIND_NAMES, + ERR_RECORD_POLL, + HIDDEN, + KACK, + NBUF, + NREC, + POISONED_TIMEOUT_NS, + POLL_TIMER_STRIDE, + TAG_MASK, + TIMEOUT_NS, + W_ACK, + W_ARRIVE, + W_E, + W_EROW, + W_ERR, + WORLD, + mailbox_bytes, + record_bytes, +) + +# host-mapped error record: [magic, kind, site, rank | peer << 8 | row << 16, epoch, +# ...] +ERR_RECORD_WORDS = 8 +ERR_MAGIC = 0x48455252 + + +@dataclass(frozen=True) +class Mbox: + """Device-side view of one mailbox instance (built inside a kernel).""" + + peers: tuple # 8 x Int64, rotated: peers[i] = base of rank (rank + i) mod 8 + rank: Int32 + max_rows: int # rows of the instance (Constexpr) + err_host: Int64 # host-mapped error record (device address) + site: Int32 # caller's site id, copied into the error record + + @property + def rec_bytes(self) -> int: + return record_bytes(self.max_rows) + + def ctrl(self, word): + """Address of control word ``word`` of this rank's own mailbox.""" + return self.peers[0] + Int64(self.rec_bytes + 4 * word) + + def rec_off(self, buf, src, m, j): + return ( + ((Int64(buf) * WORLD + Int64(src)) * self.max_rows + Int64(m)) * NREC + + Int64(j) + ) * 8 + + +@dataclass(frozen=True) +class EpochSmem: + """Shared-memory epoch snapshot of one CTA: ``e``, the poison flag and the row + epochs (``max_rows`` words).""" + + e: Any + pz: Any + erow: Any + + @staticmethod + def allocate(smem, max_rows): + return EpochSmem( + smem.allocate_tensor(Uint32, cute.make_layout(1), 4), + smem.allocate_tensor(Int32, cute.make_layout(1), 4), + smem.allocate_tensor(Uint32, cute.make_layout(max_rows), 4), + ) + + +class Transport: + """Protocol device functions. Every protocol decision point is its own method so + that a test subclass can override it (fault injection); ``delay`` is empty in the + served classes.""" + + timeout_ns = TIMEOUT_NS + poisoned_timeout_ns = POISONED_TIMEOUT_NS + + # ------------------------------------------------------- decision points + @cute.jit + def delay(self, mb: Mbox, site: cutlass.Constexpr[int]): + pass + + def tag_ok(self, rec, tag): + return (Uint64(rec) >> 48) == Uint64(tag) + + def buffer_of(self, e): + return Uint32(e) % Uint32(NBUF) + + def ack_value(self, e): + return Uint32(e) - Uint32(1) + + def check_ack_enabled(self) -> bool: + return True + + def release_ack_enabled(self) -> bool: + return True + + @cute.jit + def release_fence(self): + sys_ops.fence_acq_rel_sys() + + # ------------------------------------------------------------- failure + @cute.jit + def fail( + self, + mb: Mbox, + kind: cutlass.Constexpr[int], + code: cutlass.Constexpr[int], + peer, + row, + epoch, + ): + """Sticky error word, the host-mapped error record, then ``trap``. ``row``: the + row the timed-out poll was reading (0 for an acknowledgement wait); ``epoch``: + the awaited epoch (a record poll stores its 16-bit tag).""" + sys_ops.atom_or_b32(mb.ctrl(W_ERR), Uint32(code)) + meta = Uint32(mb.rank) | (Uint32(peer) << 8) | (Uint32(row) << 16) + sys_ops.st_relaxed_sys_v4_u32( + mb.err_host, Uint32(ERR_MAGIC), Uint32(kind), Uint32(mb.site), meta + ) + sys_ops.st_relaxed_sys_u32(mb.err_host + Int64(16), Uint32(epoch)) + sys_ops.fence_sc_sys() + sys_ops.trap() + + # --------------------------------------------------------------- epochs + @cute.jit + def epoch_begin( + self, + mb: Mbox, + es: EpochSmem, + check: cutlass.Constexpr[bool], + rt0: cutlass.Constexpr[int] = 0, + ) -> Uint32: + """Every thread, after ``griddepcontrol.wait``. Threads ``rt0 .. rt0 + 7`` read + ``E`` and one row epoch each (rows ``max_rows`` > 8 use threads ``rt0 .. rt0 + + max_rows - 1``), thread ``rt0`` the error word, and (``check``) acquire the + acknowledgement of rank ``t - rt0``; the CTA barrier publishes the snapshot.""" + t = P.thread_index() - rt0 + nread: cutlass.Constexpr = max(WORLD, mb.max_rows) + if t >= 0 and t < nread: + e = sys_ops.ld_relaxed_gpu_u32(mb.ctrl(W_E)) + er = Uint32(0) + if t < mb.max_rows: + er = sys_ops.ld_relaxed_gpu_u32(mb.ctrl(W_EROW) + Int64(4) * Int64(t)) + pz = Uint32(0) + if t == 0: + pz = sys_ops.ld_relaxed_gpu_u32(mb.ctrl(W_ERR)) + a = Uint32(0) + if cutlass.const_expr(check and self.check_ack_enabled()): # noqa: SIM102 + if t < WORLD: + a = sys_ops.ld_acquire_sys_u32( + mb.ctrl(W_ACK) + Int64(4 * 32) * Int64(t) + ) + if t < mb.max_rows: + es.erow[t] = er + if t == 0: + es.e[0] = e + es.pz[0] = Int32(pz) + if cutlass.const_expr(check and self.check_ack_enabled()): # noqa: SIM102 + if t < WORLD: + need = e - Uint32(NBUF) + if Int32(a - need) < 0: + self.delay(mb, 5) + t0 = Uint64(0) + it = Uint32(0) + while Int32(a - need) < 0: + a = sys_ops.ld_acquire_sys_u32( + mb.ctrl(W_ACK) + Int64(4 * 32) * Int64(t) + ) + it = it + Uint32(1) + if (it & Uint32(POLL_TIMER_STRIDE - 1)) == Uint32(0): + now = sys_ops.globaltimer() + if t0 == Uint64(0): + t0 = now + elif now - t0 > Uint64(self.timeout_ns): + self.fail(mb, ERR_ACK_WAIT, 2, t, 0, e) + cute.arch.sync_threads() + return es.e[0] + + @cute.jit + def epoch_arrive( + self, + mb: Mbox, + es: EpochSmem, + rows, + total, + wait_prod: cutlass.Constexpr[bool] = False, + ): + """One thread per CTA of an epoch-advancing hop, after the barrier that ordered + this CTA's reads of ``E`` / ``E_row``. The last arriver (of ``total`` CTAs) + stores the next row epochs of the ``rows`` rows of this hop, then ``E + 1``. + ``wait_prod``: the consumer of a fenced-trigger o_proj hop may start before a + slow producer CTA has read ``E``, so the last arriver first waits for the + producer grid (``griddepcontrol.wait``).""" + old = sys_ops.atom_add_acq_rel_gpu_u32(mb.ctrl(W_ARRIVE), Uint32(1)) + if old == Uint32(total) - Uint32(1): + if cutlass.const_expr(wait_prod): + self.wait_producer() + sys_ops.st_relaxed_gpu_u32(mb.ctrl(W_ARRIVE), Uint32(0)) + for m in range(rows): + sys_ops.st_relaxed_gpu_u32( + mb.ctrl(W_EROW) + Int64(4) * Int64(m), es.erow[m] + Uint32(1) + ) + sys_ops.st_release_gpu_u32(mb.ctrl(W_E), es.e[0] + Uint32(1)) + + @cute.jit + def wait_producer(self): + cute.arch.griddepcontrol_wait() + + @cute.jit + def ack_cta(self, mb: Mbox, e: Uint32): + """Thread 0 of the acknowledgement CTA: every ``KACK``-th epoch, release "every + read of epochs <= e - 1 on this rank is done" to every rank.""" + if cutlass.const_expr(self.release_ack_enabled()): # noqa: SIM102 + if P.thread_index() == 0: # noqa: SIM102 + if (e % Uint32(KACK)) == Uint32(0): + self.delay(mb, 4) + self.release_fence() + a = self.ack_value(e) + for i in cutlass.range_constexpr(WORLD): + addr = ( + mb.peers[i] + + Int64(mb.rec_bytes) + + Int64(4) * (Int64(W_ACK) + Int64(32) * Int64(mb.rank)) + ) + sys_ops.st_relaxed_sys_u32(addr, a) + + # ------------------------------------------------------------ records + @cute.jit + def pack_records(self, w0: Uint32, w1: Uint32, w2: Uint32, tag: Uint32): + """Six bf16 (three little-endian pairs) -> two records carrying ``tag``.""" + t48 = Uint64(tag) << 48 + r0 = Uint64(w0) | (Uint64(w1 & Uint32(0xFFFF)) << 32) | t48 + r1 = Uint64(w1 >> 16) | (Uint64(w2) << 16) | t48 + return r0, r1 + + @cute.jit + def publish_records(self, mb: Mbox, buf, m, j, r0: Uint64, r1: Uint64): + """Record pair ``j`` of row ``m`` into slot ``buf`` of every rank, starting with + this rank's own copy (destinations staggered across ranks).""" + off = mb.rec_off(buf, mb.rank, m, j) + for i in cutlass.range_constexpr(WORLD): + sys_ops.st_relaxed_sys_v2_b64(mb.peers[i] + off, r0, r1) + + @cute.jit + def poll_records(self, mb: Mbox, buf, m, j, s0, tag: Uint32, poisoned, x0, x1): + """Poll record pair ``j`` of row ``m`` from sources ``s0`` and ``s0 + 1`` in the + local slot ``buf`` until all four records carry ``tag``; unpack the six values + of each source into ``x0`` / ``x1`` (fp32 register tensors). Poisoned: NaN.""" + base = mb.peers[0] + a0 = base + mb.rec_off(buf, s0, m, j) + a1 = base + mb.rec_off(buf, s0 + 1, m, j) + r00, r01 = sys_ops.ld_relaxed_sys_v2_b64(a0) + r10, r11 = sys_ops.ld_relaxed_sys_v2_b64(a1) + pending = Uint32(0) + if not (self.tag_ok(r00, tag) and self.tag_ok(r01, tag)): + pending = pending | Uint32(1) + if not (self.tag_ok(r10, tag) and self.tag_ok(r11, tag)): + pending = pending | Uint32(2) + t0 = Uint64(0) + it = Uint32(0) + lim = Uint64(self.timeout_ns) + if poisoned != 0 and lim > Uint64(self.poisoned_timeout_ns): + lim = Uint64(self.poisoned_timeout_ns) + while pending != Uint32(0): + if (pending & Uint32(1)) != Uint32(0): + r00, r01 = sys_ops.ld_relaxed_sys_v2_b64(a0) + if self.tag_ok(r00, tag) and self.tag_ok(r01, tag): + pending = pending & Uint32(2) + if (pending & Uint32(2)) != Uint32(0): + r10, r11 = sys_ops.ld_relaxed_sys_v2_b64(a1) + if self.tag_ok(r10, tag) and self.tag_ok(r11, tag): + pending = pending & Uint32(1) + if pending != Uint32(0): + it = it + Uint32(1) + if (it & Uint32(POLL_TIMER_STRIDE - 1)) == Uint32(0): + now = sys_ops.globaltimer() + if t0 == Uint64(0): + t0 = now + elif now - t0 > lim: + peer = Int32(s0) + Int32((pending & Uint32(1)) == Uint32(0)) + self.fail(mb, ERR_RECORD_POLL, 1, peer, m, tag) + if poisoned != 0: + for k in cutlass.range_constexpr(6): + x0[k] = P.bits_f32(Uint32(0x7FC00000)) + x1[k] = P.bits_f32(Uint32(0x7FC00000)) + else: + self.unpack_records(r00, r01, x0) + self.unpack_records(r10, r11, x1) + + @cute.jit + def unpack_records(self, r0: Uint64, r1: Uint64, x): + x[0] = P.bits_f32(Uint32(r0 & Uint64(0xFFFF)) << 16) + x[1] = P.bits_f32(Uint32((r0 >> 16) & Uint64(0xFFFF)) << 16) + x[2] = P.bits_f32(Uint32((r0 >> 32) & Uint64(0xFFFF)) << 16) + x[3] = P.bits_f32(Uint32(r1 & Uint64(0xFFFF)) << 16) + x[4] = P.bits_f32(Uint32((r1 >> 16) & Uint64(0xFFFF)) << 16) + x[5] = P.bits_f32(Uint32((r1 >> 32) & Uint64(0xFFFF)) << 16) + + +# ====================================================================== host side +class ErrorRecord: + """Host-mapped pinned words written by a timed-out wait just before its ``trap``, so + the host can report which wait failed after the CUDA context is lost.""" + + def __init__(self, device): + from cuda.bindings import driver as cu + + self._cu = cu + flags = cu.CU_MEMHOSTALLOC_DEVICEMAP | cu.CU_MEMHOSTALLOC_PORTABLE + err, host = cu.cuMemHostAlloc(4 * ERR_RECORD_WORDS, flags) + _check(cu, err, "cuMemHostAlloc") + self.host_ptr = int(host) + err, dptr = cu.cuMemHostGetDevicePointer(self.host_ptr, 0) + _check(cu, err, "cuMemHostGetDevicePointer") + self.device_ptr = int(dptr) + self.clear() + + def _view(self): + import ctypes + + return (ctypes.c_uint32 * ERR_RECORD_WORDS).from_address(self.host_ptr) + + def clear(self): + v = self._view() + for i in range(ERR_RECORD_WORDS): + v[i] = 0 + + def read(self): + """None if no wait timed out, else the decoded record: ``kind``, ``site``, the + waiting ``rank``, the ``peer`` it waited for, the ``row`` it was polling and the + awaited ``epoch`` (see ``Transport.fail``).""" + v = list(self._view()) + if v[0] != ERR_MAGIC: + return None + return { + "kind": ERR_KIND_NAMES.get(v[1], v[1]), + "site": v[2], + "rank": v[3] & 0xFF, + "peer": (v[3] >> 8) & 0xFF, + "row": v[3] >> 16, + "epoch": v[4], + } + + +def _check(cu, err, what): + if err != cu.CUresult.CUDA_SUCCESS: + raise RuntimeError(f"{what} failed: {err}") + + +class HopMailbox: + """One symmetric mailbox instance (records for ``max_rows`` rows + control block) on + every rank of a tensor-parallel group. Collective: every rank constructs it.""" + + def __init__(self, group_name: str, max_rows: int, device, start: int = -1): + import torch.distributed._symmetric_memory as symm_mem + + self.max_rows = max_rows + self.device = torch.device(device) + self.nbytes = mailbox_bytes(max_rows) + # symmetric allocations need not be page aligned: over-allocate and use the 4 + # KiB + # aligned window of every rank's buffer (each rank's offset follows from its own + # base) + self.storage = symm_mem.empty( + self.nbytes + ALIGN, dtype=torch.uint8, device=self.device + ) + handle = symm_mem.rendezvous(self.storage, group_name) + self.rank = int(handle.rank) + if int(handle.world_size) != WORLD: + raise ValueError( + f"hop mailbox needs {WORLD} ranks, got {handle.world_size}" + ) + self.peers = [_aligned(int(p)) for p in handle.buffer_ptrs] + off = self.peers[self.rank] - self.storage.data_ptr() + self.tensor = self.storage[off : off + self.nbytes] + self.group_name = group_name + get_backend = getattr(symm_mem, "get_backend", None) + self.backend = get_backend(self.device) if get_backend is not None else None + self.init(start) + + @classmethod + def create(cls, group_name: str, max_rows: int, device=None, start: int = -1): + return cls( + group_name, + max_rows, + device or torch.accelerator.current_accelerator(), + start, + ) + + def rotated_peers(self): + return [self.peers[(self.rank + i) % WORLD] for i in range(WORLD)] + + def init(self, start: int = -1, before_last_barrier=None): + """Records zero (tag 0), acknowledgements ``start - 1``, epochs ``start`` (NBUF: + the first use of every slot has a non-zero tag), error / arrival words 0. + Collective (quiesce, init, quiesce); ``before_last_barrier`` re-initializes + state that must restart with this mailbox (the router-partial words).""" + start = NBUF if start < 0 else start + torch.accelerator.synchronize() + dist.barrier(group=_group(self.group_name)) + _init_mailbox(self.tensor, self.max_rows, start) + if before_last_barrier is not None: + before_last_barrier() + torch.accelerator.synchronize() + dist.barrier(group=_group(self.group_name)) + + +ALIGN = 4096 + + +def _aligned(ptr: int) -> int: + return ptr + (-ptr) % ALIGN + + +def _group(group_name): + from torch.distributed.distributed_c10d import _resolve_process_group + + return _resolve_process_group(group_name) + + +def _init_mailbox(tensor: torch.Tensor, max_rows: int, start: int): + """Mailbox image: written with torch ops (stream-ordered), byte-equal to the + reference init (records 0, control block 0 except ack / E / E_row).""" + rb = record_bytes(max_rows) + tensor.zero_() + ctrl = tensor[rb : rb + CTRL_BYTES].view(torch.int32) + s = start & 0xFFFFFFFF + sm1 = (start - 1) & 0xFFFFFFFF + as_i32 = lambda v: v - (1 << 32) if v >= (1 << 31) else v # noqa: E731 + for src in range(WORLD): + ctrl[W_ACK + 32 * src] = as_i32(sm1) + ctrl[W_E] = as_i32(s) + for m in range(max_rows): + ctrl[W_EROW + m] = as_i32(s) + + +# ================================================================== consumer side +@dataclass(frozen=True) +class ConsSmem: + """Shared memory of one consumer CTA (row ``m``, cluster rank ``c``).""" + + # groups 1..3: both sources' raw values (group 0 sums every rank in order) + raw: Any + # the 96 per-chunk sums of squares (768 words: the shared-memory form's footprint) + rs: Any + wsum: Any + csum: Any + mbar: Any + es: EpochSmem + + @staticmethod + def allocate(smem, max_rows): + f32 = cutlass.Float32 + return ConsSmem( + smem.allocate_tensor( + f32, cute.make_layout((3, 128, 12), stride=(128 * 12, 12, 1)), 16 + ), + smem.allocate_tensor(f32, cute.make_layout(768), 16), + smem.allocate_tensor(f32, cute.make_layout(17), 4), + smem.allocate_tensor(f32, cute.make_layout(8), 4), + smem.allocate_tensor(cutlass.Int64, cute.make_layout(1), 8), + EpochSmem.allocate(smem, max_rows), + ) + + def mbar_addr(self): + return Uint32(self.mbar.iterator.toint()) + + +class Consumer(Transport): + """Consumer epilogue of a hop: all-reduce in rank order (+ residual) (+ RMSNorm), + numerically the one-shot fused all-reduce + residual + RMSNorm order: fp32 sum from + +0 over ranks 0..7, bf16 with -0 made +0, bf16 residual add, per-8-element chunk + sums of bf16-rounded squares, xor butterflies, block reduce ``(w0 + w2) + w1``, + cluster-rank-order total, ``rsqrt(fma(total, rcp(HIDDEN), eps))``, ``y = bf16((x * + inv) * (0 + g))``.""" + + @cute.jit + def cons_prologue(self, cs: ConsSmem): + """Initialize the mbarrier, then cluster arrive (before cluster accesses).""" + t = P.thread_index() + if t == 0: + P.mbar_init(cs.mbar_addr(), 1) + P.fence_mbarrier_init_release_cluster() + cute.arch.sync_threads() + # the cluster-scope release arrive is issued by the idle epoch-warp thread + if t == 512: + P.mbar_arrive_expect_tx_release_cluster(cs.mbar_addr(), 8 * 4) + P.cluster_arrive_relaxed() + + def block_reduce(self, v0, v1, v2): + """Block sum of the three warp sums: ``(w0 + w2) + w1``.""" + return C.add_rn(C.add_rn(v0, v2), v1) + + def chunk_carry(self, run): + return run + + @cute.jit + def consume_cta( + self, + mb: Mbox, + cs: ConsSmem, + m, + c, + e, + poisoned, + ru, + gu, + out_res: Int64, + out_norm: Int64, + eps: Float32, + res: cutlass.Constexpr[bool], + norm: cutlass.Constexpr[bool], + dn_bsum=None, + ): + """The 512 poll threads of consumer CTA (row ``m``, cluster rank ``c``); thread + ``f + 128 g`` polls record pair ``f`` from sources ``2g`` and ``2g + 1``; group + 0 finalizes six elements per thread. + + With ``dn_bsum`` (the deferred-norm form, ``norm`` set): the CTA writes + ``bf16(x * (0 + g))`` to ``out_norm`` and its block sum of squares (the norm's + own value and order) to ``dn_bsum[m][c]`` instead of normalizing; the block sums + are not exchanged in the cluster, and the row's scale is applied downstream.""" + t = P.thread_index() + f = t % 128 + grp = t // 128 + buf = self.buffer_of(e) + tag = Uint32(e) & Uint32(TAG_MASK) + j = c * 256 + 2 * f + acc = cute.make_rmem_tensor((6,), Float32) + x0 = cute.make_rmem_tensor((6,), Float32) + x1 = cute.make_rmem_tensor((6,), Float32) + acc.fill(0.0) + self.delay(mb, 2) + self.poll_records(mb, buf, m, j, 2 * grp, tag, poisoned, x0, x1) + if grp > 0: + for k in cutlass.range_constexpr(6): + cs.raw[grp - 1, f, k] = x0[k] + cs.raw[grp - 1, f, 6 + k] = x1[k] + else: + for k in cutlass.range_constexpr(6): + acc[k] = C.add_rn(C.add_rn(Float32(0.0), x0[k]), x1[k]) + P.bar_sync(1, 512) + r = cute.make_rmem_tensor((6,), Float32) + r.fill(0.0) + h0 = c * 768 + 6 * f + if grp == 0: + for q in cutlass.range_constexpr(1, 4): + for k in cutlass.range_constexpr(6): + acc[k] = C.add_rn( + C.add_rn(acc[k], cs.raw[q - 1, f, k]), cs.raw[q - 1, f, 6 + k] + ) + rb = cute.make_rmem_tensor((3,), Uint32) + for k in cutlass.range_constexpr(3): + ab = P.no_neg_zero(C.cvt_rn_bf16x2_f32(acc[2 * k], acc[2 * k + 1])) + if cutlass.const_expr(res): + rb[k] = C.cvt_rn_bf16x2_f32( + C.add_rn(P.bf16_lo(ab), P.bf16_lo(ru[k])), + C.add_rn(P.bf16_hi(ab), P.bf16_hi(ru[k])), + ) + else: + rb[k] = ab + row = out_res + Int64(2) * (Int64(m) * HIDDEN + Int64(h0)) + for k in cutlass.range_constexpr(3): + sys_ops.st_global_u32(row + Int64(4 * k), rb[k]) + for k in cutlass.range_constexpr(3): + r[2 * k] = P.bf16_lo(rb[k]) + r[2 * k + 1] = P.bf16_hi(rb[k]) + if cutlass.const_expr(norm): + self.chunk_squares(cs, f, r) + if cutlass.const_expr(not norm): + if grp == 0: + sys_ops.fence_proxy_async_global() + else: + lane = t % 32 + warp = t // 32 + bs = Float32(0.0) + if grp == 0: + P.bar_sync(2, 128) + if warp == 0: + v0 = cs.rs[lane] + v1 = cs.rs[32 + lane] + v2 = cs.rs[64 + lane] + for i in cutlass.range_constexpr(5): + o: cutlass.Constexpr = 16 >> i + v0 = C.add_rn(v0, P.shfl_xor(v0, o)) + v1 = C.add_rn(v1, P.shfl_xor(v1, o)) + v2 = C.add_rn(v2, P.shfl_xor(v2, o)) + bs = self.block_reduce(v0, v1, v2) + if cutlass.const_expr(dn_bsum is not None): + if t == 0: + sys_ops.st_global_u32( + dn_bsum + Int64(4) * (Int64(m) * 8 + Int64(c)), P.f32_bits(bs) + ) + if grp == 0: + row = out_norm + Int64(2) * (Int64(m) * HIDDEN + Int64(h0)) + for k in cutlass.range_constexpr(3): + g0 = C.add_rn(Float32(0.0), P.bf16_lo(gu[k])) + g1 = C.add_rn(Float32(0.0), P.bf16_hi(gu[k])) + u = C.cvt_rn_bf16x2_f32( + C.mul_rn(r[2 * k], g0), C.mul_rn(r[2 * k + 1], g1) + ) + sys_ops.st_global_u32(row + Int64(4 * k), u) + sys_ops.fence_proxy_async_global() # thread 0 is in group 0 + # completes the cluster barrier phase the consumer prologue opened + P.cluster_wait() + else: + # cluster exchange of the 8 block sums (st.async + mbarrier complete_tx) + crank = Int32(cute.arch.block_idx_in_cluster()) + P.cluster_wait() # every CTA of the cluster started (mbarrier initialized) + if t == 0: + la = Uint32((cs.csum.iterator + crank).toint()) + lm = cs.mbar_addr() + for cc in cutlass.range_constexpr(8): + P.st_async_b32( + P.mapa_u32(la, cc), P.f32_bits(bs), P.mapa_u32(lm, cc) + ) + P.mbar_wait_cluster(cs.mbar_addr(), Uint32(0)) + P.cluster_arrive_relaxed() # exit barrier, arrive: inbound st.async landed + if grp == 0: # the output threads read the cluster's sums of squares + tot = Float32(0.0) + for cc in cutlass.range_constexpr(8): + tot = C.add_rn(tot, cs.csum[cc]) + inv = C.rsqrt_approx(C.fma_rn(tot, C.rcp_rn(Float32(6144.0)), eps)) + row = out_norm + Int64(2) * (Int64(m) * HIDDEN + Int64(h0)) + for k in cutlass.range_constexpr(3): + g0 = C.add_rn(Float32(0.0), P.bf16_lo(gu[k])) + g1 = C.add_rn(Float32(0.0), P.bf16_hi(gu[k])) + y = C.cvt_rn_bf16x2_f32( + C.mul_rn(C.mul_rn(r[2 * k], inv), g0), + C.mul_rn(C.mul_rn(r[2 * k + 1], inv), g1), + ) + sys_ops.st_global_u32(row + Int64(4 * k), y) + sys_ops.fence_proxy_async_global() + # exit barrier, wait: no CTA exits with its st.async in flight + P.cluster_wait() + + @cute.jit + def chunk_squares(self, cs: ConsSmem, f, r): + """Chunk ``p`` (elements ``[8p, 8p + 8)``) of the CTA's 768 elements is summed + from 0 in element order. Thread ``f = 4q + ph`` owns ``[24q + 6ph, +6)``: it + adds its squares into the tail of the previous thread's chunk, carried by one + ``shfl.up``, and writes the chunk it completes.""" + q = cute.make_rmem_tensor((6,), Float32) + for k in cutlass.range_constexpr(3): + w = C.cvt_rn_bf16x2_f32( + C.mul_rn(r[2 * k], r[2 * k]), C.mul_rn(r[2 * k + 1], r[2 * k + 1]) + ) + q[2 * k] = P.bf16_lo(w) + q[2 * k + 1] = P.bf16_hi(w) + ph = f % 4 + tail = Float32(0.0) + if ph == 0: + for k in cutlass.range_constexpr(6): + tail = C.add_rn(tail, q[k]) + elif ph == 1: + for k in cutlass.range_constexpr(2, 6): + tail = C.add_rn(tail, q[k]) + elif ph == 2: + tail = C.add_rn(C.add_rn(tail, q[4]), q[5]) + run = self.chunk_carry(P.shfl_up(tail, 1)) + if ph == 1: + for k in cutlass.range_constexpr(2): + run = C.add_rn(run, q[k]) + cs.rs[3 * (f // 4)] = run + elif ph == 2: + for k in cutlass.range_constexpr(4): + run = C.add_rn(run, q[k]) + cs.rs[3 * (f // 4) + 1] = run + elif ph == 3: + for k in cutlass.range_constexpr(6): + run = C.add_rn(run, q[k]) + cs.rs[3 * (f // 4) + 2] = run + + @cute.jit + def consume_cta_bsum( + self, mb: Mbox, cs: ConsSmem, m, c, e, poisoned, ru, out_res: Int64, bsum: Int64 + ): + """Consumer epilogue without the norm (deferred): all-reduce + residual, then + this CTA's block sum of bf16-rounded squares in the same order as the norm's (96 + chunks of 8 from 0, xor butterflies, ``(w0 + w2) + w1``) to ``bsum[m][c]``; the + norm is applied by a later kernel.""" + t = P.thread_index() + f = t % 128 + grp = t // 128 + buf = self.buffer_of(e) + tag = Uint32(e) & Uint32(TAG_MASK) + acc = cute.make_rmem_tensor((6,), Float32) + x0 = cute.make_rmem_tensor((6,), Float32) + x1 = cute.make_rmem_tensor((6,), Float32) + acc.fill(0.0) + self.delay(mb, 2) + self.poll_records(mb, buf, m, c * 256 + 2 * f, 2 * grp, tag, poisoned, x0, x1) + if grp > 0: + for k in cutlass.range_constexpr(6): + cs.raw[grp - 1, f, k] = x0[k] + cs.raw[grp - 1, f, 6 + k] = x1[k] + else: + for k in cutlass.range_constexpr(6): + acc[k] = C.add_rn(C.add_rn(Float32(0.0), x0[k]), x1[k]) + P.bar_sync(1, 512) + h0 = c * 768 + 6 * f + if grp == 0: + for q in cutlass.range_constexpr(1, 4): + for k in cutlass.range_constexpr(6): + acc[k] = C.add_rn( + C.add_rn(acc[k], cs.raw[q - 1, f, k]), cs.raw[q - 1, f, 6 + k] + ) + rb = cute.make_rmem_tensor((3,), Uint32) + for k in cutlass.range_constexpr(3): + ab = P.no_neg_zero(C.cvt_rn_bf16x2_f32(acc[2 * k], acc[2 * k + 1])) + rb[k] = C.cvt_rn_bf16x2_f32( + C.add_rn(P.bf16_lo(ab), P.bf16_lo(ru[k])), + C.add_rn(P.bf16_hi(ab), P.bf16_hi(ru[k])), + ) + row = out_res + Int64(2) * (Int64(m) * HIDDEN + Int64(h0)) + for k in cutlass.range_constexpr(3): + sys_ops.st_global_u32(row + Int64(4 * k), rb[k]) + for k in cutlass.range_constexpr(3): + cs.rs[6 * f + 2 * k] = P.bf16_lo(rb[k]) + cs.rs[6 * f + 2 * k + 1] = P.bf16_hi(rb[k]) + P.bar_sync(1, 512) + lane = t % 32 + warp = t // 32 + if t < 96: + ts = Float32(0.0) + for i in cutlass.range_constexpr(4): + a0 = cs.rs[8 * t + 2 * i] + a1 = cs.rs[8 * t + 2 * i + 1] + wq = C.cvt_rn_bf16x2_f32(C.mul_rn(a0, a0), C.mul_rn(a1, a1)) + ts = C.add_rn(C.add_rn(ts, P.bf16_lo(wq)), P.bf16_hi(wq)) + for i in cutlass.range_constexpr(5): + o: cutlass.Constexpr = 16 >> i + ts = C.add_rn(ts, P.shfl_xor(ts, o)) + if lane == 0: + cs.wsum[warp] = ts + P.bar_sync(1, 512) + if t == 0: + # the three warp sums, read one word each (no vector read of a fourth word) + wa = Uint32(cs.wsum.iterator.toint()) + v = self.block_reduce( + P.ld_shared_f32(wa), P.ld_shared_f32(wa + 4), P.ld_shared_f32(wa + 8) + ) + sys_ops.st_global_u32( + bsum + Int64(4) * (Int64(m) * 8 + Int64(c)), P.f32_bits(v) + ) + if grp == 0: + sys_ops.fence_proxy_async_global() diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/launch.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/launch.py new file mode 100644 index 00000000..ce88bce3 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/launch.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Launch-side helpers for the compiled kernels: the driver function of a compiled +kernel and the co-residency checks that cooperative and cluster launches rely on. + +Kernels are launched by the DSL launcher (``.launch(..., cooperative=..., cluster=..., +use_pdl=...)``), which covers every combination these kernels use (cooperative + cluster ++ programmatic dependent launch in one launch). The launcher sets no +memory-synchronization domain, so every kernel runs in the stream's default domain; the +fenced start edge of the o_proj hop (PTX ISA 9.7.15.14, release fence at gpu scope) +needs producer and consumer in the same domain, which holds as long as no stream carries +a domain attribute. + +A cooperative grid's CTAs must all be resident at once (they wait on each other), and a +cluster grid's clusters must fit the device. Both are checked once per compiled kernel +with the driver's occupancy queries on the kernel's own function and shared-memory size, +and a failed check declines the fused path (the caller falls back to the existing +implementation). +""" + +from dataclasses import dataclass + +from cuda.bindings import driver as cu + +__all__ = ["KernelHandle", "kernel_handle", "fits_cooperative", "max_active_clusters"] + + +def _ok(res, what): + err = res[0] + if err != cu.CUresult.CUDA_SUCCESS: + raise RuntimeError(f"{what} failed: {err}") + return res[1:] if len(res) > 2 else (res[1] if len(res) == 2 else None) + + +@dataclass(frozen=True) +class KernelHandle: + """The driver function of a compiled single-kernel unit and its dynamic shared + memory.""" + + function: object # CUfunction (current context) + dynamic_smem: int + num_sms: int + + +def kernel_handle(compiled, device_index: int, dynamic_smem: int) -> KernelHandle: + """Resolve the (single) kernel of a compiled unit on the current context. + ``dynamic_smem`` is the unit's dynamic shared-memory size (the kernel class states + it).""" + library = compiled.library + count = _ok(cu.cuLibraryGetKernelCount(library), "cuLibraryGetKernelCount") + if count != 1: + raise RuntimeError(f"expected one kernel in the compiled unit, found {count}") + kernels = _ok( + cu.cuLibraryEnumerateKernels(count, library), "cuLibraryEnumerateKernels" + ) + function = _ok(cu.cuKernelGetFunction(kernels[0]), "cuKernelGetFunction") + smem = dynamic_smem + dev = _ok(cu.cuDeviceGet(device_index), "cuDeviceGet") + sms = _ok( + cu.cuDeviceGetAttribute( + cu.CUdevice_attribute.CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, dev + ), + "cuDeviceGetAttribute", + ) + return KernelHandle(function, int(smem), int(sms)) + + +def fits_cooperative(handle: KernelHandle, ctas: int, threads: int) -> bool: + """True iff ``ctas`` CTAs of ``threads`` threads can all be resident at once.""" + per_sm = _ok( + cu.cuOccupancyMaxActiveBlocksPerMultiprocessor( + handle.function, threads, handle.dynamic_smem + ), + "cuOccupancyMaxActiveBlocksPerMultiprocessor", + ) + return int(per_sm) * handle.num_sms >= ctas + + +def max_active_clusters( + handle: KernelHandle, grid: int, threads: int, cluster: int, stream=None +) -> int: + """Clusters of ``cluster`` CTAs of this kernel that can be resident at once.""" + cfg = cu.CUlaunchConfig() + cfg.gridDimX, cfg.gridDimY, cfg.gridDimZ = grid, 1, 1 + cfg.blockDimX, cfg.blockDimY, cfg.blockDimZ = threads, 1, 1 + cfg.sharedMemBytes = handle.dynamic_smem + cfg.hStream = stream if stream is not None else cu.CUstream(0) + attr = cu.CUlaunchAttribute() + attr.id = cu.CUlaunchAttributeID.CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION + attr.value.clusterDim.x, attr.value.clusterDim.y, attr.value.clusterDim.z = ( + cluster, + 1, + 1, + ) + cfg.attrs = [attr] + cfg.numAttrs = 1 + return int( + _ok( + cu.cuOccupancyMaxActiveClusters(handle.function, cfg), + "cuOccupancyMaxActiveClusters", + ) + ) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/__init__.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/__init__.py new file mode 100644 index 00000000..b4896b01 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/_core_ptx.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/_core_ptx.py new file mode 100644 index 00000000..c151955d --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/_core_ptx.py @@ -0,0 +1,256 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Inline-PTX helpers the MoE kernels need beyond ``primitives/ptx_core``. + +Each is a candidate for ``primitives/ptx_core``; moved there unchanged, the MoE bodies +then import it from there.""" + +from __future__ import annotations + +from cutlass import Float32, Int32, Int64, Uint16, Uint32 + +from ..primitives.ptx_core import ptx_arith, ptx_op + +# --------------------------------------------------------- plain-form arithmetic (pure) + + +@ptx_arith(Float32) +def add_f32(a, b): + """Plain ``add.f32`` (ptxas may contract it with a preceding plain ``mul.f32``).""" + return "add.f32 $0, $1, $2;", "=f,f,f", (Float32(a), Float32(b)) + + +@ptx_arith(Float32) +def mul_f32(a, b): + """Plain ``mul.f32``.""" + return "mul.f32 $0, $1, $2;", "=f,f,f", (Float32(a), Float32(b)) + + +@ptx_arith(Float32) +def round_bf16(x): + """``float(bfloat16_rn(x))`` as the CUDA headers compute it (cvt pair).""" + code = "{ .reg .b16 h; cvt.rn.bf16.f32 h, $1; cvt.f32.bf16 $0, h; }" + return code, "=f,f", (Float32(x),) + + +@ptx_arith(Float32) +def bf16_to_f32(bits): + """FP32 of a BF16 bit pattern (``cvt.f32.bf16``).""" + return "cvt.f32.bf16 $0, $1;", "=f,h", (Uint16(bits),) + + +@ptx_arith(Int32) +def popc(x): + return "popc.b32 $0, $1;", "=r,r", (Uint32(x),) + + +@ptx_arith(Int32) +def clz(x): + return "clz.b32 $0, $1;", "=r,r", (Uint32(x),) + + +@ptx_arith(Int32) +def ctz(x): + """Trailing zero count (32 for 0).""" + return "{ .reg .b32 t; brev.b32 t, $1; clz.b32 $0, t; }", "=r,r", (Uint32(x),) + + +# ----------------------------------------------- global loads / stores (side-effecting) + + +@ptx_op(Float32) +def ld_cg_f32(address): + """Coherent-at-L2 load of data written during the step (never hoisted above a + wait).""" + return "ld.global.cg.f32 $0, [$1];", "=f,l", (Int64(address),) + + +@ptx_op(Int32) +def ld_cg_s32(address): + return "ld.global.cg.s32 $0, [$1];", "=r,l", (Int64(address),) + + +@ptx_op((Float32, Float32, Float32, Float32)) +def ld_cg_v4_f32(address): + return ( + "ld.global.cg.v4.f32 {$0, $1, $2, $3}, [$4];", + "=f,=f,=f,=f,l", + (Int64(address),), + ) + + +@ptx_op((Uint32, Uint32, Uint32, Uint32)) +def ld_cg_v4_u32(address): + return ( + "ld.global.cg.v4.u32 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,l", + (Int64(address),), + ) + + +@ptx_op((Float32, Float32, Float32, Float32)) +def ld_nc_v4_f32(address): + """Non-coherent load of a static input (weights, biases).""" + return ( + "ld.global.nc.v4.f32 {$0, $1, $2, $3}, [$4];", + "=f,=f,=f,=f,l", + (Int64(address),), + ) + + +@ptx_op() +def st_global_b32(address, word): + return "st.global.b32 [$0], $1;", "l,r", (Int64(address), Uint32(word)) + + +@ptx_op() +def st_global_b32f(address, value): + """The 32 bits of an FP32 value.""" + return "st.global.b32 [$0], $1;", "l,f", (Int64(address), Float32(value)) + + +@ptx_op() +def atom_or_global(address, bits): + """Relaxed GPU-scope atomic ``or`` into a global word (the old value is + discarded).""" + return ( + "{ .reg .b32 o; atom.global.or.b32 o, [$0], $1; }", + "l,r", + (Int64(address), Uint32(bits)), + ) + + +@ptx_op() +def atom_or_shared(addr, bits): + """Atomic ``or`` into a shared word through its generic address (the old value is + discarded).""" + code = ( + "{ .reg .b64 a, g; .reg .b32 o; cvt.u64.u32 a, $0; cvta.shared.u64 g, a; " + "atom.or.b32 o, [g], $1; }" + ) + return code, "r,r", (Int32(addr), Uint32(bits)) + + +# ------------------------------------------------------- shared memory (side-effecting) + + +@ptx_op((Float32, Float32, Float32, Float32)) +def ld_shared_v4_f32(addr): + return "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", "=f,=f,=f,=f,r", (Int32(addr),) + + +@ptx_op(Int32) +def ld_shared_s32(addr): + return "ld.shared.s32 $0, [$1];", "=r,r", (Int32(addr),) + + +@ptx_op(Int32) +def ld_shared_s16(addr): + return "ld.shared.s16 $0, [$1];", "=r,r", (Int32(addr),) + + +@ptx_op(Int32) +def ld_shared_s8(addr): + return "ld.shared.s8 $0, [$1];", "=r,r", (Int32(addr),) + + +@ptx_op(Int32) +def ld_shared_u8(addr): + return "ld.shared.u8 $0, [$1];", "=r,r", (Int32(addr),) + + +@ptx_op() +def st_shared_b8(addr, value): + return "st.shared.b8 [$0], $1;", "r,r", (Int32(addr), Uint32(value)) + + +@ptx_op() +def st_shared_v2_u32(addr, a, b): + return ( + "st.shared.v2.u32 [$0], {$1, $2};", + "r,r,r", + (Int32(addr), Uint32(a), Uint32(b)), + ) + + +# ------------------------------------------------------------- control (side-effecting) + + +@ptx_op() +def trap_if(flag): + """``trap`` when ``flag`` is non-zero (a launch-contract violation: fail closed).""" + return "{ .reg .pred p; setp.ne.b32 p, $0, 0; @p trap; }", "r", (Int32(flag),) + + +# ---------------------------------------------------- mbarrier / DSMEM (side-effecting) + + +@ptx_op() +def mbarrier_arrive_remote(remote_bar): + """Release-arrive (cluster scope) on a peer CTA's mbarrier (``mapa``-ed address).""" + return ( + "mbarrier.arrive.release.cluster.shared::cluster.b64 _, [$0];", + "r", + (Uint32(remote_bar),), + ) + + +@ptx_op() +def st_async_v4_b32(remote, remote_bar, a, b, c, d): + """16-byte store to a peer CTA's shared memory completing 16 tx-bytes on its + mbarrier.""" + code = ( + "st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 " + "[$0], {$2, $3, $4, $5}, [$1];" + ) + return ( + code, + "r,r,r,r,r,r", + ( + Uint32(remote), + Uint32(remote_bar), + Uint32(a), + Uint32(b), + Uint32(c), + Uint32(d), + ), + ) + + +@ptx_arith((Float32, Float32)) +def bf16x2_unpack(word): + """The two BF16 halves of ``word`` as FP32 via ``cvt.f32.bf16``: (bits 0-15, bits + 16-31).""" + code = ( + "{ .reg .b16 l, h; mov.b32 {l, h}, $2; " + "cvt.f32.bf16 $0, l; cvt.f32.bf16 $1, h; }" + ) + return code, "=f,=f,r", (Uint32(word),) + + +@ptx_op() +def st_global_f32(address, value): + return "st.global.f32 [$0], $1;", "l,f", (Int64(address), Float32(value)) + + +@ptx_op() +def st_async_v4_f32(remote, remote_bar, a, b, c, d): + """16 bytes of FP32 to a peer CTA's shared memory, completing 16 tx-bytes on its + mbarrier.""" + code = ( + "st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 " + "[$0], {$2, $3, $4, $5}, [$1];" + ) + return ( + code, + "r,r,f,f,f,f", + ( + Uint32(remote), + Uint32(remote_bar), + Float32(a), + Float32(b), + Float32(c), + Float32(d), + ), + ) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/api.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/api.py new file mode 100644 index 00000000..48a8095d --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/api.py @@ -0,0 +1,330 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Host side of the small-M NVFP4 MoE engines: compile specs, scratch, launch. + +Engine-neutral: every input is a torch tensor or a Python scalar; the serving engine's +glue extracts them from its MoE layer, compiles the specs through its own compile cache +and calls ``SmallMMoe``. A verify batch is ``rows_per_request`` (1..4 rows per request, +request-major, at most 4 requests): + +* one request (M <= 4): the one-request engine (mode ``g1``); +* two requests (M <= 8): the two-request engine (mode ``g2``); +* three or four requests (M <= 16): the four-request engine (``engine16``). + +``what``: bit 0 runs the engine (it writes ``partials`` / ``shared_partials`` and the +slot records), bit 1 the finalize (``out[M, 6144]`` BF16: the layer's un-reduced MoE +output). With bit 1 clear the scratch buffers are the output (the fold ABI: ``partials`` +[4][M][8][6144] FP32, ``shared_partials`` [4][M][6144] FP32, ``slot_ids`` / +``slot_weights`` / ``slot_scales`` [M][8], row stride M, a slot id < 0 is skipped).""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import dataclass + +import torch + +from . import schedule as S +from .constants import HIDDEN, NUM_EXPERTS, TOP_K + +MAX_ROWS = 16 +MAX_REQUESTS = 4 +REQUEST_ROWS = 4 +SHARED_PARTS = 4 +MIN_CLUSTERS_SHARED = 8 +ENGINE_SMEM = {"g1": 227440, "g2": 229968, "g4": 223664} + + +@dataclass +class LayerTensors: + """One layer's tensors as the engines read them (views of the engine's processed + weights, plus the two derived forms).""" + + w13: ( + torch.Tensor + ) # uint8 [256, 512, 3072]: processed FC1 weights (gated rows, block-32 shuffled) + w13_scales: torch.Tensor # uint8 [256, 512 * 384]: processed FC1 block scales + fc2_records: torch.Tensor # uint8 [256, 4, 4, 3 * 18432] (weights.fc2_records) + # uint8 [4, 4, 18 * 32768] (weights.shared_pack) or None + shared_pack: torch.Tensor | None + g1_alpha: torch.Tensor # FP32 [256]: FC1 input scale x per-expert weight scale 2 + g2_alpha: torch.Tensor # FP32 [256] + gs1: float # 1 / FC1 input scale + gs2: float # 1 / FC2 input scale + + def check(self) -> None: + def need(ok: bool, what: str) -> None: + if not ok: + raise ValueError(f"small-M MoE: {what}") + + for name, t, shape in ( + ("w13", self.w13, (NUM_EXPERTS, 512, 3072)), + ("w13_scales", self.w13_scales, (NUM_EXPERTS, 512 * 384)), + ("fc2_records", self.fc2_records, (NUM_EXPERTS, 4, 4, 3 * 18432)), + ): + need( + t.dtype == torch.uint8 + and tuple(t.shape) == shape + and t.is_contiguous(), + f"{name} must be contiguous uint8 {shape}, " + f"got {t.dtype} {tuple(t.shape)}", + ) + need( + t.is_cuda and t.data_ptr() % 16 == 0, + f"{name} must be a 16-byte aligned CUDA tensor", + ) + if self.shared_pack is not None: + sp = self.shared_pack + need( + sp.dtype == torch.uint8 + and sp.numel() == 4 * 4 * 18 * 32768 + and sp.is_contiguous() + and sp.data_ptr() % 16 == 0, + "shared_pack must be contiguous 16-byte aligned uint8 [4, 4, 589824]", + ) + for name, t in (("g1_alpha", self.g1_alpha), ("g2_alpha", self.g2_alpha)): + need( + t.dtype == torch.float32 + and tuple(t.shape) == (NUM_EXPERTS,) + and t.is_contiguous(), + f"{name} must be contiguous FP32 [{NUM_EXPERTS}]", + ) + + +class Scratch: + """Engine outputs / finalize inputs for up to 16 rows (one set, reused in stream + order).""" + + def __init__(self, device: torch.device) -> None: + f32, i32 = torch.float32, torch.int32 + self.partials = torch.empty( + 4 * MAX_ROWS * TOP_K * HIDDEN, dtype=f32, device=device + ) + self.shared_partials = torch.empty( + SHARED_PARTS * MAX_ROWS * HIDDEN, dtype=f32, device=device + ) + self.slot_ids = torch.empty(MAX_ROWS * TOP_K, dtype=i32, device=device) + self.slot_weights = torch.empty(MAX_ROWS * TOP_K, dtype=f32, device=device) + self.slot_scales = torch.empty(MAX_ROWS * TOP_K, dtype=f32, device=device) + self.err = torch.zeros(1, dtype=i32, device=device) + + +def mode_for(rows_per_request: Sequence[int]) -> str: + n = len(rows_per_request) + if not 1 <= n <= MAX_REQUESTS or any( + not 1 <= int(r) <= REQUEST_ROWS for r in rows_per_request + ): + raise ValueError( + "small-M MoE: need 1..4 requests of 1..4 rows, " + f"got {tuple(rows_per_request)}" + ) + return "g1" if n == 1 else "g2" if n == 2 else "g4" + + +def pack_rows(rows_per_request: Sequence[int]) -> int: + """The four-request engine's per-request row word: rows of request q in byte q.""" + v = 0 + for q, r in enumerate(rows_per_request): + v |= int(r) << (8 * q) + return v + + +def default_clusters(device: torch.device | None = None) -> int: + """Co-resident 4-CTA clusters at one CTA per SM (the engines' occupancy: each uses + nearly all of an SM's shared memory) on ``device`` (default: the current one).""" + import cutlass.utils + + index = ( + torch.accelerator.current_device_index() + if device is None + else torch.device(device).index + ) + # the device's primary context is current on this thread + torch.accelerator.synchronize(index) + return int(cutlass.utils.HardwareInfo(index).get_max_active_clusters(4)) + + +def compile_specs(shared: bool = True) -> dict[str, tuple[object, tuple]]: + """The served compile units: name -> (kernel object, example arguments for + ``cute.compile``).""" + from cutlass import Float32, Int32, Int64 + from cutlass.cute.runtime import make_fake_stream + + from .engine import Nvfp4MoeSmallM + from .engine16 import Nvfp4MoeSmallM16 + from .finalize import MoeFinalize + from .routing import MoeRouteTop8 + + z = Int64(0) + one = Float32(1.0) + + def eng(m: int, split: int) -> tuple: + return ( + z, + z, + z, + z, + z, + z, + z, + one, + one, + Int32(m), + Int32(split), + z, + z, + z, + z, + z, + z, + z, + z, + z, + Int32(33), + make_fake_stream(), + ) + + return { + "g1": (Nvfp4MoeSmallM("g1", shared), eng(4, 4)), + "g2": (Nvfp4MoeSmallM("g2", shared), eng(8, 4)), + "g4": ( + Nvfp4MoeSmallM16(shared), + eng(16, pack_rows((4, 4, 4, 4))), + ), + "finalize": ( + MoeFinalize(shared), + (z, z, z, z, z, z, Int32(1), make_fake_stream()), + ), + "route": ( + MoeRouteTop8(), + (z, z, Float32(2.5), z, z, Int32(1), make_fake_stream()), + ), + } + + +class SmallMMoe: + """Launches the compiled engines on one device. + + ``compiled``: name -> compiled callable for the names of ``compile_specs``; + ``nclusters``: the engine grid in 4-CTA clusters (any value >= 8 is correct; the + co-resident maximum is the fast one: 33 on a 148-SM part).""" + + def __init__( + self, + compiled: dict[str, Callable], + nclusters: int, + device: torch.device, + shared: bool = True, + ) -> None: + if shared and nclusters < MIN_CLUSTERS_SHARED: + raise ValueError( + "small-M MoE: the in-engine shared expert needs " + f">= {MIN_CLUSTERS_SHARED} clusters" + ) + self.fn = compiled + self.nclusters = nclusters + self.shared = shared + self.device = device + self.scratch = Scratch(device) + self.tables = {} + for mode, build in (("g1", S.g1_table), ("g2", S.g2_table), ("g4", S.g4_table)): + for nc in sorted({nclusters, min(nclusters, 32)}): + self.tables[(mode, nc)] = build(nc, shared).to(device) + + def clusters(self, mode: str, rows: int) -> int: + return min(self.nclusters, 32) if mode == "g1" and rows == 1 else self.nclusters + + def __call__( + self, + layer: LayerTensors, + x: torch.Tensor, + ids: torch.Tensor, + topw: torch.Tensor, + rows_per_request: Sequence[int], + out: torch.Tensor | None = None, + what: int = 3, + ) -> torch.Tensor | None: + import cuda.bindings.driver as cuda + from cutlass import Float32, Int32, Int64 + + mode = mode_for(rows_per_request) + m = sum(int(r) for r in rows_per_request) + if ( + x.dtype != torch.bfloat16 + or tuple(x.shape) != (m, HIDDEN) + or not x.is_contiguous() + or ids.dtype != torch.int32 + or tuple(ids.shape) != (m, TOP_K) + or not ids.is_contiguous() + or topw.dtype != torch.float32 + or tuple(topw.shape) != (m, TOP_K) + or not topw.is_contiguous() + ): + raise ValueError( + "small-M MoE: x bf16 [M, 6144], ids int32 [M, 8], " + "weights fp32 [M, 8], contiguous" + ) + if self.shared and layer.shared_pack is None: + raise ValueError( + "small-M MoE: this launcher runs the shared expert; " + "the layer has no shared pack" + ) + if what & 2 and ( + out is None + or out.dtype != torch.bfloat16 + or tuple(out.shape) != (m, HIDDEN) + or not out.is_contiguous() + ): + raise ValueError("small-M MoE: out must be contiguous bf16 [M, 6144]") + sc = self.scratch + nc = self.clusters(mode, m) + split = ( + m + if mode == "g1" + else int(rows_per_request[0]) + if mode == "g2" + else pack_rows(rows_per_request) + ) + stream = cuda.CUstream(torch.cuda.current_stream(x.device).cuda_stream) + + def p(t): + return Int64(t.data_ptr() if t is not None else 0) + + if what & 1: + self.fn[mode]( + p(layer.w13), + p(x), + p(ids), + p(topw), + p(layer.shared_pack if self.shared else None), + p(layer.g1_alpha), + p(layer.g2_alpha), + Float32(layer.gs1), + Float32(layer.gs2), + Int32(m), + Int32(split), + p(sc.err), + p(self.tables[(mode, nc)]), + p(sc.shared_partials), + p(sc.partials), + p(sc.slot_ids), + p(sc.slot_weights), + p(sc.slot_scales), + p(layer.w13_scales), + p(layer.fc2_records), + Int32(nc), + stream, + ) + if what & 2: + self.fn["finalize"]( + p(sc.partials), + p(sc.shared_partials if self.shared else None), + p(sc.slot_ids), + p(sc.slot_weights), + p(sc.slot_scales), + p(out), + Int32(m), + stream, + ) + return out diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/constants.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/constants.py new file mode 100644 index 00000000..f8e77008 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/constants.py @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Fixed geometry of the GLM-5.3 routed-expert layer at tensor parallel 8.""" + +HIDDEN = 6144 # model hidden size (K of FC1, N of FC2) +NUM_EXPERTS = 256 # routed experts +TOP_K = 8 # experts per token +INTER = 256 # per-rank intermediate size of one expert (gate or up rows) +QUARTERS = 4 # an expert's FC2 output is reduced from four quarter partials +SHARED_PARTS = 4 # the shared expert's output is reduced from four partials +MAX_ROWS = 16 # verify rows of up to four requests +ROUTED_SCALE = 2.5 # routed scaling factor of the checkpoint diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/engine.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/engine.py new file mode 100644 index 00000000..6329ab9c --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/engine.py @@ -0,0 +1,2083 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Small-M NVFP4 routed-expert engine (W4A4, tcgen05 ``kind::mxf4nvf4`` block16) with +the BF16 shared expert in-engine. + +Modes: ``g1`` one request (M <= 4 rows); ``g2`` two requests of 1..4 rows each (M <= 8). + +Work unit = (expert e, quarter j): gate and up rows ``64 j .. 64 j + 63`` of FC1 (128 +MMA rows -> 64 intermediate values h) and the matching 64 K columns of FC2 (all 6144 +output rows). One cluster of four CTAs runs a unit; CTA rank c owns K range ``1536 c .. ++1536`` of FC1 and output rows ``1536 c .. +1536`` of FC2. + +* FC1: 3 ring chunks per unit, each a 3D tensor-map box of the engine's processed w13 + (two 128 x 128 B slots, 128-byte swizzle, rows in the processing's in-block order) and + 4 KiB of its block scales; 24 MMAs 128 x 8 x 64 with A = weights, B = the tokens. +* K-split reduction: every CTA pushes its 128 x 4 partial to the three peers through + distributed shared memory and sums the four partials in rank order, so all four hold + the same sums; then g1 alpha, SiLU(gate) * up, NVFP4 quantization of h (global scale + ``1 / a2``) into the FC2 operand image. +* FC2: 12 tiles of 128 x 64 from compact records, one MMA each; every routed unit stores + its raw FC2 partial into the token's top-k slot of ``partials``; the shared quarter + goes to ``shared_partials``. +* Pipeline: stream / MMA / epilogue order FC1(0), FC1(1), FC2(0), FC1(2), FC2(1), ... so + the h chain of unit k overlaps the FC1 weight stream of unit k + 1. + +Mode g2: image rows 0..3 hold request A, rows 4..7 request B (MMA N stays 8). The expert +list is the union of both requests' experts, each entry with a group mask (bit 0: used +by A, bit 1: by B); per unit and group one exchange and one h chain, so an expert both +requests use streams once. A release warp returns receive buffers to the peers, the FC2 +accumulator is double-buffered, unit 0's first copies issue as soon as the union size is +known, and the shared quarter's remaining chunks are prefetched into L2 before the +dependency wait. + +Warp roles: 0 expert dedup + producer, 1 MMA issue + TMEM owner, 6 scale-factor copies, +2..5 epilogue (TMEM lane quadrant ``warp % 4``), 7 release (g2). + +Outputs (rows n < M by global row): ``partials`` [4][M][8][6144] FP32, +``shared_partials`` [4][M][6144] FP32, ``ids`` / ``weights`` / ``scales`` [M][8] +(written by CTA 0: the slot's expert id or -1, its routing weight, its FC2 alpha). +``moe_finalize`` reads exactly this.""" + +from __future__ import annotations + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass import Float32, Int32, Int64, Uint8, Uint32, Uint64 +from cutlass.experimental.cuda import ( + TensorMap, + TensorMapL2Promotion, + TensorMapSwizzle, + create_tensor_map_tiled, +) +from cutlass.memory import SmemAllocator + +from ..primitives import ptx_core as C +from ..primitives import ptx_tc as T +from . import _core_ptx as P +from .constants import HIDDEN, NUM_EXPERTS, TOP_K + +# --------------------------------------------------------- geometry shared by the modes +CLUSTER = 4 +K_CTA = HIDDEN // CLUSTER # 1536 +SLOT = 18432 # 16 KiB weights + 2 KiB scales +BIG = 2 * SLOT # one ring chunk +RING = 4 +LOOKAHEAD = 2 # FC2(k) issues after FC1(k + 2) +H_BUFS = LOOKAHEAD + 1 +FC2_TILES = 12 +SH_CHUNK = 32768 +SH_FC1_CHUNKS = 12 +SH_CHUNKS = 18 +SH_CLUSTERS = 4 +SH_RECORD = SH_CHUNKS * SH_CHUNK +FC2_RECORD = 3 * SLOT +SF13_PER_EXPERT = 512 * 384 +REQ_ROWS = 4 # image rows per request +SF_WARP = 6 +RELEASE_WARP = 7 +ERR_BAD_ID = 1 +ERR_UNIT_CAP = 2 + +# shared memory (offsets from the 1024-aligned base) +OFF_RING = 0 +OFF_XBF = OFF_RING + RING * BIG # BF16 x image [192 K cores][8 rows][16 B] +OFF_X = OFF_XBF + 24576 # NVFP4 x image [48 K cores][8 rows][16 B] +OFF_XSF = OFF_X + 6144 # x scale atoms, 24 x 128 B + 384 +OFF_RECV = OFF_XSF + 3456 # receive buffers (per mode) +H_BUF = 768 # h image 256 B + its scale atom 512 B + +# tensor memory columns +T_ACC1 = 0 # 2 x 8 +T_ACC2 = 16 # 12 x 8 +T_ACC2_B = 384 # second FC2 accumulator (g2) +T_SFB_X = 128 # 24 x 4 +T_SFB_H = 224 # H_BUFS x 4 +T_SFA = 256 # RING x 32 + +IDESC_NVF4 = T.idesc_nvf4(128, 8) +IDESC_BF16 = T.idesc_bf16(128, 8) + +# burst tables: (d column, A byte offset, B byte offset, SFA column, SFB column, +# accumulate, group) +FC1_MMAS = tuple( + ( + 0, + 16384 * h2 + 32 * kk, + 256 * (4 * h2 + kk), + 16 * h2 + 4 * kk, + 4 * (4 * h2 + kk), + "first" if h2 == kk == 0 else 1, + 0, + ) + for h2 in range(2) + for kk in range(4) +) +FC2_MMAS = tuple( + (8 * (4 * h2 + t), SLOT * h2 + 4096 * t, 0, 16 * h2 + 4 * t, 0, 0, h2) + for h2 in range(2) + for t in range(4) +) +SH_FC1_MMAS = tuple( + ( + 0, + 16384 * h2 + 256 * kk, + 256 * (4 * h2 + kk), + 0, + 0, + "first" if h2 == kk == 0 else 1, + 0, + ) + for h2 in range(2) + for kk in range(4) +) +SH_FC2_MMAS = tuple( + (8 * h2, 16384 * h2 + 256 * kk, 256 * kk, 0, 0, 0 if kk == 0 else 1, 0) + for h2 in range(2) + for kk in range(4) +) +# scale-factor copies: (TMEM column, source byte offset, group) +FC1_CPS = tuple( + (16 * h2 + 4 * kk, 2048 * h2 + 512 * kk, 0) for h2 in range(2) for kk in range(4) +) +FC2_CPS = tuple( + (16 * h2 + 4 * kk, SLOT * h2 + 512 * kk, h2) for h2 in range(2) for kk in range(4) +) +X_CPS = tuple((4 * q, 128 * q, 0) for q in range(4)) +H_CPS = ((0, 0, 0),) + + +def _align16(v: int) -> int: + return (v + 15) // 16 * 16 + + +class _Geometry: + """Per-mode constants: thread count, receive buffers, unit table, schedule rows, + shared memory map (each total leaves room for one CTA per SM).""" + + def __init__(self, mode: str): + if mode not in ("g1", "g2"): + raise ValueError(f"unknown mode {mode!r}") + two = mode == "g2" + self.two = two + self.threads = 256 if two else 224 + self.recv_bufs = 5 if two else 4 + self.recv_slots = ( + 3 if two else 4 + ) # g2: three peer slots per buffer; g1: one slot per rank + self.recv_buf = self.recv_slots * 128 * 16 + self.off_h = OFF_RECV + self.recv_bufs * self.recv_buf + self.off_hbf = self.off_h + H_BUFS * H_BUF + self.off_misc = self.off_hbf + 1024 + self.acc2_bufs = 2 if two else 1 + self.max_units = 16 if two else 32 + self.max_list = 64 if two else 32 + self.sched_row = 16 if two else 34 + self.sched_bytes = 2080 if two else 2256 + self.rows = 2 * REQ_ROWS if two else REQ_ROWS + self.pfree_count = 3 if two else 12 + self.smem = 229968 if two else 227440 + # barriers + self.b_full = 0 + self.b_empty = 32 + self.b_xready = 64 + self.b_tready = 72 + self.b_sbar = 80 + self.b_acc1_full = 88 + self.b_acc1_empty = 104 + self.b_acc2_full = 120 + self.b_acc2_empty = self.b_acc2_full + 8 * self.acc2_bufs + self.b_hready = self.b_acc2_empty + 8 * self.acc2_bufs + self.b_sfa_full = self.b_hready + 8 * H_BUFS + self.b_sfx_ready = self.b_sfa_full + 8 * RING + self.b_precv = self.b_sfx_ready + 8 + self.b_pfree = self.b_precv + 8 * self.recv_bufs + self.b_consumed = self.b_pfree + 8 * self.recv_bufs + end = self.b_consumed + (8 * self.recv_bufs if two else 0) + self.m_tmem = end + self.m_nunits = end + 4 + self.m_nex = end + 8 + self.m_elist = _align16(end + 12) + self.m_egm = self.m_elist + 4 * self.max_list + self.m_stab = _align16(self.m_egm + 4 * self.max_list) + self.m_units = _align16(self.m_stab + self.sched_bytes) + # unit record: expert, quarter, shared flag, group mask, g1 alpha, token scale + # [rows], slot [rows] + self.u_e, self.u_j, self.u_sh, self.u_gm, self.u_g1 = 0, 4, 8, 12, 16 + self.u_tok = 20 + self.u_slot = 20 + 4 * self.rows + self.unit_bytes = 20 + 8 * self.rows + used = self.off_misc + self.m_units + self.max_units * self.unit_bytes + 1024 + assert used <= self.smem, (mode, used, self.smem) + + +class Nvfp4MoeSmallM: + """``mode``: "g1" or "g2"; ``shared``: run the shared expert's four quarters + in-engine (the served form); ``merge`` (g2): one union expert list for both requests + (the served form; False lists A's experts then B's, test-only). + + Call: ``(w13, x, ids, weights, shared_pack, g1_alpha, g2_alpha, gs1, gs2, rows, + split, err, schedule, shared_partials, partials, slot_ids, slot_weights, + slot_scales, w13_scales, fc2_records, nclusters, stream)``; addresses are device + pointers, ``gs1`` / ``gs2`` the FC1 / FC2 activation global scales (1 / input + scale), ``split`` the rows of the first request (g2; ignored by g1).""" + + def __init__(self, mode: str = "g1", shared: bool = True, merge: bool = True): + self.mode = mode + self.g = _Geometry(mode) + self.shared = shared + self.merge = merge + + @cute.jit + def __call__( + self, + w13: Int64, + x: Int64, + ids: Int64, + topw: Int64, + shpack: Int64, + g1a: Int64, + g2a: Int64, + gs1: Float32, + gs2: Float32, + rows: Int32, + split: Int32, + err: Int64, + sched: Int64, + shpartials: Int64, + pq: Int64, + ids_fin: Int64, + w_fin: Int64, + g2s: Int64, + sf13: Int64, + fc2pack: Int64, + nclusters: Int32, + stream: cuda.CUstream, + ): + # processed w13 viewed as {128 B, E * 512 rows, 24 column blocks}: one box = one + # FC1 chunk (two 128 x 128 slots) + tmap = create_tensor_map_tiled( + w13, + Uint8, + [128, NUM_EXPERTS * 512, 24], + [3072 // 16, 128 // 16], + [128, 128, 2], + swizzle=TensorMapSwizzle.s128b, + l2_promotion=TensorMapL2Promotion.l2_256b, + ) + self.kernel( + tmap, + x, + ids, + topw, + shpack, + g1a, + g2a, + gs1, + gs2, + rows, + split, + err, + sched, + shpartials, + pq, + ids_fin, + w_fin, + g2s, + sf13, + fc2pack, + ).launch( + grid=[CLUSTER * nclusters, 1, 1], + block=[self.g.threads, 1, 1], + cluster=[CLUSTER, 1, 1], + max_number_threads=[self.g.threads, 1, 1], + min_blocks_per_mp=1, + stream=stream, + use_pdl=True, + ) + + # ------------------------------------------------------------ protocol hooks + # Each step of the inter-warp / inter-CTA protocol is one method here. + @cute.jit + def wait_schedule(self, misc): + """The staged schedule table is complete.""" + C.mbarrier_wait_parity(misc + self.g.b_sbar, 0) + + @cute.jit + def wait_inputs(self): + """The epilogue's activations are written by the preceding kernel (dependent + launch).""" + C.pdl_wait() + + @cute.jit + def wait_scales(self, bar, parity): + """The scale-factor warp's copies into TMEM are complete.""" + C.mbarrier_wait_parity(bar, parity) + + @cute.jit + def hready_slot(self, i): + return i % H_BUFS + + @cute.jit + def hready_phase(self, i): + return (i // H_BUFS) & 1 + + @cute.jit + def ring_tail(self, misc, it): + """Producer tail: the MMA warp's last commit on every ring slot has arrived.""" + first = it - RING + if first < 0: + first = Int32(0) + for c in cutlass.range(first, it, unroll=1): + C.mbarrier_wait_parity( + misc + self.g.b_empty + 8 * (c % RING), (c // RING) & 1 + ) + + # ------------------------------------------------------------ helpers + @cute.jit + def unit_field(self, misc, i, field: cutlass.Constexpr[int]): + return C.ld_shared_u32(misc + self.g.m_units + i * self.g.unit_bytes + field) + + @cute.jit + def fc1_chunk(self, tmap_ptr, sf13, dst, expert, quarter, crank, c, bar, policy): + """FC1 chunk c of unit (expert, quarter) on rank crank: one tensor-map box (two + slots) and the two slots' 4 KiB of scales.""" + T.tma_load_3d( + dst, + tmap_ptr, + 0, + expert * 512 + 128 * quarter, + crank * 6 + 2 * c, + bar, + policy, + ) + src = ( + sf13 + + Int64(expert) * SF13_PER_EXPERT + + Int64(96 * quarter + 24 * crank + 8 * c) * 512 + ) + C.bulk_copy_g2s(dst + 32768, src, 4096, bar, policy) + + @cute.jit + def pre_issue(self, shpack, base, misc, crank, cl, my_sh, lane, policy): + """Shared clusters: the shared quarter's first ring chunks (static weights) + before the dependency wait; g2 also prefetches the remaining chunks into L2 (one + per lane).""" + pre = Int32(0) + if cutlass.const_expr(self.shared): # noqa: SIM102 + if my_sh != 0: + src = shpack + (Int64(cl) * 4 + Int64(crank)) * SH_RECORD + if lane == 0: + for c in cutlass.range_constexpr(RING): + C.mbarrier_arrive_expect_tx( + misc + self.g.b_full + 8 * c, SH_CHUNK + ) + C.bulk_copy_g2s( + base + OFF_RING + c * BIG, + src + c * SH_CHUNK, + SH_CHUNK, + misc + self.g.b_full + 8 * c, + policy, + ) + pre = Int32(RING) + if cutlass.const_expr(self.g.two): + c = RING + lane + if c < SH_CHUNKS: + C.bulk_prefetch_l2(src + Int64(c) * SH_CHUNK, SH_CHUNK) + return pre + + @cute.jit + def write_elist( + self, misc, bits, k, lane, word_a, word_b, group: cutlass.Constexpr[int] + ): + """Set bits of ``bits`` (this lane's 32 experts) into the expert list from entry + k, with their group masks (``group`` 0: from the two request words; 1 / 2: that + value).""" + while bits != Uint32(0): + b = P.ctz(bits) + if k < self.g.max_list: + C.st_shared_u32(misc + self.g.m_elist + 4 * k, 32 * lane + b) + if cutlass.const_expr(self.g.two): + gm = Uint32(group) + if cutlass.const_expr(group == 0): + gm = ((word_a >> Uint32(b)) & Uint32(1)) | ( + ((word_b >> Uint32(b)) & Uint32(1)) << Uint32(1) + ) + C.st_shared_u32(misc + self.g.m_egm + 4 * k, gm) + k = k + 1 + bits = bits & (bits - Uint32(1)) + + @cute.jit + def scan8(self, cnt, lane): + """Inclusive prefix sum of ``cnt`` over lanes 0..7.""" + incl = cnt + for o in cutlass.range_constexpr(3): + t = cute.arch.shuffle_sync_up(incl, 1 << o) + if lane >= (1 << o): + incl = incl + t + return incl + + @cute.jit + def stream( + self, + tmap_ptr, + shpack, + sf13, + fc2pack, + base, + misc, + crank, + nunits, + pre, + pre0, + policy, + ): + """Producer lane: stream order FC1(0), FC1(1), FC2(0), FC1(2), FC2(1), ..., then + wait for the MMA warp's last commit on every ring slot.""" + it = pre + pre0 + for kk in cutlass.range(nunits + LOOKAHEAD, unroll=1): + for part in cutlass.range_constexpr(2): + i = kk if part == 0 else kk - LOOKAHEAD + if (i >= 0) & (i < nunits): + ush = self.unit_field(misc, i, self.g.u_sh) + if ush != Uint32(0): + uj = Int32(self.unit_field(misc, i, self.g.u_j)) + src = shpack + (Int64(uj) * 4 + Int64(crank)) * SH_RECORD + c0 = Int32(SH_FC1_CHUNKS) + c1 = Int32(SH_CHUNKS) + if cutlass.const_expr(part == 0): + c0 = Int32(0) + if i == 0: + c0 = pre + c1 = Int32(SH_FC1_CHUNKS) + for c in cutlass.range(c0, c1, unroll=1): + s = it % RING + if it >= RING: + C.mbarrier_wait_parity( + misc + self.g.b_empty + 8 * s, + ((it // RING) - 1) & 1, + ) + C.mbarrier_arrive_expect_tx( + misc + self.g.b_full + 8 * s, SH_CHUNK + ) + C.bulk_copy_g2s( + base + OFF_RING + s * BIG, + src + Int64(c) * SH_CHUNK, + SH_CHUNK, + misc + self.g.b_full + 8 * s, + policy, + ) + it = it + 1 + else: + u_e_idx = Int32(self.unit_field(misc, i, self.g.u_e)) + uj = Int32(self.unit_field(misc, i, self.g.u_j)) + src2 = ( + fc2pack + + ((Int64(u_e_idx) * 4 + Int64(uj)) * 4 + Int64(crank)) + * FC2_RECORD + ) + c0 = Int32(3) + c1 = Int32(5) + if cutlass.const_expr(part == 0): + c0 = Int32(0) + if i == 0: + c0 = pre0 + c1 = Int32(3) + for c in cutlass.range(c0, c1, unroll=1): + s = it % RING + if it >= RING: + C.mbarrier_wait_parity( + misc + self.g.b_empty + 8 * s, + ((it // RING) - 1) & 1, + ) + nbytes = Int32(BIG) + if c == 4: + nbytes = Int32(SLOT) + C.mbarrier_arrive_expect_tx( + misc + self.g.b_full + 8 * s, nbytes + ) + if c < 3: + # expert / quarter read per chunk from the unit + # table, as the reference engine does + ce = Int32(self.unit_field(misc, i, self.g.u_e)) + cj = Int32(self.unit_field(misc, i, self.g.u_j)) + self.fc1_chunk( + tmap_ptr, + sf13, + base + OFF_RING + s * BIG, + ce, + cj, + crank, + c, + misc + self.g.b_full + 8 * s, + policy, + ) + else: + C.bulk_copy_g2s( + base + OFF_RING + s * BIG, + src2 + Int64(2 * c - 6) * SLOT, + nbytes, + misc + self.g.b_full + 8 * s, + policy, + ) + it = it + 1 + self.ring_tail(misc, it) + + # ------------------------------------------------ warp 0: dedup, schedule, producer + @cute.jit + def producer_g1( + self, + tmap_ptr, + ids, + shpack, + rows, + err, + sched, + sf13, + fc2pack, + base, + misc, + crank, + cl, + my_sh, + lane, + ): + g = self.g + policy = C.l2_policy("evict_first") + pre = self.pre_issue(shpack, base, misc, crank, cl, my_sh, lane, policy) + if lane == 0: + C.mbarrier_arrive_expect_tx(misc + g.b_sbar, g.sched_bytes) + C.bulk_copy_g2s( + misc + g.m_stab, + sched + Int64(cl) * g.sched_bytes, + g.sched_bytes, + misc + g.b_sbar, + C.l2_policy("evict_last"), + ) + C.pdl_wait() + nid = rows * TOP_K + e = Int32(0x7FFFFFFF) + if lane < nid: + e = P.ld_cg_s32(ids + Int64(lane) * 4) + valid = Int32(0) + if (lane < nid) & (Uint32(e) < Uint32(NUM_EXPERTS)): + valid = Int32(1) + if (lane < nid) & (valid == 0): + P.atom_or_global(err, ERR_BAD_ID) + if valid == 0: + e = Int32(0x7FFFFFFF) + # 256-bit set of the distinct experts: lane w < 8 holds word w + word = Uint32(0) + for w in cutlass.range_constexpr(NUM_EXPERTS // 32): + bit = Uint32(0) + if (valid != 0) & ((e >> 5) == w): + bit = Uint32(1) << Uint32(e & 31) + ow = cute.arch.warp_redux_sync(bit, "or") + if lane == w: + word = ow + cnt = P.popc(word) + incl = self.scan8(cnt, lane) + d_raw = cute.arch.shuffle_sync(incl, 7) + if (lane == 0) & (d_raw > TOP_K * rows): + P.atom_or_global(err, ERR_BAD_ID) + d = min(d_raw, TOP_K * rows) + self.write_elist(misc, word, incl - cnt, lane, word, word, 1) + cute.arch.sync_warp() + self.wait_schedule(misc) + row = misc + g.m_stab + 2 * g.sched_row * d + nunits = P.ld_shared_s16(row) + if lane < nunits: + code = P.ld_shared_s16(row + 2 + 2 * lane) + unit = misc + g.m_units + lane * g.unit_bytes + if code < 0: + C.st_shared_u32(unit + g.u_e, Int32(-1)) + C.st_shared_u32(unit + g.u_j, cl) + C.st_shared_u32(unit + g.u_sh, Int32(1)) + else: + C.st_shared_u32( + unit + g.u_e, C.ld_shared_u32(misc + g.m_elist + 4 * (code >> 2)) + ) + C.st_shared_u32(unit + g.u_j, code & 3) + C.st_shared_u32(unit + g.u_sh, Int32(0)) + cute.arch.sync_warp() + pre0 = Int32(0) + if lane == 0: + if nunits > 0: # noqa: SIM102 + if self.unit_field(misc, 0, g.u_sh) == Uint32(0): + C.mbarrier_arrive_expect_tx(misc + g.b_full, BIG) + e0 = Int32(self.unit_field(misc, 0, g.u_e)) + j0 = Int32(self.unit_field(misc, 0, g.u_j)) + self.fc1_chunk( + tmap_ptr, + sf13, + base + OFF_RING, + e0, + j0, + crank, + 0, + misc + g.b_full, + policy, + ) + pre0 = Int32(1) + C.st_shared_u32(misc + g.m_nunits, nunits) + # Every lane arrives (count 32): each lane's own unit-record stores are + # released by its own arrive (see the init of b_tready). + C.mbarrier_arrive(misc + g.b_tready) + cute.arch.sync_warp() + if lane == 0: + self.stream( + tmap_ptr, + shpack, + sf13, + fc2pack, + base, + misc, + crank, + nunits, + pre, + pre0, + policy, + ) + cute.arch.sync_warp() + + @cute.jit + def producer_g2( + self, + tmap_ptr, + ids, + shpack, + rows, + ma, + err, + sched, + sf13, + fc2pack, + base, + misc, + crank, + cl, + my_sh, + lane, + ): + g = self.g + mb = rows - ma + policy = C.l2_policy("evict_first") + pre = self.pre_issue(shpack, base, misc, crank, cl, my_sh, lane, policy) + if lane == 0: + C.mbarrier_arrive_expect_tx(misc + g.b_sbar, g.sched_bytes) + C.bulk_copy_g2s( + misc + g.m_stab, + sched + Int64(cl) * g.sched_bytes, + g.sched_bytes, + misc + g.b_sbar, + C.l2_policy("evict_last"), + ) + C.pdl_wait() + # lane l holds (token l >> 3, slot l & 7) of request A and of request B + ea = Int32(0x7FFFFFFF) + eb = Int32(0x7FFFFFFF) + if lane < TOP_K * ma: + ea = P.ld_cg_s32(ids + Int64(lane) * 4) + if lane < TOP_K * mb: + eb = P.ld_cg_s32(ids + Int64(TOP_K * ma + lane) * 4) + va = Int32(0) + vb = Int32(0) + if (lane < TOP_K * ma) & (Uint32(ea) < Uint32(NUM_EXPERTS)): + va = Int32(1) + if (lane < TOP_K * mb) & (Uint32(eb) < Uint32(NUM_EXPERTS)): + vb = Int32(1) + if ((lane < TOP_K * ma) & (va == 0)) | ((lane < TOP_K * mb) & (vb == 0)): + P.atom_or_global(err, ERR_BAD_ID) + if va == 0: + ea = Int32(0x7FFFFFFF) + if vb == 0: + eb = Int32(0x7FFFFFFF) + wa = Uint32(0) + wb = Uint32(0) + for w in cutlass.range_constexpr(NUM_EXPERTS // 32): + ba = Uint32(0) + bb = Uint32(0) + if (va != 0) & ((ea >> 5) == w): + ba = Uint32(1) << Uint32(ea & 31) + if (vb != 0) & ((eb >> 5) == w): + bb = Uint32(1) << Uint32(eb & 31) + oa = cute.arch.warp_redux_sync(ba, "or") + ob = cute.arch.warp_redux_sync(bb, "or") + if lane == w: + wa = oa + wb = ob + w0 = wa + if cutlass.const_expr(self.merge): + w0 = wa | wb + cnt = P.popc(w0) + incl = self.scan8(cnt, lane) + d0 = cute.arch.shuffle_sync(incl, 7) + pre0 = Int32(0) + if cutlass.const_expr(self.merge): + # unit 0 resolved from the schedule row and the union bitmap; its first + # copies issue now + self.wait_schedule(misc) + row0 = misc + g.m_stab + 2 * g.sched_row * min(d0, g.max_list) + code0 = Int32(-1) + if P.ld_shared_s16(row0) > 0: + code0 = P.ld_shared_s16(row0 + 2) + if code0 >= 0: + idx = code0 >> 2 + mine = (idx >= incl - cnt) & (idx < incl) + who = Uint32(cute.arch.vote_ballot_sync(mine)) + e0 = Int32(0) + if mine: + bits = w0 + for _ in cutlass.range(idx - (incl - cnt), unroll=1): + bits = bits & (bits - Uint32(1)) + e0 = 32 * lane + P.ctz(bits) + src_lane = Int32(0) + if who != Uint32(0): + src_lane = P.ctz(who) + e0 = cute.arch.shuffle_sync(e0, src_lane) + if (lane == 0) & (who != Uint32(0)): + C.mbarrier_arrive_expect_tx(misc + g.b_full, BIG) + self.fc1_chunk( + tmap_ptr, + sf13, + base + OFF_RING, + e0, + code0 & 3, + crank, + 0, + misc + g.b_full, + policy, + ) + pre0 = Int32(1) + pre0 = cute.arch.shuffle_sync(pre0, 0) + self.write_elist(misc, w0, incl - cnt, lane, wa, wb, 0 if self.merge else 1) + d = d0 + if cutlass.const_expr(not self.merge): + cb = P.popc(wb) + ib = self.scan8(cb, lane) + db = cute.arch.shuffle_sync(ib, 7) + self.write_elist(misc, wb, d0 + ib - cb, lane, wa, wb, 2) + d = d0 + db + if (lane == 0) & (d > g.max_list): + P.atom_or_global(err, ERR_BAD_ID) + d = min(d, g.max_list) + cute.arch.sync_warp() + self.wait_schedule(misc) + row = misc + g.m_stab + 2 * g.sched_row * d + nraw = P.ld_shared_s16(row) + nunits = min(nraw, g.max_units) + if (lane == 0) & (nraw > g.max_units): + P.atom_or_global(err, ERR_UNIT_CAP) + gmu = Int32(0) + if lane < nunits: + code = P.ld_shared_s16(row + 2 + 2 * lane) + unit = misc + g.m_units + lane * g.unit_bytes + if code < 0: + C.st_shared_u32(unit + g.u_e, Int32(-1)) + C.st_shared_u32(unit + g.u_j, cl) + C.st_shared_u32(unit + g.u_sh, Int32(1)) + gmu = Int32(1) + if mb > 0: + gmu = Int32(3) + else: + C.st_shared_u32( + unit + g.u_e, C.ld_shared_u32(misc + g.m_elist + 4 * (code >> 2)) + ) + C.st_shared_u32(unit + g.u_j, code & 3) + C.st_shared_u32(unit + g.u_sh, Int32(0)) + gmu = Int32(C.ld_shared_u32(misc + g.m_egm + 4 * (code >> 2))) + C.st_shared_u32(unit + g.u_gm, gmu) + # exchanges of this cluster (one per (unit, group)), the same in its four CTAs + nx = P.popc(gmu) + for o in (16, 8, 4, 2, 1): + nx = nx + cute.arch.shuffle_sync_bfly(nx, o) + if lane == 0: + C.st_shared_u32(misc + g.m_nex, nx) + cute.arch.sync_warp() + if lane == 0: + if (pre0 == 0) & (nunits > 0): # noqa: SIM102 + if self.unit_field(misc, 0, g.u_sh) == Uint32(0): + C.mbarrier_arrive_expect_tx(misc + g.b_full, BIG) + e0 = Int32(self.unit_field(misc, 0, g.u_e)) + j0 = Int32(self.unit_field(misc, 0, g.u_j)) + self.fc1_chunk( + tmap_ptr, + sf13, + base + OFF_RING, + e0, + j0, + crank, + 0, + misc + g.b_full, + policy, + ) + pre0 = Int32(1) + C.st_shared_u32(misc + g.m_nunits, nunits) + # Every lane arrives (count 32): each lane's own unit-record stores are + # released by its own arrive (see the init of b_tready). + C.mbarrier_arrive(misc + g.b_tready) + cute.arch.sync_warp() + if lane == 0: + self.stream( + tmap_ptr, + shpack, + sf13, + fc2pack, + base, + misc, + crank, + nunits, + pre, + pre0, + policy, + ) + cute.arch.sync_warp() + + # ------------------------------------------------------------ warp 1: MMA issue + @cute.jit + def mma_warp(self, base, misc): + g = self.g + lead = T.elect_leader() + C.mbarrier_wait_parity(misc + g.b_tready, 0) + nunits = cute.arch.shuffle_sync(Int32(C.ld_shared_u32(misc + g.m_nunits)), 0) + tm = cute.arch.shuffle_sync(Int32(C.ld_shared_u32(misc + g.m_tmem)), 0) + if nunits > 0: + C.mbarrier_wait_parity(misc + g.b_xready, 0) + self.wait_scales(misc + g.b_sfx_ready, 0) + T.tc_fence_after() + s_ring = base + OFF_RING + it = Int32(0) + for k in cutlass.range(nunits + LOOKAHEAD, unroll=1): + shk = Int32(0) + if k < nunits: + shk = cute.arch.shuffle_sync(Int32(self.unit_field(misc, k, g.u_sh)), 0) + if k < nunits: + b = k & 1 + if k >= 2: + C.mbarrier_wait_parity( + misc + g.b_acc1_empty + 8 * b, ((k >> 1) - 1) & 1 + ) + T.tc_fence_after() + if shk != 0: + for c in cutlass.range(SH_FC1_CHUNKS, unroll=1): + s = it % RING + C.mbarrier_wait_parity( + misc + g.b_full + 8 * s, (it // RING) & 1 + ) + self.wait_scales(misc + g.b_sfa_full + 8 * s, (it // RING) & 1) + T.tc_fence_after() + a0 = T.smem_desc(s_ring + s * BIG, 128, 1024, 0) + b0 = T.smem_desc(base + OFF_XBF + 2048 * c, 128, 1024, 0) + T.tc_burst( + lead, + tm + T_ACC1 + 8 * b, + a0, + b0, + 0, + 0, + Int32(c != 0), + 0, + misc + g.b_empty + 8 * s, + 0, + 0, + mmas=SH_FC1_MMAS, + block_scaled=False, + idesc=IDESC_BF16, + ) + cute.arch.sync_warp() + it = it + 1 + else: + for c in cutlass.range(3, unroll=1): + s = it % RING + C.mbarrier_wait_parity( + misc + g.b_full + 8 * s, (it // RING) & 1 + ) + self.wait_scales(misc + g.b_sfa_full + 8 * s, (it // RING) & 1) + T.tc_fence_after() + a0 = T.smem_desc(s_ring + s * BIG, 16, 1024, 2) + b0 = T.smem_desc(base + OFF_X + 2048 * c, 128, 1024, 0) + T.tc_burst( + lead, + tm + T_ACC1 + 8 * b, + a0, + b0, + tm + T_SFA + 32 * s, + tm + T_SFB_X + 32 * c, + Int32(c != 0), + 0, + misc + g.b_empty + 8 * s, + 0, + 0, + mmas=FC1_MMAS, + block_scaled=True, + idesc=IDESC_NVF4, + ) + cute.arch.sync_warp() + it = it + 1 + T.tc_commit(lead, misc + g.b_acc1_full + 8 * b) + cute.arch.sync_warp() + if (k >= LOOKAHEAD) & (k - LOOKAHEAD < nunits): + i = k - LOOKAHEAD + shi = cute.arch.shuffle_sync(Int32(self.unit_field(misc, i, g.u_sh)), 0) + ab = i % g.acc2_bufs + acc2 = tm + T_ACC2 + if cutlass.const_expr(g.acc2_bufs == 2): # noqa: SIM102 + if ab == 1: + acc2 = tm + T_ACC2_B + C.mbarrier_wait_parity( + misc + g.b_hready + 8 * self.hready_slot(i), self.hready_phase(i) + ) + T.tc_fence_after() + if i >= g.acc2_bufs: + C.mbarrier_wait_parity( + misc + g.b_acc2_empty + 8 * ab, + ((i - g.acc2_bufs) // g.acc2_bufs) & 1, + ) + T.tc_fence_after() + if shi != 0: + bh = T.smem_desc(base + g.off_hbf, 128, 1024, 0) + for c in cutlass.range(SH_FC1_CHUNKS, SH_CHUNKS, unroll=1): + s = it % RING + C.mbarrier_wait_parity( + misc + g.b_full + 8 * s, (it // RING) & 1 + ) + self.wait_scales(misc + g.b_sfa_full + 8 * s, (it // RING) & 1) + T.tc_fence_after() + a0 = T.smem_desc(s_ring + s * BIG, 128, 1024, 0) + T.tc_burst( + lead, + acc2 + 16 * (c - SH_FC1_CHUNKS), + a0, + bh, + 0, + 0, + 0, + 0, + misc + g.b_empty + 8 * s, + 0, + 0, + mmas=SH_FC2_MMAS, + block_scaled=False, + idesc=IDESC_BF16, + ) + cute.arch.sync_warp() + it = it + 1 + else: + hb = i % H_BUFS + bh = T.smem_desc(base + g.off_h + H_BUF * hb, 128, 256, 0) + for c in cutlass.range(3, 5, unroll=1): + s = it % RING + C.mbarrier_wait_parity( + misc + g.b_full + 8 * s, (it // RING) & 1 + ) + self.wait_scales(misc + g.b_sfa_full + 8 * s, (it // RING) & 1) + T.tc_fence_after() + a0 = T.smem_desc(s_ring + s * BIG, 128, 256, 0) + T.tc_burst( + lead, + acc2 + 32 * (2 * c - 6), + a0, + bh, + tm + T_SFA + 32 * s, + tm + T_SFB_H + 4 * hb, + 0, + Int32(c == 3), + misc + g.b_empty + 8 * s, + 0, + 0, + mmas=FC2_MMAS, + block_scaled=True, + idesc=IDESC_NVF4, + ) + cute.arch.sync_warp() + it = it + 1 + T.tc_commit(lead, misc + g.b_acc2_full + 8 * ab) + cute.arch.sync_warp() + cute.arch.sync_warp() + + # -------------------------------------------------- warp 6: scale factors into TMEM + @cute.jit + def sf_warp(self, base, misc): + g = self.g + lead = T.elect_leader() + C.mbarrier_wait_parity(misc + g.b_tready, 0) + nunits = cute.arch.shuffle_sync(Int32(C.ld_shared_u32(misc + g.m_nunits)), 0) + tm = cute.arch.shuffle_sync(Int32(C.ld_shared_u32(misc + g.m_tmem)), 0) + if nunits > 0: + C.mbarrier_wait_parity(misc + g.b_xready, 0) + T.tc_fence_after() + d0 = T.smem_desc(base + OFF_XSF, 0, 128, 0) + for q in cutlass.range(6, unroll=1): + T.tc_burst( + lead, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + tm + T_SFB_X + 16 * q, + d0 + Uint64(32 * q), + cps=X_CPS, + commit=False, + ) + T.tc_commit(lead, misc + g.b_sfx_ready) + cute.arch.sync_warp() + it = Int32(0) + for k in cutlass.range(nunits + LOOKAHEAD, unroll=1): + for part in cutlass.range_constexpr(2): + i = k if part == 0 else k - LOOKAHEAD + if (i >= 0) & (i < nunits): + ush = cute.arch.shuffle_sync( + Int32(self.unit_field(misc, i, g.u_sh)), 0 + ) + c0 = Int32(0) + c1 = Int32(3) + if cutlass.const_expr(part == 0): + if ush != 0: + c1 = Int32(SH_FC1_CHUNKS) + else: + c0 = Int32(3) + c1 = Int32(5) + if ush != 0: + c0 = Int32(SH_FC1_CHUNKS) + c1 = Int32(SH_CHUNKS) + else: + C.mbarrier_wait_parity( + misc + g.b_hready + 8 * self.hready_slot(i), + self.hready_phase(i), + ) + T.tc_fence_after() + hb = i % H_BUFS + T.tc_burst( + lead, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + tm + T_SFB_H + 4 * hb, + T.smem_desc( + base + g.off_h + H_BUF * hb + 256, 0, 128, 0 + ), + cps=H_CPS, + commit=False, + ) + cute.arch.sync_warp() + for c in cutlass.range(c0, c1, unroll=1): + s = it % RING + C.mbarrier_wait_parity( + misc + g.b_full + 8 * s, (it // RING) & 1 + ) + T.tc_fence_after() + if ush != 0: + T.tc_commit(lead, misc + g.b_sfa_full + 8 * s) + else: + if cutlass.const_expr(part == 0): + d = T.smem_desc( + base + OFF_RING + s * BIG + 32768, 0, 128, 0 + ) + T.tc_burst( + lead, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + misc + g.b_sfa_full + 8 * s, + tm + T_SFA + 32 * s, + d, + cps=FC1_CPS, + ) + else: + d = T.smem_desc( + base + OFF_RING + s * BIG + 16384, 0, 128, 0 + ) + T.tc_burst( + lead, + 0, + 0, + 0, + 0, + 0, + 0, + Int32(c == 3), + misc + g.b_sfa_full + 8 * s, + tm + T_SFA + 32 * s, + d, + cps=FC2_CPS, + ) + cute.arch.sync_warp() + it = it + 1 + cute.arch.sync_warp() + + # ---------------------------------------------- warp 7 (g2): receive-buffer release + @cute.jit + def release_warp(self, misc, crank, lane): + g = self.g + C.mbarrier_wait_parity(misc + g.b_tready, 0) + nex = cute.arch.shuffle_sync(Int32(C.ld_shared_u32(misc + g.m_nex)), 0) + cute.arch.cluster_wait() + if lane == 0: + for e in cutlass.range(nex - g.recv_bufs, unroll=1): + p = e % g.recv_bufs + C.mbarrier_wait_parity( + misc + g.b_consumed + 8 * p, (e // g.recv_bufs) & 1 + ) + for pr in cutlass.range_constexpr(CLUSTER): + if crank != pr: + P.mbarrier_arrive_remote( + C.mapa_shared_cluster(misc + g.b_pfree + 8 * p, pr) + ) + cute.arch.sync_warp() + + # --------------------------------------- warps 2..5: x images, h chain, FC2 readout + @cute.jit + def zero_fills(self, base, et): + zero = (Uint32(0), Uint32(0), Uint32(0), Uint32(0)) + for i in cutlass.range(et, 3456 // 16, 128, unroll=1): + C.st_shared_v4(base + OFF_XSF + 16 * i, zero) + for i in cutlass.range(et, H_BUFS * H_BUF // 16, 128, unroll=1): + C.st_shared_v4(base + self.g.off_h + 16 * i, zero) + for i in cutlass.range(et, 1024 // 16, 128, unroll=1): + C.st_shared_v4(base + self.g.off_hbf + 16 * i, zero) + + @cute.jit + def quantize_x(self, x, gs1, ma, mb, base, misc, crank, et, lane): + """The activations x (BF16) of this CTA's K range into the BF16 image + (shared-expert operand) and the NVFP4 image + scale atoms (routed operand). + Image row n: request A row n (n < 4, valid n < ma) or request B row n - 4 + (valid n - 4 < mb); invalid rows are zero.""" + xbf = base + OFF_XBF + raw = cute.make_rmem_tensor(cute.make_layout(48), Uint32) + for i in cutlass.range_constexpr(6): + b = et + 128 * i + n = b // 96 + kb = b % 96 + for w in cutlass.range_constexpr(8): + raw[8 * i + w] = Uint32(0) + ok = n < ma + grow = n + if n >= REQ_ROWS: + ok = n - REQ_ROWS < mb + grow = ma + n - REQ_ROWS + if ok: + src = x + (Int64(grow) * HIDDEN + K_CTA * crank + 16 * kb) * 2 + lo = P.ld_cg_v4_u32(src) + hi = P.ld_cg_v4_u32(src + 16) + for w in cutlass.range_constexpr(4): + raw[8 * i + w] = lo[w] + raw[8 * i + 4 + w] = hi[w] + for i in cutlass.range_constexpr(6): + b = et + 128 * i + n = b // 96 + kb = b % 96 + C.st_shared_v4( + xbf + (2 * kb) * 128 + n * 16, tuple(raw[8 * i + w] for w in range(4)) + ) + C.st_shared_v4( + xbf + (2 * kb + 1) * 128 + n * 16, + tuple(raw[8 * i + 4 + w] for w in range(4)), + ) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + for i in cutlass.range(6, unroll=1): + b = et + 128 * i + n = b // 96 + kb = b % 96 + w0 = Uint32(0) + w1 = Uint32(0) + sfb = Uint32(0) + ok = n < ma + if n >= REQ_ROWS: + ok = n - REQ_ROWS < mb + if ok: + u0 = C.ld_shared_v4(xbf + (2 * kb) * 128 + n * 16) + u1 = C.ld_shared_v4(xbf + (2 * kb + 1) * 128 + n * 16) + v = [] + for word in (*u0, *u1): + lo, hi = P.bf16x2_unpack(word) + v.append(lo) + v.append(hi) + amax = Float32(0.0) + for q in cutlass.range_constexpr(16): + amax = T.max_f32(amax, T.abs_f32(v[q])) + osc, sfb = T.nvfp4_block_scale(amax, gs1) + q0 = [P.mul_f32(v[q], osc) for q in range(8)] + q1 = [P.mul_f32(v[8 + q], osc) for q in range(8)] + w0 = T.e2m1x8(*q0) + w1 = T.e2m1x8(*q1) + P.st_shared_v2_u32( + base + OFF_X + (kb >> 1) * 128 + n * 16 + (kb & 1) * 8, w0, w1 + ) + P.st_shared_b8(base + OFF_XSF + (kb >> 2) * 128 + n * 16 + (kb & 3), sfb) + C.fence_proxy_async_shared_cta() + cute.arch.sync_warp() + if lane == 0: + C.mbarrier_arrive(misc + self.g.b_xready) + + @cute.jit + def token_scales_g1( + self, ids, topw, g1a, g2a, rows, ids_fin, w_fin, g2s, misc, block, nunits, lane + ): + g = self.g + nid = rows * TOP_K + eid = Int32(-1) + wt = Float32(0.0) + if lane < nid: + eid = P.ld_cg_s32(ids + Int64(lane) * 4) + wt = P.ld_cg_f32(topw + Int64(lane) * 4) + mysh = Int32(0) + my_unit = Int32(-1) + if lane < nunits: + mysh = Int32(self.unit_field(misc, lane, g.u_sh)) + if mysh == 0: + my_unit = Int32(self.unit_field(misc, lane, g.u_e)) + g1 = Float32(0.0) + g2 = Float32(0.0) + if my_unit >= 0: + g1 = P.ld_cg_f32(g1a + Int64(my_unit) * 4) + g2 = P.ld_cg_f32(g2a + Int64(my_unit) * 4) + tk = cute.make_rmem_tensor(cute.make_layout(REQ_ROWS), Float32) + sl = cute.make_rmem_tensor(cute.make_layout(REQ_ROWS), Int32) + for n in cutlass.range_constexpr(REQ_ROWS): + tk[n] = Float32(0.0) + sl[n] = Int32(-1) + for u in cutlass.range(nunits, unroll=1): + e = Int32(self.unit_field(misc, u, g.u_e)) + msk = Uint32(cute.arch.vote_ballot_sync(eid == e)) + for n in cutlass.range_constexpr(REQ_ROWS): + bits = (msk >> Uint32(8 * n)) & Uint32(0xFF) + src = Int32(0) + if bits != Uint32(0): + src = 8 * n + P.ctz(bits) + w = cute.arch.shuffle_sync(wt, src) + if lane == u: + if bits != Uint32(0): + tk[n] = w + sl[n] = P.ctz(bits) + else: + tk[n] = Float32(0.0) + sl[n] = Int32(-1) + if (block == 0) & (lane < nid): + self.slot_record(ids_fin, w_fin, g2s, g2a, lane, eid, wt) + if lane < nunits: + unit = misc + g.m_units + lane * g.unit_bytes + if mysh != 0: + C.st_shared_f32(unit + g.u_g1, Float32(1.0)) + for n in cutlass.range_constexpr(REQ_ROWS): + one = Float32(0.0) + if n < rows: + one = Float32(1.0) + C.st_shared_f32(unit + g.u_tok + 4 * n, one) + C.st_shared_u32(unit + g.u_slot + 4 * n, Int32(-1)) + else: + C.st_shared_f32(unit + g.u_g1, g1) + for n in cutlass.range_constexpr(REQ_ROWS): + C.st_shared_f32(unit + g.u_tok + 4 * n, P.mul_f32(tk[n], g2)) + C.st_shared_u32(unit + g.u_slot + 4 * n, sl[n]) + + @cute.jit + def slot_record(self, ids_fin, w_fin, g2s, g2a, at, eid, wt): + """Slot ``at``: its expert id (-1 if out of range), routing weight and FC2 + alpha.""" + ok = Uint32(eid) < Uint32(NUM_EXPERTS) + g2v = Float32(0.0) + sid = Int32(-1) + if ok: + g2v = P.ld_cg_f32(g2a + Int64(eid) * 4) + sid = eid + P.st_global_b32(ids_fin + Int64(at) * 4, sid) + P.st_global_b32f(w_fin + Int64(at) * 4, wt) + P.st_global_b32f(g2s + Int64(at) * 4, g2v) + + @cute.jit + def token_scales_g2( + self, + ids, + topw, + g1a, + g2a, + ma, + mb, + ids_fin, + w_fin, + g2s, + misc, + block, + nunits, + lane, + ): + g = self.g + ea = Int32(-1) + wa = Float32(0.0) + eb = Int32(-1) + wb = Float32(0.0) + if lane < TOP_K * ma: + ea = P.ld_cg_s32(ids + Int64(lane) * 4) + wa = P.ld_cg_f32(topw + Int64(lane) * 4) + if lane < TOP_K * mb: + eb = P.ld_cg_s32(ids + Int64(TOP_K * ma + lane) * 4) + wb = P.ld_cg_f32(topw + Int64(TOP_K * ma + lane) * 4) + mysh = Int32(0) + my_unit = Int32(-1) + mygm = Int32(0) + if lane < nunits: + mysh = Int32(self.unit_field(misc, lane, g.u_sh)) + # only the two group bits are used: the mask's low byte + mygm = P.ld_shared_u8(misc + g.m_units + lane * g.unit_bytes + g.u_gm) + if mysh == 0: + my_unit = Int32(self.unit_field(misc, lane, g.u_e)) + g1 = Float32(0.0) + g2 = Float32(0.0) + if my_unit >= 0: + g1 = P.ld_cg_f32(g1a + Int64(my_unit) * 4) + g2 = P.ld_cg_f32(g2a + Int64(my_unit) * 4) + tk = cute.make_rmem_tensor(cute.make_layout(2 * REQ_ROWS), Float32) + sl = cute.make_rmem_tensor(cute.make_layout(2 * REQ_ROWS), Int32) + for r in cutlass.range_constexpr(2 * REQ_ROWS): + tk[r] = Float32(0.0) + sl[r] = Int32(-1) + for u in cutlass.range(nunits, unroll=1): + e = Int32(self.unit_field(misc, u, g.u_e)) + mka = Uint32(cute.arch.vote_ballot_sync(ea == e)) + mkb = Uint32(cute.arch.vote_ballot_sync(eb == e)) + for t in cutlass.range_constexpr(REQ_ROWS): + ba = (mka >> Uint32(8 * t)) & Uint32(0xFF) + bb = (mkb >> Uint32(8 * t)) & Uint32(0xFF) + sa = Int32(0) + sb = Int32(0) + if ba != Uint32(0): + sa = 8 * t + P.ctz(ba) + if bb != Uint32(0): + sb = 8 * t + P.ctz(bb) + xa = cute.arch.shuffle_sync(wa, sa) + xb = cute.arch.shuffle_sync(wb, sb) + if lane == u: + if ba != Uint32(0): + tk[t] = xa + sl[t] = P.ctz(ba) + else: + tk[t] = Float32(0.0) + sl[t] = Int32(-1) + if bb != Uint32(0): + tk[REQ_ROWS + t] = xb + sl[REQ_ROWS + t] = P.ctz(bb) + else: + tk[REQ_ROWS + t] = Float32(0.0) + sl[REQ_ROWS + t] = Int32(-1) + if block == 0: + if lane < TOP_K * ma: + self.slot_record(ids_fin, w_fin, g2s, g2a, lane, ea, wa) + if lane < TOP_K * mb: + self.slot_record(ids_fin, w_fin, g2s, g2a, TOP_K * ma + lane, eb, wb) + if lane < nunits: + unit = misc + g.m_units + lane * g.unit_bytes + if mysh != 0: + C.st_shared_f32(unit + g.u_g1, Float32(1.0)) + else: + C.st_shared_f32(unit + g.u_g1, g1) + for r in cutlass.range_constexpr(2 * REQ_ROWS): + own = (mygm >> (r // REQ_ROWS)) & 1 + tok = Float32(0.0) + slot = Int32(-1) + if mysh != 0: + valid = r < ma + if cutlass.const_expr(r >= REQ_ROWS): + valid = r - REQ_ROWS < mb + if (own != 0) & valid: + tok = Float32(1.0) + else: + if own != 0: + tok = P.mul_f32(tk[r], g2) + slot = sl[r] + C.st_shared_f32(unit + g.u_tok + 4 * r, tok) + C.st_shared_u32(unit + g.u_slot + 4 * r, slot) + + @cute.jit + def routed_h(self, acc, g1, gs2, nrows, r0, hbuf, qd, lane): + """Routed unit: h = SiLU(g1 gate) * (g1 up) of one group's 4 columns, NVFP4 into + image rows r0 .. r0 + nrows - 1 (lane i of a quadrant: gate iff i & 8, its up + partner i ^ 8).""" + gate = (lane & 8) != 0 + hv = [] + am = [] + for n in cutlass.range_constexpr(REQ_ROWS): + mine = P.mul_f32(g1, acc[n]) + up = cute.arch.shuffle_sync_bfly(mine, 8) + h = P.mul_f32(T.silu_fast(mine), up) + hv.append(h) + a = Float32(0.0) + if gate: + a = T.abs_f32(h) + am.append(a) + for n in cutlass.range_constexpr(REQ_ROWS): + for off in (16, 4, 2, 1): + am[n] = T.max_f32(am[n], cute.arch.shuffle_sync_bfly(am[n], off)) + hq = [] + sfb = [] + for n in cutlass.range_constexpr(REQ_ROWS): + osc, sf = T.nvfp4_block_scale(am[n], gs2) + hq.append(P.mul_f32(hv[n], osc)) + sfb.append(sf) + wbase = (lane >> 2) & 1 + words = [] + for n in cutlass.range_constexpr(REQ_ROWS): + q = [ + cute.arch.shuffle_sync(hq[n], 8 + 4 * wbase + (kk >> 1) + 16 * (kk & 1)) + for kk in range(8) + ] + words.append(T.e2m1x8(*q)) + for n in cutlass.range_constexpr(REQ_ROWS): + if n < nrows: + r = r0 + n + if (lane == 8) | (lane == 12): + off = Int32(0) + if lane == 12: + off = Int32(4) + C.st_shared_u32( + hbuf + (qd >> 1) * 128 + r * 16 + (qd & 1) * 8 + off, words[n] + ) + if lane == 8: + P.st_shared_b8(hbuf + 256 + r * 16 + qd, sfb[n]) + + @cute.jit + def shared_h(self, acc, nrows, r0, hbf, qd, lane): + """Shared quarter: stock BF16 rounding points (gate / up to BF16, SiLU to BF16, + product to BF16) into BF16 image rows r0 .. r0 + nrows - 1.""" + for n in cutlass.range_constexpr(REQ_ROWS): + gb = P.round_bf16(acc[n]) + ub = cute.arch.shuffle_sync_down(gb, 16) + sv = P.round_bf16(T.silu_accurate(gb)) + hb = Uint32(C.cvt_rn_bf16_f32(P.mul_f32(sv, ub))) + hn = cute.arch.shuffle_sync_down(hb, 1) + if (n < nrows) & (lane < 16) & ((lane & 1) == 0): + kc = 2 * qd + (lane >> 3) + C.st_shared_u32( + hbf + kc * 128 + (r0 + n) * 16 + (lane & 7) * 2, + hb | (hn << Uint32(16)), + ) + + @cute.jit + def epilogue_g1( + self, + x, + ids, + topw, + g1a, + g2a, + gs1, + gs2, + rows, + shpartials, + pq, + ids_fin, + w_fin, + g2s, + base, + misc, + tm, + crank, + block, + warp, + lane, + tid, + ): + g = self.g + qd = warp & 3 + et = tid - 64 + row = 32 * qd + lane + self.zero_fills(base, et) + self.wait_inputs() + self.quantize_x(x, gs1, rows, 0, base, misc, crank, et, lane) + C.mbarrier_wait_parity(misc + g.b_tready, 0) + nunits = Int32(C.ld_shared_u32(misc + g.m_nunits)) + if warp == 2: + self.token_scales_g1( + ids, + topw, + g1a, + g2a, + rows, + ids_fin, + w_fin, + g2s, + misc, + block, + nunits, + lane, + ) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + cute.arch.cluster_wait() + lane_base = tm + Int32(32 * qd) * 65536 + racc = cute.make_rmem_tensor(cute.make_layout(FC2_TILES * REQ_ROWS), Float32) + for t in cutlass.range_constexpr(FC2_TILES * REQ_ROWS): + racc[t] = Float32(0.0) + recv = base + OFF_RECV + for k in cutlass.range(nunits + LOOKAHEAD, unroll=1): + if k < nunits: + i = k + p = i % g.recv_bufs + b = i & 1 + use = i // g.recv_bufs + if et == 0: + C.mbarrier_arrive_expect_tx( + misc + g.b_precv + 8 * p, 3 * 128 * 16, ".relaxed.cluster" + ) + C.mbarrier_wait_parity(misc + g.b_acc1_full + 8 * b, (i >> 1) & 1) + T.tc_fence_after() + v = T.tmem_load4(lane_base + T_ACC1 + 8 * b) + T.tc_fence_before() + cute.arch.sync_warp() + if lane == 0: + C.mbarrier_arrive(misc + g.b_acc1_empty + 8 * b) + if use >= 1: + C.mbarrier_wait_parity( + misc + g.b_pfree + 8 * p, (use - 1) & 1, ".acquire.cluster" + ) + laddr = recv + ((p * 4 + crank) * 128 + row) * 16 + lbar = misc + g.b_precv + 8 * p + for pr in cutlass.range_constexpr(CLUSTER): + if crank != pr: + P.st_async_v4_f32( + C.mapa_shared_cluster(laddr, pr), + C.mapa_shared_cluster(lbar, pr), + *v, + ) + C.mbarrier_wait_parity( + misc + g.b_precv + 8 * p, use & 1, ".acquire.cluster" + ) + acc = [Float32(0.0), Float32(0.0), Float32(0.0), Float32(0.0)] + for src in cutlass.range_constexpr(CLUSTER): + p0, p1, p2, p3 = v + if crank != src: + p0, p1, p2, p3 = P.ld_shared_v4_f32( + recv + ((p * 4 + src) * 128 + row) * 16 + ) + acc = [ + P.add_f32(acc[0], p0), + P.add_f32(acc[1], p1), + P.add_f32(acc[2], p2), + P.add_f32(acc[3], p3), + ] + cute.arch.sync_warp() + if (lane == 0) & (i + g.recv_bufs < nunits): + for pr in cutlass.range_constexpr(CLUSTER): + if crank != pr: + P.mbarrier_arrive_remote( + C.mapa_shared_cluster(misc + g.b_pfree + 8 * p, pr) + ) + if self.unit_field(misc, i, g.u_sh) != Uint32(0): + self.shared_h(acc, rows, 0, base + g.off_hbf, qd, lane) + else: + self.routed_h( + acc, + C.ld_shared_f32(misc + g.m_units + i * g.unit_bytes + g.u_g1), + gs2, + rows, + 0, + base + g.off_h + H_BUF * (i % H_BUFS), + qd, + lane, + ) + C.fence_proxy_async_shared_cta() + cute.arch.sync_warp() + if lane == 0: + C.mbarrier_arrive(misc + g.b_hready + 8 * self.hready_slot(i)) + if (k >= LOOKAHEAD) & (k - LOOKAHEAD < nunits): + i = k - LOOKAHEAD + C.mbarrier_wait_parity(misc + g.b_acc2_full, i & 1) + T.tc_fence_after() + C.fence_proxy_async_shared_cta() + d = T.tmem_load4x12(lane_base + T_ACC2) + T.tc_fence_before() + cute.arch.sync_warp() + if lane == 0: + C.mbarrier_arrive(misc + g.b_acc2_empty) + if self.unit_field(misc, i, g.u_sh) == Uint32(0): + for n in cutlass.range_constexpr(REQ_ROWS): + sl = Int32(self.unit_field(misc, i, g.u_slot + 4 * n)) + if (n < rows) & (sl >= 0): + uj = Int32(self.unit_field(misc, i, g.u_j)) + q = ( + pq + + ( + (((Int64(uj) * rows + n) * TOP_K + sl) * HIDDEN) + + K_CTA * crank + + row + ) + * 4 + ) + for t in cutlass.range_constexpr(FC2_TILES): + P.st_global_f32(q + 512 * t, d[4 * t + n]) + else: + for n in cutlass.range_constexpr(REQ_ROWS): + s = C.ld_shared_f32( + misc + g.m_units + i * g.unit_bytes + g.u_tok + 4 * n + ) + for t in cutlass.range_constexpr(FC2_TILES): + racc[4 * t + n] = C.fma_rn(s, d[4 * t + n], racc[4 * t + n]) + if cutlass.const_expr(self.shared): # noqa: SIM102 + # the shared quarter's partial is stored apart (stock adds it after + # the routed finalize), then its accumulator restarts + if self.unit_field(misc, i, g.u_sh) != Uint32(0): + uj = Int32(self.unit_field(misc, i, g.u_j)) + q = ( + shpartials + + ((Int64(uj) * rows) * HIDDEN + K_CTA * crank + row) * 4 + ) + for n in cutlass.range_constexpr(REQ_ROWS): + if n < rows: + for t in cutlass.range_constexpr(FC2_TILES): + P.st_global_f32( + q + (Int64(n) * HIDDEN + 128 * t) * 4, + racc[4 * t + n], + ) + for t in cutlass.range_constexpr(FC2_TILES * REQ_ROWS): + racc[t] = Float32(0.0) + + @cute.jit + def epilogue_g2( + self, + x, + ids, + topw, + g1a, + g2a, + gs1, + gs2, + rows, + ma, + shpartials, + pq, + ids_fin, + w_fin, + g2s, + base, + misc, + tm, + crank, + block, + warp, + lane, + tid, + ): + g = self.g + mb = rows - ma + qd = warp & 3 + et = tid - 64 + row = 32 * qd + lane + self.zero_fills(base, et) + self.wait_inputs() + self.quantize_x(x, gs1, ma, mb, base, misc, crank, et, lane) + C.mbarrier_wait_parity(misc + g.b_tready, 0) + nunits = Int32(C.ld_shared_u32(misc + g.m_nunits)) + nex = Int32(C.ld_shared_u32(misc + g.m_nex)) + if warp == 2: + self.token_scales_g2( + ids, + topw, + g1a, + g2a, + ma, + mb, + ids_fin, + w_fin, + g2s, + misc, + block, + nunits, + lane, + ) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + cute.arch.cluster_wait() + lane_base = tm + Int32(32 * qd) * 65536 + recv = base + OFF_RECV + ex = Int32(0) + for k in cutlass.range(nunits + LOOKAHEAD, unroll=1): + if k < nunits: + i = k + b = i & 1 + gm = Int32(self.unit_field(misc, i, g.u_gm)) + ng = P.popc(gm) + g0 = Int32(1) + if (gm & 1) != 0: + g0 = Int32(0) + if et == 0: + C.mbarrier_arrive_expect_tx( + misc + g.b_precv + 8 * (ex % g.recv_bufs), + 3 * 128 * 16, + ".relaxed.cluster", + ) + if ng == 2: + C.mbarrier_arrive_expect_tx( + misc + g.b_precv + 8 * ((ex + 1) % g.recv_bufs), + 3 * 128 * 16, + ".relaxed.cluster", + ) + C.mbarrier_wait_parity(misc + g.b_acc1_full + 8 * b, (i >> 1) & 1) + T.tc_fence_after() + va, vb = T.tmem_load4_pair( + lane_base + T_ACC1 + 8 * b + 4 * g0, + lane_base + T_ACC1 + 8 * b + 4, + Int32(ng == 2), + ) + T.tc_fence_before() + cute.arch.sync_warp() + if lane == 0: + C.mbarrier_arrive(misc + g.b_acc1_empty + 8 * b) + g1 = C.ld_shared_f32(misc + g.m_units + i * g.unit_bytes + g.u_g1) + ush = self.unit_field(misc, i, g.u_sh) + for gi in cutlass.range(ng, unroll=1): + grp = g0 + if gi != 0: + grp = Int32(1) + mg = ma + if grp != 0: + mg = mb + v0, v1, v2, v3 = va + if gi != 0: + v0, v1, v2, v3 = vb + e = ex + gi + p = e % g.recv_bufs + use = e // g.recv_bufs + if use >= 1: + C.mbarrier_wait_parity( + misc + g.b_pfree + 8 * p, (use - 1) & 1, ".acquire.cluster" + ) + lbar = misc + g.b_precv + 8 * p + for pr in cutlass.range_constexpr(CLUSTER): + if crank != pr: + # our slot in peer pr's buffer: our rank, skipping the + # peer's own + sslot = crank + if crank > pr: + sslot = crank - 1 + laddr = recv + ((p * 3 + sslot) * 128 + row) * 16 + P.st_async_v4_f32( + C.mapa_shared_cluster(laddr, pr), + C.mapa_shared_cluster(lbar, pr), + v0, + v1, + v2, + v3, + ) + C.mbarrier_wait_parity( + misc + g.b_precv + 8 * p, use & 1, ".acquire.cluster" + ) + acc = [Float32(0.0), Float32(0.0), Float32(0.0), Float32(0.0)] + for src in cutlass.range_constexpr(CLUSTER): + p0, p1, p2, p3 = v0, v1, v2, v3 + if crank != src: + rslot = Int32(src) + if src > crank: + rslot = Int32(src - 1) + p0, p1, p2, p3 = P.ld_shared_v4_f32( + recv + ((p * 3 + rslot) * 128 + row) * 16 + ) + acc = [ + P.add_f32(acc[0], p0), + P.add_f32(acc[1], p1), + P.add_f32(acc[2], p2), + P.add_f32(acc[3], p3), + ] + cute.arch.sync_warp() + if (lane == 0) & (e + g.recv_bufs < nex): + C.mbarrier_arrive(misc + g.b_consumed + 8 * p) + if ush != Uint32(0): + self.shared_h( + acc, mg, REQ_ROWS * grp, base + g.off_hbf, qd, lane + ) + else: + self.routed_h( + acc, + g1, + gs2, + mg, + REQ_ROWS * grp, + base + g.off_h + H_BUF * (i % H_BUFS), + qd, + lane, + ) + ex = ex + ng + C.fence_proxy_async_shared_cta() + cute.arch.sync_warp() + if lane == 0: + C.mbarrier_arrive(misc + g.b_hready + 8 * self.hready_slot(i)) + if (k >= LOOKAHEAD) & (k - LOOKAHEAD < nunits): + i = k - LOOKAHEAD + ab = i % 2 + C.mbarrier_wait_parity(misc + g.b_acc2_full + 8 * ab, (i // 2) & 1) + T.tc_fence_after() + C.fence_proxy_async_shared_cta() + acc2 = lane_base + T_ACC2 + if ab == 1: + acc2 = lane_base + T_ACC2_B + gm = Int32(self.unit_field(misc, i, g.u_gm)) + ng = P.popc(gm) + g0 = Int32(1) + if (gm & 1) != 0: + g0 = Int32(0) + for gi in cutlass.range(ng, unroll=1): + grp = g0 + if gi != 0: + grp = Int32(1) + mg = ma + if grp != 0: + mg = mb + d = T.tmem_load4x12(acc2 + 4 * grp) + if gi == ng - 1: + T.tc_fence_before() + cute.arch.sync_warp() + if lane == 0: + C.mbarrier_arrive(misc + g.b_acc2_empty + 8 * ab) + # unit fields are read per group, where the readout uses them (the + # reference engine's order) + if self.unit_field(misc, i, g.u_sh) == Uint32(0): + for n in cutlass.range_constexpr(REQ_ROWS): + r = REQ_ROWS * grp + n + sl = Int32( + C.ld_shared_u32( + misc + + g.m_units + + i * g.unit_bytes + + g.u_slot + + 4 * r + ) + ) + if (n < mg) & (sl >= 0): + uj = Int32(self.unit_field(misc, i, g.u_j)) + grow = r + if r >= REQ_ROWS: + grow = ma + r - REQ_ROWS + q = ( + pq + + ( + ( + ((Int64(uj) * rows + grow) * TOP_K + sl) + * HIDDEN + ) + + K_CTA * crank + + row + ) + * 4 + ) + for t in cutlass.range_constexpr(FC2_TILES): + P.st_global_f32(q + 512 * t, d[4 * t + n]) + else: + if cutlass.const_expr(self.shared): + for n in cutlass.range_constexpr(REQ_ROWS): + if n < mg: + r = REQ_ROWS * grp + n + s = C.ld_shared_f32( + misc + + g.m_units + + i * g.unit_bytes + + g.u_tok + + 4 * r + ) + uj = Int32(self.unit_field(misc, i, g.u_j)) + grow = r + if r >= REQ_ROWS: + grow = ma + r - REQ_ROWS + q = ( + shpartials + + ( + (Int64(uj) * rows + grow) * HIDDEN + + K_CTA * crank + + row + ) + * 4 + ) + for t in cutlass.range_constexpr(FC2_TILES): + P.st_global_f32( + q + 512 * t, + C.fma_rn(s, d[4 * t + n], Float32(0.0)), + ) + + # ------------------------------------------------------------ kernel + @cute.kernel + def kernel( + self, + tmap: cutlass.GridConstant[TensorMap], + x: Int64, + ids: Int64, + topw: Int64, + shpack: Int64, + g1a: Int64, + g2a: Int64, + gs1: Float32, + gs2: Float32, + rows: Int32, + split: Int32, + err: Int64, + sched: Int64, + shpartials: Int64, + pq: Int64, + ids_fin: Int64, + w_fin: Int64, + g2s: Int64, + sf13: Int64, + fc2pack: Int64, + ): + g = self.g + tid, _, _ = cute.arch.thread_idx() + block, _, _ = cute.arch.block_idx() + warp = tid // 32 + lane = tid % 32 + sm = SmemAllocator().allocate_tensor(Uint8, cute.make_layout(g.smem), 1024) + base = Int32(sm.iterator.toint()) + misc = base + g.off_misc + crank = Int32(cute.arch.block_idx_in_cluster()) + cl = block // CLUSTER + # the PTX carries no cluster-shape directive (the launch sets it): refuse any + # other shape + P.trap_if(Int32(cute.arch.cluster_size() != CLUSTER)) + C.pdl_launch_dependents() + if tid == 0: + for s in cutlass.range_constexpr(RING): + C.mbarrier_init(misc + g.b_full + 8 * s, 1) + C.mbarrier_init(misc + g.b_empty + 8 * s, 1) + C.mbarrier_init(misc + g.b_xready, 4) + # one arrive per lane of the table-building warp: each lane publishes its + # own unit-record stores (PTX memory model; no reliance on the warp barrier) + C.mbarrier_init(misc + g.b_tready, 32) + C.mbarrier_init(misc + g.b_sbar, 1) + for b in cutlass.range_constexpr(2): + C.mbarrier_init(misc + g.b_acc1_full + 8 * b, 1) + C.mbarrier_init(misc + g.b_acc1_empty + 8 * b, 4) + for b in cutlass.range_constexpr(g.acc2_bufs): + C.mbarrier_init(misc + g.b_acc2_full + 8 * b, 1) + C.mbarrier_init(misc + g.b_acc2_empty + 8 * b, 4) + for s in cutlass.range_constexpr(RING): + C.mbarrier_init(misc + g.b_sfa_full + 8 * s, 1) + C.mbarrier_init(misc + g.b_sfx_ready, 1) + for b in cutlass.range_constexpr(H_BUFS): + C.mbarrier_init(misc + g.b_hready + 8 * b, 4) + for p in cutlass.range_constexpr(g.recv_bufs): + C.mbarrier_init(misc + g.b_precv + 8 * p, 1) + C.mbarrier_init(misc + g.b_pfree + 8 * p, g.pfree_count) + if cutlass.const_expr(g.two): + C.mbarrier_init(misc + g.b_consumed + 8 * p, 4) + C.fence_mbarrier_init_release_cluster() + if warp == 1: + T.tmem_alloc(misc + g.m_tmem, 512) + T.tc_fence_before() + cute.arch.sync_threads() + T.tc_fence_after() + cute.arch.cluster_arrive_relaxed(aligned=True) + # TMEM base, read by every thread once the allocation is visible + tm = Int32(C.ld_shared_u32(misc + g.m_tmem)) + my_sh = Int32(0) + if cl < SH_CLUSTERS: + my_sh = Int32(1) + if warp == 0: + if cutlass.const_expr(g.two): + self.producer_g2( + tmap.get_ptr().toint(), + ids, + shpack, + rows, + split, + err, + sched, + sf13, + fc2pack, + base, + misc, + crank, + cl, + my_sh, + lane, + ) + else: + self.producer_g1( + tmap.get_ptr().toint(), + ids, + shpack, + rows, + err, + sched, + sf13, + fc2pack, + base, + misc, + crank, + cl, + my_sh, + lane, + ) + elif warp == 1: + self.mma_warp(base, misc) + elif warp == SF_WARP: + self.sf_warp(base, misc) + elif warp == RELEASE_WARP: + if cutlass.const_expr(g.two): + self.release_warp(misc, crank, lane) + else: + if cutlass.const_expr(g.two): + self.epilogue_g2( + x, + ids, + topw, + g1a, + g2a, + gs1, + gs2, + rows, + split, + shpartials, + pq, + ids_fin, + w_fin, + g2s, + base, + misc, + tm, + crank, + block, + warp, + lane, + tid, + ) + else: + self.epilogue_g1( + x, + ids, + topw, + g1a, + g2a, + gs1, + gs2, + rows, + shpartials, + pq, + ids_fin, + w_fin, + g2s, + base, + misc, + tm, + crank, + block, + warp, + lane, + tid, + ) + if (warp < 2) | (warp == SF_WARP): + cute.arch.cluster_wait() + T.tc_fence_before() + cute.arch.sync_threads() + T.tc_fence_after() + if warp == 1: + T.tmem_free(tm, 512) + cute.arch.cluster_arrive(aligned=True) + cute.arch.cluster_wait() diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/engine16.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/engine16.py new file mode 100644 index 00000000..5abb9632 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/engine16.py @@ -0,0 +1,1573 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Small-M NVFP4 routed-expert engine, four-request form: one launch for 2..4 requests +of 1..4 rows each (M <= 16, request-major), with the BF16 shared expert in-engine. + +The unit and its arithmetic are the one-request engine's (``engine.py``); what changes +is which token columns a unit owns: + +* the token image has 16 rows (MMA N = 16): image row ``4 q + t`` is token t of request + q (rows ``t >= rows(q)`` are zero), request q's tokens are accumulator columns ``4 q + .. 4 q + 3``; the second 8-row group of every K-major image sits one group stride + after the first; +* the expert list is the union of the requests' experts, ordered by class (experts of + three or more requests, then two, then one; ascending ids within a class) so that the + round-robin schedule spreads multi-request units over the clusters; each entry carries + a group mask (bit q: request q routes to it), so an expert several requests use + streams once; +* per unit and group: one K-split exchange and one 4-token h chain; FC2 reads the + group's four columns; +* the shared-expert clusters use a 3-slot ring and keep their 16-row BF16 token image in + the space of the fourth slot (routed clusters never write it). + +Warp roles (256 threads): 0 expert union + producer, 1 MMA issue + TMEM owner, 2..5 +epilogue (TMEM lane quadrant ``warp % 4``; warp ``2 + q`` builds request q's per-unit +token data), 6 scale-factor copies, 7 receive-buffer release. + +Outputs as the one-request engine, by global row (row stride M).""" + +from __future__ import annotations + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass import Float32, Int32, Int64, Uint32, Uint64 +from cutlass.experimental.cuda import ( + TensorMap, + TensorMapL2Promotion, + TensorMapSwizzle, + create_tensor_map_tiled, +) +from cutlass.memory import SmemAllocator + +from ..primitives import ptx_core as C +from ..primitives import ptx_tc as T +from . import _core_ptx as P +from .constants import HIDDEN, NUM_EXPERTS, TOP_K +from .engine import ( + BIG, + CLUSTER, + ERR_BAD_ID, + ERR_UNIT_CAP, + FC1_CPS, + FC1_MMAS, + FC2_CPS, + FC2_RECORD, + FC2_TILES, + H_BUFS, + H_CPS, + K_CTA, + LOOKAHEAD, + RELEASE_WARP, + REQ_ROWS, + RING, + SF13_PER_EXPERT, + SF_WARP, + SH_CHUNK, + SH_CHUNKS, + SH_CLUSTERS, + SH_FC1_CHUNKS, + SH_FC1_MMAS, + SH_RECORD, + SLOT, + _align16, +) + +GROUPS = 4 +N = GROUPS * REQ_ROWS # 16 image rows +RING_SH = 3 +RECV_BUFS = 5 +RECV_BUF = 3 * 128 * 16 # [3 peer slots][128 rows][4 FP32] +X_GRP = 6144 # NVFP4 image, one 8-row group: [48 K cores][8 rows][16 B] +XBF_GRP = 24576 # BF16 image, one 8-row group: [192 K cores][8 rows][16 B] +XSF_BYTES = ( + 24 * 256 + 256 +) # per K step a 16-row scale window; +256 for the last 32-row copy +H_BUF = 1024 # NVFP4 h: [2 groups][2 K cores][128 B] = 512, scale window at +512 +HBF_BYTES = 2048 # BF16 h: [2 groups][8 K cores][128 B] + +OFF_RING = 0 +OFF_XBF = RING_SH * BIG +OFF_X = max(RING * BIG, OFF_XBF + 2 * XBF_GRP) +OFF_XSF = OFF_X + 2 * X_GRP +OFF_RECV = OFF_XSF + XSF_BYTES +OFF_H = OFF_RECV + RECV_BUFS * RECV_BUF +OFF_HBF = OFF_H + H_BUFS * H_BUF +OFF_MISC = OFF_HBF + HBF_BYTES +SMEM = 223664 # one CTA per SM + +MAX_UNITS = 17 +MAX_LIST = 128 +SCHED_ROW = 18 +SCHED_BYTES = 136 * SCHED_ROW * 2 + +# tensor memory columns +T_ACC1 = 0 # 2 x 16 +T_ACC2 = 32 # 12 x 16 +T_SFB_X = 224 # 24 x 4 +T_SFB_H = 320 # H_BUFS x 4 +T_SFA = 384 # RING x 32 + +IDESC_NVF4 = T.idesc_nvf4(128, N) +IDESC_BF16 = T.idesc_bf16(128, N) + +FC2_MMAS = tuple( + (N * (4 * h2 + t), SLOT * h2 + 4096 * t, 0, 16 * h2 + 4 * t, 0, 0, h2) + for h2 in range(2) + for t in range(4) +) +SH_FC2_MMAS = tuple( + (N * h2, 16384 * h2 + 256 * kk, 256 * kk, 0, 0, 0 if kk == 0 else 1, 0) + for h2 in range(2) + for kk in range(4) +) +X_CPS = tuple((4 * q, 256 * q, 0) for q in range(4)) + +# misc block (offsets from OFF_MISC) +B_FULL = 0 +B_EMPTY = 32 +B_XREADY = 64 +B_XBAR = 72 +B_TREADY = 80 +B_SBAR = 88 +B_ACC1_FULL = 96 +B_ACC1_EMPTY = 112 +B_ACC2_FULL = 128 +B_ACC2_EMPTY = 136 +B_HREADY = 144 +B_SFA_FULL = B_HREADY + 8 * H_BUFS +B_SFX_READY = B_SFA_FULL + 8 * RING +B_PRECV = B_SFX_READY + 8 +B_PFREE = B_PRECV + 8 * RECV_BUFS +B_CONSUMED = B_PFREE + 8 * RECV_BUFS +M_TMEM = B_CONSUMED + 8 * RECV_BUFS +M_NUNITS = M_TMEM + 4 +M_NEX = M_TMEM + 8 +M_BM = _align16(M_TMEM + 12) # [4 requests][8 words] expert bitmaps +M_ELIST = M_BM + 4 * 8 * 4 +M_EGM = M_ELIST + 4 * MAX_LIST # one byte per entry +M_STAB = _align16(M_EGM + MAX_LIST) +M_UNITS = _align16(M_STAB + SCHED_BYTES) +# unit record: expert, quarter, shared flag, group mask, g1 alpha, token scale [16] +# (FP32), slot [16] (s8) +U_E, U_J, U_SH, U_GM, U_G1, U_TOK = 0, 4, 8, 12, 16, 20 +U_SLOT = U_TOK + 4 * N +UNIT_BYTES = U_SLOT + N +assert OFF_MISC + M_UNITS + MAX_UNITS * UNIT_BYTES + 1024 <= SMEM + + +class Nvfp4MoeSmallM16: + """``shared``: run the shared expert's four quarters in-engine (needs >= 8 + clusters). The expert list is the class-ordered union (module docstring). + + Call: as ``engine.Nvfp4MoeSmallM`` with ``split`` replaced by ``rows_per_request``: + the rows of request q in byte q (1..4 for the present requests, 0 after the last). + """ + + def __init__(self, shared: bool = True): + self.shared = shared + + @cute.jit + def __call__( + self, + w13: Int64, + x: Int64, + ids: Int64, + topw: Int64, + shpack: Int64, + g1a: Int64, + g2a: Int64, + gs1: Float32, + gs2: Float32, + rows: Int32, + mqs: Int32, + err: Int64, + sched: Int64, + shpartials: Int64, + pq: Int64, + ids_fin: Int64, + w_fin: Int64, + g2s: Int64, + sf13: Int64, + fc2pack: Int64, + nclusters: Int32, + stream: cuda.CUstream, + ): + tmap = create_tensor_map_tiled( + w13, + cutlass.Uint8, + [128, NUM_EXPERTS * 512, 24], + [3072 // 16, 128 // 16], + [128, 128, 2], + swizzle=TensorMapSwizzle.s128b, + l2_promotion=TensorMapL2Promotion.l2_256b, + ) + self.kernel( + tmap, + x, + ids, + topw, + shpack, + g1a, + g2a, + gs1, + gs2, + rows, + mqs, + err, + sched, + shpartials, + pq, + ids_fin, + w_fin, + g2s, + sf13, + fc2pack, + nclusters, + ).launch( + grid=[CLUSTER * nclusters, 1, 1], + block=[256, 1, 1], + cluster=[CLUSTER, 1, 1], + max_number_threads=[256, 1, 1], + min_blocks_per_mp=1, + stream=stream, + use_pdl=True, + ) + + # ------------------------------------------------------------ protocol hooks + # As in engine.Nvfp4MoeSmallM: each protocol step is one method here. + @cute.jit + def wait_schedule(self, misc): + C.mbarrier_wait_parity(misc + B_SBAR, 0) + + @cute.jit + def wait_inputs(self): + C.pdl_wait() + + @cute.jit + def wait_scales(self, bar, parity): + C.mbarrier_wait_parity(bar, parity) + + @cute.jit + def hready_slot(self, i): + return i % H_BUFS + + @cute.jit + def hready_phase(self, i): + return (i // H_BUFS) & 1 + + @cute.jit + def order_scale_rows(self): + """The zero fill of the x scale windows and the quantization's scale-byte stores + (other threads, same 16-byte rows) are ordered.""" + cute.arch.barrier(barrier_id=2, number_of_threads=128) + + @cute.jit + def ring_tail(self, misc, it, nring): + first = it - nring + if first < 0: + first = Int32(0) + for c in cutlass.range(first, it, unroll=1): + C.mbarrier_wait_parity(misc + B_EMPTY + 8 * (c % nring), (c // nring) & 1) + + # ------------------------------------------------------------ helpers + @cute.jit + def ufield(self, misc, i, field: cutlass.Constexpr[int]): + return C.ld_shared_u32(misc + M_UNITS + i * UNIT_BYTES + field) + + @cute.jit + def rows_of(self, mqs, q): + """Rows of request q (q may be dynamic).""" + return (mqs >> (8 * q)) & 0xFF + + @cute.jit + def first_row_of(self, rbs, q): + return (rbs >> (8 * q)) & 0xFF + + @cute.jit + def fc1_chunk(self, tmap_ptr, sf13, dst, expert, quarter, crank, c, bar, policy): + T.tma_load_3d( + dst, + tmap_ptr, + 0, + expert * 512 + 128 * quarter, + crank * 6 + 2 * c, + bar, + policy, + ) + src = ( + sf13 + + Int64(expert) * SF13_PER_EXPERT + + Int64(96 * quarter + 24 * crank + 8 * c) * 512 + ) + C.bulk_copy_g2s(dst + 32768, src, 4096, bar, policy) + + @cute.jit + def scan8(self, cnt, lane): + incl = cnt + for o in cutlass.range_constexpr(3): + t = cute.arch.shuffle_sync_up(incl, 1 << o) + if lane >= (1 << o): + incl = incl + t + return incl + + @cute.jit + def stream( + self, + tmap_ptr, + shpack, + sf13, + fc2pack, + base, + misc, + crank, + nunits, + nring, + pre, + pre0, + policy, + ): + """Producer lane: stream order FC1(0), FC1(1), FC2(0), FC1(2), FC2(1), ..., on a + ring of ``nring`` slots, then wait for the MMA warp's last commit on every slot. + """ + it = pre + pre0 + for kk in cutlass.range(nunits + LOOKAHEAD, unroll=1): + for part in cutlass.range_constexpr(2): + i = kk if part == 0 else kk - LOOKAHEAD + if (i >= 0) & (i < nunits): + ush = self.ufield(misc, i, U_SH) + if ush != Uint32(0): + uj = Int32(self.ufield(misc, i, U_J)) + src = shpack + (Int64(uj) * 4 + Int64(crank)) * SH_RECORD + c0 = Int32(SH_FC1_CHUNKS) + c1 = Int32(SH_CHUNKS) + if cutlass.const_expr(part == 0): + c0 = Int32(0) + if i == 0: + c0 = pre + c1 = Int32(SH_FC1_CHUNKS) + for c in cutlass.range(c0, c1, unroll=1): + s = it % nring + if it >= nring: + C.mbarrier_wait_parity( + misc + B_EMPTY + 8 * s, ((it // nring) - 1) & 1 + ) + C.mbarrier_arrive_expect_tx(misc + B_FULL + 8 * s, SH_CHUNK) + C.bulk_copy_g2s( + base + OFF_RING + s * BIG, + src + Int64(c) * SH_CHUNK, + SH_CHUNK, + misc + B_FULL + 8 * s, + policy, + ) + it = it + 1 + else: + u_e_idx = Int32(self.ufield(misc, i, U_E)) + uj = Int32(self.ufield(misc, i, U_J)) + src2 = ( + fc2pack + + ((Int64(u_e_idx) * 4 + Int64(uj)) * 4 + Int64(crank)) + * FC2_RECORD + ) + c0 = Int32(3) + c1 = Int32(5) + if cutlass.const_expr(part == 0): + c0 = Int32(0) + if i == 0: + c0 = pre0 + c1 = Int32(3) + for c in cutlass.range(c0, c1, unroll=1): + s = it % nring + if it >= nring: + C.mbarrier_wait_parity( + misc + B_EMPTY + 8 * s, ((it // nring) - 1) & 1 + ) + nbytes = Int32(BIG) + if c == 4: + nbytes = Int32(SLOT) + C.mbarrier_arrive_expect_tx(misc + B_FULL + 8 * s, nbytes) + if c < 3: + # expert / quarter read per chunk from the unit + # table, as the reference engine does + ce = Int32(self.ufield(misc, i, U_E)) + cj = Int32(self.ufield(misc, i, U_J)) + self.fc1_chunk( + tmap_ptr, + sf13, + base + OFF_RING + s * BIG, + ce, + cj, + crank, + c, + misc + B_FULL + 8 * s, + policy, + ) + else: + C.bulk_copy_g2s( + base + OFF_RING + s * BIG, + src2 + Int64(2 * c - 6) * SLOT, + nbytes, + misc + B_FULL + 8 * s, + policy, + ) + it = it + 1 + self.ring_tail(misc, it, nring) + + # ------------------------------------------------ warp 0: union, schedule, producer + @cute.jit + def producer( + self, + tmap_ptr, + ids, + shpack, + mqs, + rbs, + gall, + err, + sched, + sf13, + fc2pack, + base, + misc, + crank, + cl, + my_sh, + nring, + lane, + ): + policy = C.l2_policy("evict_first") + pre = Int32(0) + if cutlass.const_expr(self.shared): # noqa: SIM102 + if my_sh != 0: + src = shpack + (Int64(cl) * 4 + Int64(crank)) * SH_RECORD + if lane == 0: + for c in cutlass.range_constexpr(RING_SH): + C.mbarrier_arrive_expect_tx(misc + B_FULL + 8 * c, SH_CHUNK) + C.bulk_copy_g2s( + base + OFF_RING + c * BIG, + src + c * SH_CHUNK, + SH_CHUNK, + misc + B_FULL + 8 * c, + policy, + ) + pre = Int32(RING_SH) + c = RING_SH + lane + if c < SH_CHUNKS: + C.bulk_prefetch_l2(src + Int64(c) * SH_CHUNK, SH_CHUNK) + if lane == 0: + C.mbarrier_arrive_expect_tx(misc + B_SBAR, SCHED_BYTES) + C.bulk_copy_g2s( + misc + M_STAB, + sched + Int64(cl) * SCHED_BYTES, + SCHED_BYTES, + misc + B_SBAR, + C.l2_policy("evict_last"), + ) + C.st_shared_u32(misc + M_BM + 4 * lane, Uint32(0)) + cute.arch.sync_warp() + C.pdl_wait() + # lane l holds (token l >> 3, slot l & 7) of each request; per-request bitmaps + # by shared atomics + badid = Int32(0) + for q in cutlass.range_constexpr(GROUPS): + mq = self.rows_of(mqs, q) + inside = lane < TOP_K * mq + eq = Int32(-1) + if inside: + eq = P.ld_cg_s32( + ids + Int64(TOP_K * self.first_row_of(rbs, q) + lane) * 4 + ) + ok = Int32(0) + if inside & (Uint32(eq) < Uint32(NUM_EXPERTS)): + ok = Int32(1) + if inside & (ok == 0): + badid = Int32(1) + if ok != 0: + P.atom_or_shared( + misc + M_BM + 4 * (8 * q + (eq >> 5)), Uint32(1) << Uint32(eq & 31) + ) + if badid != 0: + P.atom_or_global(err, ERR_BAD_ID) + cute.arch.sync_warp() + wq0 = Uint32(0) + wq1 = Uint32(0) + wq2 = Uint32(0) + wq3 = Uint32(0) + if lane < NUM_EXPERTS // 32: + wq0 = C.ld_shared_u32(misc + M_BM + 4 * lane) + wq1 = C.ld_shared_u32(misc + M_BM + 4 * (8 + lane)) + wq2 = C.ld_shared_u32(misc + M_BM + 4 * (16 + lane)) + wq3 = C.ld_shared_u32(misc + M_BM + 4 * (24 + lane)) + wq = [wq0, wq1, wq2, wq3] + # classes: experts of >= 3 requests, of exactly 2, of 1 (placement only) + w0 = wq[0] | wq[1] | wq[2] | wq[3] + s1 = wq[0] ^ wq[1] + c1 = wq[0] & wq[1] + s2 = s1 ^ wq[2] + c2 = s1 & wq[2] + lo = s2 ^ wq[3] + c3 = s2 & wq[3] + ge2 = c1 | c2 | c3 + ge3 = (ge2 & lo) | (wq[0] & wq[1] & wq[2] & wq[3]) + wcls = [ge3, ge2 & ~ge3, w0 & ~ge2] + ccnt = [P.popc(wcls[c]) for c in range(3)] + cincl = [self.scan8(ccnt[c], lane) for c in range(3)] + ctot = [cute.arch.shuffle_sync(cincl[c], 7) for c in range(3)] + cbase1 = ctot[0] + cbase2 = ctot[0] + ctot[1] + d0 = cbase2 + ctot[2] + # unit 0 resolved from the schedule row and the class bitmaps; its first copies + # issue now + pre0 = Int32(0) + self.wait_schedule(misc) + row0 = misc + M_STAB + 2 * SCHED_ROW * min(d0, MAX_LIST) + code0 = Int32(-1) + if P.ld_shared_s16(row0) > 0: + code0 = P.ld_shared_s16(row0 + 2) + if code0 >= 0: + idx = code0 >> 2 + r0 = idx + ci = cincl[0] + cc = ccnt[0] + cw = wcls[0] + if idx >= cbase2: + r0 = idx - cbase2 + ci = cincl[2] + cc = ccnt[2] + cw = wcls[2] + elif idx >= cbase1: + r0 = idx - cbase1 + ci = cincl[1] + cc = ccnt[1] + cw = wcls[1] + mine = (r0 >= ci - cc) & (r0 < ci) + who = Uint32(cute.arch.vote_ballot_sync(mine)) + e0 = Int32(0) + if mine: + bits = cw + for _ in cutlass.range(r0 - (ci - cc), unroll=1): + bits = bits & (bits - Uint32(1)) + e0 = 32 * lane + P.ctz(bits) + src_lane = Int32(0) + if who != Uint32(0): + src_lane = P.ctz(who) + e0 = cute.arch.shuffle_sync(e0, src_lane) + if (lane == 0) & (who != Uint32(0)): + C.mbarrier_arrive_expect_tx(misc + B_FULL, BIG) + self.fc1_chunk( + tmap_ptr, + sf13, + base + OFF_RING, + e0, + code0 & 3, + crank, + 0, + misc + B_FULL, + policy, + ) + pre0 = Int32(1) + pre0 = cute.arch.shuffle_sync(pre0, 0) + # the union list in class order, with group masks + for c in cutlass.range_constexpr(3): + bits = wcls[c] + k = cincl[c] - ccnt[c] + if cutlass.const_expr(c == 1): + k = k + cbase1 + if cutlass.const_expr(c == 2): + k = k + cbase2 + while bits != Uint32(0): + b = P.ctz(bits) + if k < MAX_LIST: + C.st_shared_u32(misc + M_ELIST + 4 * k, 32 * lane + b) + gm = Uint32(0) + for q in cutlass.range_constexpr(GROUPS): + gm = gm | (((wq[q] >> Uint32(b)) & Uint32(1)) << Uint32(q)) + P.st_shared_b8(misc + M_EGM + k, gm) + k = k + 1 + bits = bits & (bits - Uint32(1)) + cute.arch.sync_warp() + d = min(d0, MAX_LIST) + self.wait_schedule(misc) + row = misc + M_STAB + 2 * SCHED_ROW * d + nraw = P.ld_shared_s16(row) + nunits = min(nraw, MAX_UNITS) + if (lane == 0) & (nraw > MAX_UNITS): + P.atom_or_global(err, ERR_UNIT_CAP) + gmu = Int32(0) + if lane < nunits: + code = P.ld_shared_s16(row + 2 + 2 * lane) + unit = misc + M_UNITS + lane * UNIT_BYTES + if code < 0: + C.st_shared_u32(unit + U_E, Int32(-1)) + C.st_shared_u32(unit + U_J, cl) + C.st_shared_u32(unit + U_SH, Int32(1)) + gmu = gall + else: + C.st_shared_u32( + unit + U_E, C.ld_shared_u32(misc + M_ELIST + 4 * (code >> 2)) + ) + C.st_shared_u32(unit + U_J, code & 3) + C.st_shared_u32(unit + U_SH, Int32(0)) + gmu = P.ld_shared_u8(misc + M_EGM + (code >> 2)) + C.st_shared_u32(unit + U_GM, gmu) + # exchanges of this cluster (one per (unit, group)), the same in its four CTAs + nx = P.popc(gmu) + for o in (16, 8, 4, 2, 1): + nx = nx + cute.arch.shuffle_sync_bfly(nx, o) + if lane == 0: + C.st_shared_u32(misc + M_NEX, nx) + cute.arch.sync_warp() + if lane == 0: + if (pre0 == 0) & (nunits > 0): # noqa: SIM102 + if self.ufield(misc, 0, U_SH) == Uint32(0): + C.mbarrier_arrive_expect_tx(misc + B_FULL, BIG) + e0 = Int32(self.ufield(misc, 0, U_E)) + j0 = Int32(self.ufield(misc, 0, U_J)) + self.fc1_chunk( + tmap_ptr, + sf13, + base + OFF_RING, + e0, + j0, + crank, + 0, + misc + B_FULL, + policy, + ) + pre0 = Int32(1) + C.st_shared_u32(misc + M_NUNITS, nunits) + # Every lane arrives (count 32): each lane's own unit-record stores are + # released by its own arrive (see the init of B_TREADY). + C.mbarrier_arrive(misc + B_TREADY) + cute.arch.sync_warp() + if lane == 0: + self.stream( + tmap_ptr, + shpack, + sf13, + fc2pack, + base, + misc, + crank, + nunits, + nring, + pre, + pre0, + policy, + ) + cute.arch.sync_warp() + + # ------------------------------------------------------------ warp 1: MMA issue + @cute.jit + def mma_warp(self, base, misc, nring): + lead = T.elect_leader() + C.mbarrier_wait_parity(misc + B_TREADY, 0) + nunits = cute.arch.shuffle_sync(Int32(C.ld_shared_u32(misc + M_NUNITS)), 0) + tm = cute.arch.shuffle_sync(Int32(C.ld_shared_u32(misc + M_TMEM)), 0) + if nunits > 0: + C.mbarrier_wait_parity(misc + B_XREADY, 0) + self.wait_scales(misc + B_SFX_READY, 0) + T.tc_fence_after() + s_ring = base + OFF_RING + it = Int32(0) + for k in cutlass.range(nunits + LOOKAHEAD, unroll=1): + shk = Int32(0) + if k < nunits: + shk = cute.arch.shuffle_sync(Int32(self.ufield(misc, k, U_SH)), 0) + if k < nunits: + b = k & 1 + if k >= 2: + C.mbarrier_wait_parity( + misc + B_ACC1_EMPTY + 8 * b, ((k >> 1) - 1) & 1 + ) + T.tc_fence_after() + if shk != 0: + for c in cutlass.range(SH_FC1_CHUNKS, unroll=1): + s = it % nring + C.mbarrier_wait_parity(misc + B_FULL + 8 * s, (it // nring) & 1) + self.wait_scales(misc + B_SFA_FULL + 8 * s, (it // nring) & 1) + T.tc_fence_after() + a0 = T.smem_desc(s_ring + s * BIG, 128, 1024, 0) + b0 = T.smem_desc(base + OFF_XBF + 2048 * c, 128, XBF_GRP, 0) + T.tc_burst( + lead, + tm + T_ACC1 + N * b, + a0, + b0, + 0, + 0, + Int32(c != 0), + 0, + misc + B_EMPTY + 8 * s, + 0, + 0, + mmas=SH_FC1_MMAS, + block_scaled=False, + idesc=IDESC_BF16, + ) + cute.arch.sync_warp() + it = it + 1 + else: + for c in cutlass.range(3, unroll=1): + s = it % nring + C.mbarrier_wait_parity(misc + B_FULL + 8 * s, (it // nring) & 1) + self.wait_scales(misc + B_SFA_FULL + 8 * s, (it // nring) & 1) + T.tc_fence_after() + a0 = T.smem_desc(s_ring + s * BIG, 16, 1024, 2) + b0 = T.smem_desc(base + OFF_X + 2048 * c, 128, X_GRP, 0) + T.tc_burst( + lead, + tm + T_ACC1 + N * b, + a0, + b0, + tm + T_SFA + 32 * s, + tm + T_SFB_X + 32 * c, + Int32(c != 0), + 0, + misc + B_EMPTY + 8 * s, + 0, + 0, + mmas=FC1_MMAS, + block_scaled=True, + idesc=IDESC_NVF4, + ) + cute.arch.sync_warp() + it = it + 1 + T.tc_commit(lead, misc + B_ACC1_FULL + 8 * b) + cute.arch.sync_warp() + if (k >= LOOKAHEAD) & (k - LOOKAHEAD < nunits): + i = k - LOOKAHEAD + shi = cute.arch.shuffle_sync(Int32(self.ufield(misc, i, U_SH)), 0) + C.mbarrier_wait_parity( + misc + B_HREADY + 8 * self.hready_slot(i), self.hready_phase(i) + ) + T.tc_fence_after() + if i > 0: + C.mbarrier_wait_parity(misc + B_ACC2_EMPTY, (i - 1) & 1) + T.tc_fence_after() + if shi != 0: + bh = T.smem_desc(base + OFF_HBF, 128, 1024, 0) + for c in cutlass.range(SH_FC1_CHUNKS, SH_CHUNKS, unroll=1): + s = it % nring + C.mbarrier_wait_parity(misc + B_FULL + 8 * s, (it // nring) & 1) + self.wait_scales(misc + B_SFA_FULL + 8 * s, (it // nring) & 1) + T.tc_fence_after() + a0 = T.smem_desc(s_ring + s * BIG, 128, 1024, 0) + T.tc_burst( + lead, + tm + T_ACC2 + 2 * N * (c - SH_FC1_CHUNKS), + a0, + bh, + 0, + 0, + 0, + 0, + misc + B_EMPTY + 8 * s, + 0, + 0, + mmas=SH_FC2_MMAS, + block_scaled=False, + idesc=IDESC_BF16, + ) + cute.arch.sync_warp() + it = it + 1 + else: + hb = i % H_BUFS + bh = T.smem_desc(base + OFF_H + H_BUF * hb, 128, 256, 0) + for c in cutlass.range(3, 5, unroll=1): + s = it % nring + C.mbarrier_wait_parity(misc + B_FULL + 8 * s, (it // nring) & 1) + self.wait_scales(misc + B_SFA_FULL + 8 * s, (it // nring) & 1) + T.tc_fence_after() + a0 = T.smem_desc(s_ring + s * BIG, 128, 256, 0) + T.tc_burst( + lead, + tm + T_ACC2 + 4 * N * (2 * c - 6), + a0, + bh, + tm + T_SFA + 32 * s, + tm + T_SFB_H + 4 * hb, + 0, + Int32(c == 3), + misc + B_EMPTY + 8 * s, + 0, + 0, + mmas=FC2_MMAS, + block_scaled=True, + idesc=IDESC_NVF4, + ) + cute.arch.sync_warp() + it = it + 1 + T.tc_commit(lead, misc + B_ACC2_FULL) + cute.arch.sync_warp() + cute.arch.sync_warp() + + # -------------------------------------------------- warp 6: scale factors into TMEM + @cute.jit + def sf_warp(self, base, misc, nring): + lead = T.elect_leader() + C.mbarrier_wait_parity(misc + B_TREADY, 0) + nunits = cute.arch.shuffle_sync(Int32(C.ld_shared_u32(misc + M_NUNITS)), 0) + tm = cute.arch.shuffle_sync(Int32(C.ld_shared_u32(misc + M_TMEM)), 0) + if nunits > 0: + C.mbarrier_wait_parity(misc + B_XREADY, 0) + T.tc_fence_after() + d0 = T.smem_desc(base + OFF_XSF, 0, 128, 0) + for q in cutlass.range(6, unroll=1): + T.tc_burst( + lead, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + tm + T_SFB_X + 16 * q, + d0 + Uint64(64 * q), + cps=X_CPS, + commit=False, + ) + T.tc_commit(lead, misc + B_SFX_READY) + cute.arch.sync_warp() + it = Int32(0) + for k in cutlass.range(nunits + LOOKAHEAD, unroll=1): + for part in cutlass.range_constexpr(2): + i = k if part == 0 else k - LOOKAHEAD + if (i >= 0) & (i < nunits): + ush = cute.arch.shuffle_sync(Int32(self.ufield(misc, i, U_SH)), 0) + c0 = Int32(0) + c1 = Int32(3) + if cutlass.const_expr(part == 0): + if ush != 0: + c1 = Int32(SH_FC1_CHUNKS) + else: + c0 = Int32(3) + c1 = Int32(5) + if ush != 0: + c0 = Int32(SH_FC1_CHUNKS) + c1 = Int32(SH_CHUNKS) + else: + C.mbarrier_wait_parity( + misc + B_HREADY + 8 * self.hready_slot(i), + self.hready_phase(i), + ) + T.tc_fence_after() + hb = i % H_BUFS + T.tc_burst( + lead, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + tm + T_SFB_H + 4 * hb, + T.smem_desc(base + OFF_H + H_BUF * hb + 512, 0, 128, 0), + cps=H_CPS, + commit=False, + ) + cute.arch.sync_warp() + for c in cutlass.range(c0, c1, unroll=1): + s = it % nring + C.mbarrier_wait_parity(misc + B_FULL + 8 * s, (it // nring) & 1) + T.tc_fence_after() + if ush != 0: + T.tc_commit(lead, misc + B_SFA_FULL + 8 * s) + else: + if cutlass.const_expr(part == 0): + d = T.smem_desc( + base + OFF_RING + s * BIG + 32768, 0, 128, 0 + ) + T.tc_burst( + lead, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + misc + B_SFA_FULL + 8 * s, + tm + T_SFA + 32 * s, + d, + cps=FC1_CPS, + ) + else: + d = T.smem_desc( + base + OFF_RING + s * BIG + 16384, 0, 128, 0 + ) + T.tc_burst( + lead, + 0, + 0, + 0, + 0, + 0, + 0, + Int32(c == 3), + misc + B_SFA_FULL + 8 * s, + tm + T_SFA + 32 * s, + d, + cps=FC2_CPS, + ) + cute.arch.sync_warp() + it = it + 1 + cute.arch.sync_warp() + + # --------------------------------------------------- warp 7: receive-buffer release + @cute.jit + def release_warp(self, misc, crank, lane): + C.mbarrier_wait_parity(misc + B_TREADY, 0) + nex = cute.arch.shuffle_sync(Int32(C.ld_shared_u32(misc + M_NEX)), 0) + cute.arch.cluster_wait() + if lane == 0: + for e in cutlass.range(nex - RECV_BUFS, unroll=1): + p = e % RECV_BUFS + C.mbarrier_wait_parity(misc + B_CONSUMED + 8 * p, (e // RECV_BUFS) & 1) + for pr in cutlass.range_constexpr(CLUSTER): + if crank != pr: + P.mbarrier_arrive_remote( + C.mapa_shared_cluster(misc + B_PFREE + 8 * p, pr) + ) + cute.arch.sync_warp() + + # ------------------------------------------------------------ warps 2..5 + @cute.jit + def quantize_x(self, x, gs1, mqs, rbs, base, misc, crank, my_sh, et, lane): + """The activations x (BF16) of this CTA's K range into the NVFP4 image + + scale windows (routed operand) and, on the shared-expert clusters, the BF16 + image. Image row n = 4 q + t: request q row t (valid t < rows(q)); invalid + rows are zero.""" + xbf = base + OFF_XBF + for hf in cutlass.range(2, unroll=1): + raw = cute.make_rmem_tensor(cute.make_layout(48), Uint32) + for ii in cutlass.range_constexpr(6): + b = et + 128 * (hf * 6 + ii) + n = b // 96 + kb = b % 96 + for w in cutlass.range_constexpr(8): + raw[8 * ii + w] = Uint32(0) + if (n & 3) < self.rows_of(mqs, n >> 2): + grow = self.first_row_of(rbs, n >> 2) + (n & 3) + src = x + (Int64(grow) * HIDDEN + K_CTA * crank + 16 * kb) * 2 + lo = P.ld_cg_v4_u32(src) + hi = P.ld_cg_v4_u32(src + 16) + for w in cutlass.range_constexpr(4): + raw[8 * ii + w] = lo[w] + raw[8 * ii + 4 + w] = hi[w] + for ii in cutlass.range_constexpr(6): + b = et + 128 * (hf * 6 + ii) + n = b // 96 + kb = b % 96 + if cutlass.const_expr(self.shared): # noqa: SIM102 + if my_sh != 0: + gb = xbf + (n >> 3) * XBF_GRP + (n & 7) * 16 + C.st_shared_v4( + gb + (2 * kb) * 128, + tuple(raw[8 * ii + w] for w in range(4)), + ) + C.st_shared_v4( + gb + (2 * kb + 1) * 128, + tuple(raw[8 * ii + 4 + w] for w in range(4)), + ) + w0 = Uint32(0) + w1 = Uint32(0) + sfb = Uint32(0) + if (n & 3) < self.rows_of(mqs, n >> 2): + pairs = [P.bf16x2_unpack(raw[8 * ii + w]) for w in range(8)] + v = [f for pr in pairs for f in pr] + amax = Float32(0.0) + for q in cutlass.range_constexpr(16): + amax = T.max_f32(amax, T.abs_f32(v[q])) + osc, sfb = T.nvfp4_block_scale(amax, gs1) + q0 = [P.mul_f32(v[q], osc) for q in range(8)] + q1 = [P.mul_f32(v[8 + q], osc) for q in range(8)] + w0 = T.e2m1x8(*q0) + w1 = T.e2m1x8(*q1) + P.st_shared_v2_u32( + base + + OFF_X + + (n >> 3) * X_GRP + + (kb >> 1) * 128 + + (n & 7) * 16 + + (kb & 1) * 8, + w0, + w1, + ) + P.st_shared_b8( + base + OFF_XSF + (kb >> 2) * 256 + n * 16 + (kb & 3), sfb + ) + C.fence_proxy_async_shared_cta() + cute.arch.sync_warp() + if lane == 0: + C.mbarrier_arrive(misc + B_XREADY) + + @cute.jit + def token_data( + self, + ids, + topw, + g1a, + g2a, + mqs, + rbs, + ids_fin, + w_fin, + g2s, + misc, + block, + nunits, + qw, + lane, + ): + """Request qw's rows 4 qw .. 4 qw + 3 of every unit: token scale (routing weight + x FC2 alpha; shared: 1 for a valid row) and top-k slot; warp 2 (qw 0) also + writes g1 alpha. Block 0 writes the slot records of request qw.""" + mqq = self.rows_of(mqs, qw) + rbq = self.first_row_of(rbs, qw) + eid = Int32(-1) + wt = Float32(0.0) + if lane < TOP_K * mqq: + eid = P.ld_cg_s32(ids + Int64(TOP_K * rbq + lane) * 4) + wt = P.ld_cg_f32(topw + Int64(TOP_K * rbq + lane) * 4) + mysh = Int32(0) + my_unit = Int32(-1) + mygm = Int32(0) + if lane < nunits: + mysh = Int32(self.ufield(misc, lane, U_SH)) + mygm = Int32(self.ufield(misc, lane, U_GM)) + if mysh == 0: + my_unit = Int32(self.ufield(misc, lane, U_E)) + g2 = Float32(0.0) + if my_unit >= 0: + g2 = P.ld_cg_f32(g2a + Int64(my_unit) * 4) + tk = cute.make_rmem_tensor(cute.make_layout(REQ_ROWS), Float32) + sl = cute.make_rmem_tensor(cute.make_layout(REQ_ROWS), Int32) + for t in cutlass.range_constexpr(REQ_ROWS): + tk[t] = Float32(0.0) + sl[t] = Int32(-1) + for u in cutlass.range(nunits, unroll=1): + e = Int32(self.ufield(misc, u, U_E)) + msk = Uint32(cute.arch.vote_ballot_sync(eid == e)) + for t in cutlass.range_constexpr(REQ_ROWS): + bits = (msk >> Uint32(8 * t)) & Uint32(0xFF) + src = Int32(0) + if bits != Uint32(0): + src = 8 * t + P.ctz(bits) + w = cute.arch.shuffle_sync(wt, src) + if lane == u: + if bits != Uint32(0): + tk[t] = w + sl[t] = P.ctz(bits) + else: + tk[t] = Float32(0.0) + sl[t] = Int32(-1) + if (block == 0) & (lane < TOP_K * mqq): + at = TOP_K * rbq + lane + ok = Uint32(eid) < Uint32(NUM_EXPERTS) + g2v = Float32(0.0) + sid = Int32(-1) + if ok: + g2v = P.ld_cg_f32(g2a + Int64(eid) * 4) + sid = eid + P.st_global_b32(ids_fin + Int64(at) * 4, sid) + P.st_global_b32f(w_fin + Int64(at) * 4, wt) + P.st_global_b32f(g2s + Int64(at) * 4, g2v) + if lane < nunits: + unit = misc + M_UNITS + lane * UNIT_BYTES + if qw == 0: + g1 = Float32(1.0) + if mysh == 0: + g1 = Float32(0.0) + if my_unit >= 0: + g1 = P.ld_cg_f32(g1a + Int64(my_unit) * 4) + C.st_shared_f32(unit + U_G1, g1) + own = (mygm >> qw) & 1 + for t in cutlass.range_constexpr(REQ_ROWS): + r = REQ_ROWS * qw + t + tok = Float32(0.0) + slot = Int32(-1) + if mysh != 0: + if (own != 0) & (t < mqq): + tok = Float32(1.0) + else: + if own != 0: + tok = P.mul_f32(tk[t], g2) + slot = sl[t] + C.st_shared_f32(unit + U_TOK + 4 * r, tok) + P.st_shared_b8(unit + U_SLOT + r, Uint32(slot)) + + @cute.jit + def routed_h(self, acc, g1, gs2, nrows, r0, hbuf, qd, lane): + """Routed unit, one group: h = SiLU(g1 gate) * (g1 up), NVFP4 into h image rows + r0 .. r0 + nrows - 1 (lane i of a quadrant: gate iff i & 8, up partner i ^ 8); + words packed with three shuffles (lanes 8 / 12 hold them), the scale byte at + lane 8.""" + gate = (lane & 8) != 0 + hv = [] + am = [] + for n in cutlass.range_constexpr(REQ_ROWS): + mine = P.mul_f32(acc[n], g1) + up = cute.arch.shuffle_sync_bfly(mine, 8) + h = P.mul_f32(T.silu_fast(mine), up) + hv.append(h) + a = Float32(0.0) + if gate: + a = T.abs_f32(h) + am.append(a) + for n in cutlass.range_constexpr(REQ_ROWS): + for off in (16, 4, 2, 1): + am[n] = T.max_f32(am[n], cute.arch.shuffle_sync_bfly(am[n], off)) + hq = [] + sfb = [] + for n in cutlass.range_constexpr(REQ_ROWS): + osc, sf = T.nvfp4_block_scale(am[n], gs2) + hq.append(P.mul_f32(hv[n], osc)) + sfb.append(sf) + words = [] + for n in cutlass.range_constexpr(REQ_ROWS): + by = T.e2m1x2(hq[n], cute.arch.shuffle_sync_down(hq[n], 16)) + hw = by | (cute.arch.shuffle_sync_down(by, 1) << Uint32(8)) + words.append(hw | (cute.arch.shuffle_sync_down(hw, 2) << Uint32(16))) + for n in cutlass.range_constexpr(REQ_ROWS): + if n < nrows: + r = r0 + n + if (lane == 8) | (lane == 12): + off = Int32(0) + if lane == 12: + off = Int32(4) + C.st_shared_u32( + hbuf + + (r >> 3) * 256 + + (qd >> 1) * 128 + + (r & 7) * 16 + + (qd & 1) * 8 + + off, + words[n], + ) + if lane == 8: + P.st_shared_b8(hbuf + 512 + r * 16 + qd, sfb[n]) + + @cute.jit + def shared_h(self, acc, nrows, r0, hbf, qd, lane): + """Shared quarter, one group: stock BF16 rounding points into BF16 h rows r0 .. + r0 + nrows - 1.""" + for n in cutlass.range_constexpr(REQ_ROWS): + gb = P.round_bf16(acc[n]) + ub = cute.arch.shuffle_sync_down(gb, 16) + sv = P.round_bf16(T.silu_accurate(gb)) + hb = Uint32(C.cvt_rn_bf16_f32(P.mul_f32(sv, ub))) + hn = cute.arch.shuffle_sync_down(hb, 1) + if (n < nrows) & (lane < 16) & ((lane & 1) == 0): + kc = 2 * qd + (lane >> 3) + r = r0 + n + C.st_shared_u32( + hbf + (r >> 3) * 1024 + kc * 128 + (r & 7) * 16 + (lane & 7) * 2, + hb | (hn << Uint32(16)), + ) + + @cute.jit + def readout( + self, i, rows, mqs, rbs, shpartials, pq, misc, crank, lane_base, row, lane + ): + """FC2(i): per group, the 12 tiles' 4 columns -> partials slots (routed) or + shared_partials (shared quarter); release the accumulator after the last group's + loads.""" + C.mbarrier_wait_parity(misc + B_ACC2_FULL, i & 1) + T.tc_fence_after() + C.fence_proxy_async_shared_cta() + gm = Int32(self.ufield(misc, i, U_GM)) + glast = 31 - P.clz(gm) + for g in cutlass.range(GROUPS, unroll=1): + if ((gm >> g) & 1) != 0: + mg = self.rows_of(mqs, g) + rbg = self.first_row_of(rbs, g) + d = T.tmem_load4x12(lane_base + T_ACC2 + 4 * g, stride=N) + if g == glast: + T.tc_fence_before() + cute.arch.sync_warp() + if lane == 0: + C.mbarrier_arrive(misc + B_ACC2_EMPTY) + # unit fields are read per group, where the readout uses them (the + # reference engine's order) + if self.ufield(misc, i, U_SH) == Uint32(0): + for n in cutlass.range_constexpr(REQ_ROWS): + sl = P.ld_shared_s8( + misc + M_UNITS + i * UNIT_BYTES + U_SLOT + REQ_ROWS * g + n + ) + if (n < mg) & (sl >= 0): + uj = Int32(self.ufield(misc, i, U_J)) + q = ( + pq + + ( + ( + ((Int64(uj) * rows + rbg + n) * TOP_K + sl) + * HIDDEN + ) + + K_CTA * crank + + row + ) + * 4 + ) + for t in cutlass.range_constexpr(FC2_TILES): + P.st_global_f32(q + 512 * t, d[4 * t + n]) + else: + if cutlass.const_expr(self.shared): + for n in cutlass.range_constexpr(REQ_ROWS): + if n < mg: + s = C.ld_shared_f32( + misc + + M_UNITS + + i * UNIT_BYTES + + U_TOK + + 4 * (REQ_ROWS * g + n) + ) + uj = Int32(self.ufield(misc, i, U_J)) + q = ( + shpartials + + ( + (Int64(uj) * rows + rbg + n) * HIDDEN + + K_CTA * crank + + row + ) + * 4 + ) + for t in cutlass.range_constexpr(FC2_TILES): + P.st_global_f32( + q + 512 * t, + C.fma_rn(s, d[4 * t + n], Float32(0.0)), + ) + + @cute.jit + def epilogue( + self, + x, + ids, + topw, + g1a, + g2a, + gs1, + gs2, + rows, + mqs, + rbs, + shpartials, + pq, + ids_fin, + w_fin, + g2s, + base, + misc, + tm, + crank, + my_sh, + block, + warp, + lane, + tid, + ): + qd = warp & 3 + et = tid - 64 + row = 32 * qd + lane + qw = warp - 2 + zero = (Uint32(0), Uint32(0), Uint32(0), Uint32(0)) + for i in cutlass.range(et, XSF_BYTES // 16, 128, unroll=1): + C.st_shared_v4(base + OFF_XSF + 16 * i, zero) + for i in cutlass.range(et, H_BUFS * H_BUF // 16, 128, unroll=1): + C.st_shared_v4(base + OFF_H + 16 * i, zero) + for i in cutlass.range(et, HBF_BYTES // 16, 128, unroll=1): + C.st_shared_v4(base + OFF_HBF + 16 * i, zero) + # the zeroing and the quantization's scale-byte stores touch the same 16-byte + # rows + self.order_scale_rows() + self.wait_inputs() + self.quantize_x(x, gs1, mqs, rbs, base, misc, crank, my_sh, et, lane) + C.mbarrier_wait_parity(misc + B_TREADY, 0) + nunits = Int32(C.ld_shared_u32(misc + M_NUNITS)) + nex = Int32(C.ld_shared_u32(misc + M_NEX)) + self.token_data( + ids, + topw, + g1a, + g2a, + mqs, + rbs, + ids_fin, + w_fin, + g2s, + misc, + block, + nunits, + qw, + lane, + ) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + cute.arch.cluster_wait() + lane_base = tm + Int32(32 * qd) * 65536 + recv = base + OFF_RECV + ex = Int32(0) + for k in cutlass.range(nunits + LOOKAHEAD, unroll=1): + if k < nunits: + i = k + b = i & 1 + gm = Int32(self.ufield(misc, i, U_GM)) + ng = P.popc(gm) + if et == 0: + for gi in cutlass.range(ng, unroll=1): + C.mbarrier_arrive_expect_tx( + misc + B_PRECV + 8 * ((ex + gi) % RECV_BUFS), + 3 * 128 * 16, + ".relaxed.cluster", + ) + C.mbarrier_wait_parity(misc + B_ACC1_FULL + 8 * b, (i >> 1) & 1) + T.tc_fence_after() + va = T.tmem_load4_masked(lane_base + T_ACC1 + N * b, gm) + T.tc_fence_before() + cute.arch.sync_warp() + if lane == 0: + C.mbarrier_arrive(misc + B_ACC1_EMPTY + 8 * b) + g1 = C.ld_shared_f32(misc + M_UNITS + i * UNIT_BYTES + U_G1) + ush = self.ufield(misc, i, U_SH) + gi = Int32(0) + for g in cutlass.range_constexpr(GROUPS): + if ((gm >> g) & 1) != 0: + mg = self.rows_of(mqs, g) + v0, v1, v2, v3 = va[g] + e = ex + gi + p = e % RECV_BUFS + use = e // RECV_BUFS + gi = gi + 1 + if use >= 1: + C.mbarrier_wait_parity( + misc + B_PFREE + 8 * p, + (use - 1) & 1, + ".acquire.cluster", + ) + lbar = misc + B_PRECV + 8 * p + for pr in cutlass.range_constexpr(CLUSTER): + if crank != pr: + # our slot in peer pr's buffer: our rank, skipping the + # peer's own + sslot = crank + if crank > pr: + sslot = crank - 1 + laddr = recv + ((p * 3 + sslot) * 128 + row) * 16 + P.st_async_v4_f32( + C.mapa_shared_cluster(laddr, pr), + C.mapa_shared_cluster(lbar, pr), + v0, + v1, + v2, + v3, + ) + C.mbarrier_wait_parity( + misc + B_PRECV + 8 * p, use & 1, ".acquire.cluster" + ) + acc = [Float32(0.0), Float32(0.0), Float32(0.0), Float32(0.0)] + for src in cutlass.range_constexpr(CLUSTER): + p0, p1, p2, p3 = v0, v1, v2, v3 + if crank != src: + rslot = Int32(src) + if src > crank: + rslot = Int32(src - 1) + p0, p1, p2, p3 = P.ld_shared_v4_f32( + recv + ((p * 3 + rslot) * 128 + row) * 16 + ) + acc = [ + P.add_f32(acc[0], p0), + P.add_f32(acc[1], p1), + P.add_f32(acc[2], p2), + P.add_f32(acc[3], p3), + ] + cute.arch.sync_warp() + if (lane == 0) & (e + RECV_BUFS < nex): + C.mbarrier_arrive(misc + B_CONSUMED + 8 * p) + if ush != Uint32(0): + self.shared_h( + acc, mg, REQ_ROWS * g, base + OFF_HBF, qd, lane + ) + else: + self.routed_h( + acc, + g1, + gs2, + mg, + REQ_ROWS * g, + base + OFF_H + H_BUF * (i % H_BUFS), + qd, + lane, + ) + ex = ex + ng + C.fence_proxy_async_shared_cta() + cute.arch.sync_warp() + if lane == 0: + C.mbarrier_arrive(misc + B_HREADY + 8 * self.hready_slot(i)) + if (k >= LOOKAHEAD) & (k - LOOKAHEAD < nunits): + self.readout( + k - LOOKAHEAD, + rows, + mqs, + rbs, + shpartials, + pq, + misc, + crank, + lane_base, + row, + lane, + ) + + # ------------------------------------------------------------ kernel + @cute.kernel + def kernel( + self, + tmap: cutlass.GridConstant[TensorMap], + x: Int64, + ids: Int64, + topw: Int64, + shpack: Int64, + g1a: Int64, + g2a: Int64, + gs1: Float32, + gs2: Float32, + rows: Int32, + mqs: Int32, + err: Int64, + sched: Int64, + shpartials: Int64, + pq: Int64, + ids_fin: Int64, + w_fin: Int64, + g2s: Int64, + sf13: Int64, + fc2pack: Int64, + nclusters: Int32, + ): + tid, _, _ = cute.arch.thread_idx() + block, _, _ = cute.arch.block_idx() + warp = tid // 32 + lane = tid % 32 + sm = SmemAllocator().allocate_tensor( + cutlass.Uint8, cute.make_layout(SMEM), 1024 + ) + base = Int32(sm.iterator.toint()) + misc = base + OFF_MISC + crank = Int32(cute.arch.block_idx_in_cluster()) + cl = block // CLUSTER + # the PTX carries no cluster-shape directive (the launch sets it): refuse any + # other shape + P.trap_if(Int32(cute.arch.cluster_size() != CLUSTER)) + C.pdl_launch_dependents() + if tid == 0: + for s in cutlass.range_constexpr(RING): + C.mbarrier_init(misc + B_FULL + 8 * s, 1) + C.mbarrier_init(misc + B_EMPTY + 8 * s, 1) + C.mbarrier_init(misc + B_XREADY, 4) + C.mbarrier_init(misc + B_XBAR, 1) + # one arrive per lane of the table-building warp: each lane publishes its + # own unit-record stores (PTX memory model; no reliance on the warp barrier) + C.mbarrier_init(misc + B_TREADY, 32) + C.mbarrier_init(misc + B_SBAR, 1) + for b in cutlass.range_constexpr(2): + C.mbarrier_init(misc + B_ACC1_FULL + 8 * b, 1) + C.mbarrier_init(misc + B_ACC1_EMPTY + 8 * b, 4) + C.mbarrier_init(misc + B_ACC2_FULL, 1) + C.mbarrier_init(misc + B_ACC2_EMPTY, 4) + for s in cutlass.range_constexpr(RING): + C.mbarrier_init(misc + B_SFA_FULL + 8 * s, 1) + C.mbarrier_init(misc + B_SFX_READY, 1) + for b in cutlass.range_constexpr(H_BUFS): + C.mbarrier_init(misc + B_HREADY + 8 * b, 4) + for p in cutlass.range_constexpr(RECV_BUFS): + C.mbarrier_init(misc + B_PRECV + 8 * p, 1) + C.mbarrier_init(misc + B_PFREE + 8 * p, 3) + C.mbarrier_init(misc + B_CONSUMED + 8 * p, 4) + C.fence_mbarrier_init_release_cluster() + if warp == 1: + T.tmem_alloc(misc + M_TMEM, 512) + T.tc_fence_before() + cute.arch.sync_threads() + T.tc_fence_after() + cute.arch.cluster_arrive_relaxed(aligned=True) + # TMEM base, read by every thread once the allocation is visible + tm = Int32(C.ld_shared_u32(misc + M_TMEM)) + # per-request rows (byte q of mqs) and first global rows (byte q of rbs = sum of + # the earlier rows) + rbs = (mqs << 8) * 0x01010101 + gall = Int32(0) + for q in cutlass.range_constexpr(GROUPS): + if self.rows_of(mqs, q) > 0: + gall = gall | (1 << q) + my_sh = Int32(0) + nring = Int32(RING) + if cutlass.const_expr(self.shared): # noqa: SIM102 + if (nclusters >= 2 * SH_CLUSTERS) & (cl < SH_CLUSTERS): + my_sh = Int32(1) + nring = Int32(RING_SH) + if warp == 0: + self.producer( + tmap.get_ptr().toint(), + ids, + shpack, + mqs, + rbs, + gall, + err, + sched, + sf13, + fc2pack, + base, + misc, + crank, + cl, + my_sh, + nring, + lane, + ) + elif warp == 1: + self.mma_warp(base, misc, nring) + elif warp == SF_WARP: + self.sf_warp(base, misc, nring) + elif warp == RELEASE_WARP: + self.release_warp(misc, crank, lane) + else: + self.epilogue( + x, + ids, + topw, + g1a, + g2a, + gs1, + gs2, + rows, + mqs, + rbs, + shpartials, + pq, + ids_fin, + w_fin, + g2s, + base, + misc, + tm, + crank, + my_sh, + block, + warp, + lane, + tid, + ) + if (warp < 2) | (warp == SF_WARP): + cute.arch.cluster_wait() + T.tc_fence_before() + cute.arch.sync_threads() + T.tc_fence_after() + if warp == 1: + T.tmem_free(tm, 512) + cute.arch.cluster_arrive(aligned=True) + cute.arch.cluster_wait() diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/finalize.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/finalize.py new file mode 100644 index 00000000..7dcf692c --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/finalize.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Block output of the routed experts (+ the shared expert) from per-slot partials. + +For row n and output column r, with p_q = partials[q][n][k][r] (q = 0..3) the four +quarter partials of the expert in top-k slot k, g_k its FC2 output scale and w_k its +BF16 routing weight (as FP32): + + routed = fma(w_k, bf16(g_k * (((p_0 + p_1) + p_2) + p_3)), routed) + for k = 0..7 with ids[k] >= 0 + shared = (((s_0 + 0) + s_1) + s_2) + s_3, s_j = shared_partials[j][n][r] + out = bf16(bf16(routed) + bf16(shared)) (shared on) + bf16(routed) (shared off) + +which is the stock routed finalize (every expert output rounded to BF16, combined in +slot order with BF16 weights) followed by the stock routed + shared addition. One +256-thread block covers 256 consecutive outputs of one row; all loads are issued +before the sums. + +Scratch layout (FP32 unless noted), rows M <= 16: + partials [4][M][8][6144] + shared_partials [4][M][6144] + ids [M][8] int32 (-1: no expert in this slot) + weights, scales [M][8] + out [M][6144] bf16""" + +from __future__ import annotations + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass import Float32, Int32, Int64 +from cutlass.memory import SmemAllocator + +from ..primitives.ptx_core import cvt_rn_bf16_f32, fma_rn, st_global_b16 +from ._core_ptx import add_f32, ld_cg_f32, ld_cg_s32, mul_f32, round_bf16 +from .constants import HIDDEN, QUARTERS, TOP_K + +THREADS = 256 +BLOCKS_PER_ROW = HIDDEN // THREADS + + +class MoeFinalize: + """``shared``: add the shared expert's partials (the served form); off is + test-only.""" + + def __init__(self, shared: bool = True): + self.shared = shared + + @cute.jit + def __call__( + self, + partials: Int64, + shared_partials: Int64, + ids: Int64, + weights: Int64, + scales: Int64, + out: Int64, + rows: Int32, + stream: cuda.CUstream, + ): + self.kernel(partials, shared_partials, ids, weights, scales, out, rows).launch( + grid=[rows * BLOCKS_PER_ROW, 1, 1], + block=[THREADS, 1, 1], + max_number_threads=[THREADS, 1, 1], + stream=stream, + use_pdl=True, + hint_smem_base_uniform=False, + ) + + @cute.kernel + def kernel( + self, + partials: Int64, + shared_partials: Int64, + ids: Int64, + weights: Int64, + scales: Int64, + out: Int64, + rows: Int32, + ): + tid, _, _ = cute.arch.thread_idx() + block, _, _ = cute.arch.block_idx() + smem = SmemAllocator() + slot_ids = smem.allocate_tensor(Int32, cute.make_layout(TOP_K), 4) + slot_w = smem.allocate_tensor(Float32, cute.make_layout(TOP_K), 4) + slot_g = smem.allocate_tensor(Float32, cute.make_layout(TOP_K), 4) + + cute.arch.griddepcontrol_wait() + idx = block * THREADS + tid + n = (block * THREADS) // HIDDEN + r = idx - n * HIDDEN + live = idx < rows * HIDDEN + + part = cute.make_rmem_tensor(cute.make_layout(QUARTERS * TOP_K), Float32) + part.fill(0.0) + shared_sum = Float32(0.0) + if live: + column = partials + Int64(r) * 4 + for q in cutlass.range_constexpr(QUARTERS): + for k in cutlass.range_constexpr(TOP_K): + row = (Int64(q) * Int64(rows) + Int64(n)) * TOP_K + k + part[q * TOP_K + k] = ld_cg_f32(column + row * (HIDDEN * 4)) + if cutlass.const_expr(self.shared): + element = shared_partials + Int64(idx) * 4 + stride = Int64(rows) * (HIDDEN * 4) + s0 = ld_cg_f32(element) + s1 = ld_cg_f32(element + stride) + s2 = ld_cg_f32(element + 2 * stride) + s3 = ld_cg_f32(element + 3 * stride) + shared_sum = add_f32( + add_f32(add_f32(add_f32(s0, Float32(0.0)), s1), s2), s3 + ) + if (tid < TOP_K) & (n < rows): + offset = Int64(n * TOP_K + tid) * 4 + slot_ids[tid] = ld_cg_s32(ids + offset) + slot_w[tid] = ld_cg_f32(weights + offset) + slot_g[tid] = ld_cg_f32(scales + offset) + cute.arch.sync_threads() + if live: + routed = Float32(0.0) + for k in cutlass.range_constexpr(TOP_K): + if slot_ids[k] >= 0: + acc = add_f32( + add_f32(add_f32(part[k], part[TOP_K + k]), part[2 * TOP_K + k]), + part[3 * TOP_K + k], + ) + expert = round_bf16(mul_f32(acc, slot_g[k])) + routed = fma_rn(slot_w[k], expert, routed) + if cutlass.const_expr(self.shared): + routed = add_f32(round_bf16(routed), round_bf16(shared_sum)) + st_global_b16(out + Int64(idx) * 2, cvt_rn_bf16_f32(routed)) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/routing.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/routing.py new file mode 100644 index 00000000..bdf2a632 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/routing.py @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Top-8 routing of GLM-5.3 (sigmoid scores + correction bias, one expert group). + +Per token row: s_e = sigmoid(logit_e) (stock form), key_e = s_e + bias_e with NaN ranked +as -inf and -0 as +0, the 8 largest keys selected with ties going to the lower expert +id, and weights bf16(scale * s_e / (sum of the 8 selected s + 1e-20)). Every operation +and its order is the stock routing kernel's, so ids (in selection order) and weights are +bit-identical to it, except that a NaN score ranks last here (stock ranks it first). + +One warp per row; lane l holds experts 8 l .. 8 l + 7.""" + +from __future__ import annotations + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass import Float32, Int32, Int64, Uint32 + +from ..primitives.ptx_tc import ( + routing_add, + routing_key, + routing_sigmoid, + routing_weight, +) +from ._core_ptx import ld_cg_v4_f32, ld_nc_v4_f32, st_global_b32, st_global_b32f +from .constants import MAX_ROWS, NUM_EXPERTS, TOP_K + +PER_LANE = NUM_EXPERTS // 32 + + +@cute.jit +def top8_rows(scores, keys, lane: Int32): + """Select one row's top-8; the warp holds the row, ``scores`` / ``keys`` are this + lane's 8 experts. Returns (the expert chosen in round ``lane``, its score, the sum + of the 8 chosen scores); for lanes >= 8 the first two are (0, 0).""" + taken = Uint32(0) + chosen = Int32(0) + for rnd in cutlass.range(TOP_K, unroll=1): + best = Uint32(0) + best_j = Int32(0) + for j in cutlass.range_constexpr(PER_LANE - 1, -1, -1): + if (((taken >> j) & Uint32(1)) == Uint32(0)) & (keys[j] >= best): + best = keys[j] + best_j = Int32(j) + has = (taken & Uint32(0xFF)) != Uint32(0xFF) + offer = Uint32(0) + if has: + offer = best + top = cute.arch.warp_redux_sync(offer, "max") + candidate = Uint32(0xFFFFFFFF) + if has & (best == top): + candidate = Uint32(lane * PER_LANE + best_j) + winner = Int32(cute.arch.warp_redux_sync(candidate, "min")) + if lane == (winner >> 3): + taken = taken | (Uint32(1) << Uint32(winner & 7)) + if lane == rnd: + chosen = winner + source = (chosen >> 3) & 31 + mine = Float32(0.0) + for j in cutlass.range_constexpr(PER_LANE): + value = cute.arch.shuffle_sync(scores[j], source) + if (chosen & 7) == j: + mine = value + if lane >= TOP_K: + mine = Float32(0.0) + total = mine + for offset in (16, 8, 4, 2, 1): + total = routing_add(total, cute.arch.shuffle_sync_bfly(total, offset)) + return chosen, mine, total + + +class MoeRouteTop8: + """ids_out [M][8] int32 (selection order) and weights_out [M][8] fp32 (BF16 values) + from fp32 logits [M][256] and an fp32 bias [256]; M <= 16. Launches its dependents + at entry, then waits for its own inputs.""" + + @cute.jit + def __call__( + self, + logits: Int64, + bias: Int64, + scale: Float32, + ids_out: Int64, + weights_out: Int64, + rows: Int32, + stream: cuda.CUstream, + ): + self.kernel(logits, bias, scale, ids_out, weights_out, rows).launch( + grid=[1, 1, 1], + block=[32 * rows, 1, 1], + max_number_threads=[32 * MAX_ROWS, 1, 1], + stream=stream, + use_pdl=True, + ) + + @cute.kernel + def kernel( + self, + logits: Int64, + bias: Int64, + scale: Float32, + ids_out: Int64, + weights_out: Int64, + rows: Int32, + ): + cute.arch.griddepcontrol_launch_dependents() + cute.arch.griddepcontrol_wait() + tid, _, _ = cute.arch.thread_idx() + n = tid >> 5 + lane = tid & 31 + if n < rows: + src = logits + Int64(n * NUM_EXPERTS + lane * PER_LANE) * 4 + l0 = ld_cg_v4_f32(src) + l1 = ld_cg_v4_f32(src + 16) + b0 = ld_nc_v4_f32(bias + Int64(lane * PER_LANE) * 4) + b1 = ld_nc_v4_f32(bias + Int64(lane * PER_LANE) * 4 + 16) + logit = (*l0, *l1) + bias_v = (*b0, *b1) + scores = [routing_sigmoid(logit[j]) for j in range(PER_LANE)] + keys = [routing_key(scores[j], bias_v[j]) for j in range(PER_LANE)] + chosen, mine, total = top8_rows(scores, keys, lane) + if lane < TOP_K: + slot = Int64(n * TOP_K + lane) * 4 + st_global_b32(ids_out + slot, Uint32(chosen)) + st_global_b32f(weights_out + slot, routing_weight(scale, mine, total)) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/schedule.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/schedule.py new file mode 100644 index 00000000..06f6189a --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/schedule.py @@ -0,0 +1,150 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Host schedule tables of the small-M MoE engine (pure Python + torch, built once). + +A table holds, per cluster and per number D of distinct routed experts, the cluster's +unit list: [0] the unit count, [1 + k] the code of unit k, ``4 * (rank of the expert +among the distinct experts, ascending id) + quarter``, or -1 for this cluster's quarter +of the shared expert. The kernel stages its cluster's block into shared memory with one +bulk copy.""" + +from __future__ import annotations + +import torch + +SH_CLUSTERS = 4 + +# mode G1 (one request, M <= 4): D <= 32, at most 32 units per cluster +G1_ROW = 34 # [0] units, [1..32] codes, [33] partial count (unused by the engine) +G1_D = 33 +G1_MAX_UNITS = 32 +G1_BLOCK_BYTES = 2256 # 33 x 34 int16 = 2244 B, padded to a multiple of 16 + + +def g1_table(nclusters: int, shared: bool) -> torch.Tensor: + """int16 [nclusters][G1_BLOCK_BYTES / 2]. Without the shared expert, unit u goes to + cluster u % nclusters. With it (nclusters >= 8), clusters 0..3 run one shared + quarter each first; routed units fill clusters 4.. round-robin up to a per-cluster + cap, the remainder going round-robin to clusters 0..3.""" + nc = nclusters + with_shared = shared and nc >= 2 * SH_CLUSTERS + tab = torch.zeros(nc, G1_D, G1_ROW, dtype=torch.int16) + for d in range(G1_D): + units = 4 * d + lists: list[list[int]] = [[] for _ in range(nc)] + if not with_shared: + for u in range(units): + lists[u % nc].append(u) + parts = min(nc, units) + else: + routed_clusters = nc - SH_CLUSTERS + cap = min((units + 3 * SH_CLUSTERS + nc - 1) // nc, G1_MAX_UNITS) + lim = min(units, routed_clusters * cap) + for cl in range(SH_CLUSTERS): + lists[cl].append(-1) + for u in range(lim): + lists[SH_CLUSTERS + u % routed_clusters].append(u) + for u in range(lim, units): + lists[(u - lim) % SH_CLUSTERS].append(u) + parts = SH_CLUSTERS + min(routed_clusters, units) + for cl in range(nc): + if len(lists[cl]) > G1_MAX_UNITS: + raise ValueError( + f"schedule: cluster {cl} needs {len(lists[cl])} > " + f"{G1_MAX_UNITS} units at D={d}" + ) + tab[cl, d, 0] = len(lists[cl]) + for k, u in enumerate(lists[cl]): + tab[cl, d, 1 + k] = u + tab[cl, d, 33] = parts + flat = tab.view(nc, -1) + out = torch.zeros(nc, G1_BLOCK_BYTES // 2, dtype=torch.int16) + out[:, : flat.shape[1]] = flat + return out.contiguous() + + +# mode G2 (two requests, M <= 8): D <= 64 list entries, at most 15 units per cluster +G2_ROW = 16 # [0] units, [1..15] codes +G2_D = 65 +G2_MAX_UNITS = 15 +G2_BLOCK_BYTES = G2_D * G2_ROW * 2 # 2080 + + +def g2_table(nclusters: int, shared: bool) -> torch.Tensor: + """int16 [nclusters][G2_BLOCK_BYTES / 2]: the G1 assignment rule over D <= 64 + entries with a 15-unit cluster cap.""" + nc = nclusters + with_shared = shared and nc >= 2 * SH_CLUSTERS + tab = torch.zeros(nc, G2_D, G2_ROW, dtype=torch.int16) + for d in range(G2_D): + units = 4 * d + lists: list[list[int]] = [[] for _ in range(nc)] + if not with_shared: + for u in range(units): + lists[u % nc].append(u) + else: + routed_clusters = nc - SH_CLUSTERS + cap = min((units + 3 * SH_CLUSTERS + nc - 1) // nc, G2_MAX_UNITS) + lim = min(units, routed_clusters * cap) + for cl in range(SH_CLUSTERS): + lists[cl].append(-1) + for u in range(lim): + lists[SH_CLUSTERS + u % routed_clusters].append(u) + for u in range(lim, units): + lists[(u - lim) % SH_CLUSTERS].append(u) + for cl in range(nc): + if len(lists[cl]) > G2_MAX_UNITS: + raise ValueError( + f"schedule: cluster {cl} needs {len(lists[cl])} > " + f"{G2_MAX_UNITS} units at D={d}" + ) + tab[cl, d, 0] = len(lists[cl]) + for k, u in enumerate(lists[cl]): + tab[cl, d, 1 + k] = u + return tab.view(nc, -1).contiguous() + + +# mode G4 (two to four requests, M <= 16): D <= 128 list entries, at most 17 units per +# cluster; the shared quarter counts as 4 routed units in the cap +G4_ROW = 18 # [0] units, [1..17] codes +G4_D = 136 # D = 0..128 (+ padding) +G4_MAX_UNITS = 17 +G4_BLOCK_BYTES = G4_D * G4_ROW * 2 # 4896 +G4_SHARED_WEIGHT = 4 + + +def g4_table(nclusters: int, shared: bool) -> torch.Tensor: + """int16 [nclusters][G4_BLOCK_BYTES / 2]: the G1 assignment rule over D <= 128 + entries with the shared quarter weighted as 4 units and a 17-unit cluster cap.""" + nc = nclusters + with_shared = shared and nc >= 2 * SH_CLUSTERS + tab = torch.zeros(nc, G4_D, G4_ROW, dtype=torch.int16) + for d in range(129): + units = 4 * d + lists: list[list[int]] = [[] for _ in range(nc)] + if not with_shared: + for u in range(units): + lists[u % nc].append(u) + else: + routed_clusters = nc - SH_CLUSTERS + cap = min( + (units + G4_SHARED_WEIGHT * SH_CLUSTERS + nc - 1) // nc, G4_MAX_UNITS + ) + lim = min(units, routed_clusters * cap) + for cl in range(SH_CLUSTERS): + lists[cl].append(-1) + for u in range(lim): + lists[SH_CLUSTERS + u % routed_clusters].append(u) + for u in range(lim, units): + lists[(u - lim) % SH_CLUSTERS].append(u) + for cl in range(nc): + if len(lists[cl]) > G4_MAX_UNITS: + raise ValueError( + f"schedule: cluster {cl} needs {len(lists[cl])} > " + f"{G4_MAX_UNITS} units at D={d}" + ) + tab[cl, d, 0] = len(lists[cl]) + for k, u in enumerate(lists[cl]): + tab[cl, d, 1 + k] = u + return tab.view(nc, -1).contiguous() diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/weights.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/weights.py new file mode 100644 index 00000000..01d27f84 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/moe/weights.py @@ -0,0 +1,221 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Weight forms of the small-M NVFP4 MoE engine (torch only, no kernels). + +The engine reads the routed FC1 weights in place, from the engine's own processed +TRT-LLM NVFP4 tensors (``w13`` uint8 [E, 512, 3072], ``w13`` scales [E, 512 x 384] +bytes), so it relies on how that processing permutes rows: + +* FC1: processed row ``128 j + 32 q + i`` (j, q in 0..3, i in 0..31) holds the gate row + (if ``i & 8``) or the up row (otherwise) of intermediate index ``h = 64 j + 16 q + 2 + (i & 7) + (i >> 4)``; +* FC2: processed row ``32 b + p`` of ``w2`` [E, 6144, 128] holds source row ``32 b + 4 + (p & 7) + (p >> 3)``; +* block scales follow their rows and are stored in the 128 x 4 interleave: the byte of + (row, column) sits at ``((row // 128) * (C // 4) + column // 4) * 512 + (row % 32) * + 16 + ((row % 128) // 32) * 4 + column % 4`` for C scale columns. + +``layout_self_test`` runs the engine's own processing function on a synthetic layer and +refuses the single-copy form unless both maps hold byte for byte. FC2 is read from +compact per-quarter records (``fc2_records``), rebuilt from the processed tensors with +the inverse maps; the shared expert from ``shared_pack``.""" + +from __future__ import annotations + +import torch + +from .constants import HIDDEN, INTER + +FC1_ROWS = 2 * INTER # 512 (gate + up) +FC1_BYTES = HIDDEN // 2 # 3072 packed e2m1 bytes per FC1 row +FC1_SF = HIDDEN // 16 # 384 scales per FC1 row +FC2_BYTES = INTER // 2 # 128 packed bytes per FC2 row +FC2_SF = INTER // 16 # 16 scales per FC2 row +SLOT = 18432 # one record slot: 16 KiB of weights + 2 KiB of scales +FC2_SLOTS = 3 +FC2_RECORD = FC2_SLOTS * SLOT # 55,296 bytes per (expert, quarter, CTA) + + +class LayoutMismatch(RuntimeError): + """The processed weights do not have the layout the engine reads.""" + + +def fc1_sources(device=None) -> torch.Tensor: + """[512] source row of each processed FC1 row, indexing cat([gate, up]) (gate + 0..255).""" + r = torch.arange(FC1_ROWS) + i = r % 32 + h = 64 * (r // 128) + 16 * ((r // 32) % 4) + 2 * (i & 7) + (i >> 4) + return torch.where((i & 8) != 0, h, INTER + h).to(device) + + +def fc2_sources(device=None) -> torch.Tensor: + """[6144] source row of each processed FC2 row.""" + r = torch.arange(HIDDEN) + p = r % 32 + return ((r // 32) * 32 + 4 * (p & 7) + (p >> 3)).to(device) + + +def interleave_offsets(rows: int, cols: int, device=None) -> torch.Tensor: + """[rows, cols] byte offset of each scale in the 128 x 4 interleave.""" + r = torch.arange(rows).view(rows, 1) + c = torch.arange(cols).view(1, cols) + off = ( + ((r // 128) * (cols // 4) + c // 4) * 512 + + (r % 32) * 16 + + ((r % 128) // 32) * 4 + + c % 4 + ) + return off.to(device) + + +def interleave(sf: torch.Tensor) -> torch.Tensor: + """[..., rows, cols] uint8 scales -> [..., rows * cols] interleaved bytes.""" + rows, cols = sf.shape[-2:] + off = interleave_offsets(rows, cols, sf.device).flatten() + out = torch.empty(*sf.shape[:-2], rows * cols, dtype=torch.uint8, device=sf.device) + out[..., off] = sf.reshape(*sf.shape[:-2], rows * cols) + return out + + +def deinterleave(flat: torch.Tensor, rows: int, cols: int) -> torch.Tensor: + """Inverse of ``interleave``: [..., rows * cols] bytes -> [..., rows, cols].""" + off = interleave_offsets(rows, cols, flat.device).flatten() + return flat[..., off].reshape(*flat.shape[:-1], rows, cols) + + +def _u8(t: torch.Tensor) -> torch.Tensor: + return t if t.dtype == torch.uint8 else t.view(torch.uint8) + + +def expected_processed(gate, up, gate_sf, up_sf, w2, w2_sf): + """The processed tensors the engine reads, from source-layout inputs (uint8): + gate / up [E, 256, 3072], their scales [E, 256, 384], w2 [E, 6144, 128], w2_sf + [E, 6144, 16]. Returns (w13 [E, 512, 3072], w13_sf [E, 512 * 384], w2, w2_sf + [E, 6144 * 16]).""" + src1 = fc1_sources(gate.device) + src2 = fc2_sources(w2.device) + w13 = torch.cat([gate, up], dim=1)[:, src1] + s13 = interleave(torch.cat([gate_sf, up_sf], dim=1)[:, src1]) + return w13, s13, w2[:, src2], interleave(w2_sf[:, src2]) + + +def layout_self_test(process, device, experts: int = 3, seed: int = 1000) -> int: + """Run ``process(gate, up, gate_sf, up_sf, w2, w2_sf) -> (w13, w13_sf, w2, w2_sf)`` + (the engine's own stock processing, from its loader layout) on a synthetic layer of + random bytes and compare every byte with the layout the engine reads. Returns the + number of bytes compared; raises ``LayoutMismatch`` on any difference.""" + g = torch.Generator(device="cpu").manual_seed(seed) + + def rand(*shape): + return torch.randint(0, 256, shape, generator=g, dtype=torch.uint8).to(device) + + src = ( + rand(experts, INTER, FC1_BYTES), + rand(experts, INTER, FC1_BYTES), + rand(experts, INTER, FC1_SF), + rand(experts, INTER, FC1_SF), + rand(experts, HIDDEN, FC2_BYTES), + rand(experts, HIDDEN, FC2_SF), + ) + want = expected_processed(*src) + got = process(*[t.clone() for t in src]) + names = ("w13", "w13 scales", "w2", "w2 scales") + count = 0 + for name, w, t in zip(names, want, got): + t = _u8(t).reshape(experts, -1) + w = w.reshape(experts, -1) + if t.shape != w.shape or not torch.equal(t, w): + raise LayoutMismatch( + f"processed {name} does not have the layout the fused MoE reads " + f"(shape {tuple(t.shape)} vs {tuple(w.shape)})" + ) + count += w.numel() + if count == 0: + raise LayoutMismatch("layout self-test compared no bytes") + return count + + +# ---------------------------------------------------------------- FC2 records +def fc2_records(w2: torch.Tensor, w2_sf: torch.Tensor, chunk: int = 32) -> torch.Tensor: + """Compact FC2 records [E][4 j][4 c][3 x 18,432 B] from one layer's processed w2 + (uint8 [E, 6144, 128]) and its processed scales ([E, 6144 * 16] bytes), built + ``chunk`` experts at a time. Record (e, j, c), slot s, tile t (row block + ``1536 c + 512 s + 128 t``): 4 KiB of weights in the engine's K-major image + [16 row groups][2 K halves][8 rows][16 B] for K bytes ``32 j .. 32 j + 31``, then + 512 B of scales [32 l][4 q][4 kb].""" + experts = w2.shape[0] + w2 = _u8(w2).reshape(experts, HIDDEN, FC2_BYTES) + sf = _u8(w2_sf).reshape(experts, HIDDEN * FC2_SF) + inv2 = torch.argsort(fc2_sources(w2.device)) + out = torch.empty( + experts, 4, 4, FC2_SLOTS * SLOT, dtype=torch.uint8, device=w2.device + ) + for e0 in range(0, experts, chunk): + ne = min(chunk, experts - e0) + raw = w2[e0 : e0 + ne][ + :, inv2 + ] # [ne, 6144 = c*1536 + t*128 + rg*8 + rr, 128 = j*32 + kc*16 + b] + a2 = ( + raw.view(ne, 4, 12, 16, 8, 4, 2, 16) + .permute(0, 5, 1, 2, 3, 6, 4, 7) + .reshape(ne, 4, 4, 3, 4 * 4096) + ) + rs = deinterleave(sf[e0 : e0 + ne], HIDDEN, FC2_SF)[ + :, inv2 + ] # [ne, 6144 = c*1536 + t*128 + q*32 + l, 16] + s2 = ( + rs.view(ne, 4, 12, 4, 32, 4, 4) + .permute(0, 5, 1, 2, 4, 3, 6) + .reshape(ne, 4, 4, 3, 4 * 512) + ) + out[e0 : e0 + ne] = torch.cat([a2, s2], dim=-1).reshape( + ne, 4, 4, FC2_SLOTS * SLOT + ) + return out + + +# ---------------------------------------------------------------- shared expert pack +SHARED_CHUNK = 32768 +SHARED_RECORD = 18 * SHARED_CHUNK # 589,824 bytes per (quarter, CTA) + + +def shared_rows(device=None) -> torch.Tensor: + """[4 quarters x 128] row of gate_up [512, 6144] (gate 0..255, up 256..511) for each + local row: quarter sq, local row 32 q + i is gate row 64 sq + 16 q + i (i < 16) or + up row 64 sq + 16 q + i - 16.""" + sq = torch.arange(4).view(4, 1, 1) + q = torch.arange(4).view(1, 4, 1) + i = torch.arange(32).view(1, 1, 32) + h = 64 * sq + 16 * q + (i % 16) + return torch.where(i < 16, h, INTER + h).reshape(-1).to(device) + + +def shared_pack(gate_up: torch.Tensor, down: torch.Tensor) -> torch.Tensor: + """BF16 shared expert (gate_up [512, 6144], gate rows first; down [6144, 256]) -> [4 + quarters][4 CTAs][589,824 B]: FC1 as 24 boxes of 128 rows x 64 K (no-swizzle K-major + images [16][8 K groups][8 rows][16 B]) over K range ``1536 c .. +1536``, then FC2 as + 12 tiles of 128 output rows x 64 K (``1536 c + 128 t + r``, K = h in the quarter). + """ + if gate_up.dtype != torch.bfloat16 or tuple(gate_up.shape) != (FC1_ROWS, HIDDEN): + raise ValueError(f"gate_up must be bf16 [{FC1_ROWS}, {HIDDEN}]") + if down.dtype != torch.bfloat16 or tuple(down.shape) != (HIDDEN, INTER): + raise ValueError(f"down must be bf16 [{HIDDEN}, {INTER}]") + w = ( + gate_up[shared_rows(gate_up.device)].contiguous().view(torch.uint8) + ) # [512, 12288] + a = ( + w.view(4, 16, 8, 4, 24, 8, 16) + .permute(0, 3, 4, 1, 5, 2, 6) + .reshape(4, 4, 24 * 16384) + ) + d = down.contiguous().view( + torch.uint8 + ) # [6144 = c*1536 + t*128 + rg*8 + rr, 512 = sq*128 + kc*16 + b] + f2 = ( + d.view(4, 12, 16, 8, 4, 8, 16) + .permute(4, 0, 1, 2, 5, 3, 6) + .reshape(4, 4, 12 * 16384) + ) + return torch.cat([a, f2], dim=-1).contiguous() diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/packing.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/packing.py new file mode 100644 index 00000000..fd1b1f73 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/packing.py @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Weight packers for the attention kernels: pure functions of the checkpoint tensors. + +Each packer returns a new contiguous tensor (a derived copy); the input is not +modified. Shapes are those of one tensor-parallel rank (8 attention heads). +""" + +import torch + +HEADS = 8 +LATENT = 512 # kv_lora_rank +NOPE = 192 # qk_nope_head_dim +VALUE_DIM = 256 # v_head_dim +KV_B_ROWS_PER_HEAD = NOPE + VALUE_DIM + + +def pack_uv_proj(kv_b_weight: torch.Tensor, values_per_cta: int = 16) -> torch.Tensor: + """W_UV tiles for ``attn.mla_uv_proj``. + + ``kv_b_weight``: ``kv_b_proj.weight`` of one rank, ``[8 * 448, 512]`` BF16 (rows + ``[448 h + 192, 448 h + 448)`` are head ``h``'s value rows, each a contiguous + 512-element latent row). Result: flat BF16, ordered + ``[head][value tile j][latent step s][value row r][32 latent]``, so the tile of + CTA ``(h, j)`` is one contiguous ``values_per_cta x 1 KiB`` block. + """ + if kv_b_weight.dtype != torch.bfloat16: + raise ValueError(f"kv_b_proj weight must be bf16, got {kv_b_weight.dtype}") + if tuple(kv_b_weight.shape) != (HEADS * KV_B_ROWS_PER_HEAD, LATENT): + raise ValueError( + f"kv_b_proj weight must be [{HEADS * KV_B_ROWS_PER_HEAD}, {LATENT}], " + f"got {tuple(kv_b_weight.shape)}" + ) + if values_per_cta != 16: + raise ValueError("values_per_cta must be 16") + w = kv_b_weight.view(HEADS, KV_B_ROWS_PER_HEAD, LATENT)[:, NOPE:, :] + w = w.reshape(HEADS, VALUE_DIM // values_per_cta, values_per_cta, LATENT // 32, 32) + return w.permute(0, 1, 3, 2, 4).contiguous().view(-1) + + +INDEXER_HEADS = 32 +INDEXER_HEAD_DIM = 128 +Q_LORA = 2048 + + +def pack_indexer_q(wq_b: torch.Tensor) -> torch.Tensor: + """Indexer query weight tiles for ``attn.indexer_q_proj``. + + ``wq_b``: the indexer ``wq_b.weight``, ``[32 * 128, 2048]`` BF16. Result: flat BF16 + ordered ``[head][part 4][latent step 64][row 32][32 latent]``: CTA ``(head, part)`` + reads one contiguous 128 KiB block. + """ + if wq_b.dtype != torch.bfloat16 or tuple(wq_b.shape) != ( + INDEXER_HEADS * INDEXER_HEAD_DIM, + Q_LORA, + ): + raise ValueError( + "indexer wq_b must be bf16 " + f"[{INDEXER_HEADS * INDEXER_HEAD_DIM}, {Q_LORA}], " + f"got {wq_b.dtype} {tuple(wq_b.shape)}" + ) + v = wq_b.contiguous().view(INDEXER_HEADS, 4, 32, Q_LORA // 32, 32) + return v.permute(0, 1, 3, 2, 4).contiguous().view(-1) + + +QKV_A_K = 6144 + + +def pack_qkv_a(w: torch.Tensor, grid: int) -> torch.Tensor: + """Row-owned blocks for ``attn.qkv_a_proj_small_m`` launched on ``grid`` CTAs. + + ``w``: ``[N, 6144]`` BF16 (the projections' rows stacked: main rows first, then the + auxiliary rows), N even. CTA ``c`` owns rows ``[2 floor(N/2 c / grid), 2 floor(N/2 + (c + 1) / grid))``; its block is ``[k-step 192][row][32]`` and the blocks are + concatenated in CTA order. The result depends on ``grid``: pack with the grid the + kernel is launched with. + """ + n, k = w.shape + if k != QKV_A_K or n % 2 or w.dtype != torch.bfloat16: + raise ValueError( + f"weight must be bf16 [even N, {QKV_A_K}], got {w.dtype} {tuple(w.shape)}" + ) + groups = n // 2 + blocks = [] + for c in range(grid): + r0, r1 = 2 * (groups * c // grid), 2 * (groups * (c + 1) // grid) + blocks.append( + w[r0:r1] + .reshape(r1 - r0, k // 32, 32) + .permute(1, 0, 2) + .contiguous() + .view(-1) + ) + return torch.cat(blocks) + + +Q_HEAD = NOPE + 64 # q_b rows per head: nope 192 | rope 64 + + +def _pack_rows_k(w: torch.Tensor) -> torch.Tensor: + """``[N, K]`` BF16 -> 1 KiB blocks per (16-row tile, 32-wide k-step) in ``mma.sync`` + lane order: rows + ``16 tile + 8 half + g``, elements ``32 step + 8 t + e`` at + ``[tile][step][half][g][t][e]``.""" + n, k = w.shape + if n % 16 or k % 32 or w.dtype != torch.bfloat16: + raise ValueError(f"expected bf16 [16 i, 32 j], got {w.dtype} {tuple(w.shape)}") + v = w.contiguous().view(n // 16, 2, 8, k // 32, 4, 8) + return v.permute(0, 3, 1, 2, 4, 5).contiguous().view(-1) + + +def pack_q_b(w_qb: torch.Tensor) -> torch.Tensor: + """q_b tiles for ``attn.mla_prologue``: ``q_b_proj.weight`` of one rank, ``[8 * 256, + 2048]`` BF16.""" + if tuple(w_qb.shape) != (HEADS * Q_HEAD, 2048): + raise ValueError( + f"q_b_proj weight must be [{HEADS * Q_HEAD}, 2048], got {tuple(w_qb.shape)}" + ) + return _pack_rows_k(w_qb) + + +def pack_w_uk_t(kv_b_weight: torch.Tensor) -> torch.Tensor: + """W_UK^T tiles for ``attn.mla_prologue``: from ``kv_b_proj.weight`` of one rank + (``[8 * 448, 512]`` BF16, the + first 192 rows of each head are W_UK), per head ``[512 (latent), 192 (nope)]`` + packed, heads concatenated.""" + if kv_b_weight.dtype != torch.bfloat16 or tuple(kv_b_weight.shape) != ( + HEADS * KV_B_ROWS_PER_HEAD, + LATENT, + ): + raise ValueError( + f"kv_b_proj weight must be bf16 [{HEADS * KV_B_ROWS_PER_HEAD}, {LATENT}]" + ) + wt = ( + kv_b_weight.view(HEADS, KV_B_ROWS_PER_HEAD, LATENT)[:, :NOPE, :] + .transpose(1, 2) + .contiguous() + ) + return torch.cat([_pack_rows_k(wt[h]) for h in range(HEADS)]) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/__init__.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/__init__.py new file mode 100644 index 00000000..b4896b01 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/ptx_core.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/ptx_core.py new file mode 100644 index 00000000..aa5a58e0 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/ptx_core.py @@ -0,0 +1,845 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Inline-PTX helpers shared by the kernels of this package. + +Every helper names one PTX instruction (or a short fixed sequence) so that the +generated code is exactly the instruction sequence the kernel author wrote. +Two decorators build them: + +* ``ptx_op``: memory, synchronization and protocol instructions. They are + emitted with side effects and a ``memory`` clobber, so the compiler neither + removes them nor moves other memory accesses across them. +* ``ptx_arith``: arithmetic with an explicit rounding mode. They are pure (no + side effects, no clobber), so they do not serialize the surrounding loops. + Floating-point arithmetic on values that reach an output goes through these + helpers, never through Python operators or ``cute.math``: ``add.rn`` / + ``mul.rn`` are never contracted into an FMA, and ``fma.rn`` is always one. + +Barrier and warp-collective instructions (``bar.sync``, ``barrier.cluster``, +shuffles, votes) use the ``cute.arch`` intrinsics; this module only re-exports +them under one name per instruction. + +Operands are CuTe DSL numerics, Python ints / floats, or CuTe pointers (whose +address is used: 32-bit for shared memory, 64-bit for global memory). +""" + +from functools import wraps + +import cutlass +import cutlass.cute as cute +from cutlass import Boolean, Float32, Float64, Int32, Int64, Uint16, Uint32, Uint64 +from cutlass._mlir.dialects import llvm +from cutlass.cutlass_dsl import dsl_user_op + +__all__ = [ + "ptx_op", + "ptx_arith", + "shared_address", + # programmatic dependent launch + "pdl_launch_dependents", + "pdl_wait", + # fences + "fence_proxy_async_global", + "fence_proxy_async_shared_cta", + "fence_mbarrier_init_release_cluster", + # mbarrier + "mbarrier_init", + "mbarrier_arrive", + "mbarrier_arrive_expect_tx", + "mbarrier_expect_tx", + "mbarrier_try_wait_parity", + "mbarrier_wait_parity", + # bulk copies and L2 hints + "bulk_copy_g2s", + "bulk_prefetch_l2", + "l2_policy", + # loads and stores + "ld_global_v4", + "ld_shared_v4", + "ld_shared_f32", + "ld_shared_u32", + "st_shared_u32", + "st_shared_f32", + "st_shared_v2_f32", + "st_shared_v4", + "st_global_b16", + "st_global_u32", + "st_global_v4", + # cluster and distributed shared memory + "sync_threads", + "cluster_arrive_release", + "cluster_arrive_relaxed", + "cluster_wait_acquire", + "mapa_shared_cluster", + "ld_shared_cluster_u32", + "ld_shared_cluster_f32", + "ld_shared_cluster_v4", + "st_async_f32", + "st_async_v2_f32", + # warp-level matrix instructions + "mma_m16n8k16_bf16", + "mma_m16n8k32_e4m3", + "ldmatrix_x4", + "ldmatrix_x2", + # arithmetic with an explicit rounding mode + "add_rn", + "sub_rn", + "mul_rn", + "fma_rn", + "div_rn", + "div_full", + "rsqrt_approx", + "max_nan", + "cvt_rn_bf16_f32", + "cvt_rn_bf16x2_f32", + "cvt_rn_satfinite_e4m3x2_f32", + "cvt_rn_f16x2_e4m3x2", + "cvt_rn_f16_f32", + "bf16x2_to_f32x2", + "ex2_approx", + "prmt", + "e4m3x2_to_f32x2", + "st_shared_u8", + "ld_shared_v2_f32", + "ld_global_s32", + "ld_global_s64", + "ld_global_f32", + "ld_global_u16", + "st_global_f32", + "max_f32", + "abs_f32", + "f32_bits", + "bits_f32", + "bf16_to_f32", + "neg_f32", + "mbarrier_wait_parity_spin", + "st_async_v4_b32", + "ld_global_u32_if", + "ld_global_v4_if", + "ld_global_v4_no_allocate", + "st_global_u64", +] + +_BY_CONSTRAINT = { + "r": (Int32, Uint32), + "l": (Int64, Uint64), + "h": (Uint16,), + "f": (Float32,), + "d": (Float64,), +} + + +def _operand(kind, value, loc, ip): + """IR value of one asm input for constraint letter ``kind``.""" + if hasattr(value, "toint"): # a CuTe pointer: its address + value = value.toint(loc=loc, ip=ip) + accepted = _BY_CONSTRAINT[kind] + if not isinstance(value, accepted): + value = accepted[0](value) + return value.ir_value(loc=loc, ip=ip) + + +def _emit(code, constraints, operands, result, side_effects, loc, ip): + kinds = [c for c in constraints.split(",") if c and c[0] not in "=~"] + if len(kinds) != len(operands): + raise ValueError(f"asm operand count {len(operands)} != constraints {kinds}") + values = [_operand(k, v, loc, ip) for k, v in zip(kinds, operands)] + if result is None: + ir_type = None + elif isinstance(result, tuple): + ir_type = llvm.StructType.get_literal([t.mlir_type for t in result]) + else: + ir_type = result.mlir_type + out = llvm.inline_asm( + ir_type, + values, + code, + constraints, + has_side_effects=side_effects, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + if result is None: + return None + if isinstance(result, tuple): + return tuple( + t(llvm.extractvalue(t.mlir_type, out, [i], loc=loc, ip=ip)) + for i, t in enumerate(result) + ) + return result(out) + + +def _decorator(result, side_effects, clobber): + def wrap(fn): + @dsl_user_op + @wraps(fn) + def op(*args, loc=None, ip=None, **kwargs): + code, constraints, operands, *rest = fn(*args, **kwargs) + res = rest[0] if rest else result + if clobber: + constraints = ( + f"{constraints},~{{memory}}" if constraints else "~{memory}" + ) + return _emit(code, constraints, tuple(operands), res, side_effects, loc, ip) + + return op + + return wrap + + +def ptx_op(result=None): + """Decorator for memory / synchronization instructions. + + The wrapped function returns ``(code, constraints, operands[, result])``; + the instruction is emitted with side effects and a ``memory`` clobber. + """ + return _decorator(result, True, True) + + +def ptx_arith(result): + """Decorator for pure arithmetic: no side effects, no clobber.""" + return _decorator(result, False, False) + + +def shared_address(pointer): + """32-bit shared-memory address of a CuTe shared-memory pointer.""" + return Int32(pointer.toint()) + + +def _imm_or_reg(value, index): + """An immediate for a Python int, else operand ``$index``.""" + return (str(value), ()) if isinstance(value, int) else (f"${index}", (value,)) + + +# ---------------------------------------------------------------------------- +# Programmatic dependent launch + + +@ptx_op() +def pdl_launch_dependents(): + return "griddepcontrol.launch_dependents;", "", () + + +@ptx_op() +def pdl_wait(): + return "griddepcontrol.wait;", "", () + + +# ---------------------------------------------------------------------------- +# Fences + + +@ptx_op() +def fence_proxy_async_global(): + return "fence.proxy.async.global;", "", () + + +@ptx_op() +def fence_proxy_async_shared_cta(): + return "fence.proxy.async.shared::cta;", "", () + + +@ptx_op() +def fence_mbarrier_init_release_cluster(): + return "fence.mbarrier_init.release.cluster;", "", () + + +# ---------------------------------------------------------------------------- +# mbarrier (``bar`` is a 32-bit shared-memory address). ``sem_scope`` is the +# PTX qualifier string, e.g. "" (the instruction's default), ".release.cta", +# ".release.cluster", ".acquire.cluster", ".relaxed.cta". + + +@ptx_op() +def mbarrier_init(bar, count): + return "mbarrier.init.shared::cta.b64 [$0], $1;", "r,r", (bar, Int32(count)) + + +@ptx_op() +def mbarrier_arrive(bar, sem_scope=""): + return f"mbarrier.arrive{sem_scope}.shared::cta.b64 _, [$0];", "r", (bar,) + + +@ptx_op() +def mbarrier_arrive_expect_tx(bar, tx_bytes, sem_scope=""): + code = f"mbarrier.arrive.expect_tx{sem_scope}.shared::cta.b64 _, [$0], $1;" + return code, "r,r", (bar, Int32(tx_bytes)) + + +@ptx_op() +def mbarrier_expect_tx(bar, tx_bytes, sem_scope=""): + code = f"mbarrier.expect_tx{sem_scope}.shared::cta.b64 [$0], $1;" + return code, "r,r", (bar, Int32(tx_bytes)) + + +@ptx_op(Uint32) +def mbarrier_try_wait_parity(bar, parity, sem_scope=""): + """1 if the phase with ``parity`` has completed, else 0 (one probe).""" + p, extra = _imm_or_reg(parity, 2) + code = ( + "{ .reg .pred P; " + f"mbarrier.try_wait.parity{sem_scope}.shared::cta.b64 P, [$1], {p}; " + "selp.u32 $0, 1, 0, P; }" + ) + constraints = "=r,r" + (",r" if extra else "") + return code, constraints, (bar, *(Int32(v) for v in extra)) + + +@cute.jit +def mbarrier_wait_parity(bar, parity, sem_scope: cutlass.Constexpr[str] = ""): + """Probe until the phase with ``parity`` has completed.""" + done = mbarrier_try_wait_parity(bar, parity, sem_scope) + while done == Uint32(0): + done = mbarrier_try_wait_parity(bar, parity, sem_scope) + + +@ptx_op() +def mbarrier_wait_parity_spin(bar, parity, sem_scope=""): + """Probe until the phase with ``parity`` has completed, as one asm loop (the + form whose op sequence matches a kernel that spins on ``try_wait`` in asm); + ``sem_scope`` as for ``mbarrier_try_wait_parity``.""" + code = ( + "{ .reg .pred p; MBW_${:uid}: " + f"mbarrier.try_wait.parity{sem_scope}.shared::cta.b64 p, [$0], $1; " + "@!p bra MBW_${:uid}; }" + ) + return code, "r,r", (Uint32(bar), Uint32(parity)) + + +# ---------------------------------------------------------------------------- +# Bulk copies and L2 hints (``src`` / ``addr`` are 64-bit global addresses) + + +@ptx_op() +def bulk_copy_g2s(dst, src, size, bar, policy=None): + """Bulk copy of ``size`` bytes global -> shared, completing on ``bar``.""" + base = "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes" + if policy is None: + return f"{base} [$0], [$1], $2, [$3];", "r,l,r,r", (dst, src, Int32(size), bar) + code = f"{base}.L2::cache_hint [$0], [$1], $2, [$3], $4;" + return code, "r,l,r,r,l", (dst, src, Int32(size), bar, policy) + + +@ptx_op() +def bulk_prefetch_l2(addr, size, policy=None): + """Bulk prefetch of ``size`` bytes into L2 (a hint; no output depends on it).""" + if policy is None: + return "cp.async.bulk.prefetch.L2.global [$0], $1;", "l,r", (addr, Int32(size)) + code = "cp.async.bulk.prefetch.L2.global.L2::cache_hint [$0], $1, $2;" + return code, "l,r,l", (addr, Int32(size), policy) + + +def l2_policy(priority: str): + """L2 cache policy (fraction 1.0) with eviction ``priority``: "evict_last", + "evict_normal", "evict_first" or "evict_unchanged".""" + return _createpolicy(priority) + + +@ptx_arith(Uint64) +def _createpolicy(priority): + return f"createpolicy.fractional.L2::{priority}.b64 $0, 1.0;", "=l", () + + +# ---------------------------------------------------------------------------- +# Loads and stores. Global loads here are coherent (never ``.nc``): they read +# data that may be written in the same step, after the PDL wait. + + +@ptx_op((Uint32, Uint32, Uint32, Uint32)) +def ld_global_v4(addr): + return "ld.global.v4.u32 {$0, $1, $2, $3}, [$4];", "=r,=r,=r,=r,l", (addr,) + + +@ptx_op((Uint32, Uint32, Uint32, Uint32)) +def ld_shared_v4(addr): + return "ld.shared.v4.u32 {$0, $1, $2, $3}, [$4];", "=r,=r,=r,=r,r", (addr,) + + +@ptx_op(Float32) +def ld_shared_f32(addr): + return "ld.shared.f32 $0, [$1];", "=f,r", (addr,) + + +@ptx_op(Uint32) +def ld_shared_u32(addr): + return "ld.shared.u32 $0, [$1];", "=r,r", (addr,) + + +@ptx_op() +def st_shared_u32(addr, value): + return "st.shared.u32 [$0], $1;", "r,r", (addr, Uint32(value)) + + +@ptx_op() +def st_shared_f32(addr, value): + return "st.shared.f32 [$0], $1;", "r,f", (addr, value) + + +@ptx_op() +def st_shared_v2_f32(addr, a, b): + return "st.shared.v2.f32 [$0], {$1, $2};", "r,f,f", (addr, a, b) + + +@ptx_op() +def st_shared_v4(addr, words): + return "st.shared.v4.u32 [$0], {$1, $2, $3, $4};", "r,r,r,r,r", (addr, *words) + + +@ptx_op() +def st_global_b16(addr, value): + return "st.global.b16 [$0], $1;", "l,h", (addr, value) + + +@ptx_op() +def st_global_u32(addr, value): + return "st.global.u32 [$0], $1;", "l,r", (addr, Uint32(value)) + + +@ptx_op() +def st_global_v4(addr, words): + return "st.global.v4.u32 [$0], {$1, $2, $3, $4};", "l,r,r,r,r", (addr, *words) + + +# ---------------------------------------------------------------------------- +# CTA / cluster barriers (cute.arch intrinsics) and distributed shared memory + + +def sync_threads(): + """``bar.sync 0``: all threads of the CTA.""" + cute.arch.sync_threads() + + +def cluster_arrive_release(): + """``barrier.cluster.arrive`` with its default release semantics.""" + cute.arch.cluster_arrive() + + +def cluster_arrive_relaxed(): + """``barrier.cluster.arrive.relaxed``.""" + cute.arch.cluster_arrive_relaxed() + + +def cluster_wait_acquire(): + """``barrier.cluster.wait`` with its default acquire semantics.""" + cute.arch.cluster_wait() + + +@ptx_arith(Uint32) +def mapa_shared_cluster(addr, rank): + """Shared-memory address of ``addr`` in the CTA of cluster rank ``rank``.""" + return "mapa.shared::cluster.u32 $0, $1, $2;", "=r,r,r", (addr, Int32(rank)) + + +@ptx_op(Uint32) +def ld_shared_cluster_u32(addr): + return "ld.shared::cluster.u32 $0, [$1];", "=r,r", (addr,) + + +@ptx_op(Float32) +def ld_shared_cluster_f32(addr): + return "ld.shared::cluster.f32 $0, [$1];", "=f,r", (addr,) + + +@ptx_op((Uint32, Uint32, Uint32, Uint32)) +def ld_shared_cluster_v4(addr): + code = "ld.shared::cluster.v4.u32 {$0, $1, $2, $3}, [$4];" + return code, "=r,=r,=r,=r,r", (addr,) + + +@ptx_op() +def st_async_f32(remote, value, remote_bar): + """Asynchronous store to a peer CTA's shared memory, completing on its mbarrier + (both addresses already mapped with ``mapa_shared_cluster``).""" + code = "st.async.shared::cluster.mbarrier::complete_tx::bytes.f32 [$0], $1, [$2];" + return code, "r,f,r", (remote, value, remote_bar) + + +@ptx_op() +def st_async_v2_f32(remote, a, b, remote_bar): + code = ( + "st.async.shared::cluster.mbarrier::complete_tx::bytes.v2.f32 " + "[$0], {$1, $2}, [$3];" + ) + return code, "r,f,f,r", (remote, a, b, remote_bar) + + +@ptx_op() +def st_async_v4_b32(remote, w0, w1, w2, w3, remote_bar): + """16 bytes to a peer CTA's shared memory, completing on its mbarrier.""" + code = ( + "st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 " + "[$0], {$1, $2, $3, $4}, [$5];" + ) + return code, "r,r,r,r,r,r", (remote, w0, w1, w2, w3, remote_bar) + + +# ---------------------------------------------------------------------------- +# Warp-level matrix instructions + + +@ptx_arith((Float32, Float32, Float32, Float32)) +def mma_m16n8k16_bf16(a, b, c): + """``mma.sync`` m16n8k16, BF16 inputs, FP32 accumulate: a 4-tuple of A + words, a 2-tuple of B words, a 4-tuple of FP32 accumulators.""" + code = ( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};" + ) + return code, "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", (*a, *b, *c) + + +@ptx_arith((Float32, Float32, Float32, Float32)) +def mma_m16n8k32_e4m3(a, b, c): + """``mma.sync`` m16n8k32, E4M3 inputs, FP32 accumulate.""" + code = ( + "mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 " + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};" + ) + return code, "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", (*a, *b, *c) + + +@ptx_op((Uint32, Uint32, Uint32, Uint32)) +def ldmatrix_x4(addr, trans=False): + t = ".trans" if trans else "" + code = f"ldmatrix.sync.aligned.m8n8.x4{t}.shared.b16 {{$0, $1, $2, $3}}, [$4];" + return code, "=r,=r,=r,=r,r", (addr,) + + +@ptx_op((Uint32, Uint32)) +def ldmatrix_x2(addr, trans=False): + t = ".trans" if trans else "" + code = f"ldmatrix.sync.aligned.m8n8.x2{t}.shared.b16 {{$0, $1}}, [$2];" + return code, "=r,=r,r", (addr,) + + +# ---------------------------------------------------------------------------- +# Arithmetic with an explicit rounding mode (pure) + + +@ptx_arith(Float32) +def add_rn(a, b): + return "add.rn.f32 $0, $1, $2;", "=f,f,f", (a, b) + + +@ptx_arith(Float32) +def sub_rn(a, b): + return "sub.rn.f32 $0, $1, $2;", "=f,f,f", (a, b) + + +@ptx_arith(Float32) +def mul_rn(a, b): + return "mul.rn.f32 $0, $1, $2;", "=f,f,f", (a, b) + + +@ptx_arith(Float32) +def fma_rn(a, b, c): + return "fma.rn.f32 $0, $1, $2, $3;", "=f,f,f,f", (a, b, c) + + +@ptx_arith(Float32) +def div_rn(a, b): + return "div.rn.f32 $0, $1, $2;", "=f,f,f", (a, b) + + +@ptx_arith(Float32) +def div_full(a, b): + return "div.full.f32 $0, $1, $2;", "=f,f,f", (a, b) + + +@ptx_arith(Float32) +def rsqrt_approx(a, ftz=False): + """``rsqrt.approx.f32`` (``ftz``: subnormal inputs and results flush to zero).""" + op = "rsqrt.approx.ftz.f32" if ftz else "rsqrt.approx.f32" + return f"{op} $0, $1;", "=f,f", (a,) + + +@ptx_arith(Float32) +def max_nan(a, b): + return "max.NaN.f32 $0, $1, $2;", "=f,f,f", (a, b) + + +@ptx_arith(Uint16) +def cvt_rn_bf16_f32(a): + return "cvt.rn.bf16.f32 $0, $1;", "=h,f", (a,) + + +@ptx_arith(Uint32) +def cvt_rn_bf16x2_f32(lo, hi): + """BF16x2 word: ``lo`` in bits 0-15, ``hi`` in bits 16-31.""" + return "cvt.rn.bf16x2.f32 $0, $2, $1;", "=r,f,f", (lo, hi) + + +@ptx_arith(Uint16) +def cvt_rn_satfinite_e4m3x2_f32(lo, hi): + """E4M3x2 (satfinite): ``lo`` in bits 0-7, ``hi`` in bits 8-15.""" + return "cvt.rn.satfinite.e4m3x2.f32 $0, $2, $1;", "=h,f,f", (lo, hi) + + +@ptx_arith(Uint32) +def cvt_rn_f16x2_e4m3x2(pair): + return "cvt.rn.f16x2.e4m3x2 $0, $1;", "=r,h", (pair,) + + +@ptx_arith(Uint16) +def cvt_rn_f16_f32(a): + return "cvt.rn.f16.f32 $0, $1;", "=h,f", (a,) + + +@ptx_arith((Float32, Float32)) +def bf16x2_to_f32x2(word): + """The two BF16 halves of ``word`` as FP32 (exact): (bits 0-15, bits 16-31).""" + return "shl.b32 $0, $2, 16; and.b32 $1, $2, 0xffff0000;", "=f,=f,r", (word,) + + +@ptx_arith(Float32) +def ex2_approx(a): + """``ex2.approx.f32`` (not flushed to zero): the instruction ``exp2f`` lowers to.""" + return "ex2.approx.f32 $0, $1;", "=f,f", (a,) + + +@ptx_arith(Uint32) +def prmt(a, b, selector: int): + """``prmt.b32``: byte permute of the pair (``b``:``a``) by an immediate selector.""" + return f"prmt.b32 $0, $1, $2, {selector:#x};", "=r,r,r", (Uint32(a), Uint32(b)) + + +@ptx_arith((Float32, Float32)) +def e4m3x2_to_f32x2(pair): + """Two E4M3 values (bits 0-7, 8-15) as FP32, exact: (low byte, high byte).""" + code = ( + "{ .reg .b32 h; .reg .b16 l, u; cvt.rn.f16x2.e4m3x2 h, $2; mov.b32 {l, u}, h; " + "cvt.f32.f16 $0, l; cvt.f32.f16 $1, u; }" + ) + return code, "=f,=f,h", (pair,) + + +@ptx_op() +def st_shared_u8(addr, value): + return "st.shared.u8 [$0], $1;", "r,h", (addr, Uint16(value)) + + +@ptx_op((Float32, Float32)) +def ld_shared_v2_f32(addr): + return "ld.shared.v2.f32 {$0, $1}, [$2];", "=f,=f,r", (addr,) + + +@ptx_op(Int32) +def ld_global_s32(addr): + """Coherent 32-bit global load (data written in the same step, read after the PDL + wait).""" + return "ld.global.s32 $0, [$1];", "=r,l", (addr,) + + +@ptx_op(Int64) +def ld_global_s64(addr): + return "ld.global.s64 $0, [$1];", "=l,l", (addr,) + + +@ptx_op(Float32) +def ld_global_f32(addr): + return "ld.global.f32 $0, [$1];", "=f,l", (addr,) + + +@ptx_op(Uint16) +def ld_global_u16(addr): + return "ld.global.u16 $0, [$1];", "=h,l", (addr,) + + +@ptx_op() +def st_global_f32(addr, value): + return "st.global.f32 [$0], $1;", "l,f", (addr, value) + + +@ptx_arith(Float32) +def max_f32(a, b): + """``max.f32``: a NaN operand yields the other operand.""" + return "max.f32 $0, $1, $2;", "=f,f,f", (a, b) + + +@ptx_arith(Float32) +def abs_f32(a): + return "abs.f32 $0, $1;", "=f,f", (a,) + + +@ptx_arith(Uint32) +def f32_bits(a): + return "mov.b32 $0, $1;", "=r,f", (a,) + + +@ptx_arith(Float32) +def bits_f32(a): + return "mov.b32 $0, $1;", "=f,r", (Uint32(a),) + + +@ptx_arith(Float32) +def bf16_to_f32(h): + """Exact BF16 -> FP32 (``cvt.f32.bf16``).""" + return "cvt.f32.bf16 $0, $1;", "=f,h", (Uint16(h),) + + +@ptx_arith(Float32) +def neg_f32(a): + return "neg.f32 $0, $1;", "=f,f", (a,) + + +@ptx_op((Uint32, Uint32, Uint32, Uint32)) +def ld_global_nc_v4(addr): + """Read-only (``.nc``) 16-byte load: constant data only (weights, norms).""" + return "ld.global.nc.v4.u32 {$0, $1, $2, $3}, [$4];", "=r,=r,=r,=r,l", (addr,) + + +@ptx_op(Float32) +def ld_global_nc_f32(addr): + """Read-only (``.nc``) load: constant data only.""" + return "ld.global.nc.f32 $0, [$1];", "=f,l", (addr,) + + +@ptx_op(Uint32) +def ld_global_u32(addr): + return "ld.global.u32 $0, [$1];", "=r,l", (addr,) + + +@ptx_op((Uint32, Uint32)) +def ld_global_v2(addr): + return "ld.global.v2.u32 {$0, $1}, [$2];", "=r,=r,l", (addr,) + + +@ptx_op() +def st_global_v2(addr, a, b): + return "st.global.v2.u32 [$0], {$1, $2};", "l,r,r", (addr, Uint32(a), Uint32(b)) + + +@ptx_op() +def st_global_u8(addr, value): + return "st.global.u8 [$0], $1;", "l,h", (addr, Uint16(value)) + + +@ptx_op() +def st_shared_b16(addr, value): + return "st.shared.b16 [$0], $1;", "r,h", (addr, Uint16(value)) + + +@ptx_arith(Float32) +def min_f32(a, b): + """``min.f32``: a NaN operand yields the other operand.""" + return "min.f32 $0, $1, $2;", "=f,f,f", (a, b) + + +@ptx_arith(Float32) +def rcp_rn(a): + """``rcp.rn.f32``: rounded reciprocal (what ``1.0f / x`` compiles to).""" + return "rcp.rn.f32 $0, $1;", "=f,f", (a,) + + +# ---------------------------------------------------------------------------- +# Shared-memory atomics, CTA reductions and further loads / stores + + +@ptx_op(Uint32) +def atom_shared_add_u32(addr, value): + return "atom.shared.add.u32 $0, [$1], $2;", "=r,r,r", (addr, Uint32(value)) + + +@ptx_op(Uint32) +def atom_shared_and_b32(addr, value): + return "atom.shared.and.b32 $0, [$1], $2;", "=r,r,r", (addr, Uint32(value)) + + +@ptx_op(Uint32) +def atom_shared_or_b32(addr, value): + return "atom.shared.or.b32 $0, [$1], $2;", "=r,r,r", (addr, Uint32(value)) + + +@ptx_op(Int32) +def sync_threads_or(pred): + """``bar.red.or.pred`` over the CTA (barrier 0): 1 if ``pred`` is non-zero in any + thread, else 0.""" + code = ( + "{ .reg .pred P, Q; setp.ne.u32 P, $1, 0; bar.red.or.pred Q, 0, P; " + "selp.u32 $0, 1, 0, Q; }" + ) + return code, "=r,r", (Uint32(pred),) + + +@ptx_op(Uint16) +def ld_shared_u16(addr): + return "ld.shared.u16 $0, [$1];", "=h,r", (addr,) + + +@ptx_op(Uint64) +def ld_shared_u64(addr): + return "ld.shared.u64 $0, [$1];", "=l,r", (addr,) + + +@ptx_op() +def st_shared_u64(addr, value): + return "st.shared.u64 [$0], $1;", "r,l", (addr, Uint64(value)) + + +@ptx_op((Uint64, Uint64)) +def ld_global_v2_u64(addr): + return "ld.global.v2.u64 {$0, $1}, [$2];", "=l,=l,l", (addr,) + + +@ptx_op() +def st_shared_v2_u64(addr, a, b): + return "st.shared.v2.u64 [$0], {$1, $2};", "r,l,l", (addr, Uint64(a), Uint64(b)) + + +@ptx_arith(Uint32) +def lowest_set_bit(x): + """Index of the lowest set bit of a non-zero word (``__ffs(x) - 1``).""" + return "{ .reg .b32 t; brev.b32 t, $1; bfind.shiftamt.u32 $0, t; }", "=r,r", (x,) + + +# Generic-address forms (the address may be in any state space; here always global +# memory reached through a plain pointer or an integer address) + + +@ptx_op(Float32) +def ld_f32(addr): + return "ld.f32 $0, [$1];", "=f,l", (addr,) + + +@ptx_op((Uint64, Uint64)) +def ld_v2_u64(addr): + return "ld.v2.u64 {$0, $1}, [$2];", "=l,=l,l", (addr,) + + +# Predicated and streaming global accesses (coherent) + + +@ptx_op(Uint32) +def ld_global_u32_if(addr, pred): + """Coherent 32-bit load executed only where ``pred`` holds (0 otherwise).""" + code = ( + "{ .reg .pred p; setp.ne.b32 p, $2, 0; mov.b32 $0, 0; " + "@p ld.global.b32 $0, [$1]; }" + ) + return code, "=r,l,r", (addr, Int32(Boolean(pred))) + + +@ptx_op((Uint32, Uint32, Uint32, Uint32)) +def ld_global_v4_if(addr, pred): + """``ld_global_v4`` executed only where ``pred`` holds (zeros otherwise).""" + code = ( + "{ .reg .pred p; setp.ne.b32 p, $5, 0; " + "mov.b32 $0, 0; mov.b32 $1, 0; mov.b32 $2, 0; mov.b32 $3, 0; " + "@p ld.global.v4.u32 {$0,$1,$2,$3}, [$4]; }" + ) + return code, "=r,=r,=r,=r,l,r", (addr, Int32(Boolean(pred))) + + +@ptx_op((Uint32, Uint32, Uint32, Uint32)) +def ld_global_v4_no_allocate(addr): + """Four words of a static weight through the coherent path, no L1 allocation.""" + code = "ld.global.L1::no_allocate.v4.u32 {$0,$1,$2,$3}, [$4];" + return code, "=r,=r,=r,=r,l", (addr,) + + +@ptx_op() +def st_global_u64(addr, value): + return "st.global.u64 [$0], $1;", "l,l", (addr, Uint64(value)) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/ptx_sys.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/ptx_sys.py new file mode 100644 index 00000000..3a4cbc75 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/ptx_sys.py @@ -0,0 +1,401 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Inline-PTX helpers for global-memory ordering, cross-GPU signalling and control. + +Every helper emits exactly one PTX instruction (or one fixed sequence) so that the +memory-model operations of a kernel can be read off its PTX in program order. + +Two emission classes: + +* memory-model operations (scoped loads / stores, atomics, fences, proxy fences) are + side-effecting and carry a ``~{memory}`` clobber, so the compiler neither moves other + memory accesses across them nor removes them; +* plain data loads (``ld.global.cg``, the non-coherent weight load) are side-effecting + without a clobber: they keep their order relative to every other side-effecting + operation (in particular ``griddepcontrol.wait``) but do not fence unrelated accesses. + +Pointers are 64-bit generic addresses (``Int64``) unless stated otherwise. +""" + +from cutlass import Boolean, Float32, Int32, Int64, Uint32, Uint64 +from cutlass._mlir.dialects import llvm +from cutlass.cutlass_dsl import dsl_user_op + +__all__ = [ + "atom_add_acq_rel_gpu_u32", + "atom_add_u32", + "atom_or_b32", + "atom_or_sys_u32", + "atom_or_sys_u32_generic", + "cp_async_cg_16", + "cp_async_wait_all", + "create_policy_evict_first", + "exit_thread", + "fence_acq_rel_gpu", + "fence_acq_rel_sys", + "fence_proxy_async_global", + "fence_sc_sys", + "globaltimer", + "ld_acquire_gpu_u32", + "ld_acquire_sys_u32", + "ld_cg_s32", + "ld_cg_u32", + "ld_u32", + "ld_u64", + "ld_cg_v2_f32", + "ld_cg_v4_f32", + "ld_cg_v4_u32", + "ld_nc_hint_v4_u32", + "ld_nc_v4_f32", + "ld_nc_u32", + "ld_nc_v4_u32_if", + "ld_relaxed_gpu_u32", + "ld_relaxed_sys_u32", + "ld_relaxed_sys_u64", + "ld_relaxed_sys_v2_b64", + "ld_volatile_u32", + "red_add_release_gpu_u32", + "st_global_u32", + "st_relaxed_gpu_u32", + "st_relaxed_sys_u32", + "st_relaxed_sys_u64", + "st_relaxed_sys_v2_b64", + "st_relaxed_sys_v4_u32", + "st_release_gpu_u32", + "trap", + "trap_if", +] + +_MEM = "~{memory}" + + +def _emit(code, constraints, operands, results, side_effects, loc, ip): + """One ``llvm.inline_asm`` op; ``results`` is a tuple of DSL numeric types.""" + if not results: + ir_type = None + elif len(results) == 1: + ir_type = results[0].mlir_type + else: + ir_type = llvm.StructType.get_literal([r.mlir_type for r in results]) + value = llvm.inline_asm( + ir_type, + [v.ir_value(loc=loc, ip=ip) for v in operands], + code, + constraints, + has_side_effects=side_effects, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + if not results: + return None + if len(results) == 1: + return results[0](value) + return tuple( + r(llvm.extractvalue(r.mlir_type, value, [i], loc=loc, ip=ip)) + for i, r in enumerate(results) + ) + + +# --------------------------------------------------------------------- loads +@dsl_user_op +def ld_relaxed_gpu_u32(addr, *, loc=None, ip=None) -> Uint32: + code = "ld.relaxed.gpu.global.u32 $0, [$1];" + return _emit(code, "=r,l," + _MEM, (Int64(addr),), (Uint32,), True, loc, ip) + + +@dsl_user_op +def ld_acquire_gpu_u32(addr, *, loc=None, ip=None) -> Uint32: + code = "ld.acquire.gpu.global.u32 $0, [$1];" + return _emit(code, "=r,l," + _MEM, (Int64(addr),), (Uint32,), True, loc, ip) + + +@dsl_user_op +def ld_acquire_sys_u32(addr, *, loc=None, ip=None) -> Uint32: + code = "ld.acquire.sys.global.u32 $0, [$1];" + return _emit(code, "=r,l," + _MEM, (Int64(addr),), (Uint32,), True, loc, ip) + + +@dsl_user_op +def ld_relaxed_sys_u32(addr, *, loc=None, ip=None) -> Uint32: + code = "ld.relaxed.sys.global.u32 $0, [$1];" + return _emit(code, "=r,l," + _MEM, (Int64(addr),), (Uint32,), True, loc, ip) + + +@dsl_user_op +def ld_relaxed_sys_u64(addr, *, loc=None, ip=None) -> Uint64: + code = "ld.relaxed.sys.global.u64 $0, [$1];" + return _emit(code, "=l,l," + _MEM, (Int64(addr),), (Uint64,), True, loc, ip) + + +@dsl_user_op +def ld_relaxed_sys_v2_b64(addr, *, loc=None, ip=None): + """Two single-copy-atomic 64-bit records (one 16-byte vector access; PTX ISA "vector + data types": each element is its own scalar access).""" + code = "ld.relaxed.sys.global.v2.b64 {$0,$1}, [$2];" + return _emit( + code, "=l,=l,l," + _MEM, (Int64(addr),), (Uint64, Uint64), True, loc, ip + ) + + +@dsl_user_op +def ld_volatile_u32(addr, *, loc=None, ip=None) -> Uint32: + """``ld.volatile``: for reading a word once (never as a cross-GPU poll).""" + code = "ld.volatile.global.u32 $0, [$1];" + return _emit(code, "=r,l," + _MEM, (Int64(addr),), (Uint32,), True, loc, ip) + + +@dsl_user_op +def ld_u32(addr, *, loc=None, ip=None) -> Uint32: + """Plain (weak) global load of one 32-bit word.""" + code = "ld.global.u32 $0, [$1];" + return _emit(code, "=r,l", (Int64(addr),), (Uint32,), True, loc, ip) + + +@dsl_user_op +def ld_u64(addr, *, loc=None, ip=None) -> Uint64: + """Plain (weak) global load of one 64-bit word, ordered as a memory operation.""" + code = "ld.global.u64 $0, [$1];" + return _emit(code, "=l,l," + _MEM, (Int64(addr),), (Uint64,), True, loc, ip) + + +@dsl_user_op +def ld_u64_generic(addr, *, loc=None, ip=None) -> Uint64: + """``ld_u64`` through a generic address (e.g. a host-mapped word).""" + code = "ld.u64 $0, [$1];" + return _emit(code, "=l,l," + _MEM, (Int64(addr),), (Uint64,), True, loc, ip) + + +@dsl_user_op +def ld_cg_u32(addr, *, loc=None, ip=None) -> Uint32: + code = "ld.global.cg.u32 $0, [$1];" + return _emit(code, "=r,l", (Int64(addr),), (Uint32,), True, loc, ip) + + +@dsl_user_op +def ld_cg_s32(addr, *, loc=None, ip=None) -> Int32: + code = "ld.global.cg.s32 $0, [$1];" + return _emit(code, "=r,l", (Int64(addr),), (Int32,), True, loc, ip) + + +@dsl_user_op +def ld_cg_v4_u32(addr, *, loc=None, ip=None): + code = "ld.global.cg.v4.u32 {$0,$1,$2,$3}, [$4];" + return _emit(code, "=r,=r,=r,=r,l", (Int64(addr),), (Uint32,) * 4, True, loc, ip) + + +@dsl_user_op +def ld_cg_v4_f32(addr, *, loc=None, ip=None): + code = "ld.global.cg.v4.f32 {$0,$1,$2,$3}, [$4];" + return _emit(code, "=f,=f,=f,=f,l", (Int64(addr),), (Float32,) * 4, True, loc, ip) + + +@dsl_user_op +def ld_cg_v2_f32(addr, *, loc=None, ip=None): + code = "ld.global.cg.v2.f32 {$0,$1}, [$2];" + return _emit(code, "=f,=f,l", (Int64(addr),), (Float32,) * 2, True, loc, ip) + + +@dsl_user_op +def create_policy_evict_first(*, loc=None, ip=None) -> Uint64: + code = "createpolicy.fractional.L2::evict_first.b64 $0, 1.0;" + return _emit(code, "=l", (), (Uint64,), True, loc, ip) + + +@dsl_user_op +def ld_nc_v4_f32(addr, *, loc=None, ip=None): + """Read-only (non-coherent) load of four static fp32 words.""" + code = "ld.global.nc.v4.f32 {$0,$1,$2,$3}, [$4];" + return _emit(code, "=f,=f,=f,=f,l", (Int64(addr),), (Float32,) * 4, True, loc, ip) + + +@dsl_user_op +def ld_nc_hint_v4_u32(addr, policy, *, loc=None, ip=None): + """Static-weight load (non-coherent, no L1 allocation, L2 cache-policy hint).""" + code = "ld.global.nc.L1::no_allocate.L2::cache_hint.v4.u32 {$0,$1,$2,$3}, [$4], $5;" + return _emit( + code, + "=r,=r,=r,=r,l,l", + (Int64(addr), Uint64(policy)), + (Uint32,) * 4, + True, + loc, + ip, + ) + + +@dsl_user_op +def ld_nc_u32(addr, *, loc=None, ip=None) -> Uint32: + """Read-only (non-coherent) load of one static 32-bit word: only for data no grid + writes while the kernel is resident (weights, tables written before launch).""" + code = "ld.global.nc.u32 $0, [$1];" + return _emit(code, "=r,l", (Int64(addr),), (Uint32,), True, loc, ip) + + +@dsl_user_op +def ld_nc_v4_u32_if(addr, pred, *, loc=None, ip=None): + """Static-weight load (non-coherent, no L1 allocation) of four 32-bit words, + executed only where ``pred`` is non-zero; zeros otherwise.""" + code = ( + "{ .reg .pred p; setp.ne.b32 p, $5, 0; " + "mov.b32 $0, 0; mov.b32 $1, 0; mov.b32 $2, 0; mov.b32 $3, 0; " + "@p ld.global.nc.L1::no_allocate.v4.u32 {$0,$1,$2,$3}, [$4]; }" + ) + ops = (Int64(addr), Int32(Boolean(pred))) + return _emit(code, "=r,=r,=r,=r,l,r", ops, (Uint32,) * 4, True, loc, ip) + + +# -------------------------------------------------------------------- stores +@dsl_user_op +def st_global_u32(addr, value, *, loc=None, ip=None): + code = "st.global.u32 [$0], $1;" + _emit(code, "l,r," + _MEM, (Int64(addr), Uint32(value)), (), True, loc, ip) + + +@dsl_user_op +def st_relaxed_gpu_u32(addr, value, *, loc=None, ip=None): + code = "st.relaxed.gpu.global.u32 [$0], $1;" + _emit(code, "l,r," + _MEM, (Int64(addr), Uint32(value)), (), True, loc, ip) + + +@dsl_user_op +def st_release_gpu_u32(addr, value, *, loc=None, ip=None): + code = "st.release.gpu.global.u32 [$0], $1;" + _emit(code, "l,r," + _MEM, (Int64(addr), Uint32(value)), (), True, loc, ip) + + +@dsl_user_op +def st_relaxed_sys_u32(addr, value, *, loc=None, ip=None): + code = "st.relaxed.sys.global.u32 [$0], $1;" + _emit(code, "l,r," + _MEM, (Int64(addr), Uint32(value)), (), True, loc, ip) + + +@dsl_user_op +def st_relaxed_sys_u64(addr, value, *, loc=None, ip=None): + code = "st.relaxed.sys.global.u64 [$0], $1;" + _emit(code, "l,l," + _MEM, (Int64(addr), Uint64(value)), (), True, loc, ip) + + +@dsl_user_op +def st_relaxed_sys_v2_b64(addr, r0, r1, *, loc=None, ip=None): + """Two single-copy-atomic 64-bit records in one 16-byte vector store.""" + code = "st.relaxed.sys.global.v2.b64 [$0], {$1,$2};" + ops = (Int64(addr), Uint64(r0), Uint64(r1)) + _emit(code, "l,l,l," + _MEM, ops, (), True, loc, ip) + + +@dsl_user_op +def st_relaxed_sys_v4_u32(addr, w0, w1, w2, w3, *, loc=None, ip=None): + code = "st.relaxed.sys.global.v4.u32 [$0], {$1,$2,$3,$4};" + ops = (Int64(addr), Uint32(w0), Uint32(w1), Uint32(w2), Uint32(w3)) + _emit(code, "l,r,r,r,r," + _MEM, ops, (), True, loc, ip) + + +# ------------------------------------------------------------------ atomics +@dsl_user_op +def atom_add_acq_rel_gpu_u32(addr, value, *, loc=None, ip=None) -> Uint32: + code = "atom.acq_rel.gpu.global.add.u32 $0, [$1], $2;" + ops = (Int64(addr), Uint32(value)) + return _emit(code, "=r,l,r," + _MEM, ops, (Uint32,), True, loc, ip) + + +@dsl_user_op +def atom_add_u32(addr, value, *, loc=None, ip=None) -> Uint32: + """Relaxed gpu-scope ``atom.add`` on a global word; returns the old value.""" + code = "atom.global.add.u32 $0, [$1], $2;" + ops = (Int64(addr), Uint32(value)) + return _emit(code, "=r,l,r," + _MEM, ops, (Uint32,), True, loc, ip) + + +@dsl_user_op +def red_add_release_gpu_u32(addr, value, *, loc=None, ip=None): + code = "red.release.gpu.global.add.u32 [$0], $1;" + _emit(code, "l,r," + _MEM, (Int64(addr), Uint32(value)), (), True, loc, ip) + + +@dsl_user_op +def atom_or_b32(addr, value, *, loc=None, ip=None) -> Uint32: + """Relaxed gpu-scope ``atom.or`` (the sticky error word).""" + code = "atom.global.or.b32 $0, [$1], $2;" + ops = (Int64(addr), Uint32(value)) + return _emit(code, "=r,l,r," + _MEM, ops, (Uint32,), True, loc, ip) + + +@dsl_user_op +def atom_or_sys_u32(addr, value, *, loc=None, ip=None) -> Uint32: + """System-scope relaxed ``atom.or`` on a global word; returns the old value.""" + code = "atom.global.sys.or.b32 $0, [$1], $2;" + ops = (Int64(addr), Uint32(value)) + return _emit(code, "=r,l,r," + _MEM, ops, (Uint32,), True, loc, ip) + + +@dsl_user_op +def atom_or_sys_u32_generic(addr, value, *, loc=None, ip=None) -> Uint32: + """``atom_or_sys_u32`` through a generic address (a peer's mailbox word).""" + code = "atom.sys.or.b32 $0, [$1], $2;" + ops = (Int64(addr), Uint32(value)) + return _emit(code, "=r,l,r," + _MEM, ops, (Uint32,), True, loc, ip) + + +# ------------------------------------------------------------------- fences +@dsl_user_op +def fence_acq_rel_gpu(*, loc=None, ip=None): + _emit("fence.acq_rel.gpu;", _MEM, (), (), True, loc, ip) + + +@dsl_user_op +def fence_acq_rel_sys(*, loc=None, ip=None): + _emit("fence.acq_rel.sys;", _MEM, (), (), True, loc, ip) + + +@dsl_user_op +def fence_sc_sys(*, loc=None, ip=None): + _emit("fence.sc.sys;", _MEM, (), (), True, loc, ip) + + +@dsl_user_op +def fence_proxy_async_global(*, loc=None, ip=None): + """Generic <-> async proxy hand-off for global memory (PTX ISA 9.7.13.4).""" + _emit("fence.proxy.async.global;", _MEM, (), (), True, loc, ip) + + +# -------------------------------------------------------- async copies (sm80+) +@dsl_user_op +def cp_async_cg_16(smem_addr, gmem_addr, *, loc=None, ip=None): + """16-byte ``cp.async.cg`` (shared address is 32-bit).""" + code = "cp.async.cg.shared.global [$0], [$1], 16;" + ops = (Uint32(smem_addr), Int64(gmem_addr)) + _emit(code, "r,l," + _MEM, ops, (), True, loc, ip) + + +@dsl_user_op +def cp_async_wait_all(*, loc=None, ip=None): + _emit("cp.async.wait_all;", _MEM, (), (), True, loc, ip) + + +# ------------------------------------------------------------------ control +@dsl_user_op +def globaltimer(*, loc=None, ip=None) -> Uint64: + return _emit("mov.u64 $0, %globaltimer;", "=l", (), (Uint64,), True, loc, ip) + + +@dsl_user_op +def trap(*, loc=None, ip=None): + _emit("trap;", _MEM, (), (), True, loc, ip) + + +@dsl_user_op +def trap_if(cond, *, loc=None, ip=None): + """``trap`` when ``cond`` (an integer) is non-zero, without a branch.""" + code = "{ .reg .pred p; setp.ne.u32 p, $0, 0; @p trap; }" + _emit(code, "r," + _MEM, (Uint32(cond),), (), True, loc, ip) + + +@dsl_user_op +def exit_thread(*, loc=None, ip=None): + """Terminate the executing thread (PTX ``exit``). Used for whole warps or CTAs that + leave a kernel early: an exited thread no longer counts for later + ``barrier.cluster`` phases (PTX ISA 9.7.15.3), unlike a predicated-off one.""" + _emit("exit;", _MEM, (), (), True, loc, ip) diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/ptx_tc.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/ptx_tc.py new file mode 100644 index 00000000..ab28ee5f --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/primitives/ptx_tc.py @@ -0,0 +1,417 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# ---- glm_dsa_cute shared body below ---- +"""Tensor-core, tensor-map, NVFP4 and routing-scalar helpers. + +* tcgen05 issue: the MMAs, scale-factor copies and commits of one ring chunk are one asm + block (a "burst"), predicated on a leader lane that the caller elects once and keeps + in a register. Every tcgen05 operation of a role is then issued by the same lane, + which ``tcgen05.commit`` needs (it tracks the executing thread's operations). +* TMEM loads: the loads and their ``tcgen05.wait::ld`` are one asm block, so no use of a + loaded register can be scheduled above the wait. +* Arithmetic uses fixed PTX forms (routing scores and weights, the SiLU of the routed + and shared experts, NVFP4 block quantization), so the results do not depend on + compiler contraction or approximation choices.""" + +from __future__ import annotations + +from cutlass import Float32, Int32, Uint32, Uint64 +from cutlass.experimental import primitives as nvvm + +from .ptx_core import ptx_arith, ptx_op + +# ---------------------------------------------------- descriptors (host-side constants) + + +def idesc_nvf4(m: int, n: int) -> int: + """Instruction descriptor, kind::mxf4nvf4: E2M1 x E2M1, K-major, UE4M3 scales.""" + return (1 << 7) | (1 << 10) | ((n >> 3) << 17) | ((m >> 4) << 24) + + +def idesc_bf16(m: int, n: int) -> int: + """Instruction descriptor, kind::f16: BF16 x BF16 -> FP32, K-major.""" + return (1 << 4) | (1 << 7) | (1 << 10) | ((n >> 3) << 17) | ((m >> 4) << 24) + + +def smem_desc(addr, lbo: int, sbo: int, layout: int) -> Uint64: + """Shared-memory matrix descriptor (version 1) of the 32-bit shared address + ``addr``; ``layout``: 0 no swizzle, 2 128-byte swizzle.""" + static = ( + ((lbo & 0x3FFFF) >> 4) << 16 + | ((sbo & 0x3FFFF) >> 4) << 32 + | 1 << 46 + | layout << 61 + ) + return Uint64((Uint32(addr) & Uint32(0x3FFFF)) >> Uint32(4)) | Uint64(static) + + +# ---------------------------------------------------------------- leader election + + +def elect_leader() -> Int32: + """1 in the one lane ``elect.sync`` picks for the full warp, else 0 (call + converged).""" + return Int32(nvvm.elect_sync()) + + +# ---------------------------------------------------------------- tcgen05 bursts + +_MMA_NVF4 = "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16" +_MMA_BF16 = "tcgen05.mma.cta_group::1.kind::f16" +_CP_SF = "tcgen05.cp.cta_group::1.32x128b.warpx4" +_COMMIT = "tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64" + + +def _burst(mmas, cps, block_scaled, idesc, second_group, commit): + """Asm text of one burst. Operands: $0 lead, $1 d, $2 a, $3 b, $4 sfa, $5 sfb, $6 + first (accumulate flag of MMAs marked "first"), $7 second (gate of ops marked group + 1), $8 commit barrier, $9 cp TMEM base, $10 cp source descriptor. ``mmas``: (d, a, + b, sfa, sfb, accumulate, group) with byte offsets for a / b (descriptor units added + as bytes >> 4) and column offsets for d / sfa / sfb; accumulate is "first", 1 or 0. + ``cps``: (tmem column offset, source byte offset, group).""" + nm, nc = max(len(mmas), 1), max(len(cps), 1) + lines = [ + "{", + ".reg .pred e, e2, pf, pt, p0;", + f".reg .b32 d<{nm}>, sa<{nm}>, sb<{nm}>;", + f".reg .b64 a<{nm}>, b<{nm}>, c<{nc}>;", + f".reg .b32 t<{nc}>, id;", + f"mov.b32 id, {idesc};", + "setp.ne.b32 e, $0, 0;", + "setp.ne.b32 pf, $6, 0;", + "setp.eq.b32 pt, 0, 0;", + "setp.ne.b32 p0, 0, 0;", + ] + if second_group: + lines.append("setp.ne.b32 e2, $7, 0; and.pred e2, e2, e;") + for i, (_t, src, _g) in enumerate(cps): + lines.append(f"add.u32 t{i}, $9, {_t}; add.s64 c{i}, $10, {src >> 4};") + for i, (_t, _src, g) in enumerate(cps): + lines.append(f"@{'e2' if g else 'e'} {_CP_SF} [t{i}], c{i};") + for i, (d, a, b, sfa, sfb, acc, _g) in enumerate(mmas): + lines.append( + f"add.u32 d{i}, $1, {d}; add.s64 a{i}, $2, {a >> 4}; " + f"add.s64 b{i}, $3, {b >> 4};" + ) + if block_scaled: + lines.append(f"add.u32 sa{i}, $4, {sfa}; add.u32 sb{i}, $5, {sfb};") + for i, (d, a, b, sfa, sfb, acc, g) in enumerate(mmas): + p = {"first": "pf", 1: "pt", 0: "p0"}[acc] + gate = "e2" if g else "e" + if block_scaled: + lines.append( + f"@{gate} {_MMA_NVF4} [d{i}], a{i}, b{i}, id, [sa{i}], [sb{i}], {p};" + ) + else: + lines.append(f"@{gate} {_MMA_BF16} [d{i}], a{i}, b{i}, id, {p};") + if commit: + lines.append(f"@e {_COMMIT} [$8];") + lines.append("}") + return "\n".join(lines) + + +@ptx_op() +def tc_burst( + lead, + d, + a, + b, + sfa, + sfb, + first, + second, + bar, + cp_base, + cp_desc, + *, + mmas=(), + cps=(), + block_scaled=True, + idesc=0, + commit=True, +): + """One chunk's tcgen05 issue (see ``_burst``): scale-factor copies, then MMAs, then + an optional commit to ``bar``, all by the leader lane.""" + second_group = any(op[-1] for op in mmas) or any(op[-1] for op in cps) + code = _burst(tuple(mmas), tuple(cps), block_scaled, idesc, second_group, commit) + ops = ( + Int32(lead), + Uint32(d), + Uint64(a), + Uint64(b), + Uint32(sfa), + Uint32(sfb), + Int32(first), + Int32(second), + Uint32(bar), + Uint32(cp_base), + Uint64(cp_desc), + ) + return code, "r,r,l,l,r,r,r,r,r,r,l", ops + + +@ptx_op() +def tc_commit(lead, bar): + """Arrive once on ``bar`` when the leader's prior tcgen05 operations complete.""" + code = "{ .reg .pred e; setp.ne.b32 e, $0, 0; @e " + _COMMIT + " [$1]; }" + return code, "r,r", (Int32(lead), Uint32(bar)) + + +@ptx_op((Float32, Float32, Float32, Float32)) +def tmem_load4(taddr): + """Four FP32 TMEM columns of this lane (32x32b.x4), waited.""" + code = ( + "tcgen05.ld.sync.aligned.32x32b.x4.b32 {$0, $1, $2, $3}, [$4];\n" + "tcgen05.wait::ld.sync.aligned;" + ) + return code, "=f,=f,=f,=f,r", (Uint32(taddr),) + + +@ptx_op(tuple([Float32] * 8)) +def _tmem_load4_pair(taddr_a, taddr_b, second): + code = ( + "{ .reg .pred p; setp.ne.b32 p, $10, 0;\n" + "mov.f32 $4, 0f00000000; mov.f32 $5, 0f00000000; " + "mov.f32 $6, 0f00000000; mov.f32 $7, 0f00000000;\n" + "tcgen05.ld.sync.aligned.32x32b.x4.b32 {$0, $1, $2, $3}, [$8];\n" + "@p tcgen05.ld.sync.aligned.32x32b.x4.b32 {$4, $5, $6, $7}, [$9];\n" + "tcgen05.wait::ld.sync.aligned; }" + ) + return ( + code, + ",".join(["=f"] * 8 + ["r", "r", "r"]), + (Uint32(taddr_a), Uint32(taddr_b), Int32(second)), + ) + + +def tmem_load4_pair(taddr_a, taddr_b, second): + """Four FP32 columns at ``taddr_a`` and, if ``second`` (warp-uniform), four at + ``taddr_b`` (else zeros), with one wait.""" + v = _tmem_load4_pair(taddr_a, taddr_b, second) + return v[:4], v[4:] + + +@ptx_op(tuple([Float32] * 16)) +def _tmem_load4_masked(taddr, mask): + lines = ["{", ".reg .pred p<4>;", ".reg .b32 m<4>, t<4>;"] + for q in range(4): + lines.append( + f"and.b32 m{q}, $17, {1 << q}; setp.ne.b32 p{q}, m{q}, 0; " + f"add.u32 t{q}, $16, {4 * q};" + ) + for q in range(4): + lines.append(" ".join(f"mov.f32 ${4 * q + j}, 0f00000000;" for j in range(4))) + for q in range(4): + regs = ", ".join(f"${4 * q + j}" for j in range(4)) + lines.append(f"@p{q} tcgen05.ld.sync.aligned.32x32b.x4.b32 {{{regs}}}, [t{q}];") + lines.append("tcgen05.wait::ld.sync.aligned;") + lines.append("}") + return ( + "\n".join(lines), + ",".join(["=f"] * 16 + ["r", "r"]), + (Uint32(taddr), Uint32(mask)), + ) + + +def tmem_load4_masked(taddr, mask): + """For each bit q of the warp-uniform ``mask`` (q < 4): four FP32 columns at + ``taddr + 4 q`` (zeros for a clear bit), with one wait. Returns four 4-tuples.""" + v = _tmem_load4_masked(taddr, mask) + return tuple(tuple(v[4 * q : 4 * q + 4]) for q in range(4)) + + +@ptx_op(tuple([Float32] * 48)) +def tmem_load4x12(taddr, stride: int = 8): + """Twelve 32x32b.x4 TMEM loads at column offsets 0, stride, ..., 11 stride, then one + wait.""" + lines = ["{", ".reg .b32 t<12>;"] + for t in range(12): + lines.append(f"add.u32 t{t}, $48, {stride * t};") + for t in range(12): + regs = ", ".join(f"${4 * t + j}" for j in range(4)) + lines.append(f"tcgen05.ld.sync.aligned.32x32b.x4.b32 {{{regs}}}, [t{t}];") + lines.append("tcgen05.wait::ld.sync.aligned;") + lines.append("}") + return "\n".join(lines), ",".join(["=f"] * 48 + ["r"]), (Uint32(taddr),) + + +@ptx_op() +def tmem_alloc(dst, columns: int): + """Allocate ``columns`` TMEM columns (warp-wide), writing the base address to the + shared word ``dst``, then give up the allocation permit.""" + code = ( + f"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [$0], {columns};\n" + "tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;" + ) + return code, "r", (Int32(dst),) + + +@ptx_op() +def tmem_free(taddr, columns: int): + """Free a TMEM allocation (warp-wide).""" + return ( + f"tcgen05.dealloc.cta_group::1.sync.aligned.b32 $0, {columns};", + "r", + (Uint32(taddr),), + ) + + +def tc_fence_before(): + nvvm.tcgen05_fence("before_thread_sync") + + +def tc_fence_after(): + nvvm.tcgen05_fence("after_thread_sync") + + +# ---------------------------------------------------------------- tensor-map copy + + +@ptx_op() +def tma_load_3d(dst, tmap, x, y, z, bar, policy): + """3D tile copy global -> shared (``tmap``: generic address of the tensor map).""" + code = ( + "cp.async.bulk.tensor.3d.shared::cluster.global.tile" + ".mbarrier::complete_tx::bytes.L2::cache_hint " + "[$0], [$1, {$2, $3, $4}], [$5], $6;" + ) + return ( + code, + "r,l,r,r,r,r,l", + (dst, tmap, Int32(x), Int32(y), Int32(z), bar, Uint64(policy)), + ) + + +# ---------------------------------------------------------------- NVFP4 conversions + + +@ptx_arith(Uint32) +def e2m1x8(v0, v1, v2, v3, v4, v5, v6, v7): + """Eight FP32 -> eight E2M1 nibbles (RN, satfinite), element i in bits 4i..4i+3.""" + code = ( + "{ .reg .b8 b0, b1, b2, b3; " + "cvt.rn.satfinite.e2m1x2.f32 b0, $2, $1; " + "cvt.rn.satfinite.e2m1x2.f32 b1, $4, $3; " + "cvt.rn.satfinite.e2m1x2.f32 b2, $6, $5; " + "cvt.rn.satfinite.e2m1x2.f32 b3, $8, $7; " + "mov.b32 $0, {b0, b1, b2, b3}; }" + ) + return ( + code, + "=r,f,f,f,f,f,f,f,f", + tuple(Float32(v) for v in (v0, v1, v2, v3, v4, v5, v6, v7)), + ) + + +@ptx_arith(Uint32) +def e2m1x2(lo, hi): + """Two FP32 -> one E2M1 pair (RN, satfinite): ``lo`` in bits 0-3, ``hi`` in bits + 4-7.""" + code = ( + "{ .reg .b8 b; .reg .b16 h; .reg .b32 w; " + "cvt.rn.satfinite.e2m1x2.f32 b, $2, $1; mov.b16 h, {b, b}; " + "cvt.u32.u16 w, h; and.b32 $0, w, 255; }" + ) + return code, "=r,f,f", (Float32(lo), Float32(hi)) + + +@ptx_arith((Float32, Uint32)) +def nvfp4_block_scale(amax, gs): + """NVFP4 block scale of a 16-element block with absolute maximum ``amax`` and global + scale ``gs``: the E4M3 scale byte ``sf = e4m3(gs * (amax / 6))`` and the factor that + maps the block onto E2M1, ``1 / (float(sf) / gs)`` (0 for a zero scale).""" + code = ( + "{ .reg .f32 r6, a, s, f, rg, q; .reg .b16 t, u; .reg .b32 h; .reg .pred z;\n" + "mov.f32 r6, 0f40C00000; rcp.approx.ftz.f32 r6, r6;\n" + "mul.f32 a, $2, r6; mul.f32 s, $3, a;\n" + "cvt.rn.satfinite.e4m3x2.f32 t, s, s; and.b16 u, t, 255;\n" + "cvt.rn.f16x2.e4m3x2 h, u; cvt.u16.u32 t, h; cvt.f32.f16 f, t;\n" + "setp.eq.f32 z, f, 0f00000000;\n" + "rcp.approx.ftz.f32 rg, $3; mul.f32 q, f, rg; rcp.approx.ftz.f32 q, q;\n" + "selp.f32 $0, 0f00000000, q, z; cvt.u32.u16 $1, u; }" + ) + return code, "=f,=r,f,f", (Float32(amax), Float32(gs)) + + +# ---------------------------------------------------------------- activations + + +@ptx_arith(Float32) +def silu_fast(g): + """SiLU, g / (1 + exp(-g)), with the fast exponential (ex2.approx of -g log2 e) + and an IEEE divide: the routed experts' SiLU.""" + code = ( + "{ .reg .f32 t, e, d; mul.f32 t, $1, 0fBFB8AA3B; ex2.approx.f32 e, t; " + "add.f32 d, e, 0f3F800000; div.rn.f32 $0, $1, d; }" + ) + return code, "=f,f", (Float32(g),) + + +@ptx_arith(Float32) +def silu_accurate(g): + """SiLU, g / (1 + exp(-g)), with the accurate single-precision exponential (range + reduction by 2^k, two-term log2 e) and an IEEE divide: the shared expert's SiLU.""" + code = ( + "{ .reg .f32 t, s, k, kb, r, e, d; .reg .b32 sh;\n" + "fma.rn.f32 t, $1, 0fBBBB989D, 0f3F000000; cvt.sat.f32.f32 s, t;\n" + "fma.rm.f32 k, s, 0f437C0000, 0f4B400001; " + "add.f32 kb, k, 0fCB40007F; neg.f32 kb, kb;\n" + "fma.rn.f32 r, $1, 0fBFB8AA3B, kb; fma.rn.f32 r, $1, 0fB2A57060, r;\n" + "mov.b32 sh, k; shl.b32 sh, sh, 23; ex2.approx.ftz.f32 e, r;\n" + "mov.b32 d, sh; fma.rn.f32 d, e, d, 0f3F800000; div.rn.f32 $0, $1, d; }" + ) + return code, "=f,f", (Float32(g),) + + +@ptx_arith(Float32) +def abs_f32(a): + return "abs.f32 $0, $1;", "=f,f", (Float32(a),) + + +@ptx_arith(Float32) +def max_f32(a, b): + """``max.f32`` (a NaN operand yields the other operand).""" + return "max.f32 $0, $1, $2;", "=f,f,f", (Float32(a), Float32(b)) + + +# --------------------------------------------------------------- routing scalars (pure) + + +@ptx_arith(Float32) +def routing_sigmoid(x): + """0.5 * tanh(0.5 * x) + 0.5 with the approximate tanh, flush-to-zero.""" + code = ( + "{ .reg .f32 h,t; mul.ftz.f32 h,$1,0f3F000000; tanh.approx.f32 t,h; " + "fma.rn.ftz.f32 $0,t,0f3F000000,0f3F000000; }" + ) + return code, "=f,f", (Float32(x),) + + +@ptx_arith(Float32) +def routing_add(a, b): + return "add.ftz.f32 $0,$1,$2;", "=f,f,f", (Float32(a), Float32(b)) + + +@ptx_arith(Uint32) +def routing_key(score, bias): + """Order-preserving u32 of ``score + bias`` (flush-to-zero add): NaN ranks as -inf, + -0 as +0, so a larger key is a better candidate.""" + code = ( + "{ .reg .f32 s,t; .reg .b32 u,nu,pu; .reg .pred pn,pz,ps; " + "add.ftz.f32 s,$1,$2; setp.nan.f32 pn,s,s; selp.f32 t,0fFF800000,s,pn; " + "setp.eq.f32 pz,t,0f00000000; selp.b32 u,0,t,pz; not.b32 nu,u; " + "or.b32 pu,u,0x80000000; setp.lt.s32 ps,u,0; selp.b32 $0,nu,pu,ps; }" + ) + return code, "=r,f,f", (Float32(score), Float32(bias)) + + +@ptx_arith(Float32) +def routing_weight(scale, score, total): + """BF16-rounded ``scale * score / (total + 1e-20)`` (approximate divide, + flush-to-zero), returned as FP32.""" + code = ( + "{ .reg .f32 n,d,q; .reg .b16 h; mul.ftz.f32 n,$1,$2; " + "add.ftz.f32 d,$3,0f1E3CE508; " + "div.approx.ftz.f32 q,n,d; cvt.rn.bf16.f32 h,q; cvt.f32.bf16 $0,h; }" + ) + return code, "=f,f,f,f", (Float32(scale), Float32(score), Float32(total)) diff --git a/sgl_nvidia_mega/glm_dsa/moe.py b/sgl_nvidia_mega/glm_dsa/moe.py new file mode 100644 index 00000000..6b54fd05 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/moe.py @@ -0,0 +1,308 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""CuTe DSL small-M NVFP4 MoE in the verify step of the opt-in fused GLM-5 DSA path. + +At 4 / 8 / 12 / 16 verify rows (1..4 requests of 4 rows) each MoE layer runs the stock +router GEMM (``mlp.gate``), ``moe_route_top8`` (the top-8 selection of the stock TRT-LLM +NVFP4 MoE routing: sigmoid scores, correction bias, renormalized weights x the routed +scaling factor), one engine launch for the routed experts and the BF16 shared expert, and +``moe_finalize`` (the layer's un-reduced output, which the next hop step all-reduces as +it does the stock MoE output). When the layer's ``o_proj`` hop has already routed the +rows (``step.routing``), the router GEMM and ``moe_route_top8`` are skipped, the engine +runs without the finalize, and the step returns ``MoeRecords`` (the per-expert outputs) +for the MLP-output hop, which finalizes them inside its all-reduce. + +The engine reads the server's processed routed FC1 weights in place. FC2 is read from +compact per-quarter records and the shared expert from a packed copy; both are derived +at load and registered (226.5 MB + 9.4 MB per MoE layer and rank). ``build`` keeps the +MoE stock unless the engine API accepts the configuration, every MoE layer has the +expected tensors and the server's own NVFP4 weight processing passes the engine's +layout self-test. +""" + +from __future__ import annotations + +import logging +import types +from dataclasses import dataclass +from typing import Dict, Optional, Tuple + +import torch + +from sgl_nvidia_mega.glm_dsa.eligibility import ( + HIDDEN, + SUPPORTED_ROWS, + VERIFY_TOKENS_PER_REQUEST, +) +from sgl_nvidia_mega.glm_dsa.providers import ( + MlpSteps, + compile_cached, + cu_stream, +) + +logger = logging.getLogger(__name__) + +EXPERTS = 256 +TOP_K = 8 +INTER = 256 # routed and shared intermediate size per rank (TP 8) + + +@dataclass +class MoeRecords: + """The engine's per-expert outputs of one MoE layer, for a hop that finalizes them + inside its all-reduce (row stride ``rows``): ``pq`` [4, rows, 8, 6144] fp32 (the four + quarters' FC2 outputs per top-k slot), ``shpartials`` [4, rows, 6144] fp32 (shared expert), + ``ids_fin`` / ``w_fin`` / ``g2s`` [rows, 8] (slot expert id or -1, routing weight, FC2 + scale).""" + + pq: torch.Tensor + shpartials: torch.Tensor + ids_fin: torch.Tensor + w_fin: torch.Tensor + g2s: torch.Tensor + + +def _process_like_server(gate, up, gate_sf, up_sf, w2, w2_sf): + """The server's NVFP4 TRT-LLM weight processing on a synthetic layer: the loader's + w13 order for this backend (up rows first) and the same processing call.""" + from sglang.srt.layers.moe.moe_runner import flashinfer_trtllm as trtllm + + e = gate.shape[0] + f8 = torch.float8_e4m3fn + p = lambda t: torch.nn.Parameter(t, requires_grad=False) # noqa: E731 + layer = torch.nn.Module() + layer.w13_weight = p(torch.cat([up, gate], dim=1).contiguous()) + layer.w2_weight = p(w2.contiguous()) + layer.w13_weight_scale = p(torch.cat([up_sf, gate_sf], dim=1).view(f8).contiguous()) + layer.w2_weight_scale = p(w2_sf.view(f8).contiguous()) + ones = torch.ones(e, dtype=torch.float32, device=gate.device) + layer.w2_input_scale_quant = p(ones.clone()) + layer.g1_alphas = p(ones.clone()) + layer.g1_alphas_up = p(ones.clone()) + layer.moe_runner_config = types.SimpleNamespace(is_gated=True, activation="silu") + trtllm.align_fp4_moe_weights_for_flashinfer_trtllm(layer) + return ( + layer.w13_weight.data, + layer.w13_weight_scale.data, + layer.w2_weight.data, + layer.w2_weight_scale.data, + ) + + +def _u8(t: torch.Tensor) -> torch.Tensor: + return t if t.dtype == torch.uint8 else t.view(torch.uint8) + + +def _bad(t, dtype, shape=None, numel=None, contiguous=True) -> bool: + return ( + t is None + or t.dtype != dtype + or (shape is not None and tuple(t.shape) != shape) + or (numel is not None and t.numel() != numel) + or (contiguous and not t.is_contiguous()) + ) + + +def _layer_reason(mlp) -> Optional[str]: + """Why this MoE layer cannot run on the engine (None if it can).""" + ex = mlp.experts + for name, shape in ( + ("w13_weight", (EXPERTS, 2 * INTER, HIDDEN // 2)), + ("w2_weight", (EXPERTS, HIDDEN, INTER // 2)), + ): + if _bad(getattr(ex, name, None), torch.uint8, shape): + return f"{name} is not contiguous uint8 {shape}" + for name, n in ( + ("w13_weight_scale", EXPERTS * 2 * INTER * HIDDEN // 16), + ("w2_weight_scale", EXPERTS * HIDDEN * INTER // 16), + ): + if _bad(getattr(ex, name, None), torch.float8_e4m3fn, numel=n): + return f"{name} is not contiguous e4m3 with {n} elements" + for name in ("g1_alphas", "g1_alphas_up", "g2_alphas"): + if _bad(getattr(ex, name, None), torch.float32, (EXPERTS,), contiguous=False): + return f"{name} is not fp32 [{EXPERTS}]" + if not torch.equal(ex.g1_alphas, ex.g1_alphas_up): + return "gate and up GEMM1 scales differ (the engine applies one per expert)" + for name in ("w13_input_scale_quant", "w2_input_scale_quant"): + t = getattr(ex, name, None) + if t is None or t.numel() != 1: + return f"{name} is not a single scale" + cfg = ex.moe_runner_config + if ( + getattr(cfg, "activation", "silu") != "silu" + or getattr(cfg, "gemm1_clamp_limit", None) + or getattr(cfg, "swiglu_limit", None) + or getattr(cfg, "gemm1_alpha", None) is not None + ): + return "only an unclamped SiLU-gated MLP is supported" + gu, dn = mlp.shared_experts.gate_up_proj.weight, mlp.shared_experts.down_proj.weight + if _bad(gu, torch.bfloat16, (2 * INTER, HIDDEN), contiguous=False): + return "shared expert gate_up_proj is not bf16 [512, 6144]" + if _bad(dn, torch.bfloat16, (HIDDEN, INTER), contiguous=False): + return "shared expert down_proj is not bf16 [6144, 256]" + bias = mlp.gate.e_score_correction_bias + if _bad(bias, torch.float32, (EXPERTS,), contiguous=False): + return "router correction bias is not fp32 [256]" + return None + + +class MoeSteps(MlpSteps): + """The MoE layers of one verify row count on the engine; dense layers stay stock.""" + + def __init__(self, glue: MoeGlue, rows: int) -> None: + self.glue, self.rows = glue, rows + self.split = (VERIFY_TOKENS_PER_REQUEST,) * (rows // VERIFY_TOKENS_PER_REQUEST) + + def forward(self, layer, x, buffer: Optional[torch.Tensor], step): + glue = self.glue + lt = glue.layers.get(layer.layer_id) + if lt is None: + return super().forward(layer, x, buffer, step) + m = x.shape[0] + if m != self.rows or x.dtype != torch.bfloat16 or not x.is_contiguous(): + raise RuntimeError( + f"fused verify MoE: unexpected input {tuple(x.shape)} {x.dtype} at {self.rows} rows" + ) + routing, step.routing = step.routing, None + if routing is not None: + # routed by this layer's o_proj hop; its MLP-output hop finalizes the records + ids, weights = routing + glue.engine(lt, x, ids, weights, self.split, what=1) + return glue.records(m) + ids, weights = glue.ids[:m], glue.weights[:m] + gate = layer.mlp.gate + scale = layer.mlp.routed_scaling_factor + glue.route(gate(x), gate.e_score_correction_bias, scale, ids, weights) + if ( + isinstance(buffer, torch.Tensor) + and buffer.dtype == torch.bfloat16 + and buffer.dim() == 2 + and buffer.shape[0] >= m + and buffer.shape[1] == HIDDEN + and buffer.is_contiguous() + ): + out = buffer[:m] + else: + out = torch.empty(m, HIDDEN, dtype=torch.bfloat16, device=x.device) + glue.engine(lt, x, ids, weights, self.split, out=out, what=3) + return out + + +class MoeGlue: + """Derived MoE weights, compiled engines and per-row-count steps.""" + + def __init__(self, api, layers: Dict[int, object], specs) -> None: + self.api, self.specs, self.layers = api, specs, layers + self.device = dev = next(iter(layers.values())).w13.device + self.ids = torch.empty(16, TOP_K, dtype=torch.int32, device=dev) + self.weights = torch.empty(16, TOP_K, dtype=torch.float32, device=dev) + self.fn: Dict[str, object] = {} + self.engine = None + + def compile(self, rows: Tuple[int, ...]) -> None: + import cutlass.cute as cute + + per = VERIFY_TOKENS_PER_REQUEST + modes = { + self.api.mode_for((per,) * (m // per)) for m in rows if m in SUPPORTED_ROWS + } + for name in sorted({"route", "finalize"} | modes): + kernel, args = self.specs[name] + self.fn[name] = compile_cached( + kernel, + {"unit": name, "shared": True}, + lambda k=kernel, a=args: cute.compile(k, *a), + ) + nclusters = self.api.default_clusters(self.device) + self.engine = self.api.SmallMMoe(self.fn, nclusters, self.device, shared=True) + + def records(self, m: int) -> MoeRecords: + sc = self.engine.scratch + return MoeRecords( + pq=sc.partials[: 4 * m * TOP_K * HIDDEN].view(4, m, TOP_K, HIDDEN), + shpartials=sc.shared_partials[: 4 * m * HIDDEN].view(4, m, HIDDEN), + ids_fin=sc.slot_ids[: m * TOP_K].view(m, TOP_K), + w_fin=sc.slot_weights[: m * TOP_K].view(m, TOP_K), + g2s=sc.slot_scales[: m * TOP_K].view(m, TOP_K), + ) + + def route(self, logits, bias, scale, ids, weights) -> None: + from cutlass import Float32, Int32, Int64 + + m = logits.shape[0] + if ( + logits.dtype != torch.float32 + or tuple(logits.shape) != (m, EXPERTS) + or not logits.is_contiguous() + ): + raise RuntimeError( + f"fused verify MoE: router logits {tuple(logits.shape)} {logits.dtype}" + ) + self.fn["route"]( + Int64(logits.data_ptr()), + Int64(bias.data_ptr()), + Float32(float(scale)), + Int64(ids.data_ptr()), + Int64(weights.data_ptr()), + Int32(m), + cu_stream(logits.device), + ) + + def for_rows(self, m: int) -> Optional[MoeSteps]: + if m not in SUPPORTED_ROWS or self.engine is None: + return None + return MoeSteps(self, m) + + +def build(model, derived) -> Optional[MoeGlue]: + from sgl_nvidia_mega.glm_dsa.kernels.cute.moe import api + from sgl_nvidia_mega.glm_dsa.kernels.cute.moe import weights as W + from sglang.srt.models.deepseek_v2 import DeepseekV2MoE + + def stock(why) -> None: + logger.warning("GLM DSA fused decode: MoE stays stock (%s)", why) + + moe_layers = [l for l in model.model.layers if isinstance(l.mlp, DeepseekV2MoE)] + if not moe_layers: + return None + for layer in moe_layers: + why = _layer_reason(layer.mlp) + if why is not None: + return stock(f"layer {layer.layer_id}: {why}") + try: + specs = api.compile_specs(shared=True) + except ValueError as e: + return stock(e) + dev = moe_layers[0].mlp.experts.w13_weight.device + try: + W.layout_self_test(_process_like_server, dev) + except W.LayoutMismatch as e: + return stock(f"layout self-test: {e}") + layers = {} + for layer in moe_layers: + ex, sh = layer.mlp.experts, layer.mlp.shared_experts + records = derived.add( + "moe.fc2_records", + W.fc2_records( + _u8(ex.w2_weight.data).reshape(EXPERTS, HIDDEN, INTER // 2), + _u8(ex.w2_weight_scale.data).reshape(EXPERTS, -1), + ), + ) + pack = derived.add( + "moe.shared_pack", + W.shared_pack(sh.gate_up_proj.weight.data, sh.down_proj.weight.data), + ) + lt = api.LayerTensors( + w13=ex.w13_weight.data, + w13_scales=_u8(ex.w13_weight_scale.data).reshape(EXPERTS, -1), + fc2_records=records.view(EXPERTS, 4, 4, -1), + shared_pack=pack, + g1_alpha=ex.g1_alphas.data, + g2_alpha=ex.g2_alphas.data, + gs1=float(ex.w13_input_scale_quant.item()), + gs2=float(ex.w2_input_scale_quant.item()), + ) + lt.check() + layers[layer.layer_id] = lt + logger.info("GLM DSA fused decode: MoE engine on %d layers", len(layers)) + return MoeGlue(api, layers, specs) diff --git a/sgl_nvidia_mega/glm_dsa/providers.py b/sgl_nvidia_mega/glm_dsa/providers.py new file mode 100644 index 00000000..acef0138 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/providers.py @@ -0,0 +1,595 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""The step groups of one fused verify forward, their stock bodies, and kernel +compilation. + +Per decoder layer the runtime's loop calls ``HopSteps.pre_attention`` (the +previous MLP output's all-reduce, residual add and ``input_layernorm``), +``AttentionDecode.forward`` (up to the ``o_proj`` input), +``HopSteps.post_attention`` (``o_proj``, its all-reduce, residual add and +``post_attention_layernorm``) and ``MlpSteps.forward`` (un-reduced output); then +``HopSteps.final``. The default bodies copy ``DeepseekV2Model.forward`` for a +verify step under the configurations eligibility admits, so with every group on +its default the loop is bitwise equal to stock. Kernel providers subclass a group +and override only the steps they replace, handing values on ``StepContext``. +The compile key carries the kernel class's qualified name, so a test subclass +never shares a cache entry with a served kernel. +""" + +from __future__ import annotations + +import logging +import os +from contextlib import nullcontext +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, Hashable, Optional, Tuple + +import torch + +logger = logging.getLogger(__name__) + + +@dataclass +class StepContext: + """Per-forward state shared by the step groups of one verify forward.""" + + forward_batch: Any + positions: torch.Tensor + zero_allocator: Any + # Set by a routing hop: (expert ids [rows, 8] int32, weights [rows, 8] fp32); + # read and cleared by the same layer's MoE step. + routing: Any = None + # Per layer, cleared by AttentionDecode.forward: the indexer key / head-gate + # projections and the FP8 MLA query when a fused step produced them early. + index_kw: Any = None + q_fp8: Any = None + # A deferred input norm (DNORM): (layer id, block sums [rows, 8] fp32); read + # and cleared by that layer's attention. + dnorm: Any = None + + +@dataclass +class PrologueOut: + """What ``prologue`` hands on; None where a fused override consumed it. + ``q_lora`` [rows, q_lora_rank] is the indexer input; ``q_nope`` / ``q_pe`` + are per head and ``k_nope`` [rows, 1, kv_lora_rank] / ``k_pe`` before RoPE.""" + + q_lora: Optional[torch.Tensor] + q_nope: Optional[torch.Tensor] + q_pe: Optional[torch.Tensor] + k_nope: Optional[torch.Tensor] + k_pe: Optional[torch.Tensor] + + +@dataclass +class IndexQ: + """The indexer query of a layer that runs its own indexer.""" + + q_fp8: torch.Tensor # [rows, index_heads, index_head_dim], FP8 + weights: torch.Tensor # [rows, index_heads, 1], fp32 head gates + + +class AttentionDecode: + """Stock verify attention of one ``DeepseekV2AttentionMLA`` layer, in steps: + ``qkv_a`` -> ``prologue`` -> (own indexer) ``index_k`` -> ``index_q`` -> + ``indexer`` -> ``q_absorb`` -> ``rope_cache`` -> ``sparse_mla`` -> ``uv_proj``. + + The bodies copy the verify slices of ``forward_absorb_prepare`` / + ``forward_absorb_core`` (forward_mla.py), the DSA indexer's ``forward_cuda`` + and the DSA backend's ``_forward_trtllm`` at SGLang main 44b44774 for the one + configuration served (bf16 W_UK / W_UV, fused indexer projections, FP8 KV + cache on the TRT-LLM sparse kernel, fused top-k page table, full-graph + verify); eligibility refuses every other one, and the replica test checks the + copies bitwise against stock. A kernel covering several steps overrides the + first and passes the others through, handing state on ``StepContext``. + """ + + def forward(self, layer, x, prev_topk_indices, step: StepContext): + """The ``o_proj`` input ``[rows, heads * v_head_dim]`` and the top-k + indices handed to the next layer (None unless it reuses them).""" + from sglang.srt.layers.layer_boundary import get_attn_tp_context + from sglang.srt.layers.layer_boundary.adapters.attention import ( + AttentionInputs, + ) + + attn = layer.self_attn + fb = step.forward_batch + self.check_mode(attn, fb.forward_mode) + ctx = get_attn_tp_context() + # The attention reads its input through the attention-TP context. + ctx.set_attn_inputs(AttentionInputs(x, fb, attn.prepare_qkv_latent)) + step.index_kw = step.q_fp8 = None + try: + with attn.maybe_use_decode_attn_tp(fb): + qkv_latent = self.qkv_a(attn, x, step) + pro = self.prologue(attn, qkv_latent, step) + index_q = None + if attn.should_run_indexer(prev_topk_indices): + if self._stock_index_steps(): + index_q = self._index_k_and_q(attn, x, pro.q_lora, step) + else: + self.index_k(attn, x, step.positions, step) + index_q = self.index_q(attn, x, pro.q_lora, step) + topk_indices = self.indexer(attn, index_q, prev_topk_indices, step) + q_nope_out = self.q_absorb(attn, pro.q_nope, step) + q_fp8 = self.rope_cache( + attn, q_nope_out, pro.q_pe, pro.k_nope, pro.k_pe, step + ) + attn_output = self.sparse_mla(attn, q_fp8, topk_indices, step) + out = self.uv_proj(attn, attn_output) + finally: + ctx.clear_attn_inputs() + step.index_kw = step.q_fp8 = None + return out, (topk_indices if attn.next_skip_topk else None) + + def check_mode(self, attn, mode) -> None: + # The stock bodies copy the branches of these modes only (the draft's + # decode passes: draft_attention.DraftAttention). + if not (mode.is_target_verify() or mode.is_draft_extend_v2()): + raise RuntimeError( + f"AttentionDecode serves target-verify and draft-extend-v2 forwards only (got {mode})" + ) + + def qkv_a(self, attn, x, step: StepContext) -> torch.Tensor: + """``[rows, q_lora_rank + kv_lora_rank + qk_rope_head_dim]``; an override + may also leave the indexer key / head-gate projections in + ``step.index_kw``.""" + from sglang.srt.layers.layer_boundary import get_attn_tp_context + + # forward_absorb_prepare (q_lora_rank set). + return get_attn_tp_context().fetch_qkv_latent() + + def prologue( + self, attn, qkv_latent: torch.Tensor, step: StepContext + ) -> PrologueOut: + """The q_a / kv_a norms, ``q_b_proj`` and the q / key-RoPE split.""" + from sglang.srt.model_executor.runner import get_is_capture_mode + + # forward_absorb_prepare (main 44b44774), verify: the kv_a norm runs on + # the side stream during capture. + q, latent_cache = qkv_latent.split( + [attn.q_lora_rank, attn.kv_lora_rank + attn.qk_rope_head_dim], dim=-1 + ) + k_nope = latent_cache[..., : attn.kv_lora_rank] + if attn.alt_stream is not None and get_is_capture_mode(): + current_stream = torch.cuda.current_stream() + attn.alt_stream.wait_stream(current_stream) + q = attn.q_a_layernorm(q) + with torch.cuda.stream(attn.alt_stream): + k_nope = attn.kv_a_layernorm(k_nope) + current_stream.wait_stream(attn.alt_stream) + else: + q = attn.q_a_layernorm(q) + k_nope = attn.kv_a_layernorm(k_nope) + q_lora = q + k_nope = k_nope.unsqueeze(1) + q = attn.q_b_proj_forward(q) + q_nope, q_pe, k_pe = attn._split_q_nope_pe(q, latent_cache) + return PrologueOut(q_lora, q_nope, q_pe, k_nope, k_pe) + + def index_k(self, attn, x, positions: torch.Tensor, step: StepContext) -> None: + """The indexer key's norm, RoPE, FP8 quantization and index-K cache rows.""" + # _fused_q_prepare_and_store (main 44b44774), single-stream key half. + key, _ = self._index_kw(attn.indexer, x, step) + _store_index_k(attn, key, positions, step) + + def index_q(self, attn, x, q_lora: torch.Tensor, step: StepContext) -> IndexQ: + """The indexer query (``wq_b``, RoPE, FP8) and its head gates.""" + # _fused_q_prepare_and_store (main 44b44774), single-stream query half. + indexer = attn.indexer + _, weights_raw = self._index_kw(indexer, x, step) + q = indexer.wq_b(q_lora)[0].view(-1, indexer.n_heads, indexer.head_dim) + return IndexQ(*_index_q_quant(indexer, q, weights_raw, step.positions)) + + def _stock_index_steps(self) -> bool: + cls = type(self) + return ( + cls.index_k is AttentionDecode.index_k + and cls.index_q is AttentionDecode.index_q + ) + + def _index_k_and_q( + self, attn, x, q_lora: torch.Tensor, step: StepContext + ) -> IndexQ: + """``index_k`` + ``index_q`` when neither is replaced, as stock runs them.""" + indexer = attn.indexer + if indexer.alt_stream is None: + AttentionDecode.index_k(self, attn, x, step.positions, step) + return AttentionDecode.index_q(self, attn, x, q_lora, step) + # _fused_q_prepare_and_store (main 44b44774), two-stream branch: wq_b + # (side) || key / head-gate projections (current), then the query kernel + # (current) || the key kernel + index-K store (side). + positions = step.positions + current_stream = torch.cuda.current_stream() + indexer.alt_stream.wait_stream(current_stream) + with torch.cuda.stream(indexer.alt_stream): + q = indexer.wq_b(q_lora)[0].view(-1, indexer.n_heads, indexer.head_dim) + key, weights_raw = self._index_kw(indexer, x, step) + current_stream.wait_stream(indexer.alt_stream) + indexer.alt_stream.wait_stream(current_stream) + q_fp8, weights = _index_q_quant(indexer, q, weights_raw, positions) + with torch.cuda.stream(indexer.alt_stream): + _store_index_k(attn, key, positions, step) + current_stream.wait_stream(indexer.alt_stream) + return IndexQ(q_fp8, weights) + + @staticmethod + def _index_kw(indexer, x, step: StepContext): + """(key, raw head gates), projecting ``x`` unless a step already did.""" + if step.index_kw is None: + step.index_kw, _ = indexer.wk_weights_proj(x) + return step.index_kw.split([indexer.head_dim, indexer.n_heads], dim=-1) + + def indexer( + self, attn, index_q: Optional[IndexQ], prev_topk_indices, step: StepContext + ): + """Top-k cache rows per query row (page-table rows); a layer without its + own indexer (``index_q`` None) reuses the previous layer's.""" + from sglang.srt.layers.attention.dsa.dsa_indexer import ( + _broadcast_indexer_topk_from_rank0, + ) + from sglang.srt.model_executor.forward_context import get_attn_backend + from sglang.srt.state_capturer.indexer_topk import maybe_capture_indexer_topk + + if index_q is None: + # forward_absorb_prepare: a skip-top-k layer mirrors the reused + # indices into its own capture slot. + return maybe_capture_indexer_topk(attn.layer_id, prev_topk_indices) + # The DSA indexer's forward_cuda (main 44b44774), target verify inside a + # full graph (non-empty batch, logits not skipped). + fb = step.forward_batch + metadata = get_attn_backend().get_indexer_metadata(attn.layer_id, fb) + if metadata is None: + raise RuntimeError( + "fused verify step: the attention backend has no indexer metadata" + ) + topk_result = attn.indexer._get_topk_paged( + fb, attn.layer_id, index_q.q_fp8, index_q.weights, metadata + ) + topk_result = _broadcast_indexer_topk_from_rank0(topk_result) + return maybe_capture_indexer_topk(attn.layer_id, topk_result) + + def q_absorb(self, attn, q_nope: torch.Tensor, step: StepContext) -> torch.Tensor: + """``q_nope`` through W_UK: ``[rows, heads, kv_lora_rank]``.""" + # forward_absorb_prepare (main 44b44774), bf16 W_UK. + return torch.bmm(q_nope.transpose(0, 1), attn.w_kc).transpose(0, 1) + + def rope_cache( + self, attn, q_nope_out, q_pe, k_nope, k_pe, step: StepContext + ) -> torch.Tensor: + """RoPE, the FP8 query ``[rows, heads, kv_lora_rank + rope]`` (returned) + and the key's MLA cache rows.""" + from sglang.kernels.ops.attention.utils import mla_quantize_and_rope_for_fp8 + from sglang.srt.model_executor.forward_context import get_attn_backend + + backend = get_attn_backend() + layer = attn.attn_mqa + fb = step.forward_batch + # The key as RadixAttention.forward hands it to the backend. + k = k_nope.view(-1, layer.tp_k_head_num, layer.v_head_dim) + # The DSA backend's _forward_trtllm (main 44b44774), FP8 KV cache with + # RoPE fused into the quantization (no prefill CP). + q, k, k_rope = mla_quantize_and_rope_for_fp8( + q_nope_out, + q_pe, + k.squeeze(1), + k_pe.squeeze(1), + fb.positions, + attn.rotary_emb.cos_sin_cache, + attn.rotary_emb.is_neox_style, + backend.kv_lora_rank, + backend.qk_rope_head_dim, + ) + backend.token_to_kv_pool.set_mla_kv_buffer(layer, fb.out_cache_loc, k, k_rope) + return q + + def sparse_mla( + self, attn, q_fp8: torch.Tensor, topk_indices, step: StepContext, decode=False + ) -> torch.Tensor: + """Sparse MLA over the selected cache rows: ``[rows, heads, kv_lora_rank]``. + ``decode``: a draft DECODE pass.""" + import flashinfer.decode + + from sglang.srt.environ import envs + from sglang.srt.model_executor.forward_context import get_attn_backend + + backend = get_attn_backend() + layer = attn.attn_mqa + metadata = backend.forward_metadata + # The DSA backend's _forward_trtllm (main 44b44774), FP8 query, fused + # top-k page table, no DCP; seq_lens as forward_extend passes them for + # target verify. Decode (forward_decode, is_prefill False): the full cache + # lengths, and eager DP padding rows beyond the metadata's batch trimmed + # before the kernel and restored as zeros. + k_cache = backend.token_to_kv_pool.get_key_buffer(layer.layer_id) + kv_cache = k_cache.view( + -1, backend.physical_page_size, backend.kv_cache_dim + ).unsqueeze(1) + q_all = q_fp8.view(-1, layer.tp_q_head_num, layer.head_dim) + if topk_indices is not None: + topk_indices = backend._pad_topk_indices(topk_indices, q_fp8.shape[0]) + if decode: + from sglang.srt.layers.attention.dsa_backend import ( + _restore_trtllm_decode_dp_padding, + _trim_trtllm_decode_dp_padding, + ) + + q_all, topk_indices, n_pad = _trim_trtllm_decode_dp_padding( + q_all, topk_indices, metadata.cache_seqlens_int32.shape[0] + ) + page_table_1 = backend._get_fused_topk_page_table(topk_indices) + page_table_1, sparse_mla_top_k = backend._pad_trtllm_sparse_page_table( + page_table_1 + ) + k_scale = ( + layer.k_scale_float + if getattr(layer, "k_scale_float", None) is not None + else 1.0 + ) + bmm1_scale = 1.0 * k_scale * layer.scaling + batch_size, num_heads, head_dim = q_all.shape + multi_ctas_kv_counter_buffer = backend._multi_ctas_kv_counter_for(batch_size) + seq_lens = None if decode else metadata.dsa_cache_seqlens_int32 + if seq_lens is None: + seq_lens = metadata.cache_seqlens_int32 + out = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla( + query=q_all.view(batch_size, 1, num_heads, head_dim), + block_tables=page_table_1.unsqueeze(1), + seq_lens=seq_lens, + kv_cache=kv_cache.view( + -1, 1, backend.physical_page_size, backend.kv_cache_dim + ), + workspace_buffer=backend.workspace_buffer, + qk_nope_head_dim=backend.qk_nope_head_dim, + kv_lora_rank=backend.kv_lora_rank, + qk_rope_head_dim=backend.qk_rope_head_dim, + max_seq_len=metadata.max_seq_len_k, + sparse_mla_top_k=sparse_mla_top_k, + bmm1_scale=bmm1_scale, + backend="trtllm-gen", + skip_softmax_threshold_scale_factor=envs.SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR.get(), + sparse_mla_top_k_lens=None, + multi_ctas_kv_counter_buffer=multi_ctas_kv_counter_buffer, + ) + if decode: + out = _restore_trtllm_decode_dp_padding(out, n_pad) + return out.view(-1, attn.num_local_heads, attn.kv_lora_rank) + + def uv_proj(self, attn, attn_output: torch.Tensor) -> torch.Tensor: + """``[rows, heads, kv_lora]`` -> ``[rows, heads * v]``: one batched GEMM + over the heads into the ``o_proj`` input buffer.""" + out = torch.empty( + (attn_output.shape[0], attn.num_local_heads * attn.v_head_dim), + dtype=attn_output.dtype, + device=attn_output.device, + ) + torch.bmm( + attn_output.transpose(0, 1), + attn.w_vc, + out=out.view(-1, attn.num_local_heads, attn.v_head_dim).transpose(0, 1), + ) + return out + + +def _store_index_k(attn, key, positions, step: StepContext) -> None: + """The stock index-K store (``_fused_k_prepare_and_store``) of ``key``.""" + from sglang.kernels.ops.quantization.act_quant_triton import act_quant + + fb = step.forward_batch + attn.indexer._fused_k_prepare_and_store( + key, positions, fb, attn.layer_id, act_quant, out_cache_loc=fb.out_cache_loc + ) + + +def _index_q_quant( + indexer, q: torch.Tensor, weights_raw: torch.Tensor, positions: torch.Tensor +): + """The indexer query's RoPE + FP8 quantization and the scaled head gates + (_fused_q_prepare_and_store, main 44b44774).""" + from sglang.kernels.ops.attention.dsa.indexer_q import ( + fused_q_indexer_rope_first_quant, + ) + + q_scale_gate = indexer.softmax_scale * indexer.n_heads**-0.5 + return fused_q_indexer_rope_first_quant( + q.contiguous(), + weights_raw, + q_scale_gate, + indexer._indexer_cos_sin_cache, + positions, + ) + + +class HopSteps: + """Stock all-reduce + residual + norm steps between the stages.""" + + def pre_attention(self, layer, mlp_out, residual, step: StepContext): + """The attention input and the updated residual (first layer: ``mlp_out`` + is the embedding and ``residual`` None).""" + norm = layer.input_layernorm + if residual is None: + return norm(mlp_out), mlp_out + # The MLP output owes its sum over the MoE output group (the TP group). + return norm.forward_with_allreduce_fusion( + mlp_out, residual, use_attn_tp_group=False + ) + + def post_attention(self, layer, o_input, residual, step: StepContext): + """``o_proj`` + attention-TP all-reduce + residual add + norm.""" + partial, _ = layer.self_attn.o_proj(o_input) + return layer.post_attention_layernorm.forward_with_allreduce_fusion( + partial, residual, use_attn_tp_group=True + ) + + def final(self, model, mlp_out, residual, step: StepContext) -> torch.Tensor: + """The last layer's MLP output completed, added and normed.""" + from sglang.srt.layers.moe.utils import sum_post_experts_output + + hidden = sum_post_experts_output(mlp_out) + return model.model.norm(hidden, residual)[0] + + +class MlpSteps: + """Stock dense MLP / MoE of a layer, leaving the output's sum to the next step.""" + + def forward(self, layer, x, buffer: Optional[torch.Tensor], step: StepContext): + """``buffer``: this layer's input storage, dead once the attention input is + read (the MoE may write its output into it).""" + from sglang.srt.layers.moe.moe_runner.base import moe_output_buffer_ctx + from sglang.srt.models.deepseek_v2 import DeepseekV2MoE + from sglang.srt.runtime_context import get_forward + + mlp = layer.mlp + if ( + isinstance(mlp, DeepseekV2MoE) + and not mlp.experts.moe_runner_config.inplace + and buffer is not None + ): + buffer_ctx = moe_output_buffer_ctx(buffer) + else: + buffer_ctx = nullcontext() + with get_forward().scoped(defer_moe_finalize=False), buffer_ctx: + return mlp(x, step.forward_batch, None) + + +# The steps of each group, in call order (describe, rank agreement, logs). +STEPS = { + "attention": ( + "forward", + "qkv_a", + "prologue", + "index_k", + "index_q", + "indexer", + "q_absorb", + "rope_cache", + "sparse_mla", + "uv_proj", + ), + "hops": ("pre_attention", "post_attention", "final"), + "mlp": ("forward",), +} + + +@dataclass(eq=False) +class Providers: + """The three step groups served at one verify row count.""" + + attention: AttentionDecode = field(default_factory=AttentionDecode) + hops: HopSteps = field(default_factory=HopSteps) + mlp: MlpSteps = field(default_factory=MlpSteps) + + def describe(self) -> Dict[str, str]: + """Which class serves each step (logs, rank agreement, tests).""" + out = {} + for g, names in STEPS.items(): + cls = type(getattr(self, g)) + for n in names: + out[f"{g}.{n}"] = getattr(cls, n).__qualname__.rsplit(".", 1)[0] + return out + + +def build_providers(model, rows: Tuple[int, ...], derived): + """(step groups per verify row count, built families). Each family's glue + module has ``build(model, derived)``: None keeps the family stock, else an + object with ``compile(rows)`` (through ``compile_cached``) and + ``for_rows(m)`` (its step group at ``m`` rows, None for stock).""" + from sgl_nvidia_mega.glm_dsa import attention, hops, moe + + families: Dict[str, Any] = {} + for group, glue in (("attention", attention), ("hops", hops), ("mlp", moe)): + family = glue.build(model, derived) + if family is not None: + family.compile(rows) + families[group] = family + out = { + m: Providers(**compose(m, {g: f.for_rows(m) for g, f in families.items()})) + for m in rows + } + for prov in out.values(): + # the deferred input norm needs its producer (the fold hop) and its consumer + # (the fused attention's epilogue-scaled projection) at the same row count + wire = getattr(prov.hops, "defer_input_norm", None) + if wire is not None and getattr(prov.attention, "takes_deferred_norm", False): + wire() + return out, families + + +def compose(rows: int, groups: Dict[str, Any]) -> Dict[str, Any]: + """The step groups served together at ``rows`` (stock where None). A group + whose ``requires`` names a family that is stock here is replaced by its + ``fallback`` (checked the same way), or by stock without one.""" + chosen = {g: v for g, v in groups.items() if v is not None} + changed = True + while changed: + changed = False + for g, v in list(chosen.items()): + if any(r not in chosen for r in getattr(v, "requires", ())): + chosen[g] = getattr(v, "fallback", None) + if chosen[g] is None: + del chosen[g] + changed = True + return chosen + + +_jit_cache: Any = None # created on first use +# Compiles, cache hits and units the persistent cache could not store (startup log). +compile_stats: Dict[str, int] = {"compiled": 0, "cached": 0, "not_stored": 0} + + +def kernel_source_paths() -> Tuple[str, ...]: + """Sources whose content keys the persistent cache directory.""" + import sgl_nvidia_mega.glm_dsa.kernels as kernels + + return tuple( + os.path.dirname(os.path.abspath(f)) for f in (__file__, kernels.__file__) + ) + + +def compile_key(kernel: Any, constexprs: Dict[str, Hashable]) -> Tuple[Hashable, ...]: + """The cache key: ``kernel``'s class (so a test subclass caches separately) + and the compile-time constants.""" + cls = type(kernel) + return (f"{cls.__module__}.{cls.__qualname__}", tuple(sorted(constexprs.items()))) + + +def compile_cached( + kernel: Any, constexprs: Dict[str, Hashable], compile_fn: Callable[[], Any] +) -> Any: + """The compiled ``kernel`` for these constants, compiling on a miss (the + units are compiled without tvm-ffi). A unit the persistent cache cannot store + warns once and serves from memory.""" + global _jit_cache + if _jit_cache is None: + from sglang.kernels.jit.cute_aot_cache import get_jit_cache + + _jit_cache = get_jit_cache( + "glm_dsa", source_paths=kernel_source_paths(), enable_tvm_ffi=False + ) + key = compile_key(kernel, constexprs) + if key in _jit_cache: + compile_stats["cached"] += 1 + return _jit_cache[key] + compiled = compile_fn() + compile_stats["compiled"] += 1 + try: + _jit_cache[key] = compiled + except Exception as e: # the export to disk; the in-memory entry is set first + compile_stats["not_stored"] += 1 + detail = " ".join(str(e).split())[:300] + logger.warning( + "GLM DSA fused decode: the CuTe AOT cache could not store %s %s (%s: %s); " + "it is compiled again in every new process", + key[0], + dict(key[1]), + type(e).__name__, + detail, + ) + return compiled + + +def cu_stream(device=None): + """The current torch stream of ``device`` as a kernel launch argument.""" + import cuda.bindings.driver as cuda + + return cuda.CUstream(torch.cuda.current_stream(device).cuda_stream) diff --git a/sgl_nvidia_mega/glm_dsa/runtime.py b/sgl_nvidia_mega/glm_dsa/runtime.py new file mode 100644 index 00000000..b3f710df --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/runtime.py @@ -0,0 +1,316 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""The fused verify step: the decoder stack of ``GlmMoeDsaForCausalLM`` for MTP +target-verify forwards of 1-4 requests. + +``run`` owns the layer loop and the residual stream: per layer +``hops.pre_attention`` (the previous MLP output's all-reduce + residual add + +input norm), ``attention.forward`` (up to the ``o_proj`` input), +``hops.post_attention`` (``o_proj`` + all-reduce + residual add + norm) and +``mlp.forward``; then ``hops.final``. Each step runs on the provider set of the +batch's row count (``providers``); with every step stock the result is bitwise +equal to ``DeepseekV2Model.forward`` (the replica test). + +The fused step never runs eagerly. It is recorded only while the decode graph +runner captures a full graph, after that row count's eager warmups ran the +stock forward on every rank (compiling and loading every stock kernel the step +calls), and the first capture of each row count starts with a CPU barrier of +the tensor-parallel group. Its first execution is therefore a replay after +every rank has finished startup, so a rank slow to start (compiles, autotune) +cannot leave the others in a bounded cross-GPU wait. +""" + +from __future__ import annotations + +import atexit +import logging +import threading +import time +from typing import Dict, List, Optional + +import torch +import torch.distributed as dist + +from sglang.srt import runtime_context +from sgl_nvidia_mega.glm_dsa import eligibility +from sgl_nvidia_mega.glm_dsa.providers import ( + Providers, + StepContext, + build_providers, + compile_stats, +) + +logger = logging.getLogger(__name__) + +# This process's prepared target runtime (one target model per scheduler +# process); a draft model on the same TP group reaches the target's hop family +# and error reporter through it. +TARGET: Optional[GlmDsaFusedDecode] = None + + +def tp_gather(obj) -> list: + """``obj`` of every tensor-parallel rank (host only, on the CPU group; legal + inside a graph capture).""" + group = runtime_context.get_parallel().tp_group + out = [None] * group.world_size + dist.all_gather_object(out, obj, group=group.cpu_group) + return out + + +def per_rank(gathered) -> str: + return "; ".join(f"rank {r}: {', '.join(g)}" for r, g in enumerate(gathered) if g) + + +class DerivedWeights: + """Owns the packed / re-laid-out copies of stock weights (the checkpoint is + never modified): the kernels keep only their device addresses (e.g. + ``attention.LayerArgs``), so these tensors must live as long as the runtime. + Also logged by kind at startup.""" + + def __init__(self) -> None: + self.tensors: Dict[str, List[torch.Tensor]] = {} + + def add(self, kind: str, tensor: torch.Tensor) -> torch.Tensor: + self.tensors.setdefault(kind, []).append(tensor) + return tensor + + def log(self, rank: int) -> None: + nbytes = { + k: sum(t.numel() * t.element_size() for t in ts) + for k, ts in sorted(self.tensors.items()) + } + if not nbytes: + logger.info( + "GLM DSA fused decode: no derived weight copies (rank %d)", rank + ) + return + logger.info( + "GLM DSA fused decode: derived weight copies on rank %d: %s (total %.1f MiB)", + rank, + ", ".join(f"{k} {v / 2**20:.1f} MiB" for k, v in nbytes.items()), + sum(nbytes.values()) / 2**20, + ) + + +class GlmDsaFusedDecode: + """Owns the fused verify path of one target model instance.""" + + def __init__(self, model) -> None: + eligibility.check_construction(model) + self.model = model + self.derived = DerivedWeights() + self.providers: Dict[int, Providers] = {} + self.families: Dict[str, object] = {} + # Per row count, decided at its first capture on every rank alike: + # whether the fused step is captured (else the stock forward is). + self._agreed: Dict[int, bool] = {} + self._ready = False + self.errors = ErrorReporter() + + def prepare(self) -> None: + """From ``precompile_kernels_after_loading`` (before the KV pool is sized, + so its memory is charged): Tier 2, provider sets, kernel compilation.""" + global TARGET + eligibility.check_loaded(self.model) + self.providers, self.families = build_providers( + self.model, eligibility.SUPPORTED_ROWS, self.derived + ) + rank = runtime_context.get_parallel().tp_rank + self._agree_on_composition(rank) + self.derived.log(rank) + logger.info( + "GLM DSA fused decode: kernels compiled %d, loaded from cache %d (rank %d), " + "not stored in the cache %d", + compile_stats["compiled"], + compile_stats["cached"], + rank, + compile_stats["not_stored"], + ) + if rank == 0: + for rows, p in self.providers.items(): + fused = { + k: v + for k, v in p.describe().items() + if not v.startswith(("AttentionDecode", "HopSteps", "MlpSteps")) + } + logger.info( + "GLM DSA fused decode: %d verify rows: %s", + rows, + fused or "all steps stock", + ) + self._ready = True + TARGET = self + for family in self.families.values(): + if callable(getattr(family, "error_report", None)): + self.errors.add(family.error_report) + + def _agree_on_composition(self, rank: int) -> None: + """Every rank must serve the same step groups at every row count (the + cross-GPU steps pair up across ranks); if any rank composed differently + (e.g. a family declined on one rank only), every rank serves stock.""" + gathered = tp_gather({m: p.describe() for m, p in self.providers.items()}) + differing = [r for r, c in enumerate(gathered) if c != gathered[0]] + if differing: + if rank == 0: + logger.warning( + "GLM DSA fused decode: ranks %s composed different step groups " + "than rank 0; every rank serves the stock forward", + differing, + ) + self.providers = {rows: Providers() for rows in self.providers} + self.families = {} + + def engages( + self, rows: int, forward_batch, input_embeds=None, pp_proxy_tensors=None + ) -> bool: + """Tier 3 (``eligibility.step_in_graph_runner``) while the decode graph + runner captures the forward; its eager warmups run the stock forward.""" + return ( + self._ready + and eligibility.step_in_graph_runner( + self.model, rows, forward_batch, input_embeds, pp_proxy_tensors + ) + and torch.cuda.is_current_stream_capturing() + and self._first_capture_agreement(rows, forward_batch) + ) + + def _first_capture_agreement(self, rows: int, forward_batch) -> bool: + """At the first capture of ``rows`` every rank checks that the stock + attention takes the branches the attention steps copy + (``eligibility.attention_step_reasons``) and each built family's host-only + ``step_reasons``; the gather is also the barrier before that capture. A + reason on any rank captures the stock forward at ``rows`` on every rank.""" + if rows in self._agreed: + return self._agreed[rows] + reasons = list(eligibility.attention_step_reasons(self.model, forward_batch)) + for name, family in sorted(self.families.items()): + extra = getattr(family, "step_reasons", None) + if extra is not None: + reasons += [f"{name}: {r}" for r in extra(self.model, forward_batch)] + gathered = tp_gather(reasons) + self._agreed[rows] = agreed = not any(gathered) + if runtime_context.get_parallel().tp_rank == 0: + if agreed: + logger.info( + "GLM DSA fused decode: capturing the fused verify step, %d rows", + rows, + ) + else: + logger.warning( + "GLM DSA fused decode: %d verify rows captured on the stock forward: %s", + rows, + per_rank(gathered), + ) + return agreed + + @torch.no_grad() + def run(self, input_ids, positions, forward_batch) -> torch.Tensor: + """The final-normed hidden states of a verify forward.""" + from sglang.srt.layers.layer_boundary import get_attn_tp_context + + providers = self.providers[int(positions.shape[0])] + with get_attn_tp_context().maybe_input_scattered(forward_batch): + return self._stack(providers, input_ids, positions, forward_batch) + + def _stack(self, providers, input_ids, positions, forward_batch) -> torch.Tensor: + """``DeepseekV2Model.forward`` on an eligible step, on ``providers``.""" + from sglang.srt.eplb.expert_distribution import ( + get_global_expert_distribution_recorder, + ) + from sglang.srt.layers.attention.dsa.utils import ( + maybe_prefetch_next_full_attention_kv, + ) + from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState + from sglang.srt.utils import BumpAllocator + + mm = self.model.model + step = StepContext( + forward_batch=forward_batch, + positions=positions, + zero_allocator=BumpAllocator( + buffer_size=(mm.end_layer - mm.start_layer) * 2, + dtype=torch.float32, + device=positions.device, + ), + ) + topk_share = IndexTopKShareState(forward_batch, None) + recorder = get_global_expert_distribution_recorder() + attention, hops, mlp = providers.attention, providers.hops, providers.mlp + mlp_out, residual = mm.embed_tokens(input_ids), None + for i in range(mm.start_layer, mm.end_layer): + layer = mm.layers[i] + with recorder.with_current_layer(i): + # The consumed MLP output's storage may take this layer's MoE output. + buffer = mlp_out if isinstance(mlp_out, torch.Tensor) else None + x, residual = hops.pre_attention(layer, mlp_out, residual, step) + o_input, topk_indices = attention.forward( + layer, x, topk_share.topk_indices, step + ) + topk_share.update(topk_indices) + maybe_prefetch_next_full_attention_kv( + forward_batch, mm.next_full_attention_layer_id.get(i) + ) + x, residual = hops.post_attention(layer, o_input, residual, step) + mlp_out = mlp.forward(layer, x, buffer, step) + return hops.final(self.model, mlp_out, residual, step) + + +class ErrorReporter: + """Logs the first error record left by a cross-GPU wait that timed out. + + Each source returns None or a dict describing the wait (kind, layer and hop, + waiting rank, late peer, rows) read from host memory, so it works after the + trap has lost the CUDA context. Once a source exists a daemon thread checks + every ``period_s`` seconds, and once more at exit. + """ + + def __init__(self, sources=(), period_s: float = 0.5) -> None: + self.sources = list(sources) + self.record = None + self._lock = threading.Lock() + self._period_s = period_s + self._watching = False + if self.sources: + self._start() + + def add(self, source) -> None: + with self._lock: + self.sources.append(source) + self._start() + + def _start(self) -> None: + if not self._watching: + self._watching = True + atexit.register(self.check) + threading.Thread( + target=self._watch, name="glm-dsa-errors", daemon=True + ).start() + + def check(self): + with self._lock: + if self.record is not None: + return self.record + for read in list(self.sources): + try: + record = read() + except Exception as e: # noqa: BLE001 + logger.warning( + "GLM DSA fused decode: error record unreadable: %s", e + ) + self.sources.remove(read) + continue + if record: + self.record = record + logger.error( + "GLM DSA fused decode: a cross-GPU wait in the fused verify " + "step timed out and trapped; this process's CUDA context is " + "lost. Error record: %s", + record, + ) + break + return self.record + + def _watch(self) -> None: + while self.sources and self.check() is None: + time.sleep(self._period_s) + diff --git a/sgl_nvidia_mega/glm_dsa/spec.py b/sgl_nvidia_mega/glm_dsa/spec.py new file mode 100644 index 00000000..11365d4c --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/spec.py @@ -0,0 +1,184 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The GLM-5 DSA megakernel's spec: the MTP target-verify steps of 1-4 requests +(4 / 8 / 12 / 16 rows) of ``GlmMoeDsaForCausalLM`` on SM103 / SM107 at TP8. + +SGLang checks the requirements at argument resolution and the model at load +(``sglang.srt.megakernels``); an unsupported setup stops the server on every rank, +listing every reason. This module stays light: SGLang imports it in the launcher. +""" + +from __future__ import annotations + +import importlib.util + +from sglang.srt.megakernels.spec import ( + Feature, + MegakernelSpec, + Numerics, + NumericsKind, + Requirements, +) + +from sgl_nvidia_mega.glm_dsa import switches + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/megakernels/spec.py": ( + "Feature", + "MULTI_NODE_TP", + "SPECULATIVE_DECODING", + "MegakernelSpec", + "Numerics", + "NumericsKind", + "TOLERANCE", + "Requirements", + "config_disable_reasons", + "model_disable_reasons", + ), + "srt/models/glm4_moe.py": ("GlmMoeDsaForCausalLM", "GlmMoeDsaForCausalLMNextN"), + "srt/arg_groups/fields/*.py": ( + "kv_cache_dtype", + "page_size", + "attention_backend", + "dsa_decode_backend", + "moe_runner_backend", + "speculative_algorithm", + "speculative_eagle_topk", + "speculative_num_draft_tokens", + "speculative_num_steps", + "cuda_graph_config", + ), +} + +ARCHITECTURE = "GlmMoeDsaForCausalLM" +# the sglang.srt.megakernels contract this spec is written against +API_VERSION = 1 +# verify rows: 1-4 requests x 4 tokens (3 draft tokens + 1) +MAX_ROWS = 16 + +REQUIREMENTS = Requirements( + isas=frozenset({"sm_103", "sm_107"}), + tp_sizes=frozenset({8}), + dtypes=frozenset({"bfloat16"}), + # the fused step is recorded only into captured FULL decode graphs + decode_graph_backends=frozenset({"full"}), + server_args={ + "kv_cache_dtype": frozenset({"fp8_e4m3"}), + "page_size": frozenset({64}), + "attention_backend": frozenset({"dsa"}), + "dsa_decode_backend": frozenset({"trtllm"}), + "moe_runner_backend": frozenset({"flashinfer_trtllm"}), + # NEXTN resolves to EAGLE; the MTP chain of 3 draft tokens + "speculative_algorithm": frozenset({"EAGLE"}), + "speculative_eagle_topk": frozenset({1}), + "speculative_num_draft_tokens": frozenset({4}), + "speculative_num_steps": frozenset({3}), + }, + hf_config={"model_type": "glm_moe_dsa"}, + # the fused draft's embedding front reads the replicated table + env={"SGLANG_ENABLE_EMBED_REPLICATION": True} if switches.fused_draft() else {}, +) + +# The validated claim: the step's layer loop with every step on its stock body is +# bitwise SGLang's forward (the in-graph replica check). The fused kernels' own +# per-op tolerances are not stated yet. +NUMERICS = Numerics( + kind=NumericsKind.TOLERANCE, + tolerances={"loop.stock_bodies.max_abs": 0.0}, +) + + +def toolchain_reasons() -> list[str]: + reasons = [ + f"needs {what}" + for module, what in ( + ("cutlass", "CuTe DSL (nvidia-cutlass-dsl)"), + ("cuda.bindings", "the CUDA Python bindings"), + ) + if importlib.util.find_spec(module) is None + ] + if not reasons: + from sgl_nvidia_mega.glm_dsa.kernels.cute import cutlass_supported + + ok, why = cutlass_supported() + if not ok: + reasons.append(why) + return reasons + + +def _config_disable_reasons(cfg, hf_config) -> list[str]: + from sgl_nvidia_mega.glm_dsa import eligibility + + return toolchain_reasons() + eligibility.construction_reasons(hf_config, cfg) + + +def _model_disable_reasons(model, hf_config) -> list[str]: + from sgl_nvidia_mega.glm_dsa import eligibility + + try: + eligibility.check_construction(model) + eligibility.check_loaded(model) + except ValueError as e: + return [str(e)] + return [] + + +class _Built: + """The installed target forward: covered verify steps run the fused stack.""" + + draft_sampler = None + + def __init__(self, model, fused): + self.model, self.fused = model, fused + self.widths = tuple(sorted(fused.providers)) + + def summary(self) -> str: + fused = { + rows: sorted( + k + for k, v in p.describe().items() + if not v.startswith(("AttentionDecode", "HopSteps", "MlpSteps")) + ) + for rows, p in self.fused.providers.items() + } + return "GLM-5 DSA fused verify; fused steps by rows: " + "; ".join( + f"{rows}: {', '.join(names) or 'stock'}" for rows, names in fused.items() + ) + + def plan(self, input_ids, forward_batch, *args, **kwargs): + from sglang.srt.megakernels.spec import Step + + input_embeds = kwargs.get("input_embeds", args[0] if args else None) + pp_proxy = kwargs.get("pp_proxy_tensors", args[1] if len(args) > 1 else None) + rows = int(input_ids.shape[0]) + if not self.fused.engages(rows, forward_batch, input_embeds, pp_proxy): + return None + return Step(mode="target_verify", rows=rows) + + def forward(self, step, original, input_ids, positions, forward_batch, *a, **kw): + hidden_states = self.fused.run(input_ids, positions, forward_batch) + return self.model.logits_processor( + input_ids, hidden_states, self.model.lm_head, forward_batch, None + ) + + +def _build(model, hf_config, ctx): + from sgl_nvidia_mega.glm_dsa.runtime import GlmDsaFusedDecode + + fused = GlmDsaFusedDecode(model) + fused.prepare() + return _Built(model, fused) + + +SPEC = MegakernelSpec( + name="GLM-5 DSA", + requires=REQUIREMENTS, + features=frozenset({Feature.SPECULATIVE_DECODING, Feature.MULTI_NODE_TP}), + numerics=NUMERICS, + max_rows=MAX_ROWS, + build=_build, + api_version=API_VERSION, + model_disable_reasons=_model_disable_reasons, + config_disable_reasons=_config_disable_reasons, +) diff --git a/sgl_nvidia_mega/glm_dsa/switches.py b/sgl_nvidia_mega/glm_dsa/switches.py new file mode 100644 index 00000000..1520b120 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/switches.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The GLM-5 DSA megakernel's switches, read from the environment. + +With ``--enable-nvidia-megakernel`` the fused verify step always runs; these pick +the optional parts. Each is "1" / "0"; unset takes the default. +""" + +from __future__ import annotations + +import os + +# name -> default +DEFAULTS = { + # the layers without their own top-k also run qkv_a on the small-M kernel + "SGLANG_NVIDIA_MEGAKERNEL_GLM_SHARED_QKV_A_SMALL_M": True, + # with the above: the input RMSNorm deferred into the small-M qkv_a kernel + "SGLANG_NVIDIA_MEGAKERNEL_GLM_DNORM": True, + # the NextN draft layer's fused kernels (draft decode and draft extend graphs) + "SGLANG_NVIDIA_MEGAKERNEL_GLM_FUSED_DRAFT": True, + # Serve the fused verify step though SGLang's FlashInfer all-reduce + RMSNorm + # fusion is off at verify sizes (no NVLink multicast, e.g. multi-node NVL72 trays): + # the fused hops do not need multicast, but the stock step they replace then runs + # NCCL + RMSNorm, so its numerics are not the ones the hops reproduce. + "SGLANG_NVIDIA_MEGAKERNEL_GLM_ALLOW_NO_AR_FUSION": False, +} + + +def enabled(name: str) -> bool: + raw = os.environ.get(name) + if raw is None or raw == "": + return DEFAULTS[name] + return raw not in ("0", "false", "False") + + +def shared_qkv_a_small_m() -> bool: + return enabled("SGLANG_NVIDIA_MEGAKERNEL_GLM_SHARED_QKV_A_SMALL_M") + + +def dnorm() -> bool: + return shared_qkv_a_small_m() and enabled("SGLANG_NVIDIA_MEGAKERNEL_GLM_DNORM") + + +def fused_draft() -> bool: + return enabled("SGLANG_NVIDIA_MEGAKERNEL_GLM_FUSED_DRAFT") + + +def allow_no_ar_fusion() -> bool: + return enabled("SGLANG_NVIDIA_MEGAKERNEL_GLM_ALLOW_NO_AR_FUSION") diff --git a/sgl_nvidia_mega/plugin.py b/sgl_nvidia_mega/plugin.py index 456031ec..d43e5b27 100644 --- a/sgl_nvidia_mega/plugin.py +++ b/sgl_nvidia_mega/plugin.py @@ -19,6 +19,9 @@ ("DeepseekV4ForCausalLM", "sgl_nvidia_mega.deepseek_v41.spec:SPEC"), # the DSpark draft: SGLang's forward, the megakernel's folded proposal ("DeepseekV4ForCausalLMDSpark", "sgl_nvidia_mega.deepseek_v41.draft:SPEC"), + ("GlmMoeDsaForCausalLM", "sgl_nvidia_mega.glm_dsa.spec:SPEC"), + # the NextN draft: the fused draft forward and its top-1 logits + ("GlmMoeDsaForCausalLMNextN", "sgl_nvidia_mega.glm_dsa.draft_spec:SPEC"), ) # Test only: with SGLANG_NVIDIA_MEGAKERNEL_PARITY_SORT_TOPK=1, the composition check @@ -61,3 +64,9 @@ def register() -> None: for target, kind, hook in TEST_HOOKS: HookRegistry.register(target, pkgutil.resolve_name(hook), HookType[kind]) + from sgl_nvidia_mega.spec_step import switches + + if switches.any_enabled(): + from sgl_nvidia_mega.spec_step import hooks + + hooks.register() diff --git a/sgl_nvidia_mega/spec_step/__init__.py b/sgl_nvidia_mega/spec_step/__init__.py new file mode 100644 index 00000000..f84965e9 --- /dev/null +++ b/sgl_nvidia_mega/spec_step/__init__.py @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The EAGLE v2 speculative step graph and its fusions, for any model. + +* ``SGLANG_NVIDIA_MEGAKERNEL_SPEC_STEP_GRAPH=1``: each topk-1 decode step (draft, + target verify, draft extend and the work between them) replays as one CUDA graph, + with the per-stage graphs spliced in as child graphs (``runner``). +* ``SGLANG_SPEC_*`` (``switches``): fusions of the step's small launches; each has + bitwise-identical outputs. + +All default to off; ``hooks.register`` installs the hooks of the ones that are on. +""" diff --git a/sgl_nvidia_mega/spec_step/child_graph_capture.py b/sgl_nvidia_mega/spec_step/child_graph_capture.py new file mode 100644 index 00000000..66dd175e --- /dev/null +++ b/sgl_nvidia_mega/spec_step/child_graph_capture.py @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Record CUDA graph replays as child-graph nodes of an enclosing capture. + +While a ``ChildGraphCapture`` is active on the current thread, a graph backend +hands each graph it would replay to :meth:`ChildGraphCapture.add_graph`. In +splice mode the graph (captured with ``keep_graph=True``) is added as a +child-graph node at the capturing stream's current dependencies and becomes +the stream's only dependency, so work captured afterwards is ordered after it +(the ``cudaStreamGetCaptureInfo`` / ``cudaStreamUpdateCaptureDependencies`` +pattern). CUDA clones the child when it is added, so the original graph stays +valid for regular replays. In count mode the graph is launched as usual and +recorded, together with how far it advanced the CUDA generator, which lets a +caller check what a region replays before capturing it. + +The context also carries named capture overrides: a function decorated with +:func:`capture_override` calls the handler that the active capture registered +under that name, and runs unchanged otherwise. +""" + +from __future__ import annotations + +import functools +from contextvars import ContextVar +from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar + +import torch + +try: + from cuda.bindings import runtime as rt +except ImportError: + rt = None + +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.cuda_utils import ( + checkCudaErrors, +) +from sglang.srt.model_executor.runner_utils.pool import graph_pool_replay_scope + +__all__ = [ + "ChildGraphCapture", + "capture_override", + "current_child_graph_capture", + "splice_graph_into_capture", +] + +_F = TypeVar("_F", bound=Callable[..., Any]) + +_current_capture_var: ContextVar[Optional[ChildGraphCapture]] = ContextVar( + "child_graph_capture", default=None +) + + +def current_child_graph_capture() -> Optional[ChildGraphCapture]: + return _current_capture_var.get() + + +def splice_graph_into_capture(graph: torch.cuda.CUDAGraph, stream: Any) -> None: + """Add ``graph`` (captured with ``keep_graph=True``) as a child-graph node + of the capture that is active on ``stream``.""" + if rt is None: + raise RuntimeError( + "Splicing CUDA graphs requires the 'cuda-python' package. " + "Install it with: pip install cuda-python" + ) + stream_ptr = stream.cuda_stream + status, _, capturing_graph, deps, _, num_deps = checkCudaErrors( + rt.cudaStreamGetCaptureInfo(stream_ptr) + ) + if status != rt.cudaStreamCaptureStatus.cudaStreamCaptureStatusActive: + raise RuntimeError("The current stream is not capturing a CUDA graph") + # Raises unless the graph was created with keep_graph=True. + child = rt.cudaGraph_t(init_value=graph.raw_cuda_graph()) + deps = list(deps)[:num_deps] + node = checkCudaErrors( + rt.cudaGraphAddChildGraphNode(capturing_graph, deps, len(deps), child) + ) + checkCudaErrors( + rt.cudaStreamUpdateCaptureDependencies( + stream_ptr, + [node], + None, + 1, + rt.cudaStreamUpdateCaptureDependenciesFlags.cudaStreamSetCaptureDependencies, + ) + ) + + +class ChildGraphCapture: + """Active-capture context consulted by graph backends and by functions + decorated with :func:`capture_override`. + + Args: + splice: True to splice graphs into the capture on the current stream; + False to launch them as usual and record them. + overrides: handlers by override name. A handler is called as + ``handler(original_fn, *args, **kwargs)``. + """ + + def __init__( + self, + *, + splice: bool, + overrides: Optional[Dict[str, Callable[..., Any]]] = None, + ) -> None: + self.splice = splice + self.overrides: Dict[str, Callable[..., Any]] = dict(overrides or {}) + # (owner, owner.graph_epoch) per graph handed over, in call order. + self.graph_sources: List[Tuple[Any, Optional[int]]] = [] + # Count mode: CUDA generator offset advanced by each launch. + self.generator_offset_deltas: List[int] = [] + self._token = None + + @property + def num_graphs(self) -> int: + return len(self.graph_sources) + + def add_graph(self, graph: torch.cuda.CUDAGraph, owner: Any = None) -> None: + """Take over one replay of ``graph``: splice it into the capture, or + launch it and record it. ``owner`` (e.g. the graph backend) may carry a + ``graph_epoch`` that is recorded with it.""" + self.graph_sources.append((owner, getattr(owner, "graph_epoch", None))) + if self.splice: + splice_graph_into_capture(graph, torch.cuda.current_stream()) + return + generator = torch.cuda.default_generators[torch.cuda.current_device()] + offset = generator.get_offset() + with graph_pool_replay_scope(): + graph.replay() + self.generator_offset_deltas.append(generator.get_offset() - offset) + + def __enter__(self) -> ChildGraphCapture: + if _current_capture_var.get() is not None: + raise RuntimeError("ChildGraphCapture contexts cannot be nested") + self._token = _current_capture_var.set(self) + return self + + def __exit__(self, *args: object) -> bool: + _current_capture_var.reset(self._token) + self._token = None + return False + + +def capture_override(name: str) -> Callable[[_F], _F]: + """Let an active :class:`ChildGraphCapture` replace the decorated function. + + Outside such a capture, or when it registered no handler under ``name``, + the function runs unchanged. + """ + + def decorator(fn: _F) -> _F: + @functools.wraps(fn) + def wrapper(*args, **kwargs): + capture = _current_capture_var.get() + if capture is not None: + handler = capture.overrides.get(name) + if handler is not None: + return handler(fn, *args, **kwargs) + return fn(*args, **kwargs) + + return wrapper # type: ignore[return-value] + + return decorator + + +def around_override(name: str) -> Callable[..., Any]: + """:func:`capture_override` as a HookRegistry AROUND hook, for an SGLang + function that is not decorated with it.""" + + def around(original, *args, **kwargs): + capture = _current_capture_var.get() + if capture is not None: + handler = capture.overrides.get(name) + if handler is not None: + return handler(original, *args, **kwargs) + return original(*args, **kwargs) + + return around diff --git a/sgl_nvidia_mega/spec_step/csrc/include/sgl_nvidia_mega/spec_sampling.cuh b/sgl_nvidia_mega/spec_step/csrc/include/sgl_nvidia_mega/spec_sampling.cuh new file mode 100644 index 00000000..4c45bc7a --- /dev/null +++ b/sgl_nvidia_mega/spec_step/csrc/include/sgl_nvidia_mega/spec_sampling.cuh @@ -0,0 +1,512 @@ +/* + * Copyright (c) 2025 by SGLang team. + * Copyright (c) 2024-2025 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#pragma once + +#include + +namespace sglang { +namespace speculative_sampling { + +using namespace flashinfer; +using namespace flashinfer::sampling; +using namespace cub; + +template < + uint32_t BLOCK_THREADS, + BlockScanAlgorithm SCAN_ALGORITHM, + BlockReduceAlgorithm REDUCE_ALGORITHM, + uint32_t VEC_SIZE, + bool DETERMINISTIC, + typename DType, + typename IdType, + typename IdType2, + // Chains only (one candidate per tree level): draft_probs is not read or + // written. The walk remembers the one token it rejects, and the final + // sampling takes p = q there and p = 0 elsewhere, which is exactly what the + // zero-filled buffer would hold, so the outputs are the same bits. + bool NO_DRAFT_PROBS = false> +__global__ void TreeSpeculativeSamplingTargetOnly( + IdType* predicts, // mutable + IdType* accept_index, // mutable + IdType* accept_token_num, // mutable + IdType2* candidates, + IdType2* retrive_index, + IdType2* retrive_next_token, + IdType2* retrive_next_sibling, + DType* uniform_samples, + DType* uniform_samples_for_final_sampling, + DType* target_probs, + DType* draft_probs, + uint32_t batch_size, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens, + uint32_t d, + DType threshold_single, + DType threshold_acc) { + const uint32_t bx = blockIdx.x, tx = threadIdx.x; + + extern __shared__ __align__(alignof(SamplingTempStorage)) + uint8_t smem_sampling[]; + auto& temp_storage = + reinterpret_cast&>(smem_sampling); + + DType prob_acc = 0.0; + uint32_t cur_prob_offset = bx * num_draft_tokens * d; + DType coin = uniform_samples[bx * num_draft_tokens]; + IdType2 last_accepted_retrive_idx = retrive_index[bx * num_draft_tokens]; + accept_index[bx * num_speculative_tokens] = last_accepted_retrive_idx; + uint32_t num_accepted_tokens = 0; + IdType2 cur_index = 0; + // NO_DRAFT_PROBS: the token rejected in the current row, -1 if none. + IdType2 rejected_token_id = -1; + + for (uint32_t j = 1; j < num_speculative_tokens; ++j) { + cur_index = retrive_next_token[bx * num_draft_tokens + cur_index]; + while (cur_index != -1) { + IdType2 draft_index = retrive_index[bx * num_draft_tokens + cur_index]; + IdType2 draft_token_id = candidates[bx * num_draft_tokens + cur_index]; + DType target_prob_single = target_probs[cur_prob_offset + draft_token_id]; + prob_acc += target_prob_single; + + const bool has_target_mass = target_prob_single > DType(0); + if (has_target_mass && (coin < prob_acc / threshold_acc || target_prob_single >= threshold_single)) { + // accept token + prob_acc = 0.; + cur_prob_offset = (bx * num_draft_tokens + cur_index) * d; + coin = uniform_samples[bx * num_draft_tokens + cur_index]; + predicts[last_accepted_retrive_idx] = draft_token_id; + rejected_token_id = -1; + ++num_accepted_tokens; + accept_index[bx * num_speculative_tokens + num_accepted_tokens] = draft_index; + last_accepted_retrive_idx = draft_index; + break; + } else { + // FIXME: leverage draft probs + if constexpr (NO_DRAFT_PROBS) { + rejected_token_id = draft_token_id; + } else { + draft_probs[cur_prob_offset + draft_token_id] = target_probs[cur_prob_offset + draft_token_id]; + } + cur_index = retrive_next_sibling[bx * num_draft_tokens + cur_index]; + } + } + if (cur_index == -1) break; + } + accept_token_num[bx] = num_accepted_tokens; + + // we need a different coin for the final sampling + coin = uniform_samples_for_final_sampling[bx]; + + // sample from relu(target_probs - draft_probs) + DType sum_relu_q_minus_p(0); + vec_t q_vec, p_vec; + DType relu_q_minus_p[VEC_SIZE]; + for (uint32_t i = 0; i < ceil_div(d, BLOCK_THREADS * VEC_SIZE); ++i) { + q_vec.fill(DType(0)); + p_vec.fill(DType(0)); + if ((i * BLOCK_THREADS + tx) * VEC_SIZE < d) { + q_vec.load(target_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + if (num_accepted_tokens != num_speculative_tokens - 1) { + // there is no draft_probs for the bonus token + if constexpr (NO_DRAFT_PROBS) { +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + const IdType2 token = static_cast((i * BLOCK_THREADS + tx) * VEC_SIZE + j); + if (token == rejected_token_id) p_vec[j] = q_vec[j]; + } + } else { + p_vec.load(draft_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + } + } + } +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + relu_q_minus_p[j] = max(q_vec[j] - p_vec[j], DType(0)); + } + sum_relu_q_minus_p += BlockReduce(temp_storage.block_prim.reduce) + .Sum(relu_q_minus_p); + __syncthreads(); + } + if (tx == 0) { + temp_storage.block_aggregate.value = sum_relu_q_minus_p; + } + + temp_storage.sampled_id = d; + temp_storage.last_valid_id = -1; + __syncthreads(); + sum_relu_q_minus_p = temp_storage.block_aggregate.value; + DType u = coin * sum_relu_q_minus_p; + + DType aggregate_relu_q_minus_p(0); + for (uint32_t i = 0; i < ceil_div(d, BLOCK_THREADS * VEC_SIZE); ++i) { + q_vec.fill(DType(0)); + p_vec.fill(DType(0)); + if ((i * BLOCK_THREADS + tx) * VEC_SIZE < d) { + q_vec.load(target_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + if (num_accepted_tokens != num_speculative_tokens - 1) { + // there is no draft_probs for the bonus token + if constexpr (NO_DRAFT_PROBS) { +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + const IdType2 token = static_cast((i * BLOCK_THREADS + tx) * VEC_SIZE + j); + if (token == rejected_token_id) p_vec[j] = q_vec[j]; + } + } else { + p_vec.load(draft_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + } + } + } + + vec_t relu_q_minus_p_vec; +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + relu_q_minus_p_vec[j] = max(q_vec[j] - p_vec[j], DType(0)); + } + + DeviceSamplingFromProb( + i, d, [&](DType x) { return x > 0; }, u, relu_q_minus_p_vec, aggregate_relu_q_minus_p, &temp_storage); + if (aggregate_relu_q_minus_p > u) { + break; + } + } + __syncthreads(); + // This would happen when u is very close to 1 + // and the sum of probabilities is smaller than u + // In this case, we use the last valid index as the sampled id + int sampled_id = temp_storage.sampled_id; + if (sampled_id == d) { + if (temp_storage.last_valid_id == -1) { + sampled_id = d - 1; + } else { + sampled_id = temp_storage.last_valid_id; + } + } + // set the first rejected token + predicts[last_accepted_retrive_idx] = sampled_id; + // value at not used indices are undefined +} + +// --------------------------------------------------------------------------- +// Chain verify sampling in two launches (NO_DRAFT_PROBS form only). +// +// TreeSpeculativeSamplingTargetOnly<..., NO_DRAFT_PROBS = true> runs one CTA per +// request: a short accept walk, then two sequential passes over the +// vocabulary (the relu(q - p) total, then the inverse-CDF search). The two +// kernels below split the passes over one CTA per vocabulary chunk and give the +// same bits: +// * ChainVerifyChunkStats: per (request, chunk) the same BlockReduce sums the +// single-CTA kernel computes for that chunk (the total pass's sum of relu +// values, and DeviceSamplingFromProb's sum of the values > 0) and the +// chunk's last valid index (DeviceSamplingFromProb's max-reduce). +// * ChainVerifyFinish: per request the same walk and outputs; thread 0 adds +// the chunk sums in chunk order (the single-CTA kernel's thread-0 running +// sum), finds the chunk where the running aggregate first exceeds u, and the +// CTA runs DeviceSamplingFromProb on that chunk with the aggregate of the +// chunks before it. last_valid_id is the last valid index of the chunks the +// single-CTA loop would have visited. +// --------------------------------------------------------------------------- + +template +struct ChainWalk { + uint32_t cur_prob_offset; + uint32_t num_accepted_tokens; + IdType2 rejected_token_id; + IdType2 last_accepted_retrive_idx; +}; + +// The accept walk of TreeSpeculativeSamplingTargetOnly (NO_DRAFT_PROBS), same +// statements; WRITE selects whether predicts / accept_index are written. +template +__device__ __forceinline__ ChainWalk chain_verify_walk( + IdType* predicts, + IdType* accept_index, + IdType2* candidates, + IdType2* retrive_index, + IdType2* retrive_next_token, + IdType2* retrive_next_sibling, + DType* uniform_samples, + DType* target_probs, + uint32_t bx, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens, + uint32_t d, + DType threshold_single, + DType threshold_acc) { + DType prob_acc = 0.0; + uint32_t cur_prob_offset = bx * num_draft_tokens * d; + DType coin = uniform_samples[bx * num_draft_tokens]; + IdType2 last_accepted_retrive_idx = retrive_index[bx * num_draft_tokens]; + if constexpr (WRITE) accept_index[bx * num_speculative_tokens] = last_accepted_retrive_idx; + uint32_t num_accepted_tokens = 0; + IdType2 cur_index = 0; + IdType2 rejected_token_id = -1; + + for (uint32_t j = 1; j < num_speculative_tokens; ++j) { + cur_index = retrive_next_token[bx * num_draft_tokens + cur_index]; + while (cur_index != -1) { + IdType2 draft_index = retrive_index[bx * num_draft_tokens + cur_index]; + IdType2 draft_token_id = candidates[bx * num_draft_tokens + cur_index]; + DType target_prob_single = target_probs[cur_prob_offset + draft_token_id]; + prob_acc += target_prob_single; + + const bool has_target_mass = target_prob_single > DType(0); + if (has_target_mass && (coin < prob_acc / threshold_acc || target_prob_single >= threshold_single)) { + prob_acc = 0.; + cur_prob_offset = (bx * num_draft_tokens + cur_index) * d; + coin = uniform_samples[bx * num_draft_tokens + cur_index]; + if constexpr (WRITE) predicts[last_accepted_retrive_idx] = draft_token_id; + rejected_token_id = -1; + ++num_accepted_tokens; + if constexpr (WRITE) accept_index[bx * num_speculative_tokens + num_accepted_tokens] = draft_index; + last_accepted_retrive_idx = draft_index; + break; + } else { + rejected_token_id = draft_token_id; + cur_index = retrive_next_sibling[bx * num_draft_tokens + cur_index]; + } + } + if (cur_index == -1) break; + } + return {cur_prob_offset, num_accepted_tokens, rejected_token_id, last_accepted_retrive_idx}; +} + +// relu(q - p) of chunk i for this thread, as the single-CTA kernel loads it. +template +__device__ __forceinline__ void chain_verify_relu( + vec_t& out, + DType* target_probs, + const ChainWalk& w, + uint32_t i, + uint32_t d, + uint32_t num_speculative_tokens) { + const uint32_t tx = threadIdx.x; + vec_t q_vec, p_vec; + q_vec.fill(DType(0)); + p_vec.fill(DType(0)); + if ((i * BLOCK_THREADS + tx) * VEC_SIZE < d) { + q_vec.load(target_probs + w.cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + if (w.num_accepted_tokens != num_speculative_tokens - 1) { +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + const IdType2 token = static_cast((i * BLOCK_THREADS + tx) * VEC_SIZE + j); + if (token == w.rejected_token_id) p_vec[j] = q_vec[j]; + } + } + } +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + out[j] = max(q_vec[j] - p_vec[j], DType(0)); + } +} + +template < + uint32_t BLOCK_THREADS, + BlockScanAlgorithm SCAN_ALGORITHM, + BlockReduceAlgorithm REDUCE_ALGORITHM, + uint32_t VEC_SIZE, + typename DType, + typename IdType, + typename IdType2> +__global__ void ChainVerifyChunkStats( + IdType2* candidates, + IdType2* retrive_index, + IdType2* retrive_next_token, + IdType2* retrive_next_sibling, + DType* uniform_samples, + DType* target_probs, + DType* chunk_relu_sum, // [batch, num_chunks] + DType* chunk_agg_sum, // [batch, num_chunks] + int* chunk_last_valid, // [batch, num_chunks] + uint32_t num_chunks, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens, + uint32_t d, + DType threshold_single, + DType threshold_acc) { + const uint32_t i = blockIdx.x, bx = blockIdx.y, tx = threadIdx.x; + extern __shared__ __align__(alignof(SamplingTempStorage)) + uint8_t smem_sampling[]; + auto& temp_storage = + reinterpret_cast&>(smem_sampling); + + const auto w = chain_verify_walk( + nullptr, + nullptr, + candidates, + retrive_index, + retrive_next_token, + retrive_next_sibling, + uniform_samples, + target_probs, + bx, + num_speculative_tokens, + num_draft_tokens, + d, + threshold_single, + threshold_acc); + + vec_t relu_vec; + chain_verify_relu(relu_vec, target_probs, w, i, d, num_speculative_tokens); + DType relu[VEC_SIZE], gt[VEC_SIZE]; + int valid_index[VEC_SIZE]; +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + relu[j] = relu_vec[j]; + gt[j] = relu_vec[j] > 0 ? relu_vec[j] : 0; + const uint32_t idx = (i * BLOCK_THREADS + tx) * VEC_SIZE + j; + valid_index[j] = (relu_vec[j] > 0 && idx < d) ? static_cast(idx) : -1; + } + // The total pass's sum (single-CTA kernel, first loop). + DType s_relu = + BlockReduce(temp_storage.block_prim.reduce).Sum(relu); + __syncthreads(); + // DeviceSamplingFromProb's chunk aggregate and last valid index. + DType s_agg = BlockReduce(temp_storage.block_prim.reduce).Sum(gt); + __syncthreads(); + int max_valid = BlockReduce(temp_storage.block_prim.reduce_int) + .Reduce(valid_index, MaxReduceOp{}); + if (tx == 0) { + chunk_relu_sum[bx * num_chunks + i] = s_relu; + chunk_agg_sum[bx * num_chunks + i] = s_agg; + chunk_last_valid[bx * num_chunks + i] = max_valid; + } +} + +template < + uint32_t BLOCK_THREADS, + BlockScanAlgorithm SCAN_ALGORITHM, + BlockReduceAlgorithm REDUCE_ALGORITHM, + uint32_t VEC_SIZE, + bool DETERMINISTIC, + typename DType, + typename IdType, + typename IdType2> +__global__ void ChainVerifyFinish( + IdType* predicts, // mutable + IdType* accept_index, // mutable + IdType* accept_token_num, // mutable + IdType2* candidates, + IdType2* retrive_index, + IdType2* retrive_next_token, + IdType2* retrive_next_sibling, + DType* uniform_samples, + DType* uniform_samples_for_final_sampling, + DType* target_probs, + const DType* chunk_relu_sum, + const DType* chunk_agg_sum, + const int* chunk_last_valid, + uint32_t num_chunks, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens, + uint32_t d, + DType threshold_single, + DType threshold_acc) { + const uint32_t bx = blockIdx.x, tx = threadIdx.x; + extern __shared__ __align__(alignof(SamplingTempStorage)) + uint8_t smem_sampling[]; + auto& temp_storage = + reinterpret_cast&>(smem_sampling); + __shared__ uint32_t s_chunk; + __shared__ DType s_prefix, s_u; + // The per-chunk values, loaded by all threads at once so that thread 0's + // ordered accumulation below reads shared memory (num_chunks <= BLOCK_THREADS). + __shared__ DType s_relu_sum[BLOCK_THREADS], s_agg_sum[BLOCK_THREADS]; + __shared__ int s_last_valid[BLOCK_THREADS]; + + const auto w = chain_verify_walk( + predicts, + accept_index, + candidates, + retrive_index, + retrive_next_token, + retrive_next_sibling, + uniform_samples, + target_probs, + bx, + num_speculative_tokens, + num_draft_tokens, + d, + threshold_single, + threshold_acc); + accept_token_num[bx] = w.num_accepted_tokens; + + const DType coin = uniform_samples_for_final_sampling[bx]; + for (uint32_t i = tx; i < num_chunks; i += BLOCK_THREADS) { + s_relu_sum[i] = chunk_relu_sum[bx * num_chunks + i]; + s_agg_sum[i] = chunk_agg_sum[bx * num_chunks + i]; + s_last_valid[i] = chunk_last_valid[bx * num_chunks + i]; + } + __syncthreads(); + if (tx == 0) { + // The single-CTA kernel's thread-0 running sum, chunk by chunk. + DType sum_relu_q_minus_p(0); + for (uint32_t i = 0; i < num_chunks; ++i) + sum_relu_q_minus_p += s_relu_sum[i]; + const DType u = coin * sum_relu_q_minus_p; + // DeviceSamplingFromProb's aggregate: the first chunk where it exceeds u + // is where the single-CTA loop scans and stops. + DType aggregate(0); + uint32_t hit = num_chunks; + for (uint32_t i = 0; i < num_chunks; ++i) { + const DType local = s_agg_sum[i]; + if (aggregate + local > u) { + hit = i; + break; + } + aggregate += local; + } + // The last valid index among the chunks the loop visits before the hit + // chunk (the hit chunk's own update happens in DeviceSamplingFromProb). + int last_valid = -1; + const uint32_t visited = hit < num_chunks ? hit : num_chunks; + for (uint32_t i = 0; i < visited; ++i) { + const int v = s_last_valid[i]; + if (v != -1 && v < static_cast(d)) last_valid = v; + } + s_u = u; + temp_storage.sampled_id = d; + temp_storage.last_valid_id = last_valid; + s_chunk = hit; + s_prefix = aggregate; + } + __syncthreads(); + const DType u = s_u; + const uint32_t hit = s_chunk; + if (hit < num_chunks) { + DType aggregate_relu_q_minus_p = s_prefix; + vec_t relu_vec; + chain_verify_relu( + relu_vec, target_probs, w, hit, d, num_speculative_tokens); + DeviceSamplingFromProb( + hit, d, [&](DType x) { return x > 0; }, u, relu_vec, aggregate_relu_q_minus_p, &temp_storage); + } + __syncthreads(); + int sampled_id = temp_storage.sampled_id; + if (sampled_id == d) { + if (temp_storage.last_valid_id == -1) { + sampled_id = d - 1; + } else { + sampled_id = temp_storage.last_valid_id; + } + } + predicts[w.last_accepted_retrive_idx] = sampled_id; +} + +} // namespace speculative_sampling +} // namespace sglang diff --git a/sgl_nvidia_mega/spec_step/csrc/sampling.cuh b/sgl_nvidia_mega/spec_step/csrc/sampling.cuh new file mode 100644 index 00000000..863af128 --- /dev/null +++ b/sgl_nvidia_mega/spec_step/csrc/sampling.cuh @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2025 by SGLang team. + * Copyright (c) 2024-2025 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include +#include + +#include + +#include + +#include + +namespace sglang { + +/// \brief Sample and verify a draft tree using only target probabilities. +/// NoDraftProbs (chains only): draft_probs is neither read nor written; pass +/// any float CUDA tensor in its place (see the kernel's NO_DRAFT_PROBS). +template +void tree_speculative_sampling_target_only( + tvm::ffi::TensorView predicts, + tvm::ffi::TensorView accept_index, + tvm::ffi::TensorView accept_token_num, + tvm::ffi::TensorView candidates, + tvm::ffi::TensorView retrive_index, + tvm::ffi::TensorView retrive_next_token, + tvm::ffi::TensorView retrive_next_sibling, + tvm::ffi::TensorView uniform_samples, + tvm::ffi::TensorView uniform_samples_for_final_sampling, + tvm::ffi::TensorView target_probs, + tvm::ffi::TensorView draft_probs, + double threshold_single, + double threshold_acc) { + using namespace host; + using namespace flashinfer::sampling; + static_assert(VecSize == 1 || VecSize == 2 || VecSize == 4); + SymbolicSize batch_size{"batch_size"}, draft_tokens{"draft_tokens"}, spec_tokens{"spec_tokens"}, + vocab_size{"vocab_size"}; + SymbolicDevice device; + TensorMatcher({batch_size, draft_tokens}) + .with_dtype() + .with_device(device) + .verify(candidates) + .verify(retrive_index) + .verify(retrive_next_token) + .verify(retrive_next_sibling); + TensorMatcher({batch_size, spec_tokens}).with_dtype().with_device(device).verify(accept_index); + TensorMatcher({batch_size}).with_dtype().with_device(device).verify(accept_token_num); + TensorMatcher({batch_size.unwrap() * draft_tokens.unwrap()}) + .with_dtype() + .with_device(device) + .verify(predicts); + TensorMatcher({batch_size, draft_tokens}).with_dtype().with_device(device).verify(uniform_samples); + TensorMatcher({batch_size}).with_dtype().with_device(device).verify(uniform_samples_for_final_sampling); + TensorMatcher({batch_size, draft_tokens, vocab_size}).with_dtype().with_device(device).verify(target_probs); + if constexpr (!NoDraftProbs) { + TensorMatcher({batch_size, draft_tokens, vocab_size}) + .with_dtype() + .with_device(device) + .verify(draft_probs); + } + CHECK_HOST(draft_tokens.unwrap() > 0 && spec_tokens.unwrap() > 0 && vocab_size.unwrap() > 0); + CHECK_HOST(std::gcd(int64_t{4}, vocab_size.unwrap()) == VecSize); + CHECK_HOST(threshold_single >= 0 && threshold_single <= 1); + CHECK_HOST(threshold_acc >= 0 && threshold_acc <= 1); + if (batch_size.unwrap() == 0) return; + + constexpr uint32_t block_threads = 1024; + constexpr size_t smem_size = sizeof(SamplingTempStorage); + constexpr auto kernel = speculative_sampling::TreeSpeculativeSamplingTargetOnly< + block_threads, + SCAN_ALGO, + REDUCE_ALGO, + VecSize, + Deterministic, + float, + int32_t, + int64_t, + NoDraftProbs>; + runtime::set_max_dynamic_smem_per_device(device.unwrap().device_id, smem_size); + LaunchKernel(static_cast(batch_size.unwrap()), block_threads, device.unwrap(), smem_size)( + kernel, + static_cast(predicts.data_ptr()), + static_cast(accept_index.data_ptr()), + static_cast(accept_token_num.data_ptr()), + static_cast(candidates.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + static_cast(uniform_samples.data_ptr()), + static_cast(uniform_samples_for_final_sampling.data_ptr()), + static_cast(target_probs.data_ptr()), + NoDraftProbs ? nullptr : static_cast(draft_probs.data_ptr()), + static_cast(batch_size.unwrap()), + static_cast(spec_tokens.unwrap()), + static_cast(draft_tokens.unwrap()), + static_cast(vocab_size.unwrap()), + static_cast(threshold_single), + std::max(static_cast(threshold_acc), 1e-9f)); +} + +/// \brief The NoDraftProbs form of tree_speculative_sampling_target_only in two +/// launches (chains only), same outputs bit for bit: per-chunk statistics with +/// one CTA per vocabulary chunk, then one CTA per request for the walk and the +/// final sample. chunk_* are [batch_size, num_chunks] workspaces with +/// num_chunks = ceil(vocab_size / (1024 * VecSize)). +template +void chain_speculative_sampling_target_only_two_pass( + tvm::ffi::TensorView predicts, + tvm::ffi::TensorView accept_index, + tvm::ffi::TensorView accept_token_num, + tvm::ffi::TensorView candidates, + tvm::ffi::TensorView retrive_index, + tvm::ffi::TensorView retrive_next_token, + tvm::ffi::TensorView retrive_next_sibling, + tvm::ffi::TensorView uniform_samples, + tvm::ffi::TensorView uniform_samples_for_final_sampling, + tvm::ffi::TensorView target_probs, + tvm::ffi::TensorView chunk_relu_sum, + tvm::ffi::TensorView chunk_agg_sum, + tvm::ffi::TensorView chunk_last_valid, + double threshold_single, + double threshold_acc) { + using namespace host; + using namespace flashinfer::sampling; + static_assert(VecSize == 1 || VecSize == 2 || VecSize == 4); + SymbolicSize batch_size{"batch_size"}, draft_tokens{"draft_tokens"}, spec_tokens{"spec_tokens"}, + vocab_size{"vocab_size"}, num_chunks{"num_chunks"}; + SymbolicDevice device; + TensorMatcher({batch_size, draft_tokens}) + .with_dtype() + .with_device(device) + .verify(candidates) + .verify(retrive_index) + .verify(retrive_next_token) + .verify(retrive_next_sibling); + TensorMatcher({batch_size, spec_tokens}).with_dtype().with_device(device).verify(accept_index); + TensorMatcher({batch_size}).with_dtype().with_device(device).verify(accept_token_num); + TensorMatcher({batch_size.unwrap() * draft_tokens.unwrap()}) + .with_dtype() + .with_device(device) + .verify(predicts); + TensorMatcher({batch_size, draft_tokens}).with_dtype().with_device(device).verify(uniform_samples); + TensorMatcher({batch_size}).with_dtype().with_device(device).verify(uniform_samples_for_final_sampling); + TensorMatcher({batch_size, draft_tokens, vocab_size}).with_dtype().with_device(device).verify(target_probs); + TensorMatcher({batch_size, num_chunks}) + .with_dtype() + .with_device(device) + .verify(chunk_relu_sum) + .verify(chunk_agg_sum); + TensorMatcher({batch_size, num_chunks}).with_dtype().with_device(device).verify(chunk_last_valid); + CHECK_HOST(draft_tokens.unwrap() > 0 && spec_tokens.unwrap() > 0 && vocab_size.unwrap() > 0); + CHECK_HOST(std::gcd(int64_t{4}, vocab_size.unwrap()) == VecSize); + CHECK_HOST(threshold_single >= 0 && threshold_single <= 1); + CHECK_HOST(threshold_acc >= 0 && threshold_acc <= 1); + constexpr uint32_t block_threads = 1024; + CHECK_HOST(num_chunks.unwrap() == (vocab_size.unwrap() + block_threads * VecSize - 1) / (block_threads * VecSize)); + CHECK_HOST(num_chunks.unwrap() <= block_threads); + if (batch_size.unwrap() == 0) return; + + constexpr size_t smem_size = sizeof(SamplingTempStorage); + auto stats = speculative_sampling:: + ChainVerifyChunkStats; + auto finish = speculative_sampling:: + ChainVerifyFinish; + CHECK_CUDA(cudaFuncSetAttribute(stats, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + CHECK_CUDA(cudaFuncSetAttribute(finish, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + const float th_single = static_cast(threshold_single); + const float th_acc = std::max(static_cast(threshold_acc), 1e-9f); + LaunchKernel( + dim3(static_cast(num_chunks.unwrap()), static_cast(batch_size.unwrap())), + block_threads, + device.unwrap(), + smem_size)( + stats, + static_cast(candidates.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + static_cast(uniform_samples.data_ptr()), + static_cast(target_probs.data_ptr()), + static_cast(chunk_relu_sum.data_ptr()), + static_cast(chunk_agg_sum.data_ptr()), + static_cast(chunk_last_valid.data_ptr()), + static_cast(num_chunks.unwrap()), + static_cast(spec_tokens.unwrap()), + static_cast(draft_tokens.unwrap()), + static_cast(vocab_size.unwrap()), + th_single, + th_acc); + LaunchKernel(static_cast(batch_size.unwrap()), block_threads, device.unwrap(), smem_size)( + finish, + static_cast(predicts.data_ptr()), + static_cast(accept_index.data_ptr()), + static_cast(accept_token_num.data_ptr()), + static_cast(candidates.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + static_cast(uniform_samples.data_ptr()), + static_cast(uniform_samples_for_final_sampling.data_ptr()), + static_cast(target_probs.data_ptr()), + static_cast(chunk_relu_sum.data_ptr()), + static_cast(chunk_agg_sum.data_ptr()), + static_cast(chunk_last_valid.data_ptr()), + static_cast(num_chunks.unwrap()), + static_cast(spec_tokens.unwrap()), + static_cast(draft_tokens.unwrap()), + static_cast(vocab_size.unwrap()), + th_single, + th_acc); +} + +} // namespace sglang diff --git a/sgl_nvidia_mega/spec_step/hooks.py b/sgl_nvidia_mega/spec_step/hooks.py new file mode 100644 index 00000000..50e8d82d --- /dev/null +++ b/sgl_nvidia_mega/spec_step/hooks.py @@ -0,0 +1,422 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The speculative step graph and fusions, installed into SGLang as HookRegistry +hooks. ``register`` adds only the hooks of the switches that are on.""" + +from __future__ import annotations + +import logging +import types + +import torch + +from sgl_nvidia_mega.spec_step import switches + +logger = logging.getLogger(__name__) + +_BACKEND = "sglang.srt.model_executor.runner_backend.full_cuda_graph_backend.FullCudaGraphBackend" +_WORKER = "sglang.srt.speculative.eagle_worker_v2.EAGLEWorkerV2" +_DRAFT_WORKER = "sglang.srt.speculative.eagle_worker_v2.EagleDraftWorker" +_FUTURE_MAP = "sglang.srt.managers.overlap_utils.FutureMap" +_SPEC_UTILS = "sglang.srt.speculative.spec_utils" + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/model_executor/runner_backend/full_cuda_graph_backend.py": ( + "FullCudaGraphBackend", + "capture_one(shape_key)", + "replay(shape_key)", + "cleanup", + "_graphs", + "_outputs", + "torch.cuda.CUDAGraph()", + ), + "srt/model_executor/runner_backend/utils.py": ("resolve_decode_backend",), + "srt/speculative/eagle_worker_v2.py": ( + "EAGLEWorkerV2", + "init_cuda_graphs", + "forward_batch_generation", + "activate_step_by_batch", + "_hosts_draft", + "server_args", + "EagleDraftWorker", + "_draft_extend_for_decode", + ), + "srt/speculative/eagle_utils.py": ("eagle_sample",), + "srt/speculative/spec_utils.py": ( + "generate_simulated_accept_index", + "GrammarTree.from_device", + "build_grammar_vocab_mask", + ), + "kernels/ops/speculative/sampling.py": ("tree_speculative_sampling_target_only",), + "srt/managers/overlap_utils.py": ( + "FutureMap", + "_resolve_spec_extras", + "stash", + "_maybe_init_forward_bufs", + "_maybe_init_dsa_topk_indices_buf", + "_DEBUG_ASSERT", + ), + "srt/managers/scheduler_components/internal_state.py": ( + "SchedulerInternalStateController._add_speculative", + ), +} + + +# ------------------------------------------------- per-stage graph backend + + +def after_backend_init(result, self, *args, **kwargs): + self.keep_raw_graph = False + # Bumped whenever this backend's graphs are captured or released, so a step + # graph that spliced them as children can tell that it is stale. + self.graph_epoch = 0 + + +def after_resolve_decode_backend(result, *args, **kwargs): + from sglang.srt.model_executor.runner_backend.full_cuda_graph_backend import ( + FullCudaGraphBackend, + ) + + # The step graph splices the decode-path graphs into its own. + if isinstance(result, FullCudaGraphBackend): + result.keep_raw_graph = True + + +def _keep_graph_class(base): + class KeepGraph(base): + def __new__(cls, keep_graph=False, *args, **kwargs): + return super().__new__(cls, True, *args, **kwargs) + + return KeepGraph + + +def around_capture_one(original, self, shape_key, *args, **kwargs): + if not getattr(self, "keep_raw_graph", False): + original(self, shape_key, *args, **kwargs) + self.graph_epoch = getattr(self, "graph_epoch", 0) + 1 + return + base = torch.cuda.CUDAGraph + torch.cuda.CUDAGraph = _keep_graph_class(base) + try: + original(self, shape_key, *args, **kwargs) + finally: + torch.cuda.CUDAGraph = base + self._graphs[shape_key].instantiate() + self.graph_epoch += 1 + + +def around_replay(original, self, shape_key, *args, **kwargs): + from sgl_nvidia_mega.spec_step.child_graph_capture import ( + current_child_graph_capture, + ) + + capture = current_child_graph_capture() + if capture is not None: + capture.add_graph(self._graphs[shape_key], owner=self) + return self._outputs[shape_key] + return original(self, shape_key, *args, **kwargs) + + +def after_cleanup(result, self, *args, **kwargs): + self.graph_epoch = getattr(self, "graph_epoch", 0) + 1 + + +# ------------------------------------------------------------- the worker + + +def after_init_cuda_graphs(result, self, *args, **kwargs): + from sgl_nvidia_mega.spec_step.runner import ( + EagleStepCudaGraphRunner, + step_cuda_graph_unsupported_reason, + ) + + self.step_cuda_graph_runner = None + if not self._hosts_draft: + return + reason = step_cuda_graph_unsupported_reason(self.server_args) + if reason is not None: + logger.warning( + "%s disabled: %s. Falling back to the per-stage CUDA graphs.", + switches.STEP_GRAPH, + reason, + ) + return + self.step_cuda_graph_runner = EagleStepCudaGraphRunner(self) + + +def around_forward_batch_generation( + original, self, batch, on_publish=None, grammar_barrier=None, pp_proxy_tensors=None +): + runner = getattr(self, "step_cuda_graph_runner", None) + if runner is not None and not ( + batch.forward_mode.is_extend() or batch.is_extend_in_batch + ): + self.activate_step_by_batch(batch.seq_lens.shape[0]) + out = runner.try_replay(batch, on_publish, grammar_barrier, pp_proxy_tensors) + if out is not None: + return out + return original(self, batch, on_publish, grammar_barrier, pp_proxy_tensors) + + +def after_add_speculative(result, self, state, *args, **kwargs): + worker = getattr(self.scheduler, "draft_worker", None) + runner = getattr(worker, "step_cuda_graph_runner", None) + if runner is not None: + state["spec_step_cuda_graph"] = runner.stats() + + +# -------------------------------------------------------------- the relay + + +_fused_relay_logged = set() + + +def _log_fused_relay_once(part: str) -> None: + if part not in _fused_relay_logged: + _fused_relay_logged.add(part) + logger.info("%s on: %s", switches.FUSED_RELAY, part) + + +def _resolve_single_request_views(self, batch, indices) -> bool: + """A single request whose future indices are the batch's own + req_pool_indices resolves its draft inputs as row views of the relay + buffers instead of gathering copies. The rows are written back only by this + request's next stash, which follows its forward on the same stream.""" + from sglang.srt.managers import overlap_utils + + if ( + not indices.is_cuda + or indices.shape[0] != 1 + or indices is not batch.req_pool_indices + or len(batch.reqs) != 1 + or overlap_utils._DEBUG_ASSERT + ): + return False + draft_input = batch.spec_info + row = batch.reqs[0].kv.req_pool_idx + draft_input.bonus_tokens = self.output_tokens_buf.narrow(0, row, 1) + if self.need_topk: + draft_input.topk_p = self.topk_p_buf.narrow(0, row, 1) + draft_input.topk_index = self.topk_index_buf.narrow(0, row, 1) + if self.need_hidden_states: + draft_input.hidden_states = self.hidden_states_buf.narrow(0, row, 1) + if self.draft_probs_buf is not None: + draft_input.draft_probs = self.draft_probs_buf.narrow(0, row, 1) + elif self.need_hidden_states: + draft_input.hidden_states = self.hidden_states_buf.narrow(0, row, 1) + if draft_input.future_dsa_topk_indices_available: + assert self.dsa_topk_indices_buf is not None + draft_input.dsa_topk_indices = self.dsa_topk_indices_buf.narrow(0, row, 1) + else: + draft_input.dsa_topk_indices = None + _log_fused_relay_once("views") + return True + + +def around_resolve_spec_extras(original, self, batch, *args, **kwargs): + draft_input = batch.spec_info + if not self.spec_algo.is_ngram() and draft_input is not None: + indices = getattr(draft_input, "future_indices", None) + if ( + indices is not None + and indices.shape[0] != 0 + and _resolve_single_request_views(self, batch, indices) + ): + return None + return original(self, batch, *args, **kwargs) + + +def around_stash(original, self, future_indices, payload, *args, **kwargs): + indices = future_indices + if indices.shape[0] == 0 or self.spec_algo.is_ngram() or not indices.is_cuda: + return original(self, future_indices, payload, *args, **kwargs) + from sgl_nvidia_mega.spec_step.ops import scatter_spec_extras + + self._maybe_init_forward_bufs(payload) + self._maybe_init_dsa_topk_indices_buf(payload) + # Every relay row in one scatter launch (same rows, same values; fields the + # payload leaves out keep the rows they already hold). + pairs = [(self.output_tokens_buf, payload.bonus_tokens)] + if self.need_topk and payload.topk_p is not None: + assert payload.topk_index is not None + pairs.append((self.topk_p_buf, payload.topk_p)) + pairs.append((self.topk_index_buf, payload.topk_index)) + if self.need_hidden_states and payload.hidden_states is not None: + pairs.append((self.hidden_states_buf, payload.hidden_states)) + if self.draft_probs_buf is not None and payload.draft_probs is not None: + pairs.append((self.draft_probs_buf, payload.draft_probs)) + if self.dsa_topk_indices_buf is not None and payload.dsa_topk_indices is not None: + pairs.append((self.dsa_topk_indices_buf, payload.dsa_topk_indices)) + scatter_spec_extras(indices, pairs) + _log_fused_relay_once("stash") + return None + + +# ------------------------------------------------- lifted SGLang functions + + +def _rebind(module, names): + """The ``lifted`` functions ``names`` with ``module``'s globals, added to it; + returns them by name.""" + from sgl_nvidia_mega.spec_step import lifted + + g = vars(module) + out = {} + for name in names: + value = getattr(lifted, name) + if isinstance(value, types.FunctionType): + value = types.FunctionType( + value.__code__, g, value.__name__, value.__defaults__, value.__closure__ + ) + value.__kwdefaults__ = getattr(lifted, name).__kwdefaults__ + elif isinstance(value, set): + value = set() + out[name] = value + if name not in ("eagle_sample", "_draft_extend_for_decode"): + g[name] = value + g.setdefault("_spec_switches", switches) + return out + + +def _eagle_sample(): + from sglang.srt.speculative import eagle_utils + + return _rebind( + eagle_utils, + ( + "_chunked_verify_sampling_logged", + "_log_chunked_verify_sampling_once", + "_alloc_verify_outputs", + "_broadcast_verify_outputs", + "eagle_sample", + ), + )["eagle_sample"] + + +def _draft_extend_for_decode(): + from sglang.srt.speculative import eagle_worker_v2 + + from sgl_nvidia_mega.spec_step.ops import draft_extend_tail + + vars(eagle_worker_v2).setdefault("draft_extend_tail", draft_extend_tail) + return _rebind( + eagle_worker_v2, + ( + "_draft_tail_logged", + "_log_draft_tail_once", + "_fused_draft_extend_tail_eligible", + "_draft_extend_for_decode", + ), + )["_draft_extend_for_decode"] + + +def _lazy(factory): + """A REPLACE hook that builds its function at the first call, so that + registering it imports nothing.""" + built = [] + + def hook(*args, **kwargs): + if not built: + built.append(factory()) + return built[0](*args, **kwargs) + + return hook + + +def _sampling(*args, **kwargs): + from sgl_nvidia_mega.spec_step.ops import tree_speculative_sampling_target_only + + return tree_speculative_sampling_target_only(*args, **kwargs) + + +# ------------------------------------------------------------- registration + + +def hooks(): + """(target, kind, hook) for the switches that are on.""" + from sgl_nvidia_mega.spec_step.child_graph_capture import around_override + + out = [] + if switches.enabled(switches.STEP_GRAPH): + out += [ + (f"{_BACKEND}.__init__", "AFTER", after_backend_init), + ( + "sglang.srt.model_executor.runner_backend.utils.resolve_decode_backend", + "AFTER", + after_resolve_decode_backend, + ), + (f"{_BACKEND}.capture_one", "AROUND", around_capture_one), + (f"{_BACKEND}.replay", "AROUND", around_replay), + (f"{_BACKEND}.cleanup", "AFTER", after_cleanup), + (f"{_WORKER}.init_cuda_graphs", "AFTER", after_init_cuda_graphs), + ( + f"{_WORKER}.forward_batch_generation", + "AROUND", + around_forward_batch_generation, + ), + ( + "sglang.srt.managers.scheduler_components.internal_state." + "SchedulerInternalStateController._add_speculative", + "AFTER", + after_add_speculative, + ), + ( + f"{_SPEC_UTILS}.generate_simulated_accept_index", + "AROUND", + around_override("spec_simulated_accept_index"), + ), + ( + f"{_SPEC_UTILS}.GrammarTree.from_device", + "AROUND", + around_override("spec_grammar_tree_from_device"), + ), + ( + f"{_SPEC_UTILS}.build_grammar_vocab_mask", + "AROUND", + around_override("spec_grammar_vocab_mask"), + ), + ] + if switches.enabled(switches.FUSED_RELAY): + out += [ + (f"{_FUTURE_MAP}._resolve_spec_extras", "AROUND", around_resolve_spec_extras), + (f"{_FUTURE_MAP}.stash", "AROUND", around_stash), + ] + if any( + switches.enabled(name) + for name in ( + switches.PACKED_VERIFY_BROADCAST, + switches.LEAN_SAMPLED_VERIFY, + switches.CHUNKED_VERIFY_SAMPLING, + ) + ): + out += [ + ("sglang.srt.speculative.eagle_utils.eagle_sample", "REPLACE", _lazy(_eagle_sample)), + ( + "sglang.kernels.ops.speculative.sampling.tree_speculative_sampling_target_only", + "REPLACE", + _sampling, + ), + ] + if switches.enabled(switches.FUSED_DRAFT_EXTEND_TAIL): + out.append( + ( + f"{_DRAFT_WORKER}._draft_extend_for_decode", + "REPLACE", + _lazy(_draft_extend_for_decode), + ) + ) + return out + + +def register() -> None: + from sglang.srt.plugins.hook_registry import HookRegistry, HookType + + installed = hooks() + for target, kind, hook in installed: + HookRegistry.register(target, hook, HookType[kind]) + if installed: + logger.info( + "sgl-nvidia-mega: speculative step hooks on (%s)", + ", ".join(name for name in switches.ALL if switches.enabled(name)), + ) diff --git a/sgl_nvidia_mega/spec_step/lifted.py b/sgl_nvidia_mega/spec_step/lifted.py new file mode 100644 index 00000000..319bbfdc --- /dev/null +++ b/sgl_nvidia_mega/spec_step/lifted.py @@ -0,0 +1,720 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# ruff: noqa: F821 +"""SGLang functions with the speculative step fusions applied, as they read in +SGLang's tree with them: ``eagle_utils.eagle_sample`` (the packed verify broadcast, +the lean and the chunked chain verify sampling) and +``EagleDraftWorker._draft_extend_for_decode`` (the fused topk-1 draft-extend tail). + +``hooks`` installs each with its SGLang module's globals (``rebind``), so their free +names resolve there exactly as in the originals; the helpers defined here are added +to that module under the same names. Keep each body in sync with its original: a +diff against SGLang's function shows only the fusion. +""" + +import logging + +from sgl_nvidia_mega.spec_step import switches as _spec_switches +from sgl_nvidia_mega.spec_step.ops import draft_extend_tail + +logger = logging.getLogger(__name__) + +# ---------------------------------------------------------------- eagle_utils + +_chunked_verify_sampling_logged = False + + +def _log_chunked_verify_sampling_once() -> None: + global _chunked_verify_sampling_logged + if not _chunked_verify_sampling_logged: + _chunked_verify_sampling_logged = True + logger.info( + "SGLANG_SPEC_CHUNKED_VERIFY_SAMPLING on: chain verify sampling in two " + "launches (one CTA per vocabulary chunk)." + ) + + +def _alloc_verify_outputs( + num_rows: int, bs: int, max_tree_depth: int, device +) -> Tuple[Optional[torch.Tensor], torch.Tensor, torch.Tensor, torch.Tensor]: + """Allocate verify's ``predict`` (zeros), ``accept_index`` (-1) and + ``num_correct_drafts``. + + With ``SGLANG_SPEC_PACKED_VERIFY_BROADCAST`` the three are views of one + int32 buffer, returned first, so that the TP broadcast of the verify result + is one collective (see ``_broadcast_verify_outputs``). Otherwise the first + value is None and the tensors are separate, as before. + """ + if not _spec_switches.enabled("SGLANG_SPEC_PACKED_VERIFY_BROADCAST"): + predict = torch.zeros((num_rows,), dtype=torch.int32, device=device) + accept_index = torch.full( + (bs, max_tree_depth), -1, dtype=torch.int32, device=device + ) + num_correct_drafts = torch.empty((bs,), dtype=torch.int32, device=device) + return None, predict, accept_index, num_correct_drafts + num_accept = bs * max_tree_depth + packed = torch.empty( + (num_rows + num_accept + bs,), dtype=torch.int32, device=device + ) + predict = packed[:num_rows] + predict.zero_() + accept_index = packed[num_rows : num_rows + num_accept].view(bs, max_tree_depth) + accept_index.fill_(-1) + num_correct_drafts = packed[num_rows + num_accept :] + return packed, predict, accept_index, num_correct_drafts + + +def _broadcast_verify_outputs( + tp_group, + packed: Optional[torch.Tensor], + predict: torch.Tensor, + accept_index: torch.Tensor, + num_correct_drafts: torch.Tensor, +) -> None: + """Rank 0's verify result to every rank of ``tp_group``: one collective on + the packed buffer when there is one, else one per tensor.""" + if tp_group.world_size <= 1: + return + if packed is not None: + tp_group.broadcast(packed, src=0) + return + tp_group.broadcast(predict, src=0) + tp_group.broadcast(accept_index, src=0) + tp_group.broadcast(num_correct_drafts, src=0) + + +def eagle_sample( + verify_input: EagleVerifyInput, + batch: ScheduleBatch, + logits_output: LogitsProcessorOutput, + grammar_mask: Optional[GrammarMask] = None, + uno_target_max_top_k: Optional[int] = None, + *, + watermark_state: Optional[WatermarkState] = None, + watermark_full_mask: bool = True, +): + """ + Verify and find accepted tokens based on logits output and batch + (which contains spec decoding information). + """ + from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, + ) + from sglang.srt.runtime_context import get_parallel + from sglang.srt.sampling.penaltylib.repetition_penalty import ( + apply_scaling_penalties, + ) + from sglang.srt.speculative.spec_utils import ( + SIMULATE_ACC_LEN, + SIMULATE_ACC_TOKEN_MODE, + generate_simulated_accept_index, + ) + from sglang.srt.utils.async_probe import maybe_detect_nan, sanitize_nan_logits + + device = batch.device + if batch.forward_mode.is_idle(): + predict = torch.empty(0, dtype=torch.int32, device=device) + num_correct_drafts = torch.empty(0, dtype=torch.int32, device=device) + accept_index = torch.empty(0, dtype=torch.int32, device=device) + return predict, num_correct_drafts, accept_index + + bs = len(batch.seq_lens) + sampling_info = batch.sampling_info + next_token_logits = logits_output.next_token_logits + + sanitize_nan_logits(next_token_logits, "verify: target model logits") + + # Apply penalty + # This is a relaxed version of penalties for speculative decoding. + if sampling_info.acc_additive_penalties is not None: + next_token_logits.add_( + torch.repeat_interleave( + sampling_info.acc_additive_penalties, + verify_input.draft_token_num, + dim=0, + ) + ) + if sampling_info.acc_scaling_penalties is not None: + apply_scaling_penalties( + next_token_logits, + torch.repeat_interleave( + sampling_info.acc_scaling_penalties, verify_input.draft_token_num, dim=0 + ), + ) + if sampling_info.logit_bias is not None: + next_token_logits.add_( + torch.repeat_interleave( + sampling_info.logit_bias, verify_input.draft_token_num, dim=0 + ) + ) + + # Apply grammar mask if provided + if grammar_mask is not None: + grammar_mask.apply(next_token_logits) + + watermark_context_hashes = None + watermark_selected = None + if watermark_state is not None and sampling_info.has_watermark_candidates: + if batch.return_logprob: + # compute_spec_logprobs reads logits_output after verify. + next_token_logits = next_token_logits.clone() + if next_token_logits.is_cuda: + watermark_context_hashes, watermark_selected = ( + watermark_state.force_speculative_from_tree( + logits=next_token_logits, + req_pool_indices=batch.req_pool_indices, + draft_tokens=verify_input.draft_token, + custom_mask=verify_input.custom_mask, + positions=verify_input.positions, + sampling_info=sampling_info, + draft_token_num=verify_input.draft_token_num, + full_mask=watermark_full_mask, + ) + ) + else: + contexts, context_lengths = watermark_state.speculative_contexts( + req_pool_indices=batch.req_pool_indices, + draft_tokens=verify_input.draft_token, + custom_mask=verify_input.custom_mask, + positions=verify_input.positions, + draft_token_num=verify_input.draft_token_num, + full_mask=watermark_full_mask, + context_windows=watermark_state.context_windows(sampling_info), + ) + watermark_context_hashes, watermark_selected = ( + watermark_state.force_speculative( + logits=next_token_logits, + req_pool_indices=batch.req_pool_indices, + contexts=contexts, + context_lengths=context_lengths, + sampling_info=sampling_info, + draft_token_num=verify_input.draft_token_num, + ) + ) + + candidates = verify_input.draft_token.reshape(bs, verify_input.draft_token_num) + verify_outputs, predict, accept_index, num_correct_drafts = _alloc_verify_outputs( + math.prod(next_token_logits.shape[:-1]), + bs, + verify_input.max_tree_depth, + device, + ) + + # Sample tokens + target_predict = None + use_rejection_sampling = get_spec().speculative_use_rejection_sampling + if _verify_uses_greedy( + is_all_greedy=sampling_info.is_all_greedy, + is_cpu=_is_cpu, + is_hip=_is_hip, + is_xpu=_is_xpu, + use_rejection_sampling=use_rejection_sampling, + ): + target_predict = torch.argmax(next_token_logits, dim=-1) + target_predict = target_predict.reshape(bs, verify_input.draft_token_num) + predict, accept_index, num_correct_drafts = verify_tree_greedy_func( + predicts=predict, # mutable + accept_index=accept_index, # mutable + accept_token_num=num_correct_drafts, # mutable + candidates=candidates, + retrieve_index=verify_input.retrieve_index, + retrieve_next_token=verify_input.retrieve_next_token, + retrieve_next_sibling=verify_input.retrieve_next_sibling, + target_predict=target_predict, + topk=verify_input.tree_topk, + ) + + if _is_hip: + # On ROCm, the per-rank draft tokens can differ, so ranks accept a + # different number of drafts, desynchronize the committed seq_lens, and + # deadlock the next TP collective. Broadcast from rank 0 to ensure + # consistency, the same way the sampling branch below does. + tp_group = ( + get_parallel().attn_tp_group + if is_dp_attention_enabled() + else get_parallel().tp_group + ) + _broadcast_verify_outputs( + tp_group, verify_outputs, predict, accept_index, num_correct_drafts + ) + elif _can_use_sparse_uno_tree_target_sampling( + uno_target_max_top_k, + sampling_info, + ): + from sglang.srt.speculative.uno_utils import ( + sample_uno_tree_target_tokens, + ) + + target_predict = sample_uno_tree_target_tokens( + next_token_logits=next_token_logits, + sampling_info=sampling_info, + batch_size=bs, + verify_width=verify_input.draft_token_num, + max_top_k=uno_target_max_top_k, + ) + predict, accept_index, num_correct_drafts = verify_tree_greedy_func( + predicts=predict, + accept_index=accept_index, + accept_token_num=num_correct_drafts, + candidates=candidates, + retrieve_index=verify_input.retrieve_index, + retrieve_next_token=verify_input.retrieve_next_token, + retrieve_next_sibling=verify_input.retrieve_next_sibling, + target_predict=target_predict, + topk=verify_input.tree_topk, + ) + + tp_group = ( + get_parallel().attn_tp_group + if is_dp_attention_enabled() + else get_parallel().tp_group + ) + _broadcast_verify_outputs( + tp_group, verify_outputs, predict, accept_index, num_correct_drafts + ) + else: + if _is_npu: + from sgl_kernel_npu.sample import ( + chain_speculative_sampling_triton, + top_k_renorm_prob, + top_p_renorm_prob, + tree_speculative_sampling_target_only, + ) + else: + from sglang.kernels.ops.speculative.reject_sampling import ( + chain_speculative_sampling_triton, + ) + + # if/else, not a ternary: the CUDA-only name still has to resolve in the + # branch not taken, and HIP only reaches here with rejection sampling on. + if use_rejection_sampling: + sampling_fn = chain_speculative_sampling_triton + if get_spec().speculative_use_block_verification: + sampling_fn = partial(sampling_fn, block_verification=True) + else: + if _is_cuda: + from sglang.kernels.ops.speculative.sampling import ( + tree_speculative_sampling_target_only, + ) + elif not _is_npu: + from sgl_kernel import tree_speculative_sampling_target_only + + sampling_fn = tree_speculative_sampling_target_only + + if _is_hip: + # Same names, same contract: dflash_utils.py aliases these too. + from sglang.kernels.ops.sampling.renorm_triton import ( + top_k_renorm_probs_triton as top_k_renorm_prob, + ) + from sglang.kernels.ops.sampling.renorm_triton import ( + top_p_renorm_probs_triton as top_p_renorm_prob, + ) + elif _is_cuda: + from flashinfer.sampling import top_k_renorm_probs as top_k_renorm_prob + from flashinfer.sampling import top_p_renorm_probs as top_p_renorm_prob + elif not _is_npu: + from sgl_kernel import top_k_renorm_prob, top_p_renorm_prob + + # Lean form (CUDA, chains, no rejection sampling): the same division + # broadcast per request instead of expanding the temperatures, and no + # zero-filled draft_probs (the sampling kernel's target-only path). + lean = ( + _is_cuda + and not use_rejection_sampling + and verify_input.tree_topk == 1 + and next_token_logits.is_contiguous() + and _spec_switches.enabled("SGLANG_SPEC_LEAN_SAMPLED_VERIFY") + ) + if lean and _spec_switches.enabled("SGLANG_SPEC_CHUNKED_VERIFY_SAMPLING"): + _log_chunked_verify_sampling_once() + sampling_fn = partial(sampling_fn, chunked=True) + if lean: + expanded_temperature = None + target_probs = sampling_softmax( + ( + next_token_logits.view(bs, verify_input.draft_token_num, -1) + / sampling_info.temperatures.view(bs, 1, 1) + ).view(bs * verify_input.draft_token_num, -1) + ) # (bs * num_draft_tokens, vocab_size) + else: + expanded_temperature = torch.repeat_interleave( + sampling_info.temperatures, verify_input.draft_token_num, dim=0 + ) # (bs * num_draft_tokens, 1) + + target_probs = sampling_softmax( + next_token_logits, temperatures=expanded_temperature + ) # (bs * num_draft_tokens, vocab_size) + maybe_detect_nan(target_probs, "v2 verify: target_probs after softmax") + if sampling_info.need_top_k_sampling: + target_probs = top_k_renorm_prob( + target_probs, + torch.repeat_interleave( + sampling_info.top_ks, verify_input.draft_token_num, dim=0 + ), + ) # (bs * num_draft_tokens, vocab_size) + maybe_detect_nan(target_probs, "v2 verify: target_probs after top_k_renorm") + if sampling_info.need_top_p_sampling: + target_probs = top_p_renorm_prob( + target_probs, + torch.repeat_interleave( + sampling_info.top_ps, verify_input.draft_token_num, dim=0 + ), + ) + maybe_detect_nan(target_probs, "v2 verify: target_probs after top_p_renorm") + target_probs = target_probs.reshape(bs, verify_input.draft_token_num, -1) + if use_rejection_sampling: + draft_probs = verify_input.draft_probs + elif lean: + draft_probs = None + else: + draft_probs = torch.zeros_like(target_probs) + # Defense-in-depth behind the spec_hook startup allowlist: validate + # the actual kernel inputs before the Triton kernel. + if use_rejection_sampling and ( + draft_probs is None or draft_probs.shape[-1] != target_probs.shape[-1] + ): + raise ValueError( + "Rejection sampling requires a target-vocab draft proposal " + "distribution; the current speculative algorithm/draft worker " + "does not produce one (draft_probs missing or vocab-mismatched)." + ) + + coins, coins_for_final_sampling = _verify_coins( + sampling_info=sampling_info, + seq_lens=batch.seq_lens, + draft_token_num=verify_input.draft_token_num, + candidates=candidates, + device=device, + ) + sampling_fn( + predicts=predict, # mutable + accept_index=accept_index, # mutable + accept_token_num=num_correct_drafts, # mutable + candidates=candidates, + # kwarg LHS retained as `retrive_*` to match sgl_kernel op schema. + retrive_index=verify_input.retrieve_index, + retrive_next_token=verify_input.retrieve_next_token, + retrive_next_sibling=verify_input.retrieve_next_sibling, + uniform_samples=coins, + uniform_samples_for_final_sampling=coins_for_final_sampling, + target_probs=target_probs, + draft_probs=draft_probs, + threshold_single=get_spec().speculative_accept_threshold_single, + threshold_acc=get_spec().speculative_accept_threshold_acc, + deterministic=True, + ) + del ( + expanded_temperature, + target_probs, + draft_probs, + coins, + coins_for_final_sampling, + ) + + # Sync sampling results across TP ranks: different GPUs may + # produce slightly different target_probs due to floating-point + # non-determinism in softmax/top_k/top_p, causing different + # sampled tokens. Broadcast from rank 0 to ensure consistency. + tp_group = ( + get_parallel().attn_tp_group + if is_dp_attention_enabled() + else get_parallel().tp_group + ) + _broadcast_verify_outputs( + tp_group, verify_outputs, predict, accept_index, num_correct_drafts + ) + + if SIMULATE_ACC_LEN > 0: + # Do simulation. The helper builds (and returns) a replacement + # accept_index of width spec_steps + 1, so pass max_tree_depth - 1 + # to keep the simulated width identical to the real one. + if SIMULATE_ACC_TOKEN_MODE not in ("fixed", "real-draft-token"): + raise ValueError( + "Invalid SGLANG_SIMULATE_ACC_TOKEN_MODE " + f"{SIMULATE_ACC_TOKEN_MODE!r}; expected 'fixed' or " + "'real-draft-token'." + ) + + if SIMULATE_ACC_TOKEN_MODE == "real-draft-token": + if verify_input.tree_topk != 1: + raise ValueError( + "SGLANG_SIMULATE_ACC_LEN with real draft tokens currently " + "requires speculative_eagle_topk=1." + ) + + # Use target argmax as the synthetic bonus for non-greedy requests. + if target_predict is None: + target_predict = torch.argmax(next_token_logits, dim=-1).reshape( + bs, verify_input.draft_token_num + ) + accept_index = generate_simulated_accept_index( + accept_index=accept_index, + predict=predict, # mutable + num_correct_drafts=num_correct_drafts, # mutable + candidates=candidates, + target_predict=target_predict, + simulate_acc_len=SIMULATE_ACC_LEN, + simulate_acc_token_mode=SIMULATE_ACC_TOKEN_MODE, + bs=bs, + spec_steps=verify_input.max_tree_depth - 1, + ) + + # `num_correct_drafts` stays drafts-only inside this function; the returned + # tensor includes the trailing/bonus token via out-of-place +1 so the + # name no longer flips semantics mid-function (naming doc C2). + accept_lens = num_correct_drafts + 1 + if watermark_state is not None and sampling_info.has_watermark_candidates: + watermark_state.record_speculative( + batch.req_pool_indices, + watermark_context_hashes, + watermark_selected, + accept_index, + accept_lens, + ) + return predict, accept_lens, accept_index + + +# ------------------------------------------------------------ eagle_worker_v2 + +_draft_tail_logged = set() + + +def _log_draft_tail_once(message: str) -> None: + if message not in _draft_tail_logged: + _draft_tail_logged.add(message) + logger.info(message) + + +def _fused_draft_extend_tail_eligible( + self, logits: torch.Tensor, seed_capture: Optional[torch.Tensor] +) -> bool: + """SGLANG_SPEC_FUSED_DRAFT_EXTEND_TAIL applies to this draft extend. + (Eager draft extends select their rows after this check; the selected + copy keeps the dtype and is row-major.)""" + if not _spec_switches.enabled("SGLANG_SPEC_FUSED_DRAFT_EXTEND_TAIL"): + return False + reason = None + if self.topk != 1 or _is_hip or not logits.is_cuda: + reason = "not a CUDA topk-1 draft" + elif get_spec().speculative_use_rejection_sampling: + reason = "rejection sampling" + if reason is None and ( + logits.ndim != 2 + or logits.dtype != torch.float32 + or logits.stride(1) != 1 + or logits.shape[1] < 131072 + ): + reason = "draft logits are not wide FP32 rows" + if ( + reason is None + and seed_capture is not None + and (seed_capture.ndim != 2 or seed_capture.stride(1) != 1) + ): + reason = "seed buffer is not 2-D row-major" + if reason is not None: + _log_draft_tail_once( + f"SGLANG_SPEC_FUSED_DRAFT_EXTEND_TAIL declined: {reason}" + ) + return False + _log_draft_tail_once("SGLANG_SPEC_FUSED_DRAFT_EXTEND_TAIL on") + return True + + +def _draft_extend_for_decode( + self, + batch: ScheduleBatch, + batch_result: GenerationBatchResult, + *, + kv_loc_plan: Optional[KVLocPlan] = None, +): + # Cast to int64 before entering plan stream to avoid cross-stream + # synchronization issues with .to() inside the plan stream context. + if batch_result.prepared_draft_extend_inputs is not None: + num_correct_drafts, select_index, next_token_ids = ( + batch_result.prepared_draft_extend_inputs + ) + elif batch_result.accept_lens.is_cuda: + from sglang.kernels.ops.speculative.eagle import prepare_draft_extend_inputs + + num_correct_drafts, select_index, next_token_ids = ( + prepare_draft_extend_inputs( + batch_result.accept_lens, + batch_result.next_token_ids, + self.speculative_num_draft_tokens, + ) + ) + else: + num_correct_drafts = batch_result.accept_lens - 1 + select_index = ( + torch.arange( + 0, + len(batch.seq_lens) * self.speculative_num_draft_tokens, + self.speculative_num_draft_tokens, + device=self.device, + ) + + batch_result.accept_lens + - 1 + ) + + next_token_ids = batch_result.next_token_ids.to(torch.int64) + + # Batch 2: Draft extend + draft_extend_input = EagleDraftExtendInput( + hidden_states=batch_result.logits_output.hidden_states, + # accept_lens includes the bonus token; correct drafts exclude it. + num_correct_drafts=num_correct_drafts, + num_accept_tokens=batch_result.accept_lens, + # Draft-extend fills the whole tree width (num_draft_tokens) per req, + # not num_steps + 1, so DP MLP-sync padding stays consistent for topk > 1. + num_tokens_per_req=self.speculative_num_draft_tokens, + num_tokens_for_logprob_per_req=self.speculative_num_draft_tokens, + ) + # Prepare for draft extend in a separate stream + if self.plan_stream: + self.plan_stream.wait_stream( + torch.get_device_module(self.device).current_stream() + ) + with self.plan_stream_ctx: + forward_batch = prepare_for_draft_extend( + draft_extend_input, + batch, + next_token_ids, + self.speculative_num_draft_tokens, + self.draft_runner, + self.cuda_graph_runner_for_draft_extend, + return_hidden_states_before_norm=False, + kv_loc_plan=kv_loc_plan, + ) + + if self.plan_stream: + torch.get_device_module(self.device).current_stream().wait_stream( + self.plan_stream + ) + + # Run draft extend batch in the main compute stream + can_run_decode_cuda_graph = ( + self.cuda_graph_runner_for_draft_extend + and self.cuda_graph_runner_for_draft_extend.can_run_graph(forward_batch) + ) + + # Eager path publishes the indexer top-k into a worker buffer (the graph + # path uses the runner's static buffer). Gathered at select_index below. + if self.seed_dsa_topk_from_draft_extend and not can_run_decode_cuda_graph: + forward_batch.spec_info.dsa_seed_topk_capture = ( + self._get_dsa_extend_topk_buf(forward_batch.input_ids.shape[0]) + ) + + canary_ctx = ( + context_tuple( + c.with_ops_outside_graph( + single_forward_indices=[0], + maybe_inaccurate_forward_batch=forward_batch, + ), + c.with_active_single_forward_manager(0), + ) + if (c := self.draft_runner.canary_manager) is not None + else contextlib.nullcontext() + ) + with canary_ctx: + if can_run_decode_cuda_graph: + draft_logits_output = self.cuda_graph_runner_for_draft_extend.execute( + forward_batch, select_index + ) + else: + draft_logits_output = self.draft_runner.forward( + forward_batch + ).logits_output + + maybe_detect_nan( + draft_logits_output.next_token_logits, + f"draft_extend_for_decode (cuda_graph={can_run_decode_cuda_graph})", + ) + maybe_detect_inf( + draft_logits_output.next_token_logits, + f"draft_extend_for_decode (cuda_graph={can_run_decode_cuda_graph})", + ) + + # Gather the per-request last-position indexer top-k as the next loop's + # seed (select_index already picks the last accepted position per req). + dsa_seed_topk_indices = None + dsa_extend_topk_capture = None + if self.seed_dsa_topk_from_draft_extend: + if can_run_decode_cuda_graph: + dsa_extend_topk_capture = self.cuda_graph_runner_for_draft_extend.buffers.dsa_seed_topk_capture + else: + dsa_extend_topk_capture = forward_batch.spec_info.dsa_seed_topk_capture + fused_tail = _fused_draft_extend_tail_eligible( + self, + draft_logits_output.next_token_logits, dsa_extend_topk_capture + ) + if dsa_extend_topk_capture is not None and not fused_tail: + # Fancy indexing returns a fresh tensor (detached from the buffer). + dsa_seed_topk_indices = dsa_extend_topk_capture[select_index] + + # Reorganize the spec info for the next batch + if not can_run_decode_cuda_graph: + draft_logits_output.next_token_logits = ( + draft_logits_output.next_token_logits[select_index] + ) + if draft_logits_output.hidden_states is not None: + draft_logits_output.hidden_states = draft_logits_output.hidden_states[ + select_index + ] + # Selected-row top-k remains worker-owned for both graph and eager + # paths; the graph runner only moves the row selection before lm_head. + if get_spec().speculative_use_rejection_sampling: + ret_draft_probs, ret_topk_p, ret_topk_index = sample_draft_proposal( + draft_logits_output.next_token_logits, + batch.sampling_info.temperatures, + batch.sampling_info.top_ks, + ) + elif self.topk == 1 and _is_hip: + ret_topk_p, ret_topk_index = draft_topk1_argmax_only( + draft_logits_output.next_token_logits + ) + ret_draft_probs = None + elif fused_tail: + # SGLANG_SPEC_FUSED_DRAFT_EXTEND_TAIL: argmax, unit probs and the + # seed-row gather below in two launches, same values. + ret_topk_p, ret_topk_index, dsa_seed_topk_indices = draft_extend_tail( + draft_logits_output.next_token_logits, + dsa_extend_topk_capture, + select_index if dsa_extend_topk_capture is not None else None, + ) + ret_draft_probs = None + elif self.topk == 1 and not _is_hip: + # Gated to CUDA: see #26358 — ROCm's argmax tie-break corrupts + # MTP draft selection on FP8 logits. + ret_topk_index = torch.argmax( + draft_logits_output.next_token_logits, dim=-1, keepdim=True + ) + ret_topk_p = torch.ones_like(ret_topk_index, dtype=torch.float32) + ret_draft_probs = None + else: + probs = renorm_draft_probs( + draft_logits_output.next_token_logits, + batch.sampling_info, + get_spec().speculative_use_rejection_sampling, + ) + ret_topk_p, ret_topk_index = fast_topk(probs, self.topk, dim=-1) + ret_draft_probs = None + ret_hidden_states = draft_logits_output.hidden_states + + # Construct the return values + next_draft_input = batch_result.next_draft_input + ( + next_draft_input.topk_p, + next_draft_input.topk_index, + next_draft_input.hidden_states, + ) = ( + ret_topk_p, + ret_topk_index, + ret_hidden_states, + ) + if get_spec().speculative_use_rejection_sampling: + next_draft_input.draft_probs = ret_draft_probs + if self.seed_dsa_topk_from_draft_extend: + next_draft_input.dsa_topk_indices = dsa_seed_topk_indices + next_draft_input.cuda_graph_compatible = not ( + self.seed_dsa_topk_from_draft_extend and dsa_seed_topk_indices is None + ) diff --git a/sgl_nvidia_mega/spec_step/ops.py b/sgl_nvidia_mega/spec_step/ops.py new file mode 100644 index 00000000..936a3658 --- /dev/null +++ b/sgl_nvidia_mega/spec_step/ops.py @@ -0,0 +1,267 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Kernels of the speculative step fusions: the one-launch relay scatter, the topk-1 +draft-extend tail, and the chain verify sampling without a draft_probs buffer. + +The sampling kernel is SGLang's JIT kernel with a target-only chain form and a +two-pass chunked form (``csrc/``); the other two are Triton. +""" + +from __future__ import annotations + +import math +from pathlib import Path +from typing import Optional + +import torch +import triton +import triton.language as tl + +# SGLang's split-row argmax, whose partial pass and pair reduction the tail reuses +from sglang.kernels.ops.speculative.row_argmax import ( + _argmax_pair, + _medium_argmax_partial_kernel, +) + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "kernels/ops/speculative/row_argmax.py": ( + "_argmax_pair", + "_medium_argmax_partial_kernel", + ), + "kernels/jit/utils/compile/loader.py": ("load_jit", "extra_include_paths"), +} + +CSRC = Path(__file__).resolve().parent / "csrc" + + +@triton.jit +def _scatter_spec_extras_kernel( + indices, + sources, + destinations, + INDEX_STRIDE: tl.constexpr, + SOURCE_STRIDES: tl.constexpr, + WIDTHS: tl.constexpr, + SLOTS: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + dst = tl.load(indices + row * INDEX_STRIDE).to(tl.int64) + dst = tl.where(dst < 0, dst + SLOTS, dst) + col = tl.program_id(1).to(tl.int64) * BLOCK + tl.arange(0, BLOCK) + for i in tl.static_range(len(WIDTHS)): + value = tl.load(sources[i] + row * SOURCE_STRIDES[i] + col, col < WIDTHS[i]) + tl.store(destinations[i] + dst * WIDTHS[i] + col, value, col < WIDTHS[i]) + + +def scatter_spec_extras(indices, pairs): + """``dst[indices] = src`` for every (dst, src) pair in one launch; the + destinations are row-major relay buffers with as many rows as each other.""" + if indices.numel() == 0 or not pairs: + return + sources = tuple(src.reshape(src.shape[0], -1) for _, src in pairs) + sources = tuple(src if src.stride(1) == 1 else src.contiguous() for src in sources) + widths = tuple(src.shape[1] for src in sources) + block = min(1024, triton.next_power_of_2(max(widths))) + _scatter_spec_extras_kernel[(indices.numel(), triton.cdiv(max(widths), block))]( + indices, + sources, + tuple(dst for dst, _ in pairs), + indices.stride(0), + tuple(src.stride(0) for src in sources), + widths, + pairs[0][0].shape[0], + BLOCK=block, + ) + + +@triton.jit +def _draft_tail_final_kernel( + PV, + PI, + OUT, + PROBS, + SEED_SRC, + SEED_INDEX, + SEED_OUT, + SPLITS: tl.constexpr, + BLOCK: tl.constexpr, + SEED_WIDTH: tl.constexpr, + SEED_SRC_STRIDE: tl.constexpr, + SEED_INDEX_STRIDE: tl.constexpr, + SEED_BLOCK: tl.constexpr, + HAS_SEED: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + part = tl.arange(0, BLOCK) + v = tl.load(PV + row * SPLITS + part, part < SPLITS, float("-inf")) + i = tl.load(PI + row * SPLITS + part, part < SPLITS, 0x7FFFFFFF) + _, index = tl.reduce((v, i), 0, _argmax_pair) + tl.store(OUT + row, index.to(tl.int64)) + tl.store(PROBS + row, 1.0) + if HAS_SEED: + # The row's seed: SEED_SRC[SEED_INDEX[row]], as an indexed gather. + src = tl.load(SEED_INDEX + row * SEED_INDEX_STRIDE).to(tl.int64) + for start in tl.static_range(0, SEED_WIDTH, SEED_BLOCK): + cols = start + tl.arange(0, SEED_BLOCK) + m = cols < SEED_WIDTH + seed = tl.load(SEED_SRC + src * SEED_SRC_STRIDE + cols, m) + tl.store(SEED_OUT + row * SEED_WIDTH + cols, seed, m) + + +def draft_extend_tail( + logits: torch.Tensor, + seed_src: Optional[torch.Tensor] = None, + seed_index: Optional[torch.Tensor] = None, +): + """The topk-1 draft-extend tail in two launches. + + Returns ``(topk_p, topk_index, seed)`` equal to + ``torch.ones_like(i, dtype=torch.float32)``, ``i = logits.argmax(-1, + keepdim=True)`` (NaN first, lowest index on ties) and ``seed_src[seed_index]`` + (None without ``seed_src``). ``logits``: FP32 rows with a unit column + stride; ``seed_src``: 2-D with a unit column stride; ``seed_index``: 1-D + non-negative row indices, one per logits row. + """ + assert logits.ndim == 2 and logits.dtype == torch.float32 + assert logits.stride(1) == 1 + rows, n = logits.shape + assert n > 0 + has_seed = seed_src is not None + index = torch.empty((rows, 1), dtype=torch.int64, device=logits.device) + probs = torch.empty((rows, 1), dtype=torch.float32, device=logits.device) + seed = None + if has_seed: + assert seed_src.ndim == 2 and seed_src.stride(1) == 1 + assert seed_index is not None and seed_index.shape == (rows,) + seed = torch.empty( + (rows, seed_src.shape[1]), dtype=seed_src.dtype, device=seed_src.device + ) + if rows: + block = 2048 if rows <= 64 else 4096 if rows <= 256 else 8192 + splits = triton.cdiv(n, block) + pv = torch.empty((rows, splits), dtype=torch.float32, device=logits.device) + pi = torch.empty((rows, splits), dtype=torch.int32, device=logits.device) + _medium_argmax_partial_kernel[(rows, splits)]( + logits, pv, pi, n, logits.stride(0), splits, block, num_warps=4 + ) + seed_width = seed_src.shape[1] if has_seed else 1 + _draft_tail_final_kernel[(rows,)]( + pv, + pi, + index, + probs, + seed_src if has_seed else pv, + seed_index if has_seed else pi, + seed if has_seed else pv, + splits, + triton.next_power_of_2(splits), + seed_width, + seed_src.stride(0) if has_seed else 0, + seed_index.stride(0) if has_seed else 0, + min(1024, triton.next_power_of_2(seed_width)), + has_seed, + num_warps=4, + ) + return probs, index, seed + + +def _load(name: str, wrapper: str, *marker): + from sglang.kernels.jit.utils import load_jit + + return load_jit( + "sgl_nvidia_mega_spec_sampling", + name, + *marker, + cuda_files=[str(CSRC / "sampling.cuh")], + cuda_wrappers=[("sample", wrapper)], + extra_include_paths=[str(CSRC / "include")], + extra_dependencies=["flashinfer"], + ) + + +_modules: dict = {} + + +def _module(vec_size: int, deterministic: bool, *, no_draft_probs=False, chunked=False): + key = (vec_size, deterministic, no_draft_probs, chunked) + if key not in _modules: + det = "true" if deterministic else "false" + if chunked: + wrapper = f"chain_speculative_sampling_target_only_two_pass<{vec_size}, {det}>" + _modules[key] = _load("chunked", wrapper, vec_size, deterministic) + else: + ndp = "true" if no_draft_probs else "false" + wrapper = f"tree_speculative_sampling_target_only<{vec_size}, {det}, {ndp}>" + _modules[key] = _load("tree", wrapper, vec_size, deterministic, ndp) + return _modules[key] + + +# Vocabulary chunk of the sampling kernels: 1024 threads x vec_size elements. +_CHUNK_THREADS = 1024 + + +def tree_speculative_sampling_target_only( + predicts: torch.Tensor, # mutable + accept_index: torch.Tensor, # mutable + accept_token_num: torch.Tensor, # mutable + candidates: torch.Tensor, + retrive_index: torch.Tensor, + retrive_next_token: torch.Tensor, + retrive_next_sibling: torch.Tensor, + uniform_samples: torch.Tensor, + uniform_samples_for_final_sampling: torch.Tensor, + target_probs: torch.Tensor, + draft_probs: Optional[torch.Tensor], + threshold_single: float = 1.0, + threshold_acc: float = 1.0, + deterministic: bool = True, + chunked: bool = False, +) -> None: + """SGLang's ``tree_speculative_sampling_target_only``, plus ``draft_probs=None`` + (chains only: one candidate per tree level), which samples as a zero-filled + ``draft_probs`` would without the buffer, and ``chunked`` (with + ``draft_probs=None`` only): the same sampling as two launches, the vocabulary + passes split over one CTA per chunk. The outputs are the same bits.""" + vec_size = math.gcd(4, target_probs.shape[-1]) + no_draft_probs = draft_probs is None + if chunked and no_draft_probs: + bs = target_probs.shape[0] + chunk = _CHUNK_THREADS * vec_size + num_chunks = (target_probs.shape[-1] + chunk - 1) // chunk + chunk_sums = target_probs.new_empty((2, bs, num_chunks)) + chunk_last_valid = accept_token_num.new_empty((bs, num_chunks)) + _module(vec_size, deterministic, chunked=True).sample( + predicts, + accept_index, + accept_token_num, + candidates, + retrive_index, + retrive_next_token, + retrive_next_sibling, + uniform_samples, + uniform_samples_for_final_sampling, + target_probs, + chunk_sums[0], + chunk_sums[1], + chunk_last_valid, + threshold_single, + threshold_acc, + ) + return + _module(vec_size, deterministic, no_draft_probs=no_draft_probs).sample( + predicts, + accept_index, + accept_token_num, + candidates, + retrive_index, + retrive_next_token, + retrive_next_sibling, + uniform_samples, + uniform_samples_for_final_sampling, + target_probs, + target_probs if no_draft_probs else draft_probs, + threshold_single, + threshold_acc, + ) diff --git a/sgl_nvidia_mega/spec_step/runner.py b/sgl_nvidia_mega/spec_step/runner.py new file mode 100644 index 00000000..a6a2967f --- /dev/null +++ b/sgl_nvidia_mega/spec_step/runner.py @@ -0,0 +1,1146 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""One CUDA graph per EAGLE v2 decode step (``SGLANG_NVIDIA_MEGAKERNEL_SPEC_STEP_GRAPH=1``). + +At small batch sizes an EAGLE v2 decode step is three CUDA graph replays +(draft, target verify, draft extend) with many small kernels and host work +between them, so the GPU idles between launches. This runner captures the +worker's own decode step once per key (batch size, sampling setup and the +layout of the step's input tensors) and replays it as one graph: + +* The capture runs ``EAGLEWorkerV2.forward_batch_generation`` on a copy of the + batch whose per-step device tensors are runner-owned static buffers, inside + a ``BreakableCUDAGraphCapture`` on a dedicated stream with a private memory + pool per key. The three per-stage graphs are spliced in as child-graph nodes + (``ChildGraphCapture``); the eager work between them is captured from the + same code that runs it on the regular path. +* A replay copies the live input tensors into the static buffers and launches + the graph. Host work that must happen every step runs on the host at replay: + the FutureMap publish, the shared-read fence and, in the simulated + acceptance benchmarking mode, the acceptance length draw (same function, + same host RNG stream, once per step). +* Constrained decoding: the verify tree is copied to pinned host buffers inside + the graph, and the grammar bitmask is built on the host between two graph + segments (an ``eager_on_graph`` break), still overlapping the target verify + forward; the captured sampling reads the bitmask from a static buffer. +* Random draws of sampled verification are captured inside the torch graph, + whose replay advances the CUDA generator exactly like the eager draws. + +Nothing is captured at startup: a key is captured at its first eligible step +after ``_NUM_WARMUP_STEPS`` regular steps. The last warmup step counts the +per-stage graph replays; a key whose step does not replay exactly the three +per-stage graphs is never captured. Any step the runner cannot capture runs the +regular path; the reason is counted (``/get_server_info``) and logged once. +""" + +from __future__ import annotations + +import copy +import dataclasses +import gc +import logging +import time +from collections import Counter +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +import torch + +from sglang.srt.constrained.base_grammar_backend import ( + GrammarMask, + InvalidGrammarObject, +) +from sglang.srt.distributed.parallel_state import get_tp_group, graph_capture +from sglang.srt.environ import envs +from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.model_executor.runner_backend.full_cuda_graph_backend import ( + FullCudaGraphBackend, +) +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.breakable_cuda_graph import ( + BreakableCUDAGraph, + BreakableCUDAGraphCapture, + eager_on_graph, +) +from sglang.srt.model_executor.runner_utils.pool import graph_pool_replay_scope +from sglang.srt.runtime_context import get_stream +from sglang.srt.utils import is_cuda + +from sgl_nvidia_mega.spec_step import switches +from sgl_nvidia_mega.spec_step.child_graph_capture import ChildGraphCapture + +if TYPE_CHECKING: + from sglang.srt.managers.schedule_batch import ScheduleBatch + from sglang.srt.managers.utils import GenerationBatchResult + from sglang.srt.server_args import ServerArgs + from sglang.srt.speculative.eagle_worker_v2 import EAGLEWorkerV2 + +logger = logging.getLogger(__name__) + +# Largest decode batch size captured; larger batches are rarely launch bound. +_MAX_BS = 8 +# Captured keys per worker; a new key past this runs the regular step. +_MAX_KEYS = 32 +# Regular steps of a key before it is captured; the last one counts the +# per-stage graph replays. +_NUM_WARMUP_STEPS = 2 +# Per-stage graphs of one decode step: draft, target verify, draft extend. +_NUM_STAGE_GRAPHS = 3 +# Keys whose capture may fail before the runner turns itself off. +_MAX_FAILED_KEYS = 4 +# Steps whose outputs may still be read by the scheduler's device-to-host +# copy while the next steps replay. +_OUTPUT_RING_SIZE = 2 +_SIM_LEN_SLOTS = 4 +# Free device memory a capture needs beyond the largest key captured so far. +_CAPTURE_MEMORY_HEADROOM = 1 << 30 +_FIRST_KEY_MEMORY_ESTIMATE = 256 << 20 +_MEMORY_RETRY_STEPS = 256 +_MAX_WARMUP_KEYS = 1024 +_CAPTURE_STREAM_NAME = "speculative_step_cuda_graph" + +# Device tensors of the schedule batch that a decode step reads and that change +# from step to step. Every CUDA tensor of the draft input and of the sampling +# info is a per-step input as well, except the ones listed as step outputs. +_BATCH_STEP_INPUTS = ("req_pool_indices", "seq_lens", "orig_seq_lens") +# Assigned by the decode step before anything reads them (the capture copy sets +# them to None, so a read before the assignment fails the capture loudly). +_BATCH_STEP_OUTPUTS = ("input_ids", "out_cache_loc") +_DRAFT_INPUT_STEP_OUTPUTS = ("positions", "kv_indptr", "kv_indices") + +_UNSUPPORTED_ENVS = ( + envs.SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH, + envs.SGLANG_ENABLE_CUDA_GRAPH_DEDUP, + envs.SGLANG_ENABLE_METADATA_GLUE_GRAPH, + envs.SGLANG_ENABLE_GRAPH_POOL_BORROW, + envs.SGLANG_ENABLE_OVERLAP_PLAN_STREAM, + envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION, + envs.SGLANG_MEMORY_SAVER_CUDA_GRAPH, + envs.SGLANG_ENABLE_NAN_LOGITS_CHECK, + envs.SGLANG_ENABLE_ASYNC_ASSERT, + envs.SGLANG_SANITIZE_NAN_LOGITS, +) + + +def out_graph_metadata_capturable(backend, forward_mode: ForwardMode) -> bool: + """True when ``backend``'s replay-time ``init_forward_metadata_out_graph`` for + ``forward_mode`` may be recorded into an enclosing CUDA graph: device work on + device tensors only, so that replaying the recording equals calling it again. + SGLang's DSA backends are, for decode, target verify and draft extend.""" + from sglang.srt.layers.attention.dsa_backend import ( + DeepseekSparseAttnBackend, + DeepseekSparseAttnMultiStepBackend, + ) + + if isinstance(backend, DeepseekSparseAttnMultiStepBackend): + return all( + out_graph_metadata_capturable(b, forward_mode) + for b in backend.attn_backends + ) + if isinstance(backend, DeepseekSparseAttnBackend): + return ( + forward_mode.is_decode() + or forward_mode.is_target_verify() + or forward_mode.is_draft_extend_v2() + ) + return False + + +def _cuda_python_available() -> bool: + try: + from cuda.bindings import runtime # noqa: F401 + except ImportError: + return False + return True + + +def step_cuda_graph_unsupported_reason(server_args: ServerArgs) -> Optional[str]: + """Why the step CUDA graph cannot run under these server args, or None.""" + from sglang.srt.arg_groups.overrides import resolved_view, resolving_view + from sglang.srt.runtime_context import attn_dp_enabled_of + + cfg = resolving_view(server_args) + if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3", "NEXTN"): + return ( + f"speculative_algorithm={cfg.speculative_algorithm} " + "(only EAGLE / EAGLE3 / NEXTN are supported)" + ) + if cfg.enable_multi_layer_eagle: + return "enable_multi_layer_eagle is not supported" + if cfg.speculative_eagle_topk is not None and cfg.speculative_eagle_topk != 1: + return ( + f"speculative_eagle_topk={cfg.speculative_eagle_topk} (only 1 is supported)" + ) + if cfg.speculative_use_rejection_sampling or cfg.speculative_use_block_verification: + return "rejection sampling is not supported" + if cfg.speculative_adaptive: + return "speculative_adaptive is not supported" + if not is_cuda(): + return "only CUDA devices are supported" + if not _cuda_python_available(): + return "the 'cuda-python' package is not installed" + if torch.cuda.get_allocator_backend() != "native": + return "only the native CUDA caching allocator is supported" + if cfg.pp_size > 1: + return "pipeline parallelism is not supported" + if attn_dp_enabled_of(resolved_view(server_args)): + return "attention DP is not supported" + unsupported_features = ( + ("enable_two_batch_overlap", cfg.enable_two_batch_overlap), + ("enable_single_batch_overlap", cfg.enable_single_batch_overlap), + ("enable_pdmux", cfg.enable_pdmux), + ("enable_lora", cfg.enable_lora or cfg.lora_paths), + ("disaggregation_mode", cfg.disaggregation_mode != "null"), + ("enable_hisparse", cfg.enable_hisparse), + ("enable_memory_saver", cfg.enable_memory_saver), + ("debug_cuda_graph", cfg.debug_cuda_graph), + ("disable_cuda_graph", cfg.disable_cuda_graph), + ("expert_distribution_recorder_mode", cfg.expert_distribution_recorder_mode), + ("enable_eplb", cfg.enable_eplb), + ("enable_return_routed_experts", cfg.enable_return_routed_experts), + ("enable_return_indexer_topk", cfg.enable_return_indexer_topk), + ("kv_canary", cfg.kv_canary not in (None, "none")), + ("elastic_ep_backend", cfg.elastic_ep_backend), + ) + for name, value in unsupported_features: + if value: + return f"{name} is not supported" + for env in _UNSUPPORTED_ENVS: + if env.get(): + return f"{env.name} is not supported" + return None + + +def simulated_accept_index_from_device_length( + accept_index: torch.Tensor, + predict: torch.Tensor, + num_correct_drafts: torch.Tensor, + candidates: torch.Tensor, + target_predict: Optional[torch.Tensor], + spec_steps: int, + accept_len: torch.Tensor, + simulate_acc_token_mode: str, +) -> torch.Tensor: + """``spec_utils.generate_simulated_accept_index`` for an acceptance length + that was already drawn and is read from ``accept_len`` (a device int32 + tensor of shape [1]), so that every shape is independent of its value.""" + bs = accept_index.shape[0] + width = spec_steps + 1 + cols = torch.arange(width, dtype=torch.int32, device=accept_index.device).view( + 1, width + ) + length = accept_len.view(1, 1) + first = accept_index[:, :1] + nodes = first + cols + sim_accept_index = torch.where(cols < length, nodes, torch.full_like(nodes, -1)) + num_correct_drafts.copy_((accept_len - 1).expand_as(num_correct_drafts)) + if simulate_acc_token_mode != "real-draft-token": + predict.fill_(100) # some legit token id, as the regular path + return sim_accept_index + + nodes = nodes.view(-1).long() + draft_cols = (cols + 1).clamp(max=candidates.shape[1] - 1).view(-1).long() + drafts = candidates[:, draft_cols].to(predict.dtype) + bonus = target_predict.gather(1, (length - 1).long().expand(bs, 1)) + current = predict[nodes].view(bs, width) + updated = torch.where( + cols < length - 1, + drafts, + torch.where(cols == length - 1, bonus.to(predict.dtype), current), + ) + predict.index_copy_(0, nodes, updated.view(-1)) + return sim_accept_index + + +def _tp_min(*values: int) -> Tuple[int, ...]: + """Element-wise MIN over the TP group (host collective).""" + group = get_tp_group() + if group.world_size == 1: + return tuple(int(v) for v in values) + tensor = torch.tensor([int(v) for v in values], dtype=torch.int64) + torch.distributed.all_reduce( + tensor, op=torch.distributed.ReduceOp.MIN, group=group.cpu_group + ) + return tuple(tensor.tolist()) + + +def _host_copies_in_capture(stream: torch.cuda.Stream) -> List[Tuple[str, int]]: + """("read" | "write", host address) of every copy node in the graph being + captured on ``stream`` (child graphs excluded) that touches host memory. A + replay would repeat such a copy with the values or addresses of capture time.""" + from cuda.bindings import runtime as rt + + from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.cuda_utils import ( + checkCudaErrors, + ) + + status, _, graph, _, _, _ = checkCudaErrors( + rt.cudaStreamGetCaptureInfo(stream.cuda_stream) + ) + if status != rt.cudaStreamCaptureStatus.cudaStreamCaptureStatusActive: + return [] + _, num_nodes = checkCudaErrors(rt.cudaGraphGetNodes(graph, 0)) + nodes, _ = checkCudaErrors(rt.cudaGraphGetNodes(graph, num_nodes)) + host_types = ( + rt.cudaMemoryType.cudaMemoryTypeHost, + rt.cudaMemoryType.cudaMemoryTypeUnregistered, + ) + found = [] + for node in nodes or []: + node_type = checkCudaErrors(rt.cudaGraphNodeGetType(node)) + if node_type != rt.cudaGraphNodeType.cudaGraphNodeTypeMemcpy: + continue + params = checkCudaErrors(rt.cudaGraphMemcpyNodeGetParams(node)) + for side, ptr in ( + ("read", int(params.srcPtr.ptr)), + ("write", int(params.dstPtr.ptr)), + ): + attrs = checkCudaErrors(rt.cudaPointerGetAttributes(ptr)) + if attrs.type in host_types: + found.append((side, ptr)) + return found + + +def _check_host_copies(allowed_writes=()) -> None: + """Fail the capture if the segment being captured copies from host memory or + to host memory other than ``allowed_writes`` (runner-owned pinned buffers).""" + for side, ptr in _host_copies_in_capture(torch.cuda.current_stream()): + if side == "read" or ptr not in allowed_writes: + raise RuntimeError( + f"the captured step would {side} host memory at {ptr:#x} " + "with its capture-time address or value" + ) + + +def _is_device_tensor(value: Any) -> bool: + return isinstance(value, torch.Tensor) and value.is_cuda + + +def _step_tensors(batch: ScheduleBatch) -> List[Tuple[str, str, torch.Tensor]]: + """(holder, attribute, tensor) for every per-step device input of a decode step.""" + tensors = [] + for name in _BATCH_STEP_INPUTS: + tensor = getattr(batch, name, None) + if _is_device_tensor(tensor): + tensors.append(("batch", name, tensor)) + for holder, obj, outputs in ( + ("spec_info", batch.spec_info, _DRAFT_INPUT_STEP_OUTPUTS), + ("sampling_info", batch.sampling_info, ()), + ): + for name, value in sorted(vars(obj).items()): + if name not in outputs and _is_device_tensor(value): + tensors.append((holder, name, value)) + return tensors + + +def _holders(batch: ScheduleBatch) -> Dict[str, Any]: + return { + "batch": batch, + "spec_info": batch.spec_info, + "sampling_info": batch.sampling_info, + } + + +# try_small_copy's per-tensor element limit. +_SMALL_COPY_MAX_NUMEL = 8192 + + +def _ranges_overlap(a: List[torch.Tensor], b: List[torch.Tensor]) -> bool: + def span(t): + storage = t.untyped_storage() + return storage.data_ptr(), storage.data_ptr() + storage.nbytes() + + spans = sorted(span(t) for t in b if t.numel()) + for t in a: + if not t.numel(): + continue + lo, hi = span(t) + if any(s < hi and lo < e for s, e in spans): + return True + return False + + +def _copy_step_inputs( + static_inputs: List[torch.Tensor], + live_inputs: List[torch.Tensor], + small_copy=None, +) -> str: + """Copy a replay's live inputs into the graph's static input buffers. + + With ``small_copy`` (``SGLANG_SPEC_STEP_GRAPH_FUSED_INPUT_COPY``) the copies + are one launch when every tensor qualifies for it ("fused"); otherwise the + small tensors are one launch and the others one copy each ("partial", only + when no live tensor shares memory with a static buffer, so the order of the + copies cannot matter); otherwise one copy per tensor ("per_tensor"). + """ + if small_copy is not None: + if small_copy(static_inputs, live_inputs): + return "fused" + small = [ + i for i, t in enumerate(static_inputs) if t.numel() <= _SMALL_COPY_MAX_NUMEL + ] + if 2 <= len(small) < len(static_inputs) and not _ranges_overlap( + static_inputs, live_inputs + ): + if small_copy( + [static_inputs[i] for i in small], [live_inputs[i] for i in small] + ): + chosen = set(small) + # One copy_ per tensor, as torch._foreach_copy_ does for a + # mixed-dtype list (a same-dtype list would take its fused + # path, which is not a plain byte copy). + for i in range(len(static_inputs)): + if i not in chosen: + static_inputs[i].copy_(live_inputs[i], non_blocking=True) + return "partial" + torch._foreach_copy_(static_inputs, live_inputs, non_blocking=True) + return "per_tensor" + + +class _GrammarStep: + """Capture-time overrides and replay-time host work of constrained decoding.""" + + def __init__(self, runner: EagleStepCudaGraphRunner): + self.runner = runner + self.host_tree: Optional[Tuple[torch.Tensor, ...]] = None + self.tree_ready = torch.cuda.Event() + self.mask: Optional[torch.Tensor] = None + + def overrides(self) -> Dict[str, Any]: + return { + "spec_grammar_tree_from_device": self.tree_from_device, + "spec_grammar_vocab_mask": self.vocab_mask, + } + + def tree_from_device( + self, original, cls, retrieve_next_token, retrieve_next_sibling, draft_token + ): + tensors = (retrieve_next_token, retrieve_next_sibling, draft_token) + # Runner-owned pinned buffers, refilled by every replay. + self.host_tree = tuple( + torch.empty(t.shape, dtype=t.dtype, pin_memory=True) for t in tensors + ) + for host, tensor in zip(self.host_tree, tensors): + host.copy_(tensor, non_blocking=True) + _check_host_copies({host.data_ptr() for host in self.host_tree}) + eager_on_graph(self._record_tree_ready)() + return cls(self.host_tree, None) + + def _record_tree_ready(self) -> None: + self.tree_ready.record() + + def vocab_mask(self, original, *, reqs, tree, sampling_info, device, barrier): + grammar = reqs[0].grammar + host_mask = grammar.allocate_vocab_mask( + vocab_size=sampling_info.vocab_size, + batch_size=self.host_tree[2].numel(), + device="cpu", + ) + self.mask = torch.empty(host_mask.shape, dtype=host_mask.dtype, device=device) + sampling_info.grammar_mask = None + _check_host_copies() + eager_on_graph(self._build_mask, capture_stub=lambda: self.mask)() + return GrammarMask(grammar, self.mask) + + def _build_mask(self) -> torch.Tensor: + # Same steps as spec_utils.build_grammar_vocab_mask, on the live batch. + from sglang.srt.speculative.spec_utils import generate_token_bitmask + + batch = self.runner._live_batch + barrier = self.runner._live_grammar_barrier + if barrier is not None: + barrier() + self.tree_ready.synchronize() + vocab_mask, _ = generate_token_bitmask( + batch.reqs, *self.host_tree, batch.sampling_info.vocab_size + ) + self.mask.copy_(vocab_mask, non_blocking=True) + batch.sampling_info.grammar_mask = None + return self.mask + + +@dataclasses.dataclass +class _StepGraph: + """One captured decode step and everything its replays read or write.""" + + graph: BreakableCUDAGraph + # (holder, attribute) of each static input, and the static buffers. + input_names: List[Tuple[str, str]] + static_inputs: List[torch.Tensor] + # Capture-time result; replays return shallow copies of it. + result: GenerationBatchResult + # (backend, graph epoch) of each spliced per-stage graph. + stage_graphs: List[Tuple[Any, Optional[int]]] + ring_slot: torch.Tensor + ring_next_token_ids: Optional[torch.Tensor] = None + ring_accept_lens: Optional[torch.Tensor] = None + ring_results: List[Optional[GenerationBatchResult]] = dataclasses.field( + default_factory=lambda: [None] * _OUTPUT_RING_SIZE + ) + # One (next_token_ids, accept_lens) view per ring slot, handed to results. + ring_views: List[Tuple[torch.Tensor, torch.Tensor]] = dataclasses.field( + default_factory=list + ) + # Simulated acceptance: device length read by the graph and the arguments + # of the host draw. + sim_accept_len: Optional[torch.Tensor] = None + sim_draw_args: Optional[Tuple[float, str, int]] = None + grammar: Optional[_GrammarStep] = None + num_replays: int = 0 + memory_bytes: int = 0 + + +class EagleStepCudaGraphRunner: + """Captures and replays whole EAGLE v2 decode steps for one worker.""" + + def __init__(self, worker: EAGLEWorkerV2): + self.worker = worker + draft_worker = worker.draft_worker + self.target_model_runner = worker.target_worker.model_runner + self.draft_model_runner = draft_worker.draft_runner + self.stage_runners = ( + draft_worker.cuda_graph_runner, + self.target_model_runner.decode_cuda_graph_runner, + draft_worker.cuda_graph_runner_for_draft_extend, + ) + self.device = worker.device + + self._init_state() + + self.disabled_reason = self._construction_unsupported_reason() + if self.disabled_reason is not None: + logger.warning( + "%s disabled: %s. " + "Falling back to the per-stage CUDA graphs.", + switches.STEP_GRAPH, + self.disabled_reason, + ) + return + + self._draft_extend_read_ends = self._draft_extend_backend().shared_read_ends( + ForwardMode.DRAFT_EXTEND_V2 + ) + tp_group = get_tp_group() + self._sampled_broadcast_ok = tp_group.world_size == 1 or ( + tp_group.pynccl_comm is not None + ) + self._stream = get_stream(_CAPTURE_STREAM_NAME) + logger.info( + "Speculative step CUDA graph enabled: each decode step is captured " + "at its first eligible use (batch size <= %d).", + _MAX_BS, + ) + if switches.enabled(switches.STEP_GRAPH_FUSED_INPUT_COPY): + from sglang.kernels.ops.memory.small_copy import try_small_copy + + self._small_copy = try_small_copy + + # ------------------------------------------------------------------ setup + + def _init_state(self) -> None: + self._steps: Dict[tuple, _StepGraph] = {} + # Graphs that must stay alive although they are not replayed (a capture + # that a peer rank failed); released together with every key. + self._parked_steps: List[_StepGraph] = [] + self._warmup_counts: Dict[tuple, int] = {} + self._uncapturable: Dict[tuple, str] = {} + self._memory_retry_at: Dict[tuple, int] = {} + self._output_layouts: Dict[tuple, tuple] = {} + self._failed_keys = 0 + self._num_steps_seen = 0 + self._in_worker_call = False + self._fallbacks: Counter = Counter() + self._stats = Counter() + self._sim_host: Optional[torch.Tensor] = None + self._sim_events: List[Optional[torch.cuda.Event]] = [None] * _SIM_LEN_SLOTS + self._sim_slot = 0 + # SGLANG_SPEC_STEP_GRAPH_FUSED_INPUT_COPY: the one-launch input copy. + self._small_copy = None + self._logged_input_copy = set() + self._stream: Optional[torch.cuda.Stream] = None + self._sampled_broadcast_ok = True + # The batch and grammar barrier of the step being replayed, for host breaks. + self._live_batch = None + self._live_grammar_barrier = None + self._draft_extend_read_ends = SharedReadEnds.UNKNOWN + self.disabled_reason: Optional[str] = None + + def _draft_extend_backend(self): + draft_worker = self.worker.draft_worker + return ( + draft_worker.draft_extend_attn_backend + or self.draft_model_runner.attn_backend + ) + + def _construction_unsupported_reason(self) -> Optional[str]: + names = ("draft", "target verify", "draft extend") + for name, runner in zip(names, self.stage_runners): + if runner is None: + return f"no {name} CUDA graph runner" + backend = getattr(runner, "backend", None) + if not isinstance(backend, FullCudaGraphBackend): + return f"the {name} CUDA graph backend is not the full backend" + if not backend.keep_raw_graph: + return f"the {name} CUDA graphs do not keep their raw graphs" + draft_worker = self.worker.draft_worker + backends = ( + (draft_worker.draft_attn_backend, ForwardMode.DECODE), + (self.target_model_runner.attn_backend, ForwardMode.TARGET_VERIFY), + (self._draft_extend_backend(), ForwardMode.DRAFT_EXTEND_V2), + ) + for backend, mode in backends: + if not out_graph_metadata_capturable(backend, mode): + return ( + f"attention backend {type(backend).__name__} does not declare " + f"capturable {mode.name} metadata" + ) + if self.worker.plan_stream is not None: + return "the overlap plan stream is not supported" + for model_runner in (self.target_model_runner, self.draft_model_runner): + if model_runner.device_timer is not None: + return "the device timer is not supported" + if model_runner.canary_manager is not None: + return "the KV canary is not supported" + if model_runner.msprobe_debugger is not None: + return "the msprobe debugger is not supported" + return None + + # ------------------------------------------------------------- per step + + def _ineligible_reason( + self, batch: ScheduleBatch, pp_proxy_tensors: Optional[Any] + ) -> Optional[str]: + if batch.forward_mode.is_idle(): + return "idle batch" + bs = batch.seq_lens.shape[0] + if bs > _MAX_BS: + return f"batch size above {_MAX_BS}" + spec_info = batch.spec_info + if spec_info is None: + return "no draft input" + if spec_info.is_verify_input(): + return "prebuilt verify input" + if self.worker.speculative_num_steps == 0: + return "speculative_num_steps is 0" + if batch.return_logprob: + return "logprobs requested" + if batch.return_hidden_states: + return "hidden states requested" + if pp_proxy_tensors is not None: + return "pipeline proxy tensors" + if batch.has_grammar: + for req in batch.reqs: + if req.grammar is None: + return "constrained and unconstrained requests mixed" + if isinstance(req.grammar, InvalidGrammarObject): + return "invalid grammar" + if batch.seq_lens_cpu is not None or batch.seq_lens_sum is not None: + return "host sequence lengths" + if batch.hisparse_coordinator is not None: + return "hisparse" + if batch.mamba_track_indices is not None: + return "mamba state tracking" + if batch.multimodal_inputs is not None and any( + x is not None for x in batch.multimodal_inputs + ): + return "multimodal inputs" + sampling_info = batch.sampling_info + if sampling_info.has_custom_logit_processor: + return "custom logit processor" + if ( + sampling_info.acc_additive_penalties is not None + or sampling_info.acc_scaling_penalties is not None + ): + return "penalties" + if sampling_info.logit_bias is not None: + return "logit bias" + if sampling_info.has_watermark_candidates: + return "watermark" + if sampling_info.sampling_mask_batch_indices is not None: + return "sampling masks requested" + if not sampling_info.is_all_greedy and not self._sampled_broadcast_ok: + return "sampled verify without a capturable TP broadcast" + for name, value in vars(batch).items(): + if ( + name not in _BATCH_STEP_INPUTS + and name not in _BATCH_STEP_OUTPUTS + and _is_device_tensor(value) + ): + return f"batch tensor {name}" + return None + + def _key(self, batch: ScheduleBatch, step_tensors) -> tuple: + sampling_info = batch.sampling_info + spec_info = batch.spec_info + return ( + batch.seq_lens.shape[0], + sampling_info.is_all_greedy, + sampling_info.need_top_k_sampling, + sampling_info.need_top_p_sampling, + sampling_info.need_min_p_sampling, + spec_info.capture_hidden_mode, + spec_info.future_dsa_topk_indices_available, + spec_info.cuda_graph_compatible, + ( + tuple(type(req.grammar).__name__ for req in batch.reqs) + if batch.has_grammar + else None + ), + tuple( + (holder, name, tuple(t.shape), t.dtype) + for holder, name, t in step_tensors + ), + ) + + def _fallback(self, reason: str) -> None: + if reason not in self._fallbacks: + logger.info( + "Speculative step CUDA graph: running the regular step (%s); " + "logged once per reason.", + reason, + ) + self._fallbacks[reason] += 1 + + def try_replay( + self, + batch: ScheduleBatch, + on_publish=None, + grammar_barrier=None, + pp_proxy_tensors=None, + ) -> Optional[GenerationBatchResult]: + """Run this decode step as one graph. Returns None when the caller must + run the regular step instead.""" + if self._in_worker_call or self.disabled_reason is not None: + return None + self._num_steps_seen += 1 + reason = self._ineligible_reason(batch, pp_proxy_tensors) + if reason is not None: + self._fallback(reason) + return None + step_tensors = _step_tensors(batch) + key = self._key(batch, step_tensors) + + step = self._steps.get(key) + if step is not None: + if all( + backend.graph_epoch == epoch for backend, epoch in step.stage_graphs + ): + return self._replay(step, batch, on_publish, grammar_barrier) + self._reset("the per-stage CUDA graphs were recaptured") + + reason = self._uncapturable.get(key) + if reason is not None: + self._fallback(reason) + return None + if len(self._steps) >= _MAX_KEYS: + self._fallback("captured key limit") + return None + + warmed = self._warmup_counts.pop(key, 0) + if warmed < _NUM_WARMUP_STEPS: + self._warmup_counts[key] = warmed + 1 + while len(self._warmup_counts) > _MAX_WARMUP_KEYS: + self._warmup_counts.pop(next(iter(self._warmup_counts))) + if warmed + 1 == _NUM_WARMUP_STEPS: + return self._probe_step(key, batch, on_publish, grammar_barrier) + return None + self._warmup_counts[key] = warmed + if self._memory_retry_at.get(key, 0) > self._num_steps_seen: + self._fallback("not enough free memory to capture") + return None + return self._capture_and_replay( + key, batch, step_tensors, on_publish, grammar_barrier + ) + + # ------------------------------------------------------- warmup probe + + def _run_regular_step(self, batch, on_publish, grammar_barrier, pp_proxy_tensors): + self._in_worker_call = True + try: + return self.worker.forward_batch_generation( + batch, + on_publish=on_publish, + grammar_barrier=grammar_barrier, + pp_proxy_tensors=pp_proxy_tensors, + ) + finally: + self._in_worker_call = False + + def _probe_step(self, key, batch, on_publish, grammar_barrier): + """Run a regular step while counting its per-stage graph replays.""" + with ChildGraphCapture(splice=False) as probe: + result = self._run_regular_step(batch, on_publish, grammar_barrier, None) + reason = None + if probe.num_graphs != _NUM_STAGE_GRAPHS: + reason = ( + f"the step replays {probe.num_graphs} per-stage graphs " + f"(expected {_NUM_STAGE_GRAPHS})" + ) + elif any(delta != 0 for delta in probe.generator_offset_deltas): + reason = "a per-stage graph draws random numbers" + (ok,) = _tp_min(reason is None) + if not ok: + reason = reason or "a peer rank cannot capture this step" + self._uncapturable[key] = reason + self._warmup_counts.pop(key, None) + self._fallback(reason) + else: + # The output ring is allocated before the capture, from this layout. + self._output_layouts[key] = tuple( + (tuple(t.shape), t.dtype) + for t in (result.next_token_ids, result.accept_lens) + ) + return result + + # --------------------------------------------------------------- capture + + def _memory_ok(self) -> bool: + largest = max( + (s.memory_bytes for s in self._steps.values()), + default=_FIRST_KEY_MEMORY_ESTIMATE, + ) + free = torch.cuda.mem_get_info()[0] + (ok,) = _tp_min(free >= largest + _CAPTURE_MEMORY_HEADROOM) + return bool(ok) + + def _capture_and_replay( + self, key, batch, step_tensors, on_publish, grammar_barrier + ): + if not self._memory_ok(): + self._memory_retry_at[key] = self._num_steps_seen + _MEMORY_RETRY_STEPS + self._fallback("not enough free memory to capture") + return None + + # Free unreachable graphs now: destroying a graph while a capture is in + # progress invalidates the capture, and the collector may run at any time. + gc.collect() + torch.cuda.synchronize() + get_tp_group().barrier() + reserved_before = torch.cuda.memory_reserved() + start = time.perf_counter() + step, error = None, None + gc_was_enabled = gc.isenabled() + gc.disable() + try: + step = self._capture(batch, step_tensors, self._output_layouts.get(key)) + except Exception as e: # noqa: BLE001 (the key runs the regular step) + error = e + finally: + if gc_was_enabled: + gc.enable() + with torch.cuda.stream(self._stream): + if torch.cuda.is_current_stream_capturing(): + # The capture contexts end their captures, also on errors. + self.disabled_reason = "a capture left its stream capturing" + memory = max(0, torch.cuda.memory_reserved() - reserved_before) + ok, neg_memory = _tp_min(step is not None, -memory) + self._warmup_counts.pop(key, None) + if not ok: + self._failed_keys += 1 + self._stats["capture_failures"] += 1 + reason = ( + f"capture failed: {type(error).__name__}: {error}" + if error is not None + else "capture failed on a peer rank" + ) + self._uncapturable[key] = "capture failed" + logger.warning( + "Speculative step CUDA graph: capture failed for batch size %d; " + "this key runs the regular step (%s).", + batch.seq_lens.shape[0], + reason, + exc_info=error, + ) + if step is not None: + # Keep a graph that a peer rank failed to capture: graphs are + # only released all together (see _reset). + self._parked_steps.append(step) + if self._failed_keys >= _MAX_FAILED_KEYS: + self.disabled_reason = ( + f"{self._failed_keys} keys failed to capture; last: {reason}" + ) + if self.disabled_reason is not None: + logger.warning( + "Speculative step CUDA graph turned off for this process: %s", + self.disabled_reason, + ) + return None + + step.memory_bytes = -neg_memory + self._steps[key] = step + self._stats["captures"] += 1 + logger.info( + "Speculative step CUDA graph captured: batch size %d, %s, %s, " + "%d static inputs, %d graph segment(s), %.1f ms, %.1f MB (%d keys).", + batch.seq_lens.shape[0], + "greedy" if batch.sampling_info.is_all_greedy else "sampled", + "constrained" if batch.has_grammar else "unconstrained", + len(step.static_inputs), + len(step.graph._segments), + (time.perf_counter() - start) * 1e3, + step.memory_bytes / 2**20, + len(self._steps), + ) + logger.debug("Speculative step CUDA graph key: %s", key) + return self._replay(step, batch, on_publish, grammar_barrier) + + def _capture_batch(self, batch, step_tensors): + capture_batch = copy.copy(batch) + capture_batch.spec_info = copy.copy(batch.spec_info) + capture_batch.sampling_info = copy.copy(batch.sampling_info) + for name in _BATCH_STEP_OUTPUTS: + setattr(capture_batch, name, None) + for name in _DRAFT_INPUT_STEP_OUTPUTS: + if hasattr(capture_batch.spec_info, name): + setattr(capture_batch.spec_info, name, None) + holders = _holders(capture_batch) + names, static_copies = [], [] + for holder, name, tensor in step_tensors: + static = tensor.clone() + setattr(holders[holder], name, static) + names.append((holder, name)) + static_copies.append(static) + return capture_batch, names, static_copies + + def _capture(self, batch, step_tensors, output_layout) -> _StepGraph: + if output_layout is None: + raise RuntimeError("no output layout from the warm-up step") + capture_batch, names, static_copies = self._capture_batch(batch, step_tensors) + # Allocated outside the capture: a replay's outputs must survive the next + # replay, so they cannot share memory with the graph's intermediates. + ring_slot = torch.zeros(1, dtype=torch.int64, device=self.device) + ring_next_token_ids, ring_accept_lens = ( + torch.empty((_OUTPUT_RING_SIZE, *shape), dtype=dtype, device=self.device) + for shape, dtype in output_layout + ) + # Simulated acceptance (benchmarking): whenever the step applies it, the + # length comes from this device scalar, drawn on the host per replay. + sim_accept_len = torch.ones(1, dtype=torch.int32, device=self.device) + draw_args: List[Tuple[float, str, int]] = [] + overrides = { + "spec_simulated_accept_index": self._simulated_accept_override( + sim_accept_len, draw_args + ) + } + + grammar = _GrammarStep(self) if batch.has_grammar else None + if grammar is not None: + overrides.update(grammar.overrides()) + + graph = BreakableCUDAGraph() + self._in_worker_call = True + try: + with ( + graph_capture(stream=self._stream) as context, + ChildGraphCapture(splice=True, overrides=overrides) as children, + BreakableCUDAGraphCapture( + graph, + pool=torch.cuda.graph_pool_handle(), + stream=context.stream, + capture_error_mode="thread_local", + ), + ): + result = self.worker.forward_batch_generation( + capture_batch, + on_publish=None, + grammar_barrier=None, + pp_proxy_tensors=None, + ) + ring_next_token_ids.index_copy_( + 0, ring_slot, result.next_token_ids[None] + ) + ring_accept_lens.index_copy_(0, ring_slot, result.accept_lens[None]) + ring_slot.add_(1).remainder_(_OUTPUT_RING_SIZE) + _check_host_copies() + finally: + self._in_worker_call = False + # The capture recorded no real event; the scheduler must not wait on it. + self.draft_model_runner.shared_read_done_event = None + + if children.num_graphs != _NUM_STAGE_GRAPHS: + raise RuntimeError( + f"captured {children.num_graphs} per-stage graphs " + f"(expected {_NUM_STAGE_GRAPHS})" + ) + num_breaks = 2 if grammar is not None else 0 + if len(graph._break_fns) != num_breaks or ( + grammar is not None and grammar.mask is None + ): + raise RuntimeError( + f"captured {len(graph._break_fns)} host breaks (expected {num_breaks})" + ) + if len(draw_args) > 1: + raise RuntimeError( + f"simulated acceptance applied {len(draw_args)} times (expected 1)" + ) + result.extra_keep_alive_refs = [] + return _StepGraph( + graph=graph, + input_names=names, + static_inputs=static_copies, + result=result, + stage_graphs=list(children.graph_sources), + ring_slot=ring_slot, + ring_next_token_ids=ring_next_token_ids, + ring_accept_lens=ring_accept_lens, + ring_views=[ + (ring_next_token_ids[i], ring_accept_lens[i]) + for i in range(_OUTPUT_RING_SIZE) + ], + sim_accept_len=sim_accept_len if draw_args else None, + sim_draw_args=draw_args[0] if draw_args else None, + grammar=grammar, + ) + + @staticmethod + def _simulated_accept_override(sim_accept_len, draw_args): + from sglang.srt.speculative import spec_utils + + def generate_simulated_accept_index( + original, + accept_index, + predict, + num_correct_drafts, + candidates, + target_predict, + bs, + spec_steps, + simulate_acc_len: float = spec_utils.SIMULATE_ACC_LEN, + simulate_acc_method: str = spec_utils.SIMULATE_ACC_METHOD, + simulate_acc_token_mode: str = spec_utils.SIMULATE_ACC_TOKEN_MODE, + ): + assert simulate_acc_len > 0.0 + draw_args.append((simulate_acc_len, simulate_acc_method, spec_steps + 1)) + return simulated_accept_index_from_device_length( + accept_index, + predict, + num_correct_drafts, + candidates, + target_predict, + spec_steps, + sim_accept_len, + simulate_acc_token_mode, + ) + + return generate_simulated_accept_index + + # ---------------------------------------------------------------- replay + + def _load_simulated_accept_len(self, step: _StepGraph) -> None: + from sglang.srt.speculative.spec_utils import sample_simulated_acc_len + + if self._sim_host is None: + self._sim_host = torch.empty( + _SIM_LEN_SLOTS, dtype=torch.int32, pin_memory=True + ) + slot = self._sim_slot + self._sim_slot = (slot + 1) % _SIM_LEN_SLOTS + event = self._sim_events[slot] + if event is None: + event = self._sim_events[slot] = torch.cuda.Event() + else: + # The copy that last read this slot was enqueued steps ago. + event.synchronize() + self._sim_host[slot] = sample_simulated_acc_len(*step.sim_draw_args) + step.sim_accept_len.copy_(self._sim_host[slot : slot + 1], non_blocking=True) + event.record() + + def _replay( + self, step: _StepGraph, batch, on_publish, grammar_barrier + ) -> GenerationBatchResult: + holders = _holders(batch) + path = _copy_step_inputs( + step.static_inputs, + [getattr(holders[holder], name) for holder, name in step.input_names], + self._small_copy, + ) + if self._small_copy is not None: + self._stats[f"{path}_input_copies"] += 1 + key = (path, len(step.static_inputs)) + if key not in self._logged_input_copy: + self._logged_input_copy.add(key) + logger.info( + "Speculative step CUDA graph: %s input copy for %d tensors.", + path.replace("_", "-"), + len(step.static_inputs), + ) + if step.sim_accept_len is not None: + self._load_simulated_accept_len(step) + + slot = step.num_replays % _OUTPUT_RING_SIZE + previous = step.ring_results[slot] + if previous is not None: + # The scheduler copies a result to the host on its copy stream, right + # after the forward that produced it. Do not overwrite a slot that a + # copy may still read: wait for it, or detach a result whose copy was + # never issued. + if previous.copy_done is not None: + if not previous.copy_done.query(): + self._stats["ring_waits"] += 1 + previous.copy_done.synchronize() + elif previous.next_token_ids is step.ring_views[slot][0]: + self._stats["ring_detaches"] += 1 + previous.next_token_ids = previous.next_token_ids.clone() + previous.accept_lens = previous.accept_lens.clone() + + self._live_batch, self._live_grammar_barrier = batch, grammar_barrier + try: + with graph_pool_replay_scope(): + step.graph.replay() + finally: + self._live_batch = self._live_grammar_barrier = None + step.num_replays += 1 + + template = step.result + result = copy.copy(template) + result.next_token_ids, result.accept_lens = step.ring_views[slot] + result.next_draft_input = copy.copy(template.next_draft_input) + if template.logits_output is not None: + result.logits_output = copy.copy(template.logits_output) + result.extra_keep_alive_refs = [] + result.copy_done = None + step.ring_results[slot] = result + + if on_publish is not None: + on_publish(result.new_seq_lens) + if self._draft_extend_read_ends is SharedReadEnds.UNKNOWN: + self.draft_model_runner.shared_read_done_event = None + else: + # One event after the whole step covers every phase's reads. + read_done = torch.cuda.Event() + read_done.record() + self.draft_model_runner.shared_read_done_event = read_done + self.target_model_runner.forward_pass_id += 1 + self._stats["replays"] += 1 + return result + + def _reset(self, reason: str) -> None: + """Release every captured key together (never one key alone: a graph + may reference memory that an earlier capture on the same stream owns).""" + logger.info( + "Speculative step CUDA graph: dropping all captured keys (%s).", reason + ) + torch.cuda.synchronize() + self._steps.clear() + self._parked_steps.clear() + self._warmup_counts.clear() + gc.collect() + self._stats["resets"] += 1 + + # ----------------------------------------------------------------- stats + + def stats(self) -> Dict[str, Any]: + return { + "replays": self._stats["replays"], + "captures": self._stats["captures"], + "capture_failures": self._stats["capture_failures"], + "resets": self._stats["resets"], + "ring_waits": self._stats["ring_waits"], + "ring_detaches": self._stats["ring_detaches"], + "fused_input_copies": self._stats["fused_input_copies"], + "partial_input_copies": self._stats["partial_input_copies"], + "per_tensor_input_copies": self._stats["per_tensor_input_copies"], + "keys": len(self._steps), + "fallbacks": dict(self._fallbacks), + "uncapturable_keys": len(self._uncapturable), + "disabled_reason": self.disabled_reason, + } diff --git a/sgl_nvidia_mega/spec_step/switches.py b/sgl_nvidia_mega/spec_step/switches.py new file mode 100644 index 00000000..c198c28f --- /dev/null +++ b/sgl_nvidia_mega/spec_step/switches.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The environment switches of the speculative step graph and its fusions. + +Each defaults to off. ``hooks.register`` installs only the hooks of the switches +that are on, so with all of them off SGLang runs unchanged. The names are the ones +of SGLang's tree with these changes, so a launch script serves both. +""" + +from __future__ import annotations + +import os + +# One CUDA graph per EAGLE v2 decode step (SGLang's tree: +# --enable-speculative-step-cuda-graph), reusing the per-stage CUDA graphs. +STEP_GRAPH = "SGLANG_NVIDIA_MEGAKERNEL_SPEC_STEP_GRAPH" +# Copy the step's live inputs into the graph's static buffers in one launch. +STEP_GRAPH_FUSED_INPUT_COPY = "SGLANG_SPEC_STEP_GRAPH_FUSED_INPUT_COPY" +# predict / accept_index / num_correct_drafts as views of one buffer: one rank-0 +# TP broadcast of the verify result instead of three. +PACKED_VERIFY_BROADCAST = "SGLANG_SPEC_PACKED_VERIFY_BROADCAST" +# Chain sampled verify: temperatures not expanded, no zero-filled draft_probs. +LEAN_SAMPLED_VERIFY = "SGLANG_SPEC_LEAN_SAMPLED_VERIFY" +# With LEAN_SAMPLED_VERIFY: the sampling passes split over vocabulary chunks. +CHUNKED_VERIFY_SAMPLING = "SGLANG_SPEC_CHUNKED_VERIFY_SAMPLING" +# The topk-1 draft-extend argmax, unit probs and DSA seed gather in two launches. +FUSED_DRAFT_EXTEND_TAIL = "SGLANG_SPEC_FUSED_DRAFT_EXTEND_TAIL" +# The relay rows (FutureMap): one scatter per stash, row views for one request. +FUSED_RELAY = "SGLANG_SPEC_FUSED_RELAY" + +ALL = ( + STEP_GRAPH, + STEP_GRAPH_FUSED_INPUT_COPY, + PACKED_VERIFY_BROADCAST, + LEAN_SAMPLED_VERIFY, + CHUNKED_VERIFY_SAMPLING, + FUSED_DRAFT_EXTEND_TAIL, + FUSED_RELAY, +) + + +def enabled(name: str) -> bool: + return os.environ.get(name, "").strip().lower() in ("1", "true", "yes", "on") + + +def any_enabled() -> bool: + return any(enabled(name) for name in ALL) From d84981d4e6f751e5ebbce1b4fd0883e6a383bbb3 Mon Sep 17 00:00:00 2001 From: Brayden Zhong <184021590+b8zhong@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:26:22 +0000 Subject: [PATCH 2/8] [Fix] spec_step: keep the step graph on torch 2.14 and make lifted.py importable - hooks.py: torch 2.14's `_CUDAGraph` binding reads `keep_graph` in `__init__`, so overriding only `__new__` let a bare `torch.cuda.CUDAGraph()` drop its raw graph at capture end, and decode graph capture failed in cuGraphInstantiate with CUDA_ERROR_INVALID_VALUE. - lifted.py: defer annotations. `Tuple`, `Optional` and `torch` are bound only in the SGLang module each function is rebound into, so the first verify raised `NameError: Tuple`. - lifted.py: `eagle_sample` drops its inline sampler imports, which referenced `_is_npu` (removed from eagle_utils by sgl-project/sglang#45504), and uses eagle_utils' module names. - lifted.py: `_draft_extend_for_decode` passes `sampling_seed=` and `positions=` again, as current SGLang does. Tested on 8x B300, GLM-5.3-NVFP4, TP8, EAGLE 3/1/4, SGLang f4a2df3dac, all spec_step switches on. --- sgl_nvidia_mega/spec_step/hooks.py | 3 +++ sgl_nvidia_mega/spec_step/lifted.py | 39 +++-------------------------- 2 files changed, 7 insertions(+), 35 deletions(-) diff --git a/sgl_nvidia_mega/spec_step/hooks.py b/sgl_nvidia_mega/spec_step/hooks.py index 50e8d82d..03034734 100644 --- a/sgl_nvidia_mega/spec_step/hooks.py +++ b/sgl_nvidia_mega/spec_step/hooks.py @@ -88,6 +88,9 @@ class KeepGraph(base): def __new__(cls, keep_graph=False, *args, **kwargs): return super().__new__(cls, True, *args, **kwargs) + def __init__(self, keep_graph=False, *args, **kwargs): + super().__init__(True, *args, **kwargs) + return KeepGraph diff --git a/sgl_nvidia_mega/spec_step/lifted.py b/sgl_nvidia_mega/spec_step/lifted.py index 319bbfdc..ebbac5aa 100644 --- a/sgl_nvidia_mega/spec_step/lifted.py +++ b/sgl_nvidia_mega/spec_step/lifted.py @@ -12,6 +12,8 @@ diff against SGLang's function shows only the fusion. """ +from __future__ import annotations + import logging from sgl_nvidia_mega.spec_step import switches as _spec_switches @@ -273,48 +275,13 @@ def eagle_sample( tp_group, verify_outputs, predict, accept_index, num_correct_drafts ) else: - if _is_npu: - from sgl_kernel_npu.sample import ( - chain_speculative_sampling_triton, - top_k_renorm_prob, - top_p_renorm_prob, - tree_speculative_sampling_target_only, - ) - else: - from sglang.kernels.ops.speculative.reject_sampling import ( - chain_speculative_sampling_triton, - ) - - # if/else, not a ternary: the CUDA-only name still has to resolve in the - # branch not taken, and HIP only reaches here with rejection sampling on. if use_rejection_sampling: sampling_fn = chain_speculative_sampling_triton if get_spec().speculative_use_block_verification: sampling_fn = partial(sampling_fn, block_verification=True) else: - if _is_cuda: - from sglang.kernels.ops.speculative.sampling import ( - tree_speculative_sampling_target_only, - ) - elif not _is_npu: - from sgl_kernel import tree_speculative_sampling_target_only - sampling_fn = tree_speculative_sampling_target_only - if _is_hip: - # Same names, same contract: dflash_utils.py aliases these too. - from sglang.kernels.ops.sampling.renorm_triton import ( - top_k_renorm_probs_triton as top_k_renorm_prob, - ) - from sglang.kernels.ops.sampling.renorm_triton import ( - top_p_renorm_probs_triton as top_p_renorm_prob, - ) - elif _is_cuda: - from flashinfer.sampling import top_k_renorm_probs as top_k_renorm_prob - from flashinfer.sampling import top_p_renorm_probs as top_p_renorm_prob - elif not _is_npu: - from sgl_kernel import top_k_renorm_prob, top_p_renorm_prob - # Lean form (CUDA, chains, no rejection sampling): the same division # broadcast per request instead of expanding the temperatures, and no # zero-filled draft_probs (the sampling kernel's target-only path). @@ -667,6 +634,8 @@ def _draft_extend_for_decode( draft_logits_output.next_token_logits, batch.sampling_info.temperatures, batch.sampling_info.top_ks, + sampling_seed=batch.sampling_info.sampling_seed, + positions=batch_result.new_seq_lens, ) elif self.topk == 1 and _is_hip: ret_topk_p, ret_topk_index = draft_topk1_argmax_only( From de13483b00fe4bbf6479c83b5dd2c30380c5a2c9 Mon Sep 17 00:00:00 2001 From: Brayden Zhong <184021590+b8zhong@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:53:04 +0000 Subject: [PATCH 3/8] [Fix] glm_dsa: accept --speculative-algorithm NEXTN The construction check compared the raw server argument with EAGLE, so a NEXTN launch (the InferenceX GLM-5.3 B300 AgentX recipe) was refused with "speculative algorithm 'NEXTN' is not supported (EAGLE / NEXTN only)". NEXTN resolves to the same EAGLE worker; the spec's requirement and the spec_step runner already accept it. --- sgl_nvidia_mega/glm_dsa/eligibility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sgl_nvidia_mega/glm_dsa/eligibility.py b/sgl_nvidia_mega/glm_dsa/eligibility.py index ed4a61d3..71c2a33e 100644 --- a/sgl_nvidia_mega/glm_dsa/eligibility.py +++ b/sgl_nvidia_mega/glm_dsa/eligibility.py @@ -164,7 +164,7 @@ def construction_reasons(config, server_args, *, device_capability=None) -> List "requires speculative decoding (EAGLE / NEXTN with the MTP layer); " "plain decode is not supported" ) - elif str(algo).upper() != "EAGLE": + elif str(algo).upper() not in ("EAGLE", "NEXTN"): reasons.append( f"speculative algorithm {algo!r} is not supported (EAGLE / NEXTN only)" ) From 52e423de9425bea6770c06c82eabafcf6e784b0b Mon Sep 17 00:00:00 2001 From: shuw Date: Thu, 29 Oct 2026 11:51:48 -0500 Subject: [PATCH 4/8] [Perf] glm_dsa: split the verify path into packages; Rubin round-7 hop scheduling switches Split the GLM-5 DSA verify path into the attention/, boundary/, moe/ and runtime/ packages plus plan.py (runtime.py kept as runtime_legacy.py), and fix two leftovers of the split: a stray @dataclass in moe/__init__.py and a duplicated module header in boundary/__init__.py. The spec's model_disable_reasons now calls eligibility.model_construction_reasons with the HF config and the server arguments. Port two hop scheduling ideas from the TRT-LLM Rubin round-7 branch, both off by default and output-neutral: - SGLANG_NVIDIA_MEGAKERNEL_GLM_HOP_MAX_SHARED_CARVEOUT=1: every hop kernel prefers the maximum shared-memory carveout, so an SM a hop CTA lingers on can host the next layer's large-shared-memory kernels under PDL. - SGLANG_NVIDIA_MEGAKERNEL_GLM_HOP_CTAS_PER_SM=1|2: pad the o_proj consumer hop's launch shared memory (sized from the device's per-SM limits) so at most N CTAs share an SM while polling. Round 7's single-loop 16-slot polling does not apply: at 8 ranks the consumer already polls all sources in parallel. Co-Authored-By: Claude Opus 5.5 --- sgl_nvidia_mega/glm_dsa/__init__.py | 43 +- sgl_nvidia_mega/glm_dsa/attention/__init__.py | 22 + sgl_nvidia_mega/glm_dsa/attention/fused.py | 728 ++++++++++++++++++ sgl_nvidia_mega/glm_dsa/attention/step.py | 385 +++++++++ sgl_nvidia_mega/glm_dsa/boundary/__init__.py | 386 ++++++++++ sgl_nvidia_mega/glm_dsa/draft.py | 2 +- sgl_nvidia_mega/glm_dsa/draft_attention.py | 7 +- sgl_nvidia_mega/glm_dsa/eligibility.py | 15 +- sgl_nvidia_mega/glm_dsa/hops.py | 24 +- .../glm_dsa/kernels/cute/hop/api.py | 16 +- .../glm_dsa/kernels/cute/hop/o_proj.py | 10 + .../glm_dsa/kernels/cute/launch.py | 61 +- sgl_nvidia_mega/glm_dsa/moe/__init__.py | 169 ++++ sgl_nvidia_mega/glm_dsa/moe/step.py | 180 +++++ sgl_nvidia_mega/glm_dsa/plan.py | 110 +++ sgl_nvidia_mega/glm_dsa/runtime/__init__.py | 74 ++ sgl_nvidia_mega/glm_dsa/runtime/compile.py | 72 ++ sgl_nvidia_mega/glm_dsa/runtime/executor.py | 267 +++++++ sgl_nvidia_mega/glm_dsa/runtime/parity.py | 66 ++ .../glm_dsa/{runtime.py => runtime_legacy.py} | 0 sgl_nvidia_mega/glm_dsa/spec.py | 395 +++++++--- sgl_nvidia_mega/glm_dsa/switches.py | 22 + test/unit/test_contract.py | 15 +- test/unit/test_glm_dsa_config.py | 185 +++++ 24 files changed, 3132 insertions(+), 122 deletions(-) create mode 100644 sgl_nvidia_mega/glm_dsa/attention/__init__.py create mode 100644 sgl_nvidia_mega/glm_dsa/attention/fused.py create mode 100644 sgl_nvidia_mega/glm_dsa/attention/step.py create mode 100644 sgl_nvidia_mega/glm_dsa/boundary/__init__.py create mode 100644 sgl_nvidia_mega/glm_dsa/moe/__init__.py create mode 100644 sgl_nvidia_mega/glm_dsa/moe/step.py create mode 100644 sgl_nvidia_mega/glm_dsa/plan.py create mode 100644 sgl_nvidia_mega/glm_dsa/runtime/__init__.py create mode 100644 sgl_nvidia_mega/glm_dsa/runtime/compile.py create mode 100644 sgl_nvidia_mega/glm_dsa/runtime/executor.py create mode 100644 sgl_nvidia_mega/glm_dsa/runtime/parity.py rename sgl_nvidia_mega/glm_dsa/{runtime.py => runtime_legacy.py} (100%) create mode 100644 test/unit/test_glm_dsa_config.py diff --git a/sgl_nvidia_mega/glm_dsa/__init__.py b/sgl_nvidia_mega/glm_dsa/__init__.py index e5a28762..21572ebe 100644 --- a/sgl_nvidia_mega/glm_dsa/__init__.py +++ b/sgl_nvidia_mega/glm_dsa/__init__.py @@ -1,9 +1,40 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -"""GLM-5 DSA (``GlmMoeDsaForCausalLM``, NVFP4, TP8) megakernel on SM103 / SM107: the -fused MTP target-verify step of 1-4 requests and the fused NextN draft layer. +# Copyright 2026 SGLang Team +"""GLM-5 DSA (GlmMoeDsaForCausalLM, NVFP4, TP8) megakernel on SM103 / SM107. -Read ``spec`` -> ``eligibility`` -> ``runtime`` -> ``providers``; the draft is -``draft_spec`` -> ``draft``. The CuTe DSL kernels are under ``kernels/cute`` and import -no inference engine. +Covers verify decode (``spec.SPEC``) and the NextN MTP draft decode +(``draft_spec.SPEC``). ``eligibility`` admits SM103 (B300) and SM107 (Rubin / +VR NVL72); the two GPU families run the same kernels through CuTe DSL dispatch. """ + +from __future__ import annotations + +MODEL_TYPE = "glm_moe_dsa" +TP_SIZE = 8 +HIDDEN = 6144 + +# Published HuggingFace config fields for the GLM-5.2-NVFP4 / GLM-5 DSA model. +# eligibility.py validates these against the loaded model at startup. +HF_CONFIG = dict( + hidden_size=6144, + num_attention_heads=64, + q_lora_rank=2048, + kv_lora_rank=512, + qk_rope_head_dim=64, + qk_nope_head_dim=192, + v_head_dim=256, + index_n_heads=32, + index_head_dim=128, + index_topk=2048, + n_routed_experts=256, + num_experts_per_tok=8, + n_shared_experts=1, + n_group=1, + topk_group=1, + norm_topk_prob=True, + scoring_func="sigmoid", + topk_method="noaux_tc", + moe_intermediate_size=2048, + first_k_dense_replace=3, + num_hidden_layers=78, +) diff --git a/sgl_nvidia_mega/glm_dsa/attention/__init__.py b/sgl_nvidia_mega/glm_dsa/attention/__init__.py new file mode 100644 index 00000000..0d23f5c2 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/attention/__init__.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""Attention package: stock step bodies (step.py) and fused kernel family (fused.py).""" + +from sgl_nvidia_mega.glm_dsa.attention.step import ( # noqa: F401 + AttentionDecode, + IndexQ, + PrologueOut, + StepContext, +) + +# AttentionGlue and build() are the fused kernel family for this slot. +from sgl_nvidia_mega.glm_dsa.attention.fused import AttentionGlue # noqa: F401 + + +def build(model, derived): + """None if the slot is disabled; otherwise the built AttentionGlue.""" + from sgl_nvidia_mega.glm_dsa.runtime import SLOTS + + if "attention" not in SLOTS: + return None + return AttentionGlue.build(model, derived) diff --git a/sgl_nvidia_mega/glm_dsa/attention/fused.py b/sgl_nvidia_mega/glm_dsa/attention/fused.py new file mode 100644 index 00000000..f57beedb --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/attention/fused.py @@ -0,0 +1,728 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""CuTe DSL kernels in the verify attention of the opt-in fused GLM-5 DSA path. + +``FusedAttention`` serves the ``providers.AttentionDecode`` steps of one verify +row count: ``qkv_a`` on ``qkv_a_proj_small_m`` at up to 8 rows (on layers with +their own indexer together with the indexer's key / head-gate projection, left in +``step.index_kw``; elsewhere the stock fused-A GEMM, and on indexer layers the +stock key / head-gate projection right after it, which the prologue consumes); +``prologue`` on ``mla_prologue`` at SGLang's rounding points (q_a / kv_a norms, +``q_b_proj``, W_UK absorption, RoPE + FP8 query left in ``step.q_fp8``, the MLA +and index-K cache rows), so ``index_k``, ``q_absorb`` and ``rope_cache`` pass +through; ``index_q`` on ``indexer_q_proj``; ``sparse_mla`` on ``sparse_mla_decode`` +over the fused top-k page table (16 warps per CTA up to 4 rows, 8 above); +``uv_proj`` on ``mla_uv_proj``. The indexer's logits and top-k stay stock. +With SGLANG_NVIDIA_MEGAKERNEL_GLM_SHARED_QKV_A_SMALL_M (default on) the layers +without their own top-k also run ``qkv_a`` on the small-M kernel, and with +SGLANG_NVIDIA_MEGAKERNEL_GLM_DNORM (default on) on its epilogue-scaled form, which +consumes the fold hop's deferred input norm. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +import numpy as np +import torch + +from sgl_nvidia_mega.glm_dsa import switches +from sgl_nvidia_mega.glm_dsa.eligibility import HIDDEN +from sgl_nvidia_mega.glm_dsa.attention.step import ( + AttentionDecode, + IndexQ, + PrologueOut, + StepContext, +) +from sgl_nvidia_mega.glm_dsa.runtime.compile import compile_cached, cu_stream + +# One tensor-parallel rank (TP 8). +HEADS, Q_LORA, KV_LORA, ROPE, NOPE, VALUE_DIM = 8, 2048, 512, 64, 192, 256 +INDEX_HEADS, INDEX_DIM, TOPK, INDEX_PAGE = 32, 128, 2048, 64 +KV_B_ROWS = HEADS * (NOPE + VALUE_DIM) +Q_B_ROWS = HEADS * (NOPE + ROPE) +MQA_DIM = KV_LORA + ROPE +QKV_A_ROWS = Q_LORA + KV_LORA + ROPE +INDEX_KW = INDEX_DIM + INDEX_HEADS +LOG2E = 1.4426950408889634 + +logger = logging.getLogger(__name__) + + +def _prologue_numerics(): + """The prologue's numeric options: this engine's stock prologue arithmetic.""" + from sgl_nvidia_mega.glm_dsa.kernels.cute.attn.mla_prologue import PrologueNumerics + + return PrologueNumerics( + norm_order="strided_8x256", + norm_weight_plus_zero=True, + kv_norm_bf16=True, + skip_slot0=True, + index_mean_sum="sequential", + index_rope_odd="round_x1_cos", + index_k_bf16=True, + index_scale="fp32_amax", + q_rope="one_product_rounded", + ) + + +# the RoPE arithmetic of this engine's stock indexer query path +INDEX_Q_ROPE_FORM = "mul_sub" + +# L2 prefetch hints (evict_last; no output depends on them) of the value +# up-projection while it waits for the attention core: on the target's layers this +# layer's o_proj weight, which the o_proj hop reads next (of each 48-row slab, the +# rows one hop CTA reads, its first 122,880 B: 15 MiB over the 128 slabs), then the +# first 8 MiB of the next layer's q_a / kv_a weight. +NEXT_QKV_A_PREFETCH_BYTES = 8 << 20 +O_PROJ_STAGE_SLAB_BYTES = 48 * 2048 * 2 +O_PROJ_STAGE_PREFIX_BYTES = 122_880 +EVICT_LAST = 0 + +Slot = Tuple[int, int, int, int, int] +NO_SLOT: Slot = (0, 0, 0, 0, 0) + + +def kernel_rows(rows: int) -> Optional[int]: + """The row capacity (8 or 16) of the kernel forms that serve ``rows``.""" + return 8 if 1 <= rows <= 8 else 16 if rows <= 16 else None + + +def sparse_warps(rows: int) -> int: + """Warps per CTA of the sparse MLA form: one CTA per SM up to 4 rows, two above.""" + return 16 if rows <= 4 else 8 + + +def _f32(x: float) -> float: + return float(np.float32(x)) + + +def _i64(*values) -> list: + from cutlass import Int64 + + return [Int64(v) for v in values] + + +def _addr(t: Optional[torch.Tensor]) -> Tuple[int, int]: + """(address, row stride) of ``t``; (0, 0) for None.""" + return (0, 0) if t is None else (t.data_ptr(), t.stride(0)) + + +def _fail(what: str, cls=RuntimeError) -> Exception: + return cls(f"fused verify attention: {what}") + + +def _prefetch_slot(w, nbytes: int, stride: int = 0, prefix: int = 0) -> Slot: + """An evict_last L2 prefetch slot over the first ``nbytes`` of ``w`` (with + ``stride``: the first ``prefix`` bytes of every ``stride`` bytes), or no slot + unless ``w`` is a contiguous, 128-byte aligned CUDA tensor.""" + if w is None or not w.is_cuda or not w.is_contiguous() or w.data_ptr() % 128: + return NO_SLOT + size = min(nbytes, w.numel() * w.element_size()) + return (w.data_ptr(), size, stride, prefix, EVICT_LAST) + + +def _o_proj_stage_slot(w: torch.Tensor) -> Slot: + """The strided staging slot over this layer's o_proj weight, or no slot unless + it is the expected BF16 [hidden, heads * value_dim] tensor.""" + if w.dtype != torch.bfloat16 or tuple(w.shape) != (HIDDEN, HEADS * VALUE_DIM): + return NO_SLOT + return _prefetch_slot( + w, + w.numel() * w.element_size(), + O_PROJ_STAGE_SLAB_BYTES, + O_PROJ_STAGE_PREFIX_BYTES, + ) + + +def _slot_args(slot: Slot) -> tuple: + from cutlass import Int32 + + return *_i64(*slot[:4]), Int32(slot[4]) + + +def _bf16_weight(linear, shape, what: str) -> torch.Tensor: + weight = linear.weight + if weight.dtype != torch.bfloat16 or tuple(weight.shape) != shape: + raise ValueError( + f"fused verify attention needs a bf16 {what} weight of shape {shape} per " + f"rank, got {weight.dtype} {tuple(weight.shape)}" + ) + return weight + + +def _check_rows(t: torch.Tensor, rows: int, cols: int, dtype, what: str) -> None: + """A [rows, cols] tensor with unit column stride and 16-byte aligned rows.""" + if ( + t.dtype != dtype + or t.dim() != 2 + or tuple(t.shape) != (rows, cols) + or t.stride(1) != 1 + or t.data_ptr() % 16 + or (t.stride(0) * t.element_size()) % 16 + ): + raise _fail(f"unexpected {what} {tuple(t.shape)} / {t.stride()} / {t.dtype}") + + +def _check_index(t: torch.Tensor, rows: int, what: str) -> None: + if t.dtype != torch.int64 or t.dim() != 1 or t.numel() < rows or t.stride(0) != 1: + raise _fail(f"unexpected {what} {tuple(t.shape)} {t.dtype}") + + +@dataclass +class LayerArgs: + """Per-layer constants of the attention kernels: device addresses of tensors + owned by the ``DerivedWeights`` passed to ``AttentionGlue`` (kept alive there), + and the model-owned norm weights / RoPE caches (live as the model).""" + + w_uv: int + stage0: Slot + w_qb: int + w_uk: int + q_norm_w: torch.Tensor + kv_norm_w: torch.Tensor + q_eps: float + kv_eps: float + rope: torch.Tensor + # layers with their own indexer + w_qkva: int = 0 + w_iq: int = 0 + ik_w: Optional[torch.Tensor] = None + ik_b: Optional[torch.Tensor] = None + ik_eps: float = 0.0 + index_rope: Optional[torch.Tensor] = None + w_scale: float = 0.0 + stage1: Slot = NO_SLOT # the value up-projection's second staging slot + # layers without their own indexer: the packed q_a / kv_a weight of the small-M + # projection (0: not served) and the input RMSNorm epsilon (deferred input norm) + w_qkva_s: int = 0 + in_eps: float = 0.0 + + +class FusedAttention(AttentionDecode): + """Verify attention of one row count on the attention-family kernels.""" + + def __init__(self, glue: AttentionGlue, rows: int) -> None: + self.glue, fn = glue, glue.fn + self.max_rows = cap = kernel_rows(rows) + self.uv = fn[f"uv.{cap}"] + self.sparse = fn[f"sparse.{sparse_warps(rows)}"] + self.pro = fn[f"prologue.{cap}"] + # the target's layers that reuse the previous top-k: the end-trigger prologue + self.pro_end = fn.get(f"prologue_end.{cap}") + self.iq = fn[f"index_q.{cap}"] + # the small-M q_a / kv_a forms (8 rows only), if built: + self.qa = fn.get(f"qkv_a.{cap}") # indexer layers + self.qa_s = fn.get(f"qkv_a_shared.{cap}") # shared-top-k layers (opt-in) + self.qa_s_dn = fn.get(f"qkv_a_shared_dn.{cap}") # + deferred input norm + # (DNORM; the composition wires its producer when ``takes_deferred_norm``) + self.takes_deferred_norm = self.qa_s_dn is not None + + def runs_indexer(self, attn, step: StepContext) -> bool: + """Whether this layer's indexer runs in this forward: the layers with their + own top-k (the MTP draft's step group overrides it).""" + return not attn.skip_topk + + def qkv_a(self, attn, x, step: StepContext) -> torch.Tensor: + dn, step.dnorm = step.dnorm, None + if not self.runs_indexer(attn, step): + la = self.glue.layers.get(attn.layer_id) + if dn is not None: + # x is bf16(residual * gamma): only the epilogue-scaled projection + # of this very layer may consume it + if dn[0] != attn.layer_id or self.qa_s_dn is None or not la.w_qkva_s: + raise _fail( + f"layer {attn.layer_id} got the deferred input norm of " + f"layer {dn[0]}" + ) + inv_rms = (dn[1], la.in_eps) + return self._small_m(self.qa_s_dn, la.w_qkva_s, x, inv_rms)[0] + if self.qa_s is not None and la is not None and la.w_qkva_s: + return self._small_m(self.qa_s, la.w_qkva_s, x)[0] + # layers that reuse the previous indexer keep the stock fused-A GEMM + return super().qkv_a(attn, x, step) + if dn is not None: + raise _fail(f"indexer layer {attn.layer_id} got a deferred input norm") + if self.qa is None: + qkv = super().qkv_a(attn, x, step) + step.index_kw, _ = attn.indexer.wk_weights_proj(x) + return qkv + w = self.glue.layers[attn.layer_id].w_qkva + qkv, step.index_kw = self._small_m(self.qa, w, x, index_kw=True) + return qkv + + def _small_m(self, fn, w: int, x, dn=None, index_kw=False): + """``(qkv, kw)``: q_a / kv_a of ``x`` over the packed weight at ``w``, and + with ``index_kw`` the indexer key / head-gate projection (else None); with + ``dn`` (block sums, eps) ``x`` is ``bf16(residual * gamma)`` and the + epilogue applies the row's inverse RMS (deferred input norm).""" + from cutlass import Float32, Int32 + + rows = x.shape[0] + _check_rows(x, rows, HIDDEN, torch.bfloat16, "attention input") + qkv = torch.empty((rows, QKV_A_ROWS), dtype=torch.bfloat16, device=x.device) + kw = None + if index_kw: + kw = torch.empty((rows, INDEX_KW), dtype=torch.bfloat16, device=x.device) + tail = () + if dn is not None: + _check_rows(dn[0], rows, 8, torch.float32, "deferred-norm block sums") + tail = (*_i64(dn[0].data_ptr()), Float32(dn[1])) # bsum, eps + fn( + *_i64(w), # weights + *_i64(*_addr(x)), # x, x_stride + Int32(rows), + *_i64(*_addr(qkv)), # out_main, out_main_stride + *_i64(*_addr(kw)), # out_aux, out_aux_stride (0, 0: none) + *tail, + cu_stream(x.device), + ) + return qkv, kw + + def prologue( + self, attn, qkv_latent: torch.Tensor, step: StepContext + ) -> PrologueOut: + from cutlass import Float32, Int32 + + from sglang.srt.model_executor.forward_context import get_attn_backend + + rows = qkv_latent.shape[0] + _check_rows(qkv_latent, rows, QKV_A_ROWS, torch.bfloat16, "q_a / kv_a latent") + full = self.runs_indexer(attn, step) + dev = qkv_latent.device + la = self.glue.layers[attn.layer_id] + loc = step.forward_batch.out_cache_loc + pos = step.positions + _check_index(loc, rows, "cache slots") + _check_index(pos, rows, "positions") + pool = get_attn_backend().token_to_kv_pool + mla = pool.get_key_buffer(attn.layer_id) + if ( + mla.dim() != 3 + or tuple(mla.shape[1:]) != (1, MQA_DIM) + or mla.element_size() != 1 + ): + raise _fail(f"MLA cache {tuple(mla.shape)}") + mqa = torch.empty((rows, HEADS, MQA_DIM), dtype=torch.float8_e4m3fn, device=dev) + # the indexer operands, on the layers that run the indexer only + kw = qc = None + idx = idx_stride = ik_w = ik_b = irope = irope_stride = 0 + if full: + kw = step.index_kw + if kw is None: + raise _fail("no indexer key projection") + _check_rows(kw, rows, INDEX_KW, torch.bfloat16, "indexer key projection") + if hasattr(pool, "_is_layer_owned") and not pool._is_layer_owned( + attn.layer_id + ): + raise _fail("index-K rows of another owner") + # the stock index-K store's host-side bookkeeping, then its rows + if hasattr(pool, "invalidate_index_buffer_for_layer"): + pool.invalidate_index_buffer_for_layer(attn.layer_id) + buf = pool.get_index_k_with_scale_buffer(layer_id=attn.layer_id) + idx, idx_stride = buf.data_ptr(), buf.stride(0) * buf.element_size() + qc = torch.empty((rows, Q_LORA), dtype=torch.bfloat16, device=dev) + ik_w, ik_b = la.ik_w.data_ptr(), la.ik_b.data_ptr() + irope, irope_stride = _addr(la.index_rope) + base, stride = qkv_latent.data_ptr(), qkv_latent.stride(0) + ones = self.glue.ones.data_ptr() + pro = self.pro if full or self.pro_end is None else self.pro_end + pro( + *_i64(base, stride), # q_c, q_c_stride + *_i64(base + Q_LORA * 2, stride), # kv_c, kv_c_stride + *_i64(base + (Q_LORA + KV_LORA) * 2, stride), # k_pe, k_pe_stride + *_i64(*_addr(kw)), # idx_k, idx_k_stride (0, 0: no indexer) + *_i64(pos.data_ptr(), loc.data_ptr()), # positions, slot_mapping + *_i64(loc.data_ptr() if full else 0), # idx_slot_mapping + Int32(rows), + Int32(int(full)), # has_indexer + *_i64(la.w_qb, la.w_uk), + *_i64(la.q_norm_w.data_ptr(), la.kv_norm_w.data_ptr()), + Float32(la.q_eps), + Float32(la.kv_eps), + *_i64(*_addr(la.rope)), # rope_cache, rope_stride + *_i64(ones, ones), # q_scale, k_scale + *_i64(ik_w, ik_b), # index-K norm weight, bias + Float32(la.ik_eps), + *_i64(irope, irope_stride), # index_rope_cache, index_rope_stride + *_i64(*_addr(mqa)), # mqa_q, mqa_stride + *_i64(mla.data_ptr(), *mla.stride()[:2], 1), # mla_cache, strides, size + *_i64(idx, idx_stride, INDEX_PAGE), # idx_cache, block stride, size + *_i64(0, 0), # topk, topk_stride: unused + Int32(0), # topk_width: unused + *_i64(*_addr(qc)), # qc_out, qc_out_stride + cu_stream(dev), + ) + step.q_fp8 = mqa + return PrologueOut(qc, None, None, None, None) + + def _done(self, *args) -> None: + """A step the prologue did: the index-K rows, the W_UK absorption. + ``q_absorb`` returns None: the prologue left the absorbed FP8 query in + ``step.q_fp8``, which ``rope_cache`` returns.""" + + index_k = q_absorb = _done + + def index_q(self, attn, x, q_lora: torch.Tensor, step: StepContext) -> IndexQ: + from cutlass import Float32, Int32 + + rows = q_lora.shape[0] + _check_rows(q_lora, rows, Q_LORA, torch.bfloat16, "indexer query latent") + _, weights_raw = self._index_kw(attn.indexer, x, step) + la = self.glue.layers[attn.layer_id] + pos = step.positions + _check_index(pos, rows, "positions") + dev = q_lora.device + q = torch.empty( + (rows, INDEX_HEADS, INDEX_DIM), dtype=torch.float8_e4m3fn, device=dev + ) + w = torch.empty((rows, INDEX_HEADS), dtype=torch.float32, device=dev) + self.iq( + *_i64(la.w_iq), # weights + *_i64(*_addr(q_lora)), # x, x_stride + *_i64(pos.data_ptr()), # positions + *_i64(*_addr(la.index_rope)), # rope_cache, rope_stride + *_i64(*_addr(weights_raw)), # index_weights, index_weights_stride + Float32(la.w_scale), + Int32(rows), + *_i64(*_addr(q)), # q_fp8, q_stride + *_i64(*_addr(w)), # w_out, w_out_stride + cu_stream(dev), + ) + return IndexQ(q, w.unsqueeze(-1)) + + def rope_cache(self, attn, q_nope_out, q_pe, k_nope, k_pe, step: StepContext): + if step.q_fp8 is None: + raise _fail("the prologue left no FP8 query") + return step.q_fp8 + + def sparse_mla(self, attn, q_fp8: torch.Tensor, topk_indices, step: StepContext): + from cutlass import Float32, Int32 + + from sglang.srt.model_executor.forward_context import get_attn_backend + + backend = get_attn_backend() + layer = attn.attn_mqa + rows = q_fp8.shape[0] + if topk_indices is None: + raise _fail("no top-k cache rows") + # as the stock path: the fused top-k page table holds physical cache rows + ids = backend._get_fused_topk_page_table( + backend._pad_topk_indices(topk_indices, rows) + ) + if ( + ids.dtype != torch.int32 + or tuple(ids.shape) != (rows, TOPK) + or not ids.is_contiguous() + or q_fp8.dtype != torch.float8_e4m3fn + or tuple(q_fp8.shape) != (rows, HEADS, MQA_DIM) + or not q_fp8.is_contiguous() + ): + raise _fail( + "unexpected sparse MLA inputs " + f"{tuple(q_fp8.shape)} {q_fp8.dtype} / {tuple(ids.shape)} {ids.dtype}" + ) + kv = backend.token_to_kv_pool.get_key_buffer(layer.layer_id) + if kv.shape[-1] != MQA_DIM or not kv.is_contiguous() or kv.data_ptr() % 16: + raise _fail(f"MLA cache {tuple(kv.shape)}") + k_scale = getattr(layer, "k_scale_float", None) + bmm1_scale = 1.0 * (1.0 if k_scale is None else float(k_scale)) * layer.scaling + out = torch.empty( + (rows, HEADS, KV_LORA), dtype=torch.bfloat16, device=q_fp8.device + ) + self.sparse( + *_i64(q_fp8.data_ptr(), kv.data_ptr()), # q, kv + *_i64(ids.data_ptr(), out.data_ptr()), # ids, out + *_i64(kv.numel() // MQA_DIM), # kv_rows + Int32(rows), + Float32(_f32(bmm1_scale * LOG2E)), # scale_log2 + Float32(1.0), # bmm2_scale + # unused: the ids are physical cache rows (no block-table translation) + *_i64(0, 0, 0), # request_of_row, block_table, block_table_stride + Int32(0), # max_blocks + Int32(0), # block_shift + *_i64(0, 0), # block_rows, phys_out + cu_stream(q_fp8.device), + ) + return out + + def uv_proj(self, attn, attn_output: torch.Tensor) -> torch.Tensor: + from cutlass import Int32 + + rows = attn_output.shape[0] + if ( + attn_output.dtype != torch.bfloat16 + or attn_output.dim() != 3 + or tuple(attn_output.shape[1:]) != (HEADS, KV_LORA) + or attn_output.stride(2) != 1 + or not 1 <= rows <= self.max_rows + or attn_output.data_ptr() % 16 + or attn_output.stride(0) % 8 + or attn_output.stride(1) % 8 + ): + raise _fail( + "unexpected value up-projection input " + f"{tuple(attn_output.shape)} / {attn_output.stride()} / " + f"{attn_output.dtype}" + ) + out = torch.empty( + (rows, HEADS * VALUE_DIM), dtype=torch.bfloat16, device=attn_output.device + ) + la = self.glue.layers[attn.layer_id] + self.uv( + *_i64(la.w_uv), # weights + *_i64(attn_output.data_ptr(), *attn_output.stride()[:2]), # attn, strides + *_i64(*_addr(out)), # out, out_row_stride + Int32(rows), + *_slot_args(la.stage0), + *_slot_args(la.stage1), + cu_stream(attn_output.device), + ) + return out + + +def _rope_cache_fp32(cache: torch.Tensor, what: str) -> bool: + """Whether a [positions, cos 32 | sin 32] cache is FP32 (else BF16).""" + if ( + cache.dtype not in (torch.float32, torch.bfloat16) + or cache.dim() != 2 + or cache.shape[1] != ROPE + or cache.stride(1) != 1 + ): + raise _fail( + f"unsupported {what} {tuple(cache.shape)} {cache.dtype}", ValueError + ) + return cache.dtype == torch.float32 + + +class AttentionGlue: + """Derived attention weights, compiled kernels and per-row-count steps.""" + + def __init__(self, model, derived, layers=None) -> None: + from sgl_nvidia_mega.glm_dsa.kernels.cute.packing import ( + pack_indexer_q, + pack_q_b, + pack_qkv_a, + pack_uv_proj, + pack_w_uk_t, + ) + from sglang.srt.layers.layernorm import LayerNorm + + # the target model's glue (the MTP draft passes its own layers) + self.target = layers is None + layers = list(model.model.layers if layers is None else layers) + self.decoder_layers = layers + dev = layers[0].self_attn.kv_b_proj.weight.device + self.grid = torch.cuda.get_device_properties(dev).multi_processor_count + self.ones = torch.ones(1, dtype=torch.float32, device=dev) + self.layers: Dict[int, LayerArgs] = {} + # the compiled kernel forms by ``attn.api.compile_specs`` name: + # "uv|prologue|index_q.{8,16}" (row capacity), "prologue_end.{8,16}" (target + # only), "sparse.{16,8}" (warps per CTA: 16 up to 4 rows, 8 above) and + # "qkv_a[_shared[_dn]].8" + self.fn: Dict[str, object] = {} + rope_f32, index_rope_f32 = set(), set() + + def fused_a(attn) -> torch.Tensor: + qkv_a = attn.fused_qkv_a_proj_with_mqa + return _bf16_weight(qkv_a, (QKV_A_ROWS, HIDDEN), "fused q_a / kv_a") + + # the target's layers without their own indexer, on the small-M projection + # (opt-in): one packed copy of the fused q_a / kv_a weight per layer + shared_qa: Dict[int, torch.Tensor] = {} + if self.target and switches.shared_qkv_a_small_m(): + for i, layer in enumerate(layers): + attn = layer.self_attn + if getattr(attn, "indexer", None) is None and getattr( + attn, "skip_topk", False + ): + packed = pack_qkv_a(fused_a(attn), self.grid) + shared_qa[i] = derived.add("attention.qkv_a_shared", packed) + self.shared_qkv_a_layers = len(shared_qa) + for i, layer in enumerate(layers): + attn = layer.self_attn + kv_b = _bf16_weight(attn.kv_b_proj, (KV_B_ROWS, KV_LORA), "kv_b_proj") + q_b = _bf16_weight(attn.q_b_proj, (Q_B_ROWS, Q_LORA), "q_b_proj") + qkv_a = fused_a(attn) + nxt = None + if i + 1 < len(layers): + # the tensor the next layer's projection reads + nxt = shared_qa.get( + i + 1, layers[i + 1].self_attn.fused_qkv_a_proj_with_mqa.weight + ) + rope = attn.rotary_emb.cos_sin_cache + rope_f32.add(_rope_cache_fp32(rope, "RoPE cache")) + args = LayerArgs( + w_uv=derived.add("attention.uv_proj", pack_uv_proj(kv_b)).data_ptr(), + stage0=_prefetch_slot(nxt, NEXT_QKV_A_PREFETCH_BYTES), + w_qb=derived.add("attention.q_b", pack_q_b(q_b)).data_ptr(), + w_uk=derived.add("attention.w_uk_t", pack_w_uk_t(kv_b)).data_ptr(), + q_norm_w=attn.q_a_layernorm.weight, + kv_norm_w=attn.kv_a_layernorm.weight, + q_eps=float(attn.q_a_layernorm.variance_epsilon), + kv_eps=float(attn.kv_a_layernorm.variance_epsilon), + rope=rope, + ) + self.layers[attn.layer_id] = args + if i in shared_qa: + args.w_qkva_s = shared_qa[i].data_ptr() + args.in_eps = _f32(layer.input_layernorm.variance_epsilon) + if self.target: + # slot 0 stages this layer's o_proj weight, slot 1 the next q_a / kv_a + wo = _o_proj_stage_slot(attn.o_proj.weight) + if wo != NO_SLOT: + args.stage0, args.stage1 = wo, args.stage0 + # every layer with an indexer (the draft's may run it); the target's + # layers that reuse the previous top-k have none + ix = getattr(attn, "indexer", None) + if ix is None: + continue + if ( + not isinstance(ix.k_norm, LayerNorm) + or ix.k_norm.weight.dtype != torch.float32 + or ix.k_norm.bias is None + ): + raise ValueError( + "fused verify attention needs the fp32 LayerNorm index-K norm" + ) + if ix.n_heads != INDEX_HEADS or ix.head_dim != INDEX_DIM: + raise _fail("unexpected indexer shape", ValueError) + wq_b = _bf16_weight( + ix.wq_b, (INDEX_HEADS * INDEX_DIM, Q_LORA), "indexer wq_b" + ) + wkw = _bf16_weight( + ix.wk_weights_proj, (INDEX_KW, HIDDEN), "indexer key / head gate" + ) + args.w_iq = derived.add( + "attention.index_q", pack_indexer_q(wq_b) + ).data_ptr() + args.w_qkva = derived.add( + "attention.qkv_a_index", + pack_qkv_a(torch.cat([qkv_a, wkw], dim=0), self.grid), + ).data_ptr() + args.ik_w, args.ik_b = ix.k_norm.weight, ix.k_norm.bias + args.ik_eps = float(ix.k_norm.variance_epsilon) + args.index_rope = ix.rotary_emb.cos_sin_cache + index_rope_f32.add(_rope_cache_fp32(args.index_rope, "index RoPE cache")) + args.w_scale = _f32(ix.softmax_scale * ix.n_heads**-0.5) + if len(rope_f32) != 1 or len(index_rope_f32) > 1: + raise _fail("RoPE cache dtypes differ by layer", ValueError) + self.rope_f32 = rope_f32.pop() + self.index_rope_f32 = index_rope_f32.pop() if index_rope_f32 else True + + def compile(self, rows: Tuple[int, ...]) -> None: + import cutlass.cute as cute + + from sgl_nvidia_mega.glm_dsa.kernels.cute.attn import api + + shared = QKV_A_ROWS if self.shared_qkv_a_layers else None + specs = api.compile_specs( + sorted({kernel_rows(m) for m in rows} - {None}), + numerics=_prologue_numerics(), + index_q_rope_form=INDEX_Q_ROPE_FORM, + rope_cache_fp32=self.rope_f32, + index_rope_cache_fp32=self.index_rope_f32, + warps=sorted({sparse_warps(m) for m in rows if kernel_rows(m)}), + qkv_a=(QKV_A_ROWS + INDEX_KW, QKV_A_ROWS), + grid=self.grid, + prologue_end=self.target, + qkv_a_shared=shared, + qkv_a_shared_dn=bool(shared) and switches.dnorm(), + ) + for name, (kernel, key, args) in specs.items(): + fn = compile_cached( + kernel, key, lambda k=kernel, a=args: cute.compile(k, *a) + ) + self.fn[name] = _allow_cluster16(fn) if name.startswith("sparse.") else fn + if name == "qkv_a_shared.8": + logger.info( + "GLM DSA fused decode: q_a / kv_a of %d layers without their own " + "indexer on the small-M projection (packed copies %.1f MiB)", + self.shared_qkv_a_layers, + self.shared_qkv_a_layers * QKV_A_ROWS * HIDDEN * 2 / 2**20, + ) + elif name == "qkv_a_shared_dn.8": + logger.info( + "GLM DSA fused decode: deferred input norm: epilogue-scaled q_a / " + "kv_a projection built" + ) + + def step_reasons(self, model, forward_batch) -> List[str]: + """Host-only facts the attention kernels need beyond the stock copies + (read at the first capture of each row count; any reason on any rank + keeps that row count on the stock forward).""" + from sglang.kernels.ops.kvcache.dsa_fused_store_index_cache import ( + can_use_dsa_fused_store, + ) + from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz + from sglang.srt.model_executor.forward_context import get_attn_backend + + pool = get_attn_backend().token_to_kv_pool + reasons: List[str] = [] + page = getattr(pool, "index_page_size", None) + if page != INDEX_PAGE: + reasons.append(f"index-K page size {page} is not {INDEX_PAGE}") + elif is_fp8_fnuz() or not can_use_dsa_fused_store( + torch.bfloat16, torch.int64, page + ): + reasons.append("the fused index-K store is not available") + for layer in self.decoder_layers: + attn = layer.self_attn + lid = attn.layer_id + mla = pool.get_key_buffer(lid) + if ( + mla.dim() != 3 + or tuple(mla.shape[1:]) != (1, MQA_DIM) + or mla.element_size() != 1 + or not mla.is_contiguous() + ): + reasons.append(f"layer {lid}: MLA cache {tuple(mla.shape)} {mla.dtype}") + if getattr(attn, "indexer", None) is None: + continue + if hasattr(pool, "_is_layer_owned") and not pool._is_layer_owned(lid): + reasons.append(f"layer {lid}: index-K rows owned elsewhere") + idx = pool.get_index_k_with_scale_buffer(layer_id=lid) + if ( + idx.dtype != torch.uint8 + or idx.dim() != 2 + or idx.shape[1] != INDEX_PAGE * (INDEX_DIM + 4) + or idx.stride(1) != 1 + ): + reasons.append(f"layer {lid}: index-K cache {tuple(idx.shape)}") + if len(reasons) > 8: + break + return reasons + + def for_rows(self, rows: int) -> Optional[AttentionDecode]: + if f"uv.{kernel_rows(rows)}" not in self.fn: + return None + return FusedAttention(self, rows) + + +def _allow_cluster16(compiled): + """The sparse MLA kernel's 16-CTA cluster is a non-portable cluster size: + allow it on every kernel of the compiled module before its first launch.""" + import cuda.bindings.driver as cuda + + lib = getattr(compiled, "library", None) + if lib is None and hasattr(compiled, "to"): + compiled.to(torch.cuda.current_device()) + lib = getattr(compiled, "library", None) + if lib is None: + raise _fail("no module handle for the cluster size") + _, n = cuda.cuLibraryGetKernelCount(lib) + _, kernels = cuda.cuLibraryEnumerateKernels(n, lib) + dev = torch.cuda.current_device() + attr = cuda.CUfunction_attribute.CU_FUNC_ATTRIBUTE_NON_PORTABLE_CLUSTER_SIZE_ALLOWED + for k in kernels: + res = cuda.cuKernelSetAttribute(attr, 1, k, dev) + err = res[0] if isinstance(res, tuple) else res + if err != cuda.CUresult.CUDA_SUCCESS: + raise _fail(f"cluster size attribute: {err}") + return compiled + + +# the family protocol (providers.build_providers); the MTP draft passes ``layers`` +build = AttentionGlue diff --git a/sgl_nvidia_mega/glm_dsa/attention/step.py b/sgl_nvidia_mega/glm_dsa/attention/step.py new file mode 100644 index 00000000..7b6a8f66 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/attention/step.py @@ -0,0 +1,385 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""Stock verify attention steps: StepContext, PrologueOut, IndexQ, AttentionDecode.""" + +from __future__ import annotations + +import logging +from contextlib import nullcontext +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, Hashable, Optional, Tuple + +import torch + +logger = logging.getLogger(__name__) + +class StepContext: + """Per-forward state shared by the step groups of one verify forward.""" + + forward_batch: Any + positions: torch.Tensor + zero_allocator: Any + # Set by a routing hop: (expert ids [rows, 8] int32, weights [rows, 8] fp32); + # read and cleared by the same layer's MoE step. + routing: Any = None + # Per layer, cleared by AttentionDecode.forward: the indexer key / head-gate + # projections and the FP8 MLA query when a fused step produced them early. + index_kw: Any = None + q_fp8: Any = None + # A deferred input norm (DNORM): (layer id, block sums [rows, 8] fp32); read + # and cleared by that layer's attention. + dnorm: Any = None + + +@dataclass +class PrologueOut: + """What ``prologue`` hands on; None where a fused override consumed it. + ``q_lora`` [rows, q_lora_rank] is the indexer input; ``q_nope`` / ``q_pe`` + are per head and ``k_nope`` [rows, 1, kv_lora_rank] / ``k_pe`` before RoPE.""" + + q_lora: Optional[torch.Tensor] + q_nope: Optional[torch.Tensor] + q_pe: Optional[torch.Tensor] + k_nope: Optional[torch.Tensor] + k_pe: Optional[torch.Tensor] + + +@dataclass +class IndexQ: + """The indexer query of a layer that runs its own indexer.""" + + q_fp8: torch.Tensor # [rows, index_heads, index_head_dim], FP8 + weights: torch.Tensor # [rows, index_heads, 1], fp32 head gates + + +class AttentionDecode: + """Stock verify attention of one ``DeepseekV2AttentionMLA`` layer, in steps: + ``qkv_a`` -> ``prologue`` -> (own indexer) ``index_k`` -> ``index_q`` -> + ``indexer`` -> ``q_absorb`` -> ``rope_cache`` -> ``sparse_mla`` -> ``uv_proj``. + + The bodies copy the verify slices of ``forward_absorb_prepare`` / + ``forward_absorb_core`` (forward_mla.py), the DSA indexer's ``forward_cuda`` + and the DSA backend's ``_forward_trtllm`` at SGLang main 44b44774 for the one + configuration served (bf16 W_UK / W_UV, fused indexer projections, FP8 KV + cache on the TRT-LLM sparse kernel, fused top-k page table, full-graph + verify); eligibility refuses every other one, and the replica test checks the + copies bitwise against stock. A kernel covering several steps overrides the + first and passes the others through, handing state on ``StepContext``. + """ + + def forward(self, layer, x, prev_topk_indices, step: StepContext): + """The ``o_proj`` input ``[rows, heads * v_head_dim]`` and the top-k + indices handed to the next layer (None unless it reuses them).""" + from sglang.srt.layers.layer_boundary import get_attn_tp_context + from sglang.srt.layers.layer_boundary.adapters.attention import ( + AttentionInputs, + ) + + attn = layer.self_attn + fb = step.forward_batch + self.check_mode(attn, fb.forward_mode) + ctx = get_attn_tp_context() + # The attention reads its input through the attention-TP context. + ctx.set_attn_inputs(AttentionInputs(x, fb, attn.prepare_qkv_latent)) + step.index_kw = step.q_fp8 = None + try: + with attn.maybe_use_decode_attn_tp(fb): + qkv_latent = self.qkv_a(attn, x, step) + pro = self.prologue(attn, qkv_latent, step) + index_q = None + if attn.should_run_indexer(prev_topk_indices): + if self._stock_index_steps(): + index_q = self._index_k_and_q(attn, x, pro.q_lora, step) + else: + self.index_k(attn, x, step.positions, step) + index_q = self.index_q(attn, x, pro.q_lora, step) + topk_indices = self.indexer(attn, index_q, prev_topk_indices, step) + q_nope_out = self.q_absorb(attn, pro.q_nope, step) + q_fp8 = self.rope_cache( + attn, q_nope_out, pro.q_pe, pro.k_nope, pro.k_pe, step + ) + attn_output = self.sparse_mla(attn, q_fp8, topk_indices, step) + out = self.uv_proj(attn, attn_output) + finally: + ctx.clear_attn_inputs() + step.index_kw = step.q_fp8 = None + return out, (topk_indices if attn.next_skip_topk else None) + + def check_mode(self, attn, mode) -> None: + # The stock bodies copy the branches of these modes only (the draft's + # decode passes: draft_attention.DraftAttention). + if not (mode.is_target_verify() or mode.is_draft_extend_v2()): + raise RuntimeError( + f"AttentionDecode serves target-verify and draft-extend-v2 forwards only (got {mode})" + ) + + def qkv_a(self, attn, x, step: StepContext) -> torch.Tensor: + """``[rows, q_lora_rank + kv_lora_rank + qk_rope_head_dim]``; an override + may also leave the indexer key / head-gate projections in + ``step.index_kw``.""" + from sglang.srt.layers.layer_boundary import get_attn_tp_context + + # forward_absorb_prepare (q_lora_rank set). + return get_attn_tp_context().fetch_qkv_latent() + + def prologue( + self, attn, qkv_latent: torch.Tensor, step: StepContext + ) -> PrologueOut: + """The q_a / kv_a norms, ``q_b_proj`` and the q / key-RoPE split.""" + from sglang.srt.model_executor.runner import get_is_capture_mode + + # forward_absorb_prepare (main 44b44774), verify: the kv_a norm runs on + # the side stream during capture. + q, latent_cache = qkv_latent.split( + [attn.q_lora_rank, attn.kv_lora_rank + attn.qk_rope_head_dim], dim=-1 + ) + k_nope = latent_cache[..., : attn.kv_lora_rank] + if attn.alt_stream is not None and get_is_capture_mode(): + current_stream = torch.cuda.current_stream() + attn.alt_stream.wait_stream(current_stream) + q = attn.q_a_layernorm(q) + with torch.cuda.stream(attn.alt_stream): + k_nope = attn.kv_a_layernorm(k_nope) + current_stream.wait_stream(attn.alt_stream) + else: + q = attn.q_a_layernorm(q) + k_nope = attn.kv_a_layernorm(k_nope) + q_lora = q + k_nope = k_nope.unsqueeze(1) + q = attn.q_b_proj_forward(q) + q_nope, q_pe, k_pe = attn._split_q_nope_pe(q, latent_cache) + return PrologueOut(q_lora, q_nope, q_pe, k_nope, k_pe) + + def index_k(self, attn, x, positions: torch.Tensor, step: StepContext) -> None: + """The indexer key's norm, RoPE, FP8 quantization and index-K cache rows.""" + # _fused_q_prepare_and_store (main 44b44774), single-stream key half. + key, _ = self._index_kw(attn.indexer, x, step) + _store_index_k(attn, key, positions, step) + + def index_q(self, attn, x, q_lora: torch.Tensor, step: StepContext) -> IndexQ: + """The indexer query (``wq_b``, RoPE, FP8) and its head gates.""" + # _fused_q_prepare_and_store (main 44b44774), single-stream query half. + indexer = attn.indexer + _, weights_raw = self._index_kw(indexer, x, step) + q = indexer.wq_b(q_lora)[0].view(-1, indexer.n_heads, indexer.head_dim) + return IndexQ(*_index_q_quant(indexer, q, weights_raw, step.positions)) + + def _stock_index_steps(self) -> bool: + cls = type(self) + return ( + cls.index_k is AttentionDecode.index_k + and cls.index_q is AttentionDecode.index_q + ) + + def _index_k_and_q( + self, attn, x, q_lora: torch.Tensor, step: StepContext + ) -> IndexQ: + """``index_k`` + ``index_q`` when neither is replaced, as stock runs them.""" + indexer = attn.indexer + if indexer.alt_stream is None: + AttentionDecode.index_k(self, attn, x, step.positions, step) + return AttentionDecode.index_q(self, attn, x, q_lora, step) + # _fused_q_prepare_and_store (main 44b44774), two-stream branch: wq_b + # (side) || key / head-gate projections (current), then the query kernel + # (current) || the key kernel + index-K store (side). + positions = step.positions + current_stream = torch.cuda.current_stream() + indexer.alt_stream.wait_stream(current_stream) + with torch.cuda.stream(indexer.alt_stream): + q = indexer.wq_b(q_lora)[0].view(-1, indexer.n_heads, indexer.head_dim) + key, weights_raw = self._index_kw(indexer, x, step) + current_stream.wait_stream(indexer.alt_stream) + indexer.alt_stream.wait_stream(current_stream) + q_fp8, weights = _index_q_quant(indexer, q, weights_raw, positions) + with torch.cuda.stream(indexer.alt_stream): + _store_index_k(attn, key, positions, step) + current_stream.wait_stream(indexer.alt_stream) + return IndexQ(q_fp8, weights) + + @staticmethod + def _index_kw(indexer, x, step: StepContext): + """(key, raw head gates), projecting ``x`` unless a step already did.""" + if step.index_kw is None: + step.index_kw, _ = indexer.wk_weights_proj(x) + return step.index_kw.split([indexer.head_dim, indexer.n_heads], dim=-1) + + def indexer( + self, attn, index_q: Optional[IndexQ], prev_topk_indices, step: StepContext + ): + """Top-k cache rows per query row (page-table rows); a layer without its + own indexer (``index_q`` None) reuses the previous layer's.""" + from sglang.srt.layers.attention.dsa.dsa_indexer import ( + _broadcast_indexer_topk_from_rank0, + ) + from sglang.srt.model_executor.forward_context import get_attn_backend + from sglang.srt.state_capturer.indexer_topk import maybe_capture_indexer_topk + + if index_q is None: + # forward_absorb_prepare: a skip-top-k layer mirrors the reused + # indices into its own capture slot. + return maybe_capture_indexer_topk(attn.layer_id, prev_topk_indices) + # The DSA indexer's forward_cuda (main 44b44774), target verify inside a + # full graph (non-empty batch, logits not skipped). + fb = step.forward_batch + metadata = get_attn_backend().get_indexer_metadata(attn.layer_id, fb) + if metadata is None: + raise RuntimeError( + "fused verify step: the attention backend has no indexer metadata" + ) + topk_result = attn.indexer._get_topk_paged( + fb, attn.layer_id, index_q.q_fp8, index_q.weights, metadata + ) + topk_result = _broadcast_indexer_topk_from_rank0(topk_result) + return maybe_capture_indexer_topk(attn.layer_id, topk_result) + + def q_absorb(self, attn, q_nope: torch.Tensor, step: StepContext) -> torch.Tensor: + """``q_nope`` through W_UK: ``[rows, heads, kv_lora_rank]``.""" + # forward_absorb_prepare (main 44b44774), bf16 W_UK. + return torch.bmm(q_nope.transpose(0, 1), attn.w_kc).transpose(0, 1) + + def rope_cache( + self, attn, q_nope_out, q_pe, k_nope, k_pe, step: StepContext + ) -> torch.Tensor: + """RoPE, the FP8 query ``[rows, heads, kv_lora_rank + rope]`` (returned) + and the key's MLA cache rows.""" + from sglang.kernels.ops.attention.utils import mla_quantize_and_rope_for_fp8 + from sglang.srt.model_executor.forward_context import get_attn_backend + + backend = get_attn_backend() + layer = attn.attn_mqa + fb = step.forward_batch + # The key as RadixAttention.forward hands it to the backend. + k = k_nope.view(-1, layer.tp_k_head_num, layer.v_head_dim) + # The DSA backend's _forward_trtllm (main 44b44774), FP8 KV cache with + # RoPE fused into the quantization (no prefill CP). + q, k, k_rope = mla_quantize_and_rope_for_fp8( + q_nope_out, + q_pe, + k.squeeze(1), + k_pe.squeeze(1), + fb.positions, + attn.rotary_emb.cos_sin_cache, + attn.rotary_emb.is_neox_style, + backend.kv_lora_rank, + backend.qk_rope_head_dim, + ) + backend.token_to_kv_pool.set_mla_kv_buffer(layer, fb.out_cache_loc, k, k_rope) + return q + + def sparse_mla( + self, attn, q_fp8: torch.Tensor, topk_indices, step: StepContext, decode=False + ) -> torch.Tensor: + """Sparse MLA over the selected cache rows: ``[rows, heads, kv_lora_rank]``. + ``decode``: a draft DECODE pass.""" + import flashinfer.decode + + from sglang.srt.environ import envs + from sglang.srt.model_executor.forward_context import get_attn_backend + + backend = get_attn_backend() + layer = attn.attn_mqa + metadata = backend.forward_metadata + # The DSA backend's _forward_trtllm (main 44b44774), FP8 query, fused + # top-k page table, no DCP; seq_lens as forward_extend passes them for + # target verify. Decode (forward_decode, is_prefill False): the full cache + # lengths, and eager DP padding rows beyond the metadata's batch trimmed + # before the kernel and restored as zeros. + k_cache = backend.token_to_kv_pool.get_key_buffer(layer.layer_id) + kv_cache = k_cache.view( + -1, backend.physical_page_size, backend.kv_cache_dim + ).unsqueeze(1) + q_all = q_fp8.view(-1, layer.tp_q_head_num, layer.head_dim) + if topk_indices is not None: + topk_indices = backend._pad_topk_indices(topk_indices, q_fp8.shape[0]) + if decode: + from sglang.srt.layers.attention.dsa_backend import ( + _restore_trtllm_decode_dp_padding, + _trim_trtllm_decode_dp_padding, + ) + + q_all, topk_indices, n_pad = _trim_trtllm_decode_dp_padding( + q_all, topk_indices, metadata.cache_seqlens_int32.shape[0] + ) + page_table_1 = backend._get_fused_topk_page_table(topk_indices) + page_table_1, sparse_mla_top_k = backend._pad_trtllm_sparse_page_table( + page_table_1 + ) + k_scale = ( + layer.k_scale_float + if getattr(layer, "k_scale_float", None) is not None + else 1.0 + ) + bmm1_scale = 1.0 * k_scale * layer.scaling + batch_size, num_heads, head_dim = q_all.shape + multi_ctas_kv_counter_buffer = backend._multi_ctas_kv_counter_for(batch_size) + seq_lens = None if decode else metadata.dsa_cache_seqlens_int32 + if seq_lens is None: + seq_lens = metadata.cache_seqlens_int32 + out = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla( + query=q_all.view(batch_size, 1, num_heads, head_dim), + block_tables=page_table_1.unsqueeze(1), + seq_lens=seq_lens, + kv_cache=kv_cache.view( + -1, 1, backend.physical_page_size, backend.kv_cache_dim + ), + workspace_buffer=backend.workspace_buffer, + qk_nope_head_dim=backend.qk_nope_head_dim, + kv_lora_rank=backend.kv_lora_rank, + qk_rope_head_dim=backend.qk_rope_head_dim, + max_seq_len=metadata.max_seq_len_k, + sparse_mla_top_k=sparse_mla_top_k, + bmm1_scale=bmm1_scale, + backend="trtllm-gen", + skip_softmax_threshold_scale_factor=envs.SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR.get(), + sparse_mla_top_k_lens=None, + multi_ctas_kv_counter_buffer=multi_ctas_kv_counter_buffer, + ) + if decode: + out = _restore_trtllm_decode_dp_padding(out, n_pad) + return out.view(-1, attn.num_local_heads, attn.kv_lora_rank) + + def uv_proj(self, attn, attn_output: torch.Tensor) -> torch.Tensor: + """``[rows, heads, kv_lora]`` -> ``[rows, heads * v]``: one batched GEMM + over the heads into the ``o_proj`` input buffer.""" + out = torch.empty( + (attn_output.shape[0], attn.num_local_heads * attn.v_head_dim), + dtype=attn_output.dtype, + device=attn_output.device, + ) + torch.bmm( + attn_output.transpose(0, 1), + attn.w_vc, + out=out.view(-1, attn.num_local_heads, attn.v_head_dim).transpose(0, 1), + ) + return out + + +def _store_index_k(attn, key, positions, step: StepContext) -> None: + """The stock index-K store (``_fused_k_prepare_and_store``) of ``key``.""" + from sglang.kernels.ops.quantization.act_quant_triton import act_quant + + fb = step.forward_batch + attn.indexer._fused_k_prepare_and_store( + key, positions, fb, attn.layer_id, act_quant, out_cache_loc=fb.out_cache_loc + ) + + +def _index_q_quant( + indexer, q: torch.Tensor, weights_raw: torch.Tensor, positions: torch.Tensor +): + """The indexer query's RoPE + FP8 quantization and the scaled head gates + (_fused_q_prepare_and_store, main 44b44774).""" + from sglang.kernels.ops.attention.dsa.indexer_q import ( + fused_q_indexer_rope_first_quant, + ) + + q_scale_gate = indexer.softmax_scale * indexer.n_heads**-0.5 + return fused_q_indexer_rope_first_quant( + q.contiguous(), + weights_raw, + q_scale_gate, + indexer._indexer_cos_sin_cache, + positions, + ) + + diff --git a/sgl_nvidia_mega/glm_dsa/boundary/__init__.py b/sgl_nvidia_mega/glm_dsa/boundary/__init__.py new file mode 100644 index 00000000..6d229259 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/boundary/__init__.py @@ -0,0 +1,386 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""Hop step boundary: the stock ``HopSteps`` body and the CuTe DSL all-reduce hops +(``HopFamily``) of the GLM-5 DSA fused verify forward. + +Every boundary at which the stock forward sums a tensor-parallel partial (the +``o_proj`` output, the MLP / MoE output) is served by one hop: a kernel +publishes this rank's partial to every rank's symmetric-memory mailbox and +reduces the eight partials in rank order, adds the residual and applies the +RMSNorm. The ``o_proj`` hop also computes the ``o_proj`` GEMV; on MoE layers it +computes the router logits during the all-reduce and routes the rows (top-8), +and the MLP-output hop of a MoE layer folds the MoE engine's per-expert outputs +(finalize) into the all-reduce. + +Forms per verify row count (a function of the row count and the layer index +only, so every rank runs the same hop sequence on its mailboxes): + +* MoE family served, rows <= 8: the routing ``o_proj`` hop and the fold hop on + MoE layers; dense layers use the plain hops at rows <= 4, stock above. +* MoE family served, rows 9..16: the 16-row routing hop and fold on their own + mailbox; dense layers stock. +* MoE family stock at this row count (declined at startup), rows <= 8 ("far + form"): the plain hops on every boundary. Otherwise stock. + +The kernels run only inside captured verify graphs; the mailboxes are created +and initialized before the first capture. +""" + +from __future__ import annotations + +import logging +from typing import Any, Optional + +import torch +import torch.distributed as dist + +from sgl_nvidia_mega.glm_dsa import switches +from sgl_nvidia_mega.glm_dsa.eligibility import HIDDEN, TP_SIZE +from sgl_nvidia_mega.glm_dsa.attention.step import StepContext +from sgl_nvidia_mega.glm_dsa.runtime.compile import compile_cached, cu_stream + +logger = logging.getLogger(__name__) + + +class HopSteps: + """Stock all-reduce + residual + norm steps between the stages.""" + + def pre_attention(self, layer, mlp_out, residual, step: StepContext): + """The attention input and the updated residual (first layer: ``mlp_out`` + is the embedding and ``residual`` None).""" + norm = layer.input_layernorm + if residual is None: + return norm(mlp_out), mlp_out + # The MLP output owes its sum over the MoE output group (the TP group). + return norm.forward_with_allreduce_fusion( + mlp_out, residual, use_attn_tp_group=False + ) + + def post_attention(self, layer, o_input, residual, step: StepContext): + """``o_proj`` + attention-TP all-reduce + residual add + norm.""" + partial, _ = layer.self_attn.o_proj(o_input) + return layer.post_attention_layernorm.forward_with_allreduce_fusion( + partial, residual, use_attn_tp_group=True + ) + + def final(self, model, mlp_out, residual, step: StepContext) -> torch.Tensor: + """The last layer's MLP output completed, added and normed.""" + from sglang.srt.layers.moe.utils import sum_post_experts_output + + hidden = sum_post_experts_output(mlp_out) + return model.model.norm(hidden, residual)[0] + +O_PROJ_IN = 2048 +TOP_K = 8 # routed experts per row +SMALL_ROWS = 8 # mailbox instance A (instance B: 9..16 rows) +DENSE_MAX_ROWS = 4 # plain hops on the dense layers beside the fold; stock above +# Error sites: layer * 4 + {0: o_proj, 1: MLP output, 2: final}; sites from +# EXTERNAL_SITES on are other kernels' (named with ``HopFamily.register_sites``). +HOPS_PER_LAYER = 4 +EXTERNAL_SITES = 1000 + + +class HopFamily: + """Derived weights, compiled units and mailboxes of the hop kernels.""" + + def __init__(self, model: Any, derived: Any, api: Any) -> None: + from sglang.srt.distributed.parallel_state import get_tp_group + + inner, config = model.model, model.config + span = range(inner.start_layer, inner.end_layer) + self.api, self.derived = api, derived + self.layers = [inner.layers[i] for i in span] + self.first, self.last = inner.start_layer, inner.end_layer - 1 + self.eps = float(config.rms_norm_eps) + self.is_moe = [bool(layer.is_layer_sparse) for layer in self.layers] + self.rscale = float(getattr(config, "routed_scaling_factor", 1.0)) + self.device = torch.device("cuda", torch.cuda.current_device()) + self.group = get_tp_group() + self.rank = self.group.rank_in_group + self.w_o = [layer.self_attn.o_proj.weight for layer in self.layers] + self.router: list = [] + self.kernels: Any = None + self.site_names: dict = {} + # DNORM (both switches): the layers without their own top-k whose input + # hop is the MoE fold may defer their attention input norm. + dnorm = switches.dnorm() + self.dnorm_layers = frozenset( + layer.layer_id + for i, layer in enumerate(self.layers) + if dnorm + and i > 0 + and self.is_moe[i - 1] + and getattr(layer.self_attn, "skip_topk", False) + ) + + def check(self) -> str: + """Empty if this model and environment fit the kernels, else the reason.""" + if self.group.world_size != TP_SIZE: + size = self.group.world_size + return f"tensor-parallel size {size} (the hops need {TP_SIZE})" + for w in self.w_o: + if w.dtype != torch.bfloat16 or tuple(w.shape) != (HIDDEN, O_PROJ_IN): + return "o_proj weight is not a bf16 [6144, 2048] shard" + if not w.is_contiguous(): + return "o_proj weight is not contiguous" + for layer in self.layers: + for norm in (layer.input_layernorm, layer.post_attention_layernorm): + if float(norm.variance_epsilon) != self.eps: + return "the RMSNorm epsilons differ" + for layer, moe in zip(self.layers, self.is_moe): + bias = layer.mlp.gate.e_score_correction_bias if moe else None + if moe and (bias is None or bias.dtype != torch.float32): + return "the router has no fp32 score-correction bias" + import torch.distributed._symmetric_memory as symm_mem + + get_backend = getattr(symm_mem, "get_backend", None) + backend = get_backend(self.device) if get_backend is not None else None + if backend not in (None, "CUDA"): + return f"symmetric-memory backend {backend!r} (the hops need 'CUDA')" + if not self.api.fenced_trigger_supported(): + return "the CuTe DSL toolchain does not provide the o_proj hop's start edge" + return "" + + def compile(self, rows) -> None: + """Compile the units served at ``rows`` and create the mailboxes. + + Collective over the tensor-parallel group: every rank agrees (CPU + all-reduce) on whether the hops are served at all and on the split + fold's row limit, so the hop sequence is the same on every rank. + """ + reason = self.check() + small = any(m <= SMALL_ROWS for m in rows) + big = any(m > SMALL_ROWS for m in rows) + names = set() + if small: + names |= {"allreduce_norm", "o_proj_produce", "o_proj_consume"} + names |= {"o_proj_produce_router", "o_proj_consume_router"} + names |= {"moe_fold_split", "moe_fold_plain"} + if self.dnorm_layers: + names |= {"moe_fold_split_dn", "moe_fold_plain_dn"} + if big: + names |= {"o_proj_produce_router16", "o_proj_consume_router16"} + names |= {"moe_fold_block_sums16", "rmsnorm_from_block_sums16"} + compiled, probe = {}, None + if not reason: + import cutlass.cute as cute + + for spec in (s for s in self.api.unit_specs() if s.name in names): + compiled[spec.name] = compile_cached( + spec.obj, + spec.constexprs, + lambda spec=spec: cute.compile(spec.obj, *spec.args), + ) + if spec.name == "moe_fold_split" and not hasattr( + compiled[spec.name], "library" + ): + # The occupancy query needs the driver library, which a + # cache-loaded unit may not keep: query a fresh compile. + probe = cute.compile(spec.obj, *spec.args) + ok = torch.tensor([0 if reason else 1], dtype=torch.int32) + dist.all_reduce(ok, op=dist.ReduceOp.MIN, group=self.group.cpu_group) + if int(ok.item()) == 0: + logger.warning( + "GLM decode hops fall back to the stock all-reduce: %s", + reason or "another rank declined", + ) + return + # Per MoE layer, the router folded into the o_proj hop: W_router scaled by + # the post-attention norm weight, times this rank's o_proj shard, plus this + # rank's slice of the scaled router, as bf16 hi / lo (derived copies). + for layer, moe in zip(self.layers, self.is_moe): + hoist = None + if moe: + hoist = self.api.router_hoist_weights( + layer.mlp.gate.weight, + layer.post_attention_layernorm.weight, + layer.self_attn.o_proj.weight, + self.rank, + ) + for t in hoist: + self.derived.add("hops.router_hoist", t) + self.router.append(hoist) + kernels = self.api.HopKernels.build( + compiled, + self.group.device_group.group_name, + self.device, + self.eps, + self.rscale, + self.device.index, + instance_b=big, + router=True, + probe=probe, + ) + split = torch.tensor([kernels.split_max_rows], dtype=torch.int32) + dist.all_reduce(split, op=dist.ReduceOp.MIN, group=self.group.cpu_group) + kernels.split_max_rows = int(split.item()) + self.kernels = kernels + + def register_sites(self, names: dict) -> None: + """Name other kernels' sites (>= ``EXTERNAL_SITES``) in this error record.""" + for site in names: + if int(site) < EXTERNAL_SITES: + raise ValueError(f"error site {site} is in the hop family's range") + self.site_names.update({int(s): str(n) for s, n in names.items()}) + + def error_report(self) -> Optional[dict]: + """The decoded host-mapped error record of a timed-out hop wait, or None + (host memory only: readable after the CUDA context is lost).""" + rec = self.kernels.err.read() if self.kernels is not None else None + if rec is None: + return None + site = int(rec["site"]) + if site >= EXTERNAL_SITES: + return {**rec, "layer": None, "hop": self.site_names.get(site, "external")} + layer, kind = divmod(site, HOPS_PER_LAYER) + hop = ("o_proj", "mlp_output", "final")[kind] if kind < 3 else str(kind) + return {**rec, "layer": layer, "hop": hop} + + def for_rows(self, rows: int) -> Optional[HopSteps]: + """The hop step group at ``rows`` (None: stock): the fold form, whose + ``fallback`` (the far form up to ``SMALL_ROWS`` rows, else stock) the + composition serves where the MoE family is stock at ``rows``.""" + if self.kernels is None: + return None + far = _FarHops(self) if rows <= SMALL_ROWS else None + return _FoldHops(self, rows, fallback=far) + + def site(self, layer_id: int, kind: int) -> int: + return layer_id * HOPS_PER_LAYER + kind + + def o_proj(self, layer, o_input, residual, step: Optional[StepContext] = None): + """The ``o_proj`` hop; with ``step`` (MoE layers) the routing variant, whose + expert ids / weights ([rows, 8] int32 / fp32) go to ``step.routing``.""" + i, site = layer.layer_id - self.first, self.site(layer.layer_id, 0) + out_res, out_norm = torch.empty_like(residual), torch.empty_like(residual) + tail = (residual, layer.post_attention_layernorm.weight, out_res, out_norm) + if step is None: + self.kernels.o_proj(o_input, self.w_o[i], *tail, site=site) + return out_norm, out_res + rows, dev = residual.shape[0], residual.device + ids = torch.empty((rows, TOP_K), dtype=torch.int32, device=dev) + wts = torch.empty((rows, TOP_K), dtype=torch.float32, device=dev) + bias = layer.mlp.gate.e_score_correction_bias + self.kernels.o_proj_router( + o_input, self.w_o[i], *self.router[i], bias, *tail, ids, wts, site=site + ) + step.routing = (ids, wts) + return out_norm, out_res + + def allreduce_norm(self, partial, residual, gamma, site: int): + out_res, out_norm = torch.empty_like(residual), torch.empty_like(residual) + self.kernels.allreduce_norm( + partial, residual, gamma, out_res, out_norm, site=site + ) + return out_norm, out_res + + def fold(self, rec, residual, gamma, site: int, dn: bool = False): + """The fold hop of the MoE records: ``(normed, residual out)``; with ``dn`` + (<= 8 rows) the norm is deferred: ``(u = bf16(x * gamma), residual out, + block sums [rows, 8] fp32)``.""" + rows = residual.shape[0] + out_res, out = torch.empty_like(residual), torch.empty_like(residual) + bsum = None + if dn or rows > SMALL_ROWS: + bsum = torch.empty((rows, 8), dtype=torch.float32, device=residual.device) + args = (rec.pq, rec.shpartials, rec.ids_fin, rec.w_fin, rec.g2s, residual) + if dn: + self.kernels.moe_fold_dn(*args, gamma, out_res, out, bsum, site=site) + return out, out_res, bsum + self.kernels.moe_fold(*args, gamma, out_res, out, bsum=bsum, site=site) + return out, out_res + + +class _FarHops(HopSteps): + """Rows <= 8 with the stock MLP / MoE: every boundary is a hop.""" + + def __init__(self, family: HopFamily) -> None: + self.f = family + + def pre_attention(self, layer, mlp_out, residual, step: StepContext): + if residual is None: # first layer: the embedding + return super().pre_attention(layer, mlp_out, residual, step) + site = self.f.site(layer.layer_id, 1) + gamma = layer.input_layernorm.weight + return self.f.allreduce_norm(mlp_out, residual, gamma, site) + + def post_attention(self, layer, o_input, residual, step: StepContext): + return self.f.o_proj(layer, o_input, residual) + + def final(self, model, mlp_out, residual, step: StepContext) -> torch.Tensor: + site = self.f.site(self.f.last, 2) + gamma = model.model.norm.weight + return self.f.allreduce_norm(mlp_out, residual, gamma, site)[0] + + +class _FoldHops(HopSteps): + """The MoE family is served: routing ``o_proj`` hops and folds on MoE + layers, plain hops on dense layers up to ``DENSE_MAX_ROWS`` rows. Requires + the MoE family at the same row count; ``fallback`` is served where it is + stock.""" + + requires = ("mlp",) + + def __init__( + self, family: HopFamily, rows: int, fallback: Optional[HopSteps] + ) -> None: + self.f, self.rows, self.fallback = family, rows, fallback + self.dense = rows <= DENSE_MAX_ROWS + self.dnorm_layers = frozenset() + + def defer_input_norm(self) -> None: + """Serve DNORM on the family's DNORM layers; called by the composition + when this row count's attention consumes it.""" + if self.rows <= SMALL_ROWS: + self.dnorm_layers = self.f.dnorm_layers + logger.info( + "GLM DSA fused decode: deferred input norm served on %d layers at %d " + "rows", + len(self.dnorm_layers), + self.rows, + ) + + def pre_attention(self, layer, mlp_out, residual, step: StepContext): + if residual is None: + return super().pre_attention(layer, mlp_out, residual, step) + site = self.f.site(layer.layer_id, 1) + gamma = layer.input_layernorm.weight + if not isinstance(mlp_out, torch.Tensor): + if layer.layer_id in self.dnorm_layers: + u, out_res, bsum = self.f.fold(mlp_out, residual, gamma, site, dn=True) + step.dnorm = (layer.layer_id, bsum) + return u, out_res + return self.f.fold(mlp_out, residual, gamma, site) + if self.dense: + return self.f.allreduce_norm(mlp_out, residual, gamma, site) + return super().pre_attention(layer, mlp_out, residual, step) + + def post_attention(self, layer, o_input, residual, step: StepContext): + if self.f.is_moe[layer.layer_id - self.f.first]: + return self.f.o_proj(layer, o_input, residual, step) + if self.dense: + return self.f.o_proj(layer, o_input, residual) + return super().post_attention(layer, o_input, residual, step) + + def final(self, model, mlp_out, residual, step: StepContext) -> torch.Tensor: + site = self.f.site(self.f.last, 2) + gamma = model.model.norm.weight + if not isinstance(mlp_out, torch.Tensor): + return self.f.fold(mlp_out, residual, gamma, site)[0] + if self.dense: + return self.f.allreduce_norm(mlp_out, residual, gamma, site)[0] + return super().final(model, mlp_out, residual, step) + + +def build(model: Any, derived: Any) -> Optional[HopFamily]: + """The hop family of ``model``, or None to keep every boundary stock.""" + # Defensive: hop kernel modules that fail to import keep every boundary on the + # stock all-reduce (the attention / MoE kernels do not need them). + try: + from sgl_nvidia_mega.glm_dsa.kernels.cute.hop import api + except ImportError as e: + logger.warning( + "GLM decode hops unavailable (%s); using the stock all-reduce.", e + ) + return None + return HopFamily(model, derived, api) diff --git a/sgl_nvidia_mega/glm_dsa/draft.py b/sgl_nvidia_mega/glm_dsa/draft.py index 9a24d0dd..51be1ec9 100644 --- a/sgl_nvidia_mega/glm_dsa/draft.py +++ b/sgl_nvidia_mega/glm_dsa/draft.py @@ -35,7 +35,7 @@ runtime, ) from sgl_nvidia_mega.glm_dsa.eligibility import HIDDEN, TP_SIZE -from sgl_nvidia_mega.glm_dsa.providers import compile_cached, cu_stream +from sgl_nvidia_mega.glm_dsa.runtime.compile import compile_cached, cu_stream if TYPE_CHECKING: from sglang.srt.model_executor.forward_batch_info import ForwardBatch diff --git a/sgl_nvidia_mega/glm_dsa/draft_attention.py b/sgl_nvidia_mega/glm_dsa/draft_attention.py index 79213cc7..0b0ca37b 100644 --- a/sgl_nvidia_mega/glm_dsa/draft_attention.py +++ b/sgl_nvidia_mega/glm_dsa/draft_attention.py @@ -34,11 +34,8 @@ FusedAttention, kernel_rows, ) -from sgl_nvidia_mega.glm_dsa.providers import ( - AttentionDecode, - StepContext, -) -from sgl_nvidia_mega.glm_dsa.runtime import DerivedWeights +from sgl_nvidia_mega.glm_dsa.attention.step import AttentionDecode, StepContext +from sgl_nvidia_mega.glm_dsa.runtime.executor import DerivedWeights @dataclass diff --git a/sgl_nvidia_mega/glm_dsa/eligibility.py b/sgl_nvidia_mega/glm_dsa/eligibility.py index 71c2a33e..79d56269 100644 --- a/sgl_nvidia_mega/glm_dsa/eligibility.py +++ b/sgl_nvidia_mega/glm_dsa/eligibility.py @@ -27,6 +27,7 @@ # what the refusals name as the switch that asked for the path SWITCH = "--enable-nvidia-megakernel (GLM-5 DSA)" + # GLM-5 DSA checkpoint shapes the path is built for. _EXPECTED_SHAPES = { "hidden_size": HIDDEN, @@ -191,17 +192,23 @@ def construction_reasons(config, server_args, *, device_capability=None) -> List return reasons -def check_construction(model) -> None: - """Tier 1. Raise ``ValueError`` listing every unsupported setting.""" +def model_construction_reasons(model, config=None) -> List[str]: + """Tier 1 for a constructed model: ``construction_reasons`` of its config (or + ``config``) and the running server's arguments, and its shared-experts fusion.""" from sglang.srt.runtime_context import get_server_args - reasons = construction_reasons(model.config, get_server_args()) + reasons = construction_reasons(config or model.config, get_server_args()) if getattr(model, "num_fused_shared_experts", 0) != 0: reasons.append( "requires shared-experts fusion off (num_fused_shared_experts == 0; " "pass --disable-shared-experts-fusion)" ) - _refuse(SWITCH, "the configuration", reasons) + return reasons + + +def check_construction(model) -> None: + """Tier 1. Raise ``ValueError`` listing every unsupported setting.""" + _refuse(SWITCH, "the configuration", model_construction_reasons(model)) def check_loaded(model) -> None: diff --git a/sgl_nvidia_mega/glm_dsa/hops.py b/sgl_nvidia_mega/glm_dsa/hops.py index d36ecaa6..4284f3ca 100644 --- a/sgl_nvidia_mega/glm_dsa/hops.py +++ b/sgl_nvidia_mega/glm_dsa/hops.py @@ -138,18 +138,40 @@ def compile(self, rows) -> None: if not reason: import cutlass.cute as cute - for spec in (s for s in self.api.unit_specs() if s.name in names): + from sgl_nvidia_mega.glm_dsa.kernels.cute import launch + + idx = self.device.index + per_sm = switches.hop_ctas_per_sm() + pad = launch.spread_smem(idx, per_sm) if per_sm else 0 + carveout = switches.hop_max_shared_carveout() + specs = self.api.unit_specs(consume_pad_smem=pad) + for spec in (s for s in specs if s.name in names): compiled[spec.name] = compile_cached( spec.obj, spec.constexprs, lambda spec=spec: cute.compile(spec.obj, *spec.args), ) + padded = getattr(spec.obj, "pad_smem", 0) + if carveout or padded: + launch.set_shared_memory_attributes( + compiled[spec.name], + idx, + max_carveout=carveout, + max_dynamic_smem=padded, + ) if spec.name == "moe_fold_split" and not hasattr( compiled[spec.name], "library" ): # The occupancy query needs the driver library, which a # cache-loaded unit may not keep: query a fresh compile. probe = cute.compile(spec.obj, *spec.args) + if carveout or pad: + logger.info( + "GLM DSA fused decode: hops prefer the max shared carveout: %s; " + "o_proj consumer CTAs per SM: %s", + carveout, + f"<= {per_sm} ({pad} B launch shared memory)" if pad else "unpadded", + ) ok = torch.tensor([0 if reason else 1], dtype=torch.int32) dist.all_reduce(ok, op=dist.ReduceOp.MIN, group=self.group.cpu_group) if int(ok.item()) == 0: diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/api.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/api.py index 35bb0233..64caef3a 100644 --- a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/api.py +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/api.py @@ -156,17 +156,23 @@ def _args_apply(): return (*[_z()] * 4, Float32(1e-5), Int32(1), cu.CUstream(0)) -def unit_specs(*, plain_allreduce: bool = False): +def unit_specs(*, plain_allreduce: bool = False, consume_pad_smem: int = 0): """The compile units of the forward hops (instance A: up to 8 rows; instance B: - 9..16).""" + 9..16). ``consume_pad_smem``: the o_proj consumers' padded launch shared memory + (``OProjArConsume``; 0: unpadded).""" + pad = consume_pad_smem specs = [ UnitSpec( "allreduce_norm", AllreduceNormHop(residual=True, norm=True), _args_ar() ), UnitSpec("o_proj_produce", OProjArProduce(), _args_produce()), - UnitSpec("o_proj_consume", OProjArConsume(), _args_consume()), + UnitSpec("o_proj_consume", OProjArConsume(pad_smem=pad), _args_consume()), UnitSpec("o_proj_produce_router", OProjArProduce(router=True), _args_produce()), - UnitSpec("o_proj_consume_router", OProjArConsume(router=True), _args_consume()), + UnitSpec( + "o_proj_consume_router", + OProjArConsume(router=True, pad_smem=pad), + _args_consume(), + ), UnitSpec("moe_fold_split", MoeFoldArNorm(split=True), _args_fold()), UnitSpec("moe_fold_plain", MoeFoldArNorm(split=False), _args_fold()), UnitSpec( @@ -176,7 +182,7 @@ def unit_specs(*, plain_allreduce: bool = False): ), UnitSpec( "o_proj_consume_router16", - OProjArConsume(router=True, max_rows=16), + OProjArConsume(router=True, max_rows=16, pad_smem=pad), _args_consume(), ), UnitSpec("moe_fold_block_sums16", MoeFoldArBsum(), _args_bsum()), diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/o_proj.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/o_proj.py index 761e2e33..e67f1762 100644 --- a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/o_proj.py +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/o_proj.py @@ -635,7 +635,12 @@ def __init__( residual: bool = True, norm: bool = True, max_rows: int = 8, + pad_smem: int = 0, ): + """``pad_smem``: dynamic shared memory per CTA at launch (0: what the kernel + allocates). A larger size limits how many CTAs share an SM + (``launch.spread_smem``): they then compete less for its memory bandwidth + while they poll. The kernel never touches the padding; outputs are unchanged.""" if residual and not norm: raise ValueError("residual without RMSNorm is not provided") if router and not (residual and norm): @@ -650,6 +655,10 @@ def __init__( norm, max_rows, ) + self.pad_smem = pad_smem + + def launch_options(self) -> dict: + return {"smem": self.pad_smem} if self.pad_smem else {} @cute.jit def __call__( @@ -711,6 +720,7 @@ def __call__( cluster=[8, 1, 1], use_pdl=True, stream=stream, + **self.launch_options(), ) @cute.jit diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/launch.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/launch.py index ce88bce3..270b2e63 100644 --- a/sgl_nvidia_mega/glm_dsa/kernels/cute/launch.py +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/launch.py @@ -23,7 +23,14 @@ from cuda.bindings import driver as cu -__all__ = ["KernelHandle", "kernel_handle", "fits_cooperative", "max_active_clusters"] +__all__ = [ + "KernelHandle", + "kernel_handle", + "fits_cooperative", + "max_active_clusters", + "spread_smem", + "set_shared_memory_attributes", +] def _ok(res, what): @@ -101,3 +108,55 @@ def max_active_clusters( "cuOccupancyMaxActiveClusters", ) ) + + +def _device_attr(dev, attr): + return int(_ok(cu.cuDeviceGetAttribute(attr, dev), "cuDeviceGetAttribute")) + + +def spread_smem(device_index: int, ctas_per_sm: int) -> int: + """Dynamic shared memory per CTA (a multiple of 128 B) at which at most + ``ctas_per_sm`` CTAs of a kernel fit on one SM of this device. The kernel never + touches the padding; it only keeps the hardware from packing more CTAs per SM.""" + A = cu.CUdevice_attribute + dev = _ok(cu.cuDeviceGet(device_index), "cuDeviceGet") + per_sm = _device_attr(dev, A.CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_MULTIPROCESSOR) + per_block = _device_attr(dev, A.CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN) + reserved = _device_attr(dev, A.CU_DEVICE_ATTRIBUTE_RESERVED_SHARED_MEMORY_PER_BLOCK) + need = per_sm // (ctas_per_sm + 1) - reserved + 1 + return min(-(-need // 128) * 128, per_block) + + +def set_shared_memory_attributes( + compiled, device_index: int, *, max_carveout: bool, max_dynamic_smem: int = 0 +) -> None: + """Set shared-memory attributes on every kernel of a compiled unit: + ``max_carveout`` prefers the maximum shared-memory carveout (a scheduling hint: + an SM running these CTAs stays in the configuration a large-shared-memory kernel + launched next can share); ``max_dynamic_smem`` raises the dynamic shared-memory + limit to a padded launch size.""" + lib = getattr(compiled, "library", None) + if lib is None and hasattr(compiled, "to"): + compiled.to(device_index) + lib = getattr(compiled, "library", None) + if lib is None: + raise RuntimeError("no driver library for the compiled unit") + count = _ok(cu.cuLibraryGetKernelCount(lib), "cuLibraryGetKernelCount") + kernels = _ok(cu.cuLibraryEnumerateKernels(count, lib), "cuLibraryEnumerateKernels") + dev = _ok(cu.cuDeviceGet(device_index), "cuDeviceGet") + F = cu.CUfunction_attribute + attrs = [] + if max_carveout: + attrs.append( + ( + F.CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT, + int(cu.CUshared_carveout.CU_SHAREDMEM_CARVEOUT_MAX_SHARED), + ) + ) + if max_dynamic_smem: + attrs.append( + (F.CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, int(max_dynamic_smem)) + ) + for k in kernels: + for attr, value in attrs: + _ok(cu.cuKernelSetAttribute(attr, value, k, dev), "cuKernelSetAttribute") diff --git a/sgl_nvidia_mega/glm_dsa/moe/__init__.py b/sgl_nvidia_mega/glm_dsa/moe/__init__.py new file mode 100644 index 00000000..11c0830a --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/moe/__init__.py @@ -0,0 +1,169 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""CuTe DSL small-M NVFP4 MoE in the verify step of the opt-in fused GLM-5 DSA path. + +At 4 / 8 / 12 / 16 verify rows (1..4 requests of 4 rows) each MoE layer runs the stock +router GEMM (``mlp.gate``), ``moe_route_top8`` (the top-8 selection of the stock TRT-LLM +NVFP4 MoE routing: sigmoid scores, correction bias, renormalized weights x the routed +scaling factor), one engine launch for the routed experts and the BF16 shared expert, and +``moe_finalize`` (the layer's un-reduced output, which the next hop step all-reduces as +it does the stock MoE output). When the layer's ``o_proj`` hop has already routed the +rows (``step.routing``), the router GEMM and ``moe_route_top8`` are skipped, the engine +runs without the finalize, and the step returns ``MoeRecords`` (the per-expert outputs) +for the MLP-output hop, which finalizes them inside its all-reduce. + +The engine reads the server's processed routed FC1 weights in place. FC2 is read from +compact per-quarter records and the shared expert from a packed copy; both are derived +at load and registered (226.5 MB + 9.4 MB per MoE layer and rank). ``build`` keeps the +MoE stock unless the engine API accepts the configuration, every MoE layer has the +expected tensors and the server's own NVFP4 weight processing passes the engine's +layout self-test. +""" + +from __future__ import annotations + +import logging +import types +from dataclasses import dataclass +from typing import Dict, Optional, Tuple + +import torch + +from sgl_nvidia_mega.glm_dsa.eligibility import ( + HIDDEN, + SUPPORTED_ROWS, + VERIFY_TOKENS_PER_REQUEST, +) +from sgl_nvidia_mega.glm_dsa.runtime.compile import compile_cached, cu_stream +from sgl_nvidia_mega.glm_dsa.moe.step import ( + MlpSteps, +) + +logger = logging.getLogger(__name__) + +EXPERTS = 256 +TOP_K = 8 +INTER = 256 # routed and shared intermediate size per rank (TP 8) + + +from sgl_nvidia_mega.glm_dsa.moe.step import MoeRecords, MoeSteps, MlpSteps # noqa: F401 + + +class MoeGlue: + """Derived MoE weights, compiled engines and per-row-count steps.""" + + def __init__(self, api, layers: Dict[int, object], specs) -> None: + self.api, self.specs, self.layers = api, specs, layers + self.device = dev = next(iter(layers.values())).w13.device + self.ids = torch.empty(16, TOP_K, dtype=torch.int32, device=dev) + self.weights = torch.empty(16, TOP_K, dtype=torch.float32, device=dev) + self.fn: Dict[str, object] = {} + self.engine = None + + def compile(self, rows: Tuple[int, ...]) -> None: + import cutlass.cute as cute + + per = VERIFY_TOKENS_PER_REQUEST + modes = { + self.api.mode_for((per,) * (m // per)) for m in rows if m in SUPPORTED_ROWS + } + for name in sorted({"route", "finalize"} | modes): + kernel, args = self.specs[name] + self.fn[name] = compile_cached( + kernel, + {"unit": name, "shared": True}, + lambda k=kernel, a=args: cute.compile(k, *a), + ) + nclusters = self.api.default_clusters(self.device) + self.engine = self.api.SmallMMoe(self.fn, nclusters, self.device, shared=True) + + def records(self, m: int) -> MoeRecords: + sc = self.engine.scratch + return MoeRecords( + pq=sc.partials[: 4 * m * TOP_K * HIDDEN].view(4, m, TOP_K, HIDDEN), + shpartials=sc.shared_partials[: 4 * m * HIDDEN].view(4, m, HIDDEN), + ids_fin=sc.slot_ids[: m * TOP_K].view(m, TOP_K), + w_fin=sc.slot_weights[: m * TOP_K].view(m, TOP_K), + g2s=sc.slot_scales[: m * TOP_K].view(m, TOP_K), + ) + + def route(self, logits, bias, scale, ids, weights) -> None: + from cutlass import Float32, Int32, Int64 + + m = logits.shape[0] + if ( + logits.dtype != torch.float32 + or tuple(logits.shape) != (m, EXPERTS) + or not logits.is_contiguous() + ): + raise RuntimeError( + f"fused verify MoE: router logits {tuple(logits.shape)} {logits.dtype}" + ) + self.fn["route"]( + Int64(logits.data_ptr()), + Int64(bias.data_ptr()), + Float32(float(scale)), + Int64(ids.data_ptr()), + Int64(weights.data_ptr()), + Int32(m), + cu_stream(logits.device), + ) + + def for_rows(self, m: int) -> Optional[MoeSteps]: + if m not in SUPPORTED_ROWS or self.engine is None: + return None + return MoeSteps(self, m) + + +def build(model, derived) -> Optional[MoeGlue]: + from sgl_nvidia_mega.glm_dsa.kernels.cute.moe import api + from sgl_nvidia_mega.glm_dsa.kernels.cute.moe import weights as W + from sglang.srt.models.deepseek_v2 import DeepseekV2MoE + + def stock(why) -> None: + logger.warning("GLM DSA fused decode: MoE stays stock (%s)", why) + + moe_layers = [l for l in model.model.layers if isinstance(l.mlp, DeepseekV2MoE)] + if not moe_layers: + return None + for layer in moe_layers: + why = _layer_reason(layer.mlp) + if why is not None: + return stock(f"layer {layer.layer_id}: {why}") + try: + specs = api.compile_specs(shared=True) + except ValueError as e: + return stock(e) + dev = moe_layers[0].mlp.experts.w13_weight.device + try: + W.layout_self_test(_process_like_server, dev) + except W.LayoutMismatch as e: + return stock(f"layout self-test: {e}") + layers = {} + for layer in moe_layers: + ex, sh = layer.mlp.experts, layer.mlp.shared_experts + records = derived.add( + "moe.fc2_records", + W.fc2_records( + _u8(ex.w2_weight.data).reshape(EXPERTS, HIDDEN, INTER // 2), + _u8(ex.w2_weight_scale.data).reshape(EXPERTS, -1), + ), + ) + pack = derived.add( + "moe.shared_pack", + W.shared_pack(sh.gate_up_proj.weight.data, sh.down_proj.weight.data), + ) + lt = api.LayerTensors( + w13=ex.w13_weight.data, + w13_scales=_u8(ex.w13_weight_scale.data).reshape(EXPERTS, -1), + fc2_records=records.view(EXPERTS, 4, 4, -1), + shared_pack=pack, + g1_alpha=ex.g1_alphas.data, + g2_alpha=ex.g2_alphas.data, + gs1=float(ex.w13_input_scale_quant.item()), + gs2=float(ex.w2_input_scale_quant.item()), + ) + lt.check() + layers[layer.layer_id] = lt + logger.info("GLM DSA fused decode: MoE engine on %d layers", len(layers)) + return MoeGlue(api, layers, specs) diff --git a/sgl_nvidia_mega/glm_dsa/moe/step.py b/sgl_nvidia_mega/glm_dsa/moe/step.py new file mode 100644 index 00000000..3d4010f1 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/moe/step.py @@ -0,0 +1,180 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""Stock MLP step body (MlpSteps) and MoE override steps (MoeRecords, MoeSteps).""" + +from __future__ import annotations + +from contextlib import nullcontext +from dataclasses import dataclass +from typing import Any, Optional + +import torch + +from sgl_nvidia_mega.glm_dsa.attention.step import StepContext + + +class MlpSteps: + """Stock dense MLP / MoE of a layer, leaving the output's sum to the next step.""" + + def forward(self, layer, x, buffer: Optional[torch.Tensor], step: StepContext): + """``buffer``: this layer's input storage, dead once the attention input is + read (the MoE may write its output into it).""" + from sglang.srt.layers.moe.moe_runner.base import moe_output_buffer_ctx + from sglang.srt.models.deepseek_v2 import DeepseekV2MoE + from sglang.srt.runtime_context import get_forward + + mlp = layer.mlp + if ( + isinstance(mlp, DeepseekV2MoE) + and not mlp.experts.moe_runner_config.inplace + and buffer is not None + ): + buffer_ctx = moe_output_buffer_ctx(buffer) + else: + buffer_ctx = nullcontext() + with get_forward().scoped(defer_moe_finalize=False), buffer_ctx: + return mlp(x, step.forward_batch, None) + + +# The steps of each group, in call order (describe, rank agreement, logs). + +class MoeRecords: + """The engine's per-expert outputs of one MoE layer, for a hop that finalizes them + inside its all-reduce (row stride ``rows``): ``pq`` [4, rows, 8, 6144] fp32 (the four + quarters' FC2 outputs per top-k slot), ``shpartials`` [4, rows, 6144] fp32 (shared expert), + ``ids_fin`` / ``w_fin`` / ``g2s`` [rows, 8] (slot expert id or -1, routing weight, FC2 + scale).""" + + pq: torch.Tensor + shpartials: torch.Tensor + ids_fin: torch.Tensor + w_fin: torch.Tensor + g2s: torch.Tensor + + +def _process_like_server(gate, up, gate_sf, up_sf, w2, w2_sf): + """The server's NVFP4 TRT-LLM weight processing on a synthetic layer: the loader's + w13 order for this backend (up rows first) and the same processing call.""" + from sglang.srt.layers.moe.moe_runner import flashinfer_trtllm as trtllm + + e = gate.shape[0] + f8 = torch.float8_e4m3fn + p = lambda t: torch.nn.Parameter(t, requires_grad=False) # noqa: E731 + layer = torch.nn.Module() + layer.w13_weight = p(torch.cat([up, gate], dim=1).contiguous()) + layer.w2_weight = p(w2.contiguous()) + layer.w13_weight_scale = p(torch.cat([up_sf, gate_sf], dim=1).view(f8).contiguous()) + layer.w2_weight_scale = p(w2_sf.view(f8).contiguous()) + ones = torch.ones(e, dtype=torch.float32, device=gate.device) + layer.w2_input_scale_quant = p(ones.clone()) + layer.g1_alphas = p(ones.clone()) + layer.g1_alphas_up = p(ones.clone()) + layer.moe_runner_config = types.SimpleNamespace(is_gated=True, activation="silu") + trtllm.align_fp4_moe_weights_for_flashinfer_trtllm(layer) + return ( + layer.w13_weight.data, + layer.w13_weight_scale.data, + layer.w2_weight.data, + layer.w2_weight_scale.data, + ) + + +def _u8(t: torch.Tensor) -> torch.Tensor: + return t if t.dtype == torch.uint8 else t.view(torch.uint8) + + +def _bad(t, dtype, shape=None, numel=None, contiguous=True) -> bool: + return ( + t is None + or t.dtype != dtype + or (shape is not None and tuple(t.shape) != shape) + or (numel is not None and t.numel() != numel) + or (contiguous and not t.is_contiguous()) + ) + + +def _layer_reason(mlp) -> Optional[str]: + """Why this MoE layer cannot run on the engine (None if it can).""" + ex = mlp.experts + for name, shape in ( + ("w13_weight", (EXPERTS, 2 * INTER, HIDDEN // 2)), + ("w2_weight", (EXPERTS, HIDDEN, INTER // 2)), + ): + if _bad(getattr(ex, name, None), torch.uint8, shape): + return f"{name} is not contiguous uint8 {shape}" + for name, n in ( + ("w13_weight_scale", EXPERTS * 2 * INTER * HIDDEN // 16), + ("w2_weight_scale", EXPERTS * HIDDEN * INTER // 16), + ): + if _bad(getattr(ex, name, None), torch.float8_e4m3fn, numel=n): + return f"{name} is not contiguous e4m3 with {n} elements" + for name in ("g1_alphas", "g1_alphas_up", "g2_alphas"): + if _bad(getattr(ex, name, None), torch.float32, (EXPERTS,), contiguous=False): + return f"{name} is not fp32 [{EXPERTS}]" + if not torch.equal(ex.g1_alphas, ex.g1_alphas_up): + return "gate and up GEMM1 scales differ (the engine applies one per expert)" + for name in ("w13_input_scale_quant", "w2_input_scale_quant"): + t = getattr(ex, name, None) + if t is None or t.numel() != 1: + return f"{name} is not a single scale" + cfg = ex.moe_runner_config + if ( + getattr(cfg, "activation", "silu") != "silu" + or getattr(cfg, "gemm1_clamp_limit", None) + or getattr(cfg, "swiglu_limit", None) + or getattr(cfg, "gemm1_alpha", None) is not None + ): + return "only an unclamped SiLU-gated MLP is supported" + gu, dn = mlp.shared_experts.gate_up_proj.weight, mlp.shared_experts.down_proj.weight + if _bad(gu, torch.bfloat16, (2 * INTER, HIDDEN), contiguous=False): + return "shared expert gate_up_proj is not bf16 [512, 6144]" + if _bad(dn, torch.bfloat16, (HIDDEN, INTER), contiguous=False): + return "shared expert down_proj is not bf16 [6144, 256]" + bias = mlp.gate.e_score_correction_bias + if _bad(bias, torch.float32, (EXPERTS,), contiguous=False): + return "router correction bias is not fp32 [256]" + return None + + +class MoeSteps(MlpSteps): + """The MoE layers of one verify row count on the engine; dense layers stay stock.""" + + def __init__(self, glue: MoeGlue, rows: int) -> None: + self.glue, self.rows = glue, rows + self.split = (VERIFY_TOKENS_PER_REQUEST,) * (rows // VERIFY_TOKENS_PER_REQUEST) + + def forward(self, layer, x, buffer: Optional[torch.Tensor], step): + glue = self.glue + lt = glue.layers.get(layer.layer_id) + if lt is None: + return super().forward(layer, x, buffer, step) + m = x.shape[0] + if m != self.rows or x.dtype != torch.bfloat16 or not x.is_contiguous(): + raise RuntimeError( + f"fused verify MoE: unexpected input {tuple(x.shape)} {x.dtype} at {self.rows} rows" + ) + routing, step.routing = step.routing, None + if routing is not None: + # routed by this layer's o_proj hop; its MLP-output hop finalizes the records + ids, weights = routing + glue.engine(lt, x, ids, weights, self.split, what=1) + return glue.records(m) + ids, weights = glue.ids[:m], glue.weights[:m] + gate = layer.mlp.gate + scale = layer.mlp.routed_scaling_factor + glue.route(gate(x), gate.e_score_correction_bias, scale, ids, weights) + if ( + isinstance(buffer, torch.Tensor) + and buffer.dtype == torch.bfloat16 + and buffer.dim() == 2 + and buffer.shape[0] >= m + and buffer.shape[1] == HIDDEN + and buffer.is_contiguous() + ): + out = buffer[:m] + else: + out = torch.empty(m, HIDDEN, dtype=torch.bfloat16, device=x.device) + glue.engine(lt, x, ids, weights, self.split, out=out, what=3) + return out + + diff --git a/sgl_nvidia_mega/glm_dsa/plan.py b/sgl_nvidia_mega/glm_dsa/plan.py new file mode 100644 index 00000000..44eb0d89 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/plan.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""Coordination layer: which step groups cover which slots and at which row counts. + +``build_providers`` builds the kernel family objects and returns a +``Providers`` per verify row count; ``compose`` resolves dependency fallbacks +within one row count; ``families`` translates active slots into the kernel +family names the runtime passes to the build step. +""" + +from __future__ import annotations + +from typing import Any, Dict, Tuple + +from sgl_nvidia_mega.glm_dsa.eligibility import SUPPORTED_ROWS, VERIFY_TOKENS_PER_REQUEST + +MAX_ROWS = max(SUPPORTED_ROWS) + +# Steps of each group in call order (for logging, rank agreement, tests). +STEPS = { + "attention": ( + "forward", + "qkv_a", + "prologue", + "index_k", + "index_q", + "indexer", + "q_absorb", + "rope_cache", + "sparse_mla", + "uv_proj", + ), + "hops": ("pre_attention", "post_attention", "final"), + "mlp": ("forward",), +} + +# Maps a slot name to the kernel family group it populates. +FAMILY_SLOTS = {"attention": "attention", "hops": "hops", "moe": "mlp"} + + +def _attention(): + from sgl_nvidia_mega.glm_dsa.attention import AttentionGlue + return AttentionGlue + + +def _hops(): + from sgl_nvidia_mega.glm_dsa.boundary import HopFamily + return HopFamily + + +def _mlp(): + from sgl_nvidia_mega.glm_dsa.moe import MoeGlue + return MoeGlue + + +def families(slots) -> Dict[str, str]: + """Active kernel family names for the given set of slot names.""" + return {v: k for k, v in FAMILY_SLOTS.items() if k in slots} + + +def build_providers(model, rows: Tuple[int, ...], derived, slots=None): + """(step groups per verify row count, built families). + + Each family module's ``build(model, derived)`` returns None (stay stock) + or an object with ``compile(rows)`` and ``for_rows(m)``. When ``slots`` is + given only the slot-enabled families are attempted. + """ + from sgl_nvidia_mega.glm_dsa.providers import Providers, compose + + if slots is None: + from sgl_nvidia_mega.glm_dsa.runtime import SLOTS + slots = SLOTS + + glue_map = { + "attention": ("attention", lambda m, d: _build_family("attention", m, d)), + "hops": ("hops", lambda m, d: _build_family("hops", m, d)), + "moe": ("mlp", lambda m, d: _build_family("mlp", m, d)), + } + + family_objs: Dict[str, Any] = {} + for slot_name, (group, builder) in glue_map.items(): + if slot_name not in slots: + continue + fam = builder(model, derived) + if fam is not None: + fam.compile(rows) + family_objs[group] = fam + + out = { + m: Providers(**compose(m, {g: f.for_rows(m) for g, f in family_objs.items()})) + for m in rows + } + for prov in out.values(): + wire = getattr(prov.hops, "defer_input_norm", None) + if wire is not None and getattr(prov.attention, "takes_deferred_norm", False): + wire() + return out, family_objs + + +def _build_family(slot_name: str, model, derived): + if slot_name == "attention": + from sgl_nvidia_mega.glm_dsa import attention + return attention.build(model, derived) + if slot_name == "hops": + from sgl_nvidia_mega.glm_dsa import hops + return hops.build(model, derived) + if slot_name == "mlp": + from sgl_nvidia_mega.glm_dsa import moe + return moe.build(model, derived) + return None diff --git a/sgl_nvidia_mega/glm_dsa/runtime/__init__.py b/sgl_nvidia_mega/glm_dsa/runtime/__init__.py new file mode 100644 index 00000000..edb86104 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/runtime/__init__.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""Runtime package: SLOTS mechanism, entry point, re-exports of executor/compile.""" + +from __future__ import annotations + +import os +from typing import List + +# Kernel slot names shipped in this build. +SHIPPED_SLOTS = ( + "attention", + "moe", + "hops", + "qkv_a_small_m_shared", + "dnorm", + "draft", +) + +DRAFT_SLOTS = frozenset({"draft"}) + +SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" + +# (slot, prerequisite): a slot is only active if its prerequisite is also active. +SLOT_NEEDS = ( + ("qkv_a_small_m_shared", "attention"), + ("dnorm", "qkv_a_small_m_shared"), + ("dnorm", "hops"), +) + +# SGLang symbols this package reads (validated by test/unit/test_contract.py) +SGLANG_READS = { + "kernels/plugin_kernels/nvidia/spec.py|srt/megakernels/spec.py": ( + "MegakernelSpec", + "MegakernelBinding", + "ForwardView", + "KvPoolView", + ), + "srt/runtime_context.py": ( + "get_parallel", + "get_forward", + ), +} + + +def _requested_slots() -> frozenset: + raw = os.environ.get(SLOTS_ENV) + if raw is None: + return frozenset(SHIPPED_SLOTS) + return frozenset(name.strip() for name in raw.split(",")) - {""} + + +SLOTS = _requested_slots() + + +def slot_reasons() -> List[str]: + """Reasons a slot was not activated (for startup logs / spec.reasons()).""" + reasons = [] + active = set(SLOTS) + for slot, prereq in SLOT_NEEDS: + if slot in active and prereq not in active: + active.discard(slot) + reasons.append( + f"slot {slot!r} needs {prereq!r} which is not active" + ) + if SLOTS != frozenset(SHIPPED_SLOTS): + requested = os.environ.get(SLOTS_ENV, "") + skipped = frozenset(SHIPPED_SLOTS) - SLOTS + if skipped: + reasons.append( + f"{SLOTS_ENV}={requested!r}: skipped slots: " + + ", ".join(sorted(skipped)) + ) + return reasons diff --git a/sgl_nvidia_mega/glm_dsa/runtime/compile.py b/sgl_nvidia_mega/glm_dsa/runtime/compile.py new file mode 100644 index 00000000..cf604be1 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/runtime/compile.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""Kernel compilation utilities: AOT cache wrapper and CUDA stream helper.""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Callable, Dict, Hashable, Optional, Tuple + +import torch + +logger = logging.getLogger(__name__) + +_jit_cache: Any = None # created on first use + +compile_stats: Dict[str, int] = {"compiled": 0, "cached": 0, "not_stored": 0} + + +def kernel_source_paths() -> Tuple[str, ...]: + """Sources whose content keys the persistent cache directory.""" + import sgl_nvidia_mega.glm_dsa.kernels as kernels + + return tuple( + os.path.dirname(os.path.abspath(f)) for f in (__file__, kernels.__file__) + ) + + +def compile_key(kernel: Any, constexprs: Dict[str, Hashable]) -> Tuple[Hashable, ...]: + """The cache key: kernel's class (so test subclasses cache separately).""" + cls = type(kernel) + return (f"{cls.__module__}.{cls.__qualname__}", tuple(sorted(constexprs.items()))) + + +def compile_cached( + kernel: Any, constexprs: Dict[str, Hashable], compile_fn: Callable[[], Any] +) -> Any: + """The compiled ``kernel`` for these constants, compiling on a cache miss.""" + global _jit_cache + if _jit_cache is None: + from sglang.kernels.jit.cute_aot_cache import get_jit_cache + + _jit_cache = get_jit_cache( + "glm_dsa", source_paths=kernel_source_paths(), enable_tvm_ffi=False + ) + key = compile_key(kernel, constexprs) + if key in _jit_cache: + compile_stats["cached"] += 1 + return _jit_cache[key] + compiled = compile_fn() + compile_stats["compiled"] += 1 + try: + _jit_cache[key] = compiled + except Exception as e: + compile_stats["not_stored"] += 1 + detail = " ".join(str(e).split())[:300] + logger.warning( + "GLM DSA fused decode: the CuTe AOT cache could not store %s %s (%s: %s); " + "it is compiled again in every new process", + key[0], + dict(key[1]), + type(e).__name__, + detail, + ) + return compiled + + +def cu_stream(device=None): + """The current torch stream of ``device`` as a kernel launch argument.""" + import cuda.bindings.driver as cuda + + return cuda.CUstream(torch.cuda.current_stream(device).cuda_stream) diff --git a/sgl_nvidia_mega/glm_dsa/runtime/executor.py b/sgl_nvidia_mega/glm_dsa/runtime/executor.py new file mode 100644 index 00000000..ac4ed888 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/runtime/executor.py @@ -0,0 +1,267 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""Fused verify executor: DerivedWeights, GlmDsaFusedDecode, ErrorReporter.""" + +from __future__ import annotations + +import atexit +import logging +import threading +import time +from typing import Dict, List, Optional + +import torch +import torch.distributed as dist + +from sglang.srt import runtime_context +from sgl_nvidia_mega.glm_dsa import eligibility +from sgl_nvidia_mega.glm_dsa.attention.step import StepContext +from sgl_nvidia_mega.glm_dsa.providers import Providers +from sgl_nvidia_mega.glm_dsa.runtime.compile import compile_stats + +# SGLang symbols this module reads (validated by test/unit/test_contract.py) +SGLANG_READS = { + "kernels/plugin_kernels/nvidia/spec.py|srt/megakernels/spec.py": ( + "MegakernelBinding", + "ForwardView", + "KvPoolView", + "DsaView", + ), + "srt/runtime_context.py": ("get_parallel", "get_forward"), + "srt/layers/layer_boundary/__init__.py": ("get_attn_tp_context",), + "srt/layers/layer_boundary/adapters/attention.py": ("AttentionInputs",), + "srt/eplb/expert_distribution.py": ("get_global_expert_distribution_recorder",), + "srt/layers/attention/dsa/utils.py": ("maybe_prefetch_next_full_attention_kv",), + "srt/layers/attention/index_topk_share.py": ("IndexTopKShareState",), + "srt/utils.py": ("BumpAllocator",), + "srt/layers/moe/moe_runner/base.py": ("moe_output_buffer_ctx",), +} + +logger = logging.getLogger(__name__) + +TARGET: Optional["GlmDsaFusedDecode"] = None + + +def tp_gather(obj) -> list: + group = runtime_context.get_parallel().tp_group + out = [None] * group.world_size + dist.all_gather_object(out, obj, group=group.cpu_group) + return out + + +def per_rank(gathered) -> str: + return "; ".join(f"rank {r}: {', '.join(g)}" for r, g in enumerate(gathered) if g) + + +class DerivedWeights: + """Owns packed / re-laid-out copies of stock weights (checkpoint unchanged).""" + + def __init__(self) -> None: + self.tensors: Dict[str, List[torch.Tensor]] = {} + + def add(self, kind: str, tensor: torch.Tensor) -> torch.Tensor: + self.tensors.setdefault(kind, []).append(tensor) + return tensor + + def log(self, rank: int) -> None: + nbytes = { + k: sum(t.numel() * t.element_size() for t in ts) + for k, ts in sorted(self.tensors.items()) + } + if not nbytes: + logger.info("GLM DSA fused decode: no derived weight copies (rank %d)", rank) + return + logger.info( + "GLM DSA fused decode: derived weight copies on rank %d: %s (total %.1f MiB)", + rank, + ", ".join(f"{k} {v / 2**20:.1f} MiB" for k, v in nbytes.items()), + sum(nbytes.values()) / 2**20, + ) + + +class GlmDsaFusedDecode: + """Owns the fused verify path of one target model instance.""" + + def __init__(self, model) -> None: + eligibility.check_construction(model) + self.model = model + self.derived = DerivedWeights() + self.providers: Dict[int, Providers] = {} + self.families: Dict[str, object] = {} + self._agreed: Dict[int, bool] = {} + self._ready = False + self.errors = ErrorReporter() + + def prepare(self) -> None: + """From ``precompile_kernels_after_loading``: provider sets + kernel compile.""" + global TARGET + from sgl_nvidia_mega.glm_dsa.plan import build_providers + + eligibility.check_loaded(self.model) + self.providers, self.families = build_providers( + self.model, eligibility.SUPPORTED_ROWS, self.derived + ) + rank = runtime_context.get_parallel().tp_rank + self._agree_on_composition(rank) + self.derived.log(rank) + logger.info( + "GLM DSA fused decode: kernels compiled %d, loaded from cache %d (rank %d), " + "not stored in the cache %d", + compile_stats["compiled"], + compile_stats["cached"], + rank, + compile_stats["not_stored"], + ) + if rank == 0: + for rows, p in self.providers.items(): + fused = { + k: v + for k, v in p.describe().items() + if not v.startswith(("AttentionDecode", "HopSteps", "MlpSteps")) + } + logger.info( + "GLM DSA fused decode: %d verify rows: %s", + rows, + fused or "all steps stock", + ) + self._ready = True + TARGET = self + for family in self.families.values(): + if callable(getattr(family, "error_report", None)): + self.errors.add(family.error_report) + + def _agree_on_composition(self, rank: int) -> None: + gathered = tp_gather({m: p.describe() for m, p in self.providers.items()}) + differing = [r for r, c in enumerate(gathered) if c != gathered[0]] + if differing: + if rank == 0: + logger.warning( + "GLM DSA fused decode: ranks %s composed different step groups " + "than rank 0; every rank serves the stock forward", + differing, + ) + self.providers = {rows: Providers() for rows in self.providers} + self.families = {} + + def engages(self, rows: int, forward_batch, input_embeds=None, pp_proxy_tensors=None) -> bool: + return ( + self._ready + and eligibility.step_in_graph_runner( + self.model, rows, forward_batch, input_embeds, pp_proxy_tensors + ) + and torch.cuda.is_current_stream_capturing() + and self._first_capture_agreement(rows, forward_batch) + ) + + def _first_capture_agreement(self, rows: int, forward_batch) -> bool: + if rows in self._agreed: + return self._agreed[rows] + reasons = list(eligibility.attention_step_reasons(self.model, forward_batch)) + for name, family in sorted(self.families.items()): + extra = getattr(family, "step_reasons", None) + if extra is not None: + reasons += [f"{name}: {r}" for r in extra(self.model, forward_batch)] + gathered = tp_gather(reasons) + self._agreed[rows] = agreed = not any(gathered) + if runtime_context.get_parallel().tp_rank == 0: + if agreed: + logger.info("GLM DSA fused decode: capturing the fused verify step, %d rows", rows) + else: + logger.warning( + "GLM DSA fused decode: %d verify rows captured on the stock forward: %s", + rows, + per_rank(gathered), + ) + return agreed + + @torch.no_grad() + def run(self, input_ids, positions, forward_batch) -> torch.Tensor: + from sglang.srt.layers.layer_boundary import get_attn_tp_context + + providers = self.providers[int(positions.shape[0])] + with get_attn_tp_context().maybe_input_scattered(forward_batch): + return self._stack(providers, input_ids, positions, forward_batch) + + def _stack(self, providers, input_ids, positions, forward_batch) -> torch.Tensor: + from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder + from sglang.srt.layers.attention.dsa.utils import maybe_prefetch_next_full_attention_kv + from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState + from sglang.srt.utils import BumpAllocator + + mm = self.model.model + step = StepContext( + forward_batch=forward_batch, + positions=positions, + zero_allocator=BumpAllocator( + buffer_size=(mm.end_layer - mm.start_layer) * 2, + dtype=torch.float32, + device=positions.device, + ), + ) + topk_share = IndexTopKShareState(forward_batch, None) + recorder = get_global_expert_distribution_recorder() + attention, hops, mlp = providers.attention, providers.hops, providers.mlp + mlp_out, residual = mm.embed_tokens(input_ids), None + for i in range(mm.start_layer, mm.end_layer): + layer = mm.layers[i] + with recorder.with_current_layer(i): + buffer = mlp_out if isinstance(mlp_out, torch.Tensor) else None + x, residual = hops.pre_attention(layer, mlp_out, residual, step) + o_input, topk_indices = attention.forward(layer, x, topk_share.topk_indices, step) + topk_share.update(topk_indices) + maybe_prefetch_next_full_attention_kv( + forward_batch, mm.next_full_attention_layer_id.get(i) + ) + x, residual = hops.post_attention(layer, o_input, residual, step) + mlp_out = mlp.forward(layer, x, buffer, step) + return hops.final(self.model, mlp_out, residual, step) + + +class ErrorReporter: + """Daemon thread watching for cross-GPU wait timeouts.""" + + def __init__(self, sources=(), period_s: float = 0.5) -> None: + self.sources = list(sources) + self.record = None + self._lock = threading.Lock() + self._period_s = period_s + self._watching = False + if self.sources: + self._start() + + def add(self, source) -> None: + with self._lock: + self.sources.append(source) + self._start() + + def _start(self) -> None: + if not self._watching: + self._watching = True + atexit.register(self.check) + threading.Thread(target=self._watch, name="glm-dsa-errors", daemon=True).start() + + def check(self): + with self._lock: + if self.record is not None: + return self.record + for read in list(self.sources): + try: + record = read() + except Exception as e: + logger.warning("GLM DSA fused decode: error record unreadable: %s", e) + self.sources.remove(read) + continue + if record: + self.record = record + logger.error( + "GLM DSA fused decode: a cross-GPU wait in the fused verify " + "step timed out and trapped; this process's CUDA context is " + "lost. Error record: %s", + record, + ) + break + return self.record + + def _watch(self) -> None: + while self.sources and self.check() is None: + time.sleep(self._period_s) diff --git a/sgl_nvidia_mega/glm_dsa/runtime/parity.py b/sgl_nvidia_mega/glm_dsa/runtime/parity.py new file mode 100644 index 00000000..fb9f8095 --- /dev/null +++ b/sgl_nvidia_mega/glm_dsa/runtime/parity.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""Parity test mode: run SGLang's stock forward then the plugin's stock steps on +the same inputs and log the per-step max absolute difference. + +Activated by ``SGLANG_NVIDIA_MEGAKERNEL_PARITY=1``. An env var +``SGLANG_NVIDIA_MEGAKERNEL_PARITY_SORT_TOPK=1`` sorts sparse-attention indices +in both forwards before comparison (emission order varies between them). +""" + +from __future__ import annotations + +import logging +import os +from typing import Optional + +logger = logging.getLogger(__name__) + +PARITY_ENV = "SGLANG_NVIDIA_MEGAKERNEL_PARITY" +PARITY_SORT_ENV = "SGLANG_NVIDIA_MEGAKERNEL_PARITY_SORT_TOPK" + + +def mode() -> bool: + """True when parity-check mode is active.""" + return os.environ.get(PARITY_ENV, "0") not in ("", "0") + + +def eager_steps_allowed() -> bool: + """Parity mode requires the stock forward to run eagerly first.""" + return not mode() + + +def run(fused_decode, stock_fn, input_ids, positions, forward_batch) -> None: + """Run ``stock_fn`` then the plugin's stock-body steps on the same inputs, + logging the max absolute difference for each step group output.""" + import torch + + if not mode(): + return + with torch.no_grad(): + stock_out = stock_fn(input_ids, positions, forward_batch) + plugin_out = fused_decode.run(input_ids, positions, forward_batch) + diff = (stock_out - plugin_out).abs().max().item() + logger.info( + "GLM DSA parity check: hidden-state max_abs diff stock vs plugin = %.3e " + "(rows=%d)", + diff, + int(positions.shape[0]), + ) + + +def run_draft(fused_draft, stock_fn, model, input_ids, positions, forward_batch) -> None: + """Parity check for the draft forward.""" + import torch + + if not mode(): + return + with torch.no_grad(): + stock_out = stock_fn(model, input_ids, positions, forward_batch) + plugin_out = fused_draft.run(model, input_ids, positions, forward_batch) + diff = (stock_out - plugin_out).abs().max().item() + logger.info( + "GLM DSA parity check (draft): hidden-state max_abs diff = %.3e (rows=%d)", + diff, + int(positions.shape[0]), + ) diff --git a/sgl_nvidia_mega/glm_dsa/runtime.py b/sgl_nvidia_mega/glm_dsa/runtime_legacy.py similarity index 100% rename from sgl_nvidia_mega/glm_dsa/runtime.py rename to sgl_nvidia_mega/glm_dsa/runtime_legacy.py diff --git a/sgl_nvidia_mega/glm_dsa/spec.py b/sgl_nvidia_mega/glm_dsa/spec.py index 11365d4c..550c142b 100644 --- a/sgl_nvidia_mega/glm_dsa/spec.py +++ b/sgl_nvidia_mega/glm_dsa/spec.py @@ -1,103 +1,287 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -"""The GLM-5 DSA megakernel's spec: the MTP target-verify steps of 1-4 requests -(4 / 8 / 12 / 16 rows) of ``GlmMoeDsaForCausalLM`` on SM103 / SM107 at TP8. +# Copyright 2026 SGLang Team +"""The GLM-5 DSA megakernel's spec: the setups it serves and its build. It serves +GLM-5.2 (NVFP4) on SM103 (B300) and SM107 (Rubin / VR NVL72). -SGLang checks the requirements at argument resolution and the model at load -(``sglang.srt.megakernels``); an unsupported setup stops the server on every rank, -listing every reason. This module stays light: SGLang imports it in the launcher. +SGLang checks the requirements at argument resolution and the model at load; +an unsupported setup stops the server on every rank, listing every reason. +This module stays light: SGLang imports it in the launcher. + +Stated limits of this spec: + +* The hops' cross-GPU waits use a fixed bound compiled into the kernels + (``kernels/cute/hop/constants.py`` ``TIMEOUT_NS``), not ``BuildContext.deadline_s``: + the spec refuses a ``--watchdog-timeout`` whose deadline is shorter (below 40 s). +* The first capture of each width checks, on every rank, that SGLang's attention + takes the branches the stock step copies; a reason on any rank stops every rank + (``runtime/executor.py``). """ from __future__ import annotations import importlib.util -from sglang.srt.megakernels.spec import ( - Feature, - MegakernelSpec, - Numerics, - NumericsKind, - Requirements, -) +try: + from sglang.kernels.plugin_kernels.nvidia.spec import ( + Feature, + MegakernelSpec, + Numerics, + NumericsKind, + Requirements, + ) +except ModuleNotFoundError: # SGLang before sgl-project/sglang#45389 + from sglang.srt.megakernels.spec import ( + Feature, + MegakernelSpec, + Numerics, + NumericsKind, + Requirements, + ) -from sgl_nvidia_mega.glm_dsa import switches +from sgl_nvidia_mega.glm_dsa import HF_CONFIG, MODEL_TYPE, TP_SIZE +from sgl_nvidia_mega.glm_dsa.eligibility import SUPPORTED_ROWS, VERIFY_TOKENS_PER_REQUEST +from sgl_nvidia_mega.glm_dsa.kernels.cute.hop.constants import CLUSTER, TIMEOUT_NS +from sgl_nvidia_mega.glm_dsa.plan import MAX_ROWS +from sgl_nvidia_mega.glm_dsa.runtime import SLOTS, SLOTS_ENV, parity, slot_reasons # The SGLang names this module reads (test/unit/test_contract.py pins them) SGLANG_READS = { - "srt/megakernels/spec.py": ( + "kernels/plugin_kernels/nvidia/spec.py|srt/megakernels/spec.py": ( "Feature", - "MULTI_NODE_TP", "SPECULATIVE_DECODING", + "KV_OFFLOAD", "MegakernelSpec", "Numerics", "NumericsKind", "TOLERANCE", "Requirements", + "decode_graph_backends", "config_disable_reasons", "model_disable_reasons", + "EAGLE_DRAFT_CAPTURE_CHECK", + ), + "srt/kernel_plugins/nvidia/binding.py|srt/megakernels/guards.py": ( + "deadline_s", + "deadline_s(watchdog_timeout)", + ), + "srt/kernel_plugins/nvidia/binding.py|srt/megakernels/binding.py": ( + "ForwardView", + "KvPoolView", + "DsaView", + "MegakernelBinding", + ), + "srt/model_executor/cuda_graph_config.py": ("decode", "backend", '"disabled"'), + "srt/environ.py": ( + "SGLANG_DSA_FUSE_TOPK", + "SGLANG_DSA_TOPK_BROADCAST", + "SGLANG_EXPERIMENTAL_LORA_OPTI", + "SGLANG_ENABLE_DSA_Q8KV8_BORN_FP8_Q", + "SGLANG_ENABLE_EMBED_REPLICATION", ), - "srt/models/glm4_moe.py": ("GlmMoeDsaForCausalLM", "GlmMoeDsaForCausalLMNextN"), "srt/arg_groups/fields/*.py": ( + "cuda_graph_config", + "watchdog_timeout", "kv_cache_dtype", "page_size", "attention_backend", + "decode_attention_backend", "dsa_decode_backend", "moe_runner_backend", + "moe_a2a_backend", + "init_expert_location", + "expert_distribution_recorder_mode", + "moe_dp_size", + "dp_size", "speculative_algorithm", "speculative_eagle_topk", "speculative_num_draft_tokens", "speculative_num_steps", - "cuda_graph_config", + "speculative_adaptive", + "enable_multi_layer_eagle", + "enable_cp_decode_attn_tp", + "enable_kv_cache_sharding", + "enable_layernorm_sp", + "enable_attn_tp_input_scattered", + "enable_dense_mlp_attn_tp", + "enable_shared_experts_attn_tp", + "enable_quant_communications", + "enable_fused_moe_sum_all_reduce", + "enable_dp_lm_head", + "enable_dsa_cache_layer_split", + "enforce_shared_experts_fusion", + "ep_num_redundant_experts", + "disable_cuda_graph", + "disable_decode_cuda_graph", + "speculative_use_rejection_sampling", ), } ARCHITECTURE = "GlmMoeDsaForCausalLM" -# the sglang.srt.megakernels contract this spec is written against API_VERSION = 1 -# verify rows: 1-4 requests x 4 tokens (3 draft tokens + 1) -MAX_ROWS = 16 + +# The co-resident cooperative grid the served kernels need at one CTA per SM +RESIDENT_BLOCKS = MAX_ROWS * CLUSTER + 1 + +# The hops' compiled wait bound: SGLang's wait deadline must not be shorter +HOP_WAIT_S = TIMEOUT_NS / 1e9 + +_OFF = (None, False) +SERVER_ARGS = { + "kv_cache_dtype": frozenset({"fp8_e4m3"}), + "page_size": frozenset({64}), + "attention_backend": frozenset({"dsa"}), + "decode_attention_backend": frozenset({None, "dsa"}), + "dsa_decode_backend": frozenset({"trtllm"}), + "moe_runner_backend": frozenset({"flashinfer_trtllm"}), + "moe_a2a_backend": frozenset({None, "none"}), + "init_expert_location": frozenset({None, "trivial"}), + "expert_distribution_recorder_mode": frozenset({None}), + "moe_dp_size": frozenset({None, 1}), + "dp_size": frozenset({None, 1}), + "speculative_algorithm": frozenset({"EAGLE", "NEXTN"}), + "speculative_eagle_topk": frozenset({1}), + "speculative_num_draft_tokens": frozenset({VERIFY_TOKENS_PER_REQUEST}), + "speculative_num_steps": frozenset({VERIFY_TOKENS_PER_REQUEST - 1}), + "speculative_adaptive": frozenset(_OFF), + "enable_multi_layer_eagle": frozenset(_OFF), + **dict.fromkeys( + ( + "enable_cp_decode_attn_tp", + "enable_kv_cache_sharding", + "enable_layernorm_sp", + "enable_attn_tp_input_scattered", + "enable_dense_mlp_attn_tp", + "enable_shared_experts_attn_tp", + "enable_quant_communications", + "enable_fused_moe_sum_all_reduce", + "enable_dp_lm_head", + "enable_dsa_cache_layer_split", + "enforce_shared_experts_fusion", + ), + frozenset(_OFF), + ), + **dict.fromkeys( + ("disable_cuda_graph", "disable_decode_cuda_graph"), + frozenset({*_OFF, True} if parity.eager_steps_allowed() else _OFF), + ), + "ep_num_redundant_experts": frozenset({None, 0}), + **( + {"speculative_use_rejection_sampling": frozenset(_OFF)} + if "draft" in SLOTS + else {} + ), +} + +ENV = { + "SGLANG_DSA_FUSE_TOPK": True, + "SGLANG_DSA_TOPK_BROADCAST": False, + "SGLANG_EXPERIMENTAL_LORA_OPTI": False, + "SGLANG_ENABLE_DSA_Q8KV8_BORN_FP8_Q": False, + **({"SGLANG_ENABLE_EMBED_REPLICATION": True} if "draft" in SLOTS else {}), +} + + +def decode_graph_backends() -> frozenset: + return frozenset({"full", "disabled"} if parity.eager_steps_allowed() else {"full"}) + REQUIREMENTS = Requirements( isas=frozenset({"sm_103", "sm_107"}), - tp_sizes=frozenset({8}), + tp_sizes=frozenset({TP_SIZE}), + resident_blocks=RESIDENT_BLOCKS, dtypes=frozenset({"bfloat16"}), - # the fused step is recorded only into captured FULL decode graphs - decode_graph_backends=frozenset({"full"}), - server_args={ - "kv_cache_dtype": frozenset({"fp8_e4m3"}), - "page_size": frozenset({64}), - "attention_backend": frozenset({"dsa"}), - "dsa_decode_backend": frozenset({"trtllm"}), - "moe_runner_backend": frozenset({"flashinfer_trtllm"}), - # NEXTN resolves to EAGLE; the MTP chain of 3 draft tokens - "speculative_algorithm": frozenset({"EAGLE"}), - "speculative_eagle_topk": frozenset({1}), - "speculative_num_draft_tokens": frozenset({4}), - "speculative_num_steps": frozenset({3}), - }, - hf_config={"model_type": "glm_moe_dsa"}, - # the fused draft's embedding front reads the replicated table - env={"SGLANG_ENABLE_EMBED_REPLICATION": True} if switches.fused_draft() else {}, + decode_graph_backends=decode_graph_backends(), + server_args=SERVER_ARGS, + hf_config={**HF_CONFIG, "model_type": MODEL_TYPE}, + env=ENV, ) -# The validated claim: the step's layer loop with every step on its stock body is -# bitwise SGLang's forward (the in-graph replica check). The fused kernels' own -# per-op tolerances are not stated yet. +# Per-kernel output tolerances vs. SGLang's op (or an fp64 reference). +# Each key names the test (test/kernels) that measures it. +TOLERANCES = { + # q_a / kv_a (+ indexer's key / head-gate) on the small-M GEMM + "attn.qkv_a.gemm_bound": 2**-12, + # the MLA prologue: MLA rows, index-K rows + scales and normed q_a equal SGLang's + "attn.prologue.caches.max_abs": 0.0, + # its FP8 query differs only through the q_b / W_UK GEMM order + "attn.prologue.q_fp8.mean_err_ratio": 1.05, + "attn.prologue.q_fp8.max_err_ratio": 2.0, + # the indexer query: within half an E4M3 step plus one bf16 rounding of fp64 + "attn.index_q.e4m3_steps": 0.5, + "attn.index_q.weights.max_abs": 0.0, + # the sparse MLA against fp64, relative L2 per (row, head) + "attn.sparse_mla.rel_l2": 5e-3, + # the value up-projection: one bf16 ulp plus 2^-12 x |x| @ |W| of fp64 + "attn.uv_proj.gemm_bound": 2**-12, + # the MoE routing (expert ids, bf16 weights) and routed experts equal FlashInfer's + "moe.routing.max_abs": 0.0, + "moe.routed.max_abs": 0.0, + # routed + shared expert vs SGLang's MoE + shared expert + "moe.with_shared.rel": 2**-7, + # the hops: residual, normed, o_proj GEMV, fold residual + "hop.residual.max_abs": 0.0, + "hop.normed.ulp": 2, + "hop.o_proj.gemv_bound": 2**-8, + "hop.fold.residual.max_abs": 0.0, + # the o_proj hop's router + "hop.router.logit_bound": 2**-16, + "hop.router.score_abs": 2**-11, + "hop.router.stock_key_margin": 2**-6, + "hop.router.stock_logit_abs": 2**-4, + # slot dnorm: epilogue-scaled q_a / kv_a projection and fold block sums + "dnorm.qkv_a.gemm_bound": 2**-12, + "dnorm.block_sums.max_abs": 0.0, +} +DNORM_KEYS = ("dnorm.qkv_a.gemm_bound", "dnorm.block_sums.max_abs") NUMERICS = Numerics( kind=NumericsKind.TOLERANCE, - tolerances={"loop.stock_bodies.max_abs": 0.0}, + tolerances={ + key: value + for key, value in TOLERANCES.items() + if "dnorm" in SLOTS or key not in DNORM_KEYS + }, ) +# The SGLang megakernel binding members the covered steps read +BINDING_MEMBERS = { + "MegakernelBinding": ("forward_view",), + "ForwardView": ("dsa",), + "KvPoolView": ( + "get_key_buffer", + "get_index_k_with_scale_buffer", + "index_page_size", + "set_mla_kv_buffer", + ), + "DsaView": ( + "kv_lora_rank", + "qk_nope_head_dim", + "qk_rope_head_dim", + "kv_cache_dim", + "physical_page_size", + "decode_impl", + "use_mha", + "kv_cache_dtype", + "use_fused_topk", + "topk_backend", + "dcp_size", + "index_kpool", + "hisparse", + "workspace_buffer", + "multi_ctas_kv_counter_buffer", + "indexer_metadata", + ), +} -def toolchain_reasons() -> list[str]: - reasons = [ - f"needs {what}" - for module, what in ( - ("cutlass", "CuTe DSL (nvidia-cutlass-dsl)"), - ("cuda.bindings", "the CUDA Python bindings"), - ) - if importlib.util.find_spec(module) is None - ] + +def toolchain_reasons() -> list: + reasons = [] + for module, what in ( + ("cutlass", "CuTe DSL (nvidia-cutlass-dsl)"), + ("flashinfer", "FlashInfer"), + ("cuda.bindings", "the CUDA Python bindings"), + ): + if importlib.util.find_spec(module) is None: + reasons.append(f"needs {what}") if not reasons: from sgl_nvidia_mega.glm_dsa.kernels.cute import cutlass_supported @@ -107,74 +291,91 @@ def toolchain_reasons() -> list[str]: return reasons -def _config_disable_reasons(cfg, hf_config) -> list[str]: - from sgl_nvidia_mega.glm_dsa import eligibility - - return toolchain_reasons() + eligibility.construction_reasons(hf_config, cfg) +def binding_reasons() -> list: + try: + import sglang.srt.kernel_plugins.nvidia.binding as binding + except ModuleNotFoundError: + import sglang.srt.megakernels.binding as binding + missing = [ + f"{owner}.{member}" + for owner, members in BINDING_MEMBERS.items() + for member in members + if not hasattr(getattr(binding, owner, None), member) + ] + if missing: + return [ + "this SGLang's megakernel binding lacks the DSA view members " + f"{', '.join(missing)}; update SGLang" + ] + return [] -def _model_disable_reasons(model, hf_config) -> list[str]: - from sgl_nvidia_mega.glm_dsa import eligibility +def draft_capture_reasons() -> list: + if "draft" not in SLOTS: + return [] try: - eligibility.check_construction(model) - eligibility.check_loaded(model) - except ValueError as e: - return [str(e)] - return [] + import sglang.kernels.plugin_kernels.nvidia.spec as layer + except ModuleNotFoundError: + import sglang.srt.megakernels.spec as layer + if getattr(layer, "EAGLE_DRAFT_CAPTURE_CHECK", False): + return [] + return [ + "slot draft: this SGLang's EAGLE draft graph runners do not run a draft " + "spec's capture check (spec.EAGLE_DRAFT_CAPTURE_CHECK); update SGLang, or " + f"leave slot draft out of {SLOTS_ENV}" + ] -class _Built: - """The installed target forward: covered verify steps run the fused stack.""" - draft_sampler = None +def deadline_reasons(cfg) -> list: + try: + from sglang.srt.kernel_plugins.nvidia.binding import deadline_s + except ModuleNotFoundError: + from sglang.srt.megakernels.guards import deadline_s + + dl = deadline_s(cfg.watchdog_timeout) + if dl < HOP_WAIT_S: + return [ + f"--watchdog-timeout {cfg.watchdog_timeout:g} s leaves a wait deadline of " + f"{dl:g} s, under the hops' {HOP_WAIT_S:g} s; pass at least " + f"{2 * HOP_WAIT_S:g}" + ] + return [] - def __init__(self, model, fused): - self.model, self.fused = model, fused - self.widths = tuple(sorted(fused.providers)) - def summary(self) -> str: - fused = { - rows: sorted( - k - for k, v in p.describe().items() - if not v.startswith(("AttentionDecode", "HopSteps", "MlpSteps")) - ) - for rows, p in self.fused.providers.items() - } - return "GLM-5 DSA fused verify; fused steps by rows: " + "; ".join( - f"{rows}: {', '.join(names) or 'stock'}" for rows, names in fused.items() +def _config_disable_reasons(cfg, hf_config) -> list: + reasons = toolchain_reasons() + binding_reasons() + slot_reasons() + reasons += draft_capture_reasons() + deadline_reasons(cfg) + if ( + cfg.cuda_graph_config.decode.backend == "disabled" + and not parity.eager_steps_allowed() + ): + reasons.append( + "decode CUDA graphs are disabled: the covered step runs only inside " + "captured decode graphs; enable decode CUDA graphs" ) + return reasons - def plan(self, input_ids, forward_batch, *args, **kwargs): - from sglang.srt.megakernels.spec import Step - input_embeds = kwargs.get("input_embeds", args[0] if args else None) - pp_proxy = kwargs.get("pp_proxy_tensors", args[1] if len(args) > 1 else None) - rows = int(input_ids.shape[0]) - if not self.fused.engages(rows, forward_batch, input_embeds, pp_proxy): - return None - return Step(mode="target_verify", rows=rows) +def _model_disable_reasons(model, hf_config) -> list: + from sgl_nvidia_mega.glm_dsa import eligibility - def forward(self, step, original, input_ids, positions, forward_batch, *a, **kw): - hidden_states = self.fused.run(input_ids, positions, forward_batch) - return self.model.logits_processor( - input_ids, hidden_states, self.model.lm_head, forward_batch, None - ) + return eligibility.model_construction_reasons(model, hf_config) def _build(model, hf_config, ctx): - from sgl_nvidia_mega.glm_dsa.runtime import GlmDsaFusedDecode + from sgl_nvidia_mega.glm_dsa.runtime.executor import GlmDsaFusedDecode fused = GlmDsaFusedDecode(model) fused.prepare() - return _Built(model, fused) + return fused SPEC = MegakernelSpec( - name="GLM-5 DSA", + name="GLM-5.2 / GLM-5 DSA", requires=REQUIREMENTS, - features=frozenset({Feature.SPECULATIVE_DECODING, Feature.MULTI_NODE_TP}), + features=frozenset({Feature.SPECULATIVE_DECODING, Feature.KV_OFFLOAD}), numerics=NUMERICS, max_rows=MAX_ROWS, build=_build, diff --git a/sgl_nvidia_mega/glm_dsa/switches.py b/sgl_nvidia_mega/glm_dsa/switches.py index 1520b120..b352897f 100644 --- a/sgl_nvidia_mega/glm_dsa/switches.py +++ b/sgl_nvidia_mega/glm_dsa/switches.py @@ -23,8 +23,17 @@ # the fused hops do not need multicast, but the stock step they replace then runs # NCCL + RMSNorm, so its numerics are not the ones the hops reproduce. "SGLANG_NVIDIA_MEGAKERNEL_GLM_ALLOW_NO_AR_FUSION": False, + # The hop kernels prefer the maximum shared-memory carveout, so an SM a hop CTA + # lingers on stays in the configuration the next layer's large-shared-memory + # kernels (launched early under PDL) can share. A scheduling hint: outputs are + # unchanged. + "SGLANG_NVIDIA_MEGAKERNEL_GLM_HOP_MAX_SHARED_CARVEOUT": False, } +# Not a "1" / "0" switch: N in {1, 2} caps the o_proj hop's consumer CTAs at N per SM +# (padded launch shared memory, outputs unchanged); unset or 0 leaves them unpadded. +HOP_CTAS_PER_SM = "SGLANG_NVIDIA_MEGAKERNEL_GLM_HOP_CTAS_PER_SM" + def enabled(name: str) -> bool: raw = os.environ.get(name) @@ -47,3 +56,16 @@ def fused_draft() -> bool: def allow_no_ar_fusion() -> bool: return enabled("SGLANG_NVIDIA_MEGAKERNEL_GLM_ALLOW_NO_AR_FUSION") + + +def hop_max_shared_carveout() -> bool: + return enabled("SGLANG_NVIDIA_MEGAKERNEL_GLM_HOP_MAX_SHARED_CARVEOUT") + + +def hop_ctas_per_sm() -> int: + raw = os.environ.get(HOP_CTAS_PER_SM, "").strip() + if raw in ("", "0"): + return 0 + if raw not in ("1", "2"): + raise ValueError(f"{HOP_CTAS_PER_SM}={raw!r}: expected 1 or 2 (or 0 / unset)") + return int(raw) diff --git a/test/unit/test_contract.py b/test/unit/test_contract.py index 522662bf..7e6ae7c4 100644 --- a/test/unit/test_contract.py +++ b/test/unit/test_contract.py @@ -24,6 +24,17 @@ PACKAGE = Path(sgl_nvidia_mega.__file__).resolve().parent +def _files(pattern: str) -> list: + """The files a ``SGLANG_READS`` key names: its first alternative that exists. + A key like ``"new.py|old.py"`` tries the first path; if no file matches, tries + the next, so files that moved between SGLang versions are named newest first.""" + for alternative in pattern.split("|"): + files = sorted(ROOT.glob(alternative)) + if files: + return files + return [] + + @lru_cache(maxsize=None) def _tree(path: Path) -> ast.Module: return ast.parse(path.read_text()) @@ -95,8 +106,8 @@ def test_every_sglang_import_is_declared(path): def test_sglang_provides_what_the_plugin_reads(path): missing = [] for pattern, names in _declared(path).items(): - files = sorted(ROOT.glob(pattern)) - assert files, f"{_id(path)}: no file matches {pattern}" + files = _files(pattern) + assert files, f"{_id(path)}: no file matches any alternative in {pattern!r}" provided = frozenset().union(*map(_provided, files)) missing += [f"{pattern}: {n}" for n in names if n not in provided] assert not missing, ( diff --git a/test/unit/test_glm_dsa_config.py b/test/unit/test_glm_dsa_config.py new file mode 100644 index 00000000..13a5733e --- /dev/null +++ b/test/unit/test_glm_dsa_config.py @@ -0,0 +1,185 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""GLM-5.2-NVFP4 on the GLM-5 DSA specs: the published config of +nvidia/GLM-5.2-NVFP4 has the architecture and every config value the target and +draft specs pin, so SGLang's requirement check accepts it; a config that differs in +one pinned value, or lacks it, is still refused, naming it.""" + +import contextlib +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +try: + from sglang.srt.environ import envs +except ImportError: + envs = None + +try: + from sglang.srt.kernel_plugins.nvidia import binding as sglang_runtime +except ModuleNotFoundError: # SGLang before sgl-project/sglang#45389 + from sglang.srt.megakernels import runtime as sglang_runtime + +from sgl_nvidia_mega import plugin +from sgl_nvidia_mega.glm_dsa import TP_SIZE, draft_spec, spec + +# The top-level fields of config.json in nvidia/GLM-5.2-NVFP4 at revision +# 53e0691e21895a3863a606dfd12910c69eba94ab (its git blob +# b63d4c44cdd013e928cbc37a755633e8c001626f), unchanged. Left out: the per-layer lists +# and quantization_config, which no spec pins. +GLM5_2_NVFP4 = { + "architectures": ["GlmMoeDsaForCausalLM"], + "attention_bias": False, + "attention_dropout": 0.0, + "bos_token_id": 0, + "dtype": "bfloat16", + "eos_token_id": [154820, 154827, 154829], + "ep_size": 1, + "first_k_dense_replace": 3, + "head_dim": 192, + "hidden_act": "silu", + "hidden_size": 6144, + "index_head_dim": 128, + "index_n_heads": 32, + "index_share_for_mtp_iteration": True, + "index_skip_topk_offset": 3, + "index_topk": 2048, + "index_topk_freq": 4, + "index_topk_pattern": None, + "indexer_rope_interleave": True, + "initializer_range": 0.02, + "intermediate_size": 12288, + "kv_lora_rank": 512, + "max_position_embeddings": 1048576, + "mlp_bias": False, + "model_type": "glm_moe_dsa", + "moe_intermediate_size": 2048, + "moe_layer_freq": 1, + "n_group": 1, + "n_routed_experts": 256, + "n_shared_experts": 1, + "norm_topk_prob": True, + "num_attention_heads": 64, + "num_experts": 256, + "num_experts_per_tok": 8, + "num_hidden_layers": 78, + "num_key_value_heads": 64, + "num_nextn_predict_layers": 1, + "pad_token_id": 154820, + "pretraining_tp": 1, + "q_lora_rank": 2048, + "qk_head_dim": 256, + "qk_nope_head_dim": 192, + "qk_rope_head_dim": 64, + "rms_norm_eps": 1e-05, + "rope_interleave": True, + "rope_parameters": {"rope_theta": 8000000, "rope_type": "default"}, + "routed_scaling_factor": 2.5, + "scoring_func": "sigmoid", + "tie_word_embeddings": False, + "topk_group": 1, + "topk_method": "noaux_tc", + "transformers_version": "5.11.0", + "use_cache": True, + "v_head_dim": 256, + "vocab_size": 154880, +} + +SPECS = {"target": spec.SPEC, "draft": draft_spec.SPEC} +# fields each spec pins +DRAFT_PINS = { + "hidden_size", + "num_attention_heads", + "q_lora_rank", + "kv_lora_rank", + "qk_rope_head_dim", + "qk_nope_head_dim", + "v_head_dim", + "index_n_heads", + "index_head_dim", + "index_topk", + "model_type", +} +TARGET_PINS = DRAFT_PINS | { + "n_routed_experts", + "num_experts_per_tok", + "n_shared_experts", + "n_group", + "topk_group", + "norm_topk_prob", + "scoring_func", + "topk_method", + "moe_intermediate_size", + "first_k_dense_replace", + "num_hidden_layers", +} +PINS = [(which, key) for which, s in SPECS.items() for key in s.requires.hf_config] + + +def _reasons(megakernel_spec, config: dict) -> list: + """SGLang's requirement check of the spec at load on this config.""" + graphs = SimpleNamespace(decode=SimpleNamespace(backend="full")) + with contextlib.ExitStack() as stack: + stack.enter_context( + patch.object( + sglang_runtime, + "get_parallel", + return_value=SimpleNamespace(tp_size=TP_SIZE), + ) + ) + stack.enter_context( + patch.object( + sglang_runtime, + "get_exec", + return_value=SimpleNamespace( + graph=SimpleNamespace(cuda_graph_config=graphs) + ), + ) + ) + if envs is not None: + for name, value in megakernel_spec.requires.env.items(): + stack.enter_context(getattr(envs, name).override(value)) + return sglang_runtime._requirement_reasons( + spec=megakernel_spec, hf_config=SimpleNamespace(**config) + ) + + +def _changed(value): + if isinstance(value, bool): + return not value + if isinstance(value, int): + return value + 1 + return f"{value}-changed" + + +def test_the_published_config_names_the_glm_dsa_architecture(): + rows = dict(plugin.SPECS) + assert GLM5_2_NVFP4["architectures"] == [spec.ARCHITECTURE] + assert rows[spec.ARCHITECTURE] == "sgl_nvidia_mega.glm_dsa.spec:SPEC" + assert rows[draft_spec.ARCHITECTURE] == "sgl_nvidia_mega.glm_dsa.draft_spec:SPEC" + assert GLM5_2_NVFP4["dtype"] in spec.SPEC.requires.dtypes + + +def test_the_specs_pin_these_fields(): + assert set(spec.SPEC.requires.hf_config) == TARGET_PINS + assert set(draft_spec.SPEC.requires.hf_config) == DRAFT_PINS + + +@pytest.mark.parametrize("which", SPECS) +def test_the_published_config_passes_every_pin(which): + assert _reasons(SPECS[which], GLM5_2_NVFP4) == [] + + +@pytest.mark.parametrize("which, key", PINS) +def test_a_changed_pin_is_refused(which, key): + want = SPECS[which].requires.hf_config[key] + reasons = _reasons(SPECS[which], {**GLM5_2_NVFP4, key: _changed(want)}) + assert len(reasons) == 1 and reasons[0].startswith(f"config.{key} ") + + +@pytest.mark.parametrize("which", SPECS) +def test_a_config_without_a_pinned_field_is_refused(which): + config = {k: v for k, v in GLM5_2_NVFP4.items() if k != "index_topk"} + reasons = _reasons(SPECS[which], config) + assert len(reasons) == 1 and reasons[0].startswith("config.index_topk None ") From f3ebf4e293f718ca098225548d5a58096a2112e3 Mon Sep 17 00:00:00 2001 From: shuw Date: Thu, 29 Oct 2026 11:54:23 -0500 Subject: [PATCH 5/8] [Feat] glm_dsa: serve MTP with 5 draft steps (6 verify rows a request) Port sgl-project/sgl-nvidia-mega#24's 5 / 1 / 6 support (3889557, 492aa0a, 77015e7) to the GLM-5 DSA specs, which served speculative_num_draft_tokens 4 / speculative_num_steps 3 only. - eligibility: SERVED_TOKENS_PER_REQUEST = (4, 6) and MAX_ROWS = 16; supported_rows(T) gives every whole number of requests up to 16 rows (T 4: 4 / 8 / 12 / 16, T 6: 6 / 12). The executor builds only the widths of the server's speculative_num_draft_tokens; the spec and Tier 1 refuse any other value or a pair with draft tokens != steps + 1, naming 4 / 3 and 6 / 5. - MoE: moe_split groups rows by 4 with the remainder last (6 -> (4, 2) on the two-group engine, 12 -> (4, 4, 4) on the four-group engine). - Draft: the eh_proj shard rows are the draft decode's 1-4 plus the covered widths (T 6: 1, 2, 3, 4, 6, 12). - One request at 6 rows keeps the fast forms: split MoE fold hop (SPLIT_MAX_ROWS 6), plain hops on dense layers (DENSE_MAX_ROWS 6) and the 16-warp sparse MLA up to 6 rows. Kernels unchanged. At T 4 every width, row map, eh row set and kernel form is unchanged. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- sgl_nvidia_mega/glm_dsa/attention.py | 9 ++-- sgl_nvidia_mega/glm_dsa/attention/fused.py | 9 ++-- sgl_nvidia_mega/glm_dsa/boundary/__init__.py | 4 +- sgl_nvidia_mega/glm_dsa/draft.py | 31 +++++++++--- sgl_nvidia_mega/glm_dsa/draft_spec.py | 9 ++-- sgl_nvidia_mega/glm_dsa/eligibility.py | 50 ++++++++++++++++--- sgl_nvidia_mega/glm_dsa/hops.py | 4 +- .../glm_dsa/kernels/cute/hop/api.py | 4 +- sgl_nvidia_mega/glm_dsa/moe.py | 20 ++++---- sgl_nvidia_mega/glm_dsa/moe/__init__.py | 18 +++---- sgl_nvidia_mega/glm_dsa/moe/step.py | 17 ++++++- sgl_nvidia_mega/glm_dsa/plan.py | 3 +- sgl_nvidia_mega/glm_dsa/runtime/executor.py | 11 ++-- sgl_nvidia_mega/glm_dsa/runtime_legacy.py | 4 +- sgl_nvidia_mega/glm_dsa/spec.py | 14 ++++-- 16 files changed, 143 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 40b1d75f..88207c01 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,11 @@ The package loads as an SGLang [general plugin](https://github.com/sgl-project/s | Model | GPU | Tensor parallel | Coverage | | --- | --- | --- | --- | | DeepSeek-V4.1-Flash | GB300 (SM103) | 4 | DSpark target verify steps of 1 to 384 rows (batch 1 to 64 at 6 rows a request) | -| GLM-5 DSA (NVFP4) | GB300 (SM103), Rubin (SM107) | 8 | EAGLE topk-1 target verify steps of 4 to 16 rows (batch 1 to 4 at 4 rows a request), NextN draft and draft-extend steps of 1 to 16 rows | +| GLM-5 DSA (NVFP4) | GB300 (SM103), Rubin (SM107) | 8 | EAGLE topk-1 target verify steps of whole requests up to 16 rows (batch 1 to 4 at 4 rows a request, 1 to 2 at 6), NextN draft and draft-extend steps of 1 to 16 rows | Larger verify steps, prefill and extend run SGLang's standard path. The DSpark draft step runs on the plugin's draft spec. -GLM-5 DSA needs `--kv-cache-dtype fp8_e4m3 --page-size 64 --attention-backend dsa --dsa-decode-backend trtllm --moe-runner-backend flashinfer_trtllm --speculative-algorithm EAGLE --speculative-eagle-topk 1 --speculative-num-steps 3 --speculative-num-draft-tokens 4` and `SGLANG_ENABLE_EMBED_REPLICATION=1`. Without NVLink multicast (no all-reduce fusion), set `SGLANG_NVIDIA_MEGAKERNEL_GLM_ALLOW_NO_AR_FUSION=1`. +GLM-5 DSA needs `--kv-cache-dtype fp8_e4m3 --page-size 64 --attention-backend dsa --dsa-decode-backend trtllm --moe-runner-backend flashinfer_trtllm --speculative-algorithm EAGLE --speculative-eagle-topk 1 --speculative-num-steps 3 --speculative-num-draft-tokens 4` (or `--speculative-num-steps 5 --speculative-num-draft-tokens 6`) and `SGLANG_ENABLE_EMBED_REPLICATION=1`. Without NVLink multicast (no all-reduce fusion), set `SGLANG_NVIDIA_MEGAKERNEL_GLM_ALLOW_NO_AR_FUSION=1`. ### Speculative step graph (any EAGLE v2 model) diff --git a/sgl_nvidia_mega/glm_dsa/attention.py b/sgl_nvidia_mega/glm_dsa/attention.py index 81cb4c3a..460b1e71 100644 --- a/sgl_nvidia_mega/glm_dsa/attention.py +++ b/sgl_nvidia_mega/glm_dsa/attention.py @@ -11,7 +11,7 @@ ``q_b_proj``, W_UK absorption, RoPE + FP8 query left in ``step.q_fp8``, the MLA and index-K cache rows), so ``index_k``, ``q_absorb`` and ``rope_cache`` pass through; ``index_q`` on ``indexer_q_proj``; ``sparse_mla`` on ``sparse_mla_decode`` -over the fused top-k page table (16 warps per CTA up to 4 rows, 8 above); +over the fused top-k page table (16 warps per CTA up to 6 rows, 8 above); ``uv_proj`` on ``mla_uv_proj``. The indexer's logits and top-k stay stock. With SGLANG_NVIDIA_MEGAKERNEL_GLM_SHARED_QKV_A_SMALL_M (default on) the layers without their own top-k also run ``qkv_a`` on the small-M kernel, and with @@ -92,8 +92,9 @@ def kernel_rows(rows: int) -> Optional[int]: def sparse_warps(rows: int) -> int: - """Warps per CTA of the sparse MLA form: one CTA per SM up to 4 rows, two above.""" - return 16 if rows <= 4 else 8 + """Warps per CTA of the sparse MLA form: one CTA per SM up to 6 rows (one request + of up to 6 verify rows), two above.""" + return 16 if rows <= 6 else 8 def _f32(x: float) -> float: @@ -521,7 +522,7 @@ def __init__(self, model, derived, layers=None) -> None: self.layers: Dict[int, LayerArgs] = {} # the compiled kernel forms by ``attn.api.compile_specs`` name: # "uv|prologue|index_q.{8,16}" (row capacity), "prologue_end.{8,16}" (target - # only), "sparse.{16,8}" (warps per CTA: 16 up to 4 rows, 8 above) and + # only), "sparse.{16,8}" (warps per CTA: 16 up to 6 rows, 8 above) and # "qkv_a[_shared[_dn]].8" self.fn: Dict[str, object] = {} rope_f32, index_rope_f32 = set(), set() diff --git a/sgl_nvidia_mega/glm_dsa/attention/fused.py b/sgl_nvidia_mega/glm_dsa/attention/fused.py index f57beedb..bafe12e5 100644 --- a/sgl_nvidia_mega/glm_dsa/attention/fused.py +++ b/sgl_nvidia_mega/glm_dsa/attention/fused.py @@ -11,7 +11,7 @@ ``q_b_proj``, W_UK absorption, RoPE + FP8 query left in ``step.q_fp8``, the MLA and index-K cache rows), so ``index_k``, ``q_absorb`` and ``rope_cache`` pass through; ``index_q`` on ``indexer_q_proj``; ``sparse_mla`` on ``sparse_mla_decode`` -over the fused top-k page table (16 warps per CTA up to 4 rows, 8 above); +over the fused top-k page table (16 warps per CTA up to 6 rows, 8 above); ``uv_proj`` on ``mla_uv_proj``. The indexer's logits and top-k stay stock. With SGLANG_NVIDIA_MEGAKERNEL_GLM_SHARED_QKV_A_SMALL_M (default on) the layers without their own top-k also run ``qkv_a`` on the small-M kernel, and with @@ -91,8 +91,9 @@ def kernel_rows(rows: int) -> Optional[int]: def sparse_warps(rows: int) -> int: - """Warps per CTA of the sparse MLA form: one CTA per SM up to 4 rows, two above.""" - return 16 if rows <= 4 else 8 + """Warps per CTA of the sparse MLA form: one CTA per SM up to 6 rows (one request + of up to 6 verify rows), two above.""" + return 16 if rows <= 6 else 8 def _f32(x: float) -> float: @@ -520,7 +521,7 @@ def __init__(self, model, derived, layers=None) -> None: self.layers: Dict[int, LayerArgs] = {} # the compiled kernel forms by ``attn.api.compile_specs`` name: # "uv|prologue|index_q.{8,16}" (row capacity), "prologue_end.{8,16}" (target - # only), "sparse.{16,8}" (warps per CTA: 16 up to 4 rows, 8 above) and + # only), "sparse.{16,8}" (warps per CTA: 16 up to 6 rows, 8 above) and # "qkv_a[_shared[_dn]].8" self.fn: Dict[str, object] = {} rope_f32, index_rope_f32 = set(), set() diff --git a/sgl_nvidia_mega/glm_dsa/boundary/__init__.py b/sgl_nvidia_mega/glm_dsa/boundary/__init__.py index 6d229259..2c3dbafc 100644 --- a/sgl_nvidia_mega/glm_dsa/boundary/__init__.py +++ b/sgl_nvidia_mega/glm_dsa/boundary/__init__.py @@ -16,7 +16,7 @@ only, so every rank runs the same hop sequence on its mailboxes): * MoE family served, rows <= 8: the routing ``o_proj`` hop and the fold hop on - MoE layers; dense layers use the plain hops at rows <= 4, stock above. + MoE layers; dense layers use the plain hops at rows <= 6, stock above. * MoE family served, rows 9..16: the 16-row routing hop and fold on their own mailbox; dense layers stock. * MoE family stock at this row count (declined at startup), rows <= 8 ("far @@ -73,7 +73,7 @@ def final(self, model, mlp_out, residual, step: StepContext) -> torch.Tensor: O_PROJ_IN = 2048 TOP_K = 8 # routed experts per row SMALL_ROWS = 8 # mailbox instance A (instance B: 9..16 rows) -DENSE_MAX_ROWS = 4 # plain hops on the dense layers beside the fold; stock above +DENSE_MAX_ROWS = 6 # plain hops on the dense layers beside the fold; stock above # Error sites: layer * 4 + {0: o_proj, 1: MLP output, 2: final}; sites from # EXTERNAL_SITES on are other kernels' (named with ``HopFamily.register_sites``). HOPS_PER_LAYER = 4 diff --git a/sgl_nvidia_mega/glm_dsa/draft.py b/sgl_nvidia_mega/glm_dsa/draft.py index 51be1ec9..dd45d66a 100644 --- a/sgl_nvidia_mega/glm_dsa/draft.py +++ b/sgl_nvidia_mega/glm_dsa/draft.py @@ -42,8 +42,9 @@ logger = logging.getLogger(__name__) -# draft decode: one row per request; draft extend: 4 per request -EH_ROWS = (1, 2, 3, 4, 8, 12, 16) +# draft decode: one row per request (1-4 requests); the draft extend: the covered +# widths, one row per verify token (eh_rows) +DECODE_EH_ROWS = (1, 2, 3, 4) # the draft-extend graphs select one row per request before the lm_head, so both consumers see B <= 4 rows TOP1_DECODE_ROWS = 8 TOP1_EXTEND_ROWS = 8 @@ -72,6 +73,18 @@ ONE_HOT_FILL = -torch.finfo(torch.float32).max +def eh_rows(widths) -> tuple: + """The eh_proj shard's row counts: the draft decode's (``DECODE_EH_ROWS``) and the + draft extend's at the covered widths (4 rows a request: 1, 2, 3, 4, 8, 12, 16; 6: + 1, 2, 3, 4, 6, 12).""" + return tuple(sorted({*DECODE_EH_ROWS, *widths})) + + +def served_eh_rows() -> tuple: + """``eh_rows`` at the running server's draft length.""" + return eh_rows(eligibility.supported_rows(eligibility.tokens_per_request())) + + class FusedDraft: """Kernels, derived weights and top-1 mailboxes of the NextN draft layer.""" @@ -87,6 +100,7 @@ def __init__(self) -> None: self._nextn = None self._attn_family = None # the draft layer's attention (draft_attention) self._attn_agreed = {} # (mode, rows) -> every rank serves the kernels there + self.eh_rows = DECODE_EH_ROWS def arm(self, nextn) -> None: """After the target's embedding / lm_head are shared: derive, compile, rendezvous.""" @@ -110,12 +124,13 @@ def arm(self, nextn) -> None: f"GLM-DSA fused draft: eh_proj {tuple(w.shape)} {w.dtype} is not the GLM-5 shape" ) self.eh_weight = w[self.col_off : self.col_off + self.n_local].contiguous() + self.eh_rows = served_eh_rows() specs = compile_specs( - eh_rows=EH_ROWS, + eh_rows=self.eh_rows, top1_rows=(TOP1_DECODE_ROWS, TOP1_EXTEND_ROWS), eh_norm=EH_NORM_OPTIONS, ) - for rows in EH_ROWS: + for rows in self.eh_rows: spec = specs[f"eh_proj_shard_{rows}"] self._eh[rows] = (spec[0], _compile(*spec)) self._front = _compile(*specs["embed_eh_norm"]) @@ -141,7 +156,7 @@ def arm(self, nextn) -> None: logger.info( "GLM-DSA fused draft armed: eh_proj shard rows %s, top-1 rows <= %d / %d; derived eh_proj shard %.1f MB; " "hops on the target's mailbox: %s; attention kernels: %s.", - EH_ROWS, + self.eh_rows, TOP1_DECODE_ROWS, TOP1_EXTEND_ROWS, self.eh_weight.numel() * 2 / 1e6, @@ -155,7 +170,7 @@ def _arm_attention(self, nextn) -> None: family, why = None, None try: family = draft_attention.DraftAttentionGlue(nextn) - family.compile(EH_ROWS) + family.compile(self.eh_rows) except ValueError as e: family, why = None, str(e) whys = runtime.tp_gather([] if why is None else [why]) @@ -183,7 +198,9 @@ def _arm_hops(self, dev) -> None: # the eh gather is the family's plain all-reduce unit (its own cache key) hops.compiled["allreduce"] = _compile(spec.obj, spec.constexprs, spec.args) # static: only this rank's columns are ever written, the others stay +0 - self._pad = torch.zeros(max(EH_ROWS), HIDDEN, dtype=torch.bfloat16, device=dev) + self._pad = torch.zeros( + max(self.eh_rows), HIDDEN, dtype=torch.bfloat16, device=dev + ) self._hops = hops # the draft's waits write the target's record: one record, one reporter per process self.error_record = hops.err diff --git a/sgl_nvidia_mega/glm_dsa/draft_spec.py b/sgl_nvidia_mega/glm_dsa/draft_spec.py index cab031fb..a1462a8d 100644 --- a/sgl_nvidia_mega/glm_dsa/draft_spec.py +++ b/sgl_nvidia_mega/glm_dsa/draft_spec.py @@ -34,9 +34,6 @@ "srt/models/deepseek_v2.py": ("set_embed_and_head", "logits_processor", "lm_head"), } -# draft decode: one row per request; draft extend: 4 rows per request -DRAFT_ROWS = (1, 2, 3, 4, 8, 12, 16) - # The draft top-1 is the stock argmax of the same logits (the distributed argmax # keeps its NaN and tie rules); the embedding front is bitwise fused_eh_norm. NUMERICS = Numerics( @@ -49,12 +46,13 @@ class _BuiltDraft: """The installed draft forward: armed steps run the fused draft layer.""" draft_sampler = None - widths = DRAFT_ROWS def __init__(self, model): from sgl_nvidia_mega.glm_dsa import draft self.model = model + # draft decode: one row per request; draft extend: one per verify token + self.widths = draft.served_eh_rows() self.fused = draft.create_fused_draft(model) model.logits_processor = draft.make_draft_logits_processor( model.config, self.fused @@ -120,7 +118,8 @@ def _build(model, hf_config, ctx): features=frozenset({Feature.SPECULATIVE_DECODING, Feature.MULTI_NODE_TP}), numerics=NUMERICS, # 0 keeps SGLang's draft forward (no step covered, nothing installed) - max_rows=max(DRAFT_ROWS) if switches.fused_draft() else 0, + # the draft extend at the widest covered width (draft.served_eh_rows) + max_rows=target.MAX_ROWS if switches.fused_draft() else 0, build=_build, api_version=target.API_VERSION, ) diff --git a/sgl_nvidia_mega/glm_dsa/eligibility.py b/sgl_nvidia_mega/glm_dsa/eligibility.py index 79d56269..4f97b593 100644 --- a/sgl_nvidia_mega/glm_dsa/eligibility.py +++ b/sgl_nvidia_mega/glm_dsa/eligibility.py @@ -18,9 +18,11 @@ import torch -# Verify rows per forward: 1-4 requests x 4 tokens (3 draft tokens + 1). -VERIFY_TOKENS_PER_REQUEST = 4 -SUPPORTED_ROWS = (4, 8, 12, 16) +# Verify rows a request: MTP / EAGLE with top-1 drafts verifies speculative_num_steps +# draft tokens + 1 (speculative_num_draft_tokens). The values served: 3 and 5 steps. +SERVED_TOKENS_PER_REQUEST = (4, 6) +# The widest covered step (the kernels' row cap) +MAX_ROWS = 16 TP_SIZE = 8 HIDDEN = 6144 _SM_ACCEPTED = ((10, 3), (10, 7)) @@ -28,6 +30,38 @@ SWITCH = "--enable-nvidia-megakernel (GLM-5 DSA)" +def supported_rows(tokens_per_request: int) -> tuple: + """The covered widths of verify steps of ``tokens_per_request`` rows a request: + every whole number of requests up to ``MAX_ROWS`` rows (4 rows: 4 / 8 / 12 / 16; + 6 rows: 6 / 12); none for a value that is not served. Steps with more requests run + SGLang's path.""" + if tokens_per_request not in SERVED_TOKENS_PER_REQUEST: + return () + return tuple(range(tokens_per_request, MAX_ROWS + 1, tokens_per_request)) + + +def tokens_per_request(server_args=None) -> int: + """Verify rows a request of the running server (speculative_num_draft_tokens).""" + if server_args is None: + from sglang.srt.runtime_context import get_server_args + + server_args = get_server_args() + return int(getattr(server_args, "speculative_num_draft_tokens", None) or 0) + + +def draft_length_reason(server_args) -> List[str]: + """Why the server's draft length is not served (empty if it is): top-1 drafts + verify speculative_num_steps + 1 tokens, one of ``SERVED_TOKENS_PER_REQUEST``.""" + tokens = getattr(server_args, "speculative_num_draft_tokens", None) + steps = getattr(server_args, "speculative_num_steps", None) + if tokens in SERVED_TOKENS_PER_REQUEST and steps == tokens - 1: + return [] + return [ + "requires --speculative-num-draft-tokens / --speculative-num-steps " + + " or ".join(f"{t} / {t - 1}" for t in SERVED_TOKENS_PER_REQUEST) + + f" (got {tokens!r} / {steps!r})" + ] + # GLM-5 DSA checkpoint shapes the path is built for. _EXPECTED_SHAPES = { "hidden_size": HIDDEN, @@ -68,8 +102,6 @@ } _EAGLE_VALUES = { "speculative_eagle_topk": (1,), - "speculative_num_draft_tokens": (VERIFY_TOKENS_PER_REQUEST,), - "speculative_num_steps": (VERIFY_TOKENS_PER_REQUEST - 1,), } # Server arguments that must be off: features, layouts and communication # variants that change the verify step's stock op sequence. @@ -158,7 +190,8 @@ def construction_reasons(config, server_args, *, device_capability=None) -> List f"config.{key} = {getattr(config, key, None)!r}, expected {want!r}" ) reasons += _mismatches(server_args, _VALUES, _OFF) - # Speculative decoding: EAGLE / MTP chain, 3 draft tokens (NEXTN resolves to EAGLE). + # Speculative decoding: EAGLE / MTP chain, 3 or 5 draft steps (NEXTN resolves to + # EAGLE). algo = getattr(server_args, "speculative_algorithm", None) if algo is None: reasons.append( @@ -171,6 +204,7 @@ def construction_reasons(config, server_args, *, device_capability=None) -> List ) else: reasons += _mismatches(server_args, _EAGLE_VALUES, _EAGLE_OFF) + reasons += draft_length_reason(server_args) # The attention steps implement the bf16 absorbed query and the fused DSA # top-k page table (providers.AttentionDecode). if envs.SGLANG_EXPERIMENTAL_LORA_OPTI.get(): @@ -273,7 +307,7 @@ def check_loaded(model) -> None: # boundaries use at verify sizes. if uses_cutedsl_ar_fusion(): reasons.append("the CuTe DSL all-reduce fusion backend is not supported") - elif not _ar_fusion_ok(flashinfer_ar_fusion_applies, max(SUPPORTED_ROWS)): + elif not _ar_fusion_ok(flashinfer_ar_fusion_applies, MAX_ROWS): reasons.append( "requires the FlashInfer all-reduce + RMSNorm fusion at verify sizes " "(--flashinfer-allreduce-fusion-backend not disabled; without NVLink " @@ -394,7 +428,7 @@ def step_in_graph_runner( return ( forward_batch.forward_mode.is_target_verify() - and rows in SUPPORTED_ROWS + and rows in supported_rows(tokens_per_request()) and input_embeds is None and pp_proxy_tensors is None and not getattr(forward_batch, "can_run_tbo", False) diff --git a/sgl_nvidia_mega/glm_dsa/hops.py b/sgl_nvidia_mega/glm_dsa/hops.py index 4284f3ca..b1581253 100644 --- a/sgl_nvidia_mega/glm_dsa/hops.py +++ b/sgl_nvidia_mega/glm_dsa/hops.py @@ -15,7 +15,7 @@ only, so every rank runs the same hop sequence on its mailboxes): * MoE family served, rows <= 8: the routing ``o_proj`` hop and the fold hop on - MoE layers; dense layers use the plain hops at rows <= 4, stock above. + MoE layers; dense layers use the plain hops at rows <= 6, stock above. * MoE family served, rows 9..16: the 16-row routing hop and fold on their own mailbox; dense layers stock. * MoE family stock at this row count (declined at startup), rows <= 8 ("far @@ -46,7 +46,7 @@ O_PROJ_IN = 2048 TOP_K = 8 # routed experts per row SMALL_ROWS = 8 # mailbox instance A (instance B: 9..16 rows) -DENSE_MAX_ROWS = 4 # plain hops on the dense layers beside the fold; stock above +DENSE_MAX_ROWS = 6 # plain hops on the dense layers beside the fold; stock above # Error sites: layer * 4 + {0: o_proj, 1: MLP output, 2: final}; sites from # EXTERNAL_SITES on are other kernels' (named with ``HopFamily.register_sites``). HOPS_PER_LAYER = 4 diff --git a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/api.py b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/api.py index 64caef3a..e1e9e9cd 100644 --- a/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/api.py +++ b/sgl_nvidia_mega/glm_dsa/kernels/cute/hop/api.py @@ -39,8 +39,8 @@ from .transport import ErrorRecord, HopMailbox MIN_PTX_ISA = (9, 2) -# The split fold serves 1..4 rows (the plain fold above): the rows it is gated at. -SPLIT_MAX_ROWS = 4 +# The split fold serves 1..6 rows (the plain fold above): the rows it is gated at. +SPLIT_MAX_ROWS = 6 MIN_PTX_COMPILER = (13, 2) diff --git a/sgl_nvidia_mega/glm_dsa/moe.py b/sgl_nvidia_mega/glm_dsa/moe.py index 6b54fd05..31057295 100644 --- a/sgl_nvidia_mega/glm_dsa/moe.py +++ b/sgl_nvidia_mega/glm_dsa/moe.py @@ -29,11 +29,8 @@ import torch -from sgl_nvidia_mega.glm_dsa.eligibility import ( - HIDDEN, - SUPPORTED_ROWS, - VERIFY_TOKENS_PER_REQUEST, -) +from sgl_nvidia_mega.glm_dsa.eligibility import HIDDEN, MAX_ROWS +from sgl_nvidia_mega.glm_dsa.moe.step import moe_split from sgl_nvidia_mega.glm_dsa.providers import ( MlpSteps, compile_cached, @@ -151,7 +148,7 @@ class MoeSteps(MlpSteps): def __init__(self, glue: MoeGlue, rows: int) -> None: self.glue, self.rows = glue, rows - self.split = (VERIFY_TOKENS_PER_REQUEST,) * (rows // VERIFY_TOKENS_PER_REQUEST) + self.split = moe_split(rows) def forward(self, layer, x, buffer: Optional[torch.Tensor], step): glue = self.glue @@ -202,10 +199,7 @@ def __init__(self, api, layers: Dict[int, object], specs) -> None: def compile(self, rows: Tuple[int, ...]) -> None: import cutlass.cute as cute - per = VERIFY_TOKENS_PER_REQUEST - modes = { - self.api.mode_for((per,) * (m // per)) for m in rows if m in SUPPORTED_ROWS - } + modes = {self.api.mode_for(moe_split(m)) for m in rows if 1 <= m <= MAX_ROWS} for name in sorted({"route", "finalize"} | modes): kernel, args = self.specs[name] self.fn[name] = compile_cached( @@ -249,7 +243,11 @@ def route(self, logits, bias, scale, ids, weights) -> None: ) def for_rows(self, m: int) -> Optional[MoeSteps]: - if m not in SUPPORTED_ROWS or self.engine is None: + if ( + self.engine is None + or not 1 <= m <= MAX_ROWS + or self.api.mode_for(moe_split(m)) not in self.fn + ): return None return MoeSteps(self, m) diff --git a/sgl_nvidia_mega/glm_dsa/moe/__init__.py b/sgl_nvidia_mega/glm_dsa/moe/__init__.py index 11c0830a..02494b99 100644 --- a/sgl_nvidia_mega/glm_dsa/moe/__init__.py +++ b/sgl_nvidia_mega/glm_dsa/moe/__init__.py @@ -29,14 +29,11 @@ import torch -from sgl_nvidia_mega.glm_dsa.eligibility import ( - HIDDEN, - SUPPORTED_ROWS, - VERIFY_TOKENS_PER_REQUEST, -) +from sgl_nvidia_mega.glm_dsa.eligibility import HIDDEN, MAX_ROWS from sgl_nvidia_mega.glm_dsa.runtime.compile import compile_cached, cu_stream from sgl_nvidia_mega.glm_dsa.moe.step import ( MlpSteps, + moe_split, ) logger = logging.getLogger(__name__) @@ -63,10 +60,7 @@ def __init__(self, api, layers: Dict[int, object], specs) -> None: def compile(self, rows: Tuple[int, ...]) -> None: import cutlass.cute as cute - per = VERIFY_TOKENS_PER_REQUEST - modes = { - self.api.mode_for((per,) * (m // per)) for m in rows if m in SUPPORTED_ROWS - } + modes = {self.api.mode_for(moe_split(m)) for m in rows if 1 <= m <= MAX_ROWS} for name in sorted({"route", "finalize"} | modes): kernel, args = self.specs[name] self.fn[name] = compile_cached( @@ -110,7 +104,11 @@ def route(self, logits, bias, scale, ids, weights) -> None: ) def for_rows(self, m: int) -> Optional[MoeSteps]: - if m not in SUPPORTED_ROWS or self.engine is None: + if ( + self.engine is None + or not 1 <= m <= MAX_ROWS + or self.api.mode_for(moe_split(m)) not in self.fn + ): return None return MoeSteps(self, m) diff --git a/sgl_nvidia_mega/glm_dsa/moe/step.py b/sgl_nvidia_mega/glm_dsa/moe/step.py index 3d4010f1..081d724e 100644 --- a/sgl_nvidia_mega/glm_dsa/moe/step.py +++ b/sgl_nvidia_mega/glm_dsa/moe/step.py @@ -12,6 +12,21 @@ from sgl_nvidia_mega.glm_dsa.attention.step import StepContext +# The engine's row groups: at most 4 groups of at most 4 rows (one group: g1, two: g2, +# three or four: the four-group engine) +GROUP_ROWS = 4 +MAX_GROUPS = 4 + + +def moe_split(rows: int) -> tuple: + """The engine's row groups for ``rows`` verify rows: groups of 4 rows, the + remainder last (4 -> (4,), 6 -> (4, 2), 12 -> (4, 4, 4)). The engine's groups are + token sets over one expert list, so they need not follow the request boundaries.""" + split = (GROUP_ROWS,) * (rows // GROUP_ROWS) + ( + (rows % GROUP_ROWS,) if rows % GROUP_ROWS else () + ) + assert sum(split) == rows and len(split) <= MAX_GROUPS, (rows, split) + return split class MlpSteps: """Stock dense MLP / MoE of a layer, leaving the output's sum to the next step.""" @@ -141,7 +156,7 @@ class MoeSteps(MlpSteps): def __init__(self, glue: MoeGlue, rows: int) -> None: self.glue, self.rows = glue, rows - self.split = (VERIFY_TOKENS_PER_REQUEST,) * (rows // VERIFY_TOKENS_PER_REQUEST) + self.split = moe_split(rows) def forward(self, layer, x, buffer: Optional[torch.Tensor], step): glue = self.glue diff --git a/sgl_nvidia_mega/glm_dsa/plan.py b/sgl_nvidia_mega/glm_dsa/plan.py index 44eb0d89..faf97508 100644 --- a/sgl_nvidia_mega/glm_dsa/plan.py +++ b/sgl_nvidia_mega/glm_dsa/plan.py @@ -12,9 +12,8 @@ from typing import Any, Dict, Tuple -from sgl_nvidia_mega.glm_dsa.eligibility import SUPPORTED_ROWS, VERIFY_TOKENS_PER_REQUEST +from sgl_nvidia_mega.glm_dsa.eligibility import MAX_ROWS # noqa: F401 -MAX_ROWS = max(SUPPORTED_ROWS) # Steps of each group in call order (for logging, rank agreement, tests). STEPS = { diff --git a/sgl_nvidia_mega/glm_dsa/runtime/executor.py b/sgl_nvidia_mega/glm_dsa/runtime/executor.py index ac4ed888..299bd9d6 100644 --- a/sgl_nvidia_mega/glm_dsa/runtime/executor.py +++ b/sgl_nvidia_mega/glm_dsa/runtime/executor.py @@ -98,9 +98,14 @@ def prepare(self) -> None: from sgl_nvidia_mega.glm_dsa.plan import build_providers eligibility.check_loaded(self.model) - self.providers, self.families = build_providers( - self.model, eligibility.SUPPORTED_ROWS, self.derived - ) + tokens = eligibility.tokens_per_request() + rows = eligibility.supported_rows(tokens) + if not rows: + raise RuntimeError( + f"GLM DSA fused decode: {tokens} verify rows a request is not served " + f"(served: {list(eligibility.SERVED_TOKENS_PER_REQUEST)})" + ) + self.providers, self.families = build_providers(self.model, rows, self.derived) rank = runtime_context.get_parallel().tp_rank self._agree_on_composition(rank) self.derived.log(rank) diff --git a/sgl_nvidia_mega/glm_dsa/runtime_legacy.py b/sgl_nvidia_mega/glm_dsa/runtime_legacy.py index b3f710df..031c771e 100644 --- a/sgl_nvidia_mega/glm_dsa/runtime_legacy.py +++ b/sgl_nvidia_mega/glm_dsa/runtime_legacy.py @@ -113,7 +113,9 @@ def prepare(self) -> None: global TARGET eligibility.check_loaded(self.model) self.providers, self.families = build_providers( - self.model, eligibility.SUPPORTED_ROWS, self.derived + self.model, + eligibility.supported_rows(eligibility.tokens_per_request()), + self.derived, ) rank = runtime_context.get_parallel().tp_rank self._agree_on_composition(rank) diff --git a/sgl_nvidia_mega/glm_dsa/spec.py b/sgl_nvidia_mega/glm_dsa/spec.py index 550c142b..58670ee7 100644 --- a/sgl_nvidia_mega/glm_dsa/spec.py +++ b/sgl_nvidia_mega/glm_dsa/spec.py @@ -39,9 +39,12 @@ ) from sgl_nvidia_mega.glm_dsa import HF_CONFIG, MODEL_TYPE, TP_SIZE -from sgl_nvidia_mega.glm_dsa.eligibility import SUPPORTED_ROWS, VERIFY_TOKENS_PER_REQUEST +from sgl_nvidia_mega.glm_dsa.eligibility import ( + MAX_ROWS, + SERVED_TOKENS_PER_REQUEST, + draft_length_reason, +) from sgl_nvidia_mega.glm_dsa.kernels.cute.hop.constants import CLUSTER, TIMEOUT_NS -from sgl_nvidia_mega.glm_dsa.plan import MAX_ROWS from sgl_nvidia_mega.glm_dsa.runtime import SLOTS, SLOTS_ENV, parity, slot_reasons # The SGLang names this module reads (test/unit/test_contract.py pins them) @@ -140,8 +143,10 @@ "dp_size": frozenset({None, 1}), "speculative_algorithm": frozenset({"EAGLE", "NEXTN"}), "speculative_eagle_topk": frozenset({1}), - "speculative_num_draft_tokens": frozenset({VERIFY_TOKENS_PER_REQUEST}), - "speculative_num_steps": frozenset({VERIFY_TOKENS_PER_REQUEST - 1}), + # top-1 drafts of 3 or 5 steps: verify steps of 4 or 6 rows a request (the pair + # is checked in _config_disable_reasons) + "speculative_num_draft_tokens": frozenset(SERVED_TOKENS_PER_REQUEST), + "speculative_num_steps": frozenset(t - 1 for t in SERVED_TOKENS_PER_REQUEST), "speculative_adaptive": frozenset(_OFF), "enable_multi_layer_eagle": frozenset(_OFF), **dict.fromkeys( @@ -347,6 +352,7 @@ def deadline_reasons(cfg) -> list: def _config_disable_reasons(cfg, hf_config) -> list: reasons = toolchain_reasons() + binding_reasons() + slot_reasons() reasons += draft_capture_reasons() + deadline_reasons(cfg) + reasons += draft_length_reason(cfg) if ( cfg.cuda_graph_config.decode.backend == "disabled" and not parity.eager_steps_allowed() From a751ae370b1497fd6058bb1c2255a91426ac3014 Mon Sep 17 00:00:00 2001 From: shuw Date: Tue, 3 Nov 2026 17:18:11 -0600 Subject: [PATCH 6/8] [Fix] glm_dsa: run on released SGLang (megakernel API 2, multi-node TP8) - spec, draft_spec: API_VERSION 2; the built objects offer no EAGLE entries or graph variants (draft_loop, draft_extend, begin_draft_round, graph_variants are None). Restore the target's _Built (widths, summary, plan, forward), which the package split had dropped, so build returns a BuiltMegakernel again. - spec: accept Feature.MULTI_NODE_TP, so TP8 can span the two 4-GPU trays of an NVL72 rack, as the draft spec already does. - spec: drop the DsaView binding-member and EAGLE_DRAFT_CAPTURE_CHECK gates, which only an unmerged SGLang (sgl-project/sglang#46210) passes; the covered step reads the DSA backend through get_attn_backend() and never through those views. - attention: build the AttentionGlue directly (it has no build classmethod). - draft_spec: import the megakernel interface from its new location first. - glm_dsa, spec_step: declare every SGLang import in SGLANG_READS (the stricter contract test); import modules as modules (import a.b.c as c); name reads by the file that defines them. Co-Authored-By: Claude Opus 5.5 --- sgl_nvidia_mega/glm_dsa/attention.py | 8 + sgl_nvidia_mega/glm_dsa/attention/__init__.py | 2 +- sgl_nvidia_mega/glm_dsa/attention/fused.py | 8 + sgl_nvidia_mega/glm_dsa/attention/step.py | 18 +++ sgl_nvidia_mega/glm_dsa/boundary/__init__.py | 6 + sgl_nvidia_mega/glm_dsa/draft.py | 19 ++- sgl_nvidia_mega/glm_dsa/draft_attention.py | 13 +- sgl_nvidia_mega/glm_dsa/draft_spec.py | 38 +++-- sgl_nvidia_mega/glm_dsa/eligibility.py | 22 +++ sgl_nvidia_mega/glm_dsa/hops.py | 5 + sgl_nvidia_mega/glm_dsa/moe.py | 7 +- sgl_nvidia_mega/glm_dsa/moe/__init__.py | 5 + sgl_nvidia_mega/glm_dsa/moe/step.py | 9 +- sgl_nvidia_mega/glm_dsa/providers.py | 23 +++ sgl_nvidia_mega/glm_dsa/runtime/__init__.py | 6 - sgl_nvidia_mega/glm_dsa/runtime/compile.py | 5 + sgl_nvidia_mega/glm_dsa/runtime/executor.py | 13 +- sgl_nvidia_mega/glm_dsa/runtime_legacy.py | 11 +- sgl_nvidia_mega/glm_dsa/spec.py | 147 ++++++++---------- .../spec_step/child_graph_capture.py | 8 + sgl_nvidia_mega/spec_step/hooks.py | 17 +- sgl_nvidia_mega/spec_step/lifted.py | 18 +++ sgl_nvidia_mega/spec_step/ops.py | 2 +- sgl_nvidia_mega/spec_step/runner.py | 51 +++++- 24 files changed, 343 insertions(+), 118 deletions(-) diff --git a/sgl_nvidia_mega/glm_dsa/attention.py b/sgl_nvidia_mega/glm_dsa/attention.py index 460b1e71..0a785a74 100644 --- a/sgl_nvidia_mega/glm_dsa/attention.py +++ b/sgl_nvidia_mega/glm_dsa/attention.py @@ -39,6 +39,14 @@ cu_stream, ) +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/model_executor/forward_context.py": ("get_attn_backend",), + "srt/layers/layernorm.py": ("LayerNorm",), + "kernels/ops/kvcache/dsa_fused_store_index_cache.py": ("can_use_dsa_fused_store",), + "kernels/ops/quantization/fp8_kernel.py": ("is_fp8_fnuz",), +} + # One tensor-parallel rank (TP 8). HEADS, Q_LORA, KV_LORA, ROPE, NOPE, VALUE_DIM = 8, 2048, 512, 64, 192, 256 INDEX_HEADS, INDEX_DIM, TOPK, INDEX_PAGE = 32, 128, 2048, 64 diff --git a/sgl_nvidia_mega/glm_dsa/attention/__init__.py b/sgl_nvidia_mega/glm_dsa/attention/__init__.py index 0d23f5c2..6cc4c3b0 100644 --- a/sgl_nvidia_mega/glm_dsa/attention/__init__.py +++ b/sgl_nvidia_mega/glm_dsa/attention/__init__.py @@ -19,4 +19,4 @@ def build(model, derived): if "attention" not in SLOTS: return None - return AttentionGlue.build(model, derived) + return AttentionGlue(model, derived) diff --git a/sgl_nvidia_mega/glm_dsa/attention/fused.py b/sgl_nvidia_mega/glm_dsa/attention/fused.py index bafe12e5..b5be7e98 100644 --- a/sgl_nvidia_mega/glm_dsa/attention/fused.py +++ b/sgl_nvidia_mega/glm_dsa/attention/fused.py @@ -38,6 +38,14 @@ ) from sgl_nvidia_mega.glm_dsa.runtime.compile import compile_cached, cu_stream +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/model_executor/forward_context.py": ("get_attn_backend",), + "srt/layers/layernorm.py": ("LayerNorm",), + "kernels/ops/kvcache/dsa_fused_store_index_cache.py": ("can_use_dsa_fused_store",), + "kernels/ops/quantization/fp8_kernel.py": ("is_fp8_fnuz",), +} + # One tensor-parallel rank (TP 8). HEADS, Q_LORA, KV_LORA, ROPE, NOPE, VALUE_DIM = 8, 2048, 512, 64, 192, 256 INDEX_HEADS, INDEX_DIM, TOPK, INDEX_PAGE = 32, 128, 2048, 64 diff --git a/sgl_nvidia_mega/glm_dsa/attention/step.py b/sgl_nvidia_mega/glm_dsa/attention/step.py index 7b6a8f66..50a54db4 100644 --- a/sgl_nvidia_mega/glm_dsa/attention/step.py +++ b/sgl_nvidia_mega/glm_dsa/attention/step.py @@ -11,6 +11,24 @@ import torch +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "kernels/ops/quantization/act_quant_triton.py": ("act_quant",), + "kernels/ops/attention/dsa/indexer_q.py": ("fused_q_indexer_rope_first_quant",), + "srt/layers/layer_boundary/__init__.py": ("get_attn_tp_context",), + "srt/layers/layer_boundary/adapters/attention.py": ("AttentionInputs",), + "srt/model_executor/runner/__init__.py": ("get_is_capture_mode",), + "srt/layers/attention/dsa/dsa_indexer.py": ("_broadcast_indexer_topk_from_rank0",), + "srt/model_executor/forward_context.py": ("get_attn_backend",), + "srt/state_capturer/indexer_topk.py": ("maybe_capture_indexer_topk",), + "kernels/ops/attention/utils.py": ("mla_quantize_and_rope_for_fp8",), + "srt/environ.py": ("envs",), + "srt/layers/attention/dsa_backend.py": ( + "_restore_trtllm_decode_dp_padding", + "_trim_trtllm_decode_dp_padding", + ), +} + logger = logging.getLogger(__name__) class StepContext: diff --git a/sgl_nvidia_mega/glm_dsa/boundary/__init__.py b/sgl_nvidia_mega/glm_dsa/boundary/__init__.py index 2c3dbafc..a24c0983 100644 --- a/sgl_nvidia_mega/glm_dsa/boundary/__init__.py +++ b/sgl_nvidia_mega/glm_dsa/boundary/__init__.py @@ -39,6 +39,12 @@ from sgl_nvidia_mega.glm_dsa.attention.step import StepContext from sgl_nvidia_mega.glm_dsa.runtime.compile import compile_cached, cu_stream +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/layers/moe/utils.py": ("sum_post_experts_output",), + "srt/distributed/parallel_state.py": ("get_tp_group",), +} + logger = logging.getLogger(__name__) diff --git a/sgl_nvidia_mega/glm_dsa/draft.py b/sgl_nvidia_mega/glm_dsa/draft.py index dd45d66a..101addd1 100644 --- a/sgl_nvidia_mega/glm_dsa/draft.py +++ b/sgl_nvidia_mega/glm_dsa/draft.py @@ -37,6 +37,23 @@ from sgl_nvidia_mega.glm_dsa.eligibility import HIDDEN, TP_SIZE from sgl_nvidia_mega.glm_dsa.runtime.compile import compile_cached, cu_stream +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/distributed/communication_op.py": ("tensor_model_parallel_all_gather",), + "srt/distributed/parallel_state.py": ("get_tp_group",), + "srt/model_executor/forward_batch_info.py": ("ForwardBatch",), + "srt/layers/logits_processor.py": ("LogitsProcessor",), + "srt/environ.py": ("envs",), + "srt/runtime_context.py": ( + "get_parallel", + "get_spec", + ), + "kernels/ops/layernorm/fused_eh_norm.py": ("fused_eh_norm",), + "srt/eplb/expert_distribution.py": ("get_global_expert_distribution_recorder",), + "srt/layers/attention/index_topk_share.py": ("IndexTopKShareState",), + "srt/utils/common.py": ("BumpAllocator",), +} + if TYPE_CHECKING: from sglang.srt.model_executor.forward_batch_info import ForwardBatch @@ -369,7 +386,7 @@ def forward_steps(self, model, input_ids, positions, forward_batch, attention=No get_global_expert_distribution_recorder, ) from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState - from sglang.srt.layers.layer_boundary.residual import batch as residual_batch + import sglang.srt.layers.layer_boundary.residual.batch as residual_batch from sglang.srt.utils import BumpAllocator # the zero fill stays the plain launch right before the front kernel (its ids / rows are read after its diff --git a/sgl_nvidia_mega/glm_dsa/draft_attention.py b/sgl_nvidia_mega/glm_dsa/draft_attention.py index 0b0ca37b..56e1c2a1 100644 --- a/sgl_nvidia_mega/glm_dsa/draft_attention.py +++ b/sgl_nvidia_mega/glm_dsa/draft_attention.py @@ -37,6 +37,17 @@ from sgl_nvidia_mega.glm_dsa.attention.step import AttentionDecode, StepContext from sgl_nvidia_mega.glm_dsa.runtime.executor import DerivedWeights +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/model_executor/forward_context.py": ("get_attn_backend",), + "srt/model_executor/runner_utils/capture_mode.py": ( + "get_capture_attention_variant", + ), + "srt/layers/layer_boundary/__init__.py": ("get_attn_tp_context",), + "srt/models/deepseek_v2.py": ("DeepseekV2MoE",), + "srt/layers/moe/moe_runner/base.py": ("moe_output_buffer_ctx",), +} + @dataclass class DraftStepContext(StepContext): @@ -156,7 +167,7 @@ def decoder_layer_forward( ``o_proj`` (``forward_absorb_core`` ends with ``self.o_proj``). Returns ``(hidden_states, topk_indices)`` as the stock layer does.""" from sglang.srt.layers.layer_boundary import get_attn_tp_context - from sglang.srt.layers.layer_boundary.residual import access as residual_access + import sglang.srt.layers.layer_boundary.residual.access as residual_access from sglang.srt.models.deepseek_v2 import DeepseekV2MoE hidden_states_orig = residual_access.buffer(hidden_states) diff --git a/sgl_nvidia_mega/glm_dsa/draft_spec.py b/sgl_nvidia_mega/glm_dsa/draft_spec.py index a1462a8d..cfb704e7 100644 --- a/sgl_nvidia_mega/glm_dsa/draft_spec.py +++ b/sgl_nvidia_mega/glm_dsa/draft_spec.py @@ -10,20 +10,31 @@ from __future__ import annotations -from sglang.srt.megakernels.spec import ( - Feature, - MegakernelSpec, - Numerics, - NumericsKind, - Requirements, -) +try: + from sglang.kernels.plugin_kernels.nvidia.spec import ( + Feature, + MegakernelSpec, + Numerics, + NumericsKind, + Requirements, + Step, + ) +except ModuleNotFoundError: # SGLang before sgl-project/sglang#45389 + from sglang.srt.megakernels.spec import ( + Feature, + MegakernelSpec, + Numerics, + NumericsKind, + Requirements, + Step, + ) from sgl_nvidia_mega.glm_dsa import spec as target from sgl_nvidia_mega.glm_dsa import switches # The SGLang names this module reads (test/unit/test_contract.py pins them) SGLANG_READS = { - "srt/megakernels/spec.py": ( + "kernels/plugin_kernels/nvidia/spec.py|srt/megakernels/spec.py": ( "Feature", "MegakernelSpec", "Numerics", @@ -45,7 +56,12 @@ class _BuiltDraft: """The installed draft forward: armed steps run the fused draft layer.""" + # SGLang's own proposal, draft loop and draft extend; one graph per batch size draft_sampler = None + draft_loop = None + draft_extend = None + begin_draft_round = None + graph_variants = None def __init__(self, model): from sgl_nvidia_mega.glm_dsa import draft @@ -70,8 +86,6 @@ def summary(self) -> str: return "GLM-5 NextN draft: fused embedding front, eh_proj shard, distributed top-1" def plan(self, input_ids, forward_batch, *args, **kwargs): - from sglang.srt.megakernels.spec import Step - pp_proxy = kwargs.get("pp_proxy_tensors", args[0] if args else None) if pp_proxy is not None or not self.fused.engaged(): return None @@ -89,6 +103,10 @@ class _StockDraft: """SGLANG_NVIDIA_MEGAKERNEL_GLM_FUSED_DRAFT=0: no step, SGLang's draft forward.""" draft_sampler = None + draft_loop = None + draft_extend = None + begin_draft_round = None + graph_variants = None widths = () def summary(self) -> str: diff --git a/sgl_nvidia_mega/glm_dsa/eligibility.py b/sgl_nvidia_mega/glm_dsa/eligibility.py index 4f97b593..d7d67638 100644 --- a/sgl_nvidia_mega/glm_dsa/eligibility.py +++ b/sgl_nvidia_mega/glm_dsa/eligibility.py @@ -18,6 +18,28 @@ import torch +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/environ.py": ("envs",), + "srt/runtime_context.py": ("get_server_args",), + "srt/layers/flashinfer_comm_fusion.py": ("uses_cutedsl_ar_fusion",), + "srt/layers/layer_boundary/residual/add_norm.py": ("flashinfer_ar_fusion_applies",), + "srt/layers/moe/utils.py": ("post_experts_output_is_complete",), + "srt/models/deepseek_v2.py": ( + "DeepseekV2MLP", + "DeepseekV2MoE", + ), + "srt/models/deepseek_common/attention_forward_methods/forward_methods.py": ( + "AttnForwardMethod", + ), + "srt/layers/attention/dsa_backend.py": ("DeepseekSparseAttnBackend",), + "srt/model_executor/forward_context.py": ("get_attn_backend",), + "srt/model_executor/runner_backend_utils/breakable_cuda_graph/context.py": ( + "is_in_breakable_cuda_graph", + ), + "srt/model_executor/runner_utils/capture_mode.py": ("get_is_capture_mode",), +} + # Verify rows a request: MTP / EAGLE with top-1 drafts verifies speculative_num_steps # draft tokens + 1 (speculative_num_draft_tokens). The values served: 3 and 5 steps. SERVED_TOKENS_PER_REQUEST = (4, 6) diff --git a/sgl_nvidia_mega/glm_dsa/hops.py b/sgl_nvidia_mega/glm_dsa/hops.py index b1581253..a942eafe 100644 --- a/sgl_nvidia_mega/glm_dsa/hops.py +++ b/sgl_nvidia_mega/glm_dsa/hops.py @@ -41,6 +41,11 @@ compile_cached, ) +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/distributed/parallel_state.py": ("get_tp_group",), +} + logger = logging.getLogger(__name__) O_PROJ_IN = 2048 diff --git a/sgl_nvidia_mega/glm_dsa/moe.py b/sgl_nvidia_mega/glm_dsa/moe.py index 31057295..2e128e85 100644 --- a/sgl_nvidia_mega/glm_dsa/moe.py +++ b/sgl_nvidia_mega/glm_dsa/moe.py @@ -37,6 +37,11 @@ cu_stream, ) +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/deepseek_v2.py": ("DeepseekV2MoE",), +} + logger = logging.getLogger(__name__) EXPERTS = 256 @@ -62,7 +67,7 @@ class MoeRecords: def _process_like_server(gate, up, gate_sf, up_sf, w2, w2_sf): """The server's NVFP4 TRT-LLM weight processing on a synthetic layer: the loader's w13 order for this backend (up rows first) and the same processing call.""" - from sglang.srt.layers.moe.moe_runner import flashinfer_trtllm as trtllm + import sglang.srt.layers.moe.moe_runner.flashinfer_trtllm as trtllm e = gate.shape[0] f8 = torch.float8_e4m3fn diff --git a/sgl_nvidia_mega/glm_dsa/moe/__init__.py b/sgl_nvidia_mega/glm_dsa/moe/__init__.py index 02494b99..328ab1cc 100644 --- a/sgl_nvidia_mega/glm_dsa/moe/__init__.py +++ b/sgl_nvidia_mega/glm_dsa/moe/__init__.py @@ -45,6 +45,11 @@ from sgl_nvidia_mega.glm_dsa.moe.step import MoeRecords, MoeSteps, MlpSteps # noqa: F401 +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/deepseek_v2.py": ("DeepseekV2MoE",), +} + class MoeGlue: """Derived MoE weights, compiled engines and per-row-count steps.""" diff --git a/sgl_nvidia_mega/glm_dsa/moe/step.py b/sgl_nvidia_mega/glm_dsa/moe/step.py index 081d724e..e9276c9e 100644 --- a/sgl_nvidia_mega/glm_dsa/moe/step.py +++ b/sgl_nvidia_mega/glm_dsa/moe/step.py @@ -12,6 +12,13 @@ from sgl_nvidia_mega.glm_dsa.attention.step import StepContext +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/layers/moe/moe_runner/base.py": ("moe_output_buffer_ctx",), + "srt/models/deepseek_v2.py": ("DeepseekV2MoE",), + "srt/runtime_context.py": ("get_forward",), +} + # The engine's row groups: at most 4 groups of at most 4 rows (one group: g1, two: g2, # three or four: the four-group engine) GROUP_ROWS = 4 @@ -70,7 +77,7 @@ class MoeRecords: def _process_like_server(gate, up, gate_sf, up_sf, w2, w2_sf): """The server's NVFP4 TRT-LLM weight processing on a synthetic layer: the loader's w13 order for this backend (up rows first) and the same processing call.""" - from sglang.srt.layers.moe.moe_runner import flashinfer_trtllm as trtllm + import sglang.srt.layers.moe.moe_runner.flashinfer_trtllm as trtllm e = gate.shape[0] f8 = torch.float8_e4m3fn diff --git a/sgl_nvidia_mega/glm_dsa/providers.py b/sgl_nvidia_mega/glm_dsa/providers.py index acef0138..4553700a 100644 --- a/sgl_nvidia_mega/glm_dsa/providers.py +++ b/sgl_nvidia_mega/glm_dsa/providers.py @@ -26,6 +26,29 @@ import torch +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "kernels/ops/quantization/act_quant_triton.py": ("act_quant",), + "kernels/ops/attention/dsa/indexer_q.py": ("fused_q_indexer_rope_first_quant",), + "srt/layers/layer_boundary/__init__.py": ("get_attn_tp_context",), + "srt/layers/layer_boundary/adapters/attention.py": ("AttentionInputs",), + "srt/model_executor/runner/__init__.py": ("get_is_capture_mode",), + "srt/layers/attention/dsa/dsa_indexer.py": ("_broadcast_indexer_topk_from_rank0",), + "srt/model_executor/forward_context.py": ("get_attn_backend",), + "srt/state_capturer/indexer_topk.py": ("maybe_capture_indexer_topk",), + "kernels/ops/attention/utils.py": ("mla_quantize_and_rope_for_fp8",), + "srt/environ.py": ("envs",), + "srt/layers/moe/utils.py": ("sum_post_experts_output",), + "srt/layers/moe/moe_runner/base.py": ("moe_output_buffer_ctx",), + "srt/models/deepseek_v2.py": ("DeepseekV2MoE",), + "srt/runtime_context.py": ("get_forward",), + "kernels/jit/cute_aot_cache.py": ("get_jit_cache",), + "srt/layers/attention/dsa_backend.py": ( + "_restore_trtllm_decode_dp_padding", + "_trim_trtllm_decode_dp_padding", + ), +} + logger = logging.getLogger(__name__) diff --git a/sgl_nvidia_mega/glm_dsa/runtime/__init__.py b/sgl_nvidia_mega/glm_dsa/runtime/__init__.py index edb86104..e7febe31 100644 --- a/sgl_nvidia_mega/glm_dsa/runtime/__init__.py +++ b/sgl_nvidia_mega/glm_dsa/runtime/__init__.py @@ -30,12 +30,6 @@ # SGLang symbols this package reads (validated by test/unit/test_contract.py) SGLANG_READS = { - "kernels/plugin_kernels/nvidia/spec.py|srt/megakernels/spec.py": ( - "MegakernelSpec", - "MegakernelBinding", - "ForwardView", - "KvPoolView", - ), "srt/runtime_context.py": ( "get_parallel", "get_forward", diff --git a/sgl_nvidia_mega/glm_dsa/runtime/compile.py b/sgl_nvidia_mega/glm_dsa/runtime/compile.py index cf604be1..ffe77cd2 100644 --- a/sgl_nvidia_mega/glm_dsa/runtime/compile.py +++ b/sgl_nvidia_mega/glm_dsa/runtime/compile.py @@ -10,6 +10,11 @@ import torch +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "kernels/jit/cute_aot_cache.py": ("get_jit_cache",), +} + logger = logging.getLogger(__name__) _jit_cache: Any = None # created on first use diff --git a/sgl_nvidia_mega/glm_dsa/runtime/executor.py b/sgl_nvidia_mega/glm_dsa/runtime/executor.py index 299bd9d6..5062f86c 100644 --- a/sgl_nvidia_mega/glm_dsa/runtime/executor.py +++ b/sgl_nvidia_mega/glm_dsa/runtime/executor.py @@ -13,7 +13,7 @@ import torch import torch.distributed as dist -from sglang.srt import runtime_context +import sglang.srt.runtime_context as runtime_context from sgl_nvidia_mega.glm_dsa import eligibility from sgl_nvidia_mega.glm_dsa.attention.step import StepContext from sgl_nvidia_mega.glm_dsa.providers import Providers @@ -21,19 +21,16 @@ # SGLang symbols this module reads (validated by test/unit/test_contract.py) SGLANG_READS = { - "kernels/plugin_kernels/nvidia/spec.py|srt/megakernels/spec.py": ( - "MegakernelBinding", - "ForwardView", - "KvPoolView", - "DsaView", + "srt/runtime_context.py": ( + "get_parallel", + "get_forward", ), - "srt/runtime_context.py": ("get_parallel", "get_forward"), "srt/layers/layer_boundary/__init__.py": ("get_attn_tp_context",), "srt/layers/layer_boundary/adapters/attention.py": ("AttentionInputs",), "srt/eplb/expert_distribution.py": ("get_global_expert_distribution_recorder",), "srt/layers/attention/dsa/utils.py": ("maybe_prefetch_next_full_attention_kv",), "srt/layers/attention/index_topk_share.py": ("IndexTopKShareState",), - "srt/utils.py": ("BumpAllocator",), + "srt/utils/common.py": ("BumpAllocator",), "srt/layers/moe/moe_runner/base.py": ("moe_output_buffer_ctx",), } diff --git a/sgl_nvidia_mega/glm_dsa/runtime_legacy.py b/sgl_nvidia_mega/glm_dsa/runtime_legacy.py index 031c771e..8184c816 100644 --- a/sgl_nvidia_mega/glm_dsa/runtime_legacy.py +++ b/sgl_nvidia_mega/glm_dsa/runtime_legacy.py @@ -31,7 +31,7 @@ import torch import torch.distributed as dist -from sglang.srt import runtime_context +import sglang.srt.runtime_context as runtime_context from sgl_nvidia_mega.glm_dsa import eligibility from sgl_nvidia_mega.glm_dsa.providers import ( Providers, @@ -40,6 +40,15 @@ compile_stats, ) +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/layers/layer_boundary/__init__.py": ("get_attn_tp_context",), + "srt/eplb/expert_distribution.py": ("get_global_expert_distribution_recorder",), + "srt/layers/attention/dsa/utils.py": ("maybe_prefetch_next_full_attention_kv",), + "srt/layers/attention/index_topk_share.py": ("IndexTopKShareState",), + "srt/utils/common.py": ("BumpAllocator",), +} + logger = logging.getLogger(__name__) # This process's prepared target runtime (one target model per scheduler diff --git a/sgl_nvidia_mega/glm_dsa/spec.py b/sgl_nvidia_mega/glm_dsa/spec.py index 58670ee7..34caa1f8 100644 --- a/sgl_nvidia_mega/glm_dsa/spec.py +++ b/sgl_nvidia_mega/glm_dsa/spec.py @@ -15,6 +15,11 @@ * The first capture of each width checks, on every rank, that SGLang's attention takes the branches the stock step copies; a reason on any rank stops every rank (``runtime/executor.py``). +* The covered step reads the DSA backend and its KV pool through SGLang's running + attention backend (``get_attn_backend()``), not through a binding view, so it runs on + SGLang's megakernel API version 2 as released. +* TP8 may span two 4-GPU trays of one NVL72 rack (multi-node TP); without NVLink + multicast between them, see ``switches.py`` (``ALLOW_NO_AR_FUSION``). """ from __future__ import annotations @@ -28,6 +33,7 @@ Numerics, NumericsKind, Requirements, + Step, ) except ModuleNotFoundError: # SGLang before sgl-project/sglang#45389 from sglang.srt.megakernels.spec import ( @@ -36,6 +42,7 @@ Numerics, NumericsKind, Requirements, + Step, ) from sgl_nvidia_mega.glm_dsa import HF_CONFIG, MODEL_TYPE, TP_SIZE @@ -52,27 +59,26 @@ "kernels/plugin_kernels/nvidia/spec.py|srt/megakernels/spec.py": ( "Feature", "SPECULATIVE_DECODING", + "MULTI_NODE_TP", "KV_OFFLOAD", "MegakernelSpec", "Numerics", "NumericsKind", "TOLERANCE", "Requirements", + "Step", "decode_graph_backends", "config_disable_reasons", "model_disable_reasons", - "EAGLE_DRAFT_CAPTURE_CHECK", + "draft_loop", + "draft_extend", + "begin_draft_round", + "graph_variants", ), "srt/kernel_plugins/nvidia/binding.py|srt/megakernels/guards.py": ( "deadline_s", "deadline_s(watchdog_timeout)", ), - "srt/kernel_plugins/nvidia/binding.py|srt/megakernels/binding.py": ( - "ForwardView", - "KvPoolView", - "DsaView", - "MegakernelBinding", - ), "srt/model_executor/cuda_graph_config.py": ("decode", "backend", '"disabled"'), "srt/environ.py": ( "SGLANG_DSA_FUSE_TOPK", @@ -120,7 +126,8 @@ } ARCHITECTURE = "GlmMoeDsaForCausalLM" -API_VERSION = 1 +# the SGLang megakernel contract this spec is written against +API_VERSION = 2 # The co-resident cooperative grid the served kernels need at one CTA per SM RESIDENT_BLOCKS = MAX_ROWS * CLUSTER + 1 @@ -247,37 +254,6 @@ def decode_graph_backends() -> frozenset: }, ) -# The SGLang megakernel binding members the covered steps read -BINDING_MEMBERS = { - "MegakernelBinding": ("forward_view",), - "ForwardView": ("dsa",), - "KvPoolView": ( - "get_key_buffer", - "get_index_k_with_scale_buffer", - "index_page_size", - "set_mla_kv_buffer", - ), - "DsaView": ( - "kv_lora_rank", - "qk_nope_head_dim", - "qk_rope_head_dim", - "kv_cache_dim", - "physical_page_size", - "decode_impl", - "use_mha", - "kv_cache_dtype", - "use_fused_topk", - "topk_backend", - "dcp_size", - "index_kpool", - "hisparse", - "workspace_buffer", - "multi_ctas_kv_counter_buffer", - "indexer_metadata", - ), -} - - def toolchain_reasons() -> list: reasons = [] for module, what in ( @@ -296,43 +272,6 @@ def toolchain_reasons() -> list: return reasons -def binding_reasons() -> list: - try: - import sglang.srt.kernel_plugins.nvidia.binding as binding - except ModuleNotFoundError: - import sglang.srt.megakernels.binding as binding - - missing = [ - f"{owner}.{member}" - for owner, members in BINDING_MEMBERS.items() - for member in members - if not hasattr(getattr(binding, owner, None), member) - ] - if missing: - return [ - "this SGLang's megakernel binding lacks the DSA view members " - f"{', '.join(missing)}; update SGLang" - ] - return [] - - -def draft_capture_reasons() -> list: - if "draft" not in SLOTS: - return [] - try: - import sglang.kernels.plugin_kernels.nvidia.spec as layer - except ModuleNotFoundError: - import sglang.srt.megakernels.spec as layer - - if getattr(layer, "EAGLE_DRAFT_CAPTURE_CHECK", False): - return [] - return [ - "slot draft: this SGLang's EAGLE draft graph runners do not run a draft " - "spec's capture check (spec.EAGLE_DRAFT_CAPTURE_CHECK); update SGLang, or " - f"leave slot draft out of {SLOTS_ENV}" - ] - - def deadline_reasons(cfg) -> list: try: from sglang.srt.kernel_plugins.nvidia.binding import deadline_s @@ -350,8 +289,7 @@ def deadline_reasons(cfg) -> list: def _config_disable_reasons(cfg, hf_config) -> list: - reasons = toolchain_reasons() + binding_reasons() + slot_reasons() - reasons += draft_capture_reasons() + deadline_reasons(cfg) + reasons = toolchain_reasons() + slot_reasons() + deadline_reasons(cfg) reasons += draft_length_reason(cfg) if ( cfg.cuda_graph_config.decode.backend == "disabled" @@ -370,18 +308,67 @@ def _model_disable_reasons(model, hf_config) -> list: return eligibility.model_construction_reasons(model, hf_config) +class _Built: + """The installed target forward: covered verify steps run the fused stack.""" + + # SGLang's own proposal, draft loop and draft extend; one graph per batch size + draft_sampler = None + draft_loop = None + draft_extend = None + begin_draft_round = None + graph_variants = None + + def __init__(self, model, fused): + self.model, self.fused = model, fused + self.widths = tuple(sorted(fused.providers)) + + def summary(self) -> str: + fused = { + rows: sorted( + k + for k, v in p.describe().items() + if not v.startswith(("AttentionDecode", "HopSteps", "MlpSteps")) + ) + for rows, p in self.fused.providers.items() + } + return "GLM-5 DSA fused verify; fused steps by rows: " + "; ".join( + f"{rows}: {', '.join(names) or 'stock'}" for rows, names in fused.items() + ) + + def plan(self, input_ids, forward_batch, *args, **kwargs): + input_embeds = kwargs.get("input_embeds", args[0] if args else None) + pp_proxy = kwargs.get("pp_proxy_tensors", args[1] if len(args) > 1 else None) + rows = int(input_ids.shape[0]) + if not self.fused.engages(rows, forward_batch, input_embeds, pp_proxy): + return None + return Step(mode="target_verify", rows=rows) + + def forward(self, step, original, input_ids, positions, forward_batch, *a, **kw): + hidden_states = self.fused.run(input_ids, positions, forward_batch) + return self.model.logits_processor( + input_ids, hidden_states, self.model.lm_head, forward_batch, None + ) + + def _build(model, hf_config, ctx): from sgl_nvidia_mega.glm_dsa.runtime.executor import GlmDsaFusedDecode fused = GlmDsaFusedDecode(model) fused.prepare() - return fused + return _Built(model, fused) SPEC = MegakernelSpec( name="GLM-5.2 / GLM-5 DSA", requires=REQUIREMENTS, - features=frozenset({Feature.SPECULATIVE_DECODING, Feature.KV_OFFLOAD}), + features=frozenset( + { + Feature.SPECULATIVE_DECODING, + # TP8 spans the two 4-GPU trays of one NVL72 rack (Rubin, GB300 NVL72) + Feature.MULTI_NODE_TP, + Feature.KV_OFFLOAD, + } + ), numerics=NUMERICS, max_rows=MAX_ROWS, build=_build, diff --git a/sgl_nvidia_mega/spec_step/child_graph_capture.py b/sgl_nvidia_mega/spec_step/child_graph_capture.py index 66dd175e..f00d246b 100644 --- a/sgl_nvidia_mega/spec_step/child_graph_capture.py +++ b/sgl_nvidia_mega/spec_step/child_graph_capture.py @@ -36,6 +36,14 @@ ) from sglang.srt.model_executor.runner_utils.pool import graph_pool_replay_scope +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/model_executor/runner_backend_utils/breakable_cuda_graph/cuda_utils.py": ( + "checkCudaErrors", + ), + "srt/model_executor/runner_utils/pool.py": ("graph_pool_replay_scope",), +} + __all__ = [ "ChildGraphCapture", "capture_override", diff --git a/sgl_nvidia_mega/spec_step/hooks.py b/sgl_nvidia_mega/spec_step/hooks.py index 03034734..081bcf8e 100644 --- a/sgl_nvidia_mega/spec_step/hooks.py +++ b/sgl_nvidia_mega/spec_step/hooks.py @@ -29,7 +29,6 @@ "cleanup", "_graphs", "_outputs", - "torch.cuda.CUDAGraph()", ), "srt/model_executor/runner_backend/utils.py": ("resolve_decode_backend",), "srt/speculative/eagle_worker_v2.py": ( @@ -45,7 +44,8 @@ "srt/speculative/eagle_utils.py": ("eagle_sample",), "srt/speculative/spec_utils.py": ( "generate_simulated_accept_index", - "GrammarTree.from_device", + "GrammarTree", + "from_device", "build_grammar_vocab_mask", ), "kernels/ops/speculative/sampling.py": ("tree_speculative_sampling_target_only",), @@ -58,7 +58,12 @@ "_DEBUG_ASSERT", ), "srt/managers/scheduler_components/internal_state.py": ( - "SchedulerInternalStateController._add_speculative", + "SchedulerInternalStateController", + "_add_speculative", + ), + "srt/plugins/hook_registry.py": ( + "HookRegistry", + "HookType", ), } @@ -186,7 +191,7 @@ def _resolve_single_request_views(self, batch, indices) -> bool: req_pool_indices resolves its draft inputs as row views of the relay buffers instead of gathering copies. The rows are written back only by this request's next stash, which follows its forward on the same stream.""" - from sglang.srt.managers import overlap_utils + import sglang.srt.managers.overlap_utils as overlap_utils if ( not indices.is_cuda @@ -283,7 +288,7 @@ def _rebind(module, names): def _eagle_sample(): - from sglang.srt.speculative import eagle_utils + import sglang.srt.speculative.eagle_utils as eagle_utils return _rebind( eagle_utils, @@ -298,7 +303,7 @@ def _eagle_sample(): def _draft_extend_for_decode(): - from sglang.srt.speculative import eagle_worker_v2 + import sglang.srt.speculative.eagle_worker_v2 as eagle_worker_v2 from sgl_nvidia_mega.spec_step.ops import draft_extend_tail diff --git a/sgl_nvidia_mega/spec_step/lifted.py b/sgl_nvidia_mega/spec_step/lifted.py index ebbac5aa..0b740a95 100644 --- a/sgl_nvidia_mega/spec_step/lifted.py +++ b/sgl_nvidia_mega/spec_step/lifted.py @@ -19,6 +19,24 @@ from sgl_nvidia_mega.spec_step import switches as _spec_switches from sgl_nvidia_mega.spec_step.ops import draft_extend_tail +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/layers/dp_attention.py": ("is_dp_attention_enabled",), + "srt/runtime_context.py": ("get_parallel",), + "srt/sampling/penaltylib/repetition_penalty.py": ("apply_scaling_penalties",), + "srt/speculative/spec_utils.py": ( + "SIMULATE_ACC_LEN", + "SIMULATE_ACC_TOKEN_MODE", + "generate_simulated_accept_index", + ), + "srt/utils/async_probe.py": ( + "maybe_detect_nan", + "sanitize_nan_logits", + ), + "srt/speculative/uno_utils.py": ("sample_uno_tree_target_tokens",), + "kernels/ops/speculative/eagle.py": ("prepare_draft_extend_inputs",), +} + logger = logging.getLogger(__name__) # ---------------------------------------------------------------- eagle_utils diff --git a/sgl_nvidia_mega/spec_step/ops.py b/sgl_nvidia_mega/spec_step/ops.py index 936a3658..8b5fd19f 100644 --- a/sgl_nvidia_mega/spec_step/ops.py +++ b/sgl_nvidia_mega/spec_step/ops.py @@ -29,7 +29,7 @@ "_argmax_pair", "_medium_argmax_partial_kernel", ), - "kernels/jit/utils/compile/loader.py": ("load_jit", "extra_include_paths"), + "kernels/jit/utils/compile/loader.py": ("load_jit", "load_jit(extra_include_paths)"), } CSRC = Path(__file__).resolve().parent / "csrc" diff --git a/sgl_nvidia_mega/spec_step/runner.py b/sgl_nvidia_mega/spec_step/runner.py index a6a2967f..4b18badb 100644 --- a/sgl_nvidia_mega/spec_step/runner.py +++ b/sgl_nvidia_mega/spec_step/runner.py @@ -68,6 +68,55 @@ from sgl_nvidia_mega.spec_step import switches from sgl_nvidia_mega.spec_step.child_graph_capture import ChildGraphCapture +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/constrained/base_grammar_backend.py": ( + "GrammarMask", + "InvalidGrammarObject", + ), + "srt/distributed/parallel_state.py": ( + "get_tp_group", + "graph_capture", + ), + "srt/environ.py": ("envs",), + "srt/layers/attention/base_attn_backend.py": ("SharedReadEnds",), + "srt/model_executor/forward_batch_info.py": ("ForwardMode",), + "srt/model_executor/runner_backend/full_cuda_graph_backend.py": ( + "FullCudaGraphBackend", + ), + "srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py": ( + "BreakableCUDAGraph", + "BreakableCUDAGraphCapture", + "eager_on_graph", + ), + "srt/model_executor/runner_utils/pool.py": ("graph_pool_replay_scope",), + "srt/runtime_context.py": ( + "get_stream", + "attn_dp_enabled_of", + ), + "srt/utils/common.py": ("is_cuda",), + "srt/managers/schedule_batch.py": ("ScheduleBatch",), + "srt/managers/utils.py": ("GenerationBatchResult",), + "srt/server_args.py": ("ServerArgs",), + "srt/speculative/eagle_worker_v2.py": ("EAGLEWorkerV2",), + "srt/layers/attention/dsa_backend.py": ( + "DeepseekSparseAttnBackend", + "DeepseekSparseAttnMultiStepBackend", + ), + "srt/arg_groups/overrides.py": ( + "resolved_view", + "resolving_view", + ), + "srt/model_executor/runner_backend_utils/breakable_cuda_graph/cuda_utils.py": ( + "checkCudaErrors", + ), + "srt/speculative/spec_utils.py": ( + "generate_token_bitmask", + "sample_simulated_acc_len", + ), + "kernels/ops/memory/small_copy.py": ("try_small_copy",), +} + if TYPE_CHECKING: from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.managers.utils import GenerationBatchResult @@ -993,7 +1042,7 @@ def _capture(self, batch, step_tensors, output_layout) -> _StepGraph: @staticmethod def _simulated_accept_override(sim_accept_len, draw_args): - from sglang.srt.speculative import spec_utils + import sglang.srt.speculative.spec_utils as spec_utils def generate_simulated_accept_index( original, From 0a8833220414d55beb34bfc00eeccff2f4fceb81 Mon Sep 17 00:00:00 2001 From: shuw Date: Tue, 3 Nov 2026 17:29:08 -0600 Subject: [PATCH 7/8] [Fix] glm_dsa: name the NextN draft's architecture and pin its config fields draft_spec gets ARCHITECTURE and pins the fused draft layer's attention shapes and model_type (Requirements.hf_config), as test/unit/test_glm_dsa_config.py expects: a NextN config that differs in one of them is refused at load, naming it. Co-Authored-By: Claude Opus 5.5 --- sgl_nvidia_mega/glm_dsa/draft_spec.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/sgl_nvidia_mega/glm_dsa/draft_spec.py b/sgl_nvidia_mega/glm_dsa/draft_spec.py index cfb704e7..652faafb 100644 --- a/sgl_nvidia_mega/glm_dsa/draft_spec.py +++ b/sgl_nvidia_mega/glm_dsa/draft_spec.py @@ -29,6 +29,7 @@ Step, ) +from sgl_nvidia_mega.glm_dsa import HF_CONFIG, MODEL_TYPE from sgl_nvidia_mega.glm_dsa import spec as target from sgl_nvidia_mega.glm_dsa import switches @@ -45,6 +46,29 @@ "srt/models/deepseek_v2.py": ("set_embed_and_head", "logits_processor", "lm_head"), } +ARCHITECTURE = "GlmMoeDsaForCausalLMNextN" + +# The fused draft layer's shapes: its attention and the embedding / head width (its +# MoE runs SGLang's, and it has one layer whatever the target's depth) +DRAFT_HF_CONFIG = { + **{ + key: HF_CONFIG[key] + for key in ( + "hidden_size", + "num_attention_heads", + "q_lora_rank", + "kv_lora_rank", + "qk_rope_head_dim", + "qk_nope_head_dim", + "v_head_dim", + "index_n_heads", + "index_head_dim", + "index_topk", + ) + }, + "model_type": MODEL_TYPE, +} + # The draft top-1 is the stock argmax of the same logits (the distributed argmax # keeps its NaN and tie rules); the embedding front is bitwise fused_eh_norm. NUMERICS = Numerics( @@ -131,6 +155,7 @@ def _build(model, hf_config, ctx): tp_sizes=target.REQUIREMENTS.tp_sizes, dtypes=target.REQUIREMENTS.dtypes, decode_graph_backends=target.REQUIREMENTS.decode_graph_backends, + hf_config=DRAFT_HF_CONFIG, env=target.REQUIREMENTS.env, ), features=frozenset({Feature.SPECULATIVE_DECODING, Feature.MULTI_NODE_TP}), From 1f72ac557ba84828d456f89428a1dbbc52266321 Mon Sep 17 00:00:00 2001 From: shuw Date: Wed, 4 Nov 2026 01:21:42 -0600 Subject: [PATCH 8/8] [Fix] glm_dsa: repair names lost in the package split (TP8 e2e on Rubin) - draft.py: read TARGET / tp_gather / per_rank from runtime.executor; runtime is now a package and no longer exports them. - attention/step.py, moe/step.py: restore @dataclass on StepContext and MoeRecords (capture failed with "StepContext() takes no arguments"). - moe/step.py, moe/__init__.py: import the shape constants, HIDDEN and MoeGlue where they are used; moe/__init__ re-exports from moe.step. - draft_attention.py: import AttentionGlue / FusedAttention / kernel_rows from attention.fused. Verified on two Rubin trays (TP8, --nnodes 2, dummy weights): the fused verify routes the decode graphs at 4/3/1 (bs 1-4) and 6/5/1 (bs 1-2), generate requests succeed; unit tests 582 passed. Co-Authored-By: Claude Opus 5.5 --- sgl_nvidia_mega/glm_dsa/attention/step.py | 2 ++ sgl_nvidia_mega/glm_dsa/draft.py | 21 +++++++++------------ sgl_nvidia_mega/glm_dsa/draft_attention.py | 2 +- sgl_nvidia_mega/glm_dsa/moe/__init__.py | 17 +++++++++-------- sgl_nvidia_mega/glm_dsa/moe/step.py | 14 +++++++++++--- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/sgl_nvidia_mega/glm_dsa/attention/step.py b/sgl_nvidia_mega/glm_dsa/attention/step.py index 50a54db4..a046ebd0 100644 --- a/sgl_nvidia_mega/glm_dsa/attention/step.py +++ b/sgl_nvidia_mega/glm_dsa/attention/step.py @@ -31,6 +31,8 @@ logger = logging.getLogger(__name__) + +@dataclass class StepContext: """Per-forward state shared by the step groups of one verify forward.""" diff --git a/sgl_nvidia_mega/glm_dsa/draft.py b/sgl_nvidia_mega/glm_dsa/draft.py index 101addd1..19be21f0 100644 --- a/sgl_nvidia_mega/glm_dsa/draft.py +++ b/sgl_nvidia_mega/glm_dsa/draft.py @@ -29,11 +29,8 @@ from sglang.srt.distributed import tensor_model_parallel_all_gather from sglang.srt.distributed.parallel_state import get_tp_group -from sgl_nvidia_mega.glm_dsa import ( - draft_attention, - eligibility, - runtime, -) +from sgl_nvidia_mega.glm_dsa import draft_attention, eligibility +from sgl_nvidia_mega.glm_dsa.runtime import executor from sgl_nvidia_mega.glm_dsa.eligibility import HIDDEN, TP_SIZE from sgl_nvidia_mega.glm_dsa.runtime.compile import compile_cached, cu_stream @@ -159,9 +156,9 @@ def arm(self, nextn) -> None: from sgl_nvidia_mega.glm_dsa.kernels.cute.hop.transport import ErrorRecord self.error_record = ErrorRecord(dev) - if runtime.TARGET is not None: + if executor.TARGET is not None: # the target's reporter logs the draft top-1's record too - runtime.TARGET.errors.add(self.failure) + executor.TARGET.errors.add(self.failure) for name, rows in (("decode", TOP1_DECODE_ROWS), ("extend", TOP1_EXTEND_ROWS)): spec = specs[f"distributed_argmax_{rows}"] mailbox = _symm_mailbox(spec[0].mailbox_words(self.world), tp, dev) @@ -190,12 +187,12 @@ def _arm_attention(self, nextn) -> None: family.compile(self.eh_rows) except ValueError as e: family, why = None, str(e) - whys = runtime.tp_gather([] if why is None else [why]) + whys = executor.tp_gather([] if why is None else [why]) if any(whys): if self.rank == 0: logger.warning( "GLM-DSA fused draft attention stays stock on every rank: %s", - runtime.per_rank(whys), + executor.per_rank(whys), ) return self._attn_family = family @@ -353,7 +350,7 @@ def attention_for(self, rows: int, forward_batch: ForwardBatch): mode = "decode" if forward_batch.forward_mode.is_decode() else "extend" agreed = self._attn_agreed.get((mode, rows)) if agreed is None: - gathered = runtime.tp_gather( + gathered = executor.tp_gather( list(family.step_reasons(self._nextn, forward_batch)) ) agreed = self._attn_agreed[(mode, rows)] = not any(gathered) @@ -368,7 +365,7 @@ def attention_for(self, rows: int, forward_batch: ForwardBatch): "GLM-DSA fused draft attention stays stock: draft %s graph, %d rows: %s", mode, rows, - runtime.per_rank(gathered), + executor.per_rank(gathered), ) return family.for_rows(rows) if agreed else None @@ -517,7 +514,7 @@ def _compile(kernel, constexprs, args): def _target_hop_family(): """The prepared target's hop family when its fused hops are served on every rank, else None.""" - rt = runtime.TARGET + rt = executor.TARGET family = rt.families.get("hops") if rt is not None else None return family if getattr(family, "kernels", None) is not None else None diff --git a/sgl_nvidia_mega/glm_dsa/draft_attention.py b/sgl_nvidia_mega/glm_dsa/draft_attention.py index 56e1c2a1..8b5bc07c 100644 --- a/sgl_nvidia_mega/glm_dsa/draft_attention.py +++ b/sgl_nvidia_mega/glm_dsa/draft_attention.py @@ -29,7 +29,7 @@ import torch -from sgl_nvidia_mega.glm_dsa.attention import ( +from sgl_nvidia_mega.glm_dsa.attention.fused import ( AttentionGlue, FusedAttention, kernel_rows, diff --git a/sgl_nvidia_mega/glm_dsa/moe/__init__.py b/sgl_nvidia_mega/glm_dsa/moe/__init__.py index 328ab1cc..949ebe79 100644 --- a/sgl_nvidia_mega/glm_dsa/moe/__init__.py +++ b/sgl_nvidia_mega/glm_dsa/moe/__init__.py @@ -31,20 +31,21 @@ from sgl_nvidia_mega.glm_dsa.eligibility import HIDDEN, MAX_ROWS from sgl_nvidia_mega.glm_dsa.runtime.compile import compile_cached, cu_stream -from sgl_nvidia_mega.glm_dsa.moe.step import ( +from sgl_nvidia_mega.glm_dsa.moe.step import ( # noqa: F401 + EXPERTS, + INTER, + TOP_K, MlpSteps, + MoeRecords, + MoeSteps, + _layer_reason, + _process_like_server, + _u8, moe_split, ) logger = logging.getLogger(__name__) -EXPERTS = 256 -TOP_K = 8 -INTER = 256 # routed and shared intermediate size per rank (TP 8) - - -from sgl_nvidia_mega.glm_dsa.moe.step import MoeRecords, MoeSteps, MlpSteps # noqa: F401 - # The SGLang names this module reads (test/unit/test_contract.py pins them) SGLANG_READS = { "srt/models/deepseek_v2.py": ("DeepseekV2MoE",), diff --git a/sgl_nvidia_mega/glm_dsa/moe/step.py b/sgl_nvidia_mega/glm_dsa/moe/step.py index e9276c9e..29adaef6 100644 --- a/sgl_nvidia_mega/glm_dsa/moe/step.py +++ b/sgl_nvidia_mega/glm_dsa/moe/step.py @@ -4,13 +4,18 @@ from __future__ import annotations +import types from contextlib import nullcontext from dataclasses import dataclass -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, Optional import torch from sgl_nvidia_mega.glm_dsa.attention.step import StepContext +from sgl_nvidia_mega.glm_dsa.eligibility import HIDDEN + +if TYPE_CHECKING: + from sgl_nvidia_mega.glm_dsa.moe import MoeGlue # The SGLang names this module reads (test/unit/test_contract.py pins them) SGLANG_READS = { @@ -19,6 +24,10 @@ "srt/runtime_context.py": ("get_forward",), } +EXPERTS = 256 +TOP_K = 8 +INTER = 256 # routed and shared intermediate size per rank (TP 8) + # The engine's row groups: at most 4 groups of at most 4 rows (one group: g1, two: g2, # three or four: the four-group engine) GROUP_ROWS = 4 @@ -58,8 +67,7 @@ def forward(self, layer, x, buffer: Optional[torch.Tensor], step: StepContext): return mlp(x, step.forward_batch, None) -# The steps of each group, in call order (describe, rank agreement, logs). - +@dataclass class MoeRecords: """The engine's per-expert outputs of one MoE layer, for a hop that finalizes them inside its all-reduce (row stride ``rows``): ``pq`` [4, rows, 8, 6144] fp32 (the four