From 912963d5fbc9b53dc8563bb0abc9bb26a40f57b6 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:38:00 -0700 Subject: [PATCH 01/27] [Feat] [GB300] qwen3_5: NVIDIA megakernel plugin for Qwen3.5-397B-A17B MTP verify on GB300 TP8 The target-verify step of one request (7 rows) runs the plugin's stock step: SGLang's ops in SGLang's order over the weights and pools SGLang loaded, read through SGLang's EAGLE binding. The parity tool (SGLANG_NVIDIA_MEGAKERNEL_PARITY=1) compares it with SGLang's forward bit for bit and reports how far apart they are. The MTP draft spec covers no step yet, and a slot list naming verify_sample is refused with the switch that replaces it. Opt-in slots (SGLANG_NVIDIA_MEGAKERNEL_SLOTS), each bitwise: - route: the router GEMM and the NVFP4 MoE input in one launch; - mkm: on route's outputs, the top-10, FC1, SwiGLU, requantization and FC2 in one launch; - gdn_verify: the GatedDeltaNet verify chain with four launch choices changed; - gdn_pdl: its ReplaySSM verify launched as a programmatic dependent of the conv; - hop: the boundary's three fused all-reduce + residual + RMSNorm points on an NVLink multicast hop. --- .github/CODEOWNERS | 1 + NOTICE | 11 + README.md | 40 +- sgl_nvidia_mega/plugin.py | 3 + sgl_nvidia_mega/qwen3_5/__init__.py | 31 + sgl_nvidia_mega/qwen3_5/attention/__init__.py | 53 + sgl_nvidia_mega/qwen3_5/boundary/__init__.py | 91 ++ sgl_nvidia_mega/qwen3_5/boundary/hop.py | 397 ++++++ sgl_nvidia_mega/qwen3_5/draft.py | 67 + sgl_nvidia_mega/qwen3_5/gdn/__init__.py | 121 ++ sgl_nvidia_mega/qwen3_5/gdn/conv.py | 123 ++ sgl_nvidia_mega/qwen3_5/gdn/norm_fp8.py | 169 +++ sgl_nvidia_mega/qwen3_5/gdn/replayssm.py | 603 ++++++++ sgl_nvidia_mega/qwen3_5/gdn/verify.py | 347 +++++ sgl_nvidia_mega/qwen3_5/logits/__init__.py | 16 + sgl_nvidia_mega/qwen3_5/moe/__init__.py | 172 +++ sgl_nvidia_mega/qwen3_5/moe/mkm.py | 163 +++ sgl_nvidia_mega/qwen3_5/moe/route.py | 174 +++ sgl_nvidia_mega/qwen3_5/plan.py | 56 + sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 130 ++ sgl_nvidia_mega/qwen3_5/runtime/embedding.py | 44 + sgl_nvidia_mega/qwen3_5/runtime/executor.py | 220 +++ sgl_nvidia_mega/qwen3_5/runtime/parity.py | 279 ++++ sgl_nvidia_mega/qwen3_5/runtime/state.py | 42 + sgl_nvidia_mega/qwen3_5/sm103/__init__.py | 3 + sgl_nvidia_mega/qwen3_5/sm103/hop/__init__.py | 194 +++ sgl_nvidia_mega/qwen3_5/sm103/hop/src/hop.cu | 502 +++++++ sgl_nvidia_mega/qwen3_5/sm103/moe/__init__.py | 3 + sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py | 159 +++ sgl_nvidia_mega/qwen3_5/sm103/moe/router.py | 122 ++ .../qwen3_5/sm103/moe/src/mkm_moe.cu | 1021 +++++++++++++ .../qwen3_5/sm103/moe/src/router_front.cu | 1043 ++++++++++++++ sgl_nvidia_mega/qwen3_5/spec.py | 151 ++ test/e2e/qwen3_5_server.py | 53 + test/e2e/test_qwen3_5_stock_parity.py | 147 ++ test/kernels/test_qwen3_5_gdn.py | 234 +++ test/kernels/test_qwen3_5_hop.py | 180 +++ test/kernels/test_qwen3_5_moe.py | 226 +++ test/unit/test_contract.py | 90 +- test/unit/test_plugin.py | 7 +- test/unit/test_qwen3_5_runtime.py | 277 ++++ test/unit/test_qwen3_5_spec.py | 155 ++ test/unit/test_qwen3_5_steps.py | 1257 +++++++++++++++++ 43 files changed, 9168 insertions(+), 9 deletions(-) create mode 100644 sgl_nvidia_mega/qwen3_5/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/attention/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/boundary/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/boundary/hop.py create mode 100644 sgl_nvidia_mega/qwen3_5/draft.py create mode 100644 sgl_nvidia_mega/qwen3_5/gdn/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/gdn/conv.py create mode 100644 sgl_nvidia_mega/qwen3_5/gdn/norm_fp8.py create mode 100644 sgl_nvidia_mega/qwen3_5/gdn/replayssm.py create mode 100644 sgl_nvidia_mega/qwen3_5/gdn/verify.py create mode 100644 sgl_nvidia_mega/qwen3_5/logits/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/moe/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/moe/mkm.py create mode 100644 sgl_nvidia_mega/qwen3_5/moe/route.py create mode 100644 sgl_nvidia_mega/qwen3_5/plan.py create mode 100644 sgl_nvidia_mega/qwen3_5/runtime/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/runtime/embedding.py create mode 100644 sgl_nvidia_mega/qwen3_5/runtime/executor.py create mode 100644 sgl_nvidia_mega/qwen3_5/runtime/parity.py create mode 100644 sgl_nvidia_mega/qwen3_5/runtime/state.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/hop/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/hop/src/hop.cu create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/moe/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/moe/router.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/moe/src/mkm_moe.cu create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/moe/src/router_front.cu create mode 100644 sgl_nvidia_mega/qwen3_5/spec.py create mode 100644 test/e2e/qwen3_5_server.py create mode 100644 test/e2e/test_qwen3_5_stock_parity.py create mode 100644 test/kernels/test_qwen3_5_gdn.py create mode 100644 test/kernels/test_qwen3_5_hop.py create mode 100644 test/kernels/test_qwen3_5_moe.py create mode 100644 test/unit/test_qwen3_5_runtime.py create mode 100644 test/unit/test_qwen3_5_spec.py create mode 100644 test/unit/test_qwen3_5_steps.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f6fd23b3..c7bfd973 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,3 +4,4 @@ # Per model: one owner for its step code, kernels and tests. Each new model's PR # adds its line. **/*deepseek_v41* @YAMY1234 +**/*qwen3_5* @YAMY1234 diff --git a/NOTICE b/NOTICE index 68ece65e..cf64ed2b 100644 --- a/NOTICE +++ b/NOTICE @@ -7,6 +7,17 @@ decode step of DeepSeek-V4.1 (the layer loop, the mHC boundaries, the attention, MoE, Engram and the head) follows SGLang's own model code (Apache License 2.0, Copyright SGLang Team). Each adapted file names its origin in its header. +Portions are adapted from SGLang (https://github.com/sgl-project/sglang): the stock +verify step of Qwen3.5 (the layer loop, the GatedDeltaNet, the attention and the MoE) +follows SGLang's own model code (Apache License 2.0, Copyright SGLang Team). Qwen3.5's +GatedDeltaNet verify slots follow SGLang's linear-attention backend; its causal conv +update (Apache License 2.0, Copyright contributors to the vLLM project, Copyright (c) +2024 Tri Dao); its gated RMSNorm Triton kernel, which SGLang adapted from +flash-linear-attention (MIT License, Copyright (c) 2024 Tri Dao); and its ReplaySSM +verify kernel, which SGLang ported from the vLLM ReplaySSM reference implementation +(Apache License 2.0, Copyright contributors to the vLLM project). Each adapted file +names its origin in its header. + Portions are adapted from FlashInfer (https://github.com/flashinfer-ai/flashinfer): the Lamport all-reduce primitives and the low-latency all-reduce (Apache License 2.0, Copyright (c) 2026 by FlashInfer team), and the epilogue of the wo_b GEMM diff --git a/README.md b/README.md index 92a8e4ed..ff6674c1 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,9 @@ The package loads as an SGLang [general plugin](https://github.com/sgl-project/s | Model | GPU | Tensor parallel | Coverage | | --- | --- | --- | --- | | DeepSeek-V4.1-Flash | GB300 (SM103) | 4 | DSpark target verify steps of 1 to 384 rows (batch 1 to 64 at 6 rows a request) | +| Qwen3.5-397B-A17B (NVFP4) | GB300 (SM103) | 8 (two trays, one NVLink domain) | MTP target verify steps of 7 rows (batch 1) | -Larger verify steps, prefill and extend run SGLang's standard path. The DSpark draft step runs on the plugin's draft spec. +Larger verify steps, prefill and extend run SGLang's standard path. The DSpark draft step runs on the plugin's draft spec; the Qwen3.5 MTP draft runs SGLang's. ### What a covered step runs @@ -33,6 +34,18 @@ The step is the plugin's own code over the weights SGLang loaded (`deepseek_v41/ With the megakernel on, the default 36 slots run: the ones above, the attention chain, `hop_b` (the MoE post on hop A's Lamport mailbox) with its triggers, and the DSpark draft step's 16 slots. `SGLANG_NVIDIA_MEGAKERNEL_SLOTS` (comma-separated) overrides that list for tests and bisecting; empty runs every step function's stock code. `draft_embed` runs only when listed there. +Qwen3.5 (`qwen3_5/runtime`) runs the same way: per layer the stage boundary (`boundary/`), the GatedDeltaNet or the attention (`gdn/`, `attention/`) and the MoE (`moe/`), each with a stock implementation that reproduces SGLang's verify step bit for bit. It reads the running forward's metadata and pools through SGLang's EAGLE binding (`sglang.srt.megakernels.eagle_binding`). Its slots run only when listed in `SGLANG_NVIDIA_MEGAKERNEL_SLOTS`, and each is bit for bit SGLang's: + +| Slot | Rows | Kernel | +| --- | --- | --- | +| `route` | 7 | The router GEMM and the NVFP4 quantization of the MoE input in one launch | +| `mkm` (with `route`) | 7 | On `route`'s outputs, the top-10, FC1, SwiGLU, the NVFP4 requantization and FC2 in one launch | +| `gdn_verify` | 7 | The GatedDeltaNet verify chain (conv update, split, ReplaySSM verify, gated norm with the FP8 `out_proj` input) with its launch choices changed | +| `gdn_pdl` (with `gdn_verify`) | 7 | Its ReplaySSM verify launched as a programmatic dependent of the conv | +| `hop` | 7 | The layer boundary's fused all-reduce + residual add + RMSNorm points on an NVLink multicast hop | + +The sampling and acceptance after the verify, and the GatedDeltaNet extend of prefill, stay SGLang's own code. In an SGLang that has the switches, `SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH=1` runs the sampling and acceptance in CUDA graphs, and `SGLANG_ENABLE_GDN_PREFILL_GRAPH_EXTEND=1` captures the GatedDeltaNet extend inside the breakable prefill CUDA graphs. + ### Design - **Built at load.** Buffers, kernels and tactics for every covered width are allocated and compiled when the model is loaded, before the KV cache is sized. Nothing is compiled or allocated inside a step. @@ -44,9 +57,9 @@ With the megakernel on, the default 36 slots run: the ones above, the attention ### Requirements -- An SGLang with the megakernels framework (`sglang.srt.megakernels`, `--enable-nvidia-megakernel`) +- An SGLang with the megakernels framework (`sglang.srt.megakernels`, `--enable-nvidia-megakernel`); Qwen3.5 also needs its EAGLE binding (`sglang.srt.megakernels.eagle_binding`) - SGLang's CUDA 13 image: FlashInfer (its low-latency block-scaled GEMM as in 0.7.0 to 0.7.2), CuTe DSL (`nvidia-cutlass-dsl`) and the CUDA Python bindings -- 4 x GB300 +- 4 x GB300 (DeepSeek-V4.1); 8 x GB300, two trays of one NVLink domain (Qwen3.5) ### Installation @@ -74,6 +87,24 @@ python -m sglang.launch_server \ The three environment variables are required by the DeepSeek-V4.1 spec; without them the server stops at startup, naming each. +Qwen3.5, on each of the two trays (`--node-rank 0` and `1`): + +```bash +SGLANG_NVIDIA_MEGAKERNEL_SLOTS=route,mkm,gdn_verify,gdn_pdl,hop \ +python -m sglang.launch_server \ + --model-path nvidia/Qwen3.5-397B-A17B-NVFP4-V2 \ + --tp 8 --nnodes 2 --node-rank 0 --dist-init-addr : \ + --kv-cache-dtype fp8_e4m3 \ + --speculative-algorithm NEXTN --speculative-num-steps 6 \ + --speculative-eagle-topk 1 --speculative-num-draft-tokens 7 \ + --attention-backend trtllm_mha \ + --linear-attn-decode-backend triton \ + --enable-linear-replayssm-spec --mamba-ssm-dtype bfloat16 \ + --moe-runner-backend flashinfer_trtllm \ + --flashinfer-allreduce-fusion-backend cutedsl \ + --enable-nvidia-megakernel +``` + ## Repository Structure ```text @@ -81,6 +112,7 @@ sgl-nvidia-mega/ ├── sgl_nvidia_mega/ │ ├── plugin.py # registers the specs with SGLang's megakernels framework │ ├── common/ # NVRTC builds and PDL launches; CuTe DSL sync primitives +│ ├── qwen3_5/ # the same layout: spec, plan, runtime, boundary, gdn, attention, moe, logits, sm103 │ └── deepseek_v41/ │ ├── spec.py # the spec: the setups it serves and its build │ ├── plan.py # which kernel fills each slot, per width @@ -106,6 +138,8 @@ pytest test/kernels # 4 x GB300 pytest test/e2e # 4 x GB300, with SGLANG_NVIDIA_MEGAKERNEL_E2E_MODEL_PATH set ``` +Qwen3.5 runs TP8, which spans two GB300 trays: its e2e tests read a server launched on them through `SGLANG_NVIDIA_MEGAKERNEL_E2E_URL` and the files its ranks log to (`SGLANG_NVIDIA_MEGAKERNEL_E2E_LOGS`, comma-separated). + ## Contributing Open a pull request against `main`. The PR title follows `[Type] scope: title` (see `.github/pull_request_template.md`), and `pre-commit run --all-files` must pass. diff --git a/sgl_nvidia_mega/plugin.py b/sgl_nvidia_mega/plugin.py index 456031ec..dddc4115 100644 --- a/sgl_nvidia_mega/plugin.py +++ b/sgl_nvidia_mega/plugin.py @@ -19,6 +19,9 @@ ("DeepseekV4ForCausalLM", "sgl_nvidia_mega.deepseek_v41.spec:SPEC"), # the DSpark draft: SGLang's forward, the megakernel's folded proposal ("DeepseekV4ForCausalLMDSpark", "sgl_nvidia_mega.deepseek_v41.draft:SPEC"), + ("Qwen3_5MoeForConditionalGeneration", "sgl_nvidia_mega.qwen3_5.spec:SPEC"), + # the MTP draft: SGLang's forward; its built object offers no EAGLE entry yet + ("Qwen3_5ForCausalLMMTP", "sgl_nvidia_mega.qwen3_5.draft:SPEC"), ) # Test only: with SGLANG_NVIDIA_MEGAKERNEL_PARITY_SORT_TOPK=1, the composition check diff --git a/sgl_nvidia_mega/qwen3_5/__init__.py b/sgl_nvidia_mega/qwen3_5/__init__.py new file mode 100644 index 00000000..9ba3b849 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/__init__.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen3.5 (397B-A17B, NVFP4) megakernel: its spec and the setups it serves (``spec``), +the slot plan (``plan``), the executor of a covered target-verify step and its build +(``runtime``), the step code per package (``boundary``, ``gdn``, ``attention``, ``moe``, +``logits``), and the SM103 (GB300) kernels (``sm103``). The MTP draft runs SGLang's own +forward.""" + +# Qwen3.5-397B-A17B's text_config (the HF config nests it under text_config), which +# the kernels are built for +TEXT_CONFIG = dict( + model_type="qwen3_5_moe_text", + hidden_size=4096, + num_hidden_layers=60, + full_attention_interval=4, + num_attention_heads=32, + num_key_value_heads=2, + head_dim=256, + attn_output_gate=True, + linear_num_key_heads=16, + linear_num_value_heads=64, + linear_key_head_dim=128, + linear_value_head_dim=128, + linear_conv_kernel_dim=4, + num_experts=512, + num_experts_per_tok=10, + moe_intermediate_size=1024, + shared_expert_intermediate_size=1024, + hidden_act="silu", + rms_norm_eps=1e-6, +) diff --git a/sgl_nvidia_mega/qwen3_5/attention/__init__.py b/sgl_nvidia_mega/qwen3_5/attention/__init__.py new file mode 100644 index 00000000..3ac82765 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/attention/__init__.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/srt/models/qwen3_5.py (Qwen3_5AttentionDecoderLayer.self_attention). +"""The gated full attention of a covered target-verify step, over the layer's loaded +weights, SGLang's attention backend and its KV cache. + +Stock: the CUDA arm of ``Qwen3_5AttentionDecoderLayer.self_attention`` with SGLang's own +ops in its order: the qkv projection with the fused q / k Gemma RMSNorm + RoPE + gate +split, the attention through the backend (which writes this step's KV rows), the +sigmoid output gate and ``o_proj``. ``step`` returns the TP-partial output, whose +all-reduce the boundary runs. +""" + +from __future__ import annotations + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen3_5.py": ( + "_prepare_qkv_gate", + "_prepare_qkv_gate(positions)", + "_prepare_qkv_gate(hidden_states)", + "_prepare_qkv_gate(forward_batch)", + "attn", + "attn_output_gate", + "o_proj", + ), + "kernels/ops/elementwise/elementwise.py": ( + "fused_sigmoid_mul", + "fused_sigmoid_mul(inplace)", + ), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + "srt/models/qwen3_5.py:Qwen3_5AttentionDecoderLayer.self_attention": "db0f4d98e0fb1abf", +} + + +def step(layer, x, st): + """``layer``'s attention on the normed input ``x`` [rows, hidden]: the TP-partial + output [rows, hidden].""" + from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul + + q, k, v, gate = layer._prepare_qkv_gate( + positions=st.positions, hidden_states=x, forward_batch=st.fb + ) + o = layer.attn(q, k, v, st.fb) + if layer.attn_output_gate: + o = fused_sigmoid_mul(o, gate, inplace=True) + out, _ = layer.o_proj(o) + return out diff --git a/sgl_nvidia_mega/qwen3_5/boundary/__init__.py b/sgl_nvidia_mega/qwen3_5/boundary/__init__.py new file mode 100644 index 00000000..7532e13a --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/boundary/__init__.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The residual stream between sublayers: the stream's start (``open_stream``), each +layer's input norm before the mixer (``pre``), the mixer's all-reduce + residual add + +norm into the MoE input (``post_attn``), the MoE's exit (``ffn_exit``, whose ``finish`` +completes or carries its output), and the final norm (``close_stream``). + +Stock: SGLang's own stage boundaries of each layer, in the decoder layers' order, and +its ``residual_batch`` around the layers. With SGLang's CuTe DSL fusion they run +FlashInfer's fused all-reduce + residual add + RMSNorm and take the MoE's deferred +finalize. Slot ``hop`` (``hop.py``) runs those three fused points on the decode hop. +""" + +from __future__ import annotations + +from sgl_nvidia_mega.qwen3_5.boundary import hop +from sgl_nvidia_mega.qwen3_5.boundary.hop import build # noqa: F401 + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen3_5.py": ( + "attn_boundary", + "ffn_boundary", + "norm", + "flashinfer_mnnvl_cutedsl_fusion", + ), + "srt/layers/layer_boundary/stage.py": ( + "prepare", + "_prepare_attention(capture_gathered)", + "finish", + "exit", + ), + "srt/layers/layer_boundary/exit.py": ("defer_moe_finalize", "finish"), + "srt/layers/layer_boundary/residual/batch.py": ( + "start", + "final_norm", + "final_norm(finalize_norm)", + "final_norm(skip_empty)", + ), +} + + +def open_stream(st) -> None: + """Start the residual stream for the first layer.""" + from sglang.srt.layers.layer_boundary.residual.batch import start + + start(st.fb) + + +def pre(layer, hidden, st, capture=None): + """``layer``'s mixer input from the stream (its input norm).""" + if st.slots.hop and capture is None: + x = hop.layer_input(st.bound["boundary"].hop, layer, hidden, st) + if x is not None: + return x + return layer.attn_boundary.prepare(hidden, st.fb, capture_gathered=capture) + + +def post_attn(layer, part, st): + """The mixer's partial output into the stream; returns the MoE input.""" + hidden = layer.attn_boundary.finish(part, st.fb) + if st.slots.hop: + x = hop.moe_input(st.bound["boundary"].hop, layer, hidden, st) + if x is not None: + return x + return layer.ffn_boundary.prepare(hidden, st.fb) + + +def ffn_exit(layer, st): + """The MoE's exit scope: its ``defer_moe_finalize`` tells the MoE whether to hand + over its finalize, and ``finish(out)`` completes or carries the output.""" + return layer.ffn_boundary.exit(st.fb) + + +def close_stream(st, hidden): + """The final norm after the last layer, with the CuTe DSL finalize when the last + MoE handed one over.""" + from sglang.srt.layers.layer_boundary.residual.batch import final_norm + + if st.slots.hop: + x = hop.final_norm(st.bound["boundary"].hop, hidden, st) + if x is not None: + return x + m = st.model + return final_norm( + hidden, + st.fb, + m.norm, + finalize_norm=m.flashinfer_mnnvl_cutedsl_fusion, + skip_empty=True, + ) diff --git a/sgl_nvidia_mega/qwen3_5/boundary/hop.py b/sgl_nvidia_mega/qwen3_5/boundary/hop.py new file mode 100644 index 00000000..be816829 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/boundary/hop.py @@ -0,0 +1,397 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/srt/layers/layer_boundary/stage.py, prepare.py, residual/batch.py and +# fusions/cutedsl.py (the stage boundary's CuTe DSL fusion points). +"""Slot hop: the decode hop (``sm103/hop``) in place of FlashInfer's MNNVL CuTe DSL +kernels at the three points of a covered step where SGLang's stage boundary runs them: + +- ``layer_input``: the previous MoE's finalize + all-reduce + residual add + input norm; +- ``moe_input``: the mixer's all-reduce + residual add + post-attention norm; +- ``final_norm``: the last MoE's finalize + all-reduce + residual add + final norm. + +Each one mirrors the boundary's path for a covered step, which build checks on every +layer (the ordinary batch rows, the CuTe DSL kernel tried first, nothing moved, sliced +or gathered around it). It runs the hop exactly where that path runs the CuTe DSL +kernel, by SGLang's own eligibility checks, and leaves the residual stream as the +boundary does; anywhere else it returns None and SGLang's boundary runs. At these rows +the CuTe DSL fusion runs FlashInfer's low-latency kernels, whose outputs the hop's +equal bit for bit. +""" + +from __future__ import annotations + +import logging +from functools import partial +from types import MethodType +from typing import Optional + +import msgspec +import torch + +logger = logging.getLogger(__name__) + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen3_5.py": ( + "flashinfer_mnnvl_cutedsl_fusion", + "layers", + "start_layer", + "end_layer", + "norm", + "layer_id", + "attn_boundary", + "ffn_boundary", + ), + "srt/layers/layer_boundary/stage.py": ("plan", "declaration", "kind"), + "srt/layers/layer_boundary/factories.py": ("row_transport",), + "srt/layers/layer_boundary/construction.py": ( + "BatchVariant", + "paths", + "fusions", + "norm", + "qkv_latent_func", + "finishes_directly", + "_publish_lora_layout", + "variant_for", + ), + "srt/layers/layer_boundary/contracts.py": ( + "ORDINARY", + "entry", + "output", + "prepare", + "input_move", + "attn_input_adapter", + "always_partial", + "sum_impl", + ), + "srt/layers/layer_boundary/prepare.py": ( + "_run_entry", + "_run_entry(step)", + "_run_entry(carried_fusions)", + "_run_entry(carried_read_fusions)", + "_reduce_update_read", + "_reduce_update_read(gathers_residual)", + "_reduce_update_read(fusions)", + "_reduce_update_read(read_fusions)", + "_reduce_update_read(read_fusion_indices)", + "_attn_input_default", + ), + "srt/layers/layer_boundary/fusions/cutedsl.py": ( + "CuteDSLFusion", + "MoeDeferredFinalize", + "_fused_norm_gamma", + "_finalize_add_norm", + "_ffn_input_all_reduce_add_norm", + "_finalize_eligible", + "_eligible", + "hidden_size", + "top_k", + "rms_epsilon", + "routed_output", + "expert_weights", + "permuted_indices", + "gated_shared_output", + "m", + ), + "srt/layers/layer_boundary/residual/batch.py": ("stream_of", "clear"), + "srt/layers/layer_boundary/residual/stream.py": ( + "DeclaredSum", + "pending", + "owed", + "update", + "input", + "write", + "export", + "export(takes_handoff)", + ), + "srt/layers/layer_boundary/residual/add_norm.py": ("is_plain_add",), + "srt/layers/layernorm.py": ('"gemma_weight"',), + "srt/megakernels/runtime.py": ("megakernel_agree",), + "srt/runtime_context.py": ("get_parallel", '"tp_group"', "get_exec"), + "srt/distributed/parallel_state.py": ("device_group", "world_size"), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + "srt/layers/layer_boundary/stage.py:StageBoundary._prepare": "3ade6a2952490616", + "srt/layers/layer_boundary/stage.py:StageBoundary._prepare_attention": "ab2768d2e68434a5", + "srt/layers/layer_boundary/stage.py:StageBoundary._prepare_input": "53b7f2a5bd394d76", + "srt/layers/layer_boundary/stage.py:StageBoundary._select": "f317ed475a7672cf", + "srt/layers/layer_boundary/prepare.py:_run_entry": "639740496fc09d2d", + "srt/layers/layer_boundary/prepare.py:_reduce_update_read": "67badae13f96d80f", + "srt/layers/layer_boundary/fusions/cutedsl.py:CuteDSLFusion._finalize_add_norm": "46767c09458d7e1e", + "srt/layers/layer_boundary/fusions/cutedsl.py:CuteDSLFusion._ffn_input_all_reduce_add_norm": "84163a5a5a695823", + "srt/layers/layer_boundary/residual/batch.py:final_norm": "b4a1b4813ca5ac17", +} + +# the (point, rows) the slot has run at (each logged once, as engagement evidence) +_ENGAGED: set = set() + + +class Bound(msgspec.Struct, frozen=True): + """The boundary package's build result (``st.bound["boundary"]``).""" + + hop: Optional[object] = None # sm103.hop.Hop for every covered width, or None + reason: str = "" # why there is no hop + + +def build(model, ctx) -> Bound: + """The hop when slot ``hop`` is requested and every layer's boundary is the path + mirrored here. A collective: every rank builds it or none does.""" + from sglang.srt.megakernels.runtime import megakernel_agree + from sglang.srt.runtime_context import get_parallel + + from sgl_nvidia_mega.qwen3_5 import runtime + from sgl_nvidia_mega.qwen3_5.sm103.hop import Hop + + if "hop" not in runtime.SLOTS: + return Bound(reason="slot off") + m = model.model + group = get_parallel().tp_group + reasons = refusals(m, group.world_size) + # a rank without the hop would leave its peers' consumers waiting + megakernel_agree(reasons, what="hop refusals") + if reasons: + return Bound(reason="; ".join(reasons)) + hop = Hop( + max_m=max(ctx.widths), + eps=m.flashinfer_mnnvl_cutedsl_fusion.rms_epsilon, + group=group.device_group, + device=torch.device("cuda", torch.cuda.current_device()), + deadline_s=ctx.deadline_s, + ) + return Bound(hop=hop) + + +def refusals(m, world: int) -> list[str]: + """Why the hop cannot stand in for the CuTe DSL kernels of ``m`` + (Qwen3_5ForCausalLM) at TP ``world``; [] when it can.""" + from sgl_nvidia_mega.qwen3_5.sm103 import hop as kernels + + service = m.flashinfer_mnnvl_cutedsl_fusion + if service is None: + return ["SGLang runs no FlashInfer MNNVL CuTe DSL fusion"] + out = [] + if service.hidden_size != kernels.HIDDEN or service.top_k > kernels.MAX_TOP_K: + out.append( + f"hidden {service.hidden_size}, top-{service.top_k} (the hop serves " + f"hidden {kernels.HIDDEN}, up to top-{kernels.MAX_TOP_K})" + ) + if world not in kernels.WORLDS: + out.append(f"TP {world} (the hop serves TP {kernels.WORLDS})") + paths = [ + (m.layers[i].layer_id, why) + for i in range(m.start_layer, m.end_layer) + for why in path_refusals(m.layers[i]) + ] + if paths: + more = f" (and {len(paths) - 1} more)" if len(paths) > 1 else "" + out.append(f"layer {paths[0][0]}: {paths[0][1]}{more}") + return out + + +def path_refusals(layer) -> list[str]: + """Why the ordinary paths of ``layer``'s two stages are not the ones mirrored.""" + from sglang.srt.layers.layer_boundary.construction import BatchVariant + from sglang.srt.layers.layer_boundary.fusions.cutedsl import ( + CuteDSLFusion, + _fused_norm_gamma, + ) + from sglang.srt.layers.layer_boundary.prepare import ( + _attn_input_default, + _reduce_update_read, + _run_entry, + ) + + out = [] + for stage in (layer.attn_boundary, layer.ffn_boundary): + plan, name = stage.plan, stage.kind.name.lower() + if ( + not isinstance(plan.fusions, CuteDSLFusion) + or _fused_norm_gamma(plan.norm) is None + ): + out.append(f"its {name} input runs no CuTe DSL fusion") + continue + path = plan.paths.get(BatchVariant.ORDINARY) + entry = path.entry if path is not None else None + prepare = entry.prepare if entry is not None else None + if ( + not isinstance(prepare, partial) + or prepare.func is not _run_entry + or prepare.keywords["carried_read_fusions"] + or entry.input_move is not None + or entry.attn_input_adapter not in (None, _attn_input_default) + or plan.qkv_latent_func is not None + or plan._publish_lora_layout + or stage.declaration.row_transport is not None + ): + out.append(f"its {name} input moves or reads more than the mirrored path") + continue + if stage is layer.attn_boundary: + # the finalize handoff; then the mixer's sum, which the MoE input completes + takes = ( + _first_is( + prepare.keywords["carried_fusions"], + CuteDSLFusion._finalize_add_norm, + plan, + ) + and plan.finishes_directly + and path.output.always_partial + and path.output.sum_impl is None + ) + else: + step = prepare.keywords["step"] + takes = ( + isinstance(step, partial) + and step.func is _reduce_update_read + and not step.keywords["gathers_residual"] + and not step.keywords.get("read_fusions") + and not step.keywords.get("read_fusion_indices") + and _first_is( + step.keywords["fusions"], + CuteDSLFusion._ffn_input_all_reduce_add_norm, + plan, + ) + ) + if not takes: + out.append(f"its {name} input does not try the CuTe DSL kernel first") + return out + + +def _first_is(candidates, method, plan) -> bool: + """Whether the first of ``candidates`` is ``plan``'s CuTe DSL ``method`` on ``plan``.""" + first = candidates[0] if candidates else None + return ( + isinstance(first, partial) + and isinstance(first.func, MethodType) + and first.func.__func__ is method + and first.func.__self__ is plan.fusions + and first.args == (plan,) + ) + + +def layer_input(hop, layer, hidden, st): + """``attn_boundary.prepare`` of a layer whose input completes the previous MoE's + finalize handoff (``hidden``), with the hop as its kernel; None where that + prepare would not run the CuTe DSL kernel.""" + from sglang.srt.layers.layer_boundary.fusions.cutedsl import ( + MoeDeferredFinalize, + _fused_norm_gamma, + ) + from sglang.srt.layers.layer_boundary.residual.batch import stream_of + + stream = stream_of(st.fb) + pending, plan = stream.pending, layer.attn_boundary.plan + if not ( + pending is not None + and isinstance(pending.owed, MoeDeferredFinalize) + and _ordinary(plan, st, pending) + and _takes(hop, pending.owed) + and plan.fusions._finalize_eligible(plan, st.fb, pending.owed.m) + ): + return None + owed, residual = stream.input(hidden) + norm, residual = _finalize( + hop, "layer input", owed, residual, _fused_norm_gamma(plan.norm) + ) + stream.write(residual) + return norm + + +def moe_input(hop, layer, hidden, st): + """``ffn_boundary.prepare`` of the mixer's partial output (``hidden``, as + ``attn_boundary.finish`` recorded it), with the hop as its kernel; None where that + prepare would not run the CuTe DSL kernel.""" + from sglang.srt.layers.layer_boundary.fusions.cutedsl import _fused_norm_gamma + from sglang.srt.layers.layer_boundary.residual.batch import stream_of + from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum + from sglang.srt.runtime_context import get_exec + + stream = stream_of(st.fb) + pending, plan = stream.pending, layer.ffn_boundary.plan + if not ( + pending is not None + and isinstance(pending.owed, DeclaredSum) + and _ordinary(plan, st, pending) + ): + return None + part, residual = stream.input(hidden) + rows = part.shape[0] + if not ( + residual is not None + and hop.supports(rows) + and part.dtype == torch.bfloat16 + and plan.fusions._eligible(plan, st.fb, rows) + and not get_exec().comm.enable_quant_communications + ): + return None + _engage("MoE input", rows) + norm, residual = hop.all_reduce_residual_rms_norm( + local_contribution=part, residual=residual, gamma=_fused_norm_gamma(plan.norm) + ) + stream.write(residual) + return norm + + +def final_norm(hop, hidden, st): + """``residual_batch.final_norm`` of the last MoE's finalize handoff (``hidden``) + with the hop as its kernel; None when the stream holds no handoff the hop takes.""" + from sglang.srt.layers.layer_boundary.fusions.cutedsl import MoeDeferredFinalize + from sglang.srt.layers.layer_boundary.residual.batch import clear, stream_of + + stream = stream_of(st.fb) + pending = stream.pending + if not ( + pending is not None + and isinstance(pending.owed, MoeDeferredFinalize) + and pending.update.is_plain_add + and _takes(hop, pending.owed) + ): + return None + owed, residual = stream.export(hidden, takes_handoff=True) + clear(st.fb) + norm, _ = _finalize(hop, "final norm", owed, residual, st.model.norm.gemma_weight) + return norm + + +def _ordinary(plan, st, pending) -> bool: + """A plain residual add on the ordinary batch rows: the path build checked.""" + from sglang.srt.layers.layer_boundary.construction import BatchVariant + + return ( + pending.update.is_plain_add and plan.variant_for(st.fb) is BatchVariant.ORDINARY + ) + + +def _takes(hop, owed) -> bool: + """Whether the hop's finalize reads this handoff's rows as SGLang wrote them.""" + shared = owed.gated_shared_output + return ( + hop.supports(owed.m) + and owed.routed_output.dtype == torch.bfloat16 + and owed.expert_weights.dtype == torch.bfloat16 + and owed.permuted_indices.dtype == torch.int32 + and (shared is None or shared.dtype == torch.bfloat16) + ) + + +def _finalize(hop, point, owed, residual, gamma): + _engage(point, owed.m) + return hop.moe_finalize_all_reduce_rms_norm( + routed_output=owed.routed_output, + expert_weights=owed.expert_weights, + permuted_indices=owed.permuted_indices, + gated_shared_output=owed.gated_shared_output, + residual=residual, + gamma=gamma, + ) + + +def _engage(point: str, rows: int) -> None: + if (point, rows) not in _ENGAGED: + _ENGAGED.add((point, rows)) + logger.info( + "NVIDIA megakernel: Qwen3.5 slot hop runs the %s at %d rows", point, rows + ) diff --git a/sgl_nvidia_mega/qwen3_5/draft.py b/sgl_nvidia_mega/qwen3_5/draft.py new file mode 100644 index 00000000..95c6a7ce --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/draft.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen3.5's MTP draft on SM103 (GB300): the megakernel draft spec. + +The draft's forward stays SGLang's (the built object covers no step width), and the +built object offers no EAGLE entry yet (``draft_loop``, ``draft_extend``, +``begin_draft_round``): every draft phase runs SGLang's own path. +""" + +from __future__ import annotations + +from sglang.srt.megakernels.spec import Feature, MegakernelSpec, Requirements + +from sgl_nvidia_mega.qwen3_5 import spec as target + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/megakernels/spec.py": ( + "Feature", + "SPECULATIVE_DECODING", + "MegakernelSpec", + "Requirements", + ), +} + +ARCHITECTURE = "Qwen3_5ForCausalLMMTP" + + +class MtpDraft: + """The draft spec's built object: SGLang's forward on every step.""" + + widths: tuple[int, ...] = () + draft_sampler = None # SGLang's draft proposal + + def __init__(self, lm): + self.lm = lm + + def summary(self) -> str: + return "Qwen3.5 MTP draft: SGLang's forward (no draft phase ported yet)" + + def plan(self, input_ids, forward_batch, *args, **kwargs): + return None + + def forward(self, step, original, *args, **kwargs): + raise AssertionError("the MTP draft covers no step width") + + +def _build(model, hf_config, ctx) -> MtpDraft: + return MtpDraft(model) + + +SPEC = MegakernelSpec( + name="Qwen3.5 MTP draft", + # the target's devices, TP size, dtype and decode graphs: the draft shares its GPUs + requires=Requirements( + isas=target.REQUIREMENTS.isas, + tp_sizes=target.REQUIREMENTS.tp_sizes, + dtypes=target.REQUIREMENTS.dtypes, + decode_graph_backends=target.REQUIREMENTS.decode_graph_backends, + ), + features=frozenset({Feature.SPECULATIVE_DECODING}), + numerics=target.NUMERICS, + # the draft's forward stays SGLang's: no width is built + max_rows=0, + build=_build, + api_version=target.API_VERSION, +) diff --git a/sgl_nvidia_mega/qwen3_5/gdn/__init__.py b/sgl_nvidia_mega/qwen3_5/gdn/__init__.py new file mode 100644 index 00000000..7cab0c67 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/gdn/__init__.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/srt/models/qwen3_5.py (Qwen3_5GatedDeltaNet.forward). +"""The GatedDeltaNet mixer of a covered target-verify step, over the layer's loaded +weights, SGLang's linear-attention backend and its mamba pool. + +Stock: the CUDA target-verify arm of ``Qwen3_5GatedDeltaNet.forward`` with SGLang's own +ops in its order: the input projections, the fused split of q / k / v / z / b / a, the +causal conv + ReplaySSM verify through the backend (``RadixLinearAttention``), the +gated RMSNorm and ``out_proj``. ``step`` returns the TP-partial output, whose +all-reduce the boundary runs. Slot ``gdn_verify`` (``verify.py``) runs the step with +the backend's chain opened up and its launch choices changed. +""" + +from __future__ import annotations + +import msgspec + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen3_5.py": ( + "Qwen3_5LinearDecoderLayer", + "layers", + "start_layer", + "end_layer", + "linear_attn", + "_forward_input_proj", + "num_k_heads", + "num_v_heads", + "attn_tp_size", + "head_k_dim", + "head_v_dim", + "attn", + "norm", + "out_proj", + ), + "kernels/ops/attention/triton_gdn_fused_proj.py": ( + "fused_qkvzba_split_reshape_cat_contiguous", + ), + "srt/layers/radix_linear_attention.py": ( + "forward(mixed_qkv)", + "forward(a)", + "forward(b)", + ), + "srt/layers/attention/hybrid_linear_attn_backend.py": ("linear_attn_backend",), + "srt/model_executor/forward_context.py": ("get_attn_backend",), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + "srt/models/qwen3_5.py:Qwen3_5GatedDeltaNet.forward": "a5e9e3bd054ff18f", +} + + +class Bound(msgspec.Struct, frozen=True): + """The GDN package's build result (``st.bound["gdn"]``).""" + + gdn_verify: bool = False # slot gdn_verify serves every GDN layer + gdn_pdl: bool = False # and launches its verify dependent on the conv + reason: str = "" # why a requested slot is not built + + +def build(model, ctx) -> Bound: + """Slot ``gdn_verify`` for every GatedDeltaNet layer, or why a layer cannot take it.""" + from sglang.srt.models.qwen3_5 import Qwen3_5LinearDecoderLayer + + from sgl_nvidia_mega.qwen3_5.gdn import verify + from sgl_nvidia_mega.qwen3_5.runtime import SLOTS + + if "gdn_verify" not in SLOTS: + return Bound(reason="stock (SGLANG_NVIDIA_MEGAKERNEL_SLOTS)") + m = model.model + for i in range(m.start_layer, m.end_layer): + layer = m.layers[i] + if isinstance(layer, Qwen3_5LinearDecoderLayer): + reason = verify.unsupported(layer.linear_attn) + if reason: + return Bound(reason=f"layer {i}: {reason}") + return Bound(gdn_verify=True, gdn_pdl="gdn_pdl" in SLOTS) + + +def step(layer, x, st): + """``layer``'s GatedDeltaNet on the normed input ``x`` [rows, hidden]: the + TP-partial output [rows, hidden].""" + if st.slots.gdn_verify: + from sgl_nvidia_mega.qwen3_5.gdn import verify + + opts = verify.SLOT_PDL if st.slots.gdn_pdl else verify.SLOT + return verify.step(layer.linear_attn, x, st, opts) + import triton + from sglang.kernels.ops.attention.triton_gdn_fused_proj import ( + fused_qkvzba_split_reshape_cat_contiguous, + ) + from sglang.srt.model_executor.forward_context import get_attn_backend + + gdn = layer.linear_attn + qkvz, ba = gdn._forward_input_proj(x) + backend = get_attn_backend() + backend = getattr(backend, "linear_attn_backend", backend) + if getattr(backend, "supports_packed_verify", False): + # SGLang folds the split and conv into such a backend; this step does not + raise NotImplementedError("a linear-attention backend with packed verify") + mixed_qkv, z, b, a = fused_qkvzba_split_reshape_cat_contiguous( + qkvz, + ba, + triton.cdiv(gdn.num_k_heads, gdn.attn_tp_size), + triton.cdiv(gdn.num_v_heads, gdn.attn_tp_size), + gdn.head_k_dim, + gdn.head_v_dim, + ) + core = gdn.attn(st.fb, mixed_qkv=mixed_qkv, a=a, b=b) + z_shape = z.shape + core = core.reshape(-1, core.shape[-1]) + z = z.reshape(-1, z.shape[-1]) + core = gdn.norm(core, z) + core = core.reshape(z_shape) + core = core.reshape(*core.shape[:-2], core.shape[-2] * core.shape[-1]) + out, _ = gdn.out_proj(core) + return out diff --git a/sgl_nvidia_mega/qwen3_5/gdn/conv.py b/sgl_nvidia_mega/qwen3_5/gdn/conv.py new file mode 100644 index 00000000..1aeeea23 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/gdn/conv.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright (c) 2024, Tri Dao. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# This file has been modified from its original version (the channel tile and a token-major output). +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/kernels/ops/mamba/causal_conv1d_triton.py (causal_conv1d_update). +"""The causal conv update of a GDN target-verify step on SGLang's Triton kernel, with +the two launch choices ``causal_conv1d_update`` fixes: the channel tile (``block_n``) +and a token-major output, which the token-major projection views need (their rows +have gaps, so ``empty_like`` would make the output channel-major). + +Only the verify case: no circular cache_seqlens, no accept counts, no tree mask. +""" + +from __future__ import annotations + +import torch + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "kernels/ops/mamba/causal_conv1d_triton.py": ( + "_causal_conv1d_update_kernel", + "PAD_SLOT_ID", + ), + "kernels/jit/utils/__init__.py": ("is_arch_support_pdl",), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + "kernels/ops/mamba/causal_conv1d_triton.py:causal_conv1d_update": "3394efa20c88d580", +} + + +def conv_update( + x: torch.Tensor, + conv_state: torch.Tensor, + weight: torch.Tensor, + bias, + activation, + conv_state_indices: torch.Tensor, + intermediate_conv_window: torch.Tensor, + intermediate_state_indices: torch.Tensor, + *, + block_n: int = 256, + token_major_out: bool = False, +) -> torch.Tensor: + """``causal_conv1d_update`` of x (batch, dim, seqlen) for a verify step, with the + channel tile ``block_n`` and, with ``token_major_out``, an output whose + transpose(1, 2) is contiguous.""" + import triton + from sglang.kernels.jit.utils import is_arch_support_pdl + from sglang.kernels.ops.mamba.causal_conv1d_triton import ( + PAD_SLOT_ID, + _causal_conv1d_update_kernel, + ) + + if isinstance(activation, bool): + activation = "silu" if activation is True else None + batch, dim, seqlen = x.shape + _, width = weight.shape + num_cache_lines, _, state_len = conv_state.size() + out = ( + torch.empty((batch, seqlen, dim), dtype=x.dtype, device=x.device).transpose( + 1, 2 + ) + if token_major_out + else torch.empty_like(x) + ) + state_len = width - 1 + pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {} + + def grid(meta): + return (batch, triton.cdiv(dim, meta["BLOCK_N"])) + + _causal_conv1d_update_kernel[grid]( + x, + weight, + bias, + conv_state, + None, + conv_state_indices, + None, + intermediate_conv_window, + intermediate_state_indices, + None, + None, + None, + out, + batch, + dim, + seqlen, + state_len, + num_cache_lines, + *x.stride(), + *weight.stride(), + *conv_state.stride(), + conv_state_indices.stride(0), + *intermediate_conv_window.stride(), + intermediate_state_indices.stride(0), + 0, + 0, + 0, + 0, + 0, + 0, + *out.stride(), + PAD_SLOT_ID, + HAS_BIAS=bias is not None, + KERNEL_WIDTH=width, + SILU_ACTIVATION=activation in ["silu", "swish"], + IS_CONTINUOUS_BATCHING=True, + IS_SPEC_DECODING=False, + NP2_STATELEN=triton.next_power_of_2(state_len), + NP2_SEQLEN=triton.next_power_of_2(seqlen), + USE_PAD_SLOT=True, + BLOCK_N=block_n, + SAVE_INTERMEDIATE=True, + HAS_EAGLE_TREE_CUSTOM_ATTN_MASK=False, + **pdl_kwargs, + ) + return out diff --git a/sgl_nvidia_mega/qwen3_5/gdn/norm_fp8.py b/sgl_nvidia_mega/qwen3_5/gdn/norm_fp8.py new file mode 100644 index 00000000..6891ef2d --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/gdn/norm_fp8.py @@ -0,0 +1,169 @@ +# SPDX-License-Identifier: Apache-2.0 AND MIT +# SPDX-FileCopyrightText: Copyright (c) 2024, Tri Dao. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# This file has been modified from its original version (a static FP8 store). +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/kernels/ops/attention/fla/layernorm_gated.py (_layer_norm_fwd_1pass_kernel, +# _layer_norm_fwd), itself adapted from flash-linear-attention (MIT, Tri Dao). +"""The gated RMSNorm of a GDN verify step writing static FP8 for the FP8 ``out_proj``: +SGLang's kernel and launch, plus a store that rounds each output to the activation +dtype, scales it by the linear's static input scale and casts it to e4m3, as the +linear's own input quantization would. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "kernels/ops/attention/fla/layernorm_gated.py": ("calc_rows_per_block",), + "kernels/jit/utils/__init__.py": ("is_arch_support_pdl",), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + "kernels/ops/attention/fla/layernorm_gated.py:_layer_norm_fwd_1pass_kernel": "80c8989ca591478d", + "kernels/ops/attention/fla/layernorm_gated.py:_layer_norm_fwd": "01a3c1896f273538", +} + + +@triton.jit +def _layer_norm_fwd_1pass_kernel( + X, # pointer to the input + Q, # pointer to the FP8 output + W, # pointer to the weights + Z, # pointer to the other branch + S, # pointer to the static input scale of the consumer + Rstd, # pointer to the 1/std + stride_x_row, + stride_q_row, + stride_z_row, + stride_z_token, + stride_z_head, + M, # number of rows in X + N: tl.constexpr, # number of columns in X + eps, + BLOCK_N: tl.constexpr, + ROWS_PER_BLOCK: tl.constexpr, + Z_IS_3D: tl.constexpr, + Z_HEADS: tl.constexpr, + NORM_BEFORE_GATE: tl.constexpr, + ACTIVATION: tl.constexpr, + OUT_DTYPE: tl.constexpr, # the activation dtype each output is rounded to first + USE_GDC: tl.constexpr = False, +): + if USE_GDC: + tl.extra.cuda.gdc_wait() + + row_start = tl.program_id(0) * ROWS_PER_BLOCK + group = tl.program_id(1) + rows = row_start + tl.arange(0, ROWS_PER_BLOCK) + cols = tl.arange(0, BLOCK_N) + row_offsets = rows[:, None] * stride_x_row + col_offsets = cols[None, :] + group * N + X_base = X + row_offsets + col_offsets + row_mask = rows[:, None] < M + col_mask = cols[None, :] < N + mask = row_mask & col_mask + x = tl.load(X_base, mask=mask, other=0.0).to(tl.float32) + + if Z_IS_3D: + z_row_offsets = (rows[:, None] // Z_HEADS) * stride_z_token + ( + rows[:, None] % Z_HEADS + ) * stride_z_head + else: + z_row_offsets = rows[:, None] * stride_z_row + if not NORM_BEFORE_GATE: + z = tl.load(Z + z_row_offsets + col_offsets, mask=mask, other=0.0).to( + tl.float32 + ) + if ACTIVATION == "swish" or ACTIVATION == "silu": + x *= z * tl.sigmoid(z) + elif ACTIVATION == "sigmoid": + x *= tl.sigmoid(z) + + # RMS norm + xbar = tl.where(mask, x, 0.0) + var = tl.sum(xbar * xbar, axis=1) / N + rstd = tl.rsqrt(var + eps) + rstd_offsets = group * M + rows + tl.store(Rstd + rstd_offsets, rstd, mask=rows < M) + + w_offsets = cols + group * N + w = tl.load(W + w_offsets, mask=cols < N, other=0.0).to(tl.float32) + x_hat = x * rstd[:, None] + y = x_hat * w[None, :] + + if NORM_BEFORE_GATE: + z = tl.load(Z + z_row_offsets + col_offsets, mask=mask, other=0.0).to( + tl.float32 + ) + if ACTIVATION == "swish" or ACTIVATION == "silu": + y *= z * tl.sigmoid(z) + elif ACTIVATION == "sigmoid": + y *= tl.sigmoid(z) + + # the consumer's static per-tensor quantization of the rounded activation + y = y.to(OUT_DTYPE).to(tl.float32) + y_s = tl.load(S).to(tl.float32) + y_q = tl.clamp(y * (1.0 / y_s), -448.0, 448.0).to(Q.dtype.element_ty) + tl.store(Q + rows[:, None] * stride_q_row + col_offsets, y_q, mask=mask) + + if USE_GDC: + tl.extra.cuda.gdc_launch_dependents() + + +def rms_norm_gated_fp8( + x: torch.Tensor, + weight: torch.Tensor, + eps: float, + z: torch.Tensor, + scale: torch.Tensor, + *, + norm_before_gate: bool = True, + activation: str = "swish", +) -> torch.Tensor: + """e4m3 of RMSNorm(x) * act(z) [M, N] (rounded to x's dtype first) over the static + input ``scale`` [1] of the linear it feeds; z [M, N] or [tokens, heads, N].""" + from sglang.kernels.jit.utils import is_arch_support_pdl + from sglang.kernels.ops.attention.fla.layernorm_gated import calc_rows_per_block + + M, N = x.shape + z_is_3d = z.ndim == 3 + q = torch.empty((M, N), dtype=torch.float8_e4m3fn, device=x.device) + rstd = torch.empty((M,), dtype=torch.float32, device=x.device) + block_n = min(65536 // x.element_size(), triton.next_power_of_2(N)) + num_warps = min(max(block_n // 256, 1), 8) + rows_per_block = calc_rows_per_block(M, x.device) + pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {} + out_dtype = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16}[x.dtype] + _layer_norm_fwd_1pass_kernel[(triton.cdiv(M, rows_per_block), 1)]( + x, + q, + weight, + z, + scale, + rstd, + x.stride(0), + q.stride(0), + 0 if z_is_3d else z.stride(0), + z.stride(0) if z_is_3d else 0, + z.stride(1) if z_is_3d else 0, + M, + N, + eps, + BLOCK_N=block_n, + ROWS_PER_BLOCK=rows_per_block, + Z_IS_3D=z_is_3d, + Z_HEADS=z.shape[1] if z_is_3d else 1, + NORM_BEFORE_GATE=norm_before_gate, + ACTIVATION=activation, + OUT_DTYPE=out_dtype, + num_warps=num_warps, + **pdl_kwargs, + ) + return q diff --git a/sgl_nvidia_mega/qwen3_5/gdn/replayssm.py b/sgl_nvidia_mega/qwen3_5/gdn/replayssm.py new file mode 100644 index 00000000..78bc0405 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/gdn/replayssm.py @@ -0,0 +1,603 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# This file has been modified from its original version (a programmatic dependent launch). +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/kernels/ops/attention/linear/gdn_replayssm_spec_decode.py +# (gdn_replayssm_spec_circular_kernel, _launch_gdn_spec, gdn_replayssm_spec_decode), +# itself ported from the vLLM ReplaySSM reference implementation (Apache-2.0). +"""The ReplaySSM verify of a GDN target-verify step (the circular ring's non-flush +rows), SGLang's kernel and launch with one addition: a programmatic dependent launch. +The grid starts while the causal conv before it finishes, reads the gates and the +committed history, and waits for the conv's q / k / v only where it first reads them +(PTX ``griddepcontrol``). The gates and the history must be ready when it starts: +they come from launches before the conv. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = {"kernels/ops/attention/fla/utils.py": ("is_tf32_supported",)} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + # In fla/ before sgl-project/sglang#45112 regrouped SGLang's kernels and in linear/ + # after; the wildcard goes once all supported SGLangs have the move + "kernels/ops/attention/*/gdn_replayssm_spec_decode.py:gdn_replayssm_spec_circular_kernel": "9ad1193ce666552d", + "kernels/ops/attention/*/gdn_replayssm_spec_decode.py:_launch_gdn_spec": "8bf89daf973334cc", + "kernels/ops/attention/*/gdn_replayssm_spec_decode.py:gdn_replayssm_spec_decode": "c3fcd760148facd7", +} + + +@triton.jit +def gdn_replayssm_spec_circular_kernel( + q, # [total_tokens, H, K] + k, # [total_tokens, H, K] + v, # [total_tokens, HV, V] + a, # [total_tokens, HV] + b, # [total_tokens, HV] + A_log, # [HV] fp32 + dt_bias, # [HV] fp32 + o, # [total_tokens, HV, V] preallocated output + h0, # [num_slots, HV, V, K] checkpoint (read-only in verify) + d_cache, # [num_slots, HV, L, V] chunked deltas (output reconstruction only) + k_cache, # [num_slots, H, L, K] L2-normalized keys (output reconstruction only) + g_cache, # [num_slots, HV, L] fp32 + rawv_cache, # exact raw v or compact D low part + rawk_cache, # exact raw k or normalized-K low part + beta_cache, # fp32 beta for exact replay + query_start_loc, # [B+1] int packed cu_seqlens + ssm_state_indices, # [B] int physical block per request + replay_indices, # [B] int active request slot per request + write_pos, # [num_replay_slots] int32 + cache_base, # [num_replay_slots] int32 + is_flush_flags, # [num_replay_slots] int8 + scale, + stride_q_t: tl.constexpr, # per-token stride of q (= H*K) + stride_k_t: tl.constexpr, # per-token stride of k (= H*K) + stride_v_t: tl.constexpr, # per-token stride of v (= HV*V) + stride_a_t: tl.constexpr, + stride_b_t: tl.constexpr, + stride_o_t: tl.constexpr, # per-token stride of o (= HV*V) + stride_state_slot: tl.constexpr, + stride_d_slot: tl.constexpr, + stride_k_slot: tl.constexpr, + stride_g_slot: tl.constexpr, + stride_rawv_slot: tl.constexpr, + stride_rawk_slot: tl.constexpr, + stride_beta_slot: tl.constexpr, + stride_qsl: tl.constexpr, + stride_indices: tl.constexpr, + stride_replay_indices: tl.constexpr, + H: tl.constexpr, + HV: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + BS: tl.constexpr, + BC: tl.constexpr, + NK: tl.constexpr, + BKT: tl.constexpr, + MAX_CACHE_LEN: tl.constexpr, + SOFTPLUS_THRESHOLD: tl.constexpr, + USE_QK_L2NORM_IN_KERNEL: tl.constexpr, + STATE_FP32: tl.constexpr, + STORE_EXACT_RAW: tl.constexpr, + STORE_D_RESIDUAL: tl.constexpr, + STORE_K_RESIDUAL: tl.constexpr, + IS_FLUSH: tl.constexpr, + NULL_BLOCK_ID: tl.constexpr, + DOT_PRECISION: tl.constexpr, + USE_GDC: tl.constexpr = False, +): + i_v = tl.program_id(0) + i_n = tl.program_id(1) + i_hv = tl.program_id(2) + i_h = i_hv // (HV // H) + if USE_GDC: + tl.extra.cuda.gdc_launch_dependents() + + o_v = i_v * BV + tl.arange(0, BV) + o_s = tl.arange(0, BS) + o_c = tl.arange(0, BC) + mask_v = o_v < V + + # --- per-request packed window --- + bos = tl.load(query_start_loc + i_n * stride_qsl).to(tl.int64) + eos = tl.load(query_start_loc + (i_n + 1) * stride_qsl).to(tl.int64) + spec_len = eos - bos # full window length + # Clamp the token index for padding lanes (o_s >= spec_len, which exist when + # BS = max(bs_min, npow2(max_spec_len)) > spec_len) so masked loads/stores + # never form out-of-bounds token addresses into the packed varlen buffers. + # mask_s still zeroes the *values* of those lanes, so this is numerically a + # no-op -- it only keeps the computed pointers in-bounds. + mask_s = o_s < spec_len + o_s_safe = tl.where(mask_s, o_s, 0) + + state_idx = tl.load(ssm_state_indices + i_n * stride_indices).to(tl.int64) + replay_idx = tl.load(replay_indices + i_n * stride_replay_indices).to(tl.int64) + + # output pointer (packed): token (bos + o_s), value-head i_hv, dim o_v + p_o = o + (bos + o_s_safe[:, None]) * stride_o_t + i_hv * V + o_v[None, :] + + if IS_FLUSH: + if state_idx <= NULL_BLOCK_ID: + if USE_GDC: + tl.extra.cuda.gdc_wait() + return + b_is_flush = tl.load(is_flush_flags + replay_idx) != 0 + if not b_is_flush: + if USE_GDC: + tl.extra.cuda.gdc_wait() + return + else: + if state_idx <= NULL_BLOCK_ID: + if USE_GDC: + tl.extra.cuda.gdc_wait() + full_mask = (o_s < spec_len)[:, None] & mask_v[None, :] + tl.store( + p_o, + tl.zeros([BS, BV], dtype=tl.float32).to(p_o.dtype.element_ty), + mask=full_mask, + ) + return + b_is_flush = tl.load(is_flush_flags + replay_idx) != 0 + if b_is_flush: + if USE_GDC: + tl.extra.cuda.gdc_wait() + return + + b_write_pos = tl.load(write_pos + replay_idx).to(tl.int64) + b_cache_base = tl.load(cache_base + replay_idx).to(tl.int32) + + out_mask = mask_s[:, None] & mask_v[None, :] + + b_wp_i = b_write_pos.to(tl.int32) + cache_valid = o_c < b_write_pos + + # CIRCULAR physical slots (addresses only; masks/cumsums stay logical). + phys_c = (b_cache_base + o_c) & (MAX_CACHE_LEN - 1) # [BC] history + phys_spec = (b_cache_base + b_wp_i + o_s) & (MAX_CACHE_LEN - 1) # [BS] spec + + # ------------------------------------------------------------------ + # Block 0: gates / beta / local cumsum + committed-history replay decay. + # ------------------------------------------------------------------ + A_log_val = tl.load(A_log + i_hv).to(tl.float32) + dt_bias_val = tl.load(dt_bias + i_hv).to(tl.float32) + a_s = tl.load(a + (bos + o_s_safe) * stride_a_t + i_hv, mask=mask_s, other=0.0).to( + tl.float32 + ) + b_s = tl.load(b + (bos + o_s_safe) * stride_b_t + i_hv, mask=mask_s, other=0.0).to( + tl.float32 + ) + x = a_s + dt_bias_val + softplus_x = tl.where(x <= SOFTPLUS_THRESHOLD, tl.log(1.0 + tl.exp(x)), x) + g_s = tl.where(mask_s, -tl.exp(A_log_val) * softplus_x, 0.0) + # Explicit sigmoid expression (NOT tl.sigmoid): bitwise clone of + # fused_sigmoid_gating_delta_rule_update_kernel's beta so the fp32 value + # stored to beta_cache is exactly what the recurrent baseline would compute. + beta_s = tl.where(mask_s, 1.0 / (1.0 + tl.exp(-b_s)), 0.0) + G_s = tl.cumsum(g_s, axis=0) + expG_s = tl.exp(G_s) + + if not IS_FLUSH: + # Committed-history replay decay from cached g (history loads -> phys_c). + # Output reconstruction only. On flush steps the history is folded into + # the checkpoint by the exact-fold kernel (launched first), so none of + # this is needed there. + p_g_main = g_cache + replay_idx * stride_g_slot + i_hv * MAX_CACHE_LEN + phys_c + b_g_all = tl.load(p_g_main, mask=cache_valid, other=0.0).to(tl.float32) + b_g_prefix = tl.cumsum(b_g_all, axis=0) + b_g_total = tl.sum(b_g_all, axis=0) + b_replay_decay = tl.where(cache_valid, tl.exp(b_g_total - b_g_prefix), 0.0) + b_total_decay = tl.exp(b_g_total) + + p_d_main = d_cache + ( + replay_idx * stride_d_slot + + (i_hv * MAX_CACHE_LEN + phys_c[None, :]) * V + + o_v[:, None] + ) + b_d_all = tl.load( + p_d_main, mask=mask_v[:, None] & cache_valid[None, :], other=0.0 + ).to(tl.float32) + b_d_scaled_fp32 = b_d_all * b_replay_decay[None, :] + if STATE_FP32: + # Preserve the pre-existing fp32-checkpoint path: these dots use + # fp32 operands with DOT_PRECISION (TF32 by default). + b_d_scaled = b_d_scaled_fp32 + else: + b_d_scaled = b_d_scaled_fp32.to(d_cache.dtype.element_ty) + + # PTX griddepcontrol: consume the convolution's q / k / v only after it completes + if USE_GDC: + tl.extra.cuda.gdc_wait() + + if USE_QK_L2NORM_IN_KERNEL: + qnorm_acc = tl.zeros([BS], dtype=tl.float32) + knorm_acc = tl.zeros([BS], dtype=tl.float32) + for kk in range(NK): + o_kt = kk * BKT + tl.arange(0, BKT) + mask_kt = o_kt < K + ld = mask_s[:, None] & mask_kt[None, :] + qn = tl.load( + q + (bos + o_s_safe[:, None]) * stride_q_t + i_h * K + o_kt[None, :], + mask=ld, + other=0.0, + ).to(tl.float32) + knn = tl.load( + k + (bos + o_s_safe[:, None]) * stride_k_t + i_h * K + o_kt[None, :], + mask=ld, + other=0.0, + ).to(tl.float32) + qnorm_acc += tl.sum(qn * qn, axis=1) + knorm_acc += tl.sum(knn * knn, axis=1) + q_rnorm = tl.where(mask_s, 1.0 / tl.sqrt(qnorm_acc + 1e-6), 0.0) + k_rnorm = tl.where(mask_s, 1.0 / tl.sqrt(knorm_acc + 1e-6), 0.0) + else: + q_rnorm = tl.where(mask_s, 1.0, 0.0) + k_rnorm = tl.where(mask_s, 1.0, 0.0) + + # ------------------------------------------------------------------ + # K-Tiled Fused Projection and Intra-Spec Matrices (+ flush) + # ------------------------------------------------------------------ + hw_q = tl.zeros([BV, BS], dtype=tl.float32) + hw_k = tl.zeros([BV, BS], dtype=tl.float32) + if not IS_FLUSH: + scores_q = tl.zeros([BC, BS], dtype=tl.float32) + scores_k = tl.zeros([BC, BS], dtype=tl.float32) + kk_mat = tl.zeros([BS, BS], dtype=tl.float32) + kq_mat = tl.zeros([BS, BS], dtype=tl.float32) + + write_k = (i_v == 0) and (i_hv == i_h * (HV // H)) + + for kk in range(NK): + o_kt = kk * BKT + tl.arange(0, BKT) + mask_kt = o_kt < K + # NOTE: do NOT mask the token (o_s) dimension on these loads. With the + # o_s_safe clamp the padding-row addresses are in-bounds (they read + # token `bos`), and `q_rnorm`/`k_rnorm` are already 0 on padding rows + # (line above), so the multiply below zeroes them. Masking the token + # dimension here instead leaves the tl.dot shared-memory staging of the + # padding rows uninitialised -- on this Triton/MMA path that stale + # staging leaks into the *valid* output columns (manifests at + # spec_len=2). Loading the full (token-unmasked) tile keeps staging + # fully written; the result is numerically identical. + q_tile = tl.load( + q + (bos + o_s_safe[:, None]) * stride_q_t + i_h * K + o_kt[None, :], + mask=mask_kt[None, :], + other=0.0, + ).to(tl.float32) + k_tile = tl.load( + k + (bos + o_s_safe[:, None]) * stride_k_t + i_h * K + o_kt[None, :], + mask=mask_kt[None, :], + other=0.0, + ).to(tl.float32) + k_raw_tile = k_tile + q_precise = q_tile * (q_rnorm * scale)[:, None] + k_precise = k_tile * k_rnorm[:, None] + k_store_tile = k_precise.to(k_cache.dtype.element_ty) + k_residual_tile = (k_precise - k_store_tile.to(tl.float32)).to( + k.dtype.element_ty + ) + if STATE_FP32: + # Do not lower the fp32-checkpoint output path to activation dtype. + q_tile = q_precise + k_tile = k_precise + else: + q_tile = q_precise.to(q.dtype.element_ty) + k_tile = k_store_tile + + p_h0 = ( + h0 + + state_idx * stride_state_slot + + i_hv * V * K + + o_v[:, None] * K + + o_kt[None, :] + ) + sc_tile = tl.load(p_h0, mask=mask_v[:, None] & mask_kt[None, :], other=0.0) + + qT = tl.trans(q_tile) + kT = tl.trans(k_tile) + kk_mat += tl.dot(k_tile, kT, input_precision=DOT_PRECISION) + kq_mat += tl.dot(k_tile, qT, input_precision=DOT_PRECISION) + if STORE_K_RESIDUAL and not STATE_FP32: + k_residual_T = tl.trans(k_residual_tile) + kk_mat += tl.dot(k_tile, k_residual_T, input_precision=DOT_PRECISION) + kk_mat += tl.dot(k_residual_tile, kT, input_precision=DOT_PRECISION) + kk_mat += tl.dot( + k_residual_tile, k_residual_T, input_precision=DOT_PRECISION + ) + kq_mat += tl.dot(k_residual_tile, qT, input_precision=DOT_PRECISION) + + # Checkpoint projection. On flush steps the exact-fold kernel (launched + # first) has already folded the committed history into h0, so this is + # the whole non-window contribution and no d-fold happens here anymore. + if STATE_FP32: + sc_tile = sc_tile.to(tl.float32) + hw_q += tl.dot(sc_tile, qT, input_precision=DOT_PRECISION) + hw_k += tl.dot(sc_tile, kT, input_precision=DOT_PRECISION) + else: + sc_tile = sc_tile.to(k_tile.dtype) + hw_q += tl.dot(sc_tile, qT) + hw_k += tl.dot(sc_tile, kT) + if STORE_K_RESIDUAL: + hw_k += tl.dot(sc_tile, k_residual_T) + + if not IS_FLUSH: + # cached-key history load -> phys_c (output reconstruction only) + p_k = k_cache + ( + replay_idx * stride_k_slot + + (i_h * MAX_CACHE_LEN + phys_c[:, None]) * K + + o_kt[None, :] + ) + khist_tile = tl.load( + p_k, mask=cache_valid[:, None] & mask_kt[None, :], other=0.0 + ) + if STATE_FP32: + khist_tile = khist_tile.to(tl.float32) + else: + khist_tile = khist_tile.to(k_cache.dtype.element_ty) + scores_q += tl.dot(khist_tile, qT, input_precision=DOT_PRECISION) + scores_k += tl.dot(khist_tile, kT, input_precision=DOT_PRECISION) + + if write_k: + spec_kt_mask = ( + mask_s[:, None] + & mask_kt[None, :] + & ((b_write_pos + o_s[:, None]) < MAX_CACHE_LEN) + ) + if STORE_EXACT_RAW or STORE_K_RESIDUAL: + p_cur_rawk = rawk_cache + ( + replay_idx * stride_rawk_slot + + (i_h * MAX_CACHE_LEN + phys_spec[:, None]) * K + + o_kt[None, :] + ) + if STORE_EXACT_RAW: + tl.store( + p_cur_rawk, + k_raw_tile.to(p_cur_rawk.dtype.element_ty), + mask=spec_kt_mask, + ) + else: + tl.store(p_cur_rawk, k_residual_tile, mask=spec_kt_mask) + # normalized spec key store -> phys_spec (circular) + p_cur_k = k_cache + ( + replay_idx * stride_k_slot + + (i_h * MAX_CACHE_LEN + phys_spec[:, None]) * K + + o_kt[None, :] + ) + tl.store( + p_cur_k, + k_store_tile, + mask=spec_kt_mask, + ) + + if not IS_FLUSH: + hw_q = b_total_decay * hw_q + tl.dot( + b_d_scaled, scores_q.to(b_d_scaled.dtype), input_precision=DOT_PRECISION + ) + hw_k = b_total_decay * hw_k + tl.dot( + b_d_scaled, scores_k.to(b_d_scaled.dtype), input_precision=DOT_PRECISION + ) + + # ------------------------------------------------------------------ + # strictly-lower A and T = (I + A)^{-1}. + # ------------------------------------------------------------------ + lower = (o_s[:, None] > o_s[None, :]) & mask_s[:, None] & mask_s[None, :] + diff_ij = G_s[:, None] - G_s[None, :] + A_mat = tl.where(lower, beta_s[:, None] * tl.exp(diff_ij) * kk_mat, 0.0) + + b_Ai = -A_mat + for ii in range(2, BS): + row = tl.sum(tl.where((o_s == ii)[:, None], -A_mat, 0.0), axis=0) + row = tl.where(o_s < ii, row, 0.0) + row = row + tl.sum(row[:, None] * b_Ai, axis=0) + b_Ai = tl.where((o_s == ii)[:, None], row, b_Ai) + T_mat = b_Ai + (o_s[:, None] == o_s[None, :]).to(tl.float32) + + # ------------------------------------------------------------------ + # R and D_spec = R @ T^T. + # ------------------------------------------------------------------ + p_v = v + (bos + o_s_safe[None, :]) * stride_v_t + i_hv * V + o_v[:, None] + v_tile = tl.load(p_v, mask=mask_v[:, None] & mask_s[None, :], other=0.0).to( + tl.float32 + ) + R_mat = beta_s[None, :] * (v_tile - expG_s[None, :] * hw_k) + D_spec = tl.zeros([BV, BS], dtype=tl.float32) + for j in tl.static_range(BS): + Rj = tl.sum(tl.where((o_s == j)[None, :], R_mat, 0.0), axis=1) + Tj = tl.sum(tl.where((o_s == j)[None, :], T_mat, 0.0), axis=1) + D_spec += Rj[:, None] * Tj[None, :] + + # ------------------------------------------------------------------ + # outputs. + # ------------------------------------------------------------------ + causalF = (o_s[:, None] <= o_s[None, :]) & mask_s[:, None] & mask_s[None, :] + diff_ji = G_s[None, :] - G_s[:, None] + F_mat = tl.where(causalF, tl.exp(diff_ji) * kq_mat, 0.0) + DF = tl.zeros([BV, BS], dtype=tl.float32) + for j in tl.static_range(BS): + Dj = tl.sum(tl.where((o_s == j)[None, :], D_spec, 0.0), axis=1) + Fj = tl.sum(tl.where((o_s == j)[:, None], F_mat, 0.0), axis=0) + DF += Dj[:, None] * Fj[None, :] + O_tile = expG_s[None, :] * hw_q + DF + + tl.store(p_o, tl.trans(O_tile).to(p_o.dtype.element_ty), mask=out_mask) + + # ------------------------------------------------------------------ + # write speculative d / raw-v / g / beta at circular positions (phys_spec). + # ------------------------------------------------------------------ + spec_pos = b_write_pos + o_s + spec_store_mask = mask_s & (spec_pos < MAX_CACHE_LEN) + p_cur_d = d_cache + ( + replay_idx * stride_d_slot + + (i_hv * MAX_CACHE_LEN + phys_spec[None, :]) * V + + o_v[:, None] + ) + d_hi = D_spec.to(p_cur_d.dtype.element_ty) + tl.store( + p_cur_d, + d_hi, + mask=mask_v[:, None] & spec_store_mask[None, :], + ) + if STORE_EXACT_RAW or STORE_D_RESIDUAL: + p_cur_v = rawv_cache + ( + replay_idx * stride_rawv_slot + + (i_hv * MAX_CACHE_LEN + phys_spec[None, :]) * V + + o_v[:, None] + ) + if STORE_EXACT_RAW: + tl.store( + p_cur_v, + v_tile.to(p_cur_v.dtype.element_ty), + mask=mask_v[:, None] & spec_store_mask[None, :], + ) + else: + tl.store( + p_cur_v, + (D_spec - d_hi.to(tl.float32)).to(p_cur_v.dtype.element_ty), + mask=mask_v[:, None] & spec_store_mask[None, :], + ) + if i_v == 0: + p_cur_g = ( + g_cache + replay_idx * stride_g_slot + i_hv * MAX_CACHE_LEN + phys_spec + ) + tl.store(p_cur_g, g_s, mask=spec_store_mask) + if STORE_EXACT_RAW: + p_cur_beta = ( + beta_cache + + replay_idx * stride_beta_slot + + i_hv * MAX_CACHE_LEN + + phys_spec + ) + tl.store(p_cur_beta, beta_s, mask=spec_store_mask) + + +def verify( + q, + k, + v, + a, + b, + A_log, + dt_bias, + out, + checkpoint_state, + d_cache, + k_cache, + g_cache, + rawv_cache, + rawk_cache, + beta_cache, + query_start_loc, + ssm_state_indices, + replay_indices, + write_pos, + cache_base, + is_flush, + *, + max_cache_len: int, + max_spec_len: int, + scale: float, + null_block_id: int, + block_v: int, + num_warps: int, + num_stages: int = 2, + nk: int = 2, + bs_min: int = 4, + pdl: bool = False, +) -> torch.Tensor: + """``gdn_replayssm_spec_decode(launch_mode="verify", use_qk_l2norm_in_kernel=True)`` + with an optional dependent launch (``pdl``).""" + from sglang.kernels.ops.attention.fla.utils import is_tf32_supported + + if is_flush.dtype != torch.int8: + is_flush = is_flush.to(torch.int8) + num_slots, HV, V, K = checkpoint_state.shape + H = k.shape[1] + B = query_start_loc.shape[0] - 1 + BK = triton.next_power_of_2(K) + BKT = BK // nk + BV = block_v + BS = max(bs_min, triton.next_power_of_2(max_spec_len)) + BC = max(16, triton.next_power_of_2(max_cache_len)) + store_exact_raw = ( + rawv_cache is not None and rawk_cache is not None and beta_cache is not None + ) + store_residual = ( + rawv_cache is not None and rawk_cache is not None and beta_cache is None + ) + rawv_buf = rawv_cache if rawv_cache is not None else d_cache + rawk_buf = rawk_cache if rawk_cache is not None else k_cache + beta_buf = beta_cache if beta_cache is not None else g_cache + pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if pdl else {} + gdn_replayssm_spec_circular_kernel[(triton.cdiv(V, BV), B, HV)]( + q, + k, + v, + a, + b, + A_log, + dt_bias, + out, + checkpoint_state, + d_cache, + k_cache, + g_cache, + rawv_buf, + rawk_buf, + beta_buf, + query_start_loc, + ssm_state_indices, + replay_indices, + write_pos, + cache_base, + is_flush, + scale, + q.stride(0), + k.stride(0), + v.stride(0), + a.stride(0), + b.stride(0), + out.stride(0), + checkpoint_state.stride(0), + d_cache.stride(0), + k_cache.stride(0), + g_cache.stride(0), + rawv_buf.stride(0), + rawk_buf.stride(0), + beta_buf.stride(0), + query_start_loc.stride(0), + ssm_state_indices.stride(0), + replay_indices.stride(0), + H=H, + HV=HV, + K=K, + V=V, + BK=BK, + BV=BV, + BS=BS, + BC=BC, + NK=nk, + BKT=BKT, + MAX_CACHE_LEN=max_cache_len, + SOFTPLUS_THRESHOLD=20.0, + USE_QK_L2NORM_IN_KERNEL=True, + STATE_FP32=checkpoint_state.dtype == torch.float32, + STORE_EXACT_RAW=store_exact_raw, + STORE_D_RESIDUAL=store_residual, + STORE_K_RESIDUAL=store_residual, + IS_FLUSH=False, + NULL_BLOCK_ID=null_block_id, + DOT_PRECISION="tf32" if is_tf32_supported else "ieee", + num_warps=num_warps, + num_stages=num_stages, + **pdl_kwargs, + ) + return out diff --git a/sgl_nvidia_mega/qwen3_5/gdn/verify.py b/sgl_nvidia_mega/qwen3_5/gdn/verify.py new file mode 100644 index 00000000..6f8a0983 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/gdn/verify.py @@ -0,0 +1,347 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/srt/models/qwen3_5.py (Qwen3_5GatedDeltaNet.forward), +# python/sglang/srt/layers/attention/linear/gdn_backend.py (GDNAttnBackend.forward_extend, +# GDNAttnBackend._replayssm_target_verify), python/sglang/srt/layers/linear.py +# (RowParallelLinear.forward) and python/sglang/srt/layers/quantization/fp8_utils.py +# (apply_fp8_linear_bmm_flashinfer). +"""Slot ``gdn_verify``: a GatedDeltaNet target-verify step with SGLang's chain opened up, +so its launches take the choices SGLang fixes. The chain is the backend's +target-verify arm for the circular ReplaySSM ring (the causal conv update, the strided +q / k / v split, the ReplaySSM verify), reading the step's GDN metadata and the mamba +pools from the forward's view (``Step.view.gdn`` and ``.mamba``) and its request slots +from the forward batch, then the gated norm and ``out_proj``. The choices, all bitwise +to SGLang's own: + +- token-major projection views, with a token-major conv output (no split copy); +- a 128-channel conv tile; +- the ReplaySSM verify with 32-wide value tiles on 2 warps, on SGLang's kernel + (``replayssm.py``), and with slot ``gdn_pdl`` launched dependent on the conv; +- the gated norm writing the static FP8 ``out_proj`` takes (``norm_fp8.py``), which + ``out_proj``'s FlashInfer BMM then reads without quantizing it again. +""" + +from __future__ import annotations + +import logging + +import msgspec +import torch + +logger = logging.getLogger(__name__) + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen3_5.py": ( + "linear_attn", + "_forward_input_proj", + "num_k_heads", + "num_v_heads", + "attn_tp_size", + "head_k_dim", + "head_v_dim", + "attn", + "norm", + "out_proj", + ), + "kernels/ops/attention/triton_gdn_fused_proj.py": ( + "fused_qkvzba_split_reshape_cat_contiguous", + "qwen3_5_gdn_prefill_projection_views", + ), + "srt/layers/radix_linear_attention.py": ( + "layer_id", + "conv_weights", + "bias", + "activation", + "q_dim", + "k_dim", + "v_dim", + "num_q_heads", + "num_k_heads", + "num_v_heads", + "head_q_dim", + "head_k_dim", + "head_v_dim", + "A_log", + "dt_bias", + ), + "kernels/ops/attention/fla/layernorm_gated.py": ( + "eps", + "weight", + "activation", + "group_size", + "norm_before_gate", + ), + "srt/layers/quantization/modelopt_quant.py": ( + "ModelOptFp8LinearMethod", + "use_marlin", + "use_sm120_fp8", + "use_flashinfer_bmm", + '"input_scale"', + '"weight_scale"', + ), + "srt/layers/linear.py": ( + "reduce_results", + "use_dp_attention_reduce", + "use_decode_attn_tp", + "input_is_parallel", + "bias", + ), + "srt/layers/quantization/fp8_utils.py": ("flashinfer_bmm_fp8",), + "srt/distributed/device_communicators/pynccl_allocator.py": ( + "use_symmetric_memory", + ), + "srt/layers/dp_attention.py": ("is_allocation_symmetric",), + "srt/runtime_context.py": ("get_parallel",), + "srt/model_executor/forward_batch_info.py": ("req_pool_indices",), + "srt/megakernels/eagle_binding.py": ( + "mamba", + "tokens_per_request", + "gdn", + "state_indices", + "query_start_loc", + "intermediate_indices", + '"ForwardBatch.req_pool_indices"', + "layer_ids", + "conv", + "temporal", + "intermediate_conv_window", + "ring_d", + "ring_k", + "ring_g", + "ring_rawv", + "ring_rawk", + "ring_beta", + "ring_write_pos", + "ring_cache_base", + "ring_is_flush", + ), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + # Without and with the early return of the GDN extend in breakable prefill graphs, + # which target verify never takes; the first goes once all supported SGLangs have it + "srt/layers/attention/linear/gdn_backend.py:GDNAttnBackend.forward_extend": ( + "ab7b048fafb07d0e", + "12a835fc4f78af33", + ), + # Importing the ReplaySSM kernel from fla/ and, after sgl-project/sglang#45112, from + # linear/; the first goes once all supported SGLangs have the move + "srt/layers/attention/linear/gdn_backend.py:GDNAttnBackend._replayssm_target_verify": ( + "28b89fa4b48b47ad", + "739e7115c370ac01", + ), + "srt/layers/linear.py:RowParallelLinear.forward": "b291a3c452e428a5", + "srt/layers/quantization/fp8_utils.py:apply_fp8_linear_bmm_flashinfer": "c66562e1de0008db", + "srt/layers/quantization/modelopt_quant.py:ModelOptFp8LinearMethod.apply": "91b563cf074bff42", +} + +RING_LEN, MAX_DRAFT = 16, 8 # the verify tiles' bounds, as measured + +# the step widths that have run the slot (each logged once, as engagement evidence) +_ENGAGED: set = set() + + +class Options(msgspec.Struct, frozen=True): + """The launch choices; ``STOCK`` is SGLang's own.""" + + views: bool # token-major projection views and conv output + block_n: int # the conv update's channel tile + block_v: int # the ReplaySSM verify's value tile + num_warps: int + norm_fp8: bool # the gated norm writes out_proj's static FP8 input + pdl: bool = False # the verify launches dependent on the conv (slot gdn_pdl) + + +STOCK = Options(views=False, block_n=256, block_v=64, num_warps=1, norm_fp8=False) +SLOT = Options(views=True, block_n=128, block_v=32, num_warps=2, norm_fp8=True) +SLOT_PDL = msgspec.structs.replace(SLOT, pdl=True) + + +def unsupported(gdn) -> str: + """Why the slot cannot serve ``gdn`` (a Qwen3_5GatedDeltaNet) ("" when it can).""" + from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp8LinearMethod + + attn, norm, out = gdn.attn, gdn.norm, gdn.out_proj + if attn.head_k_dim != 128 or attn.head_v_dim != 128: + return f"head dims {attn.head_k_dim} / {attn.head_v_dim}" + if norm.group_size is not None or not norm.norm_before_gate: + return "a grouped or gate-first norm" + method = out.quant_method + if ( + not isinstance(method, ModelOptFp8LinearMethod) + or method.use_marlin + or method.use_sm120_fp8 + or not out.use_flashinfer_bmm + or out.input_scale.numel() != 1 + ): + return "out_proj is not a static per-tensor FP8 linear on FlashInfer's BMM" + if ( + out.reduce_results + or out.use_dp_attention_reduce + or out.use_decode_attn_tp + or not out.input_is_parallel + or out.bias is not None + ): + return "out_proj reduces its output, splits its input or adds a bias" + return "" + + +def pool_reasons(pools, tokens_per_request: int) -> list[str]: + """Why the slot cannot run on the mamba pools (``Step.view.mamba``) at + ``tokens_per_request`` verify rows ([] when it can): the circular ReplaySSM ring + of the verify tiles' bounds, BF16 checkpoints.""" + reasons = [] + if pools.ring_d is None or pools.ring_cache_base is None: + reasons.append("no circular ReplaySSM verify ring") + elif pools.ring_d.shape[-2] != RING_LEN: + reasons.append(f"ring length {pools.ring_d.shape[-2]}") + if pools.temporal.dtype != torch.bfloat16: + reasons.append(f"{pools.temporal.dtype} checkpoints") + if tokens_per_request > MAX_DRAFT: + reasons.append(f"{tokens_per_request} rows a request") + return reasons + + +def step(gdn, x: torch.Tensor, st, opts: Options = SLOT) -> torch.Tensor: + """``Qwen3_5GatedDeltaNet.forward`` of a target-verify step with the launch + choices ``opts``: the TP-partial output [rows, hidden] of the normed input x.""" + import triton + from sglang.kernels.ops.attention.triton_gdn_fused_proj import ( + fused_qkvzba_split_reshape_cat_contiguous, + qwen3_5_gdn_prefill_projection_views, + ) + + from sgl_nvidia_mega.qwen3_5.gdn.norm_fp8 import rms_norm_gated_fp8 + + if x.shape[0] not in _ENGAGED: + _ENGAGED.add(x.shape[0]) + logger.info( + "NVIDIA megakernel: Qwen3.5 slot gdn_verify runs at %d rows", x.shape[0] + ) + qkvz, ba = gdn._forward_input_proj(x) + split = ( + qwen3_5_gdn_prefill_projection_views + if opts.views + else fused_qkvzba_split_reshape_cat_contiguous + ) + mixed_qkv, z, b, a = split( + qkvz, + ba, + triton.cdiv(gdn.num_k_heads, gdn.attn_tp_size), + triton.cdiv(gdn.num_v_heads, gdn.attn_tp_size), + gdn.head_k_dim, + gdn.head_v_dim, + ) + core = _core(gdn.attn, mixed_qkv, a, b, st, opts) + core = core.reshape(-1, core.shape[-1]) + nv = gdn.num_v_heads // gdn.attn_tp_size + if opts.norm_fp8: + norm = gdn.norm + fp8 = rms_norm_gated_fp8( + core, + norm.weight, + norm.eps, + z if opts.views else z.reshape(-1, z.shape[-1]), + gdn.out_proj.input_scale, + activation=norm.activation, + ) + return _out_proj(gdn.out_proj, fp8.reshape(-1, nv * gdn.head_v_dim), core.dtype) + z_shape = z.shape + core = gdn.norm(core, z if opts.views else z.reshape(-1, z.shape[-1])) + core = core.reshape(z_shape) + core = core.reshape(*core.shape[:-2], core.shape[-2] * core.shape[-1]) + return gdn.out_proj(core)[0] + + +def _out_proj(out_proj, fp8: torch.Tensor, dtype: torch.dtype) -> torch.Tensor: + """``RowParallelLinear.forward`` of the FP8 ``out_proj`` on its pre-quantized input: + ModelOpt's FlashInfer BMM with the static input scale as the activation scale (what + its own quantization returns), the TP-partial output in SGLang's symmetric pool.""" + from sglang.srt.distributed.device_communicators.pynccl_allocator import ( + use_symmetric_memory, + ) + from sglang.srt.layers.dp_attention import is_allocation_symmetric + from sglang.srt.layers.quantization.fp8_utils import flashinfer_bmm_fp8 + from sglang.srt.runtime_context import get_parallel + + with use_symmetric_memory( + get_parallel().tp_group, disabled=not is_allocation_symmetric() + ): + return flashinfer_bmm_fp8( + fp8, out_proj.weight, out_proj.input_scale, out_proj.weight_scale, dtype + ) + + +def _core(attn, mixed_qkv, a, b, st, opts): + """The backend's target-verify arm for the circular ReplaySSM ring.""" + from sgl_nvidia_mega.qwen3_5.gdn import replayssm + from sgl_nvidia_mega.qwen3_5.gdn.conv import conv_update + + meta, pools = st.view.gdn, st.view.mamba + li = pools.layer_ids.index(attn.layer_id) + seq_len = mixed_qkv.shape[0] + draft = st.binding.tokens_per_request + bs = seq_len // draft + states = meta.state_indices + y = conv_update( + mixed_qkv.view(bs, draft, -1).transpose(1, 2), + pools.conv[0][li], + attn.conv_weights, + attn.bias, + attn.activation, + states[:bs], + pools.intermediate_conv_window[0][li], + meta.intermediate_indices[:bs], + block_n=opts.block_n, + token_major_out=opts.views, + ) + mixed_qkv = y.transpose(1, 2).view(seq_len, -1) + query, key, value = torch.split( + mixed_qkv, [attn.q_dim, attn.k_dim, attn.v_dim], dim=-1 + ) + H, K = attn.num_k_heads, attn.head_k_dim + HV, V = attn.num_v_heads, attn.head_v_dim + q = query.reshape(seq_len, H, K) + k = key.reshape(seq_len, H, K) + v = value.reshape(seq_len, HV, V) + d_cache = pools.ring_d[li] + out = q.new_empty(seq_len, HV, V) + + def ring(t): + return None if t is None else t[li] + + replayssm.verify( + q, + k, + v, + a.reshape(seq_len, HV), + b.reshape(seq_len, HV), + attn.A_log, + attn.dt_bias, + out, + pools.temporal[li], + d_cache, + pools.ring_k[li], + pools.ring_g[li], + ring(pools.ring_rawv), + ring(pools.ring_rawk), + ring(pools.ring_beta), + meta.query_start_loc, + states, + st.fb.req_pool_indices, + pools.ring_write_pos, + pools.ring_cache_base, + pools.ring_is_flush, + max_cache_len=d_cache.shape[-2], + max_spec_len=draft, + scale=K**-0.5, + null_block_id=-1, + block_v=opts.block_v, + num_warps=opts.num_warps, + pdl=opts.pdl, + ) + return out.reshape(1, seq_len, HV, V) diff --git a/sgl_nvidia_mega/qwen3_5/logits/__init__.py b/sgl_nvidia_mega/qwen3_5/logits/__init__.py new file mode 100644 index 00000000..21cd96f3 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/logits/__init__.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The logits of a covered step. Stock: SGLang's logits processor on the final hidden +rows, which also hands the speculative worker every row's hidden state.""" + +from __future__ import annotations + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = {"srt/models/qwen3_vl.py": ("logits_processor", "lm_head")} + + +def logits(st, hidden, aux): + """SGLang's ``LogitsProcessorOutput`` for the step (``aux``: the captured auxiliary + hidden states, or None).""" + lm = st.lm + return lm.logits_processor(st.input_ids, hidden, lm.lm_head, st.fb, aux) diff --git a/sgl_nvidia_mega/qwen3_5/moe/__init__.py b/sgl_nvidia_mega/qwen3_5/moe/__init__.py new file mode 100644 index 00000000..fdae24a1 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/moe/__init__.py @@ -0,0 +1,172 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/srt/models/qwen2_moe.py (Qwen2MoeSparseMoeBlock.forward and its dual-stream path). +"""The sparse MoE of a covered step, over SGLang's loaded ``Qwen2MoeSparseMoeBlock``. + +Stock: what ``Qwen2MoeSparseMoeBlock.forward`` issues on CUDA with no expert +parallelism, no all-to-all and no paged experts (SGLang pages experts on one GPU only), +calling the block's own router gate, top-k, routed +experts and gated shared expert. At CUDA graph capture it takes the dual-stream order +(with a deferred finalize: routed experts on the main stream, shared expert on the +alternate stream), otherwise the eager order. With ``defer`` (the boundary takes the +finalize) it returns SGLang's ``MoeDeferredFinalize`` handoff, otherwise the summed +rows. Slot ``route`` (``route.py``) runs the deferred routed experts on the router +front, and slot ``mkm`` (``mkm.py``) runs them in one launch on its outputs. +""" + +from __future__ import annotations + +import torch + +from sgl_nvidia_mega.qwen3_5.moe import route + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen2_moe.py": ( + "mlp", + "alt_stream", + "gate", + "topk", + "experts", + "shared_expert", + "shared_expert_gate", + "enable_shared_expert_fusion", + "supports_deferred_finalize", + "_forward_shared_experts", + "_forward_shared_experts(apply_gate)", + "_gate_shared_output_out_of_place", + "_append_shared_to_topk_output", + ), + "srt/layers/moe/topk.py": ( + "TopKOutputChecker", + "format_is_bypassed", + "format_is_standard", + ), + "srt/layers/moe/fused_moe_triton/layer.py": ("forward_deferred_finalize",), + "srt/layers/moe/utils.py": ("sum_post_experts_output",), + "srt/layers/layer_boundary/fusions/cutedsl.py": ( + "MoeDeferredFinalize", + "from_flashinfer", + "from_flashinfer(gated_shared_output)", + "from_flashinfer(m)", + "from_flashinfer(reduce)", + ), + "kernels/ops/elementwise/elementwise.py": ("fused_gate_sigmoid_mul_add",), + "srt/model_executor/runner/__init__.py": ("get_is_capture_mode",), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + "srt/models/qwen2_moe.py:Qwen2MoeSparseMoeBlock.forward": "1994cd43fba4fbde", + "srt/models/qwen2_moe.py:Qwen2MoeSparseMoeBlock.forward_normal_dual_stream": "afae959064cc8cd7", + "srt/models/qwen2_moe.py:Qwen2MoeSparseMoeBlock._forward_router_experts": "32d09f01b77d76b5", +} + + +def build(model, ctx) -> route.Bound: + """Slots ``route`` and ``mkm`` for every MoE layer; the result is + ``st.bound["moe"]``.""" + from sgl_nvidia_mega.qwen3_5.runtime import SLOTS + + if "route" not in SLOTS: + return route.Bound(reason="stock (SGLANG_NVIDIA_MEGAKERNEL_SLOTS)") + return route.build(model, max(ctx.widths), fused="mkm" in SLOTS) + + +def step(layer, x: torch.Tensor, st, defer: bool): + """``layer``'s MoE on the normed input ``x`` [rows, hidden]: the deferred-finalize + handoff with ``defer``, else the unreduced rows [rows, hidden].""" + from sglang.kernels.ops.elementwise.elementwise import fused_gate_sigmoid_mul_add + from sglang.srt.layers.layer_boundary.fusions.cutedsl import MoeDeferredFinalize + from sglang.srt.layers.moe.utils import sum_post_experts_output + from sglang.srt.model_executor.runner import get_is_capture_mode + + mlp = layer.mlp + rows, hidden = x.shape + x = x.view(-1, hidden) + # the gate is a plain linear on CUDA (no AMX, no NPU) + use_fused_gate = mlp.shared_expert_gate is not None + if ( + mlp.alt_stream is not None + and get_is_capture_mode() + and not torch.compiler.is_compiling() + ): + routed, shared = _dual_stream(mlp, x, use_fused_gate, defer, st) + else: + shared = mlp._forward_shared_experts( + x, apply_gate=not use_fused_gate and not defer + ) + if defer and shared is not None: + shared = mlp._gate_shared_output_out_of_place(x, shared) + routed = _router_experts(mlp, x, defer, st) + if defer: + if shared is None: + raise RuntimeError("Qwen deferred finalize requires shared output") + return MoeDeferredFinalize.from_flashinfer( + routed, + gated_shared_output=shared, + m=rows, + reduce=sum_post_experts_output, + ) + if shared is not None: + if use_fused_gate: + fused_gate_sigmoid_mul_add( + x, mlp.shared_expert_gate.weight.squeeze(), shared, routed + ) + else: + routed += shared + return routed.view(rows, hidden) + + +def _dual_stream(mlp, x, use_fused_gate: bool, defer: bool, st): + """``Qwen2MoeSparseMoeBlock.forward_normal_dual_stream`` (no LoRA): (routed, + shared).""" + current = torch.cuda.current_stream() + mlp.alt_stream.wait_stream(current) + if defer: + # routed FC2 stays on the current stream, the finalize's PDL dependency + routed = _router_experts(mlp, x, True, st) + with torch.cuda.stream(mlp.alt_stream): + shared = mlp._forward_shared_experts(x, apply_gate=False) + if shared is not None: + shared = mlp._gate_shared_output_out_of_place(x, shared) + current.wait_stream(mlp.alt_stream) + return routed, shared + shared = ( + mlp._forward_shared_experts(x.clone(), apply_gate=not use_fused_gate) + if mlp.shared_expert is not None + else None + ) + with torch.cuda.stream(mlp.alt_stream): + routed = _router_experts(mlp, x, False, st) + current.wait_stream(mlp.alt_stream) + return routed, shared + + +def _router_experts(mlp, x, defer: bool, st): + """``Qwen2MoeSparseMoeBlock._forward_router_experts`` on CUDA: the router gate, + the top-k and the routed experts (slots ``route`` and ``mkm``: theirs, deferred).""" + from sglang.srt.layers.moe.topk import TopKOutputChecker + + if defer and st.slots.route: + return route.routed(mlp, x, st.bound["moe"] if st.slots.mkm else None) + router_logits, _ = mlp.gate(x) + topk_output = mlp.topk(x, router_logits) + if defer: + if not mlp.supports_deferred_finalize: + raise RuntimeError( + "Qwen deferred finalize requires a compatible FlashInfer TRTLLM MoE " + "producer and a separate shared expert" + ) + if not TopKOutputChecker.format_is_bypassed(topk_output): + raise RuntimeError( + "Qwen deferred finalize requires logits-based bypassed TopK" + ) + return mlp.experts.forward_deferred_finalize(x, topk_output) + if mlp.enable_shared_expert_fusion and TopKOutputChecker.format_is_standard( + topk_output + ): + topk_output = mlp._append_shared_to_topk_output(topk_output, x) + return mlp.experts(x, topk_output) diff --git a/sgl_nvidia_mega/qwen3_5/moe/mkm.py b/sgl_nvidia_mega/qwen3_5/moe/mkm.py new file mode 100644 index 00000000..2d48bcaf --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/moe/mkm.py @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Slot ``mkm``, on slot ``route``'s outputs: the top-10, FC1, SwiGLU, the NVFP4 +requantization and FC2 of up to 7 rows in one launch (``sm103/moe/mkm.py``), in place of +the trtllm runner's routing and its two GEMMs. Its rows and expert weights reach +SGLang's deferred finalize as the runner's would, in expert order (an identity +permutation). +""" + +from __future__ import annotations + +import torch + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen2_moe.py": ("topk", "experts", "layer_id"), + "srt/layers/moe/topk.py": ( + "topk_config", + "top_k", + "renormalize", + "scoring_func", + "correction_bias", + "use_grouped_topk", + ), + "srt/layers/moe/fused_moe_triton/layer.py": ( + "num_experts", + "num_local_experts", + "moe_ep_rank", + "top_k", + "routing_method_type", + "moe_runner_config", + "dispatcher", + ), + "srt/layers/moe/moe_runner/base.py": ( + "activation", + "is_gated", + "routed_scaling_factor", + "gemm1_alpha", + "gemm1_clamp_limit", + "swiglu_limit", + ), + "srt/layers/moe/utils.py": ("RoutingMethodType", "RenormalizeNaive"), + "srt/layers/quantization/modelopt_quant.py": ( + '"w13_weight"', + '"w2_weight"', + '"w13_weight_scale"', + '"w2_weight_scale"', + '"g1_alphas"', + '"g2_alphas"', + ), + "srt/layers/moe/moe_runner/flashinfer_trtllm.py": ( + '"g1_scale_c"', + "FlashInferTrtllmDeferredFinalizeOutput", + "gemm2_out", + "expert_weights", + "expanded_idx_to_permuted_idx", + "top_k", + ), + "srt/layers/moe/token_dispatcher/standard.py": ( + "StandardCombineInput", + "combine(combine_input)", + ), +} + + +def shared_topk_slots(config) -> int: + """The trailing top-k columns SGLang's router fills with fused shared experts: + ``TopKConfig.num_shared_topk_slots``, named ``num_fused_shared_experts`` before + SGLang split its two meanings; read by name so both serve.""" + slots = getattr(config, "num_shared_topk_slots", None) + return config.num_fused_shared_experts if slots is None else slots + + +def unsupported(mlp) -> str: + """Why the fused experts cannot stand in for ``mlp``'s routed experts ("" when + they can): the TP8 shapes on the trtllm-gen weight layout, no expert parallelism, + softmax top-10 renormalized, plain SwiGLU.""" + from sglang.srt.layers.moe.utils import RoutingMethodType + + from sgl_nvidia_mega.qwen3_5.sm103.moe import mkm + + experts, cfg = mlp.experts, mlp.topk.topk_config + e, h, i = mkm.EXPERTS, mkm.HIDDEN, mkm.W13_ROWS // 2 + for name, t, dtype, shape in ( + ("w13_weight", experts.w13_weight, torch.uint8, (e, 2 * i, h // 2)), + ("w2_weight", experts.w2_weight, torch.uint8, (e, h, i // 2)), + ( + "w13_weight_scale", + experts.w13_weight_scale, + torch.float8_e4m3fn, + (e, 2 * i, h // 16), + ), + ( + "w2_weight_scale", + experts.w2_weight_scale, + torch.float8_e4m3fn, + (e, h, i // 16), + ), + ("g1_alphas", experts.g1_alphas, torch.float32, (e,)), + ("g1_scale_c", experts.g1_scale_c, torch.float32, (e,)), + ("g2_alphas", experts.g2_alphas, torch.float32, (e,)), + ): + if t.dtype != dtype or tuple(t.shape) != shape: + return f"{name} {t.dtype} {tuple(t.shape)}" + if not ( + experts.num_experts == experts.num_local_experts == e + and experts.moe_ep_rank == 0 + and experts.top_k == cfg.top_k == mkm.TOP_K + and cfg.renormalize + and cfg.scoring_func == "softmax" + and cfg.correction_bias is None + and not cfg.use_grouped_topk + and shared_topk_slots(cfg) == 0 + and experts.routing_method_type == RoutingMethodType.RenormalizeNaive + ): + return ( + "routing other than a softmax top-10, renormalized, over 512 local experts" + ) + run = experts.moe_runner_config + if not ( + run.activation == "silu" + and run.is_gated + and run.routed_scaling_factor in (None, 1.0) + and run.gemm1_alpha is None + and run.gemm1_clamp_limit is None + and run.swiglu_limit is None + ): + return "an expert activation other than a plain SwiGLU" + return "" + + +def routed(mlp, maps: tuple, identity: torch.Tensor, logits, x_fp4, x_sf): + """The deferred-finalize output of ``mlp``'s routed experts from the router front's + logits and quantized input (``maps``: the layer's weight TMA maps; ``identity``: + 0, 1, ... int32).""" + from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( + FlashInferTrtllmDeferredFinalizeOutput, + ) + from sglang.srt.layers.moe.token_dispatcher.standard import StandardCombineInput + + from sgl_nvidia_mega.qwen3_5.sm103.moe import mkm + + experts = mlp.experts + gemm2_out, expert_weights = mkm.mkm_moe( + maps, + logits, + x_fp4, + x_sf, + experts.w13_weight_scale.view(torch.uint8), + experts.w2_weight_scale.view(torch.uint8), + experts.g1_alphas, + experts.g1_scale_c, + experts.g2_alphas, + ) + deferred = FlashInferTrtllmDeferredFinalizeOutput( + gemm2_out=gemm2_out, + expert_weights=expert_weights, + expanded_idx_to_permuted_idx=identity[: logits.shape[0] * mkm.TOP_K], + top_k=mkm.TOP_K, + ) + return experts.dispatcher.combine( + combine_input=StandardCombineInput(hidden_states=deferred) + ) diff --git a/sgl_nvidia_mega/qwen3_5/moe/route.py b/sgl_nvidia_mega/qwen3_5/moe/route.py new file mode 100644 index 00000000..3315cec8 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/moe/route.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/srt/layers/moe/fused_moe_triton/layer.py (FusedMoE.forward_deferred_finalize). +"""Slot ``route``: a MoE layer's router GEMM and the NVFP4 quantization of its routed +experts' input in one launch (``sm103/moe/router.py``), then SGLang's trtllm routed +experts on that input, with the deferred finalize. + +The logits equal the gate's cuBLAS GEMM bit for bit and the quantized input equals the +trtllm runner's own quantizer, so the step's output is the stock one's. The runner +takes the quantized input through its pre-quantized (``hidden_states_scale``) path, +or slot ``mkm`` (``mkm.py``) takes the router front's outputs in its place. +""" + +from __future__ import annotations + +import logging + +import msgspec +import torch + +logger = logging.getLogger(__name__) + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen2_moe.py": ( + "Qwen2MoeSparseMoeBlock", + "gate", + "topk", + "experts", + "supports_deferred_finalize", + "layer_id", + ), + "srt/models/qwen3_5.py": ("layers", "start_layer", "end_layer", "mlp"), + "srt/layers/moe/topk.py": ("TopKOutputChecker", "format_is_bypassed"), + "srt/layers/moe/fused_moe_triton/layer.py": ( + "dispatcher", + "run_moe_core", + "run_moe_core(dispatch_output)", + "quant_method", + ), + "srt/layers/moe/token_dispatcher/standard.py": ( + "dispatch(hidden_states)", + "dispatch(topk_output)", + "combine(combine_input)", + "hidden_states_scale", + ), + "srt/layers/moe/moe_runner/flashinfer_trtllm.py": ( + "flashinfer_trtllm_deferred_finalize_context", + "trtllm_padded_hidden_size", + ), + "srt/layers/quantization/modelopt_quant.py": ( + '"w13_input_scale_quant"', + "quant_config", + "use_per_token_activation", + ), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + "srt/layers/moe/fused_moe_triton/layer.py:FusedMoE.forward_deferred_finalize": "af4b474cbaf33493", + "srt/layers/moe/fused_moe_triton/layer.py:FusedMoE._dispatch_with_pre_quant": "dcdefc28cf643081", +} + + +# the (slot, rows) that have run (each logged once, as engagement evidence) +_ENGAGED: set = set() + + +class Bound(msgspec.Struct, frozen=True): + """The MoE package's build result (``st.bound["moe"]``).""" + + route: bool = False # the router front stands in for every layer's gate + mkm: dict = msgspec.field(default_factory=dict) # layer id -> its weight TMA maps + identity: object = None # [70] int32 0, 1, ...: slot mkm's rows are in expert order + reason: str = "" # why a requested slot is not built + + +def build(model, max_rows: int, fused: bool) -> Bound: + """The router front for every MoE layer, compiled for 1..max_rows rows, and with + ``fused`` slot mkm's kernel and weight maps; or why a layer cannot take them.""" + from sgl_nvidia_mega.qwen3_5.moe import mkm + from sgl_nvidia_mega.qwen3_5.sm103.moe import mkm as mkm_kernel + from sgl_nvidia_mega.qwen3_5.sm103.moe import router + + m = model.model + layers = [m.layers[i].mlp for i in range(m.start_layer, m.end_layer)] + for i, mlp in zip(range(m.start_layer, m.end_layer), layers): + reason = unsupported(mlp) + if reason: + return Bound(reason=f"layer {i}: {reason}") + device = torch.cuda.current_device() + router.build(device, max_rows) + if not fused: + return Bound(route=True) + for i, mlp in zip(range(m.start_layer, m.end_layer), layers): + reason = mkm.unsupported(mlp) + if reason: + return Bound(route=True, reason=f"mkm: layer {i}: {reason}") + mkm_kernel.build(device) + return Bound( + route=True, + mkm={ + mlp.layer_id: mkm_kernel.weight_maps( + mlp.experts.w13_weight, mlp.experts.w2_weight + ) + for mlp in layers + }, + identity=torch.arange( + mkm_kernel.MAX_TOKENS * mkm_kernel.TOP_K, dtype=torch.int32, device="cuda" + ), + ) + + +def unsupported(mlp) -> str: + """Why the router front cannot stand in for ``mlp``'s gate and the runner's input + quantizer ("" when it can).""" + from sglang.srt.models.qwen2_moe import Qwen2MoeSparseMoeBlock + + from sgl_nvidia_mega.qwen3_5.sm103.moe import router + + if not isinstance(mlp, Qwen2MoeSparseMoeBlock): + return f"{type(mlp).__name__} is not a Qwen2MoeSparseMoeBlock" + w = mlp.gate.weight + if w.dtype != torch.bfloat16 or tuple(w.shape) != (router.EXPERTS, router.HIDDEN): + return f"router gate {w.dtype} {tuple(w.shape)}" + experts = mlp.experts + scale = experts.w13_input_scale_quant + if scale.dtype != torch.float32 or scale.numel() != 1: + return f"routed input scale {scale.dtype} {tuple(scale.shape)}" + if experts.trtllm_padded_hidden_size is not None: + return "the routed experts' hidden dim is padded" + if experts.quant_method.quant_config.use_per_token_activation: + return "per-token activation scales" + if not mlp.supports_deferred_finalize: + return "no deferred finalize" + return "" + + +def routed(mlp, x: torch.Tensor, fused: Bound | None = None): + """``FusedMoE.forward_deferred_finalize`` of ``mlp``'s routed experts on ``x`` + [rows, hidden], with the router front's logits and quantized input (``fused``: on + slot mkm's kernel).""" + from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( + flashinfer_trtllm_deferred_finalize_context, + ) + from sglang.srt.layers.moe.topk import TopKOutputChecker + + from sgl_nvidia_mega.qwen3_5.moe import mkm + from sgl_nvidia_mega.qwen3_5.sm103.moe import router + + experts = mlp.experts + slot = "route" if fused is None else "mkm" + if (slot, x.shape[0]) not in _ENGAGED: + _ENGAGED.add((slot, x.shape[0])) + logger.info( + "NVIDIA megakernel: Qwen3.5 slot %s runs at %d rows", slot, x.shape[0] + ) + logits, x_fp4, x_sf = router.router_gemm_quant( + x, mlp.gate.weight, experts.w13_input_scale_quant.reshape(1) + ) + if fused is not None: + maps = fused.mkm[mlp.layer_id] + return mkm.routed(mlp, maps, fused.identity, logits, x_fp4, x_sf) + topk_output = mlp.topk(x, logits) + if not TopKOutputChecker.format_is_bypassed(topk_output): + raise RuntimeError("the router front requires logits-based bypassed TopK") + dispatch_output = experts.dispatcher.dispatch( + hidden_states=x, topk_output=topk_output + )._replace(hidden_states=x_fp4, hidden_states_scale=x_sf.view(torch.float8_e4m3fn)) + with flashinfer_trtllm_deferred_finalize_context(): + combine_input = experts.run_moe_core(dispatch_output=dispatch_output) + return experts.dispatcher.combine(combine_input=combine_input) diff --git a/sgl_nvidia_mega/qwen3_5/plan.py b/sgl_nvidia_mega/qwen3_5/plan.py new file mode 100644 index 00000000..07d31464 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/plan.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Which kernel fills each slot of a covered step, per width (rows), fixed at build. + +A slot runs only where it is requested (``SGLANG_NVIDIA_MEGAKERNEL_SLOTS``), built and +inside its row bounds; every other op is SGLang's own. ``route`` (the router +GEMM and the routed experts' NVFP4 input in one launch) and ``mkm`` (on its outputs, the +top-10 and both expert GEMMs in one launch, up to 7 rows) cover every width, as do +``gdn_verify`` (the GatedDeltaNet verify chain with its launch choices changed) and +``hop`` (the boundary's fused all-reduce points on the decode hop). +""" + +from __future__ import annotations + +import msgspec + +MAX_ROWS = 7 # the widest covered step: one request of 7 verify rows (TP8 bs1) + + +class Slots(msgspec.Struct, frozen=True): + """The kernels that fill one width's step.""" + + route: bool = False # router GEMM + NVFP4 input quantization (sm103/moe/router.py) + mkm: bool = ( + False # top-10 + FC1 + SwiGLU + FC2 on route's outputs (sm103/moe/mkm.py) + ) + gdn_verify: bool = False # the GDN verify chain's launch choices (gdn/verify.py) + gdn_pdl: bool = False # its ReplaySSM verify launched dependent on the conv + hop: bool = False # the boundary's fused all-reduce points (boundary/hop.py) + + +class Facts(msgspec.Struct, frozen=True): + """What build found, which every rank must agree on. All off: every step + function runs its stock code.""" + + route: bool = False # the router front built for every MoE layer + mkm: bool = False # the fused routed experts built for every MoE layer + gdn_verify: bool = False # the GDN verify slot built for every GDN layer + gdn_pdl: bool = False # with its dependent verify launch + hop: bool = False # the decode hop, built for the widest covered step + + +def slots(rows: int, facts: Facts) -> Slots: + if not 1 <= rows <= MAX_ROWS: + raise ValueError(f"{rows} rows are outside the covered widths 1-{MAX_ROWS}") + return Slots( + route=facts.route, + mkm=facts.route and facts.mkm, + gdn_verify=facts.gdn_verify, + gdn_pdl=facts.gdn_verify and facts.gdn_pdl, + hop=facts.hop, + ) + + +def plan(widths: tuple[int, ...], facts: Facts) -> dict[int, Slots]: + return {w: slots(w, facts) for w in widths} diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py new file mode 100644 index 00000000..0883e391 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The megakernel runtime: the executor of a covered target-verify step (``executor``), +its per-step state (``state``), the embedding (``embedding``), the test-only parity +check (``parity``), and ``build``, which SGLang runs through the spec once the model is +prepared and before the KV cache is sized.""" + +from __future__ import annotations + +import os + +import msgspec + +# The kernel slots that run by default, and the ones that run only when requested +# (until step-level timing and accuracy runs move them to the default list) +SHIPPED_SLOTS: tuple[str, ...] = () +OPT_IN_SLOTS: tuple[str, ...] = ("route", "mkm", "gdn_verify", "gdn_pdl", "hop") +SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" + + +def _requested_slots() -> frozenset: + """``SLOTS_ENV`` (comma-separated; "" runs the stock step), else every shipped + slot.""" + raw = os.environ.get(SLOTS_ENV) + if raw is None: + return frozenset(SHIPPED_SLOTS) + return frozenset(name.strip() for name in raw.split(",")) - {""} + + +SLOTS = _requested_slots() + + +def slot_reasons() -> list[str]: + """Why the requested slots cannot be served ([] when they can).""" + known = (*SHIPPED_SLOTS, *OPT_IN_SLOTS) + reasons = [ + f"{SLOTS_ENV}: unknown slot {name!r} (known: {', '.join(known) or 'none'})" + for name in sorted(SLOTS - set(known) - {"verify_sample"}) + ] + if "verify_sample" in SLOTS: + reasons.append( + f"{SLOTS_ENV}: 'verify_sample' is no longer an entry: drop it; an SGLang " + "with post-verify graphs turns them on with " + "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH=1" + ) + if "mkm" in SLOTS and "route" not in SLOTS: + reasons.append(f"{SLOTS_ENV}: slot 'mkm' runs on slot 'route''s outputs") + if "gdn_pdl" in SLOTS and "gdn_verify" not in SLOTS: + reasons.append( + f"{SLOTS_ENV}: slot 'gdn_pdl' launches slot 'gdn_verify''s verify" + ) + return reasons + + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/megakernels/spec.py": ( + "widths", + "tokens_per_request", + "deadline_s", + "binding", + ), +} + + +class PackageContext(msgspec.Struct, frozen=True): + """What every package's ``build(model, ctx, group)`` gets.""" + + widths: tuple[int, ...] # the covered steps' rows (requests x rows a request) + tokens_per_request: int + deadline_s: float # each cross-GPU wait's deadline + + +def build(model, ctx): + """Each package's build, the slot plan per width and the executor; ``ctx`` is + SGLang's BuildContext.""" + import torch + + from sgl_nvidia_mega.qwen3_5 import boundary, gdn, moe + from sgl_nvidia_mega.qwen3_5 import plan as slot_plan + from sgl_nvidia_mega.qwen3_5.runtime import parity + from sgl_nvidia_mega.qwen3_5.runtime.executor import Qwen35Verify + + if not ctx.widths: + raise RuntimeError( + f"NVIDIA megakernel: {ctx.tokens_per_request} rows a request leave no " + "covered width" + ) + pctx = PackageContext( + widths=ctx.widths, + tokens_per_request=ctx.tokens_per_request, + deadline_s=ctx.deadline_s, + ) + bound = { + "moe": moe.build(model, pctx), + "gdn": gdn.build(model, pctx), + "boundary": boundary.build(model, pctx), + } + facts = slot_plan.Facts( + route=bound["moe"].route, + mkm=bool(bound["moe"].mkm), + gdn_verify=bound["gdn"].gdn_verify, + gdn_pdl=bound["gdn"].gdn_pdl, + hop=bound["boundary"].hop is not None, + ) + slots = slot_plan.plan(pctx.widths, facts) + built = msgspec.structs.asdict(facts) + # every package's build agrees across ranks, so this raises on every rank alike + missing = sorted(name for name in SLOTS if not built[name]) + if missing: + raise RuntimeError( + f"NVIDIA megakernel: requested slots {missing} built no width (MoE: " + f"{bound['moe'].reason or 'built'}; GDN: {bound['gdn'].reason or 'built'}; " + f"boundary: {bound['boundary'].reason or 'built'})" + ) + checker = ( + parity.Parity(pctx.widths, torch.cuda.current_device(), model) + if parity.ENABLED + else None + ) + on = sorted(name for name, ok in built.items() if ok) + return Qwen35Verify( + model, + slots, + bound=bound, + widths=pctx.widths, + binding=ctx.binding, + summary=f"slots {on or 'none (stock)'}", + parity=checker, + ) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/embedding.py b/sgl_nvidia_mega/qwen3_5/runtime/embedding.py new file mode 100644 index 00000000..cb15bf31 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/runtime/embedding.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/srt/managers/mm_utils.py (general_mm_embed_routine). +"""The token embedding of a covered step. Stock: SGLang's input preparation of the VL +model (the vocabulary-parallel lookup, copied into the batch's embedding buffer when +the batch has one), or the language model's own lookup for a language-only model.""" + +from __future__ import annotations + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/managers/mm_utils.py": ( + "prepare_mm_inputs", + "prepare_mm_inputs(multimodal_model)", + "prepare_mm_inputs(use_deepstack)", + ), + "srt/models/qwen3_vl.py": ("language_model_only", "use_deepstack"), + "srt/models/qwen3_5.py": ("embed_tokens",), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + "srt/managers/mm_utils.py:general_mm_embed_routine": "4230344c2697664e", +} + + +def embed(st): + """[rows, hidden] for ``st.input_ids``.""" + from sglang.srt.managers.mm_utils import prepare_mm_inputs + + lm = st.lm + if lm.language_model_only: + return st.model.embed_tokens(st.input_ids) + # a verify step carries no multimodal input, so kwargs gains no deepstack rows + return prepare_mm_inputs( + st.input_ids, + st.fb, + st.model, + {"positions": st.positions, "pp_proxy_tensors": None}, + multimodal_model=lm, + use_deepstack=lm.use_deepstack, + ) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/executor.py b/sgl_nvidia_mega/qwen3_5/runtime/executor.py new file mode 100644 index 00000000..c53e4890 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/runtime/executor.py @@ -0,0 +1,220 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/srt/models/qwen3_vl.py (Qwen3VLForConditionalGeneration.forward) and +# python/sglang/srt/models/qwen3_5.py (Qwen3_5ForCausalLM.forward and its layers' forward). +"""The executor: one covered target-verify step of Qwen3.5. + +The build gets SGLang's EAGLE binding (``BuildContext.binding``) and each covered step +the running forward's view (``Step.view``: the attention and GDN metadata, the KV and +mamba pools); the step's batch tensors are the forward's own arguments. +``Qwen35Verify.plan`` decides from host state alone, identical on every rank, whether a +step is covered: a target-verify step of a built width. ``Qwen35Verify.forward`` checks +the pools the slots read at the first covered step, then takes and returns what +``Qwen3_5MoeForConditionalGeneration.forward`` does. It runs the step from the +plugin's step functions over SGLang's loaded modules, never through a model-level or +decoder-layer ``forward``: + +1. the positions the layers take (M-RoPE when the model uses it) and the embedding; +2. per layer, under the expert-distribution recorder's layer scope: + ``x = boundary.pre(layer, h, st)``, ``part = gdn.step(layer, x, st)`` or + ``attention.step(...)``, ``x = boundary.post_attn(layer, part, st)``, then inside + ``boundary.ffn_exit(layer, st)``: ``out = moe.step(layer, x, st, defer)``, and + ``h = exit.finish(out)``; +3. ``boundary.close_stream`` (the final norm) and the logits (``logits``). +""" + +from __future__ import annotations + +from functools import partial + +import torch +from sglang.srt.megakernels.spec import Step + +from sgl_nvidia_mega.qwen3_5 import attention, boundary, gdn, logits, moe +from sgl_nvidia_mega.qwen3_5.gdn import verify as _gdn_verify # noqa: F401 +from sgl_nvidia_mega.qwen3_5.runtime import embedding +from sgl_nvidia_mega.qwen3_5.runtime.state import StepState + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + # Step, and the BuiltMegakernel members SGLang reads + "srt/megakernels/spec.py": ( + "Step", + "mode", + "rows", + "view", + "widths", + "draft_sampler", + "summary", + "plan", + "forward", + ), + # Qwen3VLForConditionalGeneration.forward, which the executor stands in for + "srt/models/qwen3_vl.py": ( + "forward(get_embedding)", + "forward(pp_proxy_tensors)", + "is_mrope_enabled", + "capture_aux_hidden_states", + "model", + ), + "srt/model_executor/forward_batch_info.py": ("mrope_positions", "forward_mode"), + "srt/models/qwen3_5.py": ( + "Qwen3_5LinearDecoderLayer", + "layers", + "start_layer", + "end_layer", + '"_is_layer_to_capture"', + ), + "srt/layers/aux_hidden_states.py": ("AuxHiddenStateList",), + "srt/eplb/expert_distribution.py": ( + "get_global_expert_distribution_recorder", + "with_current_layer", + ), + "srt/megakernels/eagle_binding.py": ( + "tokens_per_request", + "mamba", + ), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + "srt/models/qwen3_vl.py:Qwen3VLForConditionalGeneration.forward": "d394d79ba7fa7b31", + "srt/models/qwen3_5.py:Qwen3_5ForCausalLM.forward": "9e530b264ce30ad6", + "srt/models/qwen3_5.py:Qwen3_5LinearDecoderLayer.forward": "2298a57111f9430e", + "srt/models/qwen3_5.py:Qwen3_5AttentionDecoderLayer.forward": "6473f673a1a0796b", +} + + +class Qwen35Verify: + """One rank's covered steps: the model, each width's slots and each package's + build result.""" + + # the target offers no draft sampler (BuiltMegakernel) + draft_sampler = None + + def __init__( + self, + lm, + slots: dict, + bound: dict, + widths: tuple[int, ...], + binding=None, + summary: str = "", + parity=None, + ): + from sglang.srt.models.qwen3_5 import Qwen3_5LinearDecoderLayer + + self.lm, self.slots, self.bound = lm, slots, bound + self.widths = tuple(sorted(widths)) + self.binding = binding # SGLang's EagleBinding (BuildContext.binding) + self._summary = summary + self.parity = parity # runtime.parity.Parity under the test-only switch + self._pools_checked = False + m = lm.model + self.mixers = { + i: gdn.step + if isinstance(m.layers[i], Qwen3_5LinearDecoderLayer) + else attention.step + for i in range(m.start_layer, m.end_layer) + } + + def summary(self) -> str: + """What was built, for SGLang's startup line.""" + return self._summary + + def plan( + self, + input_ids, + forward_batch, + get_embedding=False, + pp_proxy_tensors=None, + **kwargs, + ) -> Step | None: + """The step if this forward is covered, else None (SGLang's forward runs). + An argument this executor does not know also leaves the step to SGLang.""" + if self.parity is not None: + self.parity.flush() + if kwargs or get_embedding or pp_proxy_tensors is not None: + return None + if not forward_batch.forward_mode.is_target_verify(): + return None + rows = int(input_ids.shape[0]) + return Step(mode="target_verify", rows=rows) if rows in self.widths else None + + def forward( + self, step: Step, original, input_ids, positions, forward_batch, *args, **kwargs + ): + """The covered step (``original``: the model's own forward, bound, which the + parity check runs).""" + if not self._pools_checked: + self._check_pools(step.view) + run = partial(self._forward, step, input_ids, positions, forward_batch) + if self.parity is not None: + stock = partial( + original, input_ids, positions, forward_batch, *args, **kwargs + ) + return self.parity.run(step, stock, run) + return run() + + def _check_pools(self, view): + """Refuse a slot whose pools are not the ones it was built for; the pools + exist from the first covered step on, the same on every rank.""" + if any(s.gdn_verify for s in self.slots.values()): + reasons = gdn.verify.pool_reasons( + view.mamba, self.binding.tokens_per_request + ) + if reasons: + raise RuntimeError( + "NVIDIA megakernel: slot gdn_verify cannot run: " + + "; ".join(reasons) + ) + self._pools_checked = True + + @torch.no_grad() + def _forward(self, step, input_ids, positions, forward_batch): + """``Qwen3VLForConditionalGeneration.forward`` for a covered step.""" + lm = self.lm + if lm.is_mrope_enabled: + positions = forward_batch.mrope_positions + st = StepState( + lm=lm, + forward_batch=forward_batch, + positions=positions, + input_ids=input_ids, + slots=self.slots[step.rows], + bound=self.bound, + binding=self.binding, + view=step.view, + ) + hidden, aux = self._language_model(st, embedding.embed(st)) + return logits.logits(st, hidden, aux if lm.capture_aux_hidden_states else None) + + def _language_model(self, st, hidden): + """``Qwen3_5ForCausalLM.forward``: the layers and the final norm; returns the + hidden rows and the captured auxiliary rows (None without a capture layer).""" + from sglang.srt.eplb.expert_distribution import ( + get_global_expert_distribution_recorder, + ) + from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList + + m = st.model + boundary.open_stream(st) + aux = AuxHiddenStateList() + for i in range(m.start_layer, m.end_layer): + layer = m.layers[i] + capture = aux if getattr(layer, "_is_layer_to_capture", False) else None + with get_global_expert_distribution_recorder().with_current_layer(i): + hidden = self._layer(i, layer, hidden, st, capture) + hidden = boundary.close_stream(st, hidden) + return hidden, (aux if len(aux) else None) + + def _layer(self, i, layer, hidden, st, capture): + """A decoder layer's forward: the mixer stage, then the MoE stage.""" + x = boundary.pre(layer, hidden, st, capture) + part = self.mixers[i](layer, x, st) + x = boundary.post_attn(layer, part, st) + with boundary.ffn_exit(layer, st) as ffn_exit: + out = moe.step(layer, x, st, defer=ffn_exit.defer_moe_finalize) + return ffn_exit.finish(out) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/parity.py b/sgl_nvidia_mega/qwen3_5/runtime/parity.py new file mode 100644 index 00000000..e0e72e80 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/runtime/parity.py @@ -0,0 +1,279 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Test only: the covered step against SGLang's own forward, bit for bit, in eager +steps and inside the decode CUDA graphs (``SGLANG_NVIDIA_MEGAKERNEL_PARITY``). + +``1`` runs SGLang's forward on the step, then the megakernel's, and compares the logits +and the hidden rows they return bit for bit. An eager step logs the comparison at once. +A captured step records both forwards and the comparison, which adds to device counters +on every replay; the next eager forward on the model (a new request's prefill) logs +what the replays added. ``control`` runs SGLang's forward twice instead: the run-to-run +floor of the same comparison. ``trace`` also hooks every module both forwards call and +logs the first module call whose input or output differs (eager steps only). + +Every line also reports how far the two forwards' outputs are apart, which is what a +numeric slot (not bitwise) is read by: per verify row, whether the two top-1 tokens +differ and the |difference| of the two log-softmaxes at SGLang's top-1, and the max abs +difference of the hidden rows. + +A verify forward rewrites its requests' conv windows in place (SGLang's commit later +rolls them back to the accepted token), so the second forward runs on the windows as +they were before the first: their rows are saved and put back between the two. The +verify only reads the SSM checkpoints and only writes this step's ReplaySSM window. +An eager line's valid_rows counts the step's requests with a mamba slot. The forwards +SGLang runs before capturing a graph log as ``capture-warmup``: the graph runner's +placeholder inputs give every row the same KV slot, and there two runs of SGLang's own +forward already differ (``control``), first at the first full-attention layer's +attention, so those lines say nothing about the step. +""" + +from __future__ import annotations + +import logging +import os + +import torch + +logger = logging.getLogger(__name__) + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/layers/logits_processor.py": ("next_token_logits", "hidden_states"), + "srt/model_executor/forward_context.py": ("get_attn_backend",), + "srt/layers/attention/hybrid_linear_attn_backend.py": ( + "linear_attn_backend", + "req_to_token_pool", + "forward_metadata", + ), + "srt/layers/attention/mamba/mamba2_metadata.py": ("mamba_cache_indices",), + "srt/mem_cache/memory_pool.py": ("mamba_pool", "mamba_cache", "conv"), + "srt/model_executor/runner/__init__.py": ("get_is_capture_mode",), +} + +MODE = os.environ.get("SGLANG_NVIDIA_MEGAKERNEL_PARITY", "0") +ENABLED = MODE in ("1", "control", "trace") + +# element size -> the integer type its bits are compared as +_BITS = {1: torch.uint8, 2: torch.int16, 4: torch.int32, 8: torch.int64} + + +class Parity: + """One rank's comparison counters per width: covered steps, the elements of the + logits and of the hidden rows whose bits differ, and the elements compared (zero + when a forward returned no such output, e.g. SGLang's autotune dummy run).""" + + def __init__(self, widths: tuple[int, ...], device, model=None): + self.counters = { + w: torch.zeros(5, dtype=torch.int64, device=device) for w in widths + } + self.logged = {w: [0] * 5 for w in widths} + # rows, top-1 differs, sum and max of |dlp|, hidden max abs; read at each flush + self.distance = { + w: torch.zeros(5, dtype=torch.float64, device=device) for w in widths + } + self.label = " control" if MODE == "control" else "" + self.model = model # whose modules trace mode hooks + + def run(self, step, original, megakernel): + """``original()`` then ``megakernel()`` (``original()`` again for control) on the + same step; returns the second's output.""" + traced = MODE == "trace" and not torch.cuda.is_current_stream_capturing() + slots, convs = conv_rows() + # padded rows hold -1; their slot 0 stand-in is saved and put back unchanged + rows = None if slots is None else slots.clamp(min=0) + saved = [conv[:, rows].clone() for conv in convs] + want, calls = self._traced(original) if traced else (original(), None) + want = (_clone(want.next_token_logits), _clone(want.hidden_states)) + for conv, rows_before in zip(convs, saved): + conv[:, rows] = rows_before + if traced: + got, got_calls = self._traced(megakernel) + logger.info( + "NVIDIA megakernel parity trace %s rows=%d: %s", + step.mode, + step.rows, + first_divergence(calls, got_calls), + ) + else: + got = original() if MODE == "control" else megakernel() + pairs = list(zip(want, (got.next_token_logits, got.hidden_states))) + apart = distance(*pairs[0], *pairs[1]) + if torch.cuda.is_current_stream_capturing(): + counts = self.counters[step.rows] + counts[0:1].add_(1) + for i, (a, b) in enumerate(pairs, start=1): + counts[i : i + 1].add_(differing(a, b)) + counts[i + 2 : i + 3].add_(0 if a is None else a.numel()) + if apart is not None: + d = self.distance[step.rows] + d[0:4:2].add_(apart[0:4:2]) + d[1:2].add_(apart[1:2]) + d[3:5].copy_(torch.maximum(d[3:5], apart[3:5])) + return got + from sglang.srt.model_executor.runner import get_is_capture_mode + + logger.info( + "NVIDIA megakernel parity%s %s %s rows=%d logits_differ=%d " + "hidden_differ=%d logits_compared=%d hidden_compared=%d " + "logits_max_abs=%s hidden_max_abs=%s valid_rows=%s%s", + self.label, + "capture-warmup" if get_is_capture_mode() else "eager", + step.mode, + step.rows, + *(int(differing(a, b)) for a, b in pairs), + *(0 if a is None else a.numel() for a, _ in pairs), + *(_max_abs(a, b) for a, b in pairs), + "n/a" if slots is None else int((slots >= 0).sum()), + _distance_text(apart), + ) + return got + + def flush(self) -> None: + """Log what the replays of captured steps added since the last flush; a host + sync, so only outside capture.""" + if torch.cuda.is_current_stream_capturing(): + return + for rows, counts in self.counters.items(): + now = counts.tolist() + if now == self.logged[rows]: + continue + added = [n - o for n, o in zip(now, self.logged[rows])] + self.logged[rows] = now + apart = self.distance[rows].clone() + self.distance[rows].zero_() + logger.info( + "NVIDIA megakernel parity%s graph target_verify rows=%d steps=%d " + "logits_differ=%d hidden_differ=%d logits_compared=%d " + "hidden_compared=%d%s", + self.label, + rows, + *added, + _distance_text(apart), + ) + + def _traced(self, fn): + """fn()'s output and every module call it made: (name, input, output).""" + names = {m: n or "model" for n, m in self.model.named_modules()} + calls = [] + + def hook(module, args, out): + calls.append((names[module], _first_tensor(args), _first_tensor(out))) + + handles = [m.register_forward_hook(hook) for m in names] + try: + return fn(), calls + finally: + for h in handles: + h.remove() + + +def conv_rows(): + """This step's mamba slots and the pool's conv-window tensors ([layers, slots, + ...]); (None, []) without a linear-attention backend.""" + from sglang.srt.model_executor.forward_context import get_attn_backend + + backend = get_attn_backend() + linear = getattr(backend, "linear_attn_backend", None) + if linear is None: + return None, [] + cache = linear.req_to_token_pool.mamba_pool.mamba_cache + return linear.forward_metadata.mamba_cache_indices, list(cache.conv) + + +def first_divergence(want, got) -> str: + """The first call in got whose input or output bits differ from the same module's + call of the same index in want.""" + index, seen = {}, {} + for name, a_in, a_out in want: + seen[name] = seen.get(name, -1) + 1 + index[(name, seen[name])] = (a_in, a_out) + seen, common = {}, 0 + for name, b_in, b_out in got: + seen[name] = seen.get(name, -1) + 1 + pair = index.get((name, seen[name])) + if pair is None: + continue + common += 1 + for what, a, b in (("input", pair[0], b_in), ("output", pair[1], b_out)): + if a is None or b is None: + continue + if a.shape != b.shape or a.dtype != b.dtype: + return f"{name}#{seen[name]} {what}: {a.dtype}{tuple(a.shape)} against {b.dtype}{tuple(b.shape)}" + n = int(differing(a, b)) + if n: + return f"{name}#{seen[name]} {what}: {n} of {a.numel()} elements differ" + return f"no diverging call among {common} common module calls" + + +def _first_tensor(value): + """A clone of the first tensor in value (a tensor, a tuple / list of them, or an + output with next_token_logits), or None.""" + if isinstance(value, torch.Tensor): + return value.clone() + if isinstance(value, (tuple, list)): + return next((v.clone() for v in value if isinstance(v, torch.Tensor)), None) + logits = getattr(value, "next_token_logits", None) + return logits.clone() if isinstance(logits, torch.Tensor) else None + + +def distance(logits_a, logits_b, hidden_a, hidden_b): + """[rows, rows whose top-1 differs, sum and max over rows of |dlp|, hidden max abs] + on the device, dlp being the log-softmax difference at a's top-1; None without + logits.""" + if logits_a is None or logits_b is None: + return None + top = logits_a.argmax(-1, keepdim=True) + dlp = ( + torch.log_softmax(logits_b.float(), -1).gather(-1, top) + - torch.log_softmax(logits_a.float(), -1).gather(-1, top) + ).abs() + hidden = ( + torch.zeros((), device=logits_a.device) + if hidden_a is None + else (hidden_a.float() - hidden_b.float()).abs().max() + ) + # no host-to-device copy: this also runs inside graph capture + return torch.stack( + [ + torch.full((), float(top.shape[0]), device=logits_a.device), + (logits_b.argmax(-1, keepdim=True) != top).sum().float(), + dlp.sum(), + dlp.max(), + hidden, + ] + ).double() + + +def _distance_text(apart) -> str: + if apart is None: + return "" + rows, top1, total, worst, hidden = apart.tolist() + mean = total / rows if rows else 0.0 + return ( + f" rows_compared={int(rows)} top1_differ={int(top1)} dlp_mean={mean:.3g} " + f"dlp_max={worst:.3g} hidden_dist_max={hidden:.3g}" + ) + + +def differing(a, b): + """The number of elements of a and b whose bits differ (a device scalar).""" + if a is None or b is None: + if a is None and b is None: + return 0 + raise RuntimeError("parity: only one forward returned this output") + if a.shape != b.shape or a.dtype != b.dtype: + raise RuntimeError( + f"parity: {a.dtype}{tuple(a.shape)} against {b.dtype}{tuple(b.shape)}" + ) + bits = _BITS[a.element_size()] + return (a.contiguous().view(bits) != b.contiguous().view(bits)).sum() + + +def _clone(t): + return None if t is None else t.clone() + + +def _max_abs(a, b) -> str: + if a is None: + return "n/a" + return f"{(a.float() - b.float()).abs().max().item():.3g}" diff --git a/sgl_nvidia_mega/qwen3_5/runtime/state.py b/sgl_nvidia_mega/qwen3_5/runtime/state.py new file mode 100644 index 00000000..271e9ba1 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/runtime/state.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""``StepState`` (``st``): what one covered step carries between its step functions. + +The runtime creates one per forward and passes it to every step function. Fields fixed +for the step are set at creation. A package keeps values of its own in ``scratch`` +under ``"."``, and its build result in ``bound[""]``. +""" + +from __future__ import annotations + +from typing import Any + +import torch + + +class StepState: + def __init__( + self, + *, + lm, + forward_batch, + positions: torch.Tensor, + input_ids: torch.Tensor, + slots, + bound: dict, + binding: Any = None, + view: Any = None, + ): + self.lm = lm # Qwen3_5MoeForConditionalGeneration + self.model = lm.model # Qwen3_5MoeForCausalLM: layers, embedding, final norm + self.fb = forward_batch + # what the layers take: the M-RoPE positions [3, rows] when the model uses them + self.positions = positions + self.input_ids = input_ids + self.slots = slots # plan.Slots: which kernels fill this width's step + self.bound: dict[str, Any] = bound # package -> its build() result + self.binding = binding # SGLang's EagleBinding: configuration of the TP group + # Step.view, the forward's EagleForwardView: the attention and GDN metadata and + # the KV and mamba pools, built at capture for a graph step + self.view = view + self.scratch: dict[str, Any] = {} diff --git a/sgl_nvidia_mega/qwen3_5/sm103/__init__.py b/sgl_nvidia_mega/qwen3_5/sm103/__init__.py new file mode 100644 index 00000000..945c4b97 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The SM103 (GB300) kernels of the Qwen3.5 slots and their launchers; no SGLang imports.""" diff --git a/sgl_nvidia_mega/qwen3_5/sm103/hop/__init__.py b/sgl_nvidia_mega/qwen3_5/sm103/hop/__init__.py new file mode 100644 index 00000000..974e14b2 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/hop/__init__.py @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The decode hop (``src/hop.cu``): the TP sum of a verify step's bf16 partials with the +residual add and RMSNorm (+ static FP8) for hidden 4096, through one NVLink multicast +mailbox per process. ``publish`` (or ``publish_fin``, which finalizes the routed MoE rows +first) pushes this rank's rows; ``consume`` pulls every rank's, sums them in rank order, +writes the residual and the norm and re-arms the mailbox. The outputs are those of +FlashInfer's MNNVL CuTe DSL low-latency all-reduce + residual + RMSNorm, bit for bit. + +A consumer that waits past the deadline faults at address 0x0 (fail-stop) instead of +returning: the deadline is compiled in, as SGLang's ``BuildContext.deadline_s``. +""" + +from __future__ import annotations + +import ctypes +from functools import cache +from pathlib import Path + +import torch + +from sgl_nvidia_mega.common import jit + +SOURCE = Path(__file__).parent / "src" / "hop.cu" +HIDDEN, STAGES, ROW_CTAS, ROW_THREADS, QUAD_CTAS, MAX_TOP_K = 4096, 3, 4, 128, 8, 16 +SENTINEL = -32768 # bf16 -0.0 as int16: an empty mailbox slot +WORLDS = (2, 4, 8) + + +class Site(ctypes.Structure): + _fields_ = [ + ("mc", ctypes.c_void_p), + ("uc", ctypes.c_void_p), + ("ctrl", ctypes.c_void_p), + ("rank", ctypes.c_int), + ("world", ctypes.c_int), + ("m_cap", ctypes.c_int), + ] + + +class Args(ctypes.Structure): + _fields_ = ( + [("site", Site)] + + [ + (name, ctypes.c_void_p) + for name in ( + "partial", + "routed", + "expert_w", + "permuted", + "shared", + "residual_in", + "gamma", + "fp8_scale", + "residual_out", + "norm_out", + "fp8_out", + ) + ] + + [("m", ctypes.c_int), ("top_k", ctypes.c_int), ("eps", ctypes.c_float)] + ) + + +@cache +def _kernels(device_index: int, deadline_ns: int) -> dict[str, jit.Kernel]: + major, minor = torch.cuda.get_device_capability(device_index) + cubin = jit.compile_source( + SOURCE.read_text(), + "q35_hop", + f"sm_{major}{minor}a", + ("-std=c++17", "-default-device", f"-DQ35_HOP_DEADLINE_NS={deadline_ns}ull"), + ) + names = ["q35hop_publish", "q35hop_publish_fin_top10", "q35hop_publish_fin"] + names += [f"q35hop_consume_w{w}" for w in WORLDS] + return {n: jit.load(cubin, n, (ROW_THREADS, 1, 1), 0) for n in names} + + +class Hop: + """One process's mailbox and control block, shared by every hop of every step; + construct on every rank of ``group`` (a collective) before any capture.""" + + def __init__(self, *, max_m: int, eps: float, group, device, deadline_s: float): + import torch.distributed as dist + import torch.distributed._symmetric_memory as symm_mem + + self.world = dist.get_world_size(group) + if self.world not in WORLDS: + raise ValueError(f"the hop serves TP {WORLDS}, not {self.world}") + self.rank = dist.get_rank(group) + self.max_m, self.eps, self.device = int(max_m), float(eps), torch.device(device) + self.kernels = _kernels(self.device.index, int(deadline_s * 1e9)) + if symm_mem.get_backend(self.device) is None: + # the backend FlashInfer's fusion workspace would select later in this process + symm_mem.set_backend("NCCL") + self._mailbox = symm_mem.empty( + (STAGES, self.world, self.max_m, HIDDEN), + dtype=torch.int16, + device=self.device, + ) + handle = symm_mem.rendezvous(self._mailbox, group) + multicast = int(handle.multicast_ptr or 0) + if not multicast or multicast % 16: + raise RuntimeError("the hop needs an NVLink multicast mapping") + self._mailbox.fill_(SENTINEL) + self._ctrl = torch.zeros(4, dtype=torch.int32, device=self.device) + self._handle = handle # keeps the mapping alive + self.site = Site( + multicast, + self._mailbox.data_ptr(), + self._ctrl.data_ptr(), + self.rank, + self.world, + self.max_m, + ) + torch.cuda.synchronize(self.device) + dist.barrier(group=group) + + def supports(self, m: int) -> bool: + return 1 <= m <= self.max_m + + def all_reduce_residual_rms_norm( + self, *, local_contribution, residual, gamma, fp8_scale=None + ): + """(norm, residual_out[, fp8]) [M, 4096] of sum_ranks(partial) + residual.""" + m = int(local_contribution.shape[0]) + args, outs = self._args(m, residual, gamma, fp8_scale) + args.partial = local_contribution.data_ptr() + self._launch("q35hop_publish", args, ROW_CTAS) + self._consume(args) + return outs + + def moe_finalize_all_reduce_rms_norm( + self, + *, + routed_output, + expert_weights, + permuted_indices, + gated_shared_output, + residual, + gamma, + fp8_scale=None, + ): + """The same with this rank's partial the finalized MoE rows: per token the top-k + routed rows weighted in k order, plus the gated shared output.""" + m, top_k = (int(n) for n in permuted_indices.shape) + if top_k > MAX_TOP_K: + raise ValueError(f"top-{top_k} routing (at most {MAX_TOP_K})") + args, outs = self._args(m, residual, gamma, fp8_scale) + args.routed, args.expert_w = routed_output.data_ptr(), expert_weights.data_ptr() + args.permuted, args.top_k = permuted_indices.data_ptr(), top_k + args.shared = jit.null_or_pointer(gated_shared_output).value + name = "q35hop_publish_fin_top10" if top_k == 10 else "q35hop_publish_fin" + self._launch(name, args, QUAD_CTAS) + self._consume(args) + return outs + + def _args(self, m: int, residual, gamma, fp8_scale): + if not self.supports(m): + raise ValueError(f"the hop serves 1-{self.max_m} rows, not {m}") + shape = (m, HIDDEN) + outs = [ + torch.empty(shape, dtype=torch.bfloat16, device=self.device), + torch.empty(shape, dtype=torch.bfloat16, device=self.device), + ] + if fp8_scale is not None: + outs.append( + torch.empty(shape, dtype=torch.float8_e4m3fn, device=self.device) + ) + args = Args( + site=self.site, + residual_in=residual.data_ptr(), + gamma=gamma.data_ptr(), + fp8_scale=jit.null_or_pointer(fp8_scale).value, + residual_out=outs[1].data_ptr(), + norm_out=outs[0].data_ptr(), + fp8_out=outs[2].data_ptr() if fp8_scale is not None else None, + m=m, + eps=self.eps, + ) + return args, tuple(outs) + + def _launch(self, name, args, ctas, cluster=None): + jit.launch( + self.kernels[name], + (args.m, ctas, 1), + [args], + torch.cuda.current_stream(self.device).cuda_stream, + cluster, + ) + + def _consume(self, args): + self._launch( + f"q35hop_consume_w{self.world}", args, ROW_CTAS, cluster=(1, ROW_CTAS, 1) + ) diff --git a/sgl_nvidia_mega/qwen3_5/sm103/hop/src/hop.cu b/sgl_nvidia_mega/qwen3_5/sm103/hop/src/hop.cu new file mode 100644 index 00000000..6548daa4 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/hop/src/hop.cu @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// The decode hop of a Qwen3.5 TP verify step: the TP sum of bf16 partials + residual + RMSNorm +// (+ static FP8) for hidden 4096, up to 8 ranks on one NVLink domain, bit-identical to +// FlashInfer's MNNVL CuTe DSL low-latency all-reduce + residual + RMSNorm. +// publish push this rank's bf16 partial [M, 4096] +// publish_fin finalize the routed MoE rows (top-k weighted, plus the gated shared output) and push +// consume pull + rank-order sum + residual + RMSNorm (+ static FP8), re-arm, advance +// +// Transport: every rank multicast-stores its partial into slot mailbox[stage][rank][token][...] +// of every rank; a reader polls its own copy until no bf16 -0.0 sentinel is left, sums the ranks +// in order and writes the sentinel back. Three stages rotate, one per hop. A fragment is 8 +// consecutive bf16 of a token row (frag = 0..511); a token row is finished by one cluster of +// kRowCtas CTAs x kRowThreads threads (CTA c owns fragments [128c, 128c + 128)). +// +// Built with NVRTC: C-linkage entries per instantiation; Q35_HOP_DEADLINE_NS (0: none) is the +// consumer's wait deadline, past which it faults at address 0x0 (fail-stop). + +#include + +// NVRTC has no +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; + +#ifndef Q35_HOP_DEADLINE_NS +#define Q35_HOP_DEADLINE_NS 0ull +#endif + +namespace q35hop { + + +constexpr int kHidden = 4096; +constexpr int kVec = 8; +constexpr int kFrags = kHidden / kVec; +constexpr int kStages = 3; +constexpr int kRowCtas = 4; +constexpr int kRowThreads = 128; +constexpr int kRowWarps = kRowThreads / 32; +constexpr uint32_t kNegZeroPair = 0x80008000u; +static_assert(kRowCtas * kRowThreads == kFrags, "a row is one fragment per thread of the cluster"); +static_assert(kRowWarps == 4, "row_sumsq_fold folds exactly four warp sums per CTA"); + +// Per-process control block in device memory. +struct Ctrl { + uint32_t stage; // stage of the next hop + uint32_t arrive; // CTA tickets of the running consumer launch + uint32_t seq; // hops completed + uint32_t pad; +}; + +// Passed by value as a kernel parameter. +struct Site { + void* mc; // multicast base of the mailbox [kStages][world][m_cap][kHidden] bf16 + void* uc; // this rank's unicast view of the same mailbox + Ctrl* ctrl; + int rank, world, m_cap; +}; + +// ---------------------------------------------------------------- PTX + +__device__ __forceinline__ void pdl_wait() { asm volatile("griddepcontrol.wait;" ::: "memory"); } +__device__ __forceinline__ void pdl_trigger() { asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); } +__device__ __forceinline__ unsigned long long gtimer() { + unsigned long long t; + asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t)); + return t; +} +__device__ __forceinline__ uint32_t ld_volatile_u32(const uint32_t* p) { + uint32_t v; + asm volatile("ld.volatile.global.u32 %0, [%1];" : "=r"(v) : "l"(p) : "memory"); + return v; +} +__device__ __forceinline__ uint4 ld_volatile_v4(const void* p) { + uint4 v; + asm volatile("ld.volatile.global.v4.u32 {%0,%1,%2,%3}, [%4];" + : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) + : "l"(p) + : "memory"); + return v; +} +__device__ __forceinline__ void st_v4(void* p, uint4 v) { + asm volatile("st.global.v4.u32 [%0], {%1,%2,%3,%4};" ::"l"(p), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w) + : "memory"); +} +__device__ __forceinline__ void mc_st_v4(void* mc, uint4 v) { + asm volatile("multimem.st.relaxed.sys.global.v4.bf16x2 [%0], {%1,%2,%3,%4};" ::"l"(mc), "r"(v.x), + "r"(v.y), "r"(v.z), "r"(v.w) + : "memory"); +} +__device__ __forceinline__ void mc_st_v2(void* mc, uint2 v) { + asm volatile("multimem.st.relaxed.sys.global.v2.bf16x2 [%0], {%1,%2};" ::"l"(mc), "r"(v.x), "r"(v.y) : "memory"); +} +__device__ __forceinline__ uint32_t cluster_ctarank() { + uint32_t r; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(r)); + return r; +} +__device__ __forceinline__ float div_full(float a, float b) { + float r; + asm("div.full.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} +// Two f32 to an e4m3 pair; lo lands in the low byte. +__device__ __forceinline__ uint16_t cvt_e4m3x2(float lo, float hi) { + uint16_t r; + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(r) : "f"(hi), "f"(lo)); + return r; +} + +// ---------------------------------------------------------------- bf16 + +__device__ __forceinline__ float bf_lo(uint32_t w) { return __uint_as_float(w << 16); } +__device__ __forceinline__ float bf_hi(uint32_t w) { return __uint_as_float(w & 0xFFFF0000u); } +__device__ __forceinline__ uint32_t pack_bf16x2(float lo, float hi) { + __nv_bfloat162 v = __floats2bfloat162_rn(lo, hi); + return *reinterpret_cast(&v); +} +__device__ __forceinline__ void unpack8(uint4 v, float f[8]) { + f[0] = bf_lo(v.x); f[1] = bf_hi(v.x); f[2] = bf_lo(v.y); f[3] = bf_hi(v.y); + f[4] = bf_lo(v.z); f[5] = bf_hi(v.z); f[6] = bf_lo(v.w); f[7] = bf_hi(v.w); +} +__device__ __forceinline__ uint4 pack8(const float f[8]) { + return make_uint4(pack_bf16x2(f[0], f[1]), pack_bf16x2(f[2], f[3]), pack_bf16x2(f[4], f[5]), + pack_bf16x2(f[6], f[7])); +} +__device__ __forceinline__ bool word_has_negzero(uint32_t w) { + return (w & 0xFFFFu) == 0x8000u || (w >> 16) == 0x8000u; +} +__device__ __forceinline__ bool frag_has_negzero(uint4 v) { + return word_has_negzero(v.x) | word_has_negzero(v.y) | word_has_negzero(v.z) | word_has_negzero(v.w); +} +__device__ __forceinline__ uint32_t sanitize_word(uint32_t w) { + if ((w & 0xFFFFu) == 0x8000u) w &= 0xFFFF0000u; + if ((w >> 16) == 0x8000u) w &= 0x0000FFFFu; + return w; +} + +// ---------------------------------------------------------------- stage protocol + +__device__ __forceinline__ size_t slot(const Site& s, uint32_t stage, int src, int token, int frag) { + return ((static_cast(stage) * s.world + src) * s.m_cap + token) * kHidden + + static_cast(frag) * kVec; +} + +struct Hop { + uint32_t stage; + uint32_t seq; + uint32_t ticket; // thread 0 only +}; + +// Consumer launch, every thread of every participating CTA, after griddepcontrol.wait. Thread +// 0 takes a ticket whose value is consumed only in hop_leave, so the atomic stays off the +// critical path. +__device__ __forceinline__ Hop hop_enter(const Site& s) { + Hop h; + h.stage = ld_volatile_u32(&s.ctrl->stage); + h.seq = ld_volatile_u32(&s.ctrl->seq); + h.ticket = threadIdx.x == 0 ? atomicAdd(&s.ctrl->arrive, 1u) : 0u; + return h; +} + +// Thread 0 of every participating CTA; the last arrival advances the stage. No CTA of this +// launch can read the advanced value, since every CTA read the stage before taking its ticket. +__device__ __forceinline__ void hop_leave(const Site& s, const Hop& h, uint32_t participants) { + if (threadIdx.x == 0 && h.ticket == participants - 1) { + s.ctrl->arrive = 0; + s.ctrl->stage = (h.stage + 1) % kStages; + s.ctrl->seq = h.seq + 1; + } +} + +// Producer-only launch, after griddepcontrol.wait. +__device__ __forceinline__ uint32_t stage_peek(const Site& s) { return ld_volatile_u32(&s.ctrl->stage); } + +// ---------------------------------------------------------------- producer + +// This rank's final bf16 partial for one fragment; exactly once per (hop, rank, token, frag). +// A launch that also stores its own outputs should push first: the push starts the flight. +__device__ __forceinline__ void push_frag(const Site& s, uint32_t stage, int token, int frag, uint4 v) { + v = make_uint4(sanitize_word(v.x), sanitize_word(v.y), sanitize_word(v.z), sanitize_word(v.w)); + mc_st_v4(static_cast(s.mc) + slot(s, stage, s.rank, token, frag), v); +} + +// Same for 4 consecutive bf16 (half a fragment, columns [4q, 4q + 4)); a fragment may be pushed +// as two halves by different threads. +__device__ __forceinline__ void push_quad(const Site& s, uint32_t stage, int token, int quad, uint2 v) { + v = make_uint2(sanitize_word(v.x), sanitize_word(v.y)); + mc_st_v2(static_cast(s.mc) + slot(s, stage, s.rank, token, quad >> 1) + (quad & 1) * 4, v); +} + +// ---------------------------------------------------------------- consumer + +// A peer that missed the wait deadline: store to address 0x0, then trap (fail-stop). +__device__ __forceinline__ void fault_at_null() { + asm volatile("{\n.reg .u64 a;\nmov.u64 a, 0;\nst.volatile.global.u32 [a], 0;\ntrap;\n}" ::: "memory"); +} + +// Sum over ranks 0..W-1 (fp32, from 0) of one fragment; spins on the sentinel. A wait past +// Q35_HOP_DEADLINE_NS (if not 0) stops the kernel. +template +__device__ __forceinline__ bool pull_frag(const Site& s, uint32_t stage, int token, int frag, float acc[8]) { + const uint16_t* uc = static_cast(s.uc); + uint4 v[W]; +#pragma unroll + for (int r = 0; r < W; ++r) v[r] = ld_volatile_v4(uc + slot(s, stage, r, token, frag)); + uint32_t pending = 0; +#pragma unroll + for (int r = 0; r < W; ++r) pending |= frag_has_negzero(v[r]) ? (1u << r) : 0u; + unsigned long long t0 = 0; + uint32_t spins = 0; + while (pending) { +#pragma unroll + for (int r = 0; r < W; ++r) + if (pending & (1u << r)) v[r] = ld_volatile_v4(uc + slot(s, stage, r, token, frag)); + uint32_t still = 0; +#pragma unroll + for (int r = 0; r < W; ++r) + if ((pending & (1u << r)) && frag_has_negzero(v[r])) still |= 1u << r; + pending = still; + if (Q35_HOP_DEADLINE_NS != 0 && pending && ((++spins & 255u) == 0)) { + const unsigned long long now = gtimer(); + if (t0 == 0) { + t0 = now; + } else if (now - t0 > Q35_HOP_DEADLINE_NS) { + fault_at_null(); + } + } + } +#pragma unroll + for (int e = 0; e < 8; ++e) acc[e] = 0.f; +#pragma unroll + for (int r = 0; r < W; ++r) { + float f[8]; + unpack8(v[r], f); +#pragma unroll + for (int e = 0; e < 8; ++e) acc[e] = __fadd_rn(acc[e], f[e]); + } + return true; +} + +// Write the sentinel back into the W slots this thread consumed (any time after pull_frag). +template +__device__ __forceinline__ void rearm_frag(const Site& s, uint32_t stage, int token, int frag) { + uint16_t* uc = static_cast(s.uc); + const uint4 z = make_uint4(kNegZeroPair, kNegZeroPair, kNegZeroPair, kNegZeroPair); +#pragma unroll + for (int r = 0; r < W; ++r) st_v4(uc + slot(s, stage, r, token, frag), z); +} + +// ---------------------------------------------------------------- row finish + +// prenorm = bf16(acc + residual); pre[] gets the bf16-rounded values the norm uses. +__device__ __forceinline__ uint4 residual_add(const float acc[8], uint4 residual, float pre[8]) { + float r[8]; + unpack8(residual, r); +#pragma unroll + for (int e = 0; e < 8; ++e) pre[e] = __fadd_rn(acc[e], r[e]); + const uint4 out = pack8(pre); + unpack8(out, pre); + return out; +} + +__device__ __forceinline__ float sumsq8(const float x[8]) { + float s = 0.f; +#pragma unroll + for (int e = 0; e < 8; ++e) s = __fadd_rn(s, __fmul_rn(x[e], x[e])); + return s; +} + +struct RowSmem { + float xs[kRowCtas][kRowWarps]; + uint64_t mbar; +}; + +// Prologue of every CTA of the row cluster, all threads, before griddepcontrol.wait: arm the +// exchange barrier and make it visible to the peers before any of them can send. +__device__ __forceinline__ void row_init(RowSmem& rs) { + if (threadIdx.x == 0) { + const uint32_t a = static_cast(__cvta_generic_to_shared(&rs.mbar)); + asm volatile("mbarrier.init.shared::cta.b64 [%0], 1;" ::"r"(a) : "memory"); + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(a), + "r"(static_cast(sizeof(rs.xs))) + : "memory"); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); + } + asm volatile("barrier.cluster.arrive.relaxed.aligned;\n\tbarrier.cluster.wait.acquire.aligned;" ::: "memory"); +} + +// Row total of the per-thread values: lane butterfly per warp, then each warp sends its sum to +// every CTA of the cluster with st.async; each CTA waits on its own mbarrier (no release fence, +// so this thread's earlier remote stores are not waited for) and folds per CTA as +// (w0 + w2) + (w1 + w3), then the CTAs in rank order from 0. +__device__ __forceinline__ float row_sumsq_fold(float v, RowSmem& rs) { + const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5; +#pragma unroll + for (int off = 1; off < 32; off <<= 1) v = __fadd_rn(v, __shfl_xor_sync(0xffffffffu, v, off)); + if (lane < kRowCtas) { + const uint32_t me = cluster_ctarank(); + const uint32_t slot_l = static_cast(__cvta_generic_to_shared(&rs.xs[me][warp])); + const uint32_t mbar_l = static_cast(__cvta_generic_to_shared(&rs.mbar)); + uint32_t slot_r, mbar_r; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(slot_r) : "r"(slot_l), "r"(lane)); + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(mbar_r) : "r"(mbar_l), "r"(lane)); + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.f32 [%0], %1, [%2];" ::"r"(slot_r), + "f"(v), "r"(mbar_r) + : "memory"); + } + { + const uint32_t a = static_cast(__cvta_generic_to_shared(&rs.mbar)); + asm volatile( + "{\n\t.reg .pred P;\n\tWAIT_ROW:\n\t" + "mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P, [%0], 0;\n\t" + "@!P bra WAIT_ROW;\n\t}" ::"r"(a) + : "memory"); + } + float total = 0.f; +#pragma unroll + for (int c = 0; c < kRowCtas; ++c) + total = __fadd_rn(total, __fadd_rn(__fadd_rn(rs.xs[c][0], rs.xs[c][2]), __fadd_rn(rs.xs[c][1], rs.xs[c][3]))); + return total; +} + +__device__ __forceinline__ float inv_rms(float row_sumsq, float eps) { + return rsqrtf(__fadd_rn(row_sumsq / static_cast(kHidden), eps)); +} + +// bf16((x * inv) * gamma), gamma being the applied multiplier. +__device__ __forceinline__ uint4 norm_apply(const float pre[8], float inv, const float gamma[8]) { + float out[8]; +#pragma unroll + for (int e = 0; e < 8; ++e) out[e] = __fmul_rn(__fmul_rn(pre[e], inv), gamma[e]); + return pack8(out); +} + +// The reciprocal the static per-tensor FP8 quant multiplies by. +__device__ __forceinline__ float inv_scale_of(float scale) { return div_full(1.f, scale); } + +// e4m3(clamp(x * inv_scale, +-448)) of the bf16 norm output, 8 bytes. +__device__ __forceinline__ uint2 quant_fp8(uint4 norm_bf16, float inv_scale) { + float q[8]; + unpack8(norm_bf16, q); + uint32_t w[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const float a = fminf(fmaxf(__fmul_rn(q[2 * j], inv_scale), -448.f), 448.f); + const float b = fminf(fmaxf(__fmul_rn(q[2 * j + 1], inv_scale), -448.f), 448.f); + w[j] = cvt_e4m3x2(a, b); + } + return make_uint2(w[0] | (w[1] << 16), w[2] | (w[3] << 16)); +} + +} // namespace q35hop + +// a named namespace: the C entries below name Args in their signatures +namespace q35hop_kernels { + + +using namespace q35hop; + +struct Args { + Site site; + const uint16_t* partial; // [M, H] bf16 (publish) + const uint16_t* routed; // [M*K, H] bf16 (publish_fin), rows indexed by permuted + const uint16_t* expert_w; // [M, K] bf16 + const int32_t* permuted; // [M, K]; -1 skips a slot + const uint16_t* shared; // [M, H] bf16 gated shared expert output, or null + const uint16_t* residual_in; // [M, H] bf16 + const uint16_t* gamma; // [H] bf16, the applied multiplier + const float* fp8_scale; // [1], or null + uint16_t* residual_out; // [M, H] bf16 + uint16_t* norm_out; // [M, H] bf16, or null + uint8_t* fp8_out; // [M, H] e4m3, or null + int m, top_k; + float eps; +}; + +constexpr int kMaxTopK = 16; + +__device__ __forceinline__ uint4 ld_v4(const void* p) { return *static_cast(p); } + +__device__ __forceinline__ size_t row_elem(int token, int frag) { + return static_cast(token) * kHidden + static_cast(frag) * kVec; +} + +// Finalize-publish, 4 bf16 per thread (8 CTAs x 128 threads per token): per element +// sum_k w_k * routed[perm_k] in k order with separate multiply and add, then the shared output, +// as the reference finalize does. TOPK > 0 fixes top-k at compile time, so every routed row is +// loaded into registers before any accumulation (TOPK = 0 reads top_k at run time). +template +__device__ __forceinline__ void publish_fin_quad_kernel(const Args& a) { + constexpr int kSlots = TOPK > 0 ? TOPK : kMaxTopK; + __shared__ int32_t rows_s[kSlots]; + __shared__ float wts_s[kSlots]; + const int top_k = TOPK > 0 ? TOPK : a.top_k; + const int token = blockIdx.x; + const int quad = blockIdx.y * kRowThreads + threadIdx.x; + if (threadIdx.x < top_k) { + const int i = token * top_k + threadIdx.x; + const int r = a.permuted[i]; + rows_s[threadIdx.x] = r; + wts_s[threadIdx.x] = r < 0 ? 0.f : __uint_as_float(static_cast(a.expert_w[i]) << 16); + } + __syncthreads(); + const size_t col = static_cast(quad) * 4; + pdl_wait(); + const uint32_t stage = stage_peek(a.site); + // Read after the wait: the side-stream join that produces it may be captured as a programmatic + // edge, which lets this launch start before the producer has finished. + uint2 sh = make_uint2(0, 0); + if (a.shared != nullptr) sh = *reinterpret_cast(a.shared + static_cast(token) * kHidden + col); + int rows[kSlots]; +#pragma unroll + for (int k = 0; k < kSlots; ++k) rows[k] = k < top_k ? rows_s[k] : -1; + uint2 src[kSlots]; +#pragma unroll + for (int k = 0; k < kSlots; ++k) + src[k] = rows[k] >= 0 ? *reinterpret_cast(a.routed + static_cast(rows[k]) * kHidden + col) + : make_uint2(0, 0); + float acc[4] = {0.f, 0.f, 0.f, 0.f}; +#pragma unroll + for (int k = 0; k < kSlots; ++k) { + if (rows[k] >= 0) { + const float w = wts_s[k]; + const float f[4] = {bf_lo(src[k].x), bf_hi(src[k].x), bf_lo(src[k].y), bf_hi(src[k].y)}; +#pragma unroll + for (int e = 0; e < 4; ++e) acc[e] = __fadd_rn(acc[e], __fmul_rn(f[e], w)); + } + } + if (a.shared != nullptr) { + const float f[4] = {bf_lo(sh.x), bf_hi(sh.x), bf_lo(sh.y), bf_hi(sh.y)}; +#pragma unroll + for (int e = 0; e < 4; ++e) acc[e] = __fadd_rn(acc[e], f[e]); + } + push_quad(a.site, stage, token, quad, make_uint2(pack_bf16x2(acc[0], acc[1]), pack_bf16x2(acc[2], acc[3]))); + pdl_trigger(); +} + +__device__ __forceinline__ void publish_kernel(const Args& a) { + const int token = blockIdx.x; + const int frag = blockIdx.y * kRowThreads + threadIdx.x; + pdl_wait(); + const uint32_t stage = stage_peek(a.site); + push_frag(a.site, stage, token, frag, ld_v4(a.partial + row_elem(token, frag))); + pdl_trigger(); +} + +template +__device__ __forceinline__ void consume_kernel(const Args& a) { + __shared__ RowSmem rs; + const int token = blockIdx.x; + const int frag = blockIdx.y * kRowThreads + threadIdx.x; + pdl_trigger(); + float gam[8]; + unpack8(ld_v4(a.gamma + static_cast(frag) * kVec), gam); + const float inv_s = a.fp8_out != nullptr ? inv_scale_of(*a.fp8_scale) : 1.f; + row_init(rs); + pdl_wait(); + const Hop h = hop_enter(a.site); + const uint4 res = ld_v4(a.residual_in + row_elem(token, frag)); + float acc[8]; + pull_frag(a.site, h.stage, token, frag, acc); + float pre[8]; + const uint4 res_out = residual_add(acc, res, pre); + const float total = row_sumsq_fold(sumsq8(pre), rs); + const uint4 norm = norm_apply(pre, inv_rms(total, a.eps), gam); + *reinterpret_cast(a.residual_out + row_elem(token, frag)) = res_out; + if (a.norm_out != nullptr) *reinterpret_cast(a.norm_out + row_elem(token, frag)) = norm; + if (a.fp8_out != nullptr) *reinterpret_cast(a.fp8_out + row_elem(token, frag)) = quant_fp8(norm, inv_s); + rearm_frag(a.site, h.stage, token, frag); + hop_leave(a.site, h, gridDim.x * gridDim.y); +} + +} // namespace q35hop_kernels + +using q35hop_kernels::Args; + +extern "C" __global__ void __launch_bounds__(q35hop::kRowThreads) q35hop_publish(Args a) { + q35hop_kernels::publish_kernel(a); +} + +extern "C" __global__ void __launch_bounds__(q35hop::kRowThreads) q35hop_publish_fin_top10(Args a) { + q35hop_kernels::publish_fin_quad_kernel<10>(a); +} + +extern "C" __global__ void __launch_bounds__(q35hop::kRowThreads) q35hop_publish_fin(Args a) { + q35hop_kernels::publish_fin_quad_kernel<0>(a); +} + +extern "C" __global__ void __launch_bounds__(q35hop::kRowThreads) q35hop_consume_w2(Args a) { + q35hop_kernels::consume_kernel<2>(a); +} + +extern "C" __global__ void __launch_bounds__(q35hop::kRowThreads) q35hop_consume_w4(Args a) { + q35hop_kernels::consume_kernel<4>(a); +} + +extern "C" __global__ void __launch_bounds__(q35hop::kRowThreads) q35hop_consume_w8(Args a) { + q35hop_kernels::consume_kernel<8>(a); +} diff --git a/sgl_nvidia_mega/qwen3_5/sm103/moe/__init__.py b/sgl_nvidia_mega/qwen3_5/sm103/moe/__init__.py new file mode 100644 index 00000000..4ebe4802 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/moe/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The MoE kernels: the router front (``router``).""" diff --git a/sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py b/sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py new file mode 100644 index 00000000..d21a4890 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The routed experts of up to 7 rows in one launch (``src/mkm_moe.cu``), on the router +front's outputs: the top-10 of the logits, FC1, SwiGLU, the NVFP4 requantization of the +intermediate and FC2, reading the trtllm-gen weight layout in place. It writes the +deferred-finalize rows (row ``t * 10 + k`` is expert ``k`` of token ``t``) and the +expert weights, as the trtllm runner's deferred finalize takes them. +""" + +from __future__ import annotations + +import ctypes +from functools import cache +from pathlib import Path + +import torch + +from sgl_nvidia_mega.common import jit + +SOURCE = Path(__file__).parent / "src" / "mkm_moe.cu" +HIDDEN, EXPERTS, TOP_K, MAX_TOKENS = 4096, 512, 10, 7 +W13_ROWS = 256 # an expert's gate and up rows at TP8 (intermediate 128 per rank) +W13_BOX_ROWS = 128 # kW13BoxRows: rows of one W13 TMA box +THREADS = 256 +CTAS = ( + 2 * MAX_TOKENS * TOP_K +) # a 2-CTA cluster per selectable expert; the unused ones exit +SMEM_BYTES = 211968 # sizeof(Smem) + 1024, checked when the source compiles + + +class Params(ctypes.Structure): + """qwen_mkm_moe::Params.""" + + _fields_ = [ + (name, ctypes.c_void_p) + for name in ( + "logits", + "expert_weights", + "topk_ids", + "x_sf", + "w13_sf", + "w2_sf", + "g1_alpha", + "g1_scale_c", + "g2_alpha", + "out", + "trace", + ) + ] + [("num_tokens", ctypes.c_int)] + + +@cache +def _kernel(device_index: int) -> jit.Kernel: + major, minor = torch.cuda.get_device_capability(device_index) + cubin = jit.compile_source( + SOURCE.read_text(), + "q35_mkm_moe", + f"sm_{major}{minor}a", + ("-std=c++17", "-default-device", f"-DQ35_MKM_SMEM_BYTES={SMEM_BYTES}"), + ) + return jit.load(cubin, "q35_mkm_moe", (THREADS, 1, 1), SMEM_BYTES) + + +def build(device_index: int) -> None: + """Compile the kernel (no compile inside a step).""" + _kernel(device_index) + + +def _map( + tensor: torch.Tensor, + rows: int, + row_bytes: int, + box_bytes: int, + box_rows: int, + swizzle, +): + """A 2-D uint8 TMA map of ``tensor`` as [rows, row_bytes].""" + cu = jit.driver() + u64, u32 = cu.cuuint64_t, cu.cuuint32_t + (tensor_map,) = jit.check( + cu.cuTensorMapEncodeTiled( + cu.CUtensorMapDataType.CU_TENSOR_MAP_DATA_TYPE_UINT8, + 2, + tensor.data_ptr(), + [u64(row_bytes), u64(rows)], + [u64(row_bytes)], + [u32(box_bytes), u32(box_rows)], + [u32(1), u32(1)], + cu.CUtensorMapInterleave.CU_TENSOR_MAP_INTERLEAVE_NONE, + swizzle, + cu.CUtensorMapL2promotion.CU_TENSOR_MAP_L2_PROMOTION_L2_256B, + cu.CUtensorMapFloatOOBfill.CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE, + ), + "cuTensorMapEncodeTiled", + ) + return tensor_map + + +def weight_maps(w13: torch.Tensor, w2: torch.Tensor) -> tuple: + """The TMA maps of a layer's W13 [512, 256, 2048] and W2 [512, 4096, 64] (uint8, the + trtllm-gen layout); they stay valid as long as the weights.""" + swizzle = jit.driver().CUtensorMapSwizzle + return ( + _map( + w13, + EXPERTS * W13_ROWS, + HIDDEN // 2, + 128, + W13_BOX_ROWS, + swizzle.CU_TENSOR_MAP_SWIZZLE_128B, + ), + _map(w2, EXPERTS * HIDDEN, 64, 64, 256, swizzle.CU_TENSOR_MAP_SWIZZLE_64B), + ) + + +def mkm_moe( + maps: tuple, + logits: torch.Tensor, + x_fp4: torch.Tensor, + x_sf: torch.Tensor, + w13_sf: torch.Tensor, + w2_sf: torch.Tensor, + g1_alpha: torch.Tensor, + g1_scale_c: torch.Tensor, + g2_alpha: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """(gemm2_out bf16 [M * 10, 4096], expert_weights bf16 [M, 10]) for logits [M, 512] + bf16 (M <= 7) and the router front's x_fp4 [M, 2048] / x_sf [M, 256] (uint8, linear + e4m3); w13_sf [512, 256, 256] / w2_sf [512, 4096, 8] e4m3 viewed as uint8; g1_alpha, + g1_scale_c, g2_alpha [512] fp32.""" + m = logits.shape[0] + if not 1 <= m <= MAX_TOKENS: + raise ValueError(f"mkm: {m} rows (1-{MAX_TOKENS})") + device = logits.device + gemm2_out = torch.empty((m * TOP_K, HIDDEN), dtype=torch.bfloat16, device=device) + expert_weights = torch.empty((m, TOP_K), dtype=torch.bfloat16, device=device) + topk_ids = torch.empty((m, TOP_K), dtype=torch.int32, device=device) + swizzle = jit.driver().CUtensorMapSwizzle.CU_TENSOR_MAP_SWIZZLE_128B + x_map = _map(x_fp4, m, HIDDEN // 2, 128, 8, swizzle) + p = Params( + logits=logits.data_ptr(), + expert_weights=expert_weights.data_ptr(), + topk_ids=topk_ids.data_ptr(), + x_sf=x_sf.data_ptr(), + w13_sf=w13_sf.data_ptr(), + w2_sf=w2_sf.data_ptr(), + g1_alpha=g1_alpha.data_ptr(), + g1_scale_c=g1_scale_c.data_ptr(), + g2_alpha=g2_alpha.data_ptr(), + out=gemm2_out.data_ptr(), + num_tokens=m, + ) + jit.launch( + _kernel(device.index), + (CTAS, 1, 1), + [*maps, x_map, p], + torch.cuda.current_stream().cuda_stream, + ) + return gemm2_out, expert_weights diff --git a/sgl_nvidia_mega/qwen3_5/sm103/moe/router.py b/sgl_nvidia_mega/qwen3_5/sm103/moe/router.py new file mode 100644 index 00000000..e99b8a66 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/moe/router.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The router GEMM and the NVFP4 quantization of its input in one launch, for the +trtllm MoE runner, which routes from the logits itself (``src/router_front.cu``). + +The logits equal SGLang's cuBLAS router GEMM bit for bit, and the quantization equals +the runner's own NVFP4 input quantizer (16-element blocks, e4m3 scales, linear layout). +""" + +from __future__ import annotations + +import ctypes +from functools import cache +from pathlib import Path + +import torch + +from sgl_nvidia_mega.common import jit + +SOURCE = Path(__file__).parent / "src" / "router_front.cu" +HIDDEN, EXPERTS, MAX_TOKENS = 4096, 512, 16 +THREADS, CTAS = 256, 64 # 8 experts a CTA +_ROW_BYTES = HIDDEN * 2 + 16 +_PART_BYTES = 2 * 16 * 8 * 4 +MODE_QUANT = 1 + + +def max_m_bucket(m: int) -> int: + """The compiled row capacity that serves m rows.""" + return 1 if m <= 1 else 8 if m <= 8 else 16 + + +def shared_bytes(max_m: int) -> int: + """GemvSmem::kBytes: 8 weight rows, max_m input rows, the fp32 partials and + two mbarriers.""" + return 8 * _ROW_BYTES + max_m * _ROW_BYTES + _PART_BYTES + 16 + + +class Params(ctypes.Structure): + """qwen_router_front::Params.""" + + _fields_ = [ + (name, ctypes.c_void_p) + for name in ( + "x", + "weight", + "global_scale", + "logits", + "topk_weights", + "topk_ids", + "x_fp4", + "x_sf", + "tile_idx_to_expert_idx", + "tile_idx_to_mn_limit", + "expanded_idx_to_permuted_idx", + "permuted_idx_to_expanded_idx", + "total_num_padded_tokens", + "num_non_exiting_tiles", + "arrival", + "trace", + ) + ] + [ + ("num_tokens", ctypes.c_uint32), + ("tile_log2", ctypes.c_uint32), + ("mode", ctypes.c_uint32), + ("sh_weight", ctypes.c_void_p), + ("sh_in_scale", ctypes.c_void_p), + ("sh_w_scale", ctypes.c_void_p), + ("sh_out", ctypes.c_void_p), + ("sh_n", ctypes.c_uint32), + ] + + +@cache +def _kernel(device_index: int, max_m: int) -> jit.Kernel: + major, minor = torch.cuda.get_device_capability(device_index) + cubin = jit.compile_source( + SOURCE.read_text(), + f"q35_router_front_m{max_m}", + f"sm_{major}{minor}a", + ("-std=c++17", "-default-device", f"-DQ35_ROUTER_MAX_M={max_m}"), + ) + return jit.load( + cubin, "q35_router_gemm_quant", (THREADS, 1, 1), shared_bytes(max_m) + ) + + +def build(device_index: int, max_rows: int) -> None: + """Compile every row capacity up to max_rows (no compile inside a step).""" + for m in sorted({max_m_bucket(r) for r in range(1, max_rows + 1)}): + _kernel(device_index, m) + + +def router_gemm_quant( + x: torch.Tensor, weight: torch.Tensor, global_scale: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """x [M, 4096] bf16 (M <= 16), weight [512, 4096] bf16, global_scale [1] fp32: + (logits bf16 [M, 512], x_fp4 uint8 [M, 2048], x_sf uint8 [M, 256]).""" + m = x.shape[0] + # the kernel reads x as packed rows (bulk copies) and serves at most 16 + if x.dtype != torch.bfloat16 or x.shape[1:] != (HIDDEN,) or not x.is_contiguous(): + raise ValueError( + f"router front: x {x.dtype} {tuple(x.stride())} {tuple(x.shape)}" + ) + if not 1 <= m <= MAX_TOKENS: + raise ValueError(f"router front: {m} rows (1-{MAX_TOKENS})") + logits = torch.empty((m, EXPERTS), dtype=torch.bfloat16, device=x.device) + x_fp4 = torch.empty((m, HIDDEN // 2), dtype=torch.uint8, device=x.device) + x_sf = torch.empty((m, HIDDEN // 16), dtype=torch.uint8, device=x.device) + p = Params( + x=x.data_ptr(), + weight=weight.data_ptr(), + global_scale=global_scale.data_ptr(), + logits=logits.data_ptr(), + x_fp4=x_fp4.data_ptr(), + x_sf=x_sf.data_ptr(), + num_tokens=m, + mode=MODE_QUANT, + ) + kernel = _kernel(x.device.index, max_m_bucket(m)) + jit.launch(kernel, (CTAS, 1, 1), [p], torch.cuda.current_stream().cuda_stream) + return logits, x_fp4, x_sf diff --git a/sgl_nvidia_mega/qwen3_5/sm103/moe/src/mkm_moe.cu b/sgl_nvidia_mega/qwen3_5/sm103/moe/src/mkm_moe.cu new file mode 100644 index 00000000..83925119 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/moe/src/mkm_moe.cu @@ -0,0 +1,1021 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Fused small-M NVFP4 MoE expert phase for Qwen3.5 TP8 decode verify (M <= 7 tokens, top-10, 512 experts, +// hidden 4096, intermediate 128 per rank): the top-10 of the router logits in the trtllm-gen routing's order and +// weights, FC1, SwiGLU, the NVFP4 requantization and FC2 in one launch over the trtllm-gen weight layout. Writes the +// deferred-finalize rows (row t * 10 + k, identity permutation) and the expert weights. +// +// Built with NVRTC: the kernel has C linkage (q35_mkm_moe), and the launcher's dynamic shared memory size +// (Q35_MKM_SMEM_BYTES) is checked against the layout at compile time. + +#include +#include +#include + +// NVRTC has no , or +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; +typedef unsigned long long uintptr_t; +typedef struct __align__(64) { + uint64_t opaque[16]; +} CUtensorMap; +#ifndef INFINITY +#define INFINITY __int_as_float(0x7f800000) +#endif + +// Minimal SM100 (tcgen05 / TMA / mbarrier) PTX helpers for the small-M NVFP4 MoE engine. + +namespace qwen35 { + +__device__ __forceinline__ uint32_t smem_u32(const void* p) { + return static_cast(__cvta_generic_to_shared(p)); +} + +// ---------------------------------------------------------------- mbarrier +__device__ __forceinline__ void mbar_init(uint64_t* bar, uint32_t count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" ::"r"(smem_u32(bar)), "r"(count) : "memory"); +} +__device__ __forceinline__ void fence_mbar_init() { + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); +} +__device__ __forceinline__ void mbar_expect_tx(uint64_t* bar, uint32_t bytes) { + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(smem_u32(bar)), "r"(bytes) : "memory"); +} +__device__ __forceinline__ void mbar_arrive(uint64_t* bar) { + asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];" ::"r"(smem_u32(bar)) : "memory"); +} +__device__ __forceinline__ bool mbar_try_wait(uint64_t* bar, uint32_t phase) { + uint32_t ok; + asm volatile( + "{\n .reg .pred p;\n mbarrier.try_wait.parity.shared::cta.b64 p, [%1], %2;\n selp.u32 %0, 1, 0, p;\n}\n" + : "=r"(ok) + : "r"(smem_u32(bar)), "r"(phase) + : "memory"); + return ok != 0; +} +__device__ __forceinline__ void mbar_wait(uint64_t* bar, uint32_t phase) { + while (!mbar_try_wait(bar, phase)) { + } +} + +// ---------------------------------------------------------------- TMA / bulk copies +__device__ __forceinline__ void tma_load_2d(void* dst, const void* tmap, uint64_t* bar, int c0, int c1) { + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%3, %4}], [%2];" :: + "r"(smem_u32(dst)), + "l"(tmap), "r"(smem_u32(bar)), "r"(c0), "r"(c1) + : "memory"); +} +__device__ __forceinline__ void bulk_load(void* dst, const void* src, uint32_t bytes, uint64_t* bar) { + asm volatile("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"( + smem_u32(dst)), + "l"(src), "r"(bytes), "r"(smem_u32(bar)) + : "memory"); +} +__device__ __forceinline__ void prefetch_tmap(const void* tmap) { + asm volatile("prefetch.tensormap [%0];" ::"l"(tmap) : "memory"); +} +// generic-proxy smem writes -> visible to the async proxy (UMMA / UTCCP operand reads) +__device__ __forceinline__ void fence_proxy_async_smem() { + asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); +} + +// ---------------------------------------------------------------- tcgen05 +__device__ __forceinline__ void tmem_alloc(uint32_t* dst_smem, uint32_t ncols) { + asm volatile("tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" ::"r"(smem_u32(dst_smem)), + "r"(ncols) + : "memory"); +} +__device__ __forceinline__ void tmem_relinquish() { + asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;" ::: "memory"); +} +__device__ __forceinline__ void tmem_dealloc(uint32_t taddr, uint32_t ncols) { + asm volatile("tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" ::"r"(taddr), "r"(ncols) : "memory"); +} +__device__ __forceinline__ void tc_fence_before() { + asm volatile("tcgen05.fence::before_thread_sync;" ::: "memory"); +} +__device__ __forceinline__ void tc_fence_after() { + asm volatile("tcgen05.fence::after_thread_sync;" ::: "memory"); +} +// arrive on `bar` when all prior tcgen05 async ops issued by this thread complete +__device__ __forceinline__ void tc_commit(uint64_t* bar) { + asm volatile("tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%0];" ::"r"(smem_u32(bar)) + : "memory"); +} +// smem -> TMEM copy of one 512-byte scale-factor chunk (32 rows x 16 B), broadcast to the 4 lane quarters +__device__ __forceinline__ void utccp_32x128b_warpx4(uint32_t taddr, uint64_t sdesc) { + asm volatile("tcgen05.cp.cta_group::1.32x128b.warpx4 [%0], %1;" ::"r"(taddr), "l"(sdesc) : "memory"); +} +// D(tmem) (+)= A(smem) * B(smem)^T with NVFP4 block-16 scales from TMEM +__device__ __forceinline__ void umma_nvf4(uint32_t tmem_d, uint64_t adesc, uint64_t bdesc, uint32_t idesc, + uint32_t tmem_sfa, uint32_t tmem_sfb, uint32_t accumulate) { + asm volatile( + "{\n .reg .pred p;\n setp.ne.b32 p, %4, 0;\n" + " tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16 [%0], %1, %2, %3, [%5], [%6], p;\n}\n" ::"r"( + tmem_d), + "l"(adesc), "l"(bdesc), "r"(idesc), "r"(accumulate), "r"(tmem_sfa), "r"(tmem_sfb) + : "memory"); +} +// each warp reads its 32-lane quarter: 32 lanes x 16 consecutive 32-bit columns +__device__ __forceinline__ void tmem_ld_32x32b_x16(uint32_t taddr, uint32_t (&r)[16]) { + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x16.b32 {%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15}, [%16];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]), "=r"(r[4]), "=r"(r[5]), "=r"(r[6]), "=r"(r[7]), + "=r"(r[8]), "=r"(r[9]), "=r"(r[10]), "=r"(r[11]), "=r"(r[12]), "=r"(r[13]), "=r"(r[14]), "=r"(r[15]) + : "r"(taddr) + : "memory"); +} +// L2 prefetch of a contiguous global range (bytes multiple of 16, 16-byte aligned) +__device__ __forceinline__ void bulk_prefetch_l2(const void* gptr, uint32_t bytes) { + asm volatile("cp.async.bulk.prefetch.L2.global [%0], %1;" ::"l"(gptr), "r"(bytes) : "memory"); +} +__device__ __forceinline__ void tmem_ld_wait() { asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); } +// each warp writes its 32-lane quarter: 32 lanes x 16 consecutive 32-bit columns +__device__ __forceinline__ void tmem_st_32x32b_x16(uint32_t taddr, const uint32_t (&r)[16]) { + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x16.b32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16};" ::"r"( + taddr), + "r"(r[0]), "r"(r[1]), "r"(r[2]), "r"(r[3]), "r"(r[4]), "r"(r[5]), "r"(r[6]), "r"(r[7]), "r"(r[8]), "r"(r[9]), + "r"(r[10]), "r"(r[11]), "r"(r[12]), "r"(r[13]), "r"(r[14]), "r"(r[15]) + : "memory"); +} +__device__ __forceinline__ void tmem_st_wait() { asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); } + +// ---------------------------------------------------------------- descriptors +// SM100 shared-memory matrix descriptor (version 1). layout: 0 none, 2 SW128, 4 SW64, 6 SW32. +__device__ __forceinline__ uint64_t make_sdesc(uint32_t saddr, uint32_t lbo, uint32_t sbo, uint32_t layout) { + uint64_t d = 0; + d |= (uint64_t)((saddr >> 4) & 0x3FFF); + d |= (uint64_t)((lbo >> 4) & 0x3FFF) << 16; + d |= (uint64_t)((sbo >> 4) & 0x3FFF) << 32; + d |= (uint64_t)1 << 46; // version + d |= (uint64_t)(layout & 0x7) << 61; + return d; +} +// Block-scaled instruction descriptor: E2M1 x E2M1, UE4M3 scales, both K-major, F32 accumulate. +__host__ __device__ constexpr uint32_t make_idesc_nvf4(uint32_t M, uint32_t N) { + return (1u << 7) // a_format E2M1 + | (1u << 10) // b_format E2M1 + | ((N >> 3) << 17) // n_dim + | (0u << 23) // scale format UE4M3 + | ((M >> 4) << 24); // m_dim +} + +} // namespace qwen35 + +namespace sglang { + +namespace qwen_mkm_moe { + +using namespace qwen35; + + +constexpr int kH = 4096, kE = 512, kTopK = 10, kMaxTok = 7; +constexpr int kN1 = 256; // W13 rows per expert (gate + up, TP8) +constexpr int kStageBytes = 36864; // 32 KB weights + 4 KB scale factors +constexpr int kStages = 5; +constexpr int kN1c = 128; // W13 rows per CTA (row split) +constexpr int kW13BoxRows = kN1c; // rows of one W13 TMA box; the host tensor map must use the same +constexpr int kFc1Stages = 8; // 8 k-blocks each (full K = 4096): 2 TMA boxes of 128 rows x 128 B +constexpr int kFc2Stages = 4; // 2 tiles each +constexpr int kTotStages = kFc1Stages + kFc2Stages; +constexpr int kThreads = 256; +// TMEM columns. FC1: accumulator [0,128), x SFA [128,384) (64 k-blocks x 4), SFB ring [384,512) (32 slots x 4). +// FC2: accumulator [0,256), h SFA [256,320) (16 atoms x 4), SFB ring [384,512) (16 slots x 8). +constexpr uint32_t kColSfaX = 128, kColSfa = 256, kColSfb = 384; +constexpr int kSfbSlots = 16, kSfbSlots1 = 32; +constexpr int kARow = 132; // FC1 accumulator row stride in floats: 16 B aligned, rows on distinct banks +#ifndef SF_UTCCP_FC1 +#define SF_UTCCP_FC1 0 +#endif +#ifndef DIAG_NO_SFWAIT // timing diagnostic only: the MMA issuer does not wait for staged scale factors +#define DIAG_NO_SFWAIT 0 +#endif +#ifndef L2_EVICT_FIRST // stream expert weights with an L2 evict-first policy (they are read once per layer) +#define L2_EVICT_FIRST 1 +#endif +#ifndef TOPK_RANK // 1: all-pairs rank top-10; 0: chains of warp max reductions (faster: 13.14 vs 13.32 us) +#define TOPK_RANK 0 +#endif +#ifndef PDL_TRIGGER // 1: let dependent grids launch once the FC2 MMAs are queued (only safe if they all wait) +#define PDL_TRIGGER 0 +#endif +#ifndef HOIST_SCALES // 1: per-expert dequant scales loaded right after routing; 0: at first use +#define HOIST_SCALES 1 +#endif +#ifndef FC2SF_PRE // 1: stage FC2 scale factors of stages 8, 9 during the partial exchange; 0: after h +#define FC2SF_PRE 1 +#endif +#ifndef DIAG_NO_MMA // timing diagnostic only: no MMAs, stages are released as soon as they land +#define DIAG_NO_MMA 0 +#endif // partial row stride in floats: 16 B aligned, rows land on distinct banks + +struct Params { + const uint16_t* logits; // [M, 512] bf16 router logits (router_gemm_quant output) + uint16_t* expert_weights; // [M, 10] bf16 out: DynBlock softmax-over-top-10 weights (CTA 0 writes) + int32_t* topk_ids; // [M, 10] out (CTA 0 writes) + const uint8_t* x_sf; // [M, 256] linear e4m3 + const uint8_t* w13_sf; // [E, 2 row blocks x 64 k blocks x 512 B] (128x4 interleaved) + const uint8_t* w2_sf; // [E, 32 row blocks x 2 k blocks x 512 B] + const float* g1_alpha; // [E] gate dequant + const float* g1_scale_c; // [E] up dequant * FC2 input quant scale + const float* g2_alpha; // [E] + __nv_bfloat16* out; // [M * 10, 4096] + long long* trace; // optional [grid, 16] %globaltimer stamps + int num_tokens; +}; + +struct __align__(1024) Smem { + uint8_t ring[kStages][kStageBytes]; + alignas(1024) uint8_t xa[16 * 1024]; // x operand: 8 rows x full K (2048 B), SW128 (16 atoms of 8 rows x 128 B) + float acc1[8 * kARow]; // FC1 accumulator rows 0..7 (tokens) x 128 accumulator columns + alignas(1024) uint8_t ha[1024]; // h operand: 8 rows x 128 B (64 B used), SW128 (atom must be 1024 B aligned) + uint8_t xsf[8][256]; // x scale bytes, full K (row t = token t), by bulk copy + uint32_t hsf[8][2]; // h scale words: token t, k-block kb (4 e4m3 bytes) + uint64_t full[kStages], empty[kStages], sf_full[kStages]; + uint64_t x_full, sfa_ready, fc1_done, h_peer, h_ready, fc2_done, route_done; + uint32_t tmem; + int expert; + uint32_t tokmask; + int kslot[8]; + alignas(16) uint32_t cand[8][32]; // read as uint4 by the rank-based top-k + int32_t ids[8][kTopK]; +}; + +__device__ __forceinline__ uint64_t gtime() { + uint64_t t; + asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t)::"memory"); + return t; +} +__device__ __forceinline__ uint32_t cluster_rank() { + uint32_t r; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(r)); + return r; +} +__device__ __forceinline__ void cluster_arrive() { + asm volatile("barrier.cluster.arrive.release.aligned;" ::: "memory"); +} +// no release: the phase only orders DSMEM traffic that mbarriers already completed, so pending global stores +// (stamps, routing outputs, output rows) need not drain before the arrive +__device__ __forceinline__ void cluster_arrive_relaxed() { + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); +} +__device__ __forceinline__ void cluster_wait() { + asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); +} +__device__ __forceinline__ uint32_t mapa(uint32_t saddr, uint32_t rank) { + uint32_t r; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(r) : "r"(saddr), "r"(rank)); + return r; +} +// smem (this CTA) -> smem (peer CTA in the cluster), completes tx bytes on the peer's mbarrier +__device__ __forceinline__ void bulk_s2s(uint32_t dst_cluster, uint32_t src, uint32_t bytes, uint32_t bar_cluster) { + asm volatile("cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"( + dst_cluster), + "r"(src), "r"(bytes), "r"(bar_cluster) + : "memory"); +} +// registers -> the peer CTA's shared memory; completes the bytes on the peer's mbarrier +__device__ __forceinline__ void st_async_v4(uint32_t dst, const uint4& v, uint32_t bar) { + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 [%0], {%1, %2, %3, %4}, [%5];" ::"r"(dst), + "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w), "r"(bar) + : "memory"); +} +__device__ __forceinline__ void st_async_b32(uint32_t dst, uint32_t v, uint32_t bar) { + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.b32 [%0], %1, [%2];" ::"r"(dst), "r"(v), "r"(bar) + : "memory"); +} +__device__ __forceinline__ void tmem_st_x4(uint32_t taddr, const uint4& v) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x4.b32 [%0], {%1, %2, %3, %4};" ::"r"(taddr), "r"(v.x), "r"(v.y), + "r"(v.z), "r"(v.w) + : "memory"); +} +__device__ __forceinline__ void named_bar(int id, int n) { + asm volatile("bar.sync %0, %1;" ::"r"(id), "r"(n) : "memory"); +} +__device__ __forceinline__ void tmem_ld_x32(uint32_t taddr, uint32_t (&r)[32]) { + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x32.b32 {%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16,%17,%18,%19," + "%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,%31}, [%32];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]), "=r"(r[4]), "=r"(r[5]), "=r"(r[6]), "=r"(r[7]), "=r"(r[8]), + "=r"(r[9]), "=r"(r[10]), "=r"(r[11]), "=r"(r[12]), "=r"(r[13]), "=r"(r[14]), "=r"(r[15]), "=r"(r[16]), + "=r"(r[17]), "=r"(r[18]), "=r"(r[19]), "=r"(r[20]), "=r"(r[21]), "=r"(r[22]), "=r"(r[23]), "=r"(r[24]), + "=r"(r[25]), "=r"(r[26]), "=r"(r[27]), "=r"(r[28]), "=r"(r[29]), "=r"(r[30]), "=r"(r[31]) + : "r"(taddr) + : "memory"); +} +__device__ __forceinline__ void tmem_ld_x64(uint32_t taddr, uint32_t (&r)[64]) { + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x64.b32 {%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16,%17,%18,%19," + "%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,%31,%32,%33,%34,%35,%36,%37,%38,%39,%40,%41,%42,%43,%44,%45,%46,%47," + "%48,%49,%50,%51,%52,%53,%54,%55,%56,%57,%58,%59,%60,%61,%62,%63}, [%64];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]), "=r"(r[4]), "=r"(r[5]), "=r"(r[6]), "=r"(r[7]), "=r"(r[8]), + "=r"(r[9]), "=r"(r[10]), "=r"(r[11]), "=r"(r[12]), "=r"(r[13]), "=r"(r[14]), "=r"(r[15]), "=r"(r[16]), + "=r"(r[17]), "=r"(r[18]), "=r"(r[19]), "=r"(r[20]), "=r"(r[21]), "=r"(r[22]), "=r"(r[23]), "=r"(r[24]), + "=r"(r[25]), "=r"(r[26]), "=r"(r[27]), "=r"(r[28]), "=r"(r[29]), "=r"(r[30]), "=r"(r[31]), "=r"(r[32]), + "=r"(r[33]), "=r"(r[34]), "=r"(r[35]), "=r"(r[36]), "=r"(r[37]), "=r"(r[38]), "=r"(r[39]), "=r"(r[40]), + "=r"(r[41]), "=r"(r[42]), "=r"(r[43]), "=r"(r[44]), "=r"(r[45]), "=r"(r[46]), "=r"(r[47]), "=r"(r[48]), + "=r"(r[49]), "=r"(r[50]), "=r"(r[51]), "=r"(r[52]), "=r"(r[53]), "=r"(r[54]), "=r"(r[55]), "=r"(r[56]), + "=r"(r[57]), "=r"(r[58]), "=r"(r[59]), "=r"(r[60]), "=r"(r[61]), "=r"(r[62]), "=r"(r[63]) + : "r"(taddr) + : "memory"); +} +__device__ __forceinline__ void tmem_st_x8(uint32_t taddr, const uint32_t (&r)[8]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x8.b32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8};" ::"r"(taddr), "r"(r[0]), + "r"(r[1]), "r"(r[2]), "r"(r[3]), "r"(r[4]), "r"(r[5]), "r"(r[6]), "r"(r[7]) + : "memory"); +} +__device__ __forceinline__ void tmem_st_x32(uint32_t taddr, const uint32_t (&r)[32]) { + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x32.b32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16,%17,%18," + "%19,%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,%31,%32};" ::"r"(taddr), + "r"(r[0]), "r"(r[1]), "r"(r[2]), "r"(r[3]), "r"(r[4]), "r"(r[5]), "r"(r[6]), "r"(r[7]), "r"(r[8]), "r"(r[9]), + "r"(r[10]), "r"(r[11]), "r"(r[12]), "r"(r[13]), "r"(r[14]), "r"(r[15]), "r"(r[16]), "r"(r[17]), "r"(r[18]), + "r"(r[19]), "r"(r[20]), "r"(r[21]), "r"(r[22]), "r"(r[23]), "r"(r[24]), "r"(r[25]), "r"(r[26]), "r"(r[27]), + "r"(r[28]), "r"(r[29]), "r"(r[30]), "r"(r[31]) + : "memory"); +} +__device__ __forceinline__ float ex2_approx(float x) { + float y; + asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} +__device__ __forceinline__ float mul_ftz(float a, float b) { + float y; + asm("mul.ftz.f32 %0, %1, %2;" : "=f"(y) : "f"(a), "f"(b)); + return y; +} +__device__ __forceinline__ float rcp_approx(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} +__device__ __forceinline__ uint32_t cvt_e4m3(float a) { + uint32_t out; + asm("{\n .reg .b16 pair;\n .reg .f32 zero;\n mov.f32 zero, 0f00000000;\n" + " cvt.rn.satfinite.e4m3x2.f32 pair, zero, %1;\n cvt.u32.u16 %0, pair;\n}" + : "=r"(out) + : "f"(a)); + return out & 0xFFu; +} +__device__ __forceinline__ float e4m3_to_f32(uint32_t v) { + float out; + asm("{\n .reg .b16 pair;\n .reg .b32 h2;\n .reg .b16 lo, hi;\n cvt.u16.u32 pair, %1;\n" + " cvt.rn.f16x2.e4m3x2 h2, pair;\n mov.b32 {lo, hi}, h2;\n cvt.f32.f16 %0, lo;\n}" + : "=f"(out) + : "r"(v)); + return out; +} +__device__ __forceinline__ uint32_t cvt_e2m1x8(const float* v) { + uint32_t out; + asm("{\n .reg .b8 b0, b1, b2, b3;\n" + " cvt.rn.satfinite.e2m1x2.f32 b0, %2, %1;\n cvt.rn.satfinite.e2m1x2.f32 b1, %4, %3;\n" + " cvt.rn.satfinite.e2m1x2.f32 b2, %6, %5;\n cvt.rn.satfinite.e2m1x2.f32 b3, %8, %7;\n" + " mov.b32 %0, {b0, b1, b2, b3};\n}" + : "=r"(out) + : "f"(v[0]), "f"(v[1]), "f"(v[2]), "f"(v[3]), "f"(v[4]), "f"(v[5]), "f"(v[6]), "f"(v[7])); + return out; +} +// ---- router top-k, bit-exact with FlashInfer's single-block routing (routingCustom, RenormalizeNaive on bf16): +// key = bf16 order (cub float twiddle) above 65535 - expert, i.e. the higher score first, the smaller expert on ties. +__device__ __forceinline__ uint32_t topk_key(uint32_t bits, uint32_t expert) { + const uint32_t twiddled = bits ^ ((bits & 0x8000u) ? 0xFFFFu : 0x8000u); + return (twiddled << 16) | (65535u - expert); +} +__device__ __forceinline__ void topk_unkey(uint32_t key, uint32_t& bits, uint32_t& expert) { + expert = 65535u - (key & 0xFFFFu); + const uint32_t twiddled = key >> 16; + bits = twiddled ^ ((twiddled & 0x8000u) ? 0x8000u : 0xFFFFu); +} +__device__ __forceinline__ uint32_t max16(const uint32_t (&k)[16]) { + const uint32_t a = max(max(max(k[0], k[1]), max(k[2], k[3])), max(max(k[4], k[5]), max(k[6], k[7]))); + const uint32_t b = max(max(max(k[8], k[9]), max(k[10], k[11])), max(max(k[12], k[13]), max(k[14], k[15]))); + return max(a, b); +} +__device__ __forceinline__ uint32_t warp_max_u32(uint32_t v) { + uint32_t r; + asm volatile("redux.sync.max.u32 %0, %1, 0xffffffff;" : "=r"(r) : "r"(v)); + return r; +} +__device__ __forceinline__ uint32_t warp_sum_u32(uint32_t v) { + uint32_t r; + asm volatile("redux.sync.add.u32 %0, %1, 0xffffffff;" : "=r"(r) : "r"(v)); + return r; +} +__device__ __forceinline__ uint32_t warp_topk_of_lanes(uint32_t v, uint32_t lane) { + uint32_t mine = 0; +#pragma unroll + for (uint32_t k = 0; k < kTopK; ++k) { + const uint32_t top = warp_max_u32(v); + v = v == top ? 0u : v; + mine = lane == k ? top : mine; + } + return mine; +} +__device__ __forceinline__ uint32_t count_greater(const uint32_t* s_cand, uint32_t v) { + uint32_t r = 0; +#pragma unroll + for (int j = 0; j < 32; j += 4) { + const uint4 q = *reinterpret_cast(s_cand + j); + r += (q.x > v) + (q.y > v) + (q.z > v) + (q.w > v); + } + return r; +} +// Top-10 keys of 512 (16 per lane), lane k < 10 holding the k-th largest (keys unique and nonzero). Ranks come from +// all-pairs compares over shared memory rather than chains of warp reductions. +__device__ __forceinline__ uint32_t warp_top10(uint32_t (&keys)[16], uint32_t lane, uint32_t* s_cand) { + // bound = the 10th largest lane maximum; every top-10 key is >= bound + const uint32_t m = max16(keys); + s_cand[lane] = m; + __syncwarp(); + const uint32_t r = count_greater(s_cand, m); + const uint32_t bound = __shfl_sync(0xffffffffu, m, __ffs(__ballot_sync(0xffffffffu, r == kTopK - 1)) - 1); + uint32_t count = 0; +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) count += keys[i] >= bound; + // exclusive prefix of count over the lanes, from the ballots of its bits (count <= 16) + const uint32_t lt = (1u << lane) - 1u; + uint32_t offset = 0, total = 0; +#pragma unroll + for (int b = 0; b < 5; ++b) { + const uint32_t bb = __ballot_sync(0xffffffffu, (count >> b) & 1u); + offset += (uint32_t)__popc(bb & lt) << b; + total += (uint32_t)__popc(bb) << b; + } + if (total <= 32) { + __syncwarp(); // every lane has read the lane maxima +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) { + if (keys[i] >= bound) s_cand[offset++] = keys[i]; + } + if (lane >= total) s_cand[lane] = 0u; + __syncwarp(); + const uint32_t cand = s_cand[lane]; + const uint32_t rk = count_greater(s_cand, cand); + __syncwarp(); + if (lane < total && rk < kTopK) s_cand[rk] = cand; + __syncwarp(); + return lane < kTopK ? s_cand[lane] : 0u; + } + uint32_t mine = 0; +#pragma unroll 1 + for (uint32_t k = 0; k < kTopK; ++k) { + const uint32_t top = warp_max_u32(max16(keys)); +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) keys[i] = keys[i] == top ? 0u : keys[i]; + mine = lane == k ? top : mine; + } + return mine; +} +// Top-10 keys of 512 (16 per lane), lane k holding the k-th largest (keys unique and nonzero). +__device__ __forceinline__ uint32_t warp_top10_redux(uint32_t (&keys)[16], uint32_t lane, uint32_t* s_cand) { + uint32_t v = max16(keys); + uint32_t bound = 0; +#pragma unroll + for (uint32_t k = 0; k < kTopK; ++k) { + bound = warp_max_u32(v); + v = v == bound ? 0u : v; + } + uint32_t count = 0; +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) count += keys[i] >= bound; + const uint32_t total = warp_sum_u32(count); + if (total <= 32) { + uint32_t offset = count; +#pragma unroll + for (uint32_t d = 1; d < 32; d <<= 1) { + const uint32_t up = __shfl_up_sync(0xffffffffu, offset, d); + offset += lane >= d ? up : 0u; + } + offset -= count; +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) { + if (keys[i] >= bound) s_cand[offset++] = keys[i]; + } + __syncwarp(); + const uint32_t cand = lane < total ? s_cand[lane] : 0u; + __syncwarp(); + return warp_topk_of_lanes(cand, lane); + } + uint32_t mine = 0; +#pragma unroll 1 + for (uint32_t k = 0; k < kTopK; ++k) { + const uint32_t top = warp_max_u32(max16(keys)); +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) keys[i] = keys[i] == top ? 0u : keys[i]; + mine = lane == k ? top : mine; + } + return mine; +} +// FlashInfer calcSoftmax (bf16 scores) as built with -use_fast_math: expf -> ex2.approx.ftz of x * log2(e), +// division -> div.full.ftz; reductions are cooperative_groups::reduce like the reference. +__device__ __forceinline__ float fast_expf(float x) { + float y; + asm("{\n .reg .f32 t;\n mul.ftz.f32 t, %1, 0f3FB8AA3B;\n ex2.approx.ftz.f32 %0, t;\n}" : "=f"(y) : "f"(x)); + return y; +} +__device__ __forceinline__ float fast_div(float a, float b) { + float y; + asm("div.full.ftz.f32 %0, %1, %2;" : "=f"(y) : "f"(a), "f"(b)); + return y; +} +__device__ __forceinline__ uint16_t softmax_topk(uint16_t score_bits, uint32_t lane) { + namespace cg = cooperative_groups; + const auto warp = cg::tiled_partition<32>(cg::this_thread_block()); + const float si = __uint_as_float((uint32_t)score_bits << 16); + float max_score = -INFINITY; + if (lane < kTopK) max_score = si >= max_score ? si : max_score; + max_score = cg::reduce(warp, max_score, cg::greater()); + float sum_score = 0.f, new_score = 0.f; + if (lane < kTopK) { + new_score = fast_expf(__fsub_rn(si, max_score)); + sum_score = __fadd_rn(sum_score, new_score); + } + sum_score = cg::reduce(warp, sum_score, cg::plus()); + uint16_t out = score_bits; + if (lane < kTopK) { + const __nv_bfloat16 b = __float2bfloat16_rn(fast_div(new_score, sum_score)); + out = *reinterpret_cast(&b); + } + return out; +} + +__device__ __forceinline__ uint64_t policy_evict_first() { + uint64_t pol; + asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(pol)); + return pol; +} +__device__ __forceinline__ void tma_load_2d_w(void* dst, const void* tmap, uint64_t* bar, int c0, int c1, uint64_t pol) { + if (L2_EVICT_FIRST) + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint" + " [%0], [%1, {%3, %4}], [%2], %5;" ::"r"(smem_u32(dst)), + "l"(tmap), "r"(smem_u32(bar)), "r"(c0), "r"(c1), "l"(pol) + : "memory"); + else + tma_load_2d(dst, tmap, bar, c0, c1); +} +__device__ __forceinline__ void bulk_load_w(void* dst, const void* src, uint32_t bytes, uint64_t* bar, uint64_t pol) { + if (L2_EVICT_FIRST) + asm volatile("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [%0], [%1], %2, [%3], %4;" ::"r"( + smem_u32(dst)), + "l"(src), "r"(bytes), "r"(smem_u32(bar)), "l"(pol) + : "memory"); + else + bulk_load(dst, src, bytes, bar); +} + +__device__ __forceinline__ void stamp(const Params& p, int slot) { + if (p.trace) p.trace[blockIdx.x * 16 + slot] = (long long)gtime(); +} +// stamp after `dep` is available (the branch needs its value; keys are never 0) +__device__ __forceinline__ void stamp_dep(const Params& p, int slot, uint32_t dep) { + if (dep == 0u) __trap(); + stamp(p, slot); +} +// Physical W13 column (accumulator column) holding reordered row q: block-32 shuffle of FlashInfer's +// get_shuffle_matrix_a_row_indices (epilogue_tile_m = 128); reordered row 2j is up_j, 2j + 1 is gate_j. +__device__ __forceinline__ int w13_col(int q) { return (q & ~31) + ((q & 31) % 4) * 8 + (q & 31) / 4; } +__host__ __device__ constexpr int w13_logical(int p) { + return ((p & ~31) + (p % 8) * 4 + (p & 31) / 8) % 2 ? ((p & ~31) + (p % 8) * 4 + (p & 31) / 8) / 2 + : 128 + ((p & ~31) + (p % 8) * 4 + (p & 31) / 8) / 2; +} + +// FC2 drain of TMEM lane quarter q (tiles 4 q .. 4 q + 3), column half `half` (128 accumulator columns) +__device__ __forceinline__ void fc2_drain(const Params& p, Smem& s, uint32_t tm, int q, int lane, int half, float a2, + uint32_t rank, int e) { + mbar_wait(&s.fc2_done, 0); + if (!HOIST_SCALES) a2 = p.g2_alpha[e]; + tc_fence_after(); + if (q == 0 && half == 0 && lane == 0) stamp(p, 7); + const int row = q * 32 + lane, tile = row / 8, t = row % 8; + const bool valid = t < p.num_tokens && ((s.tokmask >> t) & 1u); + __nv_bfloat16* orow = p.out + (size_t)(t * kTopK + s.kslot[t]) * kH + rank * 2048 + tile * 256 + half * 128; +#pragma unroll 1 + for (int c = 0; c < 2; ++c) { + uint32_t r[64]; + tmem_ld_x64(tm + ((uint32_t)(q * 32) << 16) + (uint32_t)(half * 128 + c * 64), r); + tmem_ld_wait(); + if (valid) { +#pragma unroll + for (int b32 = 0; b32 < 2; ++b32) { + uint32_t pk[16]; +#pragma unroll + for (int l = 0; l < 32; l += 2) { + const float v0 = __uint_as_float(r[b32 * 32 + (l % 4) * 8 + l / 4]) * a2; + const float v1 = __uint_as_float(r[b32 * 32 + ((l + 1) % 4) * 8 + (l + 1) / 4]) * a2; + const __nv_bfloat162 b2 = __floats2bfloat162_rn(v0, v1); + pk[l / 2] = *reinterpret_cast(&b2); + } +#pragma unroll + for (int i = 0; i < 4; ++i) + *reinterpret_cast(orow + c * 64 + b32 * 32 + i * 8) = + make_uint4(pk[4 * i], pk[4 * i + 1], pk[4 * i + 2], pk[4 * i + 3]); + } + } + } +} + +extern "C" __global__ void __cluster_dims__(2, 1, 1) __launch_bounds__(kThreads, 1) +q35_mkm_moe(const __grid_constant__ CUtensorMap tm13, const __grid_constant__ CUtensorMap tm2, + const __grid_constant__ CUtensorMap tmx, const Params p) { + extern __shared__ __align__(1024) uint8_t raw[]; + Smem& s = *reinterpret_cast((reinterpret_cast(raw) + 1023) & ~uintptr_t(1023)); + const int warp = threadIdx.x / 32, lane = threadIdx.x % 32; + const uint32_t rank = cluster_rank(); + const int pair = blockIdx.x / 2; + if (threadIdx.x == 0) { + stamp(p, 0); + for (int i = 0; i < kStages; ++i) { + mbar_init(&s.full[i], 1); + mbar_init(&s.empty[i], 1); + mbar_init(&s.sf_full[i], 4); + } + mbar_init(&s.x_full, 1); + mbar_init(&s.sfa_ready, 4); + mbar_init(&s.fc1_done, 1); + mbar_init(&s.h_peer, 1); + mbar_init(&s.h_ready, 4); + mbar_init(&s.fc2_done, 1); + mbar_init(&s.route_done, 1); + mbar_expect_tx(&s.h_peer, 16 * 16 + 8 * 4); // the peer's half of h (8 rows x 32 B) and its 8 scale words + fence_mbar_init(); + } + if (warp == 2) { tmem_alloc(&s.tmem, 512); tmem_relinquish(); tc_fence_before(); } + if (warp == 1 && lane == 0) { prefetch_tmap(&tm13); prefetch_tmap(&tm2); prefetch_tmap(&tmx); } + __syncthreads(); + cluster_arrive(); // waited on before the first DSMEM transfer and at exit + tc_fence_after(); + const uint32_t tm = s.tmem; + // cluster `pair` serves (token my_t, top-k slot my_k) unless an earlier pair (in t * 10 + k order) has its expert + const int my_t = pair / kTopK, my_k = pair % kTopK; + if (my_t >= p.num_tokens) { // both CTAs of the pair agree; nothing was issued + cluster_wait(); + if (warp == 2) tmem_dealloc(tm, 512); + return; + } + auto issue_stage = [&](int st, int e, uint64_t pol) { + const int b = st % kStages; + mbar_expect_tx(&s.full[b], kStageBytes); + uint8_t* dst = s.ring[b]; + if (st < kFc1Stages) { + tma_load_2d_w(dst, &tm13, &s.full[b], st * 256, e * kN1 + (int)rank * kN1c, pol); + tma_load_2d_w(dst + 16384, &tm13, &s.full[b], st * 256 + 128, e * kN1 + (int)rank * kN1c, pol); + const uint8_t* sf = p.w13_sf + (size_t)e * (kN1 * kH / 16); + bulk_load_w(dst + 32768, sf + (size_t)((int)rank * 64 + st * 8) * 512, 4096, &s.full[b], pol); + } else { + const int f = st - kFc1Stages; +#pragma unroll + for (int tl = 0; tl < 2; ++tl) { + const int j = f * 2 + tl; + const int row0 = (int)rank * 2048 + j * 256; + tma_load_2d_w(dst + tl * 16384, &tm2, &s.full[b], 0, e * kH + row0, pol); + bulk_load_w(dst + 32768 + tl * 2048, p.w2_sf + (size_t)e * (kH * 8) + (size_t)(row0 / 128) * 2 * 512, 2048, + &s.full[b], pol); + } + } + }; + // routing: warps 0, 2..7 take tokens 0..6 (DynBlock-exact top-10 + softmax); warp 1 recomputes token my_t's top-10 + // and starts this pair's weight stream at once (a duplicate pair only wastes its first ring fill, mostly L2 hits) + const int rtok = warp == 1 ? my_t : (warp == 0 ? 0 : warp - 1); + uint32_t key = 0; + asm volatile("griddepcontrol.wait;" ::: "memory"); + if (threadIdx.x == 32) stamp(p, 10); + if (warp == 3 && lane == 0) { + // x (8 atoms of this CTA's K half, rows >= M zero-filled) and its scale bytes; independent of the routing + mbar_expect_tx(&s.x_full, 16 * 1024 + p.num_tokens * 256); +#pragma unroll + for (int a = 0; a < 16; ++a) tma_load_2d(s.xa + a * 1024, &tmx, &s.x_full, a * 128, 0); + for (int t = 0; t < p.num_tokens; ++t) bulk_load(s.xsf[t], p.x_sf + (size_t)t * (kH / 16), 256, &s.x_full); + } + if (rtok < p.num_tokens) { + const uint4* src = reinterpret_cast(p.logits + (size_t)rtok * kE + lane * 16); + const uint4 v0 = src[0], v1 = src[1]; + uint32_t keys[16]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + const uint32_t w = i < 4 ? (&v0.x)[i] : (&v1.x)[i - 4]; + keys[2 * i] = topk_key(w & 0xFFFFu, (uint32_t)(lane * 16 + 2 * i)); + keys[2 * i + 1] = topk_key(w >> 16, (uint32_t)(lane * 16 + 2 * i + 1)); + } + if (threadIdx.x == 32) stamp_dep(p, 11, keys[0]); + key = TOPK_RANK ? warp_top10(keys, (uint32_t)lane, s.cand[warp]) : warp_top10_redux(keys, (uint32_t)lane, s.cand[warp]); + } + int e; + if (warp == 1) { + uint32_t bits, espec; + topk_unkey(__shfl_sync(0xffffffffu, key, my_k), bits, espec); + if (lane == 0) { + stamp(p, 9); + const uint64_t pol = policy_evict_first(); + for (int st = 0; st < kStages; ++st) issue_stage(st, (int)espec, pol); + stamp(p, 2); + } + mbar_wait(&s.route_done, 0); + e = s.expert; + } else { + if (rtok < p.num_tokens) { + uint32_t bits, expert; + topk_unkey(key, bits, expert); + const uint16_t wbits = softmax_topk((uint16_t)bits, (uint32_t)lane); + if (lane < kTopK) { + s.ids[rtok][lane] = (int32_t)expert; + if (blockIdx.x == 0) { + p.expert_weights[rtok * kTopK + lane] = wbits; + p.topk_ids[rtok * kTopK + lane] = (int32_t)expert; + } + } + } + named_bar(4, 224); + if (warp == 0) { + // this pair's expert, whether an earlier pair picked it, and which tokens use it (top-k slot per token) + const int ee = s.ids[my_t][my_k]; + bool dup = false; + for (int q = lane; q < pair; q += 32) dup |= s.ids[q / kTopK][q % kTopK] == ee; + dup = __any_sync(0xffffffffu, dup); + uint32_t mbit = 0; + int ks = 0; + if (lane < p.num_tokens) { +#pragma unroll + for (int k = 0; k < kTopK; ++k) + if (s.ids[lane][k] == ee) { mbit = 1u << lane; ks = k; } + } + const uint32_t mask = __reduce_or_sync(0xffffffffu, mbit); + if (lane < 8) s.kslot[lane] = ks; + if (lane == 0) { + s.expert = dup ? -1 : ee; + s.tokmask = mask; + stamp(p, 1); + mbar_arrive(&s.route_done); + } + } + named_bar(4, 224); + e = s.expert; + } + if (e < 0) { // duplicate pair (both CTAs agree): let the speculative loads land; nothing was sent over DSMEM + if (warp == 1) { + if (lane == 0) + for (int b = 0; b < kStages; ++b) mbar_wait(&s.full[b], 0); + __syncwarp(); + } + mbar_wait(&s.x_full, 0); + cluster_wait(); + if (warp == 2) tmem_dealloc(tm, 512); + return; + } + tc_fence_after(); + // per-expert dequant scales, loaded now so that no global read sits behind the FC1 -> h -> FC2 chain + float ag = 0.f, sc = 0.f, a2 = 0.f; + if (HOIST_SCALES) { ag = p.g1_alpha[e]; sc = p.g1_scale_c[e]; a2 = p.g2_alpha[e]; } + + if (warp == 1) { + if (lane == 0) { + // ---------------- TMA producer: the rest of the weight stream (stages 0 .. kStages - 1 went out above) + const uint64_t pol = policy_evict_first(); + for (int st = kStages; st < kTotStages; ++st) { + mbar_wait(&s.empty[st % kStages], ((st / kStages) - 1) & 1); + issue_stage(st, e, pol); + } + } + __syncwarp(); + } else if (warp == 2) { + if (lane == 0) { + // ---------------- MMA issuer (tensor pipe: MMAs only) + const uint32_t idesc = make_idesc_nvf4(128u, 256u), idesc1 = make_idesc_nvf4(128u, 128u); + mbar_wait(&s.x_full, 0); + mbar_wait(&s.sfa_ready, 0); + tc_fence_after(); + int n = 0; + for (int st = 0; st < kFc1Stages; ++st) { + const int b = st % kStages; + mbar_wait(&s.full[b], (st / kStages) & 1); + if (!SF_UTCCP_FC1 && !DIAG_NO_SFWAIT) mbar_wait(&s.sf_full[b], (st / kStages) & 1); + tc_fence_after(); + if (st == 0) stamp(p, 3); + if (DIAG_NO_MMA) { n += 4; tc_commit(&s.empty[b]); continue; } +#pragma unroll + for (int q = 0; q < 8; ++q, ++n) { + const int kb = st * 8 + q; // k-blocks in K order: one accumulator, the stock summation order + umma_nvf4(tm, make_sdesc(smem_u32(s.xa + (kb / 4) * 1024) + (kb % 4) * 32, 0, 0, 2), + make_sdesc(smem_u32(s.ring[b] + (q / 4) * 16384) + (q % 4) * 32, 0, 1024, 2), idesc1, + tm + kColSfaX + kb * 4, tm + kColSfb + (uint32_t)(n % kSfbSlots1) * 4, kb > 0); + } + tc_commit(&s.empty[b]); + } + tc_commit(&s.fc1_done); + mbar_wait(&s.h_ready, 0); + tc_fence_after(); + stamp(p, 6); + for (int st = kFc1Stages; st < kTotStages; ++st) { + const int b = st % kStages; + mbar_wait(&s.full[b], (st / kStages) & 1); + if (!DIAG_NO_SFWAIT) mbar_wait(&s.sf_full[b], (st / kStages) & 1); + tc_fence_after(); + if (DIAG_NO_MMA) { tc_commit(&s.empty[b]); continue; } +#pragma unroll + for (int tl = 0; tl < 2; ++tl) { + const int j = (st - kFc1Stages) * 2 + tl; +#pragma unroll + for (int kb = 0; kb < 2; ++kb) { + const int n2 = 32 + j * 2 + kb; // FC2 scale-factor slot, numbered as the helper stages them + umma_nvf4(tm, make_sdesc(smem_u32(s.ha) + kb * 32, 0, 0, 2), + make_sdesc(smem_u32(s.ring[b] + tl * 16384) + kb * 32, 0, 512, 4), idesc, + tm + kColSfa + (j * 2 + kb) * 4, tm + kColSfb + (uint32_t)(n2 % kSfbSlots) * 8, (j | kb) != 0); + } + } + tc_commit(&s.empty[b]); + } + tc_commit(&s.fc2_done); + if (PDL_TRIGGER) asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); + } + __syncwarp(); + } else if (warp >= 4) { + // ---------------- TMEM-side warpgroup: q = lane quarter (TMEM lanes 32 q .. 32 q + 31) + const int q = warp - 4; + const uint32_t lane_addr = (uint32_t)(q * 32) << 16; + // x scale factors: row m -> (lane m % 32, column m / 32) of each 4-column atom, every quarter holds a copy + { + mbar_wait(&s.x_full, 0); + uint32_t r[32]; +#pragma unroll 1 + for (int c4 = 0; c4 < 8; ++c4) { +#pragma unroll + for (int i = 0; i < 32; ++i) { + const int kb = c4 * 8 + i / 4, col = i % 4; + uint32_t w = 0; + if (col == 0 && lane < p.num_tokens) w = *reinterpret_cast(&s.xsf[lane][kb * 4]); + r[i] = w; + } + tmem_st_x32(tm + lane_addr + kColSfaX + c4 * 32, r); + } + tmem_st_wait(); + tc_fence_before(); + __syncwarp(); + if (lane == 0) mbar_arrive(&s.sfa_ready); + } + // weight scale factors of every stage: k-block slot (8 columns) = [atom rows 0..127 | atom rows 128..255] + auto stage_sf = [&](int st, int n0, int nkb, int atom_stride, int half_off) { + const int b = st % kStages; + mbar_wait(&s.full[b], (st / kStages) & 1); + // the slots about to be overwritten were last read by the MMAs of stage st - 4 + if (st >= 4) mbar_wait(&s.empty[(st - 4) % kStages], ((st - 4) / kStages) & 1); + const uint8_t* sf = s.ring[b] + 32768; + for (int i = 0; i < nkb; ++i) { + const int tl = i / 2, kb = i % 2; + const uint8_t* a0 = atom_stride ? sf + tl * 2048 + kb * 512 : sf + i * 512; + const uint8_t* a1 = a0 + half_off; + const uint4 v0 = *reinterpret_cast(a0 + lane * 16); + const uint4 v1 = *reinterpret_cast(a1 + lane * 16); + const uint32_t r[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w}; + tmem_st_x8(tm + lane_addr + kColSfb + (uint32_t)((n0 + i) % kSfbSlots) * 8, r); + } + tmem_st_wait(); + tc_fence_before(); + __syncwarp(); + if (lane == 0) mbar_arrive(&s.sf_full[b]); + }; + for (int st = 0; st < kFc1Stages; ++st) { + const int b = st % kStages; + mbar_wait(&s.full[b], (st / kStages) & 1); + // the slots about to be overwritten were last read by the MMAs of stage st - 4 + if (st >= 4) mbar_wait(&s.empty[(st - 4) % kStages], ((st - 4) / kStages) & 1); + const uint8_t* sf = s.ring[b] + 32768; +#pragma unroll + for (int i = 0; i < 8; ++i) + tmem_st_x4(tm + lane_addr + kColSfb + (uint32_t)((st * 8 + i) % kSfbSlots1) * 4, + *reinterpret_cast(sf + i * 512 + lane * 16)); + tmem_st_wait(); + tc_fence_before(); + __syncwarp(); + if (lane == 0) mbar_arrive(&s.sf_full[b]); + } + + // FC1 accumulator: TMEM lanes 0..7 (quarter 0) hold the token rows; quarter 0 drains all 128 columns + mbar_wait(&s.fc1_done, 0); + tc_fence_after(); + if (q == 0) { + if (lane == 0) stamp(p, 4); +#pragma unroll 1 + for (int c = 0; c < 2; ++c) { + uint32_t r[64]; + tmem_ld_x64(tm + (uint32_t)(c * 64), r); + tmem_ld_wait(); + if (lane < 8) { +#pragma unroll + for (int i = 0; i < 16; ++i) + *reinterpret_cast(s.acc1 + lane * kARow + c * 64 + 4 * i) = + make_uint4(r[4 * i], r[4 * i + 1], r[4 * i + 2], r[4 * i + 3]); + } + } + } + // FC2 scale factors of stages 8 and 9 while the partials travel (all FC1 MMAs are done, so every slot is free) + if (FC2SF_PRE) { + stage_sf(kFc1Stages, 32, 4, 1, 1024); + stage_sf(kFc1Stages + 1, 36, 4, 1, 1024); + } + if (threadIdx.x == 128) stamp(p, 15); + tc_fence_before(); + named_bar(2, 128); // quarter 0's accumulator rows are visible to the whole warpgroup + + if (threadIdx.x == 128) stamp(p, 5); + if (!HOIST_SCALES) { ag = p.g1_alpha[e]; sc = p.g1_scale_c[e]; } + if (q < 2) { + // this CTA's 64 intermediates: (token row, 16-wide block, half). Accumulator columns of each 32-block: + // [0, 8) up 2m, [8, 16) gate 2m, [16, 24) up 2m + 1, [24, 32) gate 2m + 1 (m = 0..7 within the 16 indices) + const int it = threadIdx.x - 128; // 0..63 + const int row = it / 8, blk = (it / 2) % 4, hf = it % 2; + const float4* ar = reinterpret_cast(s.acc1 + row * kARow + 32 * blk + 4 * hf); + const float4 u_even = ar[0], g_even = ar[2], u_odd = ar[4], g_odd = ar[6]; + const float ga[8] = {g_even.x, g_odd.x, g_even.y, g_odd.y, g_even.z, g_odd.z, g_even.w, g_odd.w}; + const float ua[8] = {u_even.x, u_odd.x, u_even.y, u_odd.y, u_even.z, u_odd.z, u_even.w, u_odd.w}; + // trtllm-gen's SwiGLU + NVFP4 epilogue op for op (bit for bit, test_qwen3_5_moe.py); a = FC1 dequant, + // c = a x FC2 input scale: + // h' = (u a)(g sigmoid(g a)), sigmoid = rcp(ex2(-(g (a log2e))) + 1), block scale e4m3(max|h'| c / 6), FP4 (h' / scale) c + const float a_l2e = mul_ftz(ag, 1.4426950216293334961f); + float h[8]; + float amax = 0.f; +#pragma unroll + for (int i = 0; i < 8; ++i) { + const float e = ex2_approx(-(ga[i] * a_l2e)); + const float sg = rcp_approx(e + 1.0f); + const float v = fmaf(ua[i], ag, 0.0f) * (ga[i] * sg); + h[i] = v; + amax = fmaxf(amax, fabsf(v)); + } + amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, 1)); + const uint32_t sf = cvt_e4m3(fmaxf(0.f, fabsf(amax * sc)) * 0.16666667163372039795f); + const float sff = e4m3_to_f32(sf); + const float osc = sff == 0.f ? 0.f : rcp_approx(sff); + float qv[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) qv[i] = mul_ftz(h[i], osc) * sc; + const uint32_t packed = cvt_e2m1x8(qv); + const int gblk = 4 * (int)rank + blk; // 16-wide block over the 128 intermediates; k-block = rank + const int chunk = gblk / 2; + *reinterpret_cast(s.ha + row * 128 + ((chunk ^ row) * 16) + (gblk % 2) * 8 + hf * 4) = packed; + if (hf == 0) reinterpret_cast(&s.hsf[row][gblk / 4])[gblk % 4] = (uint8_t)sf; + } + named_bar(1, 128); + if (q == 0) { + // send this CTA's k-block of h (8 rows x two 16-B chunks) and its scale words into the peer's buffers + cluster_wait(); // phase 1: the peer's barriers are initialized + const uint32_t pbar = mapa(smem_u32(&s.h_peer), rank ^ 1u); + if (lane < 16) { + const int row = lane / 2, chunk = 2 * (int)rank + lane % 2; + const uint32_t off = (uint32_t)(row * 128 + ((chunk ^ row) * 16)); + st_async_v4(mapa(smem_u32(s.ha + off), rank ^ 1u), *reinterpret_cast(s.ha + off), pbar); + } else if (lane < 24) { + const int row = lane - 16; + st_async_b32(mapa(smem_u32(&s.hsf[row][rank]), rank ^ 1u), s.hsf[row][rank], pbar); + } + __syncwarp(); + } + mbar_wait(&s.h_peer, 0); // the peer's half of h and its scale words + if (threadIdx.x == 128) stamp(p, 5); + // h scale factors: 16 atoms (tile jt, k-block kb) of 4 columns; row m = 32 c + lane is nonzero only in its tile + { + uint32_t r[32]; +#pragma unroll 1 + for (int half = 0; half < 2; ++half) { +#pragma unroll + for (int i = 0; i < 32; ++i) { + const int a = half * 8 + i / 4, col = i % 4; // atom a = 2 jt + kb + const int jt = a / 2, kb = a % 2; + const int row = 32 * col + lane; + r[i] = (row / 8 == jt) ? s.hsf[row % 8][kb] : 0u; + } + tmem_st_x32(tm + lane_addr + kColSfa + half * 32, r); + } + tmem_st_wait(); + } + fence_proxy_async_smem(); + tc_fence_before(); + __syncwarp(); + if (lane == 0) mbar_arrive(&s.h_ready); + // FC2 weight scale factors (the MMA issuer waits for each stage's) + for (int st = kFc1Stages + (FC2SF_PRE ? 2 : 0); st < kTotStages; ++st) + stage_sf(st, 32 + (st - kFc1Stages) * 4, 4, 1, 1024); + + if (q < 2) fc2_drain(p, s, tm, q, lane, 0, a2, rank, e); + } + if (warp < 2) fc2_drain(p, s, tm, warp, lane, 1, a2, rank, e); + tc_fence_before(); + __syncthreads(); + if (threadIdx.x == 0) stamp(p, 13); + // every warp but warp 4 still owes its wait on the first cluster phase; phase 2 then ends the DSMEM exchange + if (warp != 4) cluster_wait(); + cluster_arrive_relaxed(); + cluster_wait(); + if (threadIdx.x == 0) stamp(p, 8); + if (warp == 2) tmem_dealloc(tm, 512); +} + +static_assert(sizeof(Smem) + 1024 == Q35_MKM_SMEM_BYTES, "the launcher's dynamic shared memory size"); + +} // namespace qwen_mkm_moe + +} // namespace sglang diff --git a/sgl_nvidia_mega/qwen3_5/sm103/moe/src/router_front.cu b/sgl_nvidia_mega/qwen3_5/sm103/moe/src/router_front.cu new file mode 100644 index 00000000..24040b14 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/moe/src/router_front.cu @@ -0,0 +1,1043 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// MoE router front end for decode token counts in one launch: router GEMV +// (512 experts x 4096 hidden, bf16 weights, fp32 accumulation, bf16 logits), +// softmax / top-10 / renormalize, NVFP4 quantization of the activations +// (16-element blocks, e4m3 scales, linear layout) and the routing tables the +// cute-dsl grouped GEMMs consume. +// +// Every CTA computes the logits of kExpertsPerCta experts for all tokens on +// tensor cores, bitwise equal to the cuBLAS router GEMM (see gemv_chain); CTA +// b < M also quantizes token row b. The last CTA to arrive (acq_rel counter, no +// spinning) runs top-k and builds the routing tables. +// router_tail_kernel runs the same top-k, quantization and tables behind a +// router GEMM computed elsewhere (bitwise to the stock chain given its logits). +// +// Given the same bf16 logits, the softmax / top-k / renormalize arithmetic +// follows the operation order of the Triton router kernel +// (moe_fused_gate._router_triton_kernel, softmax, renormalize, 512 experts), the +// quantization follows flashinfer's NVFP4 linear-layout quantizer, and the +// routing tables equal flashinfer moe_sort's output on every entry the GEMMs +// read. +// +// Built with NVRTC: the kernel is a device function, and the extern "C" entry at the +// end instantiates only its router GEMM + NVFP4 input quantization (mode kModeQuant). + +#include + +// NVRTC has no +typedef short int16_t; +typedef int int32_t; +typedef unsigned char uint8_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; + +#define SGL_DEVICE __forceinline__ __device__ + +namespace sglang { + +using bf16_t = __nv_bfloat16; + +namespace device { + +template +SGL_DEVICE void PDLWaitPrimary() { + if constexpr (kUsePDL) { + asm volatile("griddepcontrol.wait;" ::: "memory"); + } +} + +template +SGL_DEVICE void PDLTriggerSecondary() { + if constexpr (kUsePDL) { + asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); + } +} + +} // namespace device + +} // namespace sglang + +namespace sglang { + +namespace qwen_router_front { + +inline constexpr uint32_t kHidden = 4096; +inline constexpr uint32_t kExperts = 512; +inline constexpr uint32_t kTopK = 10; +inline constexpr uint32_t kMaxTokens = 16; +inline constexpr uint32_t kThreads = 256; +inline constexpr uint32_t kWarps = kThreads / 32; +// 64 CTAs: leaves SMs free for the shared expert running concurrently on the side stream. +inline constexpr uint32_t kExpertsPerCta = 8; +inline constexpr uint32_t kNumCtas = kExperts / kExpertsPerCta; +inline constexpr uint32_t kKPerWarp = kHidden / kWarps; // 512 +inline constexpr uint32_t kKPerLane = kKPerWarp / 32; // 16: one NVFP4 block +inline constexpr uint32_t kSfVec = 16; +inline constexpr uint32_t kBlocksPerRow = kHidden / kSfVec; // 256 +static_assert(kKPerLane == kSfVec, "each lane must own exactly one quantization block"); +static_assert(kExpertsPerCta <= 8, "a CTA's experts form one n8 MMA tile"); +static_assert(kExperts == 2 * 256, "the top-k lane layout assumes 2 x 256 experts"); + +struct Params { + const bf16_t* x; // [M, kHidden] + const bf16_t* weight; // [kExperts, kHidden] + const float* global_scale; // [1], quantization-domain global scale + bf16_t* logits; // [M, kExperts] + float* topk_weights; // [M, kTopK] + int32_t* topk_ids; // [M, kTopK] + uint8_t* x_fp4; // [M, kHidden / 2] + uint8_t* x_sf; // [M, kBlocksPerRow] + int32_t* tile_idx_to_expert_idx; + int32_t* tile_idx_to_mn_limit; + int32_t* expanded_idx_to_permuted_idx; // [M, kTopK] + int32_t* permuted_idx_to_expanded_idx; + int32_t* total_num_padded_tokens; // [1] + int32_t* num_non_exiting_tiles; // [1] + uint32_t* arrival; // monotonic arrival counter, one per call site + uint64_t* trace; // optional %globaltimer stamps (instrumented build only) + uint32_t num_tokens; + uint32_t tile_log2; + uint32_t mode; // kMode* bits: input quantization, top-k, routing tables + // Shared-expert gate/up projection (static per-tensor FP8), computed alongside the router GEMV. + const uint8_t* sh_weight; // [sh_n, kHidden] e4m3 + const float* sh_in_scale; // [1] activation scale + const float* sh_w_scale; // [1] weight scale + bf16_t* sh_out; // [M, sh_n] + uint32_t sh_n; +}; + +inline constexpr uint32_t kModeQuant = 1u; +inline constexpr uint32_t kModeTopk = 2u; +inline constexpr uint32_t kModeTables = 4u; + +SGL_DEVICE float bf16_lo(uint32_t v) { + return __uint_as_float(v << 16); +} +SGL_DEVICE float bf16_hi(uint32_t v) { + return __uint_as_float(v & 0xFFFF0000u); +} + +struct U32x8 { + uint4 lo, hi; +}; + +SGL_DEVICE U32x8 ld_weight(const bf16_t* p) { + // Static weights, 32 B per lane: read-only path, not kept in L1, evict-first in L2. + U32x8 v; + asm volatile( + "ld.global.nc.L1::no_allocate.L2::evict_first.v8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(v.lo.x), "=r"(v.lo.y), "=r"(v.lo.z), "=r"(v.lo.w), "=r"(v.hi.x), "=r"(v.hi.y), "=r"(v.hi.z), "=r"(v.hi.w) + : "l"(p)); + return v; +} + +SGL_DEVICE U32x8 ld_produced32(const void* p) { + // Data written by an earlier grid: coherent load, issued after the PDL wait. + U32x8 v; + asm volatile("ld.global.v8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(v.lo.x), "=r"(v.lo.y), "=r"(v.lo.z), "=r"(v.lo.w), "=r"(v.hi.x), "=r"(v.hi.y), "=r"(v.hi.z), + "=r"(v.hi.w) + : "l"(p)); + return v; +} + +SGL_DEVICE uint4 ld_produced(const void* p) { + uint4 v; + asm volatile("ld.global.v4.u32 {%0, %1, %2, %3}, [%4];" : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) : "l"(p)); + return v; +} + +SGL_DEVICE uint64_t global_timer() { + uint64_t t; + asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t)); + return t; +} + +SGL_DEVICE float ex2_approx(float x) { + float y; + asm("ex2.approx.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + +SGL_DEVICE float div_full(float a, float b) { + float y; + asm("div.full.f32 %0, %1, %2;" : "=f"(y) : "f"(a), "f"(b)); + return y; +} + +SGL_DEVICE float rcp_approx_ftz(float a) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(a)); + return y; +} + +SGL_DEVICE uint32_t cvt_e4m3(float a) { + uint32_t out; + asm("{\n" + ".reg .b16 pair;\n" + ".reg .f32 zero;\n" + "mov.f32 zero, 0f00000000;\n" + "cvt.rn.satfinite.e4m3x2.f32 pair, zero, %1;\n" + "cvt.u32.u16 %0, pair;\n" + "}" + : "=r"(out) + : "f"(a)); + return out & 0xFFu; +} + +SGL_DEVICE float e4m3_to_f32(uint32_t v) { + float out; + asm("{\n" + ".reg .b16 pair;\n" + ".reg .b32 h2;\n" + ".reg .b16 lo, hi;\n" + "cvt.u16.u32 pair, %1;\n" + "cvt.rn.f16x2.e4m3x2 h2, pair;\n" + "mov.b32 {lo, hi}, h2;\n" + "cvt.f32.f16 %0, lo;\n" + "}" + : "=f"(out) + : "r"(v)); + return out; +} + +SGL_DEVICE uint32_t cvt_e2m1x8(const float (&v)[8]) { + uint32_t out; + asm("{\n" + ".reg .b8 b0, b1, b2, b3;\n" + "cvt.rn.satfinite.e2m1x2.f32 b0, %2, %1;\n" + "cvt.rn.satfinite.e2m1x2.f32 b1, %4, %3;\n" + "cvt.rn.satfinite.e2m1x2.f32 b2, %6, %5;\n" + "cvt.rn.satfinite.e2m1x2.f32 b3, %8, %7;\n" + "mov.b32 %0, {b0, b1, b2, b3};\n" + "}" + : "=r"(out) + : "f"(v[0]), "f"(v[1]), "f"(v[2]), "f"(v[3]), "f"(v[4]), "f"(v[5]), "f"(v[6]), "f"(v[7])); + return out; +} + +SGL_DEVICE uint32_t bf16x2_abs_max(uint32_t a, uint32_t b) { + uint32_t out; + asm("{\n" + ".reg .b32 ta, tb;\n" + "and.b32 ta, %1, 0x7FFF7FFF;\n" + "and.b32 tb, %2, 0x7FFF7FFF;\n" + "max.bf16x2 %0, ta, tb;\n" + "}" + : "=r"(out) + : "r"(a), "r"(b)); + return out; +} + +SGL_DEVICE uint32_t bf16x2_max(uint32_t a, uint32_t b) { + uint32_t out; + asm("max.bf16x2 %0, %1, %2;" : "=r"(out) : "r"(a), "r"(b)); + return out; +} + +// One 16-element NVFP4 block: e4m3 scale byte and 8 packed bytes (element 2i in +// the low nibble of byte i). +SGL_DEVICE void quantize_block(const uint4& a, const uint4& b, float gs, uint32_t& sf, uint2& packed) { + const uint32_t w[8] = {a.x, a.y, a.z, a.w, b.x, b.y, b.z, b.w}; + const uint32_t m01 = bf16x2_abs_max(w[0], w[1]); + const uint32_t m23 = bf16x2_abs_max(w[2], w[3]); + const uint32_t m45 = bf16x2_abs_max(w[4], w[5]); + const uint32_t m67 = bf16x2_abs_max(w[6], w[7]); + const uint32_t m = bf16x2_max(bf16x2_max(m01, m23), bf16x2_max(m45, m67)); + const float amax = fmaxf(bf16_lo(m), bf16_hi(m)); + const float scale = __fmul_rn(gs, __fmul_rn(amax, rcp_approx_ftz(6.0f))); + sf = cvt_e4m3(scale); + const float sf_f = e4m3_to_f32(sf); + const float out_scale = (sf_f == 0.0f) ? 0.0f : rcp_approx_ftz(__fmul_rn(sf_f, rcp_approx_ftz(gs))); + float lo[8], hi[8]; +#pragma unroll + for (int i = 0; i < 4; ++i) { + lo[2 * i] = __fmul_rn(bf16_lo(w[i]), out_scale); + lo[2 * i + 1] = __fmul_rn(bf16_hi(w[i]), out_scale); + hi[2 * i] = __fmul_rn(bf16_lo(w[4 + i]), out_scale); + hi[2 * i + 1] = __fmul_rn(bf16_hi(w[4 + i]), out_scale); + } + packed.x = cvt_e2m1x8(lo); + packed.y = cvt_e2m1x8(hi); +} + +// Monotonic 16-bit key of a bf16 value (the canonical +0 for -0 is applied by +// the caller's +0.0 add). +SGL_DEVICE uint32_t bf16_order_key(uint32_t bits16) { + return (bits16 & 0x8000u) ? (~bits16 & 0xFFFFu) : (bits16 | 0x8000u); +} + +SGL_DEVICE float bf16_from_key(uint32_t key16) { + const uint32_t bits = (key16 & 0x8000u) ? (key16 & 0x7FFFu) : (~key16 & 0xFFFFu); + return __uint_as_float(bits << 16); +} + +template +SGL_DEVICE void trace_stamp(const Params& p, uint32_t slot) { + if constexpr (kTrace) { + if (threadIdx.x == 0) p.trace[slot] = global_timer(); + } +} + +// D += A * B for one m16n8k16 bf16 tile, fp32 accumulators. +SGL_DEVICE void mma_bf16_16816(float (&d)[4], uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, + uint32_t b1) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, " + "{%0, %1, %2, %3};" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)); +} + +SGL_DEVICE uint32_t u32x8_at(const U32x8& v, int i) { + switch (i) { + case 0: return v.lo.x; + case 1: return v.lo.y; + case 2: return v.lo.z; + case 3: return v.lo.w; + case 4: return v.hi.x; + case 5: return v.hi.y; + case 6: return v.hi.z; + default: return v.hi.w; + } +} + +template +SGL_DEVICE void cycle_stamp(const Params& p, uint32_t slot, uint64_t t0) { + if constexpr (kTrace) { + if (threadIdx.x == 0) p.trace[kNumCtas * 8 + slot] = clock64() - t0; + } +} + +// Softmax / top-10 / renormalize for kNT tokens of one warp with a full per-lane +// sort (the exact fallback of topk_tokens_fast). Lane layout of the reference +// kernel: lane holds experts [8l, 8l+8) and [256+8l, 256+8l+8). +template +SGL_DEVICE void topk_tokens(const Params& p, const uint32_t (&m)[kNT], uint32_t* s_mask, int16_t* s_ids) { + const uint32_t lane = threadIdx.x % 32; + constexpr float kLog2e = 1.44269502162933349609375f; + float a[kNT][16]; + uint4 r[kNT][2]; +#pragma unroll + for (int t = 0; t < kNT; ++t) { + const bf16_t* row = p.logits + m[t] * kExperts; + r[t][0] = ld_produced(row + 8 * lane); + r[t][1] = ld_produced(row + 256 + 8 * lane); + } + float row_max[kNT], row_sum[kNT]; +#pragma unroll + for (int t = 0; t < kNT; ++t) { + const uint32_t w[8] = {r[t][0].x, r[t][0].y, r[t][0].z, r[t][0].w, r[t][1].x, r[t][1].y, r[t][1].z, r[t][1].w}; +#pragma unroll + for (int i = 0; i < 8; ++i) { + a[t][2 * i] = __fadd_rn(bf16_lo(w[i]), 0.0f); + a[t][2 * i + 1] = __fadd_rn(bf16_hi(w[i]), 0.0f); + } + float mx = a[t][0]; +#pragma unroll + for (int i = 1; i < 16; ++i) mx = fmaxf(mx, a[t][i]); + row_max[t] = mx; + } +#pragma unroll + for (int off = 16; off > 0; off >>= 1) { +#pragma unroll + for (int t = 0; t < kNT; ++t) row_max[t] = fmaxf(row_max[t], __shfl_xor_sync(0xFFFFFFFFu, row_max[t], off)); + } +#pragma unroll + for (int t = 0; t < kNT; ++t) { + float e[16]; +#pragma unroll + for (int i = 0; i < 16; ++i) e[i] = ex2_approx(__fmul_rn(__fsub_rn(a[t][i], row_max[t]), kLog2e)); + // e[0..7] are experts 8l..8l+7, e[8..15] are 256+8l..: the reference's in-thread tree. + const float lo = __fadd_rn( + __fadd_rn(__fadd_rn(e[0], e[2]), __fadd_rn(e[4], e[6])), + __fadd_rn(__fadd_rn(e[8], e[10]), __fadd_rn(e[12], e[14]))); + const float hi = __fadd_rn( + __fadd_rn(__fadd_rn(e[1], e[3]), __fadd_rn(e[5], e[7])), + __fadd_rn(__fadd_rn(e[9], e[11]), __fadd_rn(e[13], e[15]))); + row_sum[t] = __fadd_rn(lo, hi); + } +#pragma unroll + for (int off = 16; off > 0; off >>= 1) { +#pragma unroll + for (int t = 0; t < kNT; ++t) row_sum[t] = __fadd_rn(row_sum[t], __shfl_xor_sync(0xFFFFFFFFu, row_sum[t], off)); + } + + // Keys: monotonic value in the high half, (0xFFFF - expert) in the low half, so + // the warp max is the largest logit with the lowest expert id on ties. + uint32_t key[kNT][16]; +#pragma unroll + for (int t = 0; t < kNT; ++t) { +#pragma unroll + for (int i = 0; i < 16; ++i) { + const uint32_t idx = (i < 8) ? (8 * lane + i) : (256 + 8 * lane + (i - 8)); + const uint32_t bits = __float_as_uint(a[t][i]) >> 16; + key[t][i] = (a[t][i] == a[t][i]) ? ((bf16_order_key(bits) << 16) | (0xFFFFu - idx)) : (0xFFFFu - idx); + } + // Lane-local descending bitonic sort. +#pragma unroll + for (int k = 2; k <= 16; k <<= 1) { +#pragma unroll + for (int j = k >> 1; j > 0; j >>= 1) { +#pragma unroll + for (int i = 0; i < 16; ++i) { + const int l = i ^ j; + if (l > i) { + const uint32_t x = key[t][i], y = key[t][l]; + if ((i & k) == 0) { + key[t][i] = max(x, y); + key[t][l] = min(x, y); + } else { + key[t][i] = min(x, y); + key[t][l] = max(x, y); + } + } + } + } + } + } + + float sel[kNT][kTopK]; + uint32_t ids[kNT][kTopK]; +#pragma unroll + for (int rr = 0; rr < static_cast(kTopK); ++rr) { +#pragma unroll + for (int t = 0; t < kNT; ++t) { + const uint32_t winner = __reduce_max_sync(0xFFFFFFFFu, key[t][0]); + const bool owner = key[t][0] == winner; +#pragma unroll + for (int i = 0; i < 15; ++i) key[t][i] = owner ? key[t][i + 1] : key[t][i]; + key[t][15] = owner ? 0u : key[t][15]; + ids[t][rr] = 0xFFFFu - (winner & 0xFFFFu); + // The winner's softmax probability, recomputed from its logit with the + // per-expert operations instead of shuffled from its owner lane. + const float v = bf16_from_key(winner >> 16); + sel[t][rr] = div_full(ex2_approx(__fmul_rn(__fsub_rn(v, row_max[t]), kLog2e)), row_sum[t]); + } + } +#pragma unroll + for (int t = 0; t < kNT; ++t) { + // Reference reduction over the 16 top-k slots (10 used): even lanes hold slots 0..7, odd 8..15. + const float even = __fadd_rn( + __fadd_rn(__fadd_rn(sel[t][0], sel[t][2]), __fadd_rn(sel[t][4], sel[t][6])), + __fadd_rn(__fadd_rn(sel[t][1], sel[t][3]), __fadd_rn(sel[t][5], sel[t][7]))); + const float odd = __fadd_rn(sel[t][8], sel[t][9]); + const float routed_sum = __fadd_rn(even, odd); + const float norm = routed_sum > 0.0f ? routed_sum : 1.0f; + if (lane < kTopK) { + float w = 0.0f; + uint32_t id = 0; +#pragma unroll + for (int rr = 0; rr < static_cast(kTopK); ++rr) { + if (lane == static_cast(rr)) { + w = div_full(sel[t][rr], norm); + id = ids[t][rr]; + } + } + p.topk_weights[m[t] * kTopK + lane] = w; + p.topk_ids[m[t] * kTopK + lane] = static_cast(id); + s_ids[m[t] * kTopK + lane] = static_cast(id); + atomicOr(&s_mask[id], 1u << m[t]); + } + } +} + +// Out of line: rarely taken, and the tail CTA runs this code cold. +__device__ __noinline__ void topk_token_full(const Params& p, uint32_t m, uint32_t* s_mask, int16_t* s_ids) { + const uint32_t m1[1] = {m}; + topk_tokens<1, false>(p, m1, s_mask, s_ids); +} + +SGL_DEVICE void cmpx_desc(uint32_t& a, uint32_t& b) { + const uint32_t hi = max(a, b), lo = min(a, b); + a = hi; + b = lo; +} + +// Descending sort of 4 keys (5 comparators). +SGL_DEVICE void sort4_desc(uint32_t& x0, uint32_t& x1, uint32_t& x2, uint32_t& x3) { + cmpx_desc(x0, x1); + cmpx_desc(x2, x3); + cmpx_desc(x0, x2); + cmpx_desc(x1, x3); + cmpx_desc(x1, x2); +} + +// Top 4 of two descending 4-lists, descending (bitonic split + clean). +SGL_DEVICE void merge_top4_desc(uint32_t (&a)[4], const uint32_t (&b)[4]) { +#pragma unroll + for (int i = 0; i < 4; ++i) a[i] = max(a[i], b[3 - i]); + cmpx_desc(a[0], a[2]); + cmpx_desc(a[1], a[3]); + cmpx_desc(a[0], a[1]); + cmpx_desc(a[2], a[3]); +} + +// Key of a canonical (bf16-exact) fp32 logit: monotonic value bits in the high +// half, (0xFFFF - expert) in the low half, so the max key is the largest logit +// with the lowest expert id on ties. NaN ranks below every number. +SGL_DEVICE uint32_t logit_key(float v, uint32_t expert) { + const uint32_t u = __float_as_uint(v); + const uint32_t ord = u ^ (static_cast(static_cast(u) >> 31) | 0x80000000u); + return (v == v) ? ((ord & 0xFFFF0000u) | (0xFFFFu - expert)) : (0xFFFFu - expert); +} + +SGL_DEVICE float key_value(uint32_t key) { + const uint32_t hi = key & 0xFFFF0000u; + return __uint_as_float((hi & 0x80000000u) ? (hi & 0x7FFF0000u) : ~hi & 0xFFFF0000u); +} + +// Softmax / top-10 / renormalize for kNT tokens of one warp. Each lane keeps the +// top 4 of its 16 keys; a token where some lane runs out before the last round +// is recomputed with the full per-lane sort (exact either way). Arithmetic on +// the selected values follows the reference kernel's operations and order. +template +SGL_DEVICE void topk_tokens_fast( + const Params& p, const uint32_t (&m)[kNT], const bool (&valid)[kNT], uint32_t* s_mask, int16_t* s_ids) { + const uint32_t lane = threadIdx.x % 32; + const uint64_t c0 = kTrace ? clock64() : 0; + constexpr float kLog2e = 1.44269502162933349609375f; + float a[kNT][16]; + uint4 r[kNT][2]; +#pragma unroll + for (int t = 0; t < kNT; ++t) { + const bf16_t* row = p.logits + m[t] * kExperts; + r[t][0] = ld_produced(row + 8 * lane); + r[t][1] = ld_produced(row + 256 + 8 * lane); + } + if (r[0][1].w != 0x12345678u) cycle_stamp(p, 4, c0); + uint32_t top[kNT][4]; +#pragma unroll + for (int t = 0; t < kNT; ++t) { + const uint32_t w[8] = {r[t][0].x, r[t][0].y, r[t][0].z, r[t][0].w, r[t][1].x, r[t][1].y, r[t][1].z, r[t][1].w}; + uint32_t k[16]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + a[t][2 * i] = __fadd_rn(bf16_lo(w[i]), 0.0f); + a[t][2 * i + 1] = __fadd_rn(bf16_hi(w[i]), 0.0f); + } +#pragma unroll + for (int i = 0; i < 16; ++i) { + const uint32_t idx = (i < 8) ? (8 * lane + i) : (256 + 8 * lane + (i - 8)); + k[i] = logit_key(a[t][i], idx); + } +#pragma unroll + for (int g = 0; g < 4; ++g) sort4_desc(k[4 * g], k[4 * g + 1], k[4 * g + 2], k[4 * g + 3]); + uint32_t q0[4] = {k[0], k[1], k[2], k[3]}, q1[4] = {k[4], k[5], k[6], k[7]}; + uint32_t q2[4] = {k[8], k[9], k[10], k[11]}, q3[4] = {k[12], k[13], k[14], k[15]}; + merge_top4_desc(q0, q1); + merge_top4_desc(q2, q3); + merge_top4_desc(q0, q2); +#pragma unroll + for (int i = 0; i < 4; ++i) top[t][i] = q0[i]; + } + if (top[0][0] != 0x12345678u) cycle_stamp(p, 5, c0); + + float row_max[kNT], row_sum[kNT]; + uint32_t mine[kNT]; + bool bad[kNT]; + uint32_t taken[kNT]; +#pragma unroll + for (int t = 0; t < kNT; ++t) { + mine[t] = 0; + bad[t] = false; + taken[t] = 0; + } +#pragma unroll + for (int rr = 0; rr < static_cast(kTopK); ++rr) { +#pragma unroll + for (int t = 0; t < kNT; ++t) { + const uint32_t winner = __reduce_max_sync(0xFFFFFFFFu, top[t][0]); + const bool owner = top[t][0] == winner; + top[t][0] = owner ? top[t][1] : top[t][0]; + top[t][1] = owner ? top[t][2] : top[t][1]; + top[t][2] = owner ? top[t][3] : top[t][2]; + top[t][3] = owner ? 0u : top[t][3]; + taken[t] += owner ? 1u : 0u; + bad[t] = bad[t] || (owner && taken[t] == 4u && rr + 1 < static_cast(kTopK)); + mine[t] = (lane == static_cast(rr)) ? winner : mine[t]; + if (rr == 0) { + // The largest key is the row max; the reference's exp/sum tree follows. + row_max[t] = key_value(winner); + float e[16]; +#pragma unroll + for (int i = 0; i < 16; ++i) e[i] = ex2_approx(__fmul_rn(__fsub_rn(a[t][i], row_max[t]), kLog2e)); + const float lo = __fadd_rn( + __fadd_rn(__fadd_rn(e[0], e[2]), __fadd_rn(e[4], e[6])), + __fadd_rn(__fadd_rn(e[8], e[10]), __fadd_rn(e[12], e[14]))); + const float hi = __fadd_rn( + __fadd_rn(__fadd_rn(e[1], e[3]), __fadd_rn(e[5], e[7])), + __fadd_rn(__fadd_rn(e[9], e[11]), __fadd_rn(e[13], e[15]))); + row_sum[t] = __fadd_rn(lo, hi); + } + } + } +#pragma unroll + for (int off = 16; off > 0; off >>= 1) { +#pragma unroll + for (int t = 0; t < kNT; ++t) row_sum[t] = __fadd_rn(row_sum[t], __shfl_xor_sync(0xFFFFFFFFu, row_sum[t], off)); + } + if (row_sum[0] != 12345.0f) cycle_stamp(p, 6, c0); + +#pragma unroll + for (int t = 0; t < kNT; ++t) { + if (!valid[t]) continue; + if (__any_sync(0xFFFFFFFFu, bad[t])) { + topk_token_full(p, m[t], s_mask, s_ids); + continue; + } + // Lane rr < 10 owns slot rr: its probability, then the reference's 16-slot sum tree. + const float s = div_full(ex2_approx(__fmul_rn(__fsub_rn(key_value(mine[t]), row_max[t]), kLog2e)), row_sum[t]); + float w[kTopK]; +#pragma unroll + for (int j = 0; j < static_cast(kTopK); ++j) w[j] = __shfl_sync(0xFFFFFFFFu, s, j); + const float even = __fadd_rn( + __fadd_rn(__fadd_rn(w[0], w[2]), __fadd_rn(w[4], w[6])), __fadd_rn(__fadd_rn(w[1], w[3]), __fadd_rn(w[5], w[7]))); + const float routed_sum = __fadd_rn(even, __fadd_rn(w[8], w[9])); + const float norm = routed_sum > 0.0f ? routed_sum : 1.0f; + if (lane < kTopK) { + const uint32_t id = 0xFFFFu - (mine[t] & 0xFFFFu); + p.topk_weights[m[t] * kTopK + lane] = div_full(s, norm); + p.topk_ids[m[t] * kTopK + lane] = static_cast(id); + s_ids[m[t] * kTopK + lane] = static_cast(id); + atomicOr(&s_mask[id], 1u << m[t]); + } + } + cycle_stamp(p, 7, c0); +} + +template +SGL_DEVICE void topk_and_tables(const Params& p, uint32_t* s_mask, int16_t* s_tile, int16_t* s_ids, uint32_t* s_warp) { + const uint32_t warp = threadIdx.x / 32; + const uint32_t lane = threadIdx.x % 32; + const uint32_t M = p.num_tokens; + + for (uint32_t i = threadIdx.x; i < kExperts; i += kThreads) s_mask[i] = 0; + __syncthreads(); + trace_stamp(p, kNumCtas * 8 + 1); + if (warp < M) { + if constexpr (kMaxM > kWarps) { + // Warp w handles tokens w and w + 8; a missing second token repeats the first unwritten. + const bool second = warp + kWarps < M; + const uint32_t m2[2] = {warp, second ? warp + kWarps : warp}; + const bool v2[2] = {true, second}; + topk_tokens_fast<2, kTrace>(p, m2, v2, s_mask, s_ids); + } else { + const uint32_t m1[1] = {warp}; + const bool v1[1] = {true}; + topk_tokens_fast<1, kTrace>(p, m1, v1, s_mask, s_ids); + } + } + __syncthreads(); + trace_stamp(p, kNumCtas * 8 + 2); + if (!(p.mode & kModeTables)) return; + + // Tiles in ascending expert order; with M <= 16 every active expert owns one tile. + const uint32_t e0 = 2 * threadIdx.x; + const uint32_t mask0 = s_mask[e0], mask1 = s_mask[e0 + 1]; + const bool act0 = mask0 != 0, act1 = mask1 != 0; + const uint32_t bal0 = __ballot_sync(0xFFFFFFFFu, act0); + const uint32_t bal1 = __ballot_sync(0xFFFFFFFFu, act1); + const uint32_t lt = (1u << lane) - 1u; + const uint32_t in_warp = __popc(bal0 & lt) + __popc(bal1 & lt); + if (lane == 0) s_warp[warp] = __popc(bal0) + __popc(bal1); + __syncthreads(); + uint32_t base = 0, total = 0; +#pragma unroll + for (uint32_t w = 0; w < kWarps; ++w) { + const uint32_t c = s_warp[w]; + base += (w < warp) ? c : 0u; + total += c; + } + const uint32_t tile0 = base + in_warp; + const uint32_t tile1 = tile0 + (act0 ? 1u : 0u); + if (act0) { + s_tile[e0] = static_cast(tile0); + p.tile_idx_to_expert_idx[tile0] = static_cast(e0); + p.tile_idx_to_mn_limit[tile0] = static_cast((tile0 << p.tile_log2) + __popc(mask0)); + } + if (act1) { + s_tile[e0 + 1] = static_cast(tile1); + p.tile_idx_to_expert_idx[tile1] = static_cast(e0 + 1); + p.tile_idx_to_mn_limit[tile1] = static_cast((tile1 << p.tile_log2) + __popc(mask1)); + } + if (threadIdx.x == 0) { + p.total_num_padded_tokens[0] = static_cast(total << p.tile_log2); + p.num_non_exiting_tiles[0] = static_cast(total); + } + __syncthreads(); + for (uint32_t i = threadIdx.x; i < M * kTopK; i += kThreads) { + const uint32_t m = i / kTopK; + const uint32_t e = static_cast(s_ids[i]); + const uint32_t rank = __popc(s_mask[e] & ((1u << m) - 1u)); + const uint32_t permuted = (static_cast(s_tile[e]) << p.tile_log2) + rank; + p.expanded_idx_to_permuted_idx[i] = static_cast(permuted); + p.permuted_idx_to_expanded_idx[permuted] = static_cast(i); + } + trace_stamp(p, kNumCtas * 8 + 3); +} + +// Router GEMV in the order of the cuBLAS router GEMM it replaces (bitwise equal +// for this shape): K split into two halves, each half a sequential chain of +// 16-wide MMAs in increasing K, the two fp32 halves added, rounded to bf16. +// Per CTA: kExpertsPerCta experts as the n8 tile, tokens as the m16 tile (rows +// >= M repeat token 0); warp 0 runs the chain of the first half, warp 1 the +// second, operands from shared memory. +inline constexpr uint32_t kSplitK = kHidden / 2; +inline constexpr uint32_t kChainSteps = kSplitK / 16; +// Shared-memory rows are padded by 16 B so the 8 rows of an ldmatrix hit distinct banks. +inline constexpr uint32_t kRowBytes = kHidden * 2 + 16; + +// Shared gate/up stage: FP8 rows padded by 16 B like the bf16 ones; K split in kShSplits +// chains of m16n8k32 MMAs, one per warp 2 .. 2 + kShSplits - 1. +inline constexpr uint32_t kShRowBytes = kHidden + 16; +inline constexpr uint32_t kShSplits = 4; +inline constexpr uint32_t kShSteps = kHidden / kShSplits / 32; + +template +struct GemvSmem { + static constexpr uint32_t kW = 0; + static constexpr uint32_t kX = kW + kExpertsPerCta * kRowBytes; + static constexpr uint32_t kPart = kX + kMaxM * kRowBytes; // fp32 [2][16][kExpertsPerCta] + static constexpr uint32_t kShW = kPart + 2 * 16 * kExpertsPerCta * 4; + static constexpr uint32_t kXq = kShW + (kShared ? 8 * kShRowBytes : 0); + static constexpr uint32_t kShPart = kXq + (kShared ? kMaxM * kShRowBytes : 0); // fp32 [kShSplits][16][8] + static constexpr uint32_t kBarW = kShPart + (kShared ? kShSplits * 16 * 8 * 4 : 0); + static constexpr uint32_t kBarX = kBarW + 8; + static constexpr uint32_t kBarQ = kBarX + 8; // e4m3 token rows written (shared stage only) + static constexpr uint32_t kBytes = kBarQ + (kShared ? 8 : 0); +}; + +SGL_DEVICE uint32_t smem_addr(const void* p) { + return static_cast(__cvta_generic_to_shared(p)); +} + +SGL_DEVICE void ldmatrix_x4(uint32_t (&r)[4], uint32_t addr) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]) + : "r"(addr)); +} + +SGL_DEVICE void ldmatrix_x2(uint32_t (&r)[2], uint32_t addr) { + asm volatile("ldmatrix.sync.aligned.m8n8.x2.shared.b16 {%0, %1}, [%2];" : "=r"(r[0]), "=r"(r[1]) : "r"(addr)); +} + +// One K half of the GEMV for this CTA's experts; writes the fp32 partials to s_part. +template +SGL_DEVICE void gemv_chain(uint32_t half, uint32_t M, const uint8_t* smem, float* s_part) { + using L = GemvSmem; + const uint32_t lane = threadIdx.x % 32; + const uint32_t j = lane >> 3, r = lane & 7; + // ldmatrix.x4 for A: matrix j covers tokens (j & 1) * 8 + r and K offset (j >> 1) * 8. + const uint32_t tok = (j & 1) * 8 + r; + const uint32_t a_row = tok < M ? tok : 0; + const uint32_t a_addr = smem_addr(smem + L::kX + a_row * kRowBytes + (half * kSplitK + (j >> 1) * 8) * 2); + // ldmatrix.x2 for B: matrix (j & 1) covers experts r and K offset (j & 1) * 8. + const uint32_t b_row = r < kExpertsPerCta ? r : 0; + const uint32_t b_addr = smem_addr(smem + L::kW + b_row * kRowBytes + (half * kSplitK + (j & 1) * 8) * 2); + // Fragments are loaded kDepth steps ahead so shared-memory latency stays off the MMA chain. + constexpr uint32_t kDepth = 4; + uint32_t a[kDepth][4], b[kDepth][2]; +#pragma unroll + for (uint32_t i = 0; i < kDepth; ++i) { + ldmatrix_x4(a[i], a_addr + i * 32); + ldmatrix_x2(b[i], b_addr + i * 32); + } + float d[4] = {0.0f, 0.0f, 0.0f, 0.0f}; +#pragma unroll + for (uint32_t step = 0; step < kChainSteps; ++step) { + const uint32_t slot = step % kDepth; + mma_bf16_16816(d, a[slot][0], a[slot][1], a[slot][2], a[slot][3], b[slot][0], b[slot][1]); + if (step + kDepth < kChainSteps) { + ldmatrix_x4(a[slot], a_addr + (step + kDepth) * 32); + ldmatrix_x2(b[slot], b_addr + (step + kDepth) * 32); + } + } + // C fragment: d0/d1 = (token g, experts 2c, 2c+1), d2/d3 = token g + 8. + const uint32_t g = lane >> 2, c = lane & 3; + if (c < kExpertsPerCta / 2) { + float* part = s_part + half * 16 * kExpertsPerCta; + part[g * kExpertsPerCta + 2 * c] = d[0]; + part[g * kExpertsPerCta + 2 * c + 1] = d[1]; + if constexpr (kMaxM > 8) { + part[(g + 8) * kExpertsPerCta + 2 * c] = d[2]; + part[(g + 8) * kExpertsPerCta + 2 * c + 1] = d[3]; + } + } +} + +SGL_DEVICE void mma_e4m3_16832(float (&d)[4], const uint32_t (&a)[4], const uint32_t (&b)[2]) { + asm volatile( + "mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, " + "{%0, %1, %2, %3};" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1])); +} + +// Two values -> two e4m3 bytes (low byte = first), saturating, round to nearest even. +SGL_DEVICE uint32_t cvt_e4m3x2(float lo, float hi) { + uint32_t out; + asm("{\n" + ".reg .b16 pair;\n" + "cvt.rn.satfinite.e4m3x2.f32 pair, %2, %1;\n" + "cvt.u32.u16 %0, pair;\n" + "}" + : "=r"(out) + : "f"(lo), "f"(hi)); + return out; +} + +// One K split of the shared gate/up projection for this CTA's 8 rows; fp32 partials to s_part. +template +SGL_DEVICE void shared_chain(uint32_t split, uint32_t M, const uint8_t* smem, float* s_part) { + using L = GemvSmem; + const uint32_t lane = threadIdx.x % 32; + const uint32_t j = lane >> 3, r = lane & 7; + const uint32_t tok = (j & 1) * 8 + r; + const uint32_t a_row = tok < M ? tok : 0; + const uint32_t kbase = split * (kHidden / kShSplits); + // ldmatrix on byte pairs: a 16 B row segment holds 16 e4m3 values (one m16n8k32 half-K). + const uint32_t a_addr = smem_addr(smem + L::kXq + a_row * kShRowBytes + kbase + (j >> 1) * 16); + const uint32_t b_addr = smem_addr(smem + L::kShW + r * kShRowBytes + kbase + (j & 1) * 16); + constexpr uint32_t kDepth = 4; + uint32_t a[kDepth][4], b[kDepth][2]; +#pragma unroll + for (uint32_t i = 0; i < kDepth; ++i) { + ldmatrix_x4(a[i], a_addr + i * 32); + ldmatrix_x2(b[i], b_addr + i * 32); + } + float d[4] = {0.0f, 0.0f, 0.0f, 0.0f}; +#pragma unroll + for (uint32_t step = 0; step < kShSteps; ++step) { + const uint32_t slot = step % kDepth; + mma_e4m3_16832(d, a[slot], b[slot]); + if (step + kDepth < kShSteps) { + ldmatrix_x4(a[slot], a_addr + (step + kDepth) * 32); + ldmatrix_x2(b[slot], b_addr + (step + kDepth) * 32); + } + } + const uint32_t g = lane >> 2, c = lane & 3; + float* part = s_part + split * 16 * 8; + part[g * 8 + 2 * c] = d[0]; + part[g * 8 + 2 * c + 1] = d[1]; + if constexpr (kMaxM > 8) { + part[(g + 8) * 8 + 2 * c] = d[2]; + part[(g + 8) * 8 + 2 * c + 1] = d[3]; + } +} + +SGL_DEVICE void mbar_init(uint32_t bar, uint32_t count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" ::"r"(bar), "r"(count) : "memory"); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); +} + +SGL_DEVICE void mbar_arrive(uint32_t bar) { + asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];" ::"r"(bar) : "memory"); +} + +SGL_DEVICE void mbar_expect_tx(uint32_t bar, uint32_t bytes) { + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(bar), "r"(bytes) : "memory"); +} + +SGL_DEVICE void mbar_wait(uint32_t bar, uint32_t parity) { + asm volatile( + "{\n" + ".reg .pred done;\n" + "WAIT_%=:\n" + "mbarrier.try_wait.parity.shared::cta.b64 done, [%0], %1;\n" + "@!done bra WAIT_%=;\n" + "}" ::"r"(bar), + "r"(parity) + : "memory"); +} + +SGL_DEVICE void bulk_load(uint32_t dst, const void* src, uint32_t bytes, uint32_t bar) { + asm volatile( + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"(dst), + "l"(src), + "r"(bytes), + "r"(bar) + : "memory"); +} + +// Warps 2..7 of a CTA with shared rows: quantize the token rows to e4m3 the way +// static_quant_fp8 does (x * (1 / scale), saturate, round to nearest even), then +// warps 2..5 run the gate/up K splits (after all 192 threads arrive on the kBarQ +// mbarrier) while warps 6..7 quantize row b < M to NVFP4. +template +SGL_DEVICE void shared_gate_up(const Params& p, uint32_t M, uint8_t* smem, uint32_t bar_w, uint32_t bar_x) { + using L = GemvSmem; + const uint32_t t = threadIdx.x - 64; // 0 .. 191 + const uint32_t warp = threadIdx.x / 32; + mbar_wait(bar_x, 0); + const float inv = div_full(1.0f, p.sh_in_scale[0]); + for (uint32_t i = t; i < M * (kHidden / 16); i += kThreads - 64) { + const uint32_t row = i / (kHidden / 16), blk = i % (kHidden / 16); + const uint4* src = reinterpret_cast(smem + L::kX + row * kRowBytes + blk * 32); + const uint4 lo = src[0], hi = src[1]; + const uint32_t w[8] = {lo.x, lo.y, lo.z, lo.w, hi.x, hi.y, hi.z, hi.w}; + uint32_t q[4]; +#pragma unroll + for (int k = 0; k < 4; ++k) { + const float a0 = fminf(fmaxf(__fmul_rn(bf16_lo(w[2 * k]), inv), -448.0f), 448.0f); + const float a1 = fminf(fmaxf(__fmul_rn(bf16_hi(w[2 * k]), inv), -448.0f), 448.0f); + const float a2 = fminf(fmaxf(__fmul_rn(bf16_lo(w[2 * k + 1]), inv), -448.0f), 448.0f); + const float a3 = fminf(fmaxf(__fmul_rn(bf16_hi(w[2 * k + 1]), inv), -448.0f), 448.0f); + q[k] = cvt_e4m3x2(a0, a1) | (cvt_e4m3x2(a2, a3) << 16); + } + *reinterpret_cast(smem + L::kXq + row * kShRowBytes + blk * 16) = make_uint4(q[0], q[1], q[2], q[3]); + } + const uint32_t bar_q = smem_addr(smem + L::kBarQ); + mbar_arrive(bar_q); + if (warp < 2 + kShSplits) { + mbar_wait(bar_q, 0); + mbar_wait(bar_w, 0); + shared_chain(warp - 2, M, smem, reinterpret_cast(smem + L::kShPart)); + } else if ((p.mode & kModeQuant) && blockIdx.x < M) { + const uint8_t* row = smem + L::kX + blockIdx.x * kRowBytes; + for (uint32_t blk = threadIdx.x - 64 - 32 * kShSplits; blk < kBlocksPerRow; blk += kThreads - 64 - 32 * kShSplits) { + const uint4* src = reinterpret_cast(row + blk * kSfVec * 2); + uint32_t sf; + uint2 packed; + quantize_block(src[0], src[1], p.global_scale[0], sf, packed); + *reinterpret_cast(p.x_fp4 + static_cast(blockIdx.x) * (kHidden / 2) + blk * 8) = packed; + p.x_sf[static_cast(blockIdx.x) * kBlocksPerRow + blk] = static_cast(sf); + } + } +} + +template +SGL_DEVICE void router_front_kernel(const Params& p) { + using L = GemvSmem; + static_assert(!kShared || kMaxM <= 8, "the shared gate/up stage fits shared memory only for M <= 8"); + extern __shared__ __align__(128) uint8_t smem[]; + __shared__ uint32_t s_mask[kExperts]; + __shared__ int16_t s_tile[kExperts]; + __shared__ int16_t s_ids[kMaxTokens * kTopK]; + __shared__ uint32_t s_warp[kWarps]; + __shared__ uint32_t s_last; + + const uint32_t warp = threadIdx.x / 32; + const uint32_t expert0 = blockIdx.x * kExpertsPerCta; + const uint32_t M = p.num_tokens; + const uint32_t tb = blockIdx.x * 8; + float* s_part = reinterpret_cast(smem + L::kPart); + const uint32_t bar_w = smem_addr(smem + L::kBarW); + const uint32_t bar_x = smem_addr(smem + L::kBarX); + trace_stamp(p, tb + 0); + + // This CTA's 8 rows of the shared gate/up projection, when it has any. + const uint32_t sh_row0 = blockIdx.x * 8; + const bool has_sh = kShared && sh_row0 < p.sh_n; + + // Router (and shared) weights are static: their bulk copies are issued before waiting on the producer grid. + if (threadIdx.x == 0) { + mbar_init(bar_w, 1); + mbar_init(bar_x, 1); + if (has_sh) mbar_init(smem_addr(smem + L::kBarQ), kThreads - 64); + mbar_expect_tx(bar_w, kExpertsPerCta * kHidden * 2 + (has_sh ? 8 * kHidden : 0)); +#pragma unroll + for (uint32_t e = 0; e < kExpertsPerCta; ++e) { + bulk_load(smem_addr(smem + L::kW + e * kRowBytes), p.weight + static_cast(expert0 + e) * kHidden, + kHidden * 2, bar_w); + } + if (has_sh) { +#pragma unroll + for (uint32_t r = 0; r < 8; ++r) { + bulk_load(smem_addr(smem + L::kShW + r * kShRowBytes), p.sh_weight + static_cast(sh_row0 + r) * kHidden, + kHidden, bar_w); + } + } + } + __syncthreads(); + + device::PDLWaitPrimary(); + trace_stamp(p, tb + 1); + if (!(p.mode & kModeTopk)) { + // The dependent grid reads our outputs only after its own griddepcontrol.wait. + device::PDLTriggerSecondary(); + } + if (threadIdx.x == 0) { + mbar_expect_tx(bar_x, M * kHidden * 2); +#pragma unroll + for (uint32_t row = 0; row < kMaxM; ++row) { + if (row < M) { + bulk_load(smem_addr(smem + L::kX + row * kRowBytes), p.x + static_cast(row) * kHidden, kHidden * 2, + bar_x); + } + } + } + + if (warp < 2) { + mbar_wait(bar_w, 0); + mbar_wait(bar_x, 0); + trace_stamp(p, tb + 2); + gemv_chain(warp, M, smem, s_part); + } else if (has_sh) { + shared_gate_up(p, M, smem, bar_w, bar_x); + } else if ((p.mode & kModeQuant) && blockIdx.x < M) { + // CTA b < M quantizes token row b while warps 0 and 1 run the GEMV chains. + mbar_wait(bar_x, 0); + const uint8_t* row = smem + L::kX + blockIdx.x * kRowBytes; + for (uint32_t blk = threadIdx.x - 64; blk < kBlocksPerRow; blk += kThreads - 64) { + const uint4* src = reinterpret_cast(row + blk * kSfVec * 2); + uint32_t sf; + uint2 packed; + quantize_block(src[0], src[1], p.global_scale[0], sf, packed); + *reinterpret_cast(p.x_fp4 + static_cast(blockIdx.x) * (kHidden / 2) + blk * 8) = packed; + p.x_sf[static_cast(blockIdx.x) * kBlocksPerRow + blk] = static_cast(sf); + } + } + __syncthreads(); + trace_stamp(p, tb + 3); + if (threadIdx.x < M * kExpertsPerCta) { + const uint32_t m = threadIdx.x / kExpertsPerCta; + const uint32_t e = threadIdx.x % kExpertsPerCta; + const float lo = s_part[m * kExpertsPerCta + e]; + const float hi = s_part[(16 + m) * kExpertsPerCta + e]; + p.logits[m * kExperts + expert0 + e] = __float2bfloat16_rn(__fadd_rn(lo, hi)); + } + if (has_sh && threadIdx.x < M * 8) { + const uint32_t m = threadIdx.x / 8; + const uint32_t n = threadIdx.x % 8; + const float* sp = reinterpret_cast(smem + L::kShPart); + float acc = sp[m * 8 + n]; +#pragma unroll + for (uint32_t k = 1; k < kShSplits; ++k) acc = __fadd_rn(acc, sp[(k * 16 + m) * 8 + n]); + const float alpha = __fmul_rn(p.sh_in_scale[0], p.sh_w_scale[0]); + p.sh_out[static_cast(m) * p.sh_n + sh_row0 + n] = __float2bfloat16_rn(__fmul_rn(acc, alpha)); + } + if (!(p.mode & kModeTopk)) return; + + // Publish: bar.sync orders this CTA's stores before the arriving thread's + // gpu-scope release; the last arrival acquires and runs top-k. + __syncthreads(); + trace_stamp(p, tb + 4); + if (threadIdx.x == 0) { + uint32_t prev; + asm volatile("atom.add.acq_rel.gpu.u32 %0, [%1], 1;" : "=r"(prev) : "l"(p.arrival) : "memory"); + s_last = ((prev + 1u) % kNumCtas) == 0u ? 1u : 0u; + } + __syncthreads(); + trace_stamp(p, tb + 5); + // The dependent grid reads our outputs only after its own griddepcontrol.wait, + // which waits for this grid to complete; triggering here lets it launch during the tail. + device::PDLTriggerSecondary(); + if (s_last) { + trace_stamp(p, kNumCtas * 8 + 0); + topk_and_tables(p, s_mask, s_tile, s_ids, s_warp); + } +} + + +} // namespace qwen_router_front + +} // namespace sglang + +// Router logits and the NVFP4 input of up to Q35_ROUTER_MAX_M rows (1, 8 or 16, set at +// compile time) in one launch, with PDL. +extern "C" __global__ __launch_bounds__(sglang::qwen_router_front::kThreads, 1) void q35_router_gemm_quant( + const __grid_constant__ sglang::qwen_router_front::Params p) { + sglang::qwen_router_front::router_front_kernel(p); +} diff --git a/sgl_nvidia_mega/qwen3_5/spec.py b/sgl_nvidia_mega/qwen3_5/spec.py new file mode 100644 index 00000000..454ea66b --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/spec.py @@ -0,0 +1,151 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The Qwen3.5 megakernel's spec: the setups it serves and its build. + +SGLang checks the requirements at argument resolution and the model at load +(``sglang.srt.megakernels``); an unsupported setup stops the server on every rank, +listing every reason. This module stays light: SGLang imports it in the launcher. +""" + +from __future__ import annotations + +from sglang.srt.megakernels.spec import ( + Feature, + MegakernelSpec, + Numerics, + NumericsKind, + Requirements, +) + +from sgl_nvidia_mega.qwen3_5 import TEXT_CONFIG +from sgl_nvidia_mega.qwen3_5.plan import MAX_ROWS + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/megakernels/spec.py": ( + "Feature", + "SPECULATIVE_DECODING", + "MULTI_NODE_TP", + "KV_OFFLOAD", + "MegakernelSpec", + "Numerics", + "NumericsKind", + "BITWISE", + "Requirements", + "decode_graph_backends", + ), + "srt/arg_groups/fields/*.py": ( + "kv_cache_dtype", + "speculative_algorithm", + "speculative_eagle_topk", + "speculative_use_rejection_sampling", + "moe_a2a_backend", + "moe_runner_backend", + "flashinfer_allreduce_fusion_backend", + "attention_backend", + "decode_attention_backend", + "linear_attn_backend", + "linear_attn_decode_backend", + "linear_attn_verify_backend", + "enable_linear_replayssm_spec", + "mamba_ssm_dtype", + "startup_weight_load_mode", + ), + "srt/configs/qwen3_5.py": ("text_config",), + "srt/megakernels/eagle_binding.py": ("EagleForwardView",), +} + +ARCHITECTURE = "Qwen3_5MoeForConditionalGeneration" +# the sglang.srt.megakernels contract this spec is written against +API_VERSION = 1 + +REQUIREMENTS = Requirements( + isas=frozenset({"sm_103"}), + tp_sizes=frozenset({8}), + dtypes=frozenset({"bfloat16"}), + # the decode CUDA graphs the covered step is tested with (eager steps too) + decode_graph_backends=frozenset({"full", "disabled"}), + server_args={ + "kv_cache_dtype": frozenset({"fp8_e4m3"}), + # MTP (NEXTN resolves to EAGLE) as a chain: uniform verify steps of 7 rows + "speculative_algorithm": frozenset({"EAGLE"}), + "speculative_eagle_topk": frozenset({1}), + # SGLang's EAGLE entries serve a chain without rejection sampling + "speculative_use_rejection_sampling": frozenset({False}), + "moe_a2a_backend": frozenset({"none"}), + # the deferred MoE finalize the boundary takes (the measured TP8 setup) + "moe_runner_backend": frozenset({"flashinfer_trtllm"}), + "flashinfer_allreduce_fusion_backend": frozenset({"cutedsl"}), + "attention_backend": frozenset({"trtllm_mha"}), + "decode_attention_backend": frozenset({None, "trtllm_mha"}), + # the Triton GDN verify on the ReplaySSM ring with BF16 checkpoints + "linear_attn_backend": frozenset({"triton"}), + "linear_attn_decode_backend": frozenset({None, "triton"}), + "linear_attn_verify_backend": frozenset({None, "triton"}), + "enable_linear_replayssm_spec": frozenset({True}), + "mamba_ssm_dtype": frozenset({"bfloat16"}), + # the kernels read the weights SGLang loaded, after its last conversion + "startup_weight_load_mode": frozenset({"serial"}), + }, + # the text shapes live in text_config, which config_disable_reasons checks + hf_config={"model_type": "qwen3_5_moe"}, +) + +# Against SGLang's forward on the same GPU (test/e2e/test_qwen3_5_stock_parity.py): the +# stock step and every slot (route, mkm, gdn_verify, gdn_pdl, hop) reproduce it bit +# for bit +NUMERICS = Numerics(kind=NumericsKind.BITWISE) + + +def text_config_reasons(hf_config) -> list[str]: + """The text_config fields the kernels are built for that differ.""" + text = hf_config.text_config + return [ + f"config.text_config.{key} {getattr(text, key, None)!r} is not {want!r}" + for key, want in TEXT_CONFIG.items() + if getattr(text, key, None) != want + ] + + +def binding_reasons() -> list[str]: + """[] when this SGLang has the EAGLE binding whose forward view the covered steps + read (``Step.view``, an ``EagleForwardView``).""" + try: + from sglang.srt.megakernels.eagle_binding import EagleForwardView # noqa: F401 + except ImportError as e: + return [ + f"this SGLang has no EAGLE forward view (megakernels/eagle_binding.py: {e})" + ] + return [] + + +def _config_disable_reasons(cfg, hf_config) -> list[str]: + from sgl_nvidia_mega.qwen3_5 import runtime + + return text_config_reasons(hf_config) + binding_reasons() + runtime.slot_reasons() + + +def _build(model, hf_config, ctx): + from sgl_nvidia_mega.qwen3_5 import runtime + + return runtime.build(model, ctx) + + +SPEC = MegakernelSpec( + name="Qwen3.5", + requires=REQUIREMENTS, + features=frozenset( + { + Feature.SPECULATIVE_DECODING, + # TP8 spans the two 4-GPU trays of one NVL72 rack + Feature.MULTI_NODE_TP, + # HiCache: covered steps read only running requests' pages + Feature.KV_OFFLOAD, + } + ), + numerics=NUMERICS, + max_rows=MAX_ROWS, + build=_build, + api_version=API_VERSION, + config_disable_reasons=_config_disable_reasons, +) diff --git a/test/e2e/qwen3_5_server.py b/test/e2e/qwen3_5_server.py new file mode 100644 index 00000000..88cdddc2 --- /dev/null +++ b/test/e2e/qwen3_5_server.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The Qwen3.5-397B-A17B NVFP4 servers the serving tests launch: GB300, TP8, MTP (verify +steps of 7 rows a request), batch 1. TP8 spans two 4-GPU GB300 trays; a test launches +its own server only where one host has 8 GPUs, otherwise it reads a running server +(``SGLANG_NVIDIA_MEGAKERNEL_E2E_URL``) and the files its ranks log to +(``SGLANG_NVIDIA_MEGAKERNEL_E2E_LOGS``, comma-separated).""" + +import os + +MODEL = os.environ.get( + "SGLANG_NVIDIA_MEGAKERNEL_E2E_MODEL_PATH", "nvidia/Qwen3.5-397B-A17B-NVFP4" +) +URL = os.environ.get("SGLANG_NVIDIA_MEGAKERNEL_E2E_URL") +LOGS = [ + p for p in os.environ.get("SGLANG_NVIDIA_MEGAKERNEL_E2E_LOGS", "").split(",") if p +] +ENV = {} +ARGS = [ + "--trust-remote-code", + "--tp", + "8", + "--quantization", + "modelopt_fp4", + "--kv-cache-dtype", + "fp8_e4m3", + "--attention-backend", + "trtllm_mha", + "--moe-runner-backend", + "flashinfer_trtllm", + "--flashinfer-allreduce-fusion-backend", + "cutedsl", + "--linear-attn-decode-backend", + "triton", + "--enable-linear-replayssm-spec", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + "6", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "7", + # batch 1: every verify step is the covered width of 7 rows + "--max-running-requests", + "1", + "--cuda-graph-bs-decode", + "1", + "--mem-fraction-static", + "0.85", +] + os.environ.get("SGLANG_NVIDIA_MEGAKERNEL_E2E_EXTRA_ARGS", "").split() diff --git a/test/e2e/test_qwen3_5_stock_parity.py b/test/e2e/test_qwen3_5_stock_parity.py new file mode 100644 index 00000000..f33f3d1f --- /dev/null +++ b/test/e2e/test_qwen3_5_stock_parity.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The megakernel's stock step against SGLang's, bit for bit, on GB300 TP8 with MTP, in +eager verify steps and inside the decode CUDA graphs. + +With every slot off, the covered step runs each package's stock code, which must +reproduce SGLang's verify step exactly; a bitwise slot must too, so +``SGLANG_NVIDIA_MEGAKERNEL_E2E_SLOTS`` (e.g. ``route``) runs the same check with those +slots on. ``SGLANG_NVIDIA_MEGAKERNEL_PARITY=1`` (test +only) runs SGLang's forward and then the megakernel's on every covered step and +compares the logits and the hidden rows bit for bit: an eager step logs it at once; a +captured step records both forwards and the comparison, and the next eager forward (a +request's prefill) logs what the replays added. Requests run one at a time, so every +verify step is the covered width (7 rows). + +A server launched elsewhere (TP8 spans two GB300 trays) is read through +``SGLANG_NVIDIA_MEGAKERNEL_E2E_URL`` and its rank logs +(``SGLANG_NVIDIA_MEGAKERNEL_E2E_LOGS``); it must run with the parity switch and the +slots under test in ``SGLANG_NVIDIA_MEGAKERNEL_SLOTS``. +""" + +import os +import re +import tempfile +import unittest +from pathlib import Path + +import requests +from qwen3_5_server import ARGS, ENV, LOGS, MODEL, URL +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + terminate_and_kill_process_tree, +) + +from sgl_nvidia_mega.plugin import FLAG_CLI + +LAUNCH_TIMEOUT = 3600 +PARITY_ENV = { + "SGLANG_NVIDIA_MEGAKERNEL_SLOTS": os.environ.get( + "SGLANG_NVIDIA_MEGAKERNEL_E2E_SLOTS", "" + ), + "SGLANG_NVIDIA_MEGAKERNEL_PARITY": "1", +} +# an eager step on padding only (valid_rows=0) writes no output rows, so it is not +# compared; SGLang's graph-capture warmups log as capture-warmup (runtime/parity.py) +EAGER = re.compile( + r"NVIDIA megakernel parity eager target_verify rows=(\d+) " + r"logits_differ=(\d+) hidden_differ=(\d+) " + r"logits_compared=(\d+) hidden_compared=(\d+) .*valid_rows=[1-9]" +) +GRAPH = re.compile( + r"NVIDIA megakernel parity graph target_verify rows=(\d+) steps=(\d+) " + r"logits_differ=(\d+) hidden_differ=(\d+) " + r"logits_compared=(\d+) hidden_compared=(\d+)" +) +REQUESTS = 6 + + +class _StockParity(CustomTestCase): + """One server: its covered steps compared bit for bit.""" + + EXTRA_ARGS: list = [] + + @classmethod + def setUpClass(cls): + cls.process, cls.url = None, URL or DEFAULT_URL_FOR_TEST + if URL: + cls.log_paths = LOGS + return + cls.logs = tuple( + tempfile.NamedTemporaryFile("w+", suffix=s) for s in (".out", ".err") + ) + cls.log_paths = [f.name for f in cls.logs] + cls.process = popen_launch_server( + MODEL, + cls.url, + timeout=LAUNCH_TIMEOUT, + other_args=ARGS + [FLAG_CLI, *cls.EXTRA_ARGS], + env={**os.environ, **ENV, **PARITY_ENV}, + return_stdout_stderr=cls.logs, + ) + + @classmethod + def tearDownClass(cls): + if cls.process is not None: + terminate_and_kill_process_tree(cls.process) + for log in cls.logs: + log.close() + + def _generate(self, i: int) -> str: + r = requests.post( + self.url + "/generate", + json={ + "text": f"Question {i}: explain step by step whether {i} * 17 + 3 is odd.", + "sampling_params": {"temperature": 0, "max_new_tokens": 64}, + }, + timeout=600, + ) + r.raise_for_status() + return r.json()["text"] + + def _parity_lines(self): + # the last request's prefill logs what the earlier requests' replays added + for i in range(REQUESTS + 1): + self._generate(i) + text = "".join(Path(p).read_text(errors="replace") for p in self.log_paths) + return EAGER.findall(text), GRAPH.findall(text) + + def check(self, graph): + """graph: the lines of captured steps (True), of eager ones (False), or + either (None, for a server launched elsewhere).""" + eager, graphs = self._parity_lines() + found = {True: graphs, False: eager, None: eager + graphs}[graph] + self.assertTrue(found, "no parity line: the covered steps did not run") + differ = [line for line in eager + graphs if any(int(n) for n in line[-4:-2])] + self.assertEqual(differ, [], f"{len(differ)} parity lines differ") + self.assertEqual({int(line[0]) for line in found}, {7}) + # an autotune dummy run returns no logits: compare real ones too + self.assertGreater(sum(int(line[-2]) for line in found), 0) + if graphs: + self.assertGreater(sum(int(line[1]) for line in graphs), 0) + + +@unittest.skipIf(URL, "launches its own server with the parity switch") +class TestEagerStockStepIsSGLangs(_StockParity): + EXTRA_ARGS = ["--disable-decode-cuda-graph"] + + def test_every_eager_covered_step_is_bitwise(self): + self.check(graph=False) + + +@unittest.skipIf(URL, "launches its own server with the parity switch") +class TestGraphStockStepIsSGLangs(_StockParity): + def test_every_captured_covered_step_is_bitwise(self): + self.check(graph=True) + + +@unittest.skipUnless(URL, "reads a server launched with the parity switch") +class TestRunningServerStockStepIsSGLangs(_StockParity): + def test_every_covered_step_is_bitwise(self): + self.check(graph=None) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/kernels/test_qwen3_5_gdn.py b/test/kernels/test_qwen3_5_gdn.py new file mode 100644 index 00000000..d2d120a9 --- /dev/null +++ b/test/kernels/test_qwen3_5_gdn.py @@ -0,0 +1,234 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The kernels slot gdn_verify launches its own way, on one SM100-family GPU (GB300), +against SGLang's, bit for bit, at Qwen3.5's TP8 GatedDeltaNet shapes: + +- the causal conv update with the 128-channel tile and the token-major output, against + ``causal_conv1d_update`` (output, conv states and intermediate windows), on the + contiguous split and on the token-major projection views; +- the gated RMSNorm writing static FP8, against SGLang's gated norm followed by the FP8 + linear's static input quantization (``static_quant_fp8``), with z as rows and as + token-major views; +- the ReplaySSM verify launch, with and without the dependent launch, against + ``gdn_replayssm_spec_decode``'s (output and ring entries). +""" + +import pytest +import torch + +H, K, HV, V = 2, 128, 8, 128 # per rank at TP8: 16 / 64 heads over 8 +D = 2 * H * K + HV * V # the conv's q / k / v channels +DRAFT, WIDTH, SLOTS = 7, 4, 9 + + +def _gpu_ready() -> bool: + if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10: + return False + try: + import triton # noqa: F401 + except ImportError: + return False + return True + + +pytestmark = pytest.mark.skipif(not _gpu_ready(), reason="needs an SM100-family GPU") + + +def _projection(bs, views, g): + """A verify step's mixed q / k / v rows [bs * 7, D]: a contiguous copy, or the + token-major view of the q / k / v / z projection.""" + qkvz = torch.randn(bs * DRAFT, D + HV * V, device="cuda", generator=g).to( + torch.bfloat16 + ) + return qkvz[:, :D] if views else qkvz[:, :D].contiguous() + + +@pytest.mark.parametrize("views", [False, True]) +@pytest.mark.parametrize("block_n", [256, 128]) +def test_conv_update_is_sglangs(views, block_n): + from sglang.kernels.ops.mamba.causal_conv1d_triton import causal_conv1d_update + + from sgl_nvidia_mega.qwen3_5.gdn.conv import conv_update + + g = torch.Generator(device="cuda").manual_seed(block_n + views) + bs = 2 + rows = _projection(bs, views, g) + weight = torch.randn(D, WIDTH, device="cuda", generator=g).to(torch.bfloat16) + bias = torch.randn(D, device="cuda", generator=g).to(torch.bfloat16) + state = torch.randn(SLOTS, D, WIDTH - 1, device="cuda", generator=g).to( + torch.bfloat16 + ) + window = torch.zeros( + SLOTS, DRAFT, D, WIDTH - 1, dtype=torch.bfloat16, device="cuda" + ) + slots = torch.tensor([4, 7], dtype=torch.int32, device="cuda") + inter = torch.tensor([0, 1], dtype=torch.int32, device="cuda") + x = rows.view(bs, DRAFT, D).transpose(1, 2) + want_state, want_window = state.clone(), window.clone() + want = causal_conv1d_update( + x, + want_state, + weight, + bias, + "silu", + conv_state_indices=slots, + intermediate_conv_window=want_window, + intermediate_state_indices=inter, + ) + got = conv_update( + x, + state, + weight, + bias, + "silu", + slots, + window, + inter, + block_n=block_n, + token_major_out=views, + ) + assert torch.equal(got.contiguous(), want.contiguous()) + assert torch.equal(state, want_state) and torch.equal(window, want_window) + # the slot reads the output back as token rows without a copy + assert got.transpose(1, 2).is_contiguous() + + +@pytest.mark.parametrize("views", [False, True]) +def test_norm_fp8_is_sglangs_norm_then_static_quant(views): + from sglang.kernels.ops.attention.fla.layernorm_gated import RMSNorm + from sglang.kernels.ops.quantization.fp8_kernel import static_quant_fp8 + + from sgl_nvidia_mega.qwen3_5.gdn.norm_fp8 import rms_norm_gated_fp8 + + g = torch.Generator(device="cuda").manual_seed(int(views)) + tokens = 2 * DRAFT + core = torch.randn(tokens * HV, V, device="cuda", generator=g).to(torch.bfloat16) + qkvz = torch.randn(tokens, D + HV * V, device="cuda", generator=g).to( + torch.bfloat16 + ) + z = qkvz[:, D:].view(tokens, HV, V) # token-major view, rows with gaps + if not views: + z = z.reshape(-1, V) + norm = RMSNorm( + V, eps=1e-6, norm_before_gate=True, device="cuda", dtype=torch.bfloat16 + ) + norm.weight.data = torch.randn(V, device="cuda", generator=g).to(torch.bfloat16) + scale = torch.tensor([0.02], device="cuda") + want, _ = static_quant_fp8( + norm(core, z).reshape(tokens, HV * V), scale, repeat_scale=False + ) + got = rms_norm_gated_fp8( + core, norm.weight, norm.eps, z, scale, activation=norm.activation + ) + assert torch.equal( + got.view(torch.uint8).reshape(tokens, -1), want.view(torch.uint8) + ) + + +@pytest.mark.parametrize("pdl", [False, True]) +def test_replayssm_verify_is_sglangs(pdl): + """The slot's verify launch (32-wide value tiles on 2 warps, and with or without the + dependent launch) against gdn_replayssm_spec_decode's verify launch: the output and + this step's ring entries.""" + try: + from sglang.kernels.ops.attention.linear.gdn_replayssm_spec_decode import ( + gdn_replayssm_spec_decode, + ) + except ImportError: # in fla/ before sgl-project/sglang#45112 + from sglang.kernels.ops.attention.fla.gdn_replayssm_spec_decode import ( + gdn_replayssm_spec_decode, + ) + + from sgl_nvidia_mega.qwen3_5.gdn import replayssm + + g = torch.Generator(device="cuda").manual_seed(3 + int(pdl)) + bs, ring_len, slots = 2, 16, 9 + tokens = bs * DRAFT + + def rand(*shape, dtype=torch.bfloat16): + return torch.randn(*shape, device="cuda", generator=g).to(dtype) + + qkv = rand(tokens, D) # strided q / k / v, as the slot's split leaves them + q = qkv[:, : H * K].view(tokens, H, K) + k = qkv[:, H * K : 2 * H * K].view(tokens, H, K) + v = qkv[:, 2 * H * K :].view(tokens, HV, V) + ba = rand(tokens, 2 * HV) + b, a = ba[:, :HV], ba[:, HV:] + A_log, dt_bias = rand(HV, dtype=torch.float32), rand(HV, dtype=torch.float32) + state = rand(slots, HV, V, K) + ring = [ + rand(slots, HV, ring_len, V), + rand(slots, H, ring_len, K), + -torch.rand(slots, HV, ring_len, device="cuda", generator=g), + rand(slots, HV, ring_len, V), + rand(slots, H, ring_len, K), + ] + write_pos = torch.randint(0, ring_len - DRAFT, (slots,), device="cuda", generator=g) + write_pos = write_pos.to(torch.int32) + cache_base = torch.randint(0, ring_len, (slots,), device="cuda", generator=g) + cache_base = cache_base.to(torch.int32) + is_flush = torch.zeros(slots, dtype=torch.int8, device="cuda") + qsl = torch.arange(0, tokens + 1, DRAFT, dtype=torch.int32, device="cuda") + states = torch.tensor([3, 5], dtype=torch.int32, device="cuda") + replay = torch.tensor([1, 2], dtype=torch.int32, device="cuda") + want_ring, got_ring = [t.clone() for t in ring], [t.clone() for t in ring] + want, got = (q.new_empty(tokens, HV, V) for _ in range(2)) + gdn_replayssm_spec_decode( + q=q, + k=k, + v=v, + a=a, + b=b, + A_log=A_log, + dt_bias=dt_bias, + checkpoint_state=state, + d_cache=want_ring[0], + k_cache=want_ring[1], + g_cache=want_ring[2], + rawv_cache=want_ring[3], + rawk_cache=want_ring[4], + beta_cache=None, + out=want, + query_start_loc=qsl, + ssm_state_indices=states, + replay_indices=replay, + write_pos=write_pos, + cache_base=cache_base, + is_flush=is_flush, + max_cache_len=ring_len, + max_spec_len=DRAFT, + scale=K**-0.5, + use_qk_l2norm_in_kernel=True, + null_block_id=-1, + launch_mode="verify", + block_v=32, + num_warps=2, + ) + replayssm.verify( + q, + k, + v, + a, + b, + A_log, + dt_bias, + got, + state, + *got_ring, + None, + qsl, + states, + replay, + write_pos, + cache_base, + is_flush, + max_cache_len=ring_len, + max_spec_len=DRAFT, + scale=K**-0.5, + null_block_id=-1, + block_v=32, + num_warps=2, + pdl=pdl, + ) + assert torch.equal(got, want) + assert all(torch.equal(x, y) for x, y in zip(got_ring, want_ring)) diff --git a/test/kernels/test_qwen3_5_hop.py b/test/kernels/test_qwen3_5_hop.py new file mode 100644 index 00000000..c7dd7da5 --- /dev/null +++ b/test/kernels/test_qwen3_5_hop.py @@ -0,0 +1,180 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The Qwen3.5 decode hop (``qwen3_5/sm103/hop``) on 4 SM100-family GPUs (GB300) with +NVLink multicast, against SGLang's FlashInfer MNNVL CuTe DSL fusion workspace as the +verify server builds it (hidden 4096, top-10, no early shared-expert load), bit for bit: + +- the all-reduce + residual add + RMSNorm, and the MoE finalize (top-10 rows and the + gated shared output) + all-reduce + residual add + RMSNorm, at 1-15 rows, over a + sequence that cycles the three mailbox stages, mixes both operations and delays one + rank in turn; the static FP8 output equals SGLang's static quantization of the norm; +- the same operations replayed from a CUDA graph with fresh inputs. + +Run with ``python test/kernels/test_qwen3_5_hop.py --num-gpu 4``. +""" + +import os + +import pytest +import torch +from sglang.srt.distributed import parallel_state as ps +from sglang.test.kernels.utils import multigpu_pytest_main +from sglang.test.test_utils import publish_build_topology + +HIDDEN, TOP_K, EPS, MAX_M = 4096, 10, 1e-6, 15 + +pytestmark = pytest.mark.skipif( + not (torch.cuda.is_available() and torch.cuda.get_device_capability()[0] == 10), + reason="Requires SM100/SM103", +) + + +@pytest.fixture(scope="module") +def rank(): + rank = int(os.environ["RANK"]) + world_size = int(os.environ["WORLD_SIZE"]) + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + ps.set_custom_all_reduce(False) + ps.init_distributed_environment( + world_size=world_size, + rank=rank, + local_rank=local_rank, + distributed_init_method="env://", + ) + publish_build_topology(tp_size=world_size, world_rank=rank) + ps.initialize_model_parallel() + yield rank + ps.destroy_model_parallel() + ps.destroy_distributed_environment() + + +@pytest.fixture(scope="module") +def pair(rank): + """(SGLang's fusion workspace, the hop) on the TP group, built once: each one + rendezvouses its own multicast region.""" + from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( + FlashInferMNNVLCuteDSLARFusion, + ) + from sglang.srt.runtime_context import get_spec + + from sgl_nvidia_mega.qwen3_5.sm103.hop import Hop + + group = ps.get_tp_group().device_group + device = torch.device(f"cuda:{torch.cuda.current_device()}") + # speculative decoding keeps the finalize on the ordering the verify server runs + with get_spec().override(speculative_algorithm="EAGLE"): + fusion = FlashInferMNNVLCuteDSLARFusion( + hidden_size=HIDDEN, + top_k=TOP_K, + max_m=MAX_M + 1, + rms_epsilon=EPS, + weight_bias=0.0, + process_group=group, + device=device, + ) + hop = Hop(max_m=MAX_M, eps=EPS, group=group, device=device, deadline_s=30.0) + return fusion, hop + + +def _operands(m, device, rank, seed): + """One hop's inputs: this rank's partial and MoE rows, and the residual and the + norm multiplier every rank holds alike.""" + own = torch.Generator(device=device).manual_seed(seed * 100 + rank) + common = torch.Generator(device=device).manual_seed(seed) + + def randn(*shape, gen=own): + return torch.randn(*shape, device=device, generator=gen).bfloat16() + + rows = m * TOP_K + return dict( + partial=randn(m, HIDDEN), + # gemm2's output, with rows past the routed ones as the runner pads it + routed=randn(rows + 5, HIDDEN), + expert_weights=torch.rand(m, TOP_K, device=device, generator=own).bfloat16(), + permuted=torch.randperm(rows, device=device, generator=own) + .to(torch.int32) + .view(m, TOP_K), + shared=randn(m, HIDDEN), + residual=randn(m, HIDDEN, gen=common), + gamma=(1 + 0.1 * randn(HIDDEN, gen=common).float()).bfloat16(), + ) + + +def _run(engine, op, x, **kw): + if op == "all_reduce": + return engine.all_reduce_residual_rms_norm( + local_contribution=x["partial"], + residual=x["residual"], + gamma=x["gamma"], + **kw, + ) + return engine.moe_finalize_all_reduce_rms_norm( + routed_output=x["routed"], + expert_weights=x["expert_weights"], + permuted_indices=x["permuted"], + gated_shared_output=x["shared"], + residual=x["residual"], + gamma=x["gamma"], + **kw, + ) + + +def _same(got, want): + for a, b in zip(got, want): + assert torch.equal(a.view(torch.int16), b.view(torch.int16)) + + +def test_hop_matches_flashinfer_fusion(rank, pair): + from sglang.kernels.ops.quantization.fp8_kernel import static_quant_fp8 + + fusion, hop = pair + world = ps.get_tp_group().world_size + device = torch.device(f"cuda:{torch.cuda.current_device()}") + scale = torch.tensor([0.05], device=device) + calls = [(op, m) for m in (1, 2, 3, 7, 8, 15) for op in ("all_reduce", "finalize")] + with torch.inference_mode(): + for i, (op, m) in enumerate(calls): + x = _operands(m, device, rank, seed=i) + want = _run(fusion, op, x) + if i % (world + 1) == rank: + torch.cuda._sleep(2_000_000) # this rank publishes late + quantize = i % 3 == 0 + got = _run(hop, op, x, fp8_scale=scale if quantize else None) + _same(got[:2], want) + if quantize: + fp8, _ = static_quant_fp8(want[0], scale, repeat_scale=False) + assert torch.equal(got[2].view(torch.uint8), fp8.view(torch.uint8)) + torch.cuda.synchronize() + + +def test_hop_replays_from_a_cuda_graph(rank, pair): + fusion, hop = pair + device = torch.device(f"cuda:{torch.cuda.current_device()}") + m = 7 + # four hops: one more than the mailbox has stages + ops = ("all_reduce", "finalize", "all_reduce", "finalize") + with torch.inference_mode(): + static = [_operands(m, device, rank, seed=50 + i) for i in range(len(ops))] + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for op, x in zip(ops, static): + _run(hop, op, x) # eager warmup on the capture stream + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outputs = [_run(hop, op, x) for op, x in zip(ops, static)] + for replay in range(3): + for i, x in enumerate(static): + fresh = _operands(m, device, rank, seed=100 + 10 * replay + i) + for name, tensor in x.items(): + tensor.copy_(fresh[name]) + graph.replay() + for op, x, got in zip(ops, static, outputs): + _same(got, _run(fusion, op, x)) + torch.cuda.synchronize() + + +if __name__ == "__main__": + multigpu_pytest_main(__name__, __file__, num_gpus=(4,)) diff --git a/test/kernels/test_qwen3_5_moe.py b/test/kernels/test_qwen3_5_moe.py new file mode 100644 index 00000000..5136d93e --- /dev/null +++ b/test/kernels/test_qwen3_5_moe.py @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The MoE package's kernels on one SM100-family GPU (GB300), bit for bit, eager and +replayed from a CUDA graph: +- the router front (slot route): router_gemm_quant's logits against the cuBLAS router + GEMM SGLang runs (ReplicatedLinear, bf16), and its NVFP4 input against the trtllm + runner's quantizer (flashinfer.fp4_quantize, linear scale layout), at 1-16 rows; +- the fused routed experts (slot mkm), against FlashInfer's trtllm-gen NVFP4 MoE as + SGLang serves Qwen3.5 at TP8 (routing from the logits, RenormalizeNaive, deferred + finalize): each token's 10 expert rows and their weights, at 1, 4 and 7 rows.""" + +import pytest +import torch + +from sgl_nvidia_mega.qwen3_5.sm103.moe import mkm, router + +E, H, TOP_K = mkm.EXPERTS, mkm.HIDDEN, mkm.TOP_K +I = mkm.W13_ROWS // 2 # intermediate per rank +X_SCALE = 448.0 * 6.0 / 12.0 # the static input scale of the routed experts +H_SCALE = 30.0 / (448.0 * 6.0) # the static FC2 input scale + + +def _gpu_ready() -> bool: + if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10: + return False + try: + import flashinfer # noqa: F401 + from cuda.bindings import nvrtc # noqa: F401 + except ImportError: + return False + return True + + +pytestmark = pytest.mark.skipif(not _gpu_ready(), reason="needs an SM100-family GPU") + + +def _experts(g, rows, cols, std): + """NVFP4 experts [E, rows, cols]: gaussian weights with row and block spread, + quantized by FlashInfer with a global scale per expert.""" + from flashinfer import fp4_quantize + + packed = torch.empty(E, rows, cols // 2, dtype=torch.uint8, device="cuda") + sf = torch.empty(E, rows, cols // 16, dtype=torch.uint8, device="cuda") + scale = torch.empty(E, dtype=torch.float32, device="cuda") + for e in range(E): + rs = torch.exp(torch.randn(rows, 1, device="cuda", generator=g) * 0.5) + cs = torch.exp(torch.randn(1, cols // 16, device="cuda", generator=g) * 0.5) + w = torch.randn(rows, cols, device="cuda", generator=g) * std + w = (w * rs * cs.repeat_interleave(16, dim=1)).to(torch.bfloat16) + s = (448.0 * 6.0 / w.float().abs().amax()).reshape(1) + q, q_sf = fp4_quantize(w, s, 16, False, False) + packed[e] = q.view(torch.uint8).reshape(rows, cols // 2) + sf[e] = q_sf.view(torch.uint8).reshape(rows, cols // 16) + scale[e] = s + return packed, sf, scale + + +@pytest.fixture(scope="module") +def layer(): + """One TP8 Qwen3.5 MoE layer on the trtllm-gen weight layout, as SGLang loads it.""" + from sglang.srt.layers.quantization.utils import ( + prepare_static_weights_for_trtllm_fp4_moe, + ) + + g = torch.Generator(device="cuda").manual_seed(11) + w13, sf13, s13 = _experts(g, 2 * I, H, 0.02) + w2, sf2, s2 = _experts(g, H, I, 0.02) + w13, sf13, w2, sf2 = prepare_static_weights_for_trtllm_fp4_moe( + w13, w2, sf13, sf2, H, I, E, is_gated=True + ) + g1 = (1.0 / (X_SCALE * s13)).float() + return { + "w13": w13, + "w13_sf": sf13.view(torch.float8_e4m3fn).reshape(E, 2 * I, H // 16), + "w2": w2, + "w2_sf": sf2.view(torch.float8_e4m3fn).reshape(E, H, I // 16), + "g1": g1, + "g1c": (g1 / H_SCALE).float(), + "g2": (H_SCALE / s2).float(), + "maps": mkm.weight_maps(w13, w2), + } + + +def _inputs(m, seed): + from flashinfer import fp4_quantize + + g = torch.Generator(device="cuda").manual_seed(seed) + x = torch.randn(m, H, device="cuda", generator=g) + x[:, torch.randint(0, H, (8,), device="cuda", generator=g)] *= ( + 20.0 # outlier channels + ) + q, q_sf = fp4_quantize( + x.to(torch.bfloat16), torch.tensor([X_SCALE], device="cuda"), 16, False, False + ) + logits = torch.randn(m, E, device="cuda", generator=g).to(torch.bfloat16) + return ( + logits, + q.view(torch.uint8).reshape(m, H // 2), + q_sf.view(torch.uint8).reshape(m, H // 16), + ) + + +def _fused(layer, logits, x_fp4, x_sf): + return mkm.mkm_moe( + layer["maps"], + logits, + x_fp4, + x_sf, + layer["w13_sf"].view(torch.uint8), + layer["w2_sf"].view(torch.uint8), + layer["g1"], + layer["g1c"], + layer["g2"], + ) + + +def _stock(layer, logits, x_fp4, x_sf): + """The rows and weights SGLang's trtllm runner hands its deferred finalize, row + t * 10 + k for expert k of token t.""" + from flashinfer.fused_moe import trtllm_fp4_block_scale_moe + from flashinfer.fused_moe.core import ActivationType + from sglang.srt.layers.moe.utils import RoutingMethodType + + m = logits.shape[0] + rows, weights, permuted = trtllm_fp4_block_scale_moe( + routing_logits=logits, + routing_bias=None, + hidden_states=x_fp4, + hidden_states_scale=x_sf.view(torch.float8_e4m3fn), + gemm1_weights=layer["w13"], + gemm1_weights_scale=layer["w13_sf"], + gemm1_bias=None, + gemm1_alpha=None, + gemm1_beta=None, + gemm1_clamp_limit=None, + gemm2_weights=layer["w2"], + gemm2_weights_scale=layer["w2_sf"], + gemm2_bias=None, + output1_scale_scalar=layer["g1c"], + output1_scale_gate_scalar=layer["g1"], + output2_scale_scalar=layer["g2"], + per_token_scale=None, + num_experts=E, + top_k=TOP_K, + n_group=None, + topk_group=None, + intermediate_size=I, + local_expert_offset=0, + local_num_experts=E, + routed_scaling_factor=None, + routing_method_type=RoutingMethodType.RenormalizeNaive, + do_finalize=False, + activation_type=ActivationType.Swiglu, + tune_max_num_tokens=8, + enable_pdl=True, + )[:3] + return rows[permuted.reshape(-1).long()[: m * TOP_K]], weights + + +@pytest.mark.parametrize("m", [1, 4, 7]) +@pytest.mark.parametrize("seed", [0, 1]) +def test_mkm_is_the_trtllm_routed_experts(layer, m, seed): + logits, x_fp4, x_sf = _inputs(m, seed) + rows, weights = _fused(layer, logits, x_fp4, x_sf) + want_rows, want_weights = _stock(layer, logits, x_fp4, x_sf) + assert want_weights.dtype == torch.bfloat16 + assert torch.equal(weights.view(torch.int16), want_weights.view(torch.int16)) + differ = (rows.view(torch.int16) != want_rows.view(torch.int16)).any(dim=1) + assert not differ.any(), f"rows {differ.nonzero().flatten().tolist()} differ" + + +def test_mkm_replays_from_a_cuda_graph(layer): + logits, x_fp4, x_sf = _inputs(7, 2) + eager = _fused(layer, logits, x_fp4, x_sf) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = _fused(layer, logits, x_fp4, x_sf) + graph.replay() + torch.cuda.synchronize() + assert all(torch.equal(a, b) for a, b in zip(eager, captured)) + + +# ---------------------------------------------------------------- the router front + + +def _router_inputs(m: int, seed: int): + g = torch.Generator(device="cuda").manual_seed(seed) + x = torch.randn(m, router.HIDDEN, device="cuda", generator=g).to(torch.bfloat16) + x[0, :32] = 0 # an all-zero quantization block + x[-1, 32:48] *= 1e4 # a block far above the others + w = ( + torch.randn(router.EXPERTS, router.HIDDEN, device="cuda", generator=g) * 0.02 + ).to(torch.bfloat16) + scale = (448.0 * 6.0 / x.float().abs().max()).reshape(1) + return x, w, scale + + +def _router_stock(x, w, scale): + from flashinfer import fp4_quantize + + logits = torch.nn.functional.linear(x, w) + fp4, sf = fp4_quantize(x, scale, 16, False, False) + return logits, fp4.view(torch.uint8).reshape(x.shape[0], -1), sf.view(torch.uint8) + + +@pytest.mark.parametrize("m", list(range(1, router.MAX_TOKENS + 1))) +def test_router_gemm_quant_is_the_stock_router_gemm_and_quantizer(m): + x, w, scale = _router_inputs(m, seed=m) + got = router.router_gemm_quant(x, w, scale) + want = _router_stock(x, w, scale) + for name, a, b in zip(("logits", "x_fp4", "x_sf"), got, want): + assert torch.equal(a.view(torch.uint8), b.reshape(a.shape).view(torch.uint8)), ( + name + ) + + +def test_router_gemm_quant_replays_from_a_cuda_graph(): + x, w, scale = _router_inputs(7, seed=0) + router.build(x.device.index, 7) + eager = router.router_gemm_quant(x, w, scale) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = router.router_gemm_quant(x, w, scale) + graph.replay() + torch.cuda.synchronize() + assert all(torch.equal(a, b) for a, b in zip(eager, captured)) diff --git a/test/unit/test_contract.py b/test/unit/test_contract.py index 762d10d1..50ac41a3 100644 --- a/test/unit/test_contract.py +++ b/test/unit/test_contract.py @@ -7,14 +7,21 @@ ``sglang.srt.megakernels``) and its parity test hooks one SGLang function by dotted path (``plugin.TEST_HOOKS``). Every module that imports from ``sglang`` lists what it reads in ``SGLANG_READS``. A refactor that renames one fails here, naming the file to -repair, instead of failing at a GPU server's startup. The checks are static -(AST): no module of this package is imported. Run it against each SGLang this -package supports, and nightly against SGLang main. +repair, instead of failing at a GPU server's startup. A module whose stock step mirrors +the body of an SGLang function lists it in ``SGLANG_MIRRORS`` with the digest of its +code (its tokens without comments or docstring; a tuple of digests where the supported +SGLangs differ), so a changed branch fails here too. +The checks are static (AST and tokens): no module of this package is imported. Run it +against each SGLang this package supports, and nightly against SGLang main. """ import ast +import hashlib +import io +import tokenize from functools import lru_cache from pathlib import Path +from typing import Optional import pytest import sglang @@ -196,3 +203,80 @@ def test_every_binding_callable_the_plugin_reads_is_tolerated(): untolerated = names - callables - data untolerated = {n for n in untolerated if not n.startswith("get_")} assert not untolerated, f"{_id(path)}: {sorted(untolerated)}" + + +def _mirrors(path: Path) -> dict: + """The module's ``SGLANG_MIRRORS``, or {} without one.""" + for node in _tree(path).body: + if isinstance(node, ast.Assign) and any( + isinstance(t, ast.Name) and t.id == "SGLANG_MIRRORS" for t in node.targets + ): + return ast.literal_eval(node.value) + return {} + + +def mirror_digest(path: Path, qualname: str) -> Optional[str]: + """The digest of a function's code in ``path`` (``Class.method`` or ``function``): + its decorators, signature and body as tokens, without comments, blank lines or + docstring; None when the file does not define it.""" + scope = _tree(path).body + *owners, name = qualname.split(".") + for owner in owners: + cls = next( + (n for n in scope if isinstance(n, ast.ClassDef) and n.name == owner), None + ) + if cls is None: + return None + scope = cls.body + node = next( + ( + n + for n in scope + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == name + ), + None, + ) + if node is None: + return None + first = min([node.lineno, *(d.lineno for d in node.decorator_list)]) + lines = path.read_text().splitlines(keepends=True)[first - 1 : node.end_lineno] + doc = node.body[0] + has_doc = ( + isinstance(doc, ast.Expr) + and isinstance(doc.value, ast.Constant) + and isinstance(doc.value.value, str) + ) + skip = (doc.lineno - first + 1, doc.end_lineno - first + 1) if has_doc else None + digest = hashlib.sha256() + for tok in tokenize.generate_tokens(io.StringIO("".join(lines)).readline): + if tok.type in (tokenize.COMMENT, tokenize.NL): + continue + if skip and skip[0] <= tok.start[0] and tok.end[0] <= skip[1]: + continue + # indentation depth is structure, not text + text = "" if tok.type in (tokenize.INDENT, tokenize.NEWLINE) else tok.string + digest.update(f"{tok.type}:{text}\n".encode()) + return digest.hexdigest()[:16] + + +MIRRORS = [ + (path, key, digest) + for path in sorted(PACKAGE.rglob("*.py")) + for key, digest in _mirrors(path).items() +] + + +@pytest.mark.parametrize( + "path, key, digest", MIRRORS, ids=[f"{_id(p)}::{k}" for p, k, _ in MIRRORS] +) +def test_the_mirrored_sglang_code_is_unchanged(path, key, digest): + pattern, qualname = key.split(":") + files = sorted(ROOT.glob(pattern)) + assert len(files) == 1, f"{_id(path)}: {pattern} matches {len(files)} files" + got = mirror_digest(files[0], qualname) + assert got is not None, f"{_id(path)}: SGLang no longer defines {key}" + assert got in ((digest,) if isinstance(digest, str) else digest), ( + f"SGLang changed {key}, whose body {_id(path)} mirrors: re-read it, update " + f"the mirror, then pin {got!r} in its SGLANG_MIRRORS (a tuple where the " + "supported SGLangs differ)" + ) diff --git a/test/unit/test_plugin.py b/test/unit/test_plugin.py index 16775f79..9e3ee906 100644 --- a/test/unit/test_plugin.py +++ b/test/unit/test_plugin.py @@ -26,9 +26,10 @@ def test_register_registers_every_spec_and_no_hook(hooks, monkeypatch): plugin.register() assert registry._REGISTRY == dict(plugin.SPECS) assert hooks == [] - # the registered path resolves to the spec SGLang checks - spec, why = registry.find_spec(architectures=[plugin.SPECS[0][0]], isa="sm_103") - assert spec is not None, why + # each registered path resolves to the spec SGLang checks + for architecture, _ in plugin.SPECS: + spec, why = registry.find_spec(architectures=[architecture], isa="sm_103") + assert spec is not None, why def test_the_parity_sort_hook_registers_only_for_its_test(hooks, monkeypatch): diff --git a/test/unit/test_qwen3_5_runtime.py b/test/unit/test_qwen3_5_runtime.py new file mode 100644 index 00000000..c30ad842 --- /dev/null +++ b/test/unit/test_qwen3_5_runtime.py @@ -0,0 +1,277 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The Qwen3.5 megakernel's runtime on CPU: the steps it covers, the binding and the +EAGLE entries at build, the MTP draft spec, the step state and the parity check.""" + +from types import SimpleNamespace + +import pytest +import torch +from sglang.srt.megakernels import spec as registry +from sglang.srt.megakernels.spec import Step + +from sgl_nvidia_mega import plugin +from sgl_nvidia_mega.qwen3_5 import ( + runtime, +) +from sgl_nvidia_mega.qwen3_5.runtime import executor, parity +from sgl_nvidia_mega.qwen3_5.runtime.state import StepState + + +@pytest.fixture(autouse=True) +def registered(monkeypatch): + monkeypatch.setattr(registry, "_REGISTRY", {}) + monkeypatch.delenv(plugin.PARITY_SORT_ENV, raising=False) + plugin.register() + + +# ---------------------------------------------------------------- the covered steps + + +def _verify(widths=(7,), binding="binding"): + """An executor over a model with no layers (plan reads only host state), built + with SGLang's EAGLE binding.""" + lm = SimpleNamespace(model=SimpleNamespace(start_layer=0, end_layer=0, layers=[])) + return executor.Qwen35Verify(lm, {}, {}, widths, binding=binding) + + +def _batch(verify=True): + return SimpleNamespace( + forward_mode=SimpleNamespace(is_target_verify=lambda: verify) + ) + + +def test_only_target_verify_steps_of_built_widths_are_covered(): + built = _verify() + ids = torch.zeros(7, dtype=torch.int64) + assert built.plan(ids, _batch()) == Step(mode="target_verify", rows=7) + assert built.plan(ids, _batch(), pp_proxy_tensors=None) == Step( + mode="target_verify", rows=7 + ) + assert built.plan(torch.zeros(14, dtype=torch.int64), _batch()) is None + assert built.plan(ids, _batch(verify=False)) is None + assert built.plan(ids, _batch(), get_embedding=True) is None + assert built.plan(ids, _batch(), pp_proxy_tensors=object()) is None + # an argument the executor does not know leaves the step to SGLang + assert built.plan(ids, _batch(), input_embeds=None) is None + + +def test_the_build_hands_over_the_binding_and_every_entry_stays_sglangs(): + from sglang.srt.megakernels.spec import ENTRIES + + from sgl_nvidia_mega.qwen3_5 import draft + + built = _verify() + assert built.binding == "binding" + # covered from the first step on: the binding comes with the build + ids = torch.zeros(7, dtype=torch.int64) + assert built.plan(ids, _batch()) == Step(mode="target_verify", rows=7) + target, mtp = built, draft.SPEC.build("lm", None, None) + assert [n for n in ENTRIES if getattr(target, n, None) is not None] == [] + assert [n for n in ENTRIES if getattr(mtp, n, None) is not None] == [] + + +def test_verify_sample_is_refused_with_the_switch_that_replaces_it(monkeypatch): + monkeypatch.setattr(runtime, "SLOTS", frozenset({"route", "verify_sample"})) + assert [r.split(": ", 1)[1] for r in runtime.slot_reasons()] == [ + "'verify_sample' is no longer an entry: drop it; an SGLang with post-verify " + "graphs turns them on with SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH=1" + ] + + +def test_the_runtime_gives_the_executor_the_build_contexts_binding(monkeypatch): + monkeypatch.setattr(runtime, "SLOTS", frozenset()) + model = SimpleNamespace( + model=SimpleNamespace(start_layer=0, end_layer=0, layers=[]) + ) + ctx = SimpleNamespace( + widths=(7,), tokens_per_request=7, deadline_s=60.0, binding="EagleBinding" + ) + built = runtime.build(model, ctx) + assert built.binding == "EagleBinding" and built.summary() == "slots none (stock)" + + +def test_the_mtp_draft_covers_no_step(): + from sgl_nvidia_mega.qwen3_5 import draft + + built = draft.SPEC.build("lm", None, None) + assert ( + draft.SPEC.max_rows == 0 and built.widths == () and built.draft_sampler is None + ) + assert built.plan(torch.zeros(7, dtype=torch.int64), _batch()) is None + assert (draft.ARCHITECTURE, "sgl_nvidia_mega.qwen3_5.draft:SPEC") in plugin.SPECS + + +# ------------------------------------------------------------------- the parity check + + +@pytest.fixture +def eager(monkeypatch): + """Outside CUDA graph capture (a CPU build of torch has no capture to query), with + an attention backend that has no linear-attention states.""" + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + monkeypatch.setattr( + "sglang.srt.model_executor.forward_context.get_attn_backend", + lambda: SimpleNamespace(), + ) + + +def test_the_second_forward_sees_the_conv_windows_the_first_saw(eager, monkeypatch): + conv = torch.zeros(2, 3, 4) # [layers, slots, ...] + cache = SimpleNamespace(conv=[conv]) + linear = SimpleNamespace( + req_to_token_pool=SimpleNamespace( + mamba_pool=SimpleNamespace(mamba_cache=cache) + ), + forward_metadata=SimpleNamespace(mamba_cache_indices=torch.tensor([1, -1])), + ) + monkeypatch.setattr( + "sglang.srt.model_executor.forward_context.get_attn_backend", + lambda: SimpleNamespace(linear_attn_backend=linear), + ) + + def forward(): + # reads slot 1, then rewrites it in place, as a verify forward's conv update does + out = conv[:, 1].sum().reshape(1, 1).clone() + conv[:, 1] += 1.0 + return SimpleNamespace(next_token_logits=out, hidden_states=None) + + got = parity.Parity((7,), "cpu").run(Step("target_verify", 7), forward, forward) + assert got.next_token_logits.item() == 0.0 # the restored state, not 8.0 + assert conv[:, 1].eq(1.0).all() and conv[:, 0].eq(0.0).all() + + +def test_parity_compares_bits_not_values(): + a = torch.tensor([0.0, 1.0, float("nan")]) + assert int(parity.differing(a, a.clone())) == 0 + assert int(parity.differing(a, torch.tensor([-0.0, 1.0, float("nan")]))) == 1 + assert ( + int(parity.differing(a, torch.tensor([0.0, 1.0 + 2**-20, float("nan")]))) == 1 + ) + assert parity.differing(None, None) == 0 + with pytest.raises(RuntimeError): + parity.differing(a, a[:2]) + with pytest.raises(RuntimeError): + parity.differing(a, None) + + +@pytest.mark.parametrize("mode", ["1", "control"]) +def test_an_eager_parity_step_runs_sglangs_forward_first( + eager, monkeypatch, caplog, mode +): + monkeypatch.setattr(parity, "MODE", mode) + order = [] + + def output(name, value): + def run(): + order.append(name) + return SimpleNamespace( + next_token_logits=torch.full((7, 3), value), hidden_states=None + ) + + return run + + checker = parity.Parity((7,), "cpu") + with caplog.at_level("INFO"): + got = checker.run( + Step("target_verify", 7), output("sglang", 1.0), output("plugin", 2.0) + ) + assert order == (["sglang", "plugin"] if mode == "1" else ["sglang", "sglang"]) + want = 0 if mode == "control" else 21 + assert ( + f"rows=7 logits_differ={want} hidden_differ=0 logits_compared=21 " + "hidden_compared=0" in caplog.text + ) + assert got.next_token_logits[0, 0] == (1.0 if mode == "control" else 2.0) + + +def test_a_capture_warmup_is_logged_apart_from_the_eager_steps( + eager, monkeypatch, caplog +): + monkeypatch.setattr( + "sglang.srt.model_executor.runner.get_is_capture_mode", lambda: True + ) + + def output(): + return SimpleNamespace(next_token_logits=torch.zeros(7, 3), hidden_states=None) + + with caplog.at_level("INFO"): + parity.Parity((7,), "cpu").run(Step("target_verify", 7), output, output) + assert "parity capture-warmup target_verify rows=7 logits_differ=0" in caplog.text + assert " eager " not in caplog.text + + +@torch.no_grad() +def test_trace_names_the_first_module_call_that_differs(eager, monkeypatch, caplog): + monkeypatch.setattr(parity, "MODE", "trace") + model = torch.nn.Sequential(torch.nn.Linear(4, 4), torch.nn.Linear(4, 4)) + x = torch.ones(7, 4) + + def forward(shift): + def run(): + h = model[0](x) + shift # an op between two modules + return SimpleNamespace(next_token_logits=model[1](h), hidden_states=None) + + return run + + checker = parity.Parity((7,), "cpu", model) + with caplog.at_level("INFO"): + checker.run(Step("target_verify", 7), forward(0.0), forward(0.0)) + checker.run(Step("target_verify", 7), forward(0.0), forward(1.0)) + traces = [r.message for r in caplog.records if "parity trace" in r.message] + assert traces == [ + "NVIDIA megakernel parity trace target_verify rows=7: no diverging call " + "among 2 common module calls", + "NVIDIA megakernel parity trace target_verify rows=7: 1#0 input: 28 of 28 " + "elements differ", + ] + + +def test_graph_parity_counts_are_logged_once_each(eager, caplog): + checker = parity.Parity((7,), "cpu") + checker.counters[7] += torch.tensor([3, 0, 5, 63, 0]) + checker.distance[7] += torch.tensor( + [21.0, 2.0, 0.42, 0.25, 0.5], dtype=torch.float64 + ) + with caplog.at_level("INFO"): + checker.flush() + checker.flush() + lines = [r.message for r in caplog.records if "parity graph" in r.message] + assert lines == [ + "NVIDIA megakernel parity graph target_verify rows=7 steps=3 " + "logits_differ=0 hidden_differ=5 logits_compared=63 hidden_compared=0 " + "rows_compared=21 top1_differ=2 dlp_mean=0.02 dlp_max=0.25 hidden_dist_max=0.5" + ] + # the distance restarts with every flush + assert checker.distance[7].tolist() == [0.0] * 5 + + +def test_the_parity_distance_reads_top1_flips_and_log_prob_gaps(): + a = torch.tensor([[2.0, 0.0, 0.0], [0.0, 3.0, 0.0]]) + b = torch.tensor([[2.0, 0.0, 0.0], [0.0, 1.0, 2.0]]) # row 1's top-1 moves + rows, top1, total, worst, hidden = parity.distance( + a, b, torch.zeros(2, 4), torch.full((2, 4), 0.25) + ).tolist() + gap = torch.log_softmax(a[1], -1)[1] - torch.log_softmax(b[1], -1)[1] + assert (rows, top1, hidden) == (2.0, 1.0, 0.25) + assert total == pytest.approx(gap.item()) and worst == pytest.approx(gap.item()) + assert parity.distance(None, None, None, None) is None + + +def test_the_step_state_carries_the_binding(): + lm = SimpleNamespace(model="model") + st = StepState( + lm=lm, + forward_batch="fb", + positions="pos", + input_ids="ids", + slots="slots", + bound={}, + binding="EagleBinding", + view="EagleForwardView", + ) + assert (st.model, st.binding, st.view) == ( + "model", + "EagleBinding", + "EagleForwardView", + ) diff --git a/test/unit/test_qwen3_5_spec.py b/test/unit/test_qwen3_5_spec.py new file mode 100644 index 00000000..138a520c --- /dev/null +++ b/test/unit/test_qwen3_5_spec.py @@ -0,0 +1,155 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The Qwen3.5 megakernel's spec on CPU: the setups SGLang's startup checks accept +and refuse with it, the slot requests it refuses before the load, and the slot plan.""" + +import sys +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +from sglang.srt.megakernels import guards +from sglang.srt.megakernels import spec as registry +from sglang.srt.model_executor.cuda_graph_config import ( + Backend, + Phase, + default_cuda_graph_config, + with_phase, +) +from sglang.srt.server_args import ServerArgs + +from sgl_nvidia_mega import plugin +from sgl_nvidia_mega.qwen3_5 import ( + TEXT_CONFIG, + plan, + runtime, + spec, +) + + +@pytest.fixture(autouse=True) +def registered(monkeypatch): + monkeypatch.setattr(registry, "_REGISTRY", {}) + monkeypatch.delenv(plugin.PARITY_SORT_ENV, raising=False) + plugin.register() + + +# The measured TP8 server (the speculative algorithm as resolved: NEXTN is EAGLE) +DOCUMENTED = dict( + model_path="dummy", + enable_nvidia_megakernel=True, + tp_size=8, + nnodes=2, + kv_cache_dtype="fp8_e4m3", + speculative_algorithm="EAGLE", + speculative_num_steps=6, + speculative_eagle_topk=1, + speculative_num_draft_tokens=7, + moe_runner_backend="flashinfer_trtllm", + flashinfer_allreduce_fusion_backend="cutedsl", + attention_backend="trtllm_mha", + linear_attn_decode_backend="triton", + enable_linear_replayssm_spec=True, + mamba_ssm_dtype="bfloat16", + enable_hierarchical_cache=True, + cuda_graph_config=default_cuda_graph_config(), +) + + +def _refusal(text=None, dtype="bfloat16", isa="sm_103", **args) -> str: + """SGLang's resolution step on these arguments: "" when it accepts them, else its + error message.""" + hf_config = SimpleNamespace( + architectures=[spec.ARCHITECTURE], + model_type="qwen3_5_moe", + text_config=SimpleNamespace(**{**TEXT_CONFIG, **(text or {})}), + ) + with ( + patch.object(guards, "plugin_reasons", return_value=[]), + patch.object(guards, "device_isa", return_value=isa), + patch.object(guards, "model_config_of") as model_config, + ): + model_config.return_value = SimpleNamespace(hf_config=hf_config, dtype=dtype) + try: + guards.handle_nvidia_megakernel(ServerArgs(**{**DOCUMENTED, **args})) + except ValueError as e: + return str(e) + return "" + + +def test_the_documented_setup_is_accepted(): + assert _refusal() == "" + + +BREAKABLE = with_phase( + default_cuda_graph_config(), Phase.DECODE, backend=Backend.BREAKABLE +) + + +@pytest.mark.parametrize( + "kw, want", + [ + (dict(tp_size=4), "tp_size 4"), + (dict(pp_size=2), "pipeline parallelism is in use"), + (dict(enable_dp_attention=True, dp_size=8), "DP attention is in use"), + (dict(kv_cache_dtype="auto"), "kv_cache_dtype 'auto'"), + (dict(speculative_algorithm=None), "speculative_algorithm None"), + (dict(speculative_eagle_topk=4), "speculative_eagle_topk 4"), + ( + dict(speculative_use_rejection_sampling=True), + "speculative_use_rejection_sampling True", + ), + (dict(moe_runner_backend="flashinfer_cutedsl"), "moe_runner_backend"), + (dict(flashinfer_allreduce_fusion_backend=None), "flashinfer_allreduce_fusion"), + (dict(attention_backend="triton"), "attention_backend 'triton'"), + (dict(linear_attn_decode_backend="flashinfer"), "linear_attn_decode_backend"), + (dict(enable_linear_replayssm_spec=False), "enable_linear_replayssm_spec"), + (dict(mamba_ssm_dtype="float32"), "mamba_ssm_dtype 'float32'"), + (dict(startup_weight_load_mode="overlap"), "startup_weight_load_mode"), + (dict(cuda_graph_config=BREAKABLE), "decode CUDA graphs"), + (dict(text=dict(num_experts=256)), "config.text_config.num_experts 256"), + (dict(text=dict(hidden_size=5120)), "config.text_config.hidden_size 5120"), + (dict(dtype="float16"), "dtype 'float16'"), + (dict(isa="sm_100"), "not sm_100"), + ], +) +def test_each_refusal_names_its_reason(kw, want): + assert want in _refusal(**kw) + + +def test_an_sglang_without_the_eagle_forward_view_is_refused(monkeypatch): + # an import of this name now raises ImportError + monkeypatch.setitem(sys.modules, "sglang.srt.megakernels.eagle_binding", None) + assert "this SGLang has no EAGLE forward view" in _refusal() + + +@pytest.mark.parametrize( + "slots, want", + [ + ({"hop2"}, "unknown slot 'hop2'"), + ({"verify_sample"}, "'verify_sample' is no longer an entry: drop it"), + ({"mkm"}, "slot 'mkm' runs on slot 'route''s outputs"), + ({"gdn_pdl"}, "slot 'gdn_pdl' launches slot 'gdn_verify''s verify"), + ], +) +def test_a_slot_set_that_cannot_run_is_refused_before_the_load( + monkeypatch, slots, want +): + monkeypatch.setattr(runtime, "SLOTS", frozenset(slots)) + assert want in _refusal() + + +def test_every_reason_is_listed_at_once(): + message = _refusal(tp_size=4, kv_cache_dtype="auto", pp_size=2) + assert message.count("\n - ") == 3, message + + +def test_the_slot_plan_covers_one_request_of_7_rows(): + assert plan.plan((7,), plan.Facts()) == {7: plan.Slots()} + assert plan.plan((7,), plan.Facts(route=True)) == {7: plan.Slots(route=True)} + assert plan.plan((7,), plan.Facts(route=True, mkm=True)) == { + 7: plan.Slots(route=True, mkm=True) + } + assert plan.plan((7,), plan.Facts(hop=True)) == {7: plan.Slots(hop=True)} + with pytest.raises(ValueError): + plan.slots(plan.MAX_ROWS + 1, plan.Facts()) diff --git a/test/unit/test_qwen3_5_steps.py b/test/unit/test_qwen3_5_steps.py new file mode 100644 index 00000000..7947fbd5 --- /dev/null +++ b/test/unit/test_qwen3_5_steps.py @@ -0,0 +1,1257 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The Qwen3.5 megakernel's step packages on CPU, against SGLang's own forward on +stubbed modules (the same ops in the same order, on the same streams, with the same +arguments): the decoder layers, the GatedDeltaNet, the attention, the MoE, the layer +loop, and the slots route, mkm, gdn_verify and hop.""" + +import contextlib +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch +from sglang.srt.megakernels import spec as registry +from sglang.srt.megakernels.spec import Step +from sglang.srt.models import qwen2_moe, qwen3_5 + +from sgl_nvidia_mega import plugin +from sgl_nvidia_mega.qwen3_5 import ( + attention, + gdn, + moe, + plan, + runtime, +) +from sgl_nvidia_mega.qwen3_5.boundary import hop as boundary_hop +from sgl_nvidia_mega.qwen3_5.runtime import executor + + +@pytest.fixture(autouse=True) +def registered(monkeypatch): + monkeypatch.setattr(registry, "_REGISTRY", {}) + monkeypatch.delenv(plugin.PARITY_SORT_ENV, raising=False) + plugin.register() + + +# ------------------------------------------------- the stock step against SGLang's + + +def _verify(widths=(7,), binding="binding"): + """An executor over a model with no layers (plan reads only host state), built + with SGLang's EAGLE binding.""" + lm = SimpleNamespace(model=SimpleNamespace(start_layer=0, end_layer=0, layers=[])) + return executor.Qwen35Verify(lm, {}, {}, widths, binding=binding) + + +class _Log(list): + def tag(self, value): + """A short stand-in for a value: a known tensor's name and the shape it is + viewed in, a known object's name, or the value itself.""" + if isinstance(value, torch.Tensor): + for name, known in self.known.items(): + if ( + isinstance(known, torch.Tensor) + and value.data_ptr() == known.data_ptr() + ): + return f"{name}{tuple(value.shape)}" + return f"?{tuple(value.shape)}" + for name, known in self.known.items(): + if value is known: + return name + return value if isinstance(value, (str, bool, int, type(None))) else "?" + + def op(self, name, out=None): + def run(*args, **kwargs): + kw = tuple(sorted((k, self.tag(v)) for k, v in kwargs.items())) + self.append((name, self.stream, tuple(map(self.tag, args)), kw)) + return out + + return run + + +def _log(**known): + log = _Log() + log.known, log.stream = dict(known), "main" + return log + + +class _Stream: + def __init__(self, name, log): + self.name, self.log = name, log + + def wait_stream(self, other): + self.log.append(("wait", self.name, other.name)) + + +@contextlib.contextmanager +def _streams(log, capture): + main = _Stream("main", log) + + @contextlib.contextmanager + def on(stream): + prev, log.stream = log.stream, stream.name + try: + yield + finally: + log.stream = prev + + with ( + patch.object(torch.cuda, "current_stream", return_value=main), + patch.object(torch.cuda, "stream", side_effect=on), + patch.object(qwen2_moe, "get_is_capture_mode", return_value=capture), + patch( + "sglang.srt.model_executor.runner.get_is_capture_mode", return_value=capture + ), + ): + yield + + +class _Callable: + def __init__(self, fn, **attrs): + self._fn = fn + self.__dict__.update(attrs) + + def __call__(self, *args, **kwargs): + return self._fn(*args, **kwargs) + + +def _moe_block(log, x, *, alt_stream=True, shared=True, deferred_ok=True): + """A Qwen2MoeSparseMoeBlock whose gate, top-k, experts and shared expert log.""" + mlp = qwen2_moe.Qwen2MoeSparseMoeBlock.__new__(qwen2_moe.Qwen2MoeSparseMoeBlock) + torch.nn.Module.__init__(mlp) + bypassed = SimpleNamespace(format="bypassed") + routed = torch.zeros(7, 4) + log.known["routed"] = routed + experts = _Callable( + log.op("experts", routed), + supports_deferred_finalize=deferred_ok, + forward_deferred_finalize=log.op("experts_deferred", "routed_d"), + ) + mlp.__dict__.update( + alt_stream=_Stream("alt", log) if alt_stream else None, + reduce_results=False, + enable_shared_expert_fusion=False, + _use_mega_moe=False, + layer_id=3, + gate=_Callable(log.op("gate", ("logits", None))), + topk=_Callable(log.op("topk", bypassed)), + experts=experts, + shared_expert=_Callable(log.op("shared", "shared_out")) if shared else None, + shared_expert_gate=_Callable( + log.op("shared_gate", "gate_out"), weight=torch.ones(1, 4) + ), + ) + return mlp + + +@contextlib.contextmanager +def _moe_kernels(log): + with ( + patch.object( + qwen2_moe, "fused_gate_sigmoid_mul", log.op("gate_mul", "gated_shared") + ), + patch.object(qwen2_moe, "fused_gate_sigmoid_mul_add", log.op("gate_mul_add")), + patch( + "sglang.kernels.ops.elementwise.elementwise.fused_gate_sigmoid_mul_add", + log.op("gate_mul_add"), + ), + patch.object(qwen2_moe, "is_npu", return_value=False), + # SGLang pages experts on one GPU only, never under the spec's TP8 + patch.object( + qwen2_moe, + "get_exec", + return_value=SimpleNamespace( + moe=SimpleNamespace(enable_paged_experts=False) + ), + ), + patch.object(qwen2_moe, "use_intel_amx_backend", return_value=False), + patch.object(qwen2_moe, "_use_aiter", False), + patch.object(qwen2_moe.TopKOutputChecker, "format_is_bypassed", lambda o: True), + patch.object( + qwen2_moe.TopKOutputChecker, "format_is_standard", lambda o: False + ), + patch( + "sglang.srt.layers.layer_boundary.fusions.cutedsl.MoeDeferredFinalize." + "from_flashinfer", + log.op("deferred_finalize", "handoff"), + ), + patch.object( + qwen2_moe, "get_moe_a2a_backend", return_value=_Callable(None, **_NO_A2A) + ), + ): + yield + + +_NO_A2A = { + name: (lambda: False) + for name in ("is_deepep", "is_deepep_v2", "is_mori", "is_ascend_fuseep") +} + + +@pytest.mark.parametrize("capture", [False, True]) +@pytest.mark.parametrize("defer", [False, True]) +@pytest.mark.parametrize("alt_stream", [False, True]) +def test_the_moe_step_issues_what_sglang_issues(capture, defer, alt_stream): + runs = [] + for which in ("sglang", "plugin"): + x = torch.zeros(7, 4) + log = _log(x=x) + mlp = _moe_block(log, x, alt_stream=alt_stream) + with _streams(log, capture), _moe_kernels(log): + if which == "sglang": + out = mlp.forward(x, None, defer_finalize=defer) + else: + out = moe.step(SimpleNamespace(mlp=mlp), x, _st(), defer=defer) + runs.append((log, log.tag(out) if not isinstance(out, torch.Tensor) else out)) + (want_log, want), (got_log, got) = runs + assert got_log == want_log + assert want == got if not isinstance(want, torch.Tensor) else torch.equal(want, got) + + +def _st(route=False): + return SimpleNamespace(slots=plan.Slots(route=route)) + + +def _router_front(log, mlp): + """The block's experts as the route slot reads them (the dispatcher, the runner + core, the input scale), and its router front's outputs.""" + from sgl_nvidia_mega.qwen3_5.sm103.moe import router + + class _Dispatched: + def _replace(self, **kwargs): + return log.op("replace", "dispatched_fp4")(**kwargs) + + log.known.update( + gate_w=torch.zeros(router.EXPERTS, router.HIDDEN, dtype=torch.bfloat16), + scale=torch.ones(()), + logits=torch.zeros(7, router.EXPERTS, dtype=torch.bfloat16), + x_fp4=torch.zeros(7, router.HIDDEN // 2, dtype=torch.uint8), + x_sf=torch.zeros(7, router.HIDDEN // 16, dtype=torch.uint8), + ) + k = log.known + mlp.gate.weight = k["gate_w"] + mlp.experts.__dict__.update( + w13_input_scale_quant=k["scale"], + dispatcher=SimpleNamespace( + dispatch=log.op("dispatch", _Dispatched()), + combine=log.op("combine", "routed_d"), + ), + run_moe_core=log.op("run_moe_core", "combine_input"), + ) + + @contextlib.contextmanager + def deferred(): + log.append(("deferred_context", log.stream)) + yield + + def front(x, w, scale): + log.op("router_front")(x, w, scale) + assert scale.shape == (1,) and scale.data_ptr() == k["scale"].data_ptr() + return k["logits"], k["x_fp4"], k["x_sf"] + + return ( + contextlib.ExitStack(), + patch.object(router, "router_gemm_quant", front), + patch( + "sglang.srt.layers.moe.moe_runner.flashinfer_trtllm." + "flashinfer_trtllm_deferred_finalize_context", + deferred, + ), + ) + + +@pytest.mark.parametrize("capture", [False, True]) +def test_the_route_slot_replaces_only_the_gate_and_the_routed_experts(capture): + """Slot route: the router front in place of the gate, then FusedMoE's + deferred-finalize path on its quantized input; every other op is the stock one.""" + runs = [] + for route in (False, True): + x = torch.zeros(7, 4) + log = _log(x=x) + mlp = _moe_block(log, x) + _, front, context = _router_front(log, mlp) + with _streams(log, capture), _moe_kernels(log), front, context: + out = moe.step(SimpleNamespace(mlp=mlp), x, _st(route), defer=True) + runs.append((log, log.tag(out))) + (stock, stock_out), (got, got_out) = runs + start = [e[0] for e in stock].index("gate") + assert [e[0] for e in stock[start : start + 3]] == [ + "gate", + "topk", + "experts_deferred", + ] + on_front = [ + ("router_front", "main", ("x(7, 4)", "gate_w(512, 4096)", "scale(1,)"), ()), + ("topk", "main", ("x(7, 4)", "logits(7, 512)"), ()), + ("dispatch", "main", (), (("hidden_states", "x(7, 4)"), ("topk_output", "?"))), + ( + "replace", + "main", + (), + ( + ("hidden_states", "x_fp4(7, 2048)"), + ("hidden_states_scale", "x_sf(7, 256)"), + ), + ), + ("deferred_context", "main"), + ("run_moe_core", "main", (), (("dispatch_output", "dispatched_fp4"),)), + ("combine", "main", (), (("combine_input", "combine_input"),)), + ] + assert got == stock[:start] + on_front + stock[start + 3 :] + assert got_out == stock_out == "handoff" + + +@pytest.mark.parametrize("capture", [False, True]) +def test_the_mkm_slot_runs_the_routed_experts_on_the_router_fronts_outputs(capture): + """Slot mkm: the router front, then one fused launch whose rows and weights reach + the deferred finalize in expert order; every other op is the stock one.""" + from sgl_nvidia_mega.qwen3_5.sm103.moe import mkm as kernel + + runs = [] + for slots in (plan.Slots(), plan.Slots(route=True, mkm=True)): + x = torch.zeros(7, 4) + log = _log(x=x) + mlp = _moe_block(log, x) + _, front, context = _router_front(log, mlp) + k = log.known + k.update( + w13_sf=torch.zeros(4, dtype=torch.float8_e4m3fn), + w2_sf=torch.zeros(8, dtype=torch.float8_e4m3fn), + g1=torch.ones(512), + c=torch.ones(512), + g2=torch.ones(512), + rows=torch.zeros(70, 4, dtype=torch.bfloat16), + weights=torch.zeros(7, 10, dtype=torch.bfloat16), + ) + mlp.experts.__dict__.update( + w13_weight_scale=k["w13_sf"], + w2_weight_scale=k["w2_sf"], + g1_alphas=k["g1"], + g1_scale_c=k["c"], + g2_alphas=k["g2"], + ) + combined = [] + + def combine(combine_input, log=log, combined=combined): + log.append(("combine", log.stream)) + combined.append(combine_input) + return "routed_d" + + mlp.experts.dispatcher.combine = combine + bound = SimpleNamespace( + mkm={3: "maps"}, identity=torch.arange(70, dtype=torch.int32) + ) + st = SimpleNamespace(slots=slots, bound={"moe": bound}) + fused = patch.object( + kernel, "mkm_moe", log.op("mkm", (k["rows"], k["weights"])) + ) + with _streams(log, capture), _moe_kernels(log), front, context, fused: + out = moe.step(SimpleNamespace(mlp=mlp), x, st, defer=True) + runs.append((log, log.tag(out), combined)) + (stock, stock_out, _), (got, got_out, combined) = runs + start = [e[0] for e in stock].index("gate") + assert [e[0] for e in stock[start : start + 3]] == [ + "gate", + "topk", + "experts_deferred", + ] + on_mkm = [ + ("router_front", "main", ("x(7, 4)", "gate_w(512, 4096)", "scale(1,)"), ()), + ( + "mkm", + "main", + ( + "maps", + "logits(7, 512)", + "x_fp4(7, 2048)", + "x_sf(7, 256)", + "w13_sf(4,)", + "w2_sf(8,)", + "g1(512,)", + "c(512,)", + "g2(512,)", + ), + (), + ), + ("combine", "main"), + ] + assert got == stock[:start] + on_mkm + stock[start + 3 :] + assert got_out == stock_out == "handoff" + (deferred,) = (c.hidden_states for c in combined) + assert deferred.gemm2_out is log.known["rows"] + assert deferred.expert_weights is log.known["weights"] + assert torch.equal( + deferred.expanded_idx_to_permuted_idx, torch.arange(70, dtype=torch.int32) + ) + assert deferred.top_k == 10 + + +def test_the_mkm_slot_names_what_a_layer_lacks(): + from sglang.srt.layers.moe.utils import RoutingMethodType + + from sgl_nvidia_mega.qwen3_5.moe import mkm + + def meta(dtype, *shape): + return torch.empty(shape, dtype=dtype, device="meta") + + fp8 = torch.float8_e4m3fn + experts = SimpleNamespace( + w13_weight=meta(torch.uint8, 512, 256, 2048), + w2_weight=meta(torch.uint8, 512, 4096, 64), + w13_weight_scale=meta(fp8, 512, 256, 256), + w2_weight_scale=meta(fp8, 512, 4096, 8), + g1_alphas=meta(torch.float32, 512), + g1_scale_c=meta(torch.float32, 512), + g2_alphas=meta(torch.float32, 512), + num_experts=512, + num_local_experts=512, + moe_ep_rank=0, + top_k=10, + routing_method_type=RoutingMethodType.RenormalizeNaive, + moe_runner_config=SimpleNamespace( + activation="silu", + is_gated=True, + routed_scaling_factor=None, + gemm1_alpha=None, + gemm1_clamp_limit=None, + swiglu_limit=None, + ), + ) + cfg = SimpleNamespace( + top_k=10, + renormalize=True, + scoring_func="softmax", + correction_bias=None, + use_grouped_topk=False, + num_fused_shared_experts=0, + ) + mlp = SimpleNamespace(experts=experts, topk=SimpleNamespace(topk_config=cfg)) + assert mkm.unsupported(mlp) == "" + experts.w13_weight = meta(torch.uint8, 512, 512, 2048) # TP4's intermediate + assert mkm.unsupported(mlp) == "w13_weight torch.uint8 (512, 512, 2048)" + experts.w13_weight = meta(torch.uint8, 512, 256, 2048) + cfg.scoring_func = "sigmoid" + assert mkm.unsupported(mlp).startswith("routing other than") + cfg.scoring_func = "softmax" + experts.moe_runner_config.swiglu_limit = 7.0 + assert mkm.unsupported(mlp).startswith("an expert activation") + + +def test_the_route_slot_is_not_built_for_a_layer_it_cannot_serve(monkeypatch): + from sgl_nvidia_mega.qwen3_5.moe import route + + mlp = _moe_block(_log(), torch.zeros(7, 4)) + mlp.gate.weight = torch.zeros(512, 4096, dtype=torch.float16) + layers = [SimpleNamespace(mlp=mlp)] + model = SimpleNamespace( + model=SimpleNamespace(start_layer=0, end_layer=1, layers=layers) + ) + monkeypatch.setattr(runtime, "SLOTS", frozenset({"route"})) + ctx = SimpleNamespace(widths=(7,), tokens_per_request=7, deadline_s=60.0) + want = "layer 0: router gate torch.float16 (512, 4096)" + assert route.build(model, 7, fused=False).reason == want + with pytest.raises( + RuntimeError, match="requested slots \\['route'\\] built no width" + ): + runtime.build(model, ctx) + + +def _gdn_module(log, x): + """A Qwen3_5GatedDeltaNet whose projections, backend call, norm and out_proj log.""" + mod = qwen3_5.Qwen3_5GatedDeltaNet.__new__(qwen3_5.Qwen3_5GatedDeltaNet) + torch.nn.Module.__init__(mod) + qkvz, ba = torch.zeros(7, 10), torch.zeros(7, 2) + z = torch.zeros(7, 2, 3) + core = torch.zeros(7, 2, 3) + log.known.update(qkvz=qkvz, ba=ba, z=z, core=core) + mod.__dict__.update( + num_k_heads=16, + num_v_heads=64, + attn_tp_size=8, + head_k_dim=128, + head_v_dim=128, + _forward_input_proj=log.op("in_proj", (qkvz, ba)), + attn=log.op("linear_attention", core), + norm=log.op("gated_norm", torch.ones(14, 3)), + out_proj=log.op("out_proj", ("out", None)), + ) + split = log.op("split", ("mixed", z, "b", "a")) + return mod, split + + +@pytest.mark.parametrize("packed_verify", [False, True]) +def test_the_gdn_step_issues_what_sglang_issues(packed_verify): + runs = [] + for which in ("sglang", "plugin"): + x = torch.zeros(7, 4) + log = _log(x=x) + mod, split = _gdn_module(log, x) + backend = SimpleNamespace( + linear_attn_backend=SimpleNamespace(supports_packed_verify=packed_verify) + ) + mode = SimpleNamespace( + is_target_verify=lambda: True, + is_decode=lambda: False, + is_extend_without_speculative=lambda: False, + ) + # a verify batch is never relabeled: its logical mode is its label + batch = SimpleNamespace(forward_mode=mode, logical_forward_mode=mode) + with ( + patch.object(qwen3_5, "_is_xpu", False), + patch.object(qwen3_5, "_is_cpu", False), + patch.object(qwen3_5, "_is_cuda", True), + patch.object(qwen3_5, "_gdn_decode_fused_proj_conv", True), + patch.object(qwen3_5, "fused_qkvzba_split_reshape_cat_contiguous", split), + patch( + "sglang.kernels.ops.attention.triton_gdn_fused_proj." + "fused_qkvzba_split_reshape_cat_contiguous", + split, + ), + patch.object(qwen3_5, "_fp8_tuple_input", return_value=False), + patch( + "sglang.srt.model_executor.forward_context.get_attn_backend", + return_value=backend, + ), + ): + try: + if which == "sglang": + out = mod.forward(x, batch) + else: + st = SimpleNamespace(fb=batch, slots=plan.Slots()) + out = gdn.step(SimpleNamespace(linear_attn=mod), x, st) + except (NotImplementedError, RuntimeError, AttributeError) as e: + out = type(e).__name__ + runs.append((log, out)) + (want_log, want), (got_log, got) = runs + if packed_verify: + # SGLang folds the split and conv into such a backend; the step refuses it + assert got == "NotImplementedError" and want != got + return + assert got_log == want_log and want == got == "out" + + +def _gdn_verify_setup(log, *, ring_len=16, temporal_dtype=torch.bfloat16, draft=7): + """A GatedDeltaNet whose projections, norm and out_proj log, and the binding of a + one-request verify step (layer 2 is the pools' third GDN layer).""" + from sgl_nvidia_mega.qwen3_5.gdn import verify + + H, K, HV, V = 2, 4, 4, 4 + D = 2 * H * K + HV * V + k = log.known + k.update( + qkvz=torch.zeros(draft, D + HV * V), + ba=torch.zeros(draft, 2 * HV), + scale=torch.ones(1), + states=torch.tensor([5]), + inter=torch.tensor([0]), + qsl=torch.tensor([0, draft]), + req=torch.tensor([3]), + conv=torch.zeros(4, 9, D, 3), + window=torch.zeros(4, 9, draft, D, 3), + temporal=torch.zeros(4, 9, HV, K, V, dtype=temporal_dtype), + ring_d=torch.zeros(4, 9, HV, ring_len, V), + ring_k=torch.zeros(4, 9, H, ring_len, K), + ring_g=torch.zeros(4, 9, HV, ring_len), + write_pos=torch.zeros(9, dtype=torch.int32), + cache_base=torch.zeros(9, dtype=torch.int32), + is_flush=torch.zeros(9, dtype=torch.int32), + ) + # layer 2's slices: the pools' third GDN layer + k.update({f"{n}2": k[n][2] for n in ("conv", "window", "temporal", "ring_d")}) + attn = SimpleNamespace( + layer_id=2, + conv_weights=torch.zeros(D, 4), + bias=None, + activation="silu", + q_dim=H * K, + k_dim=H * K, + v_dim=HV * V, + num_q_heads=H, + num_k_heads=H, + num_v_heads=HV, + head_q_dim=K, + head_k_dim=K, + head_v_dim=V, + A_log=torch.zeros(HV), + dt_bias=torch.zeros(HV), + ) + norm = SimpleNamespace(weight=torch.ones(V), eps=1e-6, activation="silu") + k.update(out_w=torch.zeros(HV * V, 8), out_s=torch.ones(1)) + out_proj = SimpleNamespace( + weight=k["out_w"], weight_scale=k["out_s"], input_scale=k["scale"] + ) + gdn = SimpleNamespace( + attn=attn, + norm=norm, + out_proj=out_proj, + num_k_heads=H, + num_v_heads=HV, + attn_tp_size=1, + head_k_dim=K, + head_v_dim=V, + _forward_input_proj=lambda x: (k["qkvz"], k["ba"]), + ) + meta = SimpleNamespace( + state_indices=k["states"], + intermediate_indices=k["inter"], + query_start_loc=k["qsl"], + ) + pools = SimpleNamespace( + layer_ids=(0, 1, 2, 4), + conv=(k["conv"],), + intermediate_conv_window=(k["window"],), + temporal=k["temporal"], + ring_d=k["ring_d"], + ring_k=k["ring_k"], + ring_g=k["ring_g"], + ring_rawv=None, + ring_rawk=None, + ring_beta=None, + ring_write_pos=k["write_pos"], + ring_cache_base=k["cache_base"], + ring_is_flush=k["is_flush"], + ) + st = SimpleNamespace( + fb=SimpleNamespace(req_pool_indices=k["req"]), + binding=SimpleNamespace(tokens_per_request=draft), + view=SimpleNamespace(gdn=meta, mamba=pools), + scratch={}, + ) + return verify, gdn, st + + +def test_the_gdn_verify_slot_runs_the_backend_chain_on_the_forward_views_rows(): + from sgl_nvidia_mega.qwen3_5.gdn import conv, norm_fp8, replayssm + + log = _log() + verify, gdn, st = _gdn_verify_setup(log) + k = log.known + + def views(qkvz, ba, nk, nv, hk, hv): + log.append(("views", log.tag(qkvz), log.tag(ba), nk, nv, hk, hv)) + return qkvz[:, :32], qkvz[:, 32:].view(7, nv, hv), ba[:, :4], ba[:, 4:] + + def conv_update(x, state, weight, bias, act, states, window, inter, **kw): + log.append( + ( + "conv", + tuple(x.shape), + log.tag(state), + log.tag(states), + log.tag(window), + log.tag(inter), + tuple(sorted(kw.items())), + ) + ) + return torch.zeros(1, 7, 32).transpose(1, 2) + + names = ( + "q", + "k", + "v", + "a", + "b", + "A_log", + "dt_bias", + "out", + "checkpoint_state", + "d_cache", + "k_cache", + "g_cache", + "rawv_cache", + "rawk_cache", + "beta_cache", + "query_start_loc", + "ssm_state_indices", + "replay_indices", + "write_pos", + "cache_base", + "is_flush", + ) + + def spec_decode(*args, **kw): + tags = {n: log.tag(v) for n, v in zip(names, args)} + log.append(("verify", tags, kw)) + + def norm(x, weight, eps, z, scale, **kw): + log.append( + ( + "norm_fp8", + tuple(x.shape), + tuple(z.shape), + log.tag(scale), + tuple(sorted(kw.items())), + ) + ) + return torch.zeros(28, 4) + + @contextlib.contextmanager + def symmetric(group, disabled=False): + log.append(("symmetric",)) + yield + + with ( + patch( + "sglang.kernels.ops.attention.triton_gdn_fused_proj.qwen3_5_gdn_prefill_projection_views", + views, + ), + patch.object(conv, "conv_update", conv_update), + patch.object(replayssm, "verify", spec_decode), + patch.object(norm_fp8, "rms_norm_gated_fp8", norm), + patch( + "sglang.srt.distributed.device_communicators.pynccl_allocator.use_symmetric_memory", + symmetric, + ), + patch("sglang.srt.layers.dp_attention.is_allocation_symmetric", lambda: True), + patch( + "sglang.srt.runtime_context.get_parallel", + lambda: SimpleNamespace(tp_group="tp"), + ), + patch( + "sglang.srt.layers.quantization.fp8_utils.flashinfer_bmm_fp8", + log.op("bmm_fp8"), + create=True, + ), + ): + for _ in range(2): # two layers of one step + verify.step(gdn, torch.zeros(7, 8), st) + one_layer = ["conv", "verify", "norm_fp8", "symmetric", "bmm_fp8", "views"] + assert [e[0] for e in log] == ["views", *one_layer, *one_layer[:-1]] + _, shape, state, states, window, inter, kw = log[1] + assert (shape, states, inter) == ((1, 32, 7), "states(1,)", "inter(1,)") + assert state == "conv2(9, 32, 3)" and window == "window2(9, 7, 32, 3)" + assert dict(kw) == {"block_n": 128, "token_major_out": True} + tags, options = log[2][1], log[2][2] + assert ( + tags["ssm_state_indices"] == "states(1,)" + and tags["replay_indices"] == "req(1,)" + ) + assert tags["query_start_loc"] == "qsl(2,)" and tags["write_pos"] == "write_pos(9,)" + assert ( + tags["rawv_cache"] is None + and tags["checkpoint_state"] == "temporal2(9, 4, 4, 4)" + ) + assert tags["d_cache"] == "ring_d2(9, 4, 16, 4)" + assert options == dict( + max_cache_len=16, + max_spec_len=7, + scale=0.5, + null_block_id=-1, + block_v=32, + num_warps=2, + pdl=False, + ) + assert log[3] == ( + "norm_fp8", + (28, 4), + (7, 4, 4), + "scale(1,)", + (("activation", "silu"),), + ) + # out_proj's BMM on the FP8 rows with its static input scale, output in the activation dtype + assert log[5] == ( + "bmm_fp8", + "main", + ("?(7, 16)", "out_w(16, 8)", "scale(1,)", "out_s(1,)", "?"), + (), + ) + + +@pytest.mark.parametrize( + "kw, want", + [ + (dict(ring_len=8), "ring length 8"), + (dict(temporal_dtype=torch.float32), "torch.float32 checkpoints"), + (dict(draft=9), "9 rows a request"), + ], +) +def test_the_gdn_verify_slot_names_the_pools_it_cannot_serve(kw, want): + verify, _, st = _gdn_verify_setup(_log(), **kw) + assert verify.pool_reasons(st.view.mamba, st.binding.tokens_per_request) == [want] + # the first covered step refuses, before it runs anything + built = _verify(binding=st.binding) + built.slots = {7: plan.Slots(gdn_verify=True)} + step = Step(mode="target_verify", rows=7, view=st.view) + with pytest.raises(RuntimeError, match=f"slot gdn_verify cannot run: {want}"): + built.forward(step, None, torch.zeros(7, dtype=torch.int64), None, None) + + +def test_the_pools_are_checked_once_at_the_first_covered_step(monkeypatch): + verify, _, st = _gdn_verify_setup(_log()) + built = _verify(binding=st.binding) + built.slots = {7: plan.Slots(gdn_verify=True)} + checks = [] + real = verify.pool_reasons + monkeypatch.setattr(verify, "pool_reasons", lambda *a: checks.append(a) or real(*a)) + monkeypatch.setattr(built, "_forward", lambda *a: "logits") + step = Step(mode="target_verify", rows=7, view=st.view) + for _ in range(2): + assert built.forward(step, None, "ids", "pos", "fb") == "logits" + assert checks == [(st.view.mamba, 7)] + + +def _attention_layer(log, x): + layer = qwen3_5.Qwen3_5AttentionDecoderLayer.__new__( + qwen3_5.Qwen3_5AttentionDecoderLayer + ) + torch.nn.Module.__init__(layer) + q, k, v, gate, o = (torch.zeros(7, 4) for _ in range(5)) + log.known.update(q=q, k=k, v=v, gate=gate, o=o) + layer.__dict__.update( + attn_output_gate=True, + forward_prepare_cuda_fused=log.op("prepare", (q, k, v, gate)), + attn=log.op("attention", o), + o_proj=log.op("o_proj", ("out", None)), + ) + return layer + + +def test_the_attention_step_issues_what_sglang_issues(): + runs = [] + for which in ("sglang", "plugin"): + x = torch.zeros(7, 4) + log = _log(x=x, positions="pos", batch="fb") + layer = _attention_layer(log, x) + mul = log.op("sigmoid_mul", "gated") + log.known["gated"] = "gated" + with ( + patch.object(qwen3_5, "_is_cuda", True), + patch.object(qwen3_5, "_is_npu", False), + patch.object(qwen3_5, "_fp8_tuple_input", return_value=False), + patch.object(qwen3_5, "fused_sigmoid_mul", mul), + patch("sglang.kernels.ops.elementwise.elementwise.fused_sigmoid_mul", mul), + ): + if which == "sglang": + out = layer.self_attention( + positions="pos", hidden_states=x, forward_batch="fb" + ) + else: + st = SimpleNamespace(positions="pos", fb="fb") + out = attention.step(layer, x, st) + runs.append((log, out)) + (want_log, want), (got_log, got) = runs + assert got_log == want_log and want == got == "out" + + +class _Exit: + def __init__(self, log, defer): + self.log, self.defer_moe_finalize = log, defer + + def __enter__(self): + self.log.append(("ffn_exit.enter",)) + return self + + def __exit__(self, *exc): + self.log.append(("ffn_exit.exit",)) + + def finish(self, out): + self.log.append(("ffn_exit.finish", self.log.tag(out))) + return "layer_out" + + +class _MoeBlock(qwen2_moe.Qwen2MoeSparseMoeBlock): + def __init__(self, log): + torch.nn.Module.__init__(self) + self.__dict__["log"] = log + + def forward(self, x, forward_batch=None, defer_finalize=False): + self.log.append(("moe", self.log.tag(x), defer_finalize)) + return "moe_out" + + +def _decoder_layer(kind, log, defer): + layer = kind.__new__(kind) + torch.nn.Module.__init__(layer) + a_in, m_in = torch.zeros(7, 4), torch.zeros(7, 4) + log.known.update(a_in=a_in, m_in=m_in) + layer.mlp = _MoeBlock(log) + layer.__dict__.update( + attn_boundary=SimpleNamespace( + prepare=log.op("attn.prepare", a_in), finish=log.op("attn.finish", "h1") + ), + ffn_boundary=SimpleNamespace( + prepare=log.op("ffn.prepare", m_in), exit=lambda fb: _Exit(log, defer) + ), + linear_attn=lambda x, fb: log.append(("mixer", log.tag(x))) or "part", + self_attention=lambda positions, hidden_states, forward_batch: ( + log.append(("mixer", log.tag(hidden_states))) or "part" + ), + ) + return layer + + +@pytest.mark.parametrize( + "kind", [qwen3_5.Qwen3_5LinearDecoderLayer, qwen3_5.Qwen3_5AttentionDecoderLayer] +) +@pytest.mark.parametrize("defer", [False, True]) +def test_a_layer_runs_what_sglangs_decoder_layer_runs(kind, defer): + batch = SimpleNamespace(forward_mode=SimpleNamespace(is_idle=lambda: False)) + runs = [] + for which in ("sglang", "plugin"): + log = _log(hidden="hidden", batch=batch, positions="pos", h1="h1", part="part") + layer = _decoder_layer(kind, log, defer) + if which == "sglang": + out = layer.forward( + positions="pos", + hidden_states="hidden", + forward_batch=batch, + captured_last_layer_outputs=None, + ) + else: + st = SimpleNamespace(fb=batch, positions="pos", slots=plan.Slots()) + built = _verify() + built.mixers = { + 0: lambda layer, x, st: log.append(("mixer", log.tag(x))) or "part" + } + with patch.object( + executor.moe, + "step", + lambda layer, x, st, defer: ( + log.append(("moe", log.tag(x), defer)) or "moe_out" + ), + ): + out = built._layer(0, layer, "hidden", st, None) + runs.append((log, out)) + (want_log, want), (got_log, got) = runs + assert got_log == want_log and want == got == "layer_out" + + +def test_the_layer_loop_opens_the_stream_scopes_each_layer_and_closes_it(monkeypatch): + log = [] + layers = [ + kind.__new__(kind) + for kind in ( + qwen3_5.Qwen3_5LinearDecoderLayer, + qwen3_5.Qwen3_5AttentionDecoderLayer, + ) + ] + for layer in layers: + torch.nn.Module.__init__(layer) + model = SimpleNamespace(start_layer=0, end_layer=2, layers=layers) + lm = SimpleNamespace( + model=model, is_mrope_enabled=True, capture_aux_hidden_states=False + ) + built = executor.Qwen35Verify(lm, {7: plan.Slots()}, {}, (7,)) + assert built.mixers == {0: gdn.step, 1: attention.step} + built.mixers = {i: (lambda i: lambda *a: f"part{i}")(i) for i in range(2)} + + @contextlib.contextmanager + def scope(i): + log.append(("layer", i)) + yield + + recorder = SimpleNamespace(with_current_layer=scope) + monkeypatch.setattr( + "sglang.srt.eplb.expert_distribution.get_global_expert_distribution_recorder", + lambda: recorder, + ) + monkeypatch.setattr(executor.embedding, "embed", lambda st: "embeds") + monkeypatch.setattr( + executor.boundary, "open_stream", lambda st: log.append(("open",)) + ) + monkeypatch.setattr( + executor.boundary, + "close_stream", + lambda st, h: log.append(("close", h)) or "final", + ) + monkeypatch.setattr( + executor.Qwen35Verify, + "_layer", + lambda self, i, layer, h, st, capture: ( + log.append(("run", i, h, capture)) or f"h{i}" + ), + ) + monkeypatch.setattr( + executor.logits, "logits", lambda st, h, aux: (st.positions, st.slots, h, aux) + ) + batch = SimpleNamespace(mrope_positions="mrope") + out = built.forward(Step("target_verify", 7), None, "ids", "pos", batch) + assert out == ("mrope", plan.Slots(), "final", None) + assert log == [ + ("open",), + ("layer", 0), + ("run", 0, "embeds", None), + ("layer", 1), + ("run", 1, "h0", None), + ("close", "h1"), + ] + + +# ------------------------------------------------------------------------- slot hop + + +class _Fused: + """SGLang's CuTe DSL workspace, or the hop: the same outputs from the same operands + (fresh tensors), and a log of the operands.""" + + def __init__(self, log, rows=7): + self.log, self.rows = log, rows + + def supports(self, m): + return m <= self.rows + + def finalize(self, handoff, residual, gamma): + return self.moe_finalize_all_reduce_rms_norm( + routed_output=handoff.routed_output, + expert_weights=handoff.expert_weights, + permuted_indices=handoff.permuted_indices, + gated_shared_output=handoff.gated_shared_output, + residual=residual, + gamma=gamma, + ) + + def moe_finalize_all_reduce_rms_norm(self, *, residual, gamma, **rows): + self.log.append(("finalize", *rows.values(), residual, gamma)) + return residual + gamma, residual * 2 + + def all_reduce_residual_rms_norm(self, *, local_contribution, residual, gamma): + self.log.append(("all_reduce", local_contribution, residual, gamma)) + return residual + gamma, residual * 2 + + +@pytest.fixture +def cutedsl_gates(monkeypatch): + """SGLang's CuTe DSL eligibility as at TP8 for a verify step: the workspace's rows + (``gates["open"] = False`` closes it).""" + from sglang.srt.layers.layer_boundary.fusions import cutedsl + from sglang.test.boundary_fixtures import parallel_of + + gates = {"open": True} + monkeypatch.setattr( + cutedsl.CuteDSLFusion, + "_eligible", + lambda self, plan, fb, m: gates["open"] and self.service.supports(m), + ) + monkeypatch.setattr( + cutedsl, "get_parallel", lambda: parallel_of(attn_dp=1, attn_tp=8) + ) + comm = SimpleNamespace(comm=SimpleNamespace(enable_quant_communications=False)) + monkeypatch.setattr(cutedsl, "get_exec", lambda: comm) + monkeypatch.setattr("sglang.srt.runtime_context.get_exec", lambda: comm) + return gates + + +def _hop_layer(service, fusions=True): + """A Qwen3.5 MoE layer's two stages as SGLang binds them at TP8, with the CuTe DSL + fusion on ``service`` unless not ``fusions``.""" + from sglang.srt.layers.layer_boundary import declare_attn, declare_ffn + from sglang.srt.layers.layer_boundary.fusions import cutedsl + from sglang.srt.layers.layer_boundary.residual.add_norm import ( + Fp8Input, + NormQuantReadout, + ) + from sglang.srt.layers.layernorm import GemmaRMSNorm + from sglang.test.boundary_fixtures import build_stages, parallel_of, planning + + fusion = cutedsl.CuteDSLFusion() if fusions else None + if fusions: + fusion.install(service, defers_finalize=True, terminal_finalize=True) + moe_ffn = dict(sparse=True, next_layer_sparse=True) + parallel = parallel_of(attn_dp=1, attn_tp=8) + with planning(parallel), patch.object(cutedsl, "get_parallel", lambda: parallel): + attn, ffn = build_stages( + ( + declare_attn(read=NormQuantReadout(fp8_input=Fp8Input.TUPLE_AND_BF16)), + GemmaRMSNorm(16), + {"fusions": fusion}, + ), + (declare_ffn(**moe_ffn), GemmaRMSNorm(16), {"fusions": fusion}), + previous=declare_ffn(**moe_ffn), + ) + return SimpleNamespace(attn_boundary=attn, ffn_boundary=ffn, layer_id=3) + + +def _handoff(rows=7, weights=torch.bfloat16): + from sglang.srt.layers.layer_boundary.fusions.cutedsl import MoeDeferredFinalize + + return MoeDeferredFinalize( + routed_output=torch.randn(rows * 10, 16).bfloat16(), + expert_weights=torch.rand(rows, 10).to(weights), + permuted_indices=torch.arange(rows * 10, dtype=torch.int32).view(rows, 10), + gated_shared_output=torch.randn(rows, 16).bfloat16(), + m=rows, + finish=lambda: pytest.fail("the handoff was completed unfused"), + ) + + +def _verify_batch(stream, rows=7): + from sglang.srt.model_executor.forward_batch_info import ForwardMode + + return SimpleNamespace( + forward_mode=ForwardMode.TARGET_VERIFY, + input_ids=torch.zeros(rows), + residual_stream=stream, + ) + + +def _same_calls(a, b): + """The same calls on the same tensors (by identity).""" + assert len(a) == len(b) == 1 + assert a[0][0] == b[0][0] and all(x is y for x, y in zip(a[0][1:], b[0][1:])) + + +@pytest.mark.parametrize("point", ["layer input", "MoE input", "final norm"]) +def test_the_hop_slot_runs_where_sglang_runs_its_cutedsl_kernel(cutedsl_gates, point): + """At each point, SGLang's stage boundary with the CuTe DSL workspace and the + slot's mirror with the hop make the same call on the same tensors and leave the + residual stream alike.""" + from sglang.srt.layers.layer_boundary import PLAIN_ADD + from sglang.srt.layers.layer_boundary.residual import batch as residual_batch + from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream + from sglang.srt.layers.layernorm import GemmaRMSNorm + + logs = {"sglang": [], "plugin": []} + service, hop = _Fused(logs["sglang"]), _Fused(logs["plugin"]) + layer, owed = _hop_layer(service), _handoff() + residual, part = torch.randn(7, 16).bfloat16(), torch.randn(7, 16).bfloat16() + final = GemmaRMSNorm(16) + runs = {} + for run in ("sglang", "plugin"): + stream = ResidualStream(residual) + fb = _verify_batch(stream) + st = SimpleNamespace(fb=fb, model=SimpleNamespace(norm=final)) + if point == "MoE input": + handle = layer.attn_boundary.finish(part, fb) + x = ( + layer.ffn_boundary.prepare(handle, fb) + if run == "sglang" + else boundary_hop.moe_input(hop, layer, handle, st) + ) + else: + handle = stream.record(owed, PLAIN_ADD) + if point == "layer input": + x = ( + layer.attn_boundary.prepare(handle, fb) + if run == "sglang" + else boundary_hop.layer_input(hop, layer, handle, st) + ) + else: + x = ( + residual_batch.final_norm( + handle, fb, final, finalize_norm=service, skip_empty=True + ) + if run == "sglang" + else boundary_hop.final_norm(hop, handle, st) + ) + runs[run] = (x, fb.residual_stream, stream.residual, stream.pending) + _same_calls(logs["sglang"], logs["plugin"]) + (x, stream, written, pending), (y, stream2, written2, pending2) = runs.values() + assert torch.equal(x, y) and pending is pending2 is None + if point == "final norm": + assert stream is stream2 is None + else: + assert torch.equal(written, written2) and written is not residual + + +@pytest.mark.parametrize("why", ["sglang declines", "fp32 weights", "too many rows"]) +def test_the_hop_slot_leaves_to_sglang_a_finalize_it_cannot_stand_in_for( + cutedsl_gates, why +): + from sglang.srt.layers.layer_boundary import PLAIN_ADD + from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream + + log = [] + layer = _hop_layer(_Fused([])) + cutedsl_gates["open"] = why != "sglang declines" + owed = _handoff(weights=torch.float32 if why == "fp32 weights" else torch.bfloat16) + hop = _Fused(log, rows=6 if why == "too many rows" else 7) + stream = ResidualStream(torch.zeros(7, 16)) + handle = stream.record(owed, PLAIN_ADD) + st = SimpleNamespace(fb=_verify_batch(stream), model=SimpleNamespace(norm=None)) + assert boundary_hop.layer_input(hop, layer, handle, st) is None + if why != "sglang declines": + assert boundary_hop.final_norm(hop, handle, st) is None + assert log == [] and stream.pending.owed is owed + + +def test_the_hop_slot_leaves_to_sglang_a_mixer_sum_it_declines(cutedsl_gates): + from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream + + log = [] + layer = _hop_layer(_Fused([])) + cutedsl_gates["open"] = False + stream = ResidualStream(torch.zeros(7, 16)) + st = SimpleNamespace(fb=_verify_batch(stream)) + handle = layer.attn_boundary.finish(torch.ones(7, 16).bfloat16(), st.fb) + assert boundary_hop.moe_input(_Fused(log), layer, handle, st) is None + assert log == [] and stream.pending is not None + + +def test_the_hop_slot_mirrors_the_paths_sglang_binds_for_qwen3_5(): + assert boundary_hop.path_refusals(_hop_layer(SimpleNamespace())) == [] + assert boundary_hop.path_refusals(_hop_layer(None, fusions=False)) == [ + "its attention input runs no CuTe DSL fusion", + "its ffn input runs no CuTe DSL fusion", + ] + + +def test_the_hop_slot_names_what_it_cannot_serve(): + service = SimpleNamespace(hidden_size=4096, top_k=10) + layers = [_hop_layer(SimpleNamespace()), _hop_layer(None, fusions=False)] + m = SimpleNamespace( + flashinfer_mnnvl_cutedsl_fusion=service, + layers=layers, + start_layer=0, + end_layer=1, + ) + assert boundary_hop.refusals(m, 8) == [] + assert boundary_hop.refusals(m, 16) == ["TP 16 (the hop serves TP (2, 4, 8))"] + service.hidden_size = 5120 + assert boundary_hop.refusals(m, 8) == [ + "hidden 5120, top-10 (the hop serves hidden 4096, up to top-16)" + ] + service.hidden_size, m.end_layer = 4096, 2 + assert boundary_hop.refusals(m, 8) == [ + "layer 3: its attention input runs no CuTe DSL fusion (and 1 more)" + ] + m.flashinfer_mnnvl_cutedsl_fusion = None + assert boundary_hop.refusals(m, 8) == [ + "SGLang runs no FlashInfer MNNVL CuTe DSL fusion" + ] + + +@pytest.mark.parametrize("taken", [False, True]) +def test_with_slot_hop_each_point_falls_back_to_sglang_where_the_hop_declines( + monkeypatch, taken +): + from sgl_nvidia_mega.qwen3_5 import boundary + + log = [] + for point in ("layer_input", "moe_input", "final_norm"): + monkeypatch.setattr( + boundary_hop, + point, + lambda hop, *a, point=point: ( + log.append((point, hop)) or (point if taken else None) + ), + ) + monkeypatch.setattr( + "sglang.srt.layers.layer_boundary.residual.batch.final_norm", + lambda *a, **k: "sglang final norm", + ) + layer = SimpleNamespace( + attn_boundary=SimpleNamespace( + prepare=lambda h, fb, capture_gathered: "sglang layer input", + finish=lambda part, fb: "handle", + ), + ffn_boundary=SimpleNamespace(prepare=lambda h, fb: "sglang MoE input"), + ) + st = SimpleNamespace( + fb="fb", + slots=plan.Slots(hop=True), + bound={"boundary": boundary_hop.Bound(hop="the hop")}, + model=SimpleNamespace(norm="norm", flashinfer_mnnvl_cutedsl_fusion=None), + ) + got = ( + boundary.pre(layer, "h", st), + boundary.post_attn(layer, "part", st), + boundary.close_stream(st, "h"), + ) + if taken: + assert got == ("layer_input", "moe_input", "final_norm") + else: + assert got == ("sglang layer input", "sglang MoE input", "sglang final norm") + assert log == [(p, "the hop") for p in ("layer_input", "moe_input", "final_norm")] + # an auxiliary capture reads SGLang's own prepare + assert boundary.pre(layer, "h", st, capture=[]) == "sglang layer input" From ef31703e2ad619baf1d851016371fe720a08103b Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:39:00 -0700 Subject: [PATCH 02/27] [Perf] [GB300] qwen3_5: Slot gdn_push: the GatedDeltaNet out_proj as the hop's push GEMM (numeric) The GatedDeltaNet's static-FP8 out_proj (1-8 rows) runs as one GEMM whose epilogue pushes the bf16 partial into the decode hop's mailbox, and the MoE input's all-reduce then only consumes it. The hop's device protocol moves into q35_hop.cuh so other kernels can push from their epilogues. Slot gdn_push (with gdn_verify and hop); numeric: it sums in another order than FlashInfer's BMM. --- README.md | 6 + sgl_nvidia_mega/qwen3_5/boundary/hop.py | 24 +- sgl_nvidia_mega/qwen3_5/gdn/__init__.py | 39 +- sgl_nvidia_mega/qwen3_5/gdn/norm_fp8.py | 16 +- sgl_nvidia_mega/qwen3_5/gdn/verify.py | 23 +- sgl_nvidia_mega/qwen3_5/plan.py | 9 +- sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 15 +- sgl_nvidia_mega/qwen3_5/sm103/hop/__init__.py | 112 ++++- sgl_nvidia_mega/qwen3_5/sm103/hop/src/hop.cu | 439 ++++-------------- .../qwen3_5/sm103/hop/src/q35_hop.cuh | 348 ++++++++++++++ sgl_nvidia_mega/qwen3_5/spec.py | 33 +- test/kernels/test_qwen3_5_hop.py | 65 +++ test/unit/test_qwen3_5_spec.py | 19 + test/unit/test_qwen3_5_steps.py | 123 ++++- 14 files changed, 894 insertions(+), 377 deletions(-) create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/hop/src/q35_hop.cuh diff --git a/README.md b/README.md index ff6674c1..bda129ac 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,12 @@ Qwen3.5 (`qwen3_5/runtime`) runs the same way: per layer the stage boundary (`bo | `gdn_pdl` (with `gdn_verify`) | 7 | Its ReplaySSM verify launched as a programmatic dependent of the conv | | `hop` | 7 | The layer boundary's fused all-reduce + residual add + RMSNorm points on an NVLink multicast hop | +These slots change the arithmetic (numeric: each sums in another order than SGLang's op, within the tolerances `spec.py` declares): + +| Slot | Rows | Kernel | +| --- | --- | --- | +| `gdn_push` (with `gdn_verify`, `hop`) | 7 | The GatedDeltaNet `out_proj` as the hop's GEMM, whose epilogue pushes the partial into the hop's mailbox | + The sampling and acceptance after the verify, and the GatedDeltaNet extend of prefill, stay SGLang's own code. In an SGLang that has the switches, `SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH=1` runs the sampling and acceptance in CUDA graphs, and `SGLANG_ENABLE_GDN_PREFILL_GRAPH_EXTEND=1` captures the GatedDeltaNet extend inside the breakable prefill CUDA graphs. ### Design diff --git a/sgl_nvidia_mega/qwen3_5/boundary/hop.py b/sgl_nvidia_mega/qwen3_5/boundary/hop.py index be816829..94083545 100644 --- a/sgl_nvidia_mega/qwen3_5/boundary/hop.py +++ b/sgl_nvidia_mega/qwen3_5/boundary/hop.py @@ -300,14 +300,30 @@ def layer_input(hop, layer, hidden, st): return norm +def takes_moe_input(hop, layer, st, rows: int) -> bool: + """Whether ``moe_input`` runs the hop for a mixer output of ``rows`` rows, as far as + the batch decides it (the stream's sum and update are checked there): a mixer may + push its partial into the hop only then.""" + from sglang.srt.layers.layer_boundary.construction import BatchVariant + from sglang.srt.runtime_context import get_exec + + plan = layer.ffn_boundary.plan + return ( + hop.supports(rows) + and plan.variant_for(st.fb) is BatchVariant.ORDINARY + and plan.fusions._eligible(plan, st.fb, rows) + and not get_exec().comm.enable_quant_communications + ) + + def moe_input(hop, layer, hidden, st): """``ffn_boundary.prepare`` of the mixer's partial output (``hidden``, as ``attn_boundary.finish`` recorded it), with the hop as its kernel; None where that - prepare would not run the CuTe DSL kernel.""" + prepare would not run the CuTe DSL kernel. A partial the mixer already pushed is + only consumed.""" from sglang.srt.layers.layer_boundary.fusions.cutedsl import _fused_norm_gamma from sglang.srt.layers.layer_boundary.residual.batch import stream_of from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum - from sglang.srt.runtime_context import get_exec stream = stream_of(st.fb) pending, plan = stream.pending, layer.ffn_boundary.plan @@ -321,10 +337,8 @@ def moe_input(hop, layer, hidden, st): rows = part.shape[0] if not ( residual is not None - and hop.supports(rows) and part.dtype == torch.bfloat16 - and plan.fusions._eligible(plan, st.fb, rows) - and not get_exec().comm.enable_quant_communications + and takes_moe_input(hop, layer, st, rows) ): return None _engage("MoE input", rows) diff --git a/sgl_nvidia_mega/qwen3_5/gdn/__init__.py b/sgl_nvidia_mega/qwen3_5/gdn/__init__.py index 7cab0c67..edbae058 100644 --- a/sgl_nvidia_mega/qwen3_5/gdn/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/gdn/__init__.py @@ -10,7 +10,8 @@ causal conv + ReplaySSM verify through the backend (``RadixLinearAttention``), the gated RMSNorm and ``out_proj``. ``step`` returns the TP-partial output, whose all-reduce the boundary runs. Slot ``gdn_verify`` (``verify.py``) runs the step with -the backend's chain opened up and its launch choices changed. +the backend's chain opened up and its launch choices changed; slot ``gdn_push`` pushes +``out_proj``'s partial into the decode hop from its GEMM (numeric). """ from __future__ import annotations @@ -59,6 +60,8 @@ class Bound(msgspec.Struct, frozen=True): gdn_verify: bool = False # slot gdn_verify serves every GDN layer gdn_pdl: bool = False # and launches its verify dependent on the conv + # slot gdn_push: layer id -> out_proj's weights packed for the hop's push GEMM + packed: dict = msgspec.field(default_factory=dict) reason: str = "" # why a requested slot is not built @@ -72,22 +75,42 @@ def build(model, ctx) -> Bound: if "gdn_verify" not in SLOTS: return Bound(reason="stock (SGLANG_NVIDIA_MEGAKERNEL_SLOTS)") m = model.model - for i in range(m.start_layer, m.end_layer): - layer = m.layers[i] - if isinstance(layer, Qwen3_5LinearDecoderLayer): - reason = verify.unsupported(layer.linear_attn) - if reason: - return Bound(reason=f"layer {i}: {reason}") - return Bound(gdn_verify=True, gdn_pdl="gdn_pdl" in SLOTS) + gdns = [ + (i, m.layers[i].linear_attn) + for i in range(m.start_layer, m.end_layer) + if isinstance(m.layers[i], Qwen3_5LinearDecoderLayer) + ] + for i, gdn in gdns: + reason = verify.unsupported(gdn) + if reason: + return Bound(reason=f"layer {i}: {reason}") + packed = {} + if "gdn_push" in SLOTS: + from sgl_nvidia_mega.qwen3_5.sm103.hop import pack_out_proj + + for i, gdn in gdns: + # the method keeps the weight as the [K, N] transpose of a row-major [N, K] + try: + packed[gdn.attn.layer_id] = pack_out_proj(gdn.out_proj.weight.t()) + except ValueError as e: + return Bound(reason=f"layer {i}: {e}") + return Bound(gdn_verify=True, gdn_pdl="gdn_pdl" in SLOTS, packed=packed) def step(layer, x, st): """``layer``'s GatedDeltaNet on the normed input ``x`` [rows, hidden]: the TP-partial output [rows, hidden].""" if st.slots.gdn_verify: + from sgl_nvidia_mega.qwen3_5.boundary import hop from sgl_nvidia_mega.qwen3_5.gdn import verify opts = verify.SLOT_PDL if st.slots.gdn_pdl else verify.SLOT + # push only where the MoE input's all-reduce will consume on the hop + if st.slots.gdn_push and hop.takes_moe_input( + st.bound["boundary"].hop, layer, st, x.shape[0] + ): + opts = msgspec.structs.replace(opts, push=True) + verify.engaged("gdn_push", x.shape[0]) return verify.step(layer.linear_attn, x, st, opts) import triton from sglang.kernels.ops.attention.triton_gdn_fused_proj import ( diff --git a/sgl_nvidia_mega/qwen3_5/gdn/norm_fp8.py b/sgl_nvidia_mega/qwen3_5/gdn/norm_fp8.py index 6891ef2d..b5a99a0d 100644 --- a/sgl_nvidia_mega/qwen3_5/gdn/norm_fp8.py +++ b/sgl_nvidia_mega/qwen3_5/gdn/norm_fp8.py @@ -55,9 +55,13 @@ def _layer_norm_fwd_1pass_kernel( ACTIVATION: tl.constexpr, OUT_DTYPE: tl.constexpr, # the activation dtype each output is rounded to first USE_GDC: tl.constexpr = False, + EARLY_DEPENDENTS: tl.constexpr = False, ): if USE_GDC: tl.extra.cuda.gdc_wait() + # the next launch loads only static data until its own wait + if EARLY_DEPENDENTS: + tl.extra.cuda.gdc_launch_dependents() row_start = tl.program_id(0) * ROWS_PER_BLOCK group = tl.program_id(1) @@ -113,7 +117,7 @@ def _layer_norm_fwd_1pass_kernel( y_q = tl.clamp(y * (1.0 / y_s), -448.0, 448.0).to(Q.dtype.element_ty) tl.store(Q + rows[:, None] * stride_q_row + col_offsets, y_q, mask=mask) - if USE_GDC: + if USE_GDC and not EARLY_DEPENDENTS: tl.extra.cuda.gdc_launch_dependents() @@ -126,9 +130,11 @@ def rms_norm_gated_fp8( *, norm_before_gate: bool = True, activation: str = "swish", + early_dependents: bool = False, ) -> torch.Tensor: """e4m3 of RMSNorm(x) * act(z) [M, N] (rounded to x's dtype first) over the static - input ``scale`` [1] of the linear it feeds; z [M, N] or [tokens, heads, N].""" + input ``scale`` [1] of the linear it feeds; z [M, N] or [tokens, heads, N]. With + ``early_dependents`` the next launch may start (and load its weights) at once.""" from sglang.kernels.jit.utils import is_arch_support_pdl from sglang.kernels.ops.attention.fla.layernorm_gated import calc_rows_per_block @@ -139,7 +145,11 @@ def rms_norm_gated_fp8( block_n = min(65536 // x.element_size(), triton.next_power_of_2(N)) num_warps = min(max(block_n // 256, 1), 8) rows_per_block = calc_rows_per_block(M, x.device) - pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {} + pdl_kwargs = ( + {"USE_GDC": True, "launch_pdl": True, "EARLY_DEPENDENTS": early_dependents} + if is_arch_support_pdl() + else {} + ) out_dtype = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16}[x.dtype] _layer_norm_fwd_1pass_kernel[(triton.cdiv(M, rows_per_block), 1)]( x, diff --git a/sgl_nvidia_mega/qwen3_5/gdn/verify.py b/sgl_nvidia_mega/qwen3_5/gdn/verify.py index 6f8a0983..6853b722 100644 --- a/sgl_nvidia_mega/qwen3_5/gdn/verify.py +++ b/sgl_nvidia_mega/qwen3_5/gdn/verify.py @@ -20,6 +20,10 @@ (``replayssm.py``), and with slot ``gdn_pdl`` launched dependent on the conv; - the gated norm writing the static FP8 ``out_proj`` takes (``norm_fp8.py``), which ``out_proj``'s FlashInfer BMM then reads without quantizing it again. + +Slot ``gdn_push`` (numeric, not bitwise) runs ``out_proj`` on the decode hop's GEMM +instead, whose epilogue pushes the partial into the hop's mailbox; the MoE input's +all-reduce then only consumes it. """ from __future__ import annotations @@ -154,6 +158,7 @@ class Options(msgspec.Struct, frozen=True): num_warps: int norm_fp8: bool # the gated norm writes out_proj's static FP8 input pdl: bool = False # the verify launches dependent on the conv (slot gdn_pdl) + push: bool = False # out_proj pushes its partial into the hop (slot gdn_push) STOCK = Options(views=False, block_n=256, block_v=64, num_warps=1, norm_fp8=False) @@ -161,6 +166,13 @@ class Options(msgspec.Struct, frozen=True): SLOT_PDL = msgspec.structs.replace(SLOT, pdl=True) +def engaged(slot: str, rows: int) -> None: + """Log once per (slot, rows) that a GDN slot ran, as engagement evidence.""" + if (slot, rows) not in _ENGAGED: + _ENGAGED.add((slot, rows)) + logger.info("NVIDIA megakernel: Qwen3.5 slot %s runs at %d rows", slot, rows) + + def unsupported(gdn) -> str: """Why the slot cannot serve ``gdn`` (a Qwen3_5GatedDeltaNet) ("" when it can).""" from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp8LinearMethod @@ -248,8 +260,17 @@ def step(gdn, x: torch.Tensor, st, opts: Options = SLOT) -> torch.Tensor: z if opts.views else z.reshape(-1, z.shape[-1]), gdn.out_proj.input_scale, activation=norm.activation, + early_dependents=opts.push, ) - return _out_proj(gdn.out_proj, fp8.reshape(-1, nv * gdn.head_v_dim), core.dtype) + x = fp8.reshape(-1, nv * gdn.head_v_dim) + if opts.push: + return st.bound["boundary"].hop.out_proj_push( + x_fp8=x, + packed_w=st.bound["gdn"].packed[gdn.attn.layer_id], + input_scale=gdn.out_proj.input_scale, + weight_scale=gdn.out_proj.weight_scale, + ) + return _out_proj(gdn.out_proj, x, core.dtype) z_shape = z.shape core = gdn.norm(core, z if opts.views else z.reshape(-1, z.shape[-1])) core = core.reshape(z_shape) diff --git a/sgl_nvidia_mega/qwen3_5/plan.py b/sgl_nvidia_mega/qwen3_5/plan.py index 07d31464..7042f618 100644 --- a/sgl_nvidia_mega/qwen3_5/plan.py +++ b/sgl_nvidia_mega/qwen3_5/plan.py @@ -6,8 +6,9 @@ inside its row bounds; every other op is SGLang's own. ``route`` (the router GEMM and the routed experts' NVFP4 input in one launch) and ``mkm`` (on its outputs, the top-10 and both expert GEMMs in one launch, up to 7 rows) cover every width, as do -``gdn_verify`` (the GatedDeltaNet verify chain with its launch choices changed) and -``hop`` (the boundary's fused all-reduce points on the decode hop). +``gdn_verify`` (the GatedDeltaNet verify chain with its launch choices changed), +``hop`` (the boundary's fused all-reduce points on the decode hop) and ``gdn_push`` +(numeric: out_proj as the hop's push GEMM, on gdn_verify's FP8 norm). """ from __future__ import annotations @@ -27,6 +28,8 @@ class Slots(msgspec.Struct, frozen=True): gdn_verify: bool = False # the GDN verify chain's launch choices (gdn/verify.py) gdn_pdl: bool = False # its ReplaySSM verify launched dependent on the conv hop: bool = False # the boundary's fused all-reduce points (boundary/hop.py) + # out_proj pushes its partial into the hop from its GEMM (gdn/verify.py; numeric) + gdn_push: bool = False class Facts(msgspec.Struct, frozen=True): @@ -38,6 +41,7 @@ class Facts(msgspec.Struct, frozen=True): gdn_verify: bool = False # the GDN verify slot built for every GDN layer gdn_pdl: bool = False # with its dependent verify launch hop: bool = False # the decode hop, built for the widest covered step + gdn_push: bool = False # out_proj packed for the push GEMM on every GDN layer def slots(rows: int, facts: Facts) -> Slots: @@ -49,6 +53,7 @@ def slots(rows: int, facts: Facts) -> Slots: gdn_verify=facts.gdn_verify, gdn_pdl=facts.gdn_verify and facts.gdn_pdl, hop=facts.hop, + gdn_push=facts.gdn_verify and facts.hop and facts.gdn_push, ) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index 0883e391..ed6b3fe3 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -14,7 +14,14 @@ # The kernel slots that run by default, and the ones that run only when requested # (until step-level timing and accuracy runs move them to the default list) SHIPPED_SLOTS: tuple[str, ...] = () -OPT_IN_SLOTS: tuple[str, ...] = ("route", "mkm", "gdn_verify", "gdn_pdl", "hop") +OPT_IN_SLOTS: tuple[str, ...] = ( + "route", + "mkm", + "gdn_verify", + "gdn_pdl", + "hop", + "gdn_push", +) SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" @@ -45,6 +52,11 @@ def slot_reasons() -> list[str]: ) if "mkm" in SLOTS and "route" not in SLOTS: reasons.append(f"{SLOTS_ENV}: slot 'mkm' runs on slot 'route''s outputs") + if "gdn_push" in SLOTS and not {"gdn_verify", "hop"} <= SLOTS: + reasons.append( + f"{SLOTS_ENV}: slot 'gdn_push' pushes slot 'gdn_verify''s out_proj into " + "slot 'hop'" + ) if "gdn_pdl" in SLOTS and "gdn_verify" not in SLOTS: reasons.append( f"{SLOTS_ENV}: slot 'gdn_pdl' launches slot 'gdn_verify''s verify" @@ -102,6 +114,7 @@ def build(model, ctx): gdn_verify=bound["gdn"].gdn_verify, gdn_pdl=bound["gdn"].gdn_pdl, hop=bound["boundary"].hop is not None, + gdn_push=bool(bound["gdn"].packed), ) slots = slot_plan.plan(pctx.widths, facts) built = msgspec.structs.asdict(facts) diff --git a/sgl_nvidia_mega/qwen3_5/sm103/hop/__init__.py b/sgl_nvidia_mega/qwen3_5/sm103/hop/__init__.py index 974e14b2..aa7a6033 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/hop/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/hop/__init__.py @@ -1,11 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -"""The decode hop (``src/hop.cu``): the TP sum of a verify step's bf16 partials with the -residual add and RMSNorm (+ static FP8) for hidden 4096, through one NVLink multicast -mailbox per process. ``publish`` (or ``publish_fin``, which finalizes the routed MoE rows -first) pushes this rank's rows; ``consume`` pulls every rank's, sums them in rank order, -writes the residual and the norm and re-arms the mailbox. The outputs are those of -FlashInfer's MNNVL CuTe DSL low-latency all-reduce + residual + RMSNorm, bit for bit. +"""The decode hop (``src/hop.cu`` on the device protocol ``src/q35_hop.cuh``): the TP sum +of a verify step's bf16 partials with the residual add and RMSNorm (+ static FP8) for +hidden 4096, through one NVLink multicast mailbox per process. ``publish`` (or +``publish_fin``, which finalizes the routed MoE rows first) pushes this rank's rows; +``consume`` pulls every rank's, sums them in rank order, writes the residual and the norm +and re-arms the mailbox. The outputs are those of FlashInfer's MNNVL CuTe DSL low-latency +all-reduce + residual + RMSNorm, bit for bit. + +A producer may push its partial itself, from its own epilogue: ``out_proj_push`` (the +GatedDeltaNet's static-FP8 ``out_proj``) does, and so may another package's kernel built +with ``headers()``, given ``Hop.site``, which then calls ``Hop.mark_pushed``. The next +all-reduce of that partial only consumes. A consumer that waits past the deadline faults at address 0x0 (fail-stop) instead of returning: the deadline is compiled in, as SGLang's ``BuildContext.deadline_s``. @@ -22,7 +28,9 @@ from sgl_nvidia_mega.common import jit SOURCE = Path(__file__).parent / "src" / "hop.cu" +HEADER = Path(__file__).parent / "src" / "q35_hop.cuh" HIDDEN, STAGES, ROW_CTAS, ROW_THREADS, QUAD_CTAS, MAX_TOP_K = 4096, 3, 4, 128, 8, 16 +OPROJ_THREADS, OPROJ_MAX_M, OPROJ_STEPS = 256, 8, (4, 8) # K = 256 * steps SENTINEL = -32768 # bf16 -0.0 as int16: an empty mailbox slot WORLDS = (2, 4, 8) @@ -61,6 +69,34 @@ class Args(ctypes.Structure): ) +class OprojArgs(ctypes.Structure): + _fields_ = ( + [("site", Site)] + + [(name, ctypes.c_void_p) for name in ("w", "x", "sx", "sw", "out")] + + [("m", ctypes.c_int)] + ) + + +def pack_out_proj(weight_nk: torch.Tensor) -> torch.Tensor: + """[4096, K] e4m3 row-major -> the push kernel's mma.m16n8k32 A fragments: + [cta 128][warp 8][k-step S][block 2][lane 32][16 B], K = 8 * S * 32.""" + w = weight_nk.contiguous().view(torch.uint8) + n, k = w.shape + steps = k // 256 + if n != HIDDEN or steps not in OPROJ_STEPS or k != 256 * steps: + raise ValueError( + f"out_proj packing needs [4096, 256 * S], got {tuple(w.shape)}" + ) + v = w.view(n // 32, 2, 2, 8, 8, steps, 2, 4, 4) + return v.permute(0, 4, 5, 1, 3, 7, 6, 2, 8).contiguous().view(-1) + + +def headers() -> dict[str, str]: + """The NVRTC header of the hop's device protocol, for a kernel that pushes into the + hop (``#include "q35_hop.cuh"``).""" + return {HEADER.name: HEADER.read_text()} + + @cache def _kernels(device_index: int, deadline_ns: int) -> dict[str, jit.Kernel]: major, minor = torch.cuda.get_device_capability(device_index) @@ -69,10 +105,15 @@ def _kernels(device_index: int, deadline_ns: int) -> dict[str, jit.Kernel]: "q35_hop", f"sm_{major}{minor}a", ("-std=c++17", "-default-device", f"-DQ35_HOP_DEADLINE_NS={deadline_ns}ull"), + headers(), ) names = ["q35hop_publish", "q35hop_publish_fin_top10", "q35hop_publish_fin"] names += [f"q35hop_consume_w{w}" for w in WORLDS] - return {n: jit.load(cubin, n, (ROW_THREADS, 1, 1), 0) for n in names} + kernels = {n: jit.load(cubin, n, (ROW_THREADS, 1, 1), 0) for n in names} + for s in OPROJ_STEPS: + name = f"q35hop_oproj_push_s{s}" + kernels[name] = jit.load(cubin, name, (OPROJ_THREADS, 1, 1), 0) + return kernels class Hop: @@ -104,6 +145,9 @@ def __init__(self, *, max_m: int, eps: float, group, device, deadline_s: float): self._mailbox.fill_(SENTINEL) self._ctrl = torch.zeros(4, dtype=torch.int32, device=self.device) self._handle = handle # keeps the mapping alive + # (address, shape) of a partial its producer already pushed; its all-reduce + # only consumes + self._pushed = None self.site = Site( multicast, self._mailbox.data_ptr(), @@ -121,14 +165,61 @@ def supports(self, m: int) -> bool: def all_reduce_residual_rms_norm( self, *, local_contribution, residual, gamma, fp8_scale=None ): - """(norm, residual_out[, fp8]) [M, 4096] of sum_ranks(partial) + residual.""" + """(norm, residual_out[, fp8]) [M, 4096] of sum_ranks(partial) + residual; a + partial out_proj_push returned is already in the mailbox.""" m = int(local_contribution.shape[0]) args, outs = self._args(m, residual, gamma, fp8_scale) - args.partial = local_contribution.data_ptr() - self._launch("q35hop_publish", args, ROW_CTAS) + if self.has_pushed(local_contribution): + self._pushed = None + else: + self._check_consumed() + args.partial = local_contribution.data_ptr() + self._launch("q35hop_publish", args, ROW_CTAS) self._consume(args) return outs + def out_proj_push(self, *, x_fp8, packed_w, input_scale, weight_scale): + """The bf16 partial [M, 4096] of a static-FP8 out_proj on x_fp8 [M, K], also + pushed into the mailbox: its all-reduce must come next.""" + m = int(x_fp8.shape[0]) + steps = packed_w.numel() // (HIDDEN * 256) + if not 1 <= m <= min(OPROJ_MAX_M, self.max_m) or steps not in OPROJ_STEPS: + raise ValueError(f"the out_proj push serves 1-{OPROJ_MAX_M} rows, not {m}") + self._check_consumed() + out = torch.empty((m, HIDDEN), dtype=torch.bfloat16, device=self.device) + args = OprojArgs( + site=self.site, + w=packed_w.data_ptr(), + x=x_fp8.data_ptr(), + sx=input_scale.data_ptr(), + sw=weight_scale.data_ptr(), + out=out.data_ptr(), + m=m, + ) + jit.launch( + self.kernels[f"q35hop_oproj_push_s{steps}"], + (HIDDEN // 32, 1, 1), + [args], + torch.cuda.current_stream(self.device).cuda_stream, + ) + self.mark_pushed(out) + return out + + def mark_pushed(self, partial: torch.Tensor) -> None: + """Record that a producer kernel, launched on this stream after the last + all-reduce, pushed the bf16 ``partial`` [M, 4096] into the mailbox (through + ``site``): its all-reduce, which must come next, only consumes.""" + self._check_consumed() + self._pushed = (partial.data_ptr(), partial.shape) + + def has_pushed(self, partial: torch.Tensor) -> bool: + return self._pushed == (partial.data_ptr(), partial.shape) + + def _check_consumed(self): + # a pushed partial left unconsumed would leave its mailbox stage dirty + if self._pushed is not None: + raise RuntimeError("the hop pushed a partial that was never consumed") + def moe_finalize_all_reduce_rms_norm( self, *, @@ -145,6 +236,7 @@ def moe_finalize_all_reduce_rms_norm( m, top_k = (int(n) for n in permuted_indices.shape) if top_k > MAX_TOP_K: raise ValueError(f"top-{top_k} routing (at most {MAX_TOP_K})") + self._check_consumed() args, outs = self._args(m, residual, gamma, fp8_scale) args.routed, args.expert_w = routed_output.data_ptr(), expert_weights.data_ptr() args.permuted, args.top_k = permuted_indices.data_ptr(), top_k diff --git a/sgl_nvidia_mega/qwen3_5/sm103/hop/src/hop.cu b/sgl_nvidia_mega/qwen3_5/sm103/hop/src/hop.cu index 6548daa4..331575a0 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/hop/src/hop.cu +++ b/sgl_nvidia_mega/qwen3_5/sm103/hop/src/hop.cu @@ -13,346 +13,12 @@ // consecutive bf16 of a token row (frag = 0..511); a token row is finished by one cluster of // kRowCtas CTAs x kRowThreads threads (CTA c owns fragments [128c, 128c + 128)). // +// oproj_push the GatedDeltaNet out_proj (static FP8, M <= 8) whose epilogue pushes its partial +// // Built with NVRTC: C-linkage entries per instantiation; Q35_HOP_DEADLINE_NS (0: none) is the // consumer's wait deadline, past which it faults at address 0x0 (fail-stop). -#include - -// NVRTC has no -typedef unsigned char uint8_t; -typedef unsigned short uint16_t; -typedef int int32_t; -typedef unsigned int uint32_t; -typedef unsigned long long uint64_t; - -#ifndef Q35_HOP_DEADLINE_NS -#define Q35_HOP_DEADLINE_NS 0ull -#endif - -namespace q35hop { - - -constexpr int kHidden = 4096; -constexpr int kVec = 8; -constexpr int kFrags = kHidden / kVec; -constexpr int kStages = 3; -constexpr int kRowCtas = 4; -constexpr int kRowThreads = 128; -constexpr int kRowWarps = kRowThreads / 32; -constexpr uint32_t kNegZeroPair = 0x80008000u; -static_assert(kRowCtas * kRowThreads == kFrags, "a row is one fragment per thread of the cluster"); -static_assert(kRowWarps == 4, "row_sumsq_fold folds exactly four warp sums per CTA"); - -// Per-process control block in device memory. -struct Ctrl { - uint32_t stage; // stage of the next hop - uint32_t arrive; // CTA tickets of the running consumer launch - uint32_t seq; // hops completed - uint32_t pad; -}; - -// Passed by value as a kernel parameter. -struct Site { - void* mc; // multicast base of the mailbox [kStages][world][m_cap][kHidden] bf16 - void* uc; // this rank's unicast view of the same mailbox - Ctrl* ctrl; - int rank, world, m_cap; -}; - -// ---------------------------------------------------------------- PTX - -__device__ __forceinline__ void pdl_wait() { asm volatile("griddepcontrol.wait;" ::: "memory"); } -__device__ __forceinline__ void pdl_trigger() { asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); } -__device__ __forceinline__ unsigned long long gtimer() { - unsigned long long t; - asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t)); - return t; -} -__device__ __forceinline__ uint32_t ld_volatile_u32(const uint32_t* p) { - uint32_t v; - asm volatile("ld.volatile.global.u32 %0, [%1];" : "=r"(v) : "l"(p) : "memory"); - return v; -} -__device__ __forceinline__ uint4 ld_volatile_v4(const void* p) { - uint4 v; - asm volatile("ld.volatile.global.v4.u32 {%0,%1,%2,%3}, [%4];" - : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) - : "l"(p) - : "memory"); - return v; -} -__device__ __forceinline__ void st_v4(void* p, uint4 v) { - asm volatile("st.global.v4.u32 [%0], {%1,%2,%3,%4};" ::"l"(p), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w) - : "memory"); -} -__device__ __forceinline__ void mc_st_v4(void* mc, uint4 v) { - asm volatile("multimem.st.relaxed.sys.global.v4.bf16x2 [%0], {%1,%2,%3,%4};" ::"l"(mc), "r"(v.x), - "r"(v.y), "r"(v.z), "r"(v.w) - : "memory"); -} -__device__ __forceinline__ void mc_st_v2(void* mc, uint2 v) { - asm volatile("multimem.st.relaxed.sys.global.v2.bf16x2 [%0], {%1,%2};" ::"l"(mc), "r"(v.x), "r"(v.y) : "memory"); -} -__device__ __forceinline__ uint32_t cluster_ctarank() { - uint32_t r; - asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(r)); - return r; -} -__device__ __forceinline__ float div_full(float a, float b) { - float r; - asm("div.full.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); - return r; -} -// Two f32 to an e4m3 pair; lo lands in the low byte. -__device__ __forceinline__ uint16_t cvt_e4m3x2(float lo, float hi) { - uint16_t r; - asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(r) : "f"(hi), "f"(lo)); - return r; -} - -// ---------------------------------------------------------------- bf16 - -__device__ __forceinline__ float bf_lo(uint32_t w) { return __uint_as_float(w << 16); } -__device__ __forceinline__ float bf_hi(uint32_t w) { return __uint_as_float(w & 0xFFFF0000u); } -__device__ __forceinline__ uint32_t pack_bf16x2(float lo, float hi) { - __nv_bfloat162 v = __floats2bfloat162_rn(lo, hi); - return *reinterpret_cast(&v); -} -__device__ __forceinline__ void unpack8(uint4 v, float f[8]) { - f[0] = bf_lo(v.x); f[1] = bf_hi(v.x); f[2] = bf_lo(v.y); f[3] = bf_hi(v.y); - f[4] = bf_lo(v.z); f[5] = bf_hi(v.z); f[6] = bf_lo(v.w); f[7] = bf_hi(v.w); -} -__device__ __forceinline__ uint4 pack8(const float f[8]) { - return make_uint4(pack_bf16x2(f[0], f[1]), pack_bf16x2(f[2], f[3]), pack_bf16x2(f[4], f[5]), - pack_bf16x2(f[6], f[7])); -} -__device__ __forceinline__ bool word_has_negzero(uint32_t w) { - return (w & 0xFFFFu) == 0x8000u || (w >> 16) == 0x8000u; -} -__device__ __forceinline__ bool frag_has_negzero(uint4 v) { - return word_has_negzero(v.x) | word_has_negzero(v.y) | word_has_negzero(v.z) | word_has_negzero(v.w); -} -__device__ __forceinline__ uint32_t sanitize_word(uint32_t w) { - if ((w & 0xFFFFu) == 0x8000u) w &= 0xFFFF0000u; - if ((w >> 16) == 0x8000u) w &= 0x0000FFFFu; - return w; -} - -// ---------------------------------------------------------------- stage protocol - -__device__ __forceinline__ size_t slot(const Site& s, uint32_t stage, int src, int token, int frag) { - return ((static_cast(stage) * s.world + src) * s.m_cap + token) * kHidden + - static_cast(frag) * kVec; -} - -struct Hop { - uint32_t stage; - uint32_t seq; - uint32_t ticket; // thread 0 only -}; - -// Consumer launch, every thread of every participating CTA, after griddepcontrol.wait. Thread -// 0 takes a ticket whose value is consumed only in hop_leave, so the atomic stays off the -// critical path. -__device__ __forceinline__ Hop hop_enter(const Site& s) { - Hop h; - h.stage = ld_volatile_u32(&s.ctrl->stage); - h.seq = ld_volatile_u32(&s.ctrl->seq); - h.ticket = threadIdx.x == 0 ? atomicAdd(&s.ctrl->arrive, 1u) : 0u; - return h; -} - -// Thread 0 of every participating CTA; the last arrival advances the stage. No CTA of this -// launch can read the advanced value, since every CTA read the stage before taking its ticket. -__device__ __forceinline__ void hop_leave(const Site& s, const Hop& h, uint32_t participants) { - if (threadIdx.x == 0 && h.ticket == participants - 1) { - s.ctrl->arrive = 0; - s.ctrl->stage = (h.stage + 1) % kStages; - s.ctrl->seq = h.seq + 1; - } -} - -// Producer-only launch, after griddepcontrol.wait. -__device__ __forceinline__ uint32_t stage_peek(const Site& s) { return ld_volatile_u32(&s.ctrl->stage); } - -// ---------------------------------------------------------------- producer - -// This rank's final bf16 partial for one fragment; exactly once per (hop, rank, token, frag). -// A launch that also stores its own outputs should push first: the push starts the flight. -__device__ __forceinline__ void push_frag(const Site& s, uint32_t stage, int token, int frag, uint4 v) { - v = make_uint4(sanitize_word(v.x), sanitize_word(v.y), sanitize_word(v.z), sanitize_word(v.w)); - mc_st_v4(static_cast(s.mc) + slot(s, stage, s.rank, token, frag), v); -} - -// Same for 4 consecutive bf16 (half a fragment, columns [4q, 4q + 4)); a fragment may be pushed -// as two halves by different threads. -__device__ __forceinline__ void push_quad(const Site& s, uint32_t stage, int token, int quad, uint2 v) { - v = make_uint2(sanitize_word(v.x), sanitize_word(v.y)); - mc_st_v2(static_cast(s.mc) + slot(s, stage, s.rank, token, quad >> 1) + (quad & 1) * 4, v); -} - -// ---------------------------------------------------------------- consumer - -// A peer that missed the wait deadline: store to address 0x0, then trap (fail-stop). -__device__ __forceinline__ void fault_at_null() { - asm volatile("{\n.reg .u64 a;\nmov.u64 a, 0;\nst.volatile.global.u32 [a], 0;\ntrap;\n}" ::: "memory"); -} - -// Sum over ranks 0..W-1 (fp32, from 0) of one fragment; spins on the sentinel. A wait past -// Q35_HOP_DEADLINE_NS (if not 0) stops the kernel. -template -__device__ __forceinline__ bool pull_frag(const Site& s, uint32_t stage, int token, int frag, float acc[8]) { - const uint16_t* uc = static_cast(s.uc); - uint4 v[W]; -#pragma unroll - for (int r = 0; r < W; ++r) v[r] = ld_volatile_v4(uc + slot(s, stage, r, token, frag)); - uint32_t pending = 0; -#pragma unroll - for (int r = 0; r < W; ++r) pending |= frag_has_negzero(v[r]) ? (1u << r) : 0u; - unsigned long long t0 = 0; - uint32_t spins = 0; - while (pending) { -#pragma unroll - for (int r = 0; r < W; ++r) - if (pending & (1u << r)) v[r] = ld_volatile_v4(uc + slot(s, stage, r, token, frag)); - uint32_t still = 0; -#pragma unroll - for (int r = 0; r < W; ++r) - if ((pending & (1u << r)) && frag_has_negzero(v[r])) still |= 1u << r; - pending = still; - if (Q35_HOP_DEADLINE_NS != 0 && pending && ((++spins & 255u) == 0)) { - const unsigned long long now = gtimer(); - if (t0 == 0) { - t0 = now; - } else if (now - t0 > Q35_HOP_DEADLINE_NS) { - fault_at_null(); - } - } - } -#pragma unroll - for (int e = 0; e < 8; ++e) acc[e] = 0.f; -#pragma unroll - for (int r = 0; r < W; ++r) { - float f[8]; - unpack8(v[r], f); -#pragma unroll - for (int e = 0; e < 8; ++e) acc[e] = __fadd_rn(acc[e], f[e]); - } - return true; -} - -// Write the sentinel back into the W slots this thread consumed (any time after pull_frag). -template -__device__ __forceinline__ void rearm_frag(const Site& s, uint32_t stage, int token, int frag) { - uint16_t* uc = static_cast(s.uc); - const uint4 z = make_uint4(kNegZeroPair, kNegZeroPair, kNegZeroPair, kNegZeroPair); -#pragma unroll - for (int r = 0; r < W; ++r) st_v4(uc + slot(s, stage, r, token, frag), z); -} - -// ---------------------------------------------------------------- row finish - -// prenorm = bf16(acc + residual); pre[] gets the bf16-rounded values the norm uses. -__device__ __forceinline__ uint4 residual_add(const float acc[8], uint4 residual, float pre[8]) { - float r[8]; - unpack8(residual, r); -#pragma unroll - for (int e = 0; e < 8; ++e) pre[e] = __fadd_rn(acc[e], r[e]); - const uint4 out = pack8(pre); - unpack8(out, pre); - return out; -} - -__device__ __forceinline__ float sumsq8(const float x[8]) { - float s = 0.f; -#pragma unroll - for (int e = 0; e < 8; ++e) s = __fadd_rn(s, __fmul_rn(x[e], x[e])); - return s; -} - -struct RowSmem { - float xs[kRowCtas][kRowWarps]; - uint64_t mbar; -}; - -// Prologue of every CTA of the row cluster, all threads, before griddepcontrol.wait: arm the -// exchange barrier and make it visible to the peers before any of them can send. -__device__ __forceinline__ void row_init(RowSmem& rs) { - if (threadIdx.x == 0) { - const uint32_t a = static_cast(__cvta_generic_to_shared(&rs.mbar)); - asm volatile("mbarrier.init.shared::cta.b64 [%0], 1;" ::"r"(a) : "memory"); - asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(a), - "r"(static_cast(sizeof(rs.xs))) - : "memory"); - asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); - } - asm volatile("barrier.cluster.arrive.relaxed.aligned;\n\tbarrier.cluster.wait.acquire.aligned;" ::: "memory"); -} - -// Row total of the per-thread values: lane butterfly per warp, then each warp sends its sum to -// every CTA of the cluster with st.async; each CTA waits on its own mbarrier (no release fence, -// so this thread's earlier remote stores are not waited for) and folds per CTA as -// (w0 + w2) + (w1 + w3), then the CTAs in rank order from 0. -__device__ __forceinline__ float row_sumsq_fold(float v, RowSmem& rs) { - const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5; -#pragma unroll - for (int off = 1; off < 32; off <<= 1) v = __fadd_rn(v, __shfl_xor_sync(0xffffffffu, v, off)); - if (lane < kRowCtas) { - const uint32_t me = cluster_ctarank(); - const uint32_t slot_l = static_cast(__cvta_generic_to_shared(&rs.xs[me][warp])); - const uint32_t mbar_l = static_cast(__cvta_generic_to_shared(&rs.mbar)); - uint32_t slot_r, mbar_r; - asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(slot_r) : "r"(slot_l), "r"(lane)); - asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(mbar_r) : "r"(mbar_l), "r"(lane)); - asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.f32 [%0], %1, [%2];" ::"r"(slot_r), - "f"(v), "r"(mbar_r) - : "memory"); - } - { - const uint32_t a = static_cast(__cvta_generic_to_shared(&rs.mbar)); - asm volatile( - "{\n\t.reg .pred P;\n\tWAIT_ROW:\n\t" - "mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P, [%0], 0;\n\t" - "@!P bra WAIT_ROW;\n\t}" ::"r"(a) - : "memory"); - } - float total = 0.f; -#pragma unroll - for (int c = 0; c < kRowCtas; ++c) - total = __fadd_rn(total, __fadd_rn(__fadd_rn(rs.xs[c][0], rs.xs[c][2]), __fadd_rn(rs.xs[c][1], rs.xs[c][3]))); - return total; -} - -__device__ __forceinline__ float inv_rms(float row_sumsq, float eps) { - return rsqrtf(__fadd_rn(row_sumsq / static_cast(kHidden), eps)); -} - -// bf16((x * inv) * gamma), gamma being the applied multiplier. -__device__ __forceinline__ uint4 norm_apply(const float pre[8], float inv, const float gamma[8]) { - float out[8]; -#pragma unroll - for (int e = 0; e < 8; ++e) out[e] = __fmul_rn(__fmul_rn(pre[e], inv), gamma[e]); - return pack8(out); -} - -// The reciprocal the static per-tensor FP8 quant multiplies by. -__device__ __forceinline__ float inv_scale_of(float scale) { return div_full(1.f, scale); } - -// e4m3(clamp(x * inv_scale, +-448)) of the bf16 norm output, 8 bytes. -__device__ __forceinline__ uint2 quant_fp8(uint4 norm_bf16, float inv_scale) { - float q[8]; - unpack8(norm_bf16, q); - uint32_t w[4]; -#pragma unroll - for (int j = 0; j < 4; ++j) { - const float a = fminf(fmaxf(__fmul_rn(q[2 * j], inv_scale), -448.f), 448.f); - const float b = fminf(fmaxf(__fmul_rn(q[2 * j + 1], inv_scale), -448.f), 448.f); - w[j] = cvt_e4m3x2(a, b); - } - return make_uint2(w[0] | (w[1] << 16), w[2] | (w[3] << 16)); -} - -} // namespace q35hop +#include "q35_hop.cuh" // a named namespace: the C entries below name Args in their signatures namespace q35hop_kernels { @@ -473,9 +139,100 @@ __device__ __forceinline__ void consume_kernel(const Args& a) { hop_leave(a.site, h, gridDim.x * gridDim.y); } +// Row-parallel FP8 out_proj (M <= 8, N = 4096, K = 8 * S * 32) whose epilogue pushes the bf16 +// partial: CTA c owns output rows [32c, 32c + 32) over the full K, warp w takes k-steps +// [w*S, (w+1)*S), the weights are mma.m16n8k32 e4m3 A fragments loaded before the PDL wait, and +// the warp partials are summed in warp order. +struct OprojArgs { + Site site; + const uint4* w; // packed [128 cta][8 warp][S step][2 block][32 lane] x 16 B + const uint8_t* x; // [M, K] e4m3 activation + const float* sx; // input scale + const float* sw; // weight scale + uint16_t* out; // [M, 4096] bf16 copy of the partial + int m; +}; + +__device__ __forceinline__ void mma_e4m3(float (&d)[4], const uint4& a, uint32_t b0, uint32_t b1) { + asm volatile( + "mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, " + "{%0,%1,%2,%3};" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a.x), "r"(a.y), "r"(a.z), "r"(a.w), "r"(b0), "r"(b1)); +} + +// Weights are static for the life of the process. +__device__ __forceinline__ uint4 ldg_nc_v4(const uint4* q) { + uint4 r; + asm volatile("ld.global.nc.L1::no_allocate.v4.u32 {%0,%1,%2,%3}, [%4];" + : "=r"(r.x), "=r"(r.y), "=r"(r.z), "=r"(r.w) + : "l"(q)); + return r; +} + +template +__device__ __forceinline__ void oproj_push_kernel(const OprojArgs& o) { + constexpr int kK = 8 * S * 32; + __shared__ __align__(16) float red[8][32][8]; + const int cta = blockIdx.x; + const int tid = threadIdx.x, warp = tid >> 5, lane = tid & 31; + const int g = lane >> 2, t = lane & 3; + uint4 a[S][2]; + { + const uint4* wb = o.w + (static_cast(cta) * 8 + warp) * (S * 2 * 32) + lane; +#pragma unroll + for (int st = 0; st < S; ++st) +#pragma unroll + for (int b = 0; b < 2; ++b) a[st][b] = ldg_nc_v4(wb + (st * 2 + b) * 32); + } + const float alpha = __fmul_rn(*o.sx, *o.sw); + pdl_wait(); + const int m = o.m; + const bool tok_ok = g < m; + const uint8_t* xr = o.x + static_cast(g) * kK + warp * S * 32 + 4 * t; + float ch[2][2][4] = {}; +#pragma unroll + for (int st = 0; st < S; ++st) { + const uint32_t b0 = tok_ok ? *reinterpret_cast(xr + st * 32) : 0u; + const uint32_t b1 = tok_ok ? *reinterpret_cast(xr + st * 32 + 16) : 0u; + mma_e4m3(ch[0][st & 1], a[st][0], b0, b1); + mma_e4m3(ch[1][st & 1], a[st][1], b0, b1); + } + { + float4* dst = reinterpret_cast(&red[warp][lane][0]); + dst[0] = make_float4(__fadd_rn(ch[0][0][0], ch[0][1][0]), __fadd_rn(ch[0][0][1], ch[0][1][1]), + __fadd_rn(ch[0][0][2], ch[0][1][2]), __fadd_rn(ch[0][0][3], ch[0][1][3])); + dst[1] = make_float4(__fadd_rn(ch[1][0][0], ch[1][1][0]), __fadd_rn(ch[1][0][1], ch[1][1][1]), + __fadd_rn(ch[1][0][2], ch[1][1][2]), __fadd_rn(ch[1][0][3], ch[1][1][3])); + } + __syncthreads(); + // Thread (tok, j) owns rows [8j, 8j + 8) of token tok. Accumulator element (row r, token k) sits + // at lane (r % 8) * 4 + k / 2, entry (r / 16) * 4 + ((r % 16) / 8) * 2 + k % 2 of the mma layout. + const int tok = tid >> 2, j = tid & 3; + if (tok < m) { + float v[8]; +#pragma unroll + for (int q = 0; q < 8; ++q) { + const int r = 8 * j + q; + const int l = (r % 8) * 4 + tok / 2; + const int e = (r / 16) * 4 + ((r % 16) / 8) * 2 + (tok % 2); + float x = red[0][l][e]; +#pragma unroll + for (int w = 1; w < 8; ++w) x = __fadd_rn(x, red[w][l][e]); + v[q] = __fmul_rn(x, alpha); + } + const uint4 frag = pack8(v); + const int fr = cta * 4 + j; + push_frag(o.site, stage_peek(o.site), tok, fr, frag); + *reinterpret_cast(o.out + row_elem(tok, fr)) = frag; + } + pdl_trigger(); +} + } // namespace q35hop_kernels using q35hop_kernels::Args; +using q35hop_kernels::OprojArgs; extern "C" __global__ void __launch_bounds__(q35hop::kRowThreads) q35hop_publish(Args a) { q35hop_kernels::publish_kernel(a); @@ -500,3 +257,11 @@ extern "C" __global__ void __launch_bounds__(q35hop::kRowThreads) q35hop_consume extern "C" __global__ void __launch_bounds__(q35hop::kRowThreads) q35hop_consume_w8(Args a) { q35hop_kernels::consume_kernel<8>(a); } + +extern "C" __global__ void __launch_bounds__(256, 1) q35hop_oproj_push_s4(OprojArgs o) { + q35hop_kernels::oproj_push_kernel<4>(o); +} + +extern "C" __global__ void __launch_bounds__(256, 1) q35hop_oproj_push_s8(OprojArgs o) { + q35hop_kernels::oproj_push_kernel<8>(o); +} diff --git a/sgl_nvidia_mega/qwen3_5/sm103/hop/src/q35_hop.cuh b/sgl_nvidia_mega/qwen3_5/sm103/hop/src/q35_hop.cuh new file mode 100644 index 00000000..9614f84f --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/hop/src/q35_hop.cuh @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// The decode hop's device protocol (namespace q35hop): the mailbox site, the stage protocol, the +// producer's push of a bf16 fragment and the consumer's rank-order pull and row finish. hop.cu +// builds the stand-alone hop launches on it; a kernel that pushes its own partial into the hop +// from its epilogue includes it too (NVRTC header "q35_hop.cuh", sm103/hop.headers()) and pushes +// each fragment with push_frag(site, stage_peek(site), token, frag, v) after griddepcontrol.wait. +// Includers that typedef the fixed-width integers themselves must use these same types. + +#pragma once + +#include + +// NVRTC has no +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; + +#ifndef Q35_HOP_DEADLINE_NS +#define Q35_HOP_DEADLINE_NS 0ull +#endif + +namespace q35hop { + + +constexpr int kHidden = 4096; +constexpr int kVec = 8; +constexpr int kFrags = kHidden / kVec; +constexpr int kStages = 3; +constexpr int kRowCtas = 4; +constexpr int kRowThreads = 128; +constexpr int kRowWarps = kRowThreads / 32; +constexpr uint32_t kNegZeroPair = 0x80008000u; +static_assert(kRowCtas * kRowThreads == kFrags, "a row is one fragment per thread of the cluster"); +static_assert(kRowWarps == 4, "row_sumsq_fold folds exactly four warp sums per CTA"); + +// Per-process control block in device memory. +struct Ctrl { + uint32_t stage; // stage of the next hop + uint32_t arrive; // CTA tickets of the running consumer launch + uint32_t seq; // hops completed + uint32_t pad; +}; + +// Passed by value as a kernel parameter. +struct Site { + void* mc; // multicast base of the mailbox [kStages][world][m_cap][kHidden] bf16 + void* uc; // this rank's unicast view of the same mailbox + Ctrl* ctrl; + int rank, world, m_cap; +}; + +// ---------------------------------------------------------------- PTX + +__device__ __forceinline__ void pdl_wait() { asm volatile("griddepcontrol.wait;" ::: "memory"); } +__device__ __forceinline__ void pdl_trigger() { asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); } +__device__ __forceinline__ unsigned long long gtimer() { + unsigned long long t; + asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t)); + return t; +} +__device__ __forceinline__ uint32_t ld_volatile_u32(const uint32_t* p) { + uint32_t v; + asm volatile("ld.volatile.global.u32 %0, [%1];" : "=r"(v) : "l"(p) : "memory"); + return v; +} +__device__ __forceinline__ uint4 ld_volatile_v4(const void* p) { + uint4 v; + asm volatile("ld.volatile.global.v4.u32 {%0,%1,%2,%3}, [%4];" + : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) + : "l"(p) + : "memory"); + return v; +} +__device__ __forceinline__ void st_v4(void* p, uint4 v) { + asm volatile("st.global.v4.u32 [%0], {%1,%2,%3,%4};" ::"l"(p), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w) + : "memory"); +} +__device__ __forceinline__ void mc_st_v4(void* mc, uint4 v) { + asm volatile("multimem.st.relaxed.sys.global.v4.bf16x2 [%0], {%1,%2,%3,%4};" ::"l"(mc), "r"(v.x), + "r"(v.y), "r"(v.z), "r"(v.w) + : "memory"); +} +__device__ __forceinline__ void mc_st_v2(void* mc, uint2 v) { + asm volatile("multimem.st.relaxed.sys.global.v2.bf16x2 [%0], {%1,%2};" ::"l"(mc), "r"(v.x), "r"(v.y) : "memory"); +} +__device__ __forceinline__ uint32_t cluster_ctarank() { + uint32_t r; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(r)); + return r; +} +__device__ __forceinline__ float div_full(float a, float b) { + float r; + asm("div.full.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} +// Two f32 to an e4m3 pair; lo lands in the low byte. +__device__ __forceinline__ uint16_t cvt_e4m3x2(float lo, float hi) { + uint16_t r; + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(r) : "f"(hi), "f"(lo)); + return r; +} + +// ---------------------------------------------------------------- bf16 + +__device__ __forceinline__ float bf_lo(uint32_t w) { return __uint_as_float(w << 16); } +__device__ __forceinline__ float bf_hi(uint32_t w) { return __uint_as_float(w & 0xFFFF0000u); } +__device__ __forceinline__ uint32_t pack_bf16x2(float lo, float hi) { + __nv_bfloat162 v = __floats2bfloat162_rn(lo, hi); + return *reinterpret_cast(&v); +} +__device__ __forceinline__ void unpack8(uint4 v, float f[8]) { + f[0] = bf_lo(v.x); f[1] = bf_hi(v.x); f[2] = bf_lo(v.y); f[3] = bf_hi(v.y); + f[4] = bf_lo(v.z); f[5] = bf_hi(v.z); f[6] = bf_lo(v.w); f[7] = bf_hi(v.w); +} +__device__ __forceinline__ uint4 pack8(const float f[8]) { + return make_uint4(pack_bf16x2(f[0], f[1]), pack_bf16x2(f[2], f[3]), pack_bf16x2(f[4], f[5]), + pack_bf16x2(f[6], f[7])); +} +__device__ __forceinline__ bool word_has_negzero(uint32_t w) { + return (w & 0xFFFFu) == 0x8000u || (w >> 16) == 0x8000u; +} +__device__ __forceinline__ bool frag_has_negzero(uint4 v) { + return word_has_negzero(v.x) | word_has_negzero(v.y) | word_has_negzero(v.z) | word_has_negzero(v.w); +} +__device__ __forceinline__ uint32_t sanitize_word(uint32_t w) { + if ((w & 0xFFFFu) == 0x8000u) w &= 0xFFFF0000u; + if ((w >> 16) == 0x8000u) w &= 0x0000FFFFu; + return w; +} + +// ---------------------------------------------------------------- stage protocol + +__device__ __forceinline__ size_t slot(const Site& s, uint32_t stage, int src, int token, int frag) { + return ((static_cast(stage) * s.world + src) * s.m_cap + token) * kHidden + + static_cast(frag) * kVec; +} + +struct Hop { + uint32_t stage; + uint32_t seq; + uint32_t ticket; // thread 0 only +}; + +// Consumer launch, every thread of every participating CTA, after griddepcontrol.wait. Thread +// 0 takes a ticket whose value is consumed only in hop_leave, so the atomic stays off the +// critical path. +__device__ __forceinline__ Hop hop_enter(const Site& s) { + Hop h; + h.stage = ld_volatile_u32(&s.ctrl->stage); + h.seq = ld_volatile_u32(&s.ctrl->seq); + h.ticket = threadIdx.x == 0 ? atomicAdd(&s.ctrl->arrive, 1u) : 0u; + return h; +} + +// Thread 0 of every participating CTA; the last arrival advances the stage. No CTA of this +// launch can read the advanced value, since every CTA read the stage before taking its ticket. +__device__ __forceinline__ void hop_leave(const Site& s, const Hop& h, uint32_t participants) { + if (threadIdx.x == 0 && h.ticket == participants - 1) { + s.ctrl->arrive = 0; + s.ctrl->stage = (h.stage + 1) % kStages; + s.ctrl->seq = h.seq + 1; + } +} + +// Producer-only launch, after griddepcontrol.wait. +__device__ __forceinline__ uint32_t stage_peek(const Site& s) { return ld_volatile_u32(&s.ctrl->stage); } + +// ---------------------------------------------------------------- producer + +// This rank's final bf16 partial for one fragment; exactly once per (hop, rank, token, frag). +// A launch that also stores its own outputs should push first: the push starts the flight. +__device__ __forceinline__ void push_frag(const Site& s, uint32_t stage, int token, int frag, uint4 v) { + v = make_uint4(sanitize_word(v.x), sanitize_word(v.y), sanitize_word(v.z), sanitize_word(v.w)); + mc_st_v4(static_cast(s.mc) + slot(s, stage, s.rank, token, frag), v); +} + +// Same for 4 consecutive bf16 (half a fragment, columns [4q, 4q + 4)); a fragment may be pushed +// as two halves by different threads. +__device__ __forceinline__ void push_quad(const Site& s, uint32_t stage, int token, int quad, uint2 v) { + v = make_uint2(sanitize_word(v.x), sanitize_word(v.y)); + mc_st_v2(static_cast(s.mc) + slot(s, stage, s.rank, token, quad >> 1) + (quad & 1) * 4, v); +} + +// ---------------------------------------------------------------- consumer + +// A peer that missed the wait deadline: store to address 0x0, then trap (fail-stop). +__device__ __forceinline__ void fault_at_null() { + asm volatile("{\n.reg .u64 a;\nmov.u64 a, 0;\nst.volatile.global.u32 [a], 0;\ntrap;\n}" ::: "memory"); +} + +// Sum over ranks 0..W-1 (fp32, from 0) of one fragment; spins on the sentinel. A wait past +// Q35_HOP_DEADLINE_NS (if not 0) stops the kernel. +template +__device__ __forceinline__ bool pull_frag(const Site& s, uint32_t stage, int token, int frag, float acc[8]) { + const uint16_t* uc = static_cast(s.uc); + uint4 v[W]; +#pragma unroll + for (int r = 0; r < W; ++r) v[r] = ld_volatile_v4(uc + slot(s, stage, r, token, frag)); + uint32_t pending = 0; +#pragma unroll + for (int r = 0; r < W; ++r) pending |= frag_has_negzero(v[r]) ? (1u << r) : 0u; + unsigned long long t0 = 0; + uint32_t spins = 0; + while (pending) { +#pragma unroll + for (int r = 0; r < W; ++r) + if (pending & (1u << r)) v[r] = ld_volatile_v4(uc + slot(s, stage, r, token, frag)); + uint32_t still = 0; +#pragma unroll + for (int r = 0; r < W; ++r) + if ((pending & (1u << r)) && frag_has_negzero(v[r])) still |= 1u << r; + pending = still; + if (Q35_HOP_DEADLINE_NS != 0 && pending && ((++spins & 255u) == 0)) { + const unsigned long long now = gtimer(); + if (t0 == 0) { + t0 = now; + } else if (now - t0 > Q35_HOP_DEADLINE_NS) { + fault_at_null(); + } + } + } +#pragma unroll + for (int e = 0; e < 8; ++e) acc[e] = 0.f; +#pragma unroll + for (int r = 0; r < W; ++r) { + float f[8]; + unpack8(v[r], f); +#pragma unroll + for (int e = 0; e < 8; ++e) acc[e] = __fadd_rn(acc[e], f[e]); + } + return true; +} + +// Write the sentinel back into the W slots this thread consumed (any time after pull_frag). +template +__device__ __forceinline__ void rearm_frag(const Site& s, uint32_t stage, int token, int frag) { + uint16_t* uc = static_cast(s.uc); + const uint4 z = make_uint4(kNegZeroPair, kNegZeroPair, kNegZeroPair, kNegZeroPair); +#pragma unroll + for (int r = 0; r < W; ++r) st_v4(uc + slot(s, stage, r, token, frag), z); +} + +// ---------------------------------------------------------------- row finish + +// prenorm = bf16(acc + residual); pre[] gets the bf16-rounded values the norm uses. +__device__ __forceinline__ uint4 residual_add(const float acc[8], uint4 residual, float pre[8]) { + float r[8]; + unpack8(residual, r); +#pragma unroll + for (int e = 0; e < 8; ++e) pre[e] = __fadd_rn(acc[e], r[e]); + const uint4 out = pack8(pre); + unpack8(out, pre); + return out; +} + +__device__ __forceinline__ float sumsq8(const float x[8]) { + float s = 0.f; +#pragma unroll + for (int e = 0; e < 8; ++e) s = __fadd_rn(s, __fmul_rn(x[e], x[e])); + return s; +} + +struct RowSmem { + float xs[kRowCtas][kRowWarps]; + uint64_t mbar; +}; + +// Prologue of every CTA of the row cluster, all threads, before griddepcontrol.wait: arm the +// exchange barrier and make it visible to the peers before any of them can send. +__device__ __forceinline__ void row_init(RowSmem& rs) { + if (threadIdx.x == 0) { + const uint32_t a = static_cast(__cvta_generic_to_shared(&rs.mbar)); + asm volatile("mbarrier.init.shared::cta.b64 [%0], 1;" ::"r"(a) : "memory"); + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(a), + "r"(static_cast(sizeof(rs.xs))) + : "memory"); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); + } + asm volatile("barrier.cluster.arrive.relaxed.aligned;\n\tbarrier.cluster.wait.acquire.aligned;" ::: "memory"); +} + +// Row total of the per-thread values: lane butterfly per warp, then each warp sends its sum to +// every CTA of the cluster with st.async; each CTA waits on its own mbarrier (no release fence, +// so this thread's earlier remote stores are not waited for) and folds per CTA as +// (w0 + w2) + (w1 + w3), then the CTAs in rank order from 0. +__device__ __forceinline__ float row_sumsq_fold(float v, RowSmem& rs) { + const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5; +#pragma unroll + for (int off = 1; off < 32; off <<= 1) v = __fadd_rn(v, __shfl_xor_sync(0xffffffffu, v, off)); + if (lane < kRowCtas) { + const uint32_t me = cluster_ctarank(); + const uint32_t slot_l = static_cast(__cvta_generic_to_shared(&rs.xs[me][warp])); + const uint32_t mbar_l = static_cast(__cvta_generic_to_shared(&rs.mbar)); + uint32_t slot_r, mbar_r; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(slot_r) : "r"(slot_l), "r"(lane)); + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(mbar_r) : "r"(mbar_l), "r"(lane)); + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.f32 [%0], %1, [%2];" ::"r"(slot_r), + "f"(v), "r"(mbar_r) + : "memory"); + } + { + const uint32_t a = static_cast(__cvta_generic_to_shared(&rs.mbar)); + asm volatile( + "{\n\t.reg .pred P;\n\tWAIT_ROW:\n\t" + "mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P, [%0], 0;\n\t" + "@!P bra WAIT_ROW;\n\t}" ::"r"(a) + : "memory"); + } + float total = 0.f; +#pragma unroll + for (int c = 0; c < kRowCtas; ++c) + total = __fadd_rn(total, __fadd_rn(__fadd_rn(rs.xs[c][0], rs.xs[c][2]), __fadd_rn(rs.xs[c][1], rs.xs[c][3]))); + return total; +} + +__device__ __forceinline__ float inv_rms(float row_sumsq, float eps) { + return rsqrtf(__fadd_rn(row_sumsq / static_cast(kHidden), eps)); +} + +// bf16((x * inv) * gamma), gamma being the applied multiplier. +__device__ __forceinline__ uint4 norm_apply(const float pre[8], float inv, const float gamma[8]) { + float out[8]; +#pragma unroll + for (int e = 0; e < 8; ++e) out[e] = __fmul_rn(__fmul_rn(pre[e], inv), gamma[e]); + return pack8(out); +} + +// The reciprocal the static per-tensor FP8 quant multiplies by. +__device__ __forceinline__ float inv_scale_of(float scale) { return div_full(1.f, scale); } + +// e4m3(clamp(x * inv_scale, +-448)) of the bf16 norm output, 8 bytes. +__device__ __forceinline__ uint2 quant_fp8(uint4 norm_bf16, float inv_scale) { + float q[8]; + unpack8(norm_bf16, q); + uint32_t w[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const float a = fminf(fmaxf(__fmul_rn(q[2 * j], inv_scale), -448.f), 448.f); + const float b = fminf(fmaxf(__fmul_rn(q[2 * j + 1], inv_scale), -448.f), 448.f); + w[j] = cvt_e4m3x2(a, b); + } + return make_uint2(w[0] | (w[1] << 16), w[2] | (w[3] << 16)); +} + +} // namespace q35hop diff --git a/sgl_nvidia_mega/qwen3_5/spec.py b/sgl_nvidia_mega/qwen3_5/spec.py index 454ea66b..1f0dd44d 100644 --- a/sgl_nvidia_mega/qwen3_5/spec.py +++ b/sgl_nvidia_mega/qwen3_5/spec.py @@ -17,7 +17,7 @@ Requirements, ) -from sgl_nvidia_mega.qwen3_5 import TEXT_CONFIG +from sgl_nvidia_mega.qwen3_5 import TEXT_CONFIG, runtime from sgl_nvidia_mega.qwen3_5.plan import MAX_ROWS # The SGLang names this module reads (test/unit/test_contract.py pins them) @@ -31,6 +31,7 @@ "Numerics", "NumericsKind", "BITWISE", + "TOLERANCE", "Requirements", "decode_graph_backends", ), @@ -92,9 +93,31 @@ ) # Against SGLang's forward on the same GPU (test/e2e/test_qwen3_5_stock_parity.py): the -# stock step and every slot (route, mkm, gdn_verify, gdn_pdl, hop) reproduce it bit -# for bit -NUMERICS = Numerics(kind=NumericsKind.BITWISE) +# stock step and the bitwise slots (route, mkm, gdn_verify, gdn_pdl, hop) reproduce it +# bit for bit. A numeric slot's outputs are held to fp64 on the same quantized operands +# (test/kernels), at most these multiples of the error of SGLang's own op. +NUMERIC_TOLERANCES = { + "gdn_push": { + "gdn_push.out_proj.rel_l2_vs_stock": 1.05, + "gdn_push.out_proj.max_abs_vs_stock": 2.0, + }, +} + + +def numerics(slots) -> Numerics: + """The claim for the requested ``slots``: bitwise unless a numeric slot is on.""" + tolerances = { + k: v + for slot, held in NUMERIC_TOLERANCES.items() + if slot in slots + for k, v in held.items() + } + if not tolerances: + return Numerics(kind=NumericsKind.BITWISE) + return Numerics(kind=NumericsKind.TOLERANCE, tolerances=tolerances) + + +NUMERICS = numerics(runtime.SLOTS) def text_config_reasons(hf_config) -> list[str]: @@ -120,8 +143,6 @@ def binding_reasons() -> list[str]: def _config_disable_reasons(cfg, hf_config) -> list[str]: - from sgl_nvidia_mega.qwen3_5 import runtime - return text_config_reasons(hf_config) + binding_reasons() + runtime.slot_reasons() diff --git a/test/kernels/test_qwen3_5_hop.py b/test/kernels/test_qwen3_5_hop.py index c7dd7da5..39993e8d 100644 --- a/test/kernels/test_qwen3_5_hop.py +++ b/test/kernels/test_qwen3_5_hop.py @@ -10,6 +10,11 @@ rank in turn; the static FP8 output equals SGLang's static quantization of the norm; - the same operations replayed from a CUDA graph with fresh inputs. +And the GatedDeltaNet out_proj push (slot gdn_push, numeric): its partial against an +fp64 reference on the same FP8 operands, at most 1.05 x the mean relative L2 and 2 x the +max abs error of FlashInfer's BMM (SGLang's op), and its all-reduce, which only consumes, +bit for bit the fusion's on the same partial. + Run with ``python test/kernels/test_qwen3_5_hop.py --num-gpu 4``. """ @@ -176,5 +181,65 @@ def test_hop_replays_from_a_cuda_graph(rank, pair): torch.cuda.synchronize() +def _fp8(rows, cols, device, gen, *, outliers=False, scale=1.0): + x = torch.randn(rows, cols, device=device, generator=gen) * scale + if outliers: + x[:, :: cols // 8] *= 32 # a few large channels, inside e4m3's range + return x.clamp(-440, 440).to(torch.float8_e4m3fn) + + +def _errors(got, ref): + diff = got.double() - ref + return (diff.norm() / ref.norm()).item(), diff.abs().max().item() + + +def test_out_proj_push_holds_to_fp64_and_its_all_reduce_only_consumes(rank, pair): + from sglang.srt.layers.quantization.fp8_utils import flashinfer_bmm_fp8 + + from sgl_nvidia_mega.qwen3_5.sm103.hop import pack_out_proj + + fusion, hop = pair + device = torch.device(f"cuda:{torch.cuda.current_device()}") + k = 1024 # out_proj's input per rank at TP8 + errors = {"push": [], "stock": []} + with torch.inference_mode(): + for seed in range(3): + gen = torch.Generator(device=device).manual_seed(1000 * seed + rank) + w = _fp8(HIDDEN, k, device, gen, scale=0.5) + packed = pack_out_proj(w) + sw = torch.tensor([0.01], device=device) + sx = torch.tensor([0.02], device=device) + for m in (1, 2, 7): + for outliers in (False, True): + x = _fp8(m, k, device, gen, outliers=outliers) + partial = hop.out_proj_push( + x_fp8=x, packed_w=packed, input_scale=sx, weight_scale=sw + ) + x2 = _operands(m, device, rank, seed=seed) + got = hop.all_reduce_residual_rms_norm( + local_contribution=partial, + residual=x2["residual"], + gamma=x2["gamma"], + ) + _same(got, _run(fusion, "all_reduce", dict(x2, partial=partial))) + ref = (x.double() @ w.double().t()) * (sx.double() * sw.double()) + stock = flashinfer_bmm_fp8(x, w.t(), sx, sw, torch.bfloat16) + errors["push"].append(_errors(partial, ref)) + errors["stock"].append(_errors(stock, ref)) + torch.cuda.synchronize() + push, stock = ( + torch.tensor(errors[n], dtype=torch.float64) for n in ("push", "stock") + ) + print( + f"T0 gdn_push out_proj rank {rank}: {len(push)} cases, mean rel-L2 " + f"{push[:, 0].mean():.4g} vs stock {stock[:, 0].mean():.4g}, max abs " + f"{push[:, 1].max():.4g} vs stock {stock[:, 1].max():.4g}", + flush=True, + ) + # the numeric-slot protocol's T0 rule + assert push[:, 0].mean() <= 1.05 * stock[:, 0].mean() + assert push[:, 1].max() <= 2 * stock[:, 1].max() + + if __name__ == "__main__": multigpu_pytest_main(__name__, __file__, num_gpus=(4,)) diff --git a/test/unit/test_qwen3_5_spec.py b/test/unit/test_qwen3_5_spec.py index 138a520c..0bed03b5 100644 --- a/test/unit/test_qwen3_5_spec.py +++ b/test/unit/test_qwen3_5_spec.py @@ -130,6 +130,10 @@ def test_an_sglang_without_the_eagle_forward_view_is_refused(monkeypatch): ({"verify_sample"}, "'verify_sample' is no longer an entry: drop it"), ({"mkm"}, "slot 'mkm' runs on slot 'route''s outputs"), ({"gdn_pdl"}, "slot 'gdn_pdl' launches slot 'gdn_verify''s verify"), + ( + {"gdn_verify", "gdn_push"}, + "slot 'gdn_push' pushes slot 'gdn_verify''s out_proj into slot 'hop'", + ), ], ) def test_a_slot_set_that_cannot_run_is_refused_before_the_load( @@ -151,5 +155,20 @@ def test_the_slot_plan_covers_one_request_of_7_rows(): 7: plan.Slots(route=True, mkm=True) } assert plan.plan((7,), plan.Facts(hop=True)) == {7: plan.Slots(hop=True)} + # gdn_push pushes gdn_verify's out_proj into the hop: it needs both built + both = dict(gdn_verify=True, hop=True) + assert plan.plan((7,), plan.Facts(**both, gdn_push=True)) == { + 7: plan.Slots(**both, gdn_push=True) + } + assert not plan.slots(7, plan.Facts(gdn_verify=True, gdn_push=True)).gdn_push with pytest.raises(ValueError): plan.slots(plan.MAX_ROWS + 1, plan.Facts()) + + +def test_a_numeric_slot_turns_the_claim_into_its_tolerances(): + from sglang.srt.megakernels.spec import NumericsKind + + assert spec.numerics(frozenset({"route", "hop"})).kind is NumericsKind.BITWISE + claim = spec.numerics(frozenset({"gdn_verify", "hop", "gdn_push"})) + assert claim.kind is NumericsKind.TOLERANCE + assert dict(claim.tolerances) == spec.NUMERIC_TOLERANCES["gdn_push"] diff --git a/test/unit/test_qwen3_5_steps.py b/test/unit/test_qwen3_5_steps.py index 7947fbd5..5db30d70 100644 --- a/test/unit/test_qwen3_5_steps.py +++ b/test/unit/test_qwen3_5_steps.py @@ -621,12 +621,24 @@ def _gdn_verify_setup(log, *, ring_len=16, temporal_dtype=torch.bfloat16, draft= return verify, gdn, st -def test_the_gdn_verify_slot_runs_the_backend_chain_on_the_forward_views_rows(): +@pytest.mark.parametrize("push", [False, True]) +def test_the_gdn_verify_slot_runs_the_backend_chain_on_the_forward_views_rows(push): + """With ``push`` (slot gdn_push) the norm releases its dependents early and + out_proj runs as the hop's push GEMM on the layer's packed weights.""" + import msgspec + from sgl_nvidia_mega.qwen3_5.gdn import conv, norm_fp8, replayssm log = _log() verify, gdn, st = _gdn_verify_setup(log) k = log.known + k["packed2"] = torch.zeros(16, dtype=torch.uint8) + hop = SimpleNamespace(out_proj_push=log.op("out_proj_push", torch.zeros(7, 8))) + st.bound = { + "boundary": SimpleNamespace(hop=hop), + "gdn": SimpleNamespace(packed={2: k["packed2"]}), + } + opts = msgspec.structs.replace(verify.SLOT, push=push) def views(qkvz, ba, nk, nv, hk, hv): log.append(("views", log.tag(qkvz), log.tag(ba), nk, nv, hk, hv)) @@ -715,8 +727,9 @@ def symmetric(group, disabled=False): ), ): for _ in range(2): # two layers of one step - verify.step(gdn, torch.zeros(7, 8), st) - one_layer = ["conv", "verify", "norm_fp8", "symmetric", "bmm_fp8", "views"] + verify.step(gdn, torch.zeros(7, 8), st, opts) + proj = ["out_proj_push"] if push else ["symmetric", "bmm_fp8"] + one_layer = ["conv", "verify", "norm_fp8", *proj, "views"] assert [e[0] for e in log] == ["views", *one_layer, *one_layer[:-1]] _, shape, state, states, window, inter, kw = log[1] assert (shape, states, inter) == ((1, 32, 7), "states(1,)", "inter(1,)") @@ -747,8 +760,22 @@ def symmetric(group, disabled=False): (28, 4), (7, 4, 4), "scale(1,)", - (("activation", "silu"),), + (("activation", "silu"), ("early_dependents", push)), ) + if push: + # the push GEMM on the FP8 rows and the layer's packed weights and scales + assert log[4] == ( + "out_proj_push", + "main", + (), + ( + ("input_scale", "scale(1,)"), + ("packed_w", "packed2(16,)"), + ("weight_scale", "out_s(1,)"), + ("x_fp8", "?(7, 16)"), + ), + ) + return # out_proj's BMM on the FP8 rows with its static input scale, output in the activation dtype assert log[5] == ( "bmm_fp8", @@ -758,6 +785,32 @@ def symmetric(group, disabled=False): ) +@pytest.mark.parametrize("takes", [False, True]) +def test_the_gdn_push_slot_pushes_only_where_the_moe_input_consumes( + monkeypatch, caplog, takes +): + import msgspec + + from sgl_nvidia_mega.qwen3_5 import gdn + from sgl_nvidia_mega.qwen3_5.gdn import verify + + monkeypatch.setattr(verify, "_ENGAGED", set()) + seen = [] + monkeypatch.setattr(boundary_hop, "takes_moe_input", lambda hop, *a: takes) + monkeypatch.setattr(verify, "step", lambda mod, x, st, opts: seen.append(opts)) + layer = SimpleNamespace(linear_attn="gdn") + st = SimpleNamespace( + slots=plan.Slots(gdn_verify=True, hop=True, gdn_push=True), + bound={"boundary": boundary_hop.Bound(hop="hop")}, + ) + with caplog.at_level("INFO"): + gdn.step(layer, torch.zeros(7, 4), st) + assert seen == [msgspec.structs.replace(verify.SLOT, push=takes)] + # the push is logged once, where it is chosen + logged = "NVIDIA megakernel: Qwen3.5 slot gdn_push runs at 7 rows" in caplog.text + assert logged == takes + + @pytest.mark.parametrize( "kw, want", [ @@ -1091,6 +1144,68 @@ def _same_calls(a, b): assert a[0][0] == b[0][0] and all(x is y for x, y in zip(a[0][1:], b[0][1:])) +def test_out_proj_packs_into_the_push_gemms_fragments(): + from sgl_nvidia_mega.qwen3_5.sm103 import hop + + w = torch.arange(4096 * 1024, dtype=torch.int64).remainder(251).to(torch.uint8) + packed = hop.pack_out_proj(w.view(4096, 1024).view(torch.float8_e4m3fn)) + assert packed.dtype == torch.uint8 and packed.numel() == 4096 * 1024 + # CTA 0, warp 0, k-step 0, block 0, lane 0: rows 0 and 8, K bytes 0-3 and 16-19 + frag = packed[:16].view(4, 4) + want = w.view(4096, 1024) + assert torch.equal(frag[0], want[0, 0:4]) and torch.equal(frag[1], want[8, 0:4]) + assert torch.equal(frag[2], want[0, 16:20]) and torch.equal(frag[3], want[8, 16:20]) + with pytest.raises(ValueError, match="out_proj packing"): + hop.pack_out_proj(torch.zeros(4096, 512, dtype=torch.float8_e4m3fn)) + + +def _pushing_hop(log): + """sm103's Hop without its mailbox: the launches it would issue, logged.""" + from sgl_nvidia_mega.qwen3_5.sm103 import hop as sm103_hop + + hop = object.__new__(sm103_hop.Hop) + hop.max_m, hop._pushed = 8, None + hop._args = lambda m, residual, gamma, scale: (SimpleNamespace(m=m), ("out",)) + hop._launch = lambda name, args, ctas, cluster=None: log.append(name) + hop._consume = lambda args: log.append("consume") + return hop + + +def test_a_pushed_partials_all_reduce_only_consumes(): + log = [] + hop = _pushing_hop(log) + pushed, other = torch.zeros(7, 16), torch.zeros(7, 16) + hop.mark_pushed(pushed) + assert hop.has_pushed(pushed) and not hop.has_pushed(other) + for partial in (pushed, other): # the push consumed, the next one publishes + hop.all_reduce_residual_rms_norm( + local_contribution=partial, residual="r", gamma="g" + ) + assert log == ["consume", "q35hop_publish", "consume"] + assert hop._pushed is None + + +def test_a_push_left_unconsumed_stops_the_next_hop(): + hop = _pushing_hop([]) + pushed, other = torch.zeros(7, 16), torch.zeros(7, 16) + hop.mark_pushed(pushed) + with pytest.raises(RuntimeError, match="never consumed"): + hop.mark_pushed(other) + with pytest.raises(RuntimeError, match="never consumed"): + hop.all_reduce_residual_rms_norm( + local_contribution=other, residual="r", gamma="g" + ) + with pytest.raises(RuntimeError, match="never consumed"): + hop.moe_finalize_all_reduce_rms_norm( + routed_output=None, + expert_weights=None, + permuted_indices=torch.zeros(7, 10, dtype=torch.int32), + gated_shared_output=None, + residual="r", + gamma="g", + ) + + @pytest.mark.parametrize("point", ["layer input", "MoE input", "final norm"]) def test_the_hop_slot_runs_where_sglang_runs_its_cutedsl_kernel(cutedsl_gates, point): """At each point, SGLang's stage boundary with the CuTe DSL workspace and the From c696d4575be8118a3e05ff2c62b7c647026c1a9e Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:40:00 -0700 Subject: [PATCH 03/27] [Perf] [GB300] qwen3_5: Slot gdn_front: the GatedDeltaNet projections and conv in one launch (numeric) One launch of 11 clusters runs the static-FP8 in_proj_qkvz and BF16 in_proj_ba projections, the conv update and an L2 prefetch of the verify's state; the ReplaySSM verify and the gated norm into out_proj's FP8 input follow as programmatic dependents. Slot gdn_front (with gdn_verify and gdn_pdl, one request of 7 rows); numeric: the projections sum in another order than SGLang's GEMMs. --- README.md | 1 + sgl_nvidia_mega/qwen3_5/gdn/__init__.py | 33 +- sgl_nvidia_mega/qwen3_5/gdn/front.py | 294 +++++++++++ sgl_nvidia_mega/qwen3_5/gdn/front_norm.py | 243 +++++++++ sgl_nvidia_mega/qwen3_5/gdn/replayssm.py | 24 +- sgl_nvidia_mega/qwen3_5/plan.py | 12 +- sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 6 + sgl_nvidia_mega/qwen3_5/runtime/executor.py | 9 + sgl_nvidia_mega/qwen3_5/sm103/gdn/__init__.py | 3 + sgl_nvidia_mega/qwen3_5/sm103/gdn/front.py | 238 +++++++++ .../qwen3_5/sm103/gdn/src/front.cu | 487 ++++++++++++++++++ sgl_nvidia_mega/qwen3_5/spec.py | 6 + test/kernels/test_qwen3_5_gdn.py | 221 +++++++- test/unit/test_qwen3_5_spec.py | 14 + test/unit/test_qwen3_5_steps.py | 357 ++++++++++++- 15 files changed, 1933 insertions(+), 15 deletions(-) create mode 100644 sgl_nvidia_mega/qwen3_5/gdn/front.py create mode 100644 sgl_nvidia_mega/qwen3_5/gdn/front_norm.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/gdn/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/gdn/front.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/gdn/src/front.cu diff --git a/README.md b/README.md index bda129ac..e0710675 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ These slots change the arithmetic (numeric: each sums in another order than SGLa | Slot | Rows | Kernel | | --- | --- | --- | | `gdn_push` (with `gdn_verify`, `hop`) | 7 | The GatedDeltaNet `out_proj` as the hop's GEMM, whose epilogue pushes the partial into the hop's mailbox | +| `gdn_front` (with `gdn_verify`, `gdn_pdl`) | 7 (one request) | The GatedDeltaNet `in_proj_qkvz` (static FP8) and `in_proj_ba` (BF16) projections, the conv update and an L2 prefetch of the verify's state in one launch, then the ReplaySSM verify and the gated norm writing `out_proj`'s FP8 input, each launched dependent on the one before | The sampling and acceptance after the verify, and the GatedDeltaNet extend of prefill, stay SGLang's own code. In an SGLang that has the switches, `SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH=1` runs the sampling and acceptance in CUDA graphs, and `SGLANG_ENABLE_GDN_PREFILL_GRAPH_EXTEND=1` captures the GatedDeltaNet extend inside the breakable prefill CUDA graphs. diff --git a/sgl_nvidia_mega/qwen3_5/gdn/__init__.py b/sgl_nvidia_mega/qwen3_5/gdn/__init__.py index edbae058..05f0ea73 100644 --- a/sgl_nvidia_mega/qwen3_5/gdn/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/gdn/__init__.py @@ -11,7 +11,9 @@ gated RMSNorm and ``out_proj``. ``step`` returns the TP-partial output, whose all-reduce the boundary runs. Slot ``gdn_verify`` (``verify.py``) runs the step with the backend's chain opened up and its launch choices changed; slot ``gdn_push`` pushes -``out_proj``'s partial into the decode hop from its GEMM (numeric). +``out_proj``'s partial into the decode hop from its GEMM (numeric), and slot ``gdn_front`` +(``front.py``) runs the projections and the conv of one request of 7 rows in one launch +(numeric). """ from __future__ import annotations @@ -62,6 +64,9 @@ class Bound(msgspec.Struct, frozen=True): gdn_pdl: bool = False # and launches its verify dependent on the conv # slot gdn_push: layer id -> out_proj's weights packed for the hop's push GEMM packed: dict = msgspec.field(default_factory=dict) + # slot gdn_front: layer id -> in_proj_qkvz's weights packed for the front + front: dict = msgspec.field(default_factory=dict) + error: object = None # the front's sticky error word, int32, zero reason: str = "" # why a requested slot is not built @@ -94,7 +99,27 @@ def build(model, ctx) -> Bound: packed[gdn.attn.layer_id] = pack_out_proj(gdn.out_proj.weight.t()) except ValueError as e: return Bound(reason=f"layer {i}: {e}") - return Bound(gdn_verify=True, gdn_pdl="gdn_pdl" in SLOTS, packed=packed) + front_packed, error = {}, None + if "gdn_front" in SLOTS: + import torch + + from sgl_nvidia_mega.qwen3_5.gdn import front + + if ctx.tokens_per_request != front.ROWS: + return Bound(reason=f"gdn_front: {ctx.tokens_per_request} rows a request") + for i, gdn in gdns: + reason = front.unsupported(gdn) + if reason: + return Bound(reason=f"gdn_front: layer {i}: {reason}") + front_packed = front.build(model, gdns) + error = torch.zeros(1, dtype=torch.int32, device="cuda") + return Bound( + gdn_verify=True, + gdn_pdl="gdn_pdl" in SLOTS, + packed=packed, + front=front_packed, + error=error, + ) def step(layer, x, st): @@ -111,6 +136,10 @@ def step(layer, x, st): ): opts = msgspec.structs.replace(opts, push=True) verify.engaged("gdn_push", x.shape[0]) + if st.slots.gdn_front: + from sgl_nvidia_mega.qwen3_5.gdn import front + + return front.step(layer.linear_attn, x, st, opts) return verify.step(layer.linear_attn, x, st, opts) import triton from sglang.kernels.ops.attention.triton_gdn_fused_proj import ( diff --git a/sgl_nvidia_mega/qwen3_5/gdn/front.py b/sgl_nvidia_mega/qwen3_5/gdn/front.py new file mode 100644 index 00000000..8bc4fa88 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/gdn/front.py @@ -0,0 +1,294 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from python/sglang/srt/models/qwen3_5.py (Qwen3_5GatedDeltaNet._try_fused_verify) +# of SGLang branch nvidia/qwen35-agentx-gb300 and SGLang's +# python/sglang/srt/layers/attention/linear/gdn_backend.py (GDNAttnBackend.forward_extend). +"""Slot ``gdn_front`` (numeric): slot gdn_verify's step for one request of 7 rows with +the fused GDN front of SGLang branch nvidia/qwen35-agentx-gb300 (SGLANG_Q35MF_MKG_FUSED + +MKG_BA_TC + MKM_GDN_FRONT): + +1. ``sm103/gdn/front.py`` in one launch: the static-FP8 ``in_proj_qkvz`` projection of + the BF16 rows, the conv update of its q / k / v channels, the BF16 ``in_proj_ba`` + projection, and an L2 prefetch of the verify's checkpoint and history ring; +2. the ReplaySSM verify on the conv output, launched dependent on the front and waiting + for it before it reads the gates b / a (``replayssm.verify``); +3. the gated RMSNorm writing ``out_proj``'s static FP8 input (``front_norm.py``), + released dependents at entry; +4. ``out_proj``: FlashInfer's BMM, or with slot gdn_push the hop's push GEMM. + +The two projections sum in another order than SGLang's GEMMs (numeric); every launch +is the branch's kernel with the branch's launch, bit for bit on the same inputs. +""" + +from __future__ import annotations + +import logging + +import torch + +logger = logging.getLogger(__name__) + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen3_5.py": ( + "in_proj_qkvz", + "in_proj_ba", + "attn_tp_size", + "conv_kernel_size", + "activation", + "head_k_dim", + "head_v_dim", + "attn", + "norm", + "out_proj", + ), + "srt/layers/radix_linear_attention.py": ( + "layer_id", + "conv_weights", + "bias", + "num_k_heads", + "num_v_heads", + "q_dim", + "k_dim", + "v_dim", + ), + "kernels/ops/attention/fla/layernorm_gated.py": ( + "eps", + "weight", + "bias", + "activation", + "group_size", + "norm_before_gate", + ), + "srt/layers/quantization/modelopt_quant.py": ( + "ModelOptFp8LinearMethod", + "use_marlin", + "use_sm120_fp8", + '"input_scale"', + '"weight_scale"', + ), + "srt/layers/quantization/unquant.py": ("UnquantizedLinearMethod",), + "srt/megakernels/eagle_binding.py": ( + "gdn", + "mamba", + "state_indices", + "intermediate_indices", + "pad_slot_id", + "layer_ids", + "conv", + "temporal", + "intermediate_conv_window", + "ring_d", + "ring_k", + "ring_g", + '"ForwardBatch.req_pool_indices"', + ), + "srt/model_executor/forward_batch_info.py": ("req_pool_indices",), +} + +ROWS = 7 # one request of 7 verify rows +CONV_CHANNELS, CONV_WIDTH = 1536, 4 + +# the step widths that have run the slot (each logged once, as engagement evidence) +_ENGAGED: set = set() + + +def _fp8_static(linear, weight_shape) -> bool: + from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp8LinearMethod + + method = linear.quant_method + return ( + isinstance(method, ModelOptFp8LinearMethod) + and not method.use_marlin + and not method.use_sm120_fp8 + and linear.weight.dtype == torch.float8_e4m3fn + and tuple(linear.weight.shape) == weight_shape + and linear.input_scale.numel() == 1 + and linear.weight_scale.numel() == 1 + and linear.input_scale.dtype == torch.float32 + and linear.weight_scale.dtype == torch.float32 + ) + + +def unsupported(gdn) -> str: + """Why the front cannot serve ``gdn`` (a Qwen3_5GatedDeltaNet) ("" when it can): + the TP8 shapes and static FP8 / BF16 projections the branch's kernel is built for.""" + from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod + + attn, norm, ba = gdn.attn, gdn.norm, gdn.in_proj_ba + if gdn.attn_tp_size != 8 or (attn.num_k_heads, attn.num_v_heads) != (2, 8): + return ( + f"TP {gdn.attn_tp_size} with {attn.num_k_heads} / {attn.num_v_heads} heads" + ) + if (gdn.head_k_dim, gdn.head_v_dim) != (128, 128) or gdn.conv_kernel_size != 4: + return "head dims or conv width" + if gdn.activation not in ("silu", "swish") or norm.activation not in ( + "silu", + "swish", + ): + return "an activation other than SiLU" + if ( + not norm.norm_before_gate + or norm.bias is not None + or norm.group_size is not None + ): + return "a grouped, biased or gate-first norm" + if attn.bias is not None: + return "a conv bias" + w = attn.conv_weights + if ( + tuple(w.shape) != (CONV_CHANNELS, CONV_WIDTH) + or w.dtype != torch.bfloat16 + or not w.is_contiguous() + ): + return f"conv weights {w.dtype} {tuple(w.shape)}" + if not _fp8_static(gdn.in_proj_qkvz, (4096, 2560)): + return "in_proj_qkvz is not a static per-tensor FP8 [4096, 2560] linear" + if not _fp8_static(gdn.out_proj, (1024, 4096)): + return "out_proj is not a static per-tensor FP8 [1024, 4096] linear" + if ( + not isinstance(ba.quant_method, UnquantizedLinearMethod) + or ba.bias is not None + or ba.weight.dtype != torch.bfloat16 + or tuple(ba.weight.shape) != (16, 4096) + or not ba.weight.is_contiguous() + ): + return "in_proj_ba is not an unbiased BF16 [16, 4096] linear" + return "" + + +def pool_reasons(pools) -> list[str]: + """Why the front cannot run on the mamba pools (``Step.view.mamba``) ([] when it + can): BF16 conv states [slots, 1536, 3] and deduplicated BF16 windows.""" + reasons = [] + conv, windows = pools.conv[0], pools.intermediate_conv_window[0] + if conv.dtype != torch.bfloat16 or tuple(conv.shape[2:]) != (CONV_CHANNELS, 3): + reasons.append(f"conv states {conv.dtype} {tuple(conv.shape)}") + if ( + windows.dtype != torch.bfloat16 + or tuple(windows.shape[2:]) != (ROWS, CONV_CHANNELS, 3) + or tuple(windows.stride()[2:]) != (1, 9, 1) + or windows.stride(1) < CONV_CHANNELS * 9 + ): + reasons.append( + f"conv windows {tuple(windows.shape)} {tuple(windows.stride())} are not " + "the deduplicated BF16 windows of 7 rows" + ) + return reasons + + +def build(model, gdns) -> dict: + """Each GDN layer's in_proj_qkvz packed for the front, by layer id; compiles the + front.""" + from sgl_nvidia_mega.qwen3_5.sm103.gdn import front + + front.build(torch.cuda.current_device()) + # the method keeps the weight as the [K, N] transpose of a row-major [N, K] + return { + gdn.attn.layer_id: front.pack(gdn.in_proj_qkvz.weight.t()) for _, gdn in gdns + } + + +def step(gdn, x: torch.Tensor, st, opts) -> torch.Tensor: + """``Qwen3_5GatedDeltaNet.forward`` of a target-verify step of 7 rows on the front: + the TP-partial output [7, hidden] of the normed BF16 rows x; ``opts`` is slot + gdn_verify's (the verify's tiles, and ``push`` for slot gdn_push).""" + from sgl_nvidia_mega.qwen3_5.gdn import replayssm + from sgl_nvidia_mega.qwen3_5.gdn.front_norm import norm + from sgl_nvidia_mega.qwen3_5.gdn.verify import _out_proj + from sgl_nvidia_mega.qwen3_5.sm103.gdn import front + + if x.shape[0] not in _ENGAGED: + _ENGAGED.add(x.shape[0]) + logger.info( + "NVIDIA megakernel: Qwen3.5 slot gdn_front runs at %d rows", x.shape[0] + ) + attn, bound = gdn.attn, st.bound["gdn"] + meta, pools = st.view.gdn, st.view.mamba + li = pools.layer_ids.index(attn.layer_id) + projected = x.new_empty((ROWS, 2560)) + conv = x.new_empty((ROWS, CONV_CHANNELS)) + ba = x.new_empty((ROWS, 16)) + front.front( + x, + bound.front[attn.layer_id], + gdn.in_proj_qkvz.input_scale, + gdn.in_proj_qkvz.weight_scale, + attn.conv_weights, + pools.conv[0][li], + pools.intermediate_conv_window[0][li], + meta.state_indices[:1], + meta.intermediate_indices[:1], + projected, + conv, + bound.error, + ba_weight=gdn.in_proj_ba.weight, + ba_out=ba, + checkpoint=pools.temporal[li], + history_d=pools.ring_d[li], + history_k=pools.ring_k[li], + history_g=pools.ring_g[li], + ring_indices=st.fb.req_pool_indices[:1], + pad_slot=meta.pad_slot_id, + ) + query, key, value = torch.split(conv, [attn.q_dim, attn.k_dim, attn.v_dim], dim=-1) + H, K = attn.num_k_heads, attn.head_k_dim + HV, V = attn.num_v_heads, attn.head_v_dim + d_cache = pools.ring_d[li] + core = query.new_empty(ROWS, HV, V) + + def ring(t): + return None if t is None else t[li] + + # b / a in the front's b8 / a8 order + replayssm.verify( + query.reshape(ROWS, H, K), + key.reshape(ROWS, H, K), + value.reshape(ROWS, HV, V), + ba[:, 8:], + ba[:, :8], + attn.A_log, + attn.dt_bias, + core, + pools.temporal[li], + d_cache, + pools.ring_k[li], + pools.ring_g[li], + ring(pools.ring_rawv), + ring(pools.ring_rawk), + ring(pools.ring_beta), + meta.query_start_loc, + meta.state_indices, + st.fb.req_pool_indices, + pools.ring_write_pos, + pools.ring_cache_base, + pools.ring_is_flush, + max_cache_len=d_cache.shape[-2], + max_spec_len=ROWS, + scale=K**-0.5, + null_block_id=-1, + block_v=opts.block_v, + num_warps=opts.num_warps, + pdl=True, + wait_for_gate_producer=True, + ) + quant = torch.empty((ROWS * HV, V), dtype=torch.float8_e4m3fn, device=x.device) + rstd = torch.empty((ROWS * HV,), dtype=torch.float32, device=x.device) + norm( + core.view(ROWS * HV, V), + projected[:, CONV_CHANNELS:].view(ROWS, HV, V), + gdn.norm.weight, + gdn.out_proj.input_scale, + quant, + rstd, + gdn.norm.eps, + ) + x_fp8 = quant.view(ROWS, HV * V) + if opts.push: + return st.bound["boundary"].hop.out_proj_push( + x_fp8=x_fp8, + packed_w=bound.packed[attn.layer_id], + input_scale=gdn.out_proj.input_scale, + weight_scale=gdn.out_proj.weight_scale, + ) + return _out_proj(gdn.out_proj, x_fp8, x.dtype) diff --git a/sgl_nvidia_mega/qwen3_5/gdn/front_norm.py b/sgl_nvidia_mega/qwen3_5/gdn/front_norm.py new file mode 100644 index 00000000..da969eda --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/gdn/front_norm.py @@ -0,0 +1,243 @@ +# SPDX-License-Identifier: Apache-2.0 AND MIT +# SPDX-FileCopyrightText: Copyright (c) 2024, Tri Dao. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# This file has been modified from its original version (a static FP8 store, preallocated +# outputs, and a dependent launch released at entry). +# Adapted from python/sglang/kernels/ops/attention/gdn_pdl_norm.py of SGLang branch +# nvidia/qwen35-agentx-gb300, itself adapted from SGLang's +# kernels/ops/attention/fla/layernorm_gated.py and flash-linear-attention (MIT, Tri Dao). +"""The gated RMSNorm after slot gdn_front's verify, writing out_proj's static FP8 input +into the caller's buffers: ``gdn_pdl_norm`` of SGLang branch nvidia/qwen35-agentx-gb300 +(the kernel and its launch). It releases its programmatic dependents at entry and waits +before it reads, and rounds through BF16 before the static FP8 conversion, as the +unfused norm and quantization do. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "kernels/ops/attention/fla/layernorm_gated.py": ("calc_rows_per_block",), +} + + +@triton.jit +def _gated_norm_early_kernel( + X, # pointer to the input + Y, # pointer to the output + W, # pointer to the weights + B, # pointer to the biases + Z, # pointer to the other branch + Mean, # pointer to the mean + Rstd, # pointer to the 1/std + stride_x_row, # how much to increase the pointer when moving by 1 row + stride_y_row, + stride_z_row, + stride_z_token, + stride_z_head, + M, # number of rows in X + N: tl.constexpr, # number of columns in X + eps, # epsilon to avoid division by zero + BLOCK_N: tl.constexpr, + ROWS_PER_BLOCK: tl.constexpr, + HAS_BIAS: tl.constexpr, + HAS_Z: tl.constexpr, + Z_IS_3D: tl.constexpr, + Z_HEADS: tl.constexpr, + NORM_BEFORE_GATE: tl.constexpr, + IS_RMS_NORM: tl.constexpr, + ACTIVATION: tl.constexpr, + USE_GDC: tl.constexpr = False, + Q=None, + S=None, + QUANT: tl.constexpr = False, + STATIC_QUANT: tl.constexpr = False, +): + if USE_GDC: + tl.extra.cuda.gdc_launch_dependents() + tl.extra.cuda.gdc_wait() + + # Map the program id to the starting row of X and Y it should compute. + row_start = tl.program_id(0) * ROWS_PER_BLOCK + group = tl.program_id(1) + + # Create 2D tile: [ROWS_PER_BLOCK, BLOCK_N] + rows = row_start + tl.arange(0, ROWS_PER_BLOCK) + cols = tl.arange(0, BLOCK_N) + + # Compute offsets for 2D tile + row_offsets = rows[:, None] * stride_x_row + col_offsets = cols[None, :] + group * N + + # Base pointers + X_base = X + row_offsets + col_offsets + Y_base = Y + rows[:, None] * stride_y_row + col_offsets + + # Create mask for valid rows and columns + row_mask = rows[:, None] < M + col_mask = cols[None, :] < N + mask = row_mask & col_mask + + # Load input data with 2D tile + x = tl.load(X_base, mask=mask, other=0.0).to(tl.float32) + + if HAS_Z and not NORM_BEFORE_GATE: + if Z_IS_3D: + z_row_offsets = (rows[:, None] // Z_HEADS) * stride_z_token + ( + rows[:, None] % Z_HEADS + ) * stride_z_head + else: + z_row_offsets = rows[:, None] * stride_z_row + Z_base = Z + z_row_offsets + col_offsets + z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32) + if ACTIVATION == "swish" or ACTIVATION == "silu": + x *= z * tl.sigmoid(z) + elif ACTIVATION == "sigmoid": + x *= tl.sigmoid(z) + + # Compute mean and variance per row (reduce along axis 1) + if not IS_RMS_NORM: + mean = tl.sum(x, axis=1) / N # Shape: [ROWS_PER_BLOCK] + # Store mean for each row + mean_offsets = group * M + rows + mean_mask = rows < M + tl.store(Mean + mean_offsets, mean, mask=mean_mask) + # Broadcast mean back to 2D for subtraction + xbar = tl.where(mask, x - mean[:, None], 0.0) + var = tl.sum(xbar * xbar, axis=1) / N # Shape: [ROWS_PER_BLOCK] + else: + xbar = tl.where(mask, x, 0.0) + var = tl.sum(xbar * xbar, axis=1) / N # Shape: [ROWS_PER_BLOCK] + mean = 0.0 # Placeholder for RMS norm + + rstd = tl.rsqrt(var + eps) # Shape: [ROWS_PER_BLOCK] + + # Store rstd for each row + rstd_offsets = group * M + rows + rstd_mask = rows < M + tl.store(Rstd + rstd_offsets, rstd, mask=rstd_mask) + + # Load weights and biases (broadcast across rows) + w_offsets = cols + group * N + w_mask = cols < N + w = tl.load(W + w_offsets, mask=w_mask, other=0.0).to(tl.float32) + + if HAS_BIAS: + b = tl.load(B + w_offsets, mask=w_mask, other=0.0).to(tl.float32) + + # Normalize and apply linear transformation + if not IS_RMS_NORM: + x_hat = (x - mean[:, None]) * rstd[:, None] + else: + x_hat = x * rstd[:, None] + + y = x_hat * w[None, :] + b[None, :] if HAS_BIAS else x_hat * w[None, :] + + if HAS_Z and NORM_BEFORE_GATE: + if Z_IS_3D: + z_row_offsets = (rows[:, None] // Z_HEADS) * stride_z_token + ( + rows[:, None] % Z_HEADS + ) * stride_z_head + else: + z_row_offsets = rows[:, None] * stride_z_row + Z_base = Z + z_row_offsets + col_offsets + z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32) + if ACTIVATION == "swish" or ACTIVATION == "silu": + y *= z * tl.sigmoid(z) + elif ACTIVATION == "sigmoid": + y *= tl.sigmoid(z) + + if STATIC_QUANT: + y = y.to(Y.dtype.element_ty).to(tl.float32) + y_s = tl.load(S).to(tl.float32) + y_q = tl.clamp(y * (1.0 / y_s), -448.0, 448.0).to(Q.dtype.element_ty) + tl.store(Q + rows[:, None] * stride_y_row + col_offsets, y_q, mask=mask) + elif QUANT: # the block is one token: per-token FP8 as _per_token_group_quant_8bit + y = tl.where(mask, y.to(Y.dtype.element_ty).to(tl.float32), 0.0) + y_s = tl.maximum(tl.max(tl.abs(y)), 1e-10) / 448.0 + y_q = tl.clamp(y * (1.0 / y_s), -448.0, 448.0).to(Q.dtype.element_ty) + tl.store(Q + rows[:, None] * N + col_offsets, y_q, mask=mask) + tl.store(S + tl.program_id(0), y_s) + else: + # Write output + tl.store(Y_base, y, mask=mask) + + +def norm( + x: torch.Tensor, + z: torch.Tensor, + weight: torch.Tensor, + scale: torch.Tensor, + quant_out: torch.Tensor, + rstd: torch.Tensor, + eps: float, +) -> None: + """Write gated RMSNorm followed by static FP8 conversion into caller buffers. + + ``x`` is BF16 with shape ``[tokens * 8, 128]``. The BF16 gate ``z`` + has shape ``[tokens, 8, 128]`` and may have strided token/head axes. + ``weight`` is contiguous with 128 elements, ``scale`` is a device FP32 + scalar, and both destinations are contiguous. The caller must provide + non-overlapping output storage on the same CUDA device. + + This entry point requires programmatic dependent launch support. It + publishes its dependent at entry, then waits before reading activations. + Normalization rounds through BF16 before static FP8 conversion, as in + the unfused operator. No output or workspace is allocated. + """ + assert x.ndim == 2 and x.shape[1] == 128 + assert x.shape[0] > 0 and x.shape[0] % 8 == 0 + assert x.dtype == torch.bfloat16 and x.is_contiguous() + assert z.shape == (x.shape[0] // 8, 8, 128) + assert z.dtype == torch.bfloat16 and z.stride(-1) == 1 + assert weight.shape == (128,) and weight.stride(0) == 1 + assert weight.dtype in (torch.bfloat16, torch.float32) + assert scale.numel() == 1 and scale.dtype == torch.float32 + assert quant_out.shape == x.shape and quant_out.is_contiguous() + assert quant_out.dtype == torch.float8_e4m3fn + assert rstd.shape == (x.shape[0],) and rstd.is_contiguous() + assert rstd.dtype == torch.float32 + assert x.is_cuda and all( + tensor.device == x.device for tensor in (z, weight, scale, quant_out, rstd) + ) + from sglang.kernels.ops.attention.fla.layernorm_gated import calc_rows_per_block + + rows = x.shape[0] + rows_per_block = calc_rows_per_block(rows, x.device) + # Y supplies the activation dtype only; STATIC_QUANT does not write it. + _gated_norm_early_kernel[(triton.cdiv(rows, rows_per_block), 1)]( + x, + x, + weight, + None, + z, + None, + rstd, + x.stride(0), + quant_out.stride(0), + 0, + z.stride(0), + z.stride(1), + rows, + 128, + eps, + BLOCK_N=128, + ROWS_PER_BLOCK=rows_per_block, + HAS_BIAS=False, + HAS_Z=True, + Z_IS_3D=True, + Z_HEADS=8, + NORM_BEFORE_GATE=True, + IS_RMS_NORM=True, + ACTIVATION="swish", + USE_GDC=True, + Q=quant_out, + S=scale, + STATIC_QUANT=True, + num_warps=1, + launch_pdl=True, + ) diff --git a/sgl_nvidia_mega/qwen3_5/gdn/replayssm.py b/sgl_nvidia_mega/qwen3_5/gdn/replayssm.py index 78bc0405..54d42057 100644 --- a/sgl_nvidia_mega/qwen3_5/gdn/replayssm.py +++ b/sgl_nvidia_mega/qwen3_5/gdn/replayssm.py @@ -11,7 +11,10 @@ The grid starts while the causal conv before it finishes, reads the gates and the committed history, and waits for the conv's q / k / v only where it first reads them (PTX ``griddepcontrol``). The gates and the history must be ready when it starts: -they come from launches before the conv. +they come from launches before the conv. Where the launch before it also writes the +gates b / a (slot gdn_front's verify front), ``wait_for_gate_producer`` moves the wait +before the gates' reads, as the kernel on SGLang branch nvidia/qwen35-agentx-gb300 does; +the arithmetic is the same. """ from __future__ import annotations @@ -95,6 +98,7 @@ def gdn_replayssm_spec_circular_kernel( NULL_BLOCK_ID: tl.constexpr, DOT_PRECISION: tl.constexpr, USE_GDC: tl.constexpr = False, + WAIT_FOR_GATE_PRODUCER: tl.constexpr = False, ): i_v = tl.program_id(0) i_n = tl.program_id(1) @@ -170,6 +174,8 @@ def gdn_replayssm_spec_circular_kernel( # ------------------------------------------------------------------ A_log_val = tl.load(A_log + i_hv).to(tl.float32) dt_bias_val = tl.load(dt_bias + i_hv).to(tl.float32) + if USE_GDC and WAIT_FOR_GATE_PRODUCER: + tl.extra.cuda.gdc_wait() a_s = tl.load(a + (bos + o_s_safe) * stride_a_t + i_hv, mask=mask_s, other=0.0).to( tl.float32 ) @@ -215,7 +221,7 @@ def gdn_replayssm_spec_circular_kernel( b_d_scaled = b_d_scaled_fp32.to(d_cache.dtype.element_ty) # PTX griddepcontrol: consume the convolution's q / k / v only after it completes - if USE_GDC: + if USE_GDC and not WAIT_FOR_GATE_PRODUCER: tl.extra.cuda.gdc_wait() if USE_QK_L2NORM_IN_KERNEL: @@ -512,9 +518,11 @@ def verify( nk: int = 2, bs_min: int = 4, pdl: bool = False, + wait_for_gate_producer: bool = False, ) -> torch.Tensor: """``gdn_replayssm_spec_decode(launch_mode="verify", use_qk_l2norm_in_kernel=True)`` - with an optional dependent launch (``pdl``).""" + with an optional dependent launch (``pdl``), whose wait comes before the gates' + reads with ``wait_for_gate_producer`` (the launch before it writes b / a).""" from sglang.kernels.ops.attention.fla.utils import is_tf32_supported if is_flush.dtype != torch.int8: @@ -536,7 +544,15 @@ def verify( rawv_buf = rawv_cache if rawv_cache is not None else d_cache rawk_buf = rawk_cache if rawk_cache is not None else k_cache beta_buf = beta_cache if beta_cache is not None else g_cache - pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if pdl else {} + pdl_kwargs = ( + { + "USE_GDC": True, + "launch_pdl": True, + "WAIT_FOR_GATE_PRODUCER": wait_for_gate_producer, + } + if pdl + else {} + ) gdn_replayssm_spec_circular_kernel[(triton.cdiv(V, BV), B, HV)]( q, k, diff --git a/sgl_nvidia_mega/qwen3_5/plan.py b/sgl_nvidia_mega/qwen3_5/plan.py index 7042f618..15e30aff 100644 --- a/sgl_nvidia_mega/qwen3_5/plan.py +++ b/sgl_nvidia_mega/qwen3_5/plan.py @@ -8,7 +8,9 @@ top-10 and both expert GEMMs in one launch, up to 7 rows) cover every width, as do ``gdn_verify`` (the GatedDeltaNet verify chain with its launch choices changed), ``hop`` (the boundary's fused all-reduce points on the decode hop) and ``gdn_push`` -(numeric: out_proj as the hop's push GEMM, on gdn_verify's FP8 norm). +(numeric: out_proj as the hop's push GEMM, on gdn_verify's FP8 norm). ``gdn_front`` +(numeric: the GDN projections and conv in one launch, then gdn_pdl's verify) covers one +request of 7 rows. """ from __future__ import annotations @@ -16,6 +18,7 @@ import msgspec MAX_ROWS = 7 # the widest covered step: one request of 7 verify rows (TP8 bs1) +GDN_FRONT_ROWS = 7 # the rows slot gdn_front's front is built for class Slots(msgspec.Struct, frozen=True): @@ -30,6 +33,8 @@ class Slots(msgspec.Struct, frozen=True): hop: bool = False # the boundary's fused all-reduce points (boundary/hop.py) # out_proj pushes its partial into the hop from its GEMM (gdn/verify.py; numeric) gdn_push: bool = False + # the projections and the conv in one launch (gdn/front.py; numeric) + gdn_front: bool = False class Facts(msgspec.Struct, frozen=True): @@ -42,6 +47,7 @@ class Facts(msgspec.Struct, frozen=True): gdn_pdl: bool = False # with its dependent verify launch hop: bool = False # the decode hop, built for the widest covered step gdn_push: bool = False # out_proj packed for the push GEMM on every GDN layer + gdn_front: bool = False # in_proj_qkvz packed for the front on every GDN layer def slots(rows: int, facts: Facts) -> Slots: @@ -54,6 +60,10 @@ def slots(rows: int, facts: Facts) -> Slots: gdn_pdl=facts.gdn_verify and facts.gdn_pdl, hop=facts.hop, gdn_push=facts.gdn_verify and facts.hop and facts.gdn_push, + gdn_front=facts.gdn_verify + and facts.gdn_pdl + and facts.gdn_front + and rows == GDN_FRONT_ROWS, ) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index ed6b3fe3..fa0e0cb3 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -21,6 +21,7 @@ "gdn_pdl", "hop", "gdn_push", + "gdn_front", ) SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" @@ -57,6 +58,10 @@ def slot_reasons() -> list[str]: f"{SLOTS_ENV}: slot 'gdn_push' pushes slot 'gdn_verify''s out_proj into " "slot 'hop'" ) + if "gdn_front" in SLOTS and not {"gdn_verify", "gdn_pdl"} <= SLOTS: + reasons.append( + f"{SLOTS_ENV}: slot 'gdn_front' runs slot 'gdn_pdl''s dependent verify" + ) if "gdn_pdl" in SLOTS and "gdn_verify" not in SLOTS: reasons.append( f"{SLOTS_ENV}: slot 'gdn_pdl' launches slot 'gdn_verify''s verify" @@ -115,6 +120,7 @@ def build(model, ctx): gdn_pdl=bound["gdn"].gdn_pdl, hop=bound["boundary"].hop is not None, gdn_push=bool(bound["gdn"].packed), + gdn_front=bool(bound["gdn"].front), ) slots = slot_plan.plan(pctx.widths, facts) built = msgspec.structs.asdict(facts) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/executor.py b/sgl_nvidia_mega/qwen3_5/runtime/executor.py index c53e4890..34174dca 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/executor.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/executor.py @@ -170,6 +170,15 @@ def _check_pools(self, view): "NVIDIA megakernel: slot gdn_verify cannot run: " + "; ".join(reasons) ) + if any(s.gdn_front for s in self.slots.values()): + from sgl_nvidia_mega.qwen3_5.gdn import front + + reasons = front.pool_reasons(view.mamba) + if reasons: + raise RuntimeError( + "NVIDIA megakernel: slot gdn_front cannot run: " + + "; ".join(reasons) + ) self._pools_checked = True @torch.no_grad() diff --git a/sgl_nvidia_mega/qwen3_5/sm103/gdn/__init__.py b/sgl_nvidia_mega/qwen3_5/sm103/gdn/__init__.py new file mode 100644 index 00000000..08ed8217 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/gdn/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The GatedDeltaNet kernels: the verify front (``front``).""" diff --git a/sgl_nvidia_mega/qwen3_5/sm103/gdn/front.py b/sgl_nvidia_mega/qwen3_5/sm103/gdn/front.py new file mode 100644 index 00000000..ec5e748c --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/gdn/front.py @@ -0,0 +1,238 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The GatedDeltaNet verify front of one request of 7 rows at TP8 in one launch +(``src/front.cu``): the static-FP8 ``in_proj_qkvz`` projection of the BF16 rows, the +causal conv update of its q / k / v channels (output, rolling state, deduplicated +intermediate windows), the BF16 ``in_proj_ba`` side projection and an L2 prefetch of the +ReplaySSM verify's checkpoint and history ring. ``gdn_pdl_front_ba_mkm`` of SGLang branch +nvidia/qwen35-agentx-gb300 (the kernel and its launch); the projections sum in another +order than SGLang's GEMMs (numeric), the conv equals SGLang's on the projected rows. + +It launches as a programmatic dependent and releases its own dependents at entry: the +next launch must wait (``griddepcontrol.wait``) before it reads any output, b / a +included. +""" + +from __future__ import annotations + +import ctypes +from functools import cache +from pathlib import Path + +import torch + +from sgl_nvidia_mega.common import jit + +SOURCE = Path(__file__).parent / "src" / "front.cu" +ROWS, HIDDEN, QKVZ, CONV, BA = 7, 4096, 2560, 1536, 16 +THREADS, CLUSTER, CTAS = ( + 256, + 8, + 88, +) # 10 projection clusters + 1 side-projection cluster +TIMEOUT = ( + 500_000_000 # clock cycles a CTA waits for its cluster's partials before trapping +) + + +class Prefetch(ctypes.Structure): + """gdn_ba::Prefetch.""" + + _fields_ = [ + ("base", ctypes.c_void_p), + ("slot_stride", ctypes.c_int64), + ("bytes", ctypes.c_int64), + ("ring", ctypes.c_int), + ] + + +class BaParams(ctypes.Structure): + """gdn_ba::Params.""" + + _fields_ = [ + ("input", ctypes.c_void_p), + ("weight", ctypes.c_void_p), + ("output", ctypes.c_void_p), + ("rows", ctypes.c_int), + ("input_stride", ctypes.c_int64), + ("prefetch", Prefetch * 4), + ("prefetches", ctypes.c_int), + ("state_indices", ctypes.c_void_p), + ("ring_indices", ctypes.c_void_p), + ("state_index64", ctypes.c_bool), + ("ring_index64", ctypes.c_bool), + ("pad_slot", ctypes.c_int64), + ] + + +class Params(ctypes.Structure): + """gdn_pdl_front_ba_mkm_detail::Params.""" + + _fields_ = [ + ("x", ctypes.c_void_p), + ("weight", ctypes.c_void_p), + ("sx", ctypes.c_void_p), + ("sw", ctypes.c_void_p), + ("output", ctypes.c_void_p), + ("error", ctypes.c_void_p), + ("timeout", ctypes.c_uint64), + ("m", ctypes.c_int), + ("conv_weight", ctypes.c_void_p), + ("conv_state", ctypes.c_void_p), + ("windows", ctypes.c_void_p), + ("conv_output", ctypes.c_void_p), + ("state_indices", ctypes.c_void_p), + ("replay_indices", ctypes.c_void_p), + ("state_stride", ctypes.c_int64 * 3), + ("window_stride", ctypes.c_int64 * 2), + ("state_index64", ctypes.c_bool), + ("replay_index64", ctypes.c_bool), + ("pad_slot", ctypes.c_int64), + ("ba", BaParams), + ] + + +def pack(weight_nk: torch.Tensor) -> torch.Tensor: + """in_proj_qkvz's FP8 weights [2560, 4096] (row-major [N, K]) packed once, as the + front's tensor-core fragments take them.""" + if weight_nk.dtype != torch.float8_e4m3fn or tuple(weight_nk.shape) != ( + QKVZ, + HIDDEN, + ): + raise ValueError( + f"the GDN front packs FP8 E4M3 [{QKVZ}, {HIDDEN}] weights, not " + f"{weight_nk.dtype} {tuple(weight_nk.shape)}" + ) + value = weight_nk.contiguous().view(torch.uint8) + value = value.reshape(10, 8, 2, 2, 8, 8, 16, 2, 4, 4) + return value.permute(0, 5, 1, 6, 2, 4, 8, 7, 3, 9).contiguous().view(-1) + + +@cache +def _kernel(device_index: int) -> jit.Kernel: + major, minor = torch.cuda.get_device_capability(device_index) + cubin = jit.compile_source( + SOURCE.read_text(), + "q35_gdn_front", + f"sm_{major}{minor}a", + ("-std=c++17", "-default-device"), + ) + return jit.load(cubin, "q35_gdn_front", (THREADS, 1, 1), 0) + + +def build(device_index: int) -> None: + """Compile the front (no compile inside a step).""" + _kernel(device_index) + + +def _prefetch(t: torch.Tensor, ring: int): + """One L2 prefetch region (slot ``index`` of a [slots, ...] tensor), or None when + a slot is not contiguous or not 16-byte sized and aligned (as the branch skips it).""" + if t is None or t.dim() < 2: + return None + contiguous, expect = True, 1 + for d in range(t.dim() - 1, 0, -1): + contiguous = contiguous and (t.shape[d] == 1 or t.stride(d) == expect) + expect *= t.shape[d] + nbytes = expect * t.element_size() + if not contiguous or nbytes % 16 or t.data_ptr() % 16: + return None + return Prefetch(t.data_ptr(), t.stride(0) * t.element_size(), nbytes, ring) + + +def front( + x: torch.Tensor, + packed: torch.Tensor, + input_scale: torch.Tensor, + weight_scale: torch.Tensor, + conv_weight: torch.Tensor, + conv_state: torch.Tensor, + windows: torch.Tensor, + state_indices: torch.Tensor, + replay_indices: torch.Tensor, + projected: torch.Tensor, + conv_out: torch.Tensor, + error: torch.Tensor, + *, + ba_weight: torch.Tensor, + ba_out: torch.Tensor, + checkpoint: torch.Tensor, + history_d: torch.Tensor, + history_k: torch.Tensor, + history_g: torch.Tensor, + ring_indices: torch.Tensor, + pad_slot: int = -1, +) -> None: + """The front of the BF16 rows ``x`` [7, 4096] (contiguous) into the caller's + buffers: ``projected`` [7, 2560] BF16 (q / k / v / z), ``conv_out`` [7, 1536] BF16, + ``ba_out`` [7, 16] BF16 (b then a), and the conv update of ``conv_state`` + [slots, 1536, 3] and ``windows`` [slots, 7, 1536, 3] (strides [s, 1, 9, 1]) at + ``state_indices[0]`` / ``replay_indices[0]`` (one int32 / int64 each; a state index + of ``pad_slot`` skips the conv writes). ``error`` is an int32 word, zero before + capture, that a timed-out wait sets before trapping. The ReplaySSM verify's + ``checkpoint`` (by state index) and ``history_*`` rings (by ``ring_indices[0]``) are + prefetched into L2: a hint, nothing is written.""" + if ( + x.dtype != torch.bfloat16 + or tuple(x.shape) != (ROWS, HIDDEN) + or not x.is_contiguous() + ): + raise ValueError( + f"the GDN front takes BF16 [7, 4096] rows, not {x.dtype} {tuple(x.shape)}" + ) + if windows.stride()[1:] != (1, 9, 1) or windows.stride(0) < CONV * 9: + raise ValueError(f"windows {tuple(windows.stride())} are not deduplicated") + regions = [ + r + for r in ( + _prefetch(checkpoint, 0), + _prefetch(history_d, 1), + _prefetch(history_k, 1), + _prefetch(history_g, 1), + ) + if r is not None + ] + ba = BaParams( + input=x.data_ptr(), + weight=ba_weight.data_ptr(), + output=ba_out.data_ptr(), + rows=ROWS, + input_stride=x.stride(0), + prefetches=len(regions), + state_indices=state_indices.data_ptr(), + ring_indices=ring_indices.data_ptr(), + state_index64=state_indices.dtype == torch.int64, + ring_index64=ring_indices.dtype == torch.int64, + pad_slot=pad_slot, + ) + for i, region in enumerate(regions): + ba.prefetch[i] = region + p = Params( + x=x.data_ptr(), + weight=packed.data_ptr(), + sx=input_scale.data_ptr(), + sw=weight_scale.data_ptr(), + output=projected.data_ptr(), + error=error.data_ptr(), + timeout=TIMEOUT, + m=ROWS, + conv_weight=conv_weight.data_ptr(), + conv_state=conv_state.data_ptr(), + windows=windows.data_ptr(), + conv_output=conv_out.data_ptr(), + state_indices=state_indices.data_ptr(), + replay_indices=replay_indices.data_ptr(), + state_index64=state_indices.dtype == torch.int64, + replay_index64=replay_indices.dtype == torch.int64, + pad_slot=pad_slot, + ba=ba, + ) + p.state_stride[:] = list(conv_state.stride()) + p.window_stride[:] = [windows.stride(0), 9] + jit.launch( + _kernel(x.device.index), + (CTAS, 1, 1), + [p], + torch.cuda.current_stream(x.device).cuda_stream, + (CLUSTER, 1, 1), + ) diff --git a/sgl_nvidia_mega/qwen3_5/sm103/gdn/src/front.cu b/sgl_nvidia_mega/qwen3_5/sm103/gdn/src/front.cu new file mode 100644 index 00000000..265e86c3 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/gdn/src/front.cu @@ -0,0 +1,487 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// The GatedDeltaNet verify front of Qwen3.5 at TP8 for one request of 7 rows, in one launch of 11 +// clusters of 8 CTAs: the static-FP8 in_proj_qkvz projection (BF16 input quantized in-kernel, e4m3 +// tensor-core split-K reduced in fixed cluster-rank order), the causal conv update of its q / k / v +// channels (output, rolling state, deduplicated intermediate windows), the BF16 in_proj_ba side +// projection on tensor cores (cluster 10), and an L2 prefetch of the ReplaySSM verify's checkpoint +// and history ring. Ported from gdn_pdl_front_ba_mkm.cuh of SGLang branch nvidia/qwen35-agentx-gb300 +// (SGLANG_Q35MF_MKG_FUSED + MKG_BA_TC + MKM_GDN_FRONT); the device code is unchanged. +// +// Built with NVRTC: front_kernel is a device function, and the extern "C" entry at the end +// instantiates its BF16-input form with the fused conv and the inline side projection. + +#include + +// NVRTC has no +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef long long int64_t; +typedef unsigned long long uint64_t; + +namespace sglang { +namespace gdn_pdl_front_ba_mkm_detail { + +namespace gdn_ba { + +// One L2 prefetch region per recurrent operand: slot `index` of [slots, ...] with contiguous slots. +struct Prefetch { + const char* base; + int64_t slot_stride; // bytes + int64_t bytes; // bytes per slot, multiple of 16 + int ring; // 0: physical state index, 1: request ring index +}; +struct Params { + const __nv_bfloat16* input; + const __nv_bfloat16* weight; + __nv_bfloat16* output; + int rows; + int64_t input_stride; + Prefetch prefetch[4]; + int prefetches; + const void* state_indices; + const void* ring_indices; + bool state_index64; + bool ring_index64; + int64_t pad_slot; +}; + +// A hint only: the consumer still reads these operands after its own dependency wait. +__device__ __forceinline__ void prefetch_l2(const char* p, int64_t bytes) { + for (int64_t offset = 0; offset < bytes; offset += 32768) { + const int64_t chunk = bytes - offset < 32768 ? bytes - offset : 32768; + asm volatile("cp.async.bulk.prefetch.L2.global [%0], %1;" ::"l"(p + offset), "r"((unsigned)chunk) : "memory"); + } +} + +__device__ __forceinline__ unsigned shared_address(const void* p) { + return static_cast(__cvta_generic_to_shared(p)); +} +__device__ __forceinline__ unsigned peer_address(unsigned address, int rank) { + unsigned result; + asm("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(result) : "r"(address), "r"(rank)); + return result; +} +__device__ __forceinline__ void expect(uint64_t* barrier, unsigned bytes) { + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" + :: "r"(shared_address(barrier)), "r"(bytes) : "memory"); +} +__device__ __forceinline__ void push(unsigned target, float x, float y, float z, + float w, unsigned barrier) { + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.f32 [%0], {%1,%2,%3,%4}, [%5];" + :: "r"(target), "f"(x), "f"(y), "f"(z), "f"(w), "r"(barrier) : "memory"); +} +__device__ __forceinline__ void wait(uint64_t* barrier, unsigned* error, uint64_t timeout) { + const uint64_t start = clock64(); + unsigned polls = 0, ready = 0; + do { + asm volatile("{ .reg .pred p; mbarrier.try_wait.parity.shared::cta.b64 p, [%1], 0; selp.u32 %0, 1, 0, p; }" + : "=r"(ready) : "r"(shared_address(barrier)) : "memory"); + if (!ready && (clock64() - start > timeout || ++polls >= (1u << 24))) { + if (atomicCAS(error, 0u, 1u) == 0u) printf("side projection inbox timeout block=%u\n", blockIdx.x); + asm volatile("trap;"); + } + } while (!ready); +} +__device__ __forceinline__ void mma(float (&d)[4], unsigned a0, unsigned a1, + unsigned a2, unsigned a3, unsigned b0, unsigned b1) { + asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)); +} + +// All eight CTAs enter this body. Scratch belongs exclusively to this role. +// The receiver acknowledges each source only after its final shared read. +template +__device__ __forceinline__ void ba_cluster_stage( + const Params& p, int rank, int tid, float* partial, float* inbox, + uint64_t* barrier, unsigned* error, uint64_t timeout) { + static_assert(Threads == 256); + const int lane = tid & 31, warp = tid >> 5; + const int group = lane >> 2, pair = 2 * (lane & 3); + const int split = warp >> 1, column_tile = warp & 1; + if (tid == 0) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], 1;" :: "r"(shared_address(barrier)) : "memory"); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); + } + __syncthreads(); + if (tid == 0) expect(barrier, rank == 0 ? 8 * p.rows * 16 * sizeof(float) : 16); + __syncthreads(); + asm volatile("barrier.cluster.arrive.aligned;" ::: "memory"); + + unsigned weight[8][2]; +#pragma unroll + for (int step = 0; step < 8; ++step) { + const int k = rank * 512 + split * 128 + step * 16 + pair; + const auto* row = p.weight + (column_tile * 8 + group) * 4096; + weight[step][0] = *reinterpret_cast(row + k); + weight[step][1] = *reinterpret_cast(row + k + 8); + } + asm volatile("griddepcontrol.wait;" ::: "memory"); + if (tid == 0 && *reinterpret_cast(error)) asm volatile("trap;"); + float acc[4] = {}; +#pragma unroll + for (int step = 0; step < 8; ++step) { + unsigned low = 0, high = 0; + if (group < p.rows) { + const auto* row = p.input + group * p.input_stride; + const int k = rank * 512 + split * 128 + step * 16 + pair; + low = *reinterpret_cast(row + k); + high = *reinterpret_cast(row + k + 8); + } + mma(acc, low, 0, high, 0, weight[step][0], weight[step][1]); + } + if (group < p.rows) { + const int output = group * 16 + column_tile * 8 + pair; + partial[split * 112 + output] = acc[0]; + partial[split * 112 + output + 1] = acc[1]; + } + __syncthreads(); + asm volatile("barrier.cluster.wait.aligned;" ::: "memory"); + if (tid < p.rows * 4) { + const int offset = tid * 4; + float values[4] = {}; +#pragma unroll + for (int j = 0; j < 4; ++j) +#pragma unroll + for (int k = 0; k < 4; ++k) values[j] = __fadd_rn(values[j], partial[k * 112 + offset + j]); + push(peer_address(shared_address(inbox + rank * 256 + offset), 0), + values[0], values[1], values[2], values[3], peer_address(shared_address(barrier), 0)); + } + if (tid == 0) wait(barrier, error, timeout); + __syncthreads(); + if (rank == 7 && tid < p.prefetches) { + const int64_t state = p.state_index64 ? static_cast(p.state_indices)[0] + : static_cast(p.state_indices)[0]; + if (state != p.pad_slot) { + const Prefetch f = p.prefetch[tid]; + const int64_t slot = f.ring ? (p.ring_index64 ? static_cast(p.ring_indices)[0] + : static_cast(p.ring_indices)[0]) + : state; + if (slot >= 0) prefetch_l2(f.base + slot * f.slot_stride, f.bytes); + } + } + if (rank == 0) { + if (tid < p.rows * 16) { + float value = 0.f; +#pragma unroll + for (int source = 0; source < 8; ++source) value = __fadd_rn(value, inbox[source * 256 + tid]); + p.output[tid] = __float2bfloat16_rn(value); + } + __syncthreads(); + if (tid > 0 && tid < 8) + push(peer_address(shared_address(partial), tid), 0.f, 0.f, 0.f, 0.f, + peer_address(shared_address(barrier), tid)); + } +} + +} // namespace gdn_ba + +constexpr int kThreads = 256; +constexpr int kCluster = 8; +__device__ __forceinline__ void pdl_wait() { asm volatile("griddepcontrol.wait;" ::: "memory"); } +__device__ __forceinline__ void pdl_trigger() { asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); } +__device__ __forceinline__ void cluster_arrive() { asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); } +__device__ __forceinline__ void cluster_wait() { asm volatile("barrier.cluster.wait.aligned;" ::: "memory"); } +__device__ __forceinline__ uint4 ldg_nc_v4(const uint4* p) { + uint4 r; + asm volatile("ld.global.nc.L1::no_allocate.v4.u32 {%0,%1,%2,%3}, [%4];" + : "=r"(r.x), "=r"(r.y), "=r"(r.z), "=r"(r.w) + : "l"(p)); + return r; +} + +__device__ __forceinline__ uint32_t smem_u32(const void* ptr) { + return static_cast(__cvta_generic_to_shared(ptr)); +} +__device__ __forceinline__ void mbar_init(uint64_t* bar, uint32_t count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" ::"r"(smem_u32(bar)), "r"(count) : "memory"); +} +__device__ __forceinline__ void mbar_init_fence() { + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); +} +__device__ __forceinline__ void mbar_expect_tx(uint64_t* bar, uint32_t bytes) { + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(smem_u32(bar)), "r"(bytes) + : "memory"); +} +__device__ __forceinline__ float div_full(float a, float b) { + float r; + asm("div.full.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} +// low byte = e4m3(lo), high byte = e4m3(hi) (cvt d, a=hi, b=lo) +__device__ __forceinline__ uint32_t e4m3x2(float lo, float hi) { + uint16_t r; + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(r) : "f"(hi), "f"(lo)); + return r; +} +__device__ __forceinline__ uint8_t e4m3(float v) { return (uint8_t)(e4m3x2(v, 0.f) & 0xff); } + +__device__ __forceinline__ void mma_e4m3(float (&d)[4], const uint4& a, uint32_t b0, uint32_t b1) { + asm volatile( + "mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, " + "{%0,%1,%2,%3};" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a.x), "r"(a.y), "r"(a.z), "r"(a.w), "r"(b0), "r"(b1)); +} + +// stock _static_quant_fp8: y * div.full(1, s), clamp +-448, cvt.rn.satfinite.e4m3 +__device__ __forceinline__ uint32_t quant4_static(const __nv_bfloat16* p, float inv) { + const uint2 raw = *reinterpret_cast(p); + const __nv_bfloat162 v01 = *reinterpret_cast(&raw.x); + const __nv_bfloat162 v23 = *reinterpret_cast(&raw.y); + float f[4] = {__low2float(v01), __high2float(v01), __low2float(v23), __high2float(v23)}; +#pragma unroll + for (int i = 0; i < 4; ++i) f[i] = fminf(fmaxf(__fmul_rn(f[i], inv), -448.f), 448.f); + return e4m3x2(f[0], f[1]) | (e4m3x2(f[2], f[3]) << 16); +} + + +__device__ __forceinline__ unsigned cluster_rank() { + unsigned value; + asm("mov.u32 %0, %%cluster_ctarank;" : "=r"(value)); + return value; +} +__device__ __forceinline__ unsigned peer_address(unsigned value, unsigned rank) { + unsigned result; + asm("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(result) : "r"(value), "r"(rank)); + return result; +} +__device__ __forceinline__ void push4(unsigned dst, float a, float b, float c, float d, unsigned bar) { + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.f32 [%0], {%1,%2,%3,%4}, [%5];" + :: "r"(dst), "f"(a), "f"(b), "f"(c), "f"(d), "r"(bar) : "memory"); +} +__device__ __forceinline__ void wait_inbox(uint64_t* bar, unsigned* error, uint64_t timeout) { + const uint64_t start = clock64(); + unsigned polls = 0, ready = 0; + do { + asm volatile("{ .reg .pred p; mbarrier.try_wait.parity.shared::cta.b64 p, [%1], 0; selp.u32 %0, 1, 0, p; }" + : "=r"(ready) : "r"(smem_u32(bar)) : "memory"); + if (!ready && (clock64() - start > timeout || ++polls >= (1u << 24))) { + if (atomicCAS(error, 0u, 1u) == 0u) printf("projection inbox timeout block=%u\n", blockIdx.x); + asm volatile("trap;"); + } + } while (!ready); +} +struct Params { + const void* x; + const uint4* weight; + const float* sx; + const float* sw; + __nv_bfloat16* output; + unsigned* error; + uint64_t timeout; + int m; + const __nv_bfloat16* conv_weight; + __nv_bfloat16* conv_state; + __nv_bfloat16* windows; + __nv_bfloat16* conv_output; + const void* state_indices; + const void* replay_indices; + int64_t state_stride[3]; + int64_t window_stride[2]; + bool state_index64; + bool replay_index64; + int64_t pad_slot; + gdn_ba::Params ba; +}; + +// Each cluster reduces its split-K partials in fixed rank order. +template +__device__ __forceinline__ void front_kernel(const Params& p) { + if (threadIdx.x == 0) pdl_trigger(); + __shared__ __align__(16) float inbox[kCluster][256]; + __shared__ __align__(16) uint2 fragments[16][32]; + __shared__ __align__(8) uint64_t barrier; + __shared__ float conv_stage[32][8]; // per channel: state taps 0..2, weights 3..6 + __shared__ int64_t conv_meta[3]; // valid, state slot, replay slot + const int rank = cluster_rank(), unit = blockIdx.x / kCluster; + const int tid = threadIdx.x, warp = tid >> 5, lane = tid & 31; + const int g = lane >> 2, t = lane & 3; + if constexpr (InlineBa) { + // The side projection uses the original BF16 activation independently of FP8 input. + if (unit == 10) { + gdn_ba::ba_cluster_stage(p.ba,rank,tid, + reinterpret_cast(fragments),reinterpret_cast(inbox), + &barrier,p.error,p.timeout); + return; + } + } + if (tid == 0) { mbar_init(&barrier, 1); mbar_init_fence(); } + __syncthreads(); + cluster_arrive(); + // Convolution owners: warp 0 reads the weights, indices and rolling state off the projection path. + const bool conv_reader = FuseConv && unit < 6 && warp == 0; + const int conv_channel = unit * 256 + rank * 32 + (tid >> 2) + 8 * (tid & 3); + // Raw BF16 until the staging write, so no thread waits on these reads before the projection. + __nv_bfloat16 conv_tap[4] = {}; + if (conv_reader) { +#pragma unroll + for (int tap = 0; tap < 4; ++tap) conv_tap[tap] = p.conv_weight[conv_channel * 4 + tap]; + } + uint4 weights[16][2]; + const uint4* wp = p.weight + (size_t)(unit * kCluster + rank) * (8 * 16 * 2 * 32) + warp * 1024 + lane; +#pragma unroll + for (int step = 0; step < 16; ++step) +#pragma unroll + for (int block = 0; block < 2; ++block) + weights[step][block] = ldg_nc_v4(wp + (step * 2 + block) * 32); + const float sx = __ldg(p.sx), alpha = __fmul_rn(sx, __ldg(p.sw)); + pdl_wait(); + // Issued now, tested before the first push: no store precedes the check. + const unsigned sticky = tid == 0 ? *reinterpret_cast(p.error) : 0u; + // Both indices are read now (independently) and used only after the projection. + int64_t conv_slot = 0, conv_replay = 0; + if (conv_reader) { + conv_slot = p.state_index64 ? static_cast(p.state_indices)[0] + : static_cast(p.state_indices)[0]; + conv_replay = p.replay_index64 ? static_cast(p.replay_indices)[0] + : static_cast(p.replay_indices)[0]; + } + const float inverse = div_full(1.0f, sx); +#pragma unroll + for (int i = 0; i < 2; ++i) { + const int step = 2 * warp + i; + uint2 value = make_uint2(0u, 0u); + if (g < p.m) { + const int offset = g * 4096 + rank * 512 + step * 32 + 4 * t; + if constexpr (Quantized) { + const uint8_t* x = static_cast(p.x) + offset; + value = make_uint2(*reinterpret_cast(x), + *reinterpret_cast(x + 16)); + } else { + const auto* x = static_cast(p.x) + offset; + value = make_uint2(quant4_static(x, inverse), quant4_static(x + 16, inverse)); + } + } + fragments[step][lane] = value; + } + __syncthreads(); + float chains[2][4][4] = {}; +#pragma unroll + for (int step = 0; step < 16; ++step) { + const uint2 b = fragments[step][lane]; + mma_e4m3(chains[0][step & 3], weights[step][0], b.x, b.y); + mma_e4m3(chains[1][step & 3], weights[step][1], b.x, b.y); + } + float acc[2][4]; +#pragma unroll + for (int block = 0; block < 2; ++block) +#pragma unroll + for (int i = 0; i < 4; ++i) + acc[block][i] = __fadd_rn(__fadd_rn(chains[block][0][i], chains[block][1][i]), + __fadd_rn(chains[block][2][i], chains[block][3][i])); + const bool feature_owner = FuseConv && unit < 6; + const bool owner = feature_owner || rank < p.m; + const bool conv_valid = conv_reader && conv_slot != p.pad_slot; + __nv_bfloat16 conv_old[3] = {}; + if (conv_valid) { + const int64_t base = conv_slot * p.state_stride[0] + conv_channel * p.state_stride[1]; +#pragma unroll + for (int j = 0; j < 3; ++j) conv_old[j] = p.conv_state[base + j * p.state_stride[2]]; + } + const unsigned bytes = feature_owner ? kCluster * p.m * 32 * sizeof(float) + : kCluster * 256 * sizeof(float); + if (tid == 0 && owner) mbar_expect_tx(&barrier, bytes); + if (sticky) asm volatile("trap;"); + cluster_wait(); +#pragma unroll + for (int j = 0; j < 2; ++j) { + const int token = 2 * t + j; + if (token < p.m) { + const int slot = feature_owner ? token * 32 + g * 4 : warp * 32 + g * 4; + const unsigned destination = smem_u32(&inbox[rank][slot]); + const int peer = feature_owner ? warp : token; + push4(peer_address(destination, peer), acc[0][j], acc[0][2+j], acc[1][j], acc[1][2+j], + peer_address(smem_u32(&barrier), peer)); + } + } + if (!owner) return; + if (conv_reader) { +#pragma unroll + for (int j = 0; j < 3; ++j) conv_stage[lane][j] = __bfloat162float(conv_old[j]); +#pragma unroll + for (int j = 0; j < 4; ++j) conv_stage[lane][3 + j] = __bfloat162float(conv_tap[j]); + if (lane == 0) { + conv_meta[0] = conv_slot != p.pad_slot; + conv_meta[1] = conv_slot; + conv_meta[2] = conv_replay; + } + } + if (tid == 0) wait_inbox(&barrier, p.error, p.timeout); + __syncthreads(); + if (feature_owner) { + // Token `warp` of 32 channels; neighbours' projected values are recomputed with the same arithmetic. + if (warp >= 7) return; + const int token = warp; + const int channel = unit * 256 + rank * 32 + (lane >> 2) + 8 * (lane & 3); + const bool valid = conv_meta[0] != 0; + float value[4]; // projected BF16 values of tokens token-3 .. token (state taps before token 0) + __nv_bfloat16 rounded = __float2bfloat16_rn(0.f); +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int t = token - 3 + j; + if (t < 0) { + value[j] = conv_stage[lane][t + 3]; + } else { + float sum = 0.f; +#pragma unroll + for (int source = 0; source < kCluster; ++source) + sum = __fadd_rn(sum, inbox[source][t * 32 + lane]); + const __nv_bfloat16 r = __float2bfloat16_rn(__fmul_rn(sum, alpha)); + value[j] = __bfloat162float(r); + if (j == 3) rounded = r; + } + } + p.output[token * 2560 + channel] = rounded; + if (valid) { + const int64_t state_base = conv_meta[1] * p.state_stride[0] + channel * p.state_stride[1]; + __nv_bfloat16* window = p.windows + conv_meta[2] * p.window_stride[0] + channel * p.window_stride[1]; + const float weight[4] = {conv_stage[lane][3], conv_stage[lane][4], conv_stage[lane][5], conv_stage[lane][6]}; + if (token == 0) { + // The overlapping logical windows have one physical writer per channel. + window[0] = __float2bfloat16_rn(value[1]); + window[1] = __float2bfloat16_rn(value[2]); + } + // Preserve the BF16 product boundary before the stock FP32 additions. + const float p0 = __bfloat162float(__float2bfloat16_rn(__fmul_rn(value[0],weight[0]))); + const float p1 = __bfloat162float(__float2bfloat16_rn(__fmul_rn(value[1],weight[1]))); + const float p2 = __bfloat162float(__float2bfloat16_rn(__fmul_rn(value[2],weight[2]))); + const float p3 = __bfloat162float(__float2bfloat16_rn(__fmul_rn(value[3],weight[3]))); + float accumulator = __fadd_rn(p0,0.f); + accumulator = __fadd_rn(accumulator,p1); + accumulator = __fadd_rn(accumulator,p2); + accumulator = __fadd_rn(accumulator,p3); + float exponential; + const float exponent = __fmul_rn(accumulator,__int_as_float(0xBFB8AA3B)); + asm("ex2.approx.f32 %0, %1;" : "=f"(exponential) : "f"(exponent)); + const float activated = div_full(accumulator, __fadd_rn(1.f, exponential)); + p.conv_output[token * 1536 + channel] = __float2bfloat16_rn(activated); + window[token + 2] = rounded; + if (token == 6) { + p.conv_state[state_base] = __float2bfloat16_rn(value[1]); + p.conv_state[state_base + p.state_stride[2]] = __float2bfloat16_rn(value[2]); + p.conv_state[state_base + 2 * p.state_stride[2]] = __float2bfloat16_rn(value[3]); + } + } + } else { + float sum = 0.f; +#pragma unroll + for (int source = 0; source < kCluster; ++source) sum = __fadd_rn(sum, inbox[source][tid]); + const int feature = 32 * warp + ((lane >> 2) & 7) + 8 * (lane & 3); + p.output[rank * 2560 + unit * 256 + feature] = __float2bfloat16_rn(__fmul_rn(sum, alpha)); + } +} + +} // namespace gdn_pdl_front_ba_mkm_detail + +} // namespace sglang + +// The verify front of 7 BF16 rows: grid 88 (11 clusters of 8), 256 threads, with PDL. +extern "C" __global__ __launch_bounds__(sglang::gdn_pdl_front_ba_mkm_detail::kThreads, 1) void q35_gdn_front( + const __grid_constant__ sglang::gdn_pdl_front_ba_mkm_detail::Params p) { + sglang::gdn_pdl_front_ba_mkm_detail::front_kernel(p); +} diff --git a/sgl_nvidia_mega/qwen3_5/spec.py b/sgl_nvidia_mega/qwen3_5/spec.py index 1f0dd44d..95ff334d 100644 --- a/sgl_nvidia_mega/qwen3_5/spec.py +++ b/sgl_nvidia_mega/qwen3_5/spec.py @@ -101,6 +101,12 @@ "gdn_push.out_proj.rel_l2_vs_stock": 1.05, "gdn_push.out_proj.max_abs_vs_stock": 2.0, }, + "gdn_front": { + "gdn_front.in_proj_qkvz.rel_l2_vs_stock": 1.05, + "gdn_front.in_proj_qkvz.max_abs_vs_stock": 2.0, + "gdn_front.in_proj_ba.rel_l2_vs_stock": 1.05, + "gdn_front.in_proj_ba.max_abs_vs_stock": 2.0, + }, } diff --git a/test/kernels/test_qwen3_5_gdn.py b/test/kernels/test_qwen3_5_gdn.py index d2d120a9..2da8a4d4 100644 --- a/test/kernels/test_qwen3_5_gdn.py +++ b/test/kernels/test_qwen3_5_gdn.py @@ -9,8 +9,20 @@ - the gated RMSNorm writing static FP8, against SGLang's gated norm followed by the FP8 linear's static input quantization (``static_quant_fp8``), with z as rows and as token-major views; -- the ReplaySSM verify launch, with and without the dependent launch, against - ``gdn_replayssm_spec_decode``'s (output and ring entries). +- the ReplaySSM verify launch, with and without the dependent launch (and its wait + before the gates, slot gdn_front's), against ``gdn_replayssm_spec_decode``'s (output and + ring entries). + +And slot gdn_front's kernels (numeric: those of SGLang branch nvidia/qwen35-agentx-gb300, +checked bit for bit against that branch on the same inputs, not here): + +- the verify front: its in_proj_qkvz and in_proj_ba rows against fp64 on the same + operands, at most 1.05 x the mean relative L2 and 2 x the max abs error of SGLang's ops + (FlashInfer's FP8 BMM on SGLang's static quantization, cuBLAS's BF16 GEMM); its conv + output, rolling state and deduplicated windows bit for bit ``causal_conv1d_update``'s + on its own projected rows; a padded request leaves the state and windows alone; graph + replays equal eager launches; +- its gated norm writing static FP8, bit for bit slot gdn_verify's. """ import pytest @@ -125,11 +137,11 @@ def test_norm_fp8_is_sglangs_norm_then_static_quant(views): ) -@pytest.mark.parametrize("pdl", [False, True]) -def test_replayssm_verify_is_sglangs(pdl): +@pytest.mark.parametrize("pdl, gate", [(False, False), (True, False), (True, True)]) +def test_replayssm_verify_is_sglangs(pdl, gate): """The slot's verify launch (32-wide value tiles on 2 warps, and with or without the - dependent launch) against gdn_replayssm_spec_decode's verify launch: the output and - this step's ring entries.""" + dependent launch, waiting before the gates with ``gate``) against + gdn_replayssm_spec_decode's verify launch: the output and this step's ring entries.""" try: from sglang.kernels.ops.attention.linear.gdn_replayssm_spec_decode import ( gdn_replayssm_spec_decode, @@ -141,7 +153,7 @@ def test_replayssm_verify_is_sglangs(pdl): from sgl_nvidia_mega.qwen3_5.gdn import replayssm - g = torch.Generator(device="cuda").manual_seed(3 + int(pdl)) + g = torch.Generator(device="cuda").manual_seed(3 + int(pdl) + 2 * int(gate)) bs, ring_len, slots = 2, 16, 9 tokens = bs * DRAFT @@ -229,6 +241,201 @@ def rand(*shape, dtype=torch.bfloat16): block_v=32, num_warps=2, pdl=pdl, + wait_for_gate_producer=gate, ) assert torch.equal(got, want) assert all(torch.equal(x, y) for x, y in zip(got_ring, want_ring)) + + +# ------------------------------------------------------------------- slot gdn_front + +FRONT_SLOTS = 5 + + +def _front_operands(seed, *, pad=False, index_dtype=torch.int32): + """One request of 7 rows at TP8: the normed rows, the static FP8 in_proj_qkvz + [2560, 4096] with its scales, the BF16 in_proj_ba [16, 4096], the conv weights, the + pools (rolling states, deduplicated windows, the verify's checkpoint and ring) and + the step's slots.""" + g = torch.Generator(device="cuda").manual_seed(seed) + + def rand(*shape, scale=1.0): + return torch.randn(*shape, device="cuda", generator=g) * scale + + phys = rand(FRONT_SLOTS, D, 9).to(torch.bfloat16) + index = lambda i: torch.tensor([i], dtype=index_dtype, device="cuda") # noqa: E731 + return dict( + x=rand(DRAFT, 4096).to(torch.bfloat16), + w=rand(2560, 4096, scale=0.05).clamp(-440, 440).to(torch.float8_e4m3fn), + sx=torch.tensor([0.01], device="cuda"), + sw=torch.tensor([0.002], device="cuda"), + ba_w=rand(16, 4096, scale=0.02).to(torch.bfloat16), + conv_w=rand(D, WIDTH, scale=0.5).to(torch.bfloat16), + state=rand(FRONT_SLOTS, D, WIDTH - 1).to(torch.bfloat16), + phys=phys, + windows=phys.as_strided((FRONT_SLOTS, DRAFT, D, WIDTH - 1), (D * 9, 1, 9, 1)), + checkpoint=rand(FRONT_SLOTS, HV, V, K).to(torch.bfloat16), + ring_d=rand(FRONT_SLOTS, HV, 16, V).to(torch.bfloat16), + ring_k=rand(FRONT_SLOTS, H, 16, K).to(torch.bfloat16), + ring_g=rand(FRONT_SLOTS, HV, 16), + states=index(-1 if pad else 3), + inter=index(1), + req=index(2), + ) + + +def _run_front(o, packed, outs): + from sgl_nvidia_mega.qwen3_5.sm103.gdn import front + + front.front( + o["x"], + packed, + o["sx"], + o["sw"], + o["conv_w"], + o["state"], + o["windows"], + o["states"], + o["inter"], + outs["projected"], + outs["conv"], + outs["error"], + ba_weight=o["ba_w"], + ba_out=outs["ba"], + checkpoint=o["checkpoint"], + history_d=o["ring_d"], + history_k=o["ring_k"], + history_g=o["ring_g"], + ring_indices=o["req"], + ) + + +def _front_outputs(): + return dict( + projected=torch.empty(DRAFT, 2560, dtype=torch.bfloat16, device="cuda"), + conv=torch.empty(DRAFT, D, dtype=torch.bfloat16, device="cuda"), + ba=torch.empty(DRAFT, 16, dtype=torch.bfloat16, device="cuda"), + error=torch.zeros(1, dtype=torch.int32, device="cuda"), + ) + + +def _errors(got, ref): + diff = got.double() - ref + return (diff.norm() / ref.norm()).item(), diff.abs().max().item() + + +@pytest.mark.parametrize("index_dtype", [torch.int32, torch.int64]) +def test_front_holds_its_projections_to_fp64_and_its_conv_is_sglangs(index_dtype): + from sglang.kernels.ops.mamba.causal_conv1d_triton import causal_conv1d_update + from sglang.kernels.ops.quantization.fp8_kernel import static_quant_fp8 + from sglang.srt.layers.quantization.fp8_utils import flashinfer_bmm_fp8 + + from sgl_nvidia_mega.qwen3_5.sm103.gdn import front + + errors = {n: [] for n in ("qkvz", "qkvz_stock", "ba", "ba_stock")} + for seed in range(3): + o = _front_operands(seed, index_dtype=index_dtype) + state0, phys0 = o["state"].clone(), o["phys"].clone() + outs = _front_outputs() + _run_front(o, front.pack(o["w"]), outs) + torch.cuda.synchronize() + assert outs["error"].item() == 0 + # the projections against fp64 on the same FP8 / BF16 operands + xq, _ = static_quant_fp8(o["x"], o["sx"], repeat_scale=False) + ref = (xq.double() @ o["w"].double().t()) * ( + o["sx"].double() * o["sw"].double() + ) + stock = flashinfer_bmm_fp8(xq, o["w"].t(), o["sx"], o["sw"], torch.bfloat16) + errors["qkvz"].append(_errors(outs["projected"], ref)) + errors["qkvz_stock"].append(_errors(stock, ref)) + ref = o["x"].double() @ o["ba_w"].double().t() + errors["ba"].append(_errors(outs["ba"], ref)) + errors["ba_stock"].append(_errors(o["x"] @ o["ba_w"].t(), ref)) + # the conv on the front's own projected q / k / v rows, as SGLang's update + state, phys = state0.clone(), phys0.clone() + windows = phys.as_strided(o["windows"].shape, o["windows"].stride()) + want = causal_conv1d_update( + outs["projected"][:, :D].view(1, DRAFT, D).transpose(1, 2), + state, + o["conv_w"], + None, + "silu", + conv_state_indices=o["states"], + intermediate_conv_window=windows, + intermediate_state_indices=o["inter"], + ) + assert torch.equal(outs["conv"], want.transpose(1, 2).reshape(DRAFT, D)) + assert torch.equal(o["state"], state) and torch.equal(o["phys"], phys) + e = {n: torch.tensor(v, dtype=torch.float64) for n, v in errors.items()} + print( + f"T0 gdn_front qkvz mean rel-L2 {e['qkvz'][:, 0].mean():.4g} vs stock " + f"{e['qkvz_stock'][:, 0].mean():.4g}, max abs {e['qkvz'][:, 1].max():.4g} vs " + f"{e['qkvz_stock'][:, 1].max():.4g}; ba {e['ba'][:, 0].mean():.4g} vs " + f"{e['ba_stock'][:, 0].mean():.4g}, {e['ba'][:, 1].max():.4g} vs " + f"{e['ba_stock'][:, 1].max():.4g}", + flush=True, + ) + for name in ("qkvz", "ba"): + got, stock = e[name], e[f"{name}_stock"] + assert got[:, 0].mean() <= 1.05 * stock[:, 0].mean(), name + assert got[:, 1].max() <= 2 * stock[:, 1].max(), name + + +def test_front_leaves_a_padded_requests_state_alone(): + from sgl_nvidia_mega.qwen3_5.sm103.gdn import front + + o = _front_operands(7, pad=True) + state0, phys0 = o["state"].clone(), o["phys"].clone() + outs = _front_outputs() + _run_front(o, front.pack(o["w"]), outs) + torch.cuda.synchronize() + assert outs["error"].item() == 0 + assert torch.equal(o["state"], state0) and torch.equal(o["phys"], phys0) + + +def test_front_replays_from_a_cuda_graph(): + from sgl_nvidia_mega.qwen3_5.sm103.gdn import front + + o = _front_operands(11) + packed = front.pack(o["w"]) + pools = ("state", "phys") + start = {n: o[n].clone() for n in pools} + outs = _front_outputs() + _run_front(o, packed, outs) # warm up (and compile) outside capture + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + _run_front(o, packed, outs) + for seed in (12, 13): + fresh = _front_operands(seed) + o["x"].copy_(fresh["x"]) + for n in pools: + o[n].copy_(start[n]) + graph.replay() + torch.cuda.synchronize() + got = {n: t.clone() for n, t in outs.items()} + got.update({n: o[n].clone() for n in pools}) + for n in pools: + o[n].copy_(start[n]) + _run_front(o, packed, outs) + torch.cuda.synchronize() + for n, t in outs.items(): + assert torch.equal(got[n], t), n + for n in pools: + assert torch.equal(got[n], o[n]), n + + +def test_front_norm_is_slot_gdn_verifys(): + from sgl_nvidia_mega.qwen3_5.gdn.front_norm import norm + from sgl_nvidia_mega.qwen3_5.gdn.norm_fp8 import rms_norm_gated_fp8 + + g = torch.Generator(device="cuda").manual_seed(5) + core = torch.randn(DRAFT * HV, V, device="cuda", generator=g).to(torch.bfloat16) + projected = torch.randn(DRAFT, 2560, device="cuda", generator=g).to(torch.bfloat16) + z = projected[:, D:].view(DRAFT, HV, V) + weight = torch.randn(V, device="cuda", generator=g).to(torch.bfloat16) + scale = torch.tensor([0.02], device="cuda") + quant = torch.empty(DRAFT * HV, V, dtype=torch.float8_e4m3fn, device="cuda") + rstd = torch.empty(DRAFT * HV, dtype=torch.float32, device="cuda") + norm(core, z, weight, scale, quant, rstd, 1e-6) + want = rms_norm_gated_fp8(core, weight, 1e-6, z, scale, activation="silu") + assert torch.equal(quant.view(torch.uint8), want.view(torch.uint8).view(-1, V)) diff --git a/test/unit/test_qwen3_5_spec.py b/test/unit/test_qwen3_5_spec.py index 0bed03b5..3c4b653d 100644 --- a/test/unit/test_qwen3_5_spec.py +++ b/test/unit/test_qwen3_5_spec.py @@ -134,6 +134,10 @@ def test_an_sglang_without_the_eagle_forward_view_is_refused(monkeypatch): {"gdn_verify", "gdn_push"}, "slot 'gdn_push' pushes slot 'gdn_verify''s out_proj into slot 'hop'", ), + ( + {"gdn_verify", "gdn_front"}, + "slot 'gdn_front' runs slot 'gdn_pdl''s dependent verify", + ), ], ) def test_a_slot_set_that_cannot_run_is_refused_before_the_load( @@ -161,6 +165,11 @@ def test_the_slot_plan_covers_one_request_of_7_rows(): 7: plan.Slots(**both, gdn_push=True) } assert not plan.slots(7, plan.Facts(gdn_verify=True, gdn_push=True)).gdn_push + # gdn_front runs gdn_pdl's verify, for one request of 7 rows only + front = dict(gdn_verify=True, gdn_pdl=True, gdn_front=True) + assert plan.plan((7,), plan.Facts(**front)) == {7: plan.Slots(**front)} + assert not plan.slots(4, plan.Facts(**front)).gdn_front + assert not plan.slots(7, plan.Facts(gdn_verify=True, gdn_front=True)).gdn_front with pytest.raises(ValueError): plan.slots(plan.MAX_ROWS + 1, plan.Facts()) @@ -172,3 +181,8 @@ def test_a_numeric_slot_turns_the_claim_into_its_tolerances(): claim = spec.numerics(frozenset({"gdn_verify", "hop", "gdn_push"})) assert claim.kind is NumericsKind.TOLERANCE assert dict(claim.tolerances) == spec.NUMERIC_TOLERANCES["gdn_push"] + both = spec.numerics(frozenset({"gdn_verify", "gdn_pdl", "gdn_front", "gdn_push"})) + assert dict(both.tolerances) == { + **spec.NUMERIC_TOLERANCES["gdn_push"], + **spec.NUMERIC_TOLERANCES["gdn_front"], + } diff --git a/test/unit/test_qwen3_5_steps.py b/test/unit/test_qwen3_5_steps.py index 5db30d70..7a050c40 100644 --- a/test/unit/test_qwen3_5_steps.py +++ b/test/unit/test_qwen3_5_steps.py @@ -806,11 +806,366 @@ def test_the_gdn_push_slot_pushes_only_where_the_moe_input_consumes( with caplog.at_level("INFO"): gdn.step(layer, torch.zeros(7, 4), st) assert seen == [msgspec.structs.replace(verify.SLOT, push=takes)] - # the push is logged once, where it is chosen + # the push is logged once where it is chosen, for slot gdn_verify's and gdn_front's paths logged = "NVIDIA megakernel: Qwen3.5 slot gdn_push runs at 7 rows" in caplog.text assert logged == takes +def _gdn_front_setup(log): + """A Qwen3.5 TP8 GatedDeltaNet (layer 2, the pools' third GDN layer) for slot + gdn_front, with the binding of a one-request verify step of 7 rows.""" + H, K, HV, V, D = 2, 128, 8, 128, 1536 + k = log.known + k.update( + x=torch.zeros(7, 4096, dtype=torch.bfloat16), + packed2=torch.zeros(16, dtype=torch.uint8), + front2=torch.zeros(16, dtype=torch.uint8), + error=torch.zeros(1, dtype=torch.int32), + in_s=torch.ones(1), + in_ws=torch.ones(1), + ba_w=torch.zeros(16, 4096, dtype=torch.bfloat16), + conv_w=torch.zeros(D, 4, dtype=torch.bfloat16), + out_s=torch.ones(1), + out_ws=torch.ones(1), + norm_w=torch.ones(V), + states=torch.tensor([5]), + inter=torch.tensor([0]), + qsl=torch.tensor([0, 7]), + req=torch.tensor([2]), + conv=torch.zeros(4, 3, D, 3), + window=torch.zeros(4, 3, 7, D, 3), + temporal=torch.zeros(4, 3, HV, K, V), + ring_d=torch.zeros(4, 3, HV, 16, V), + ring_k=torch.zeros(4, 3, H, 16, K), + ring_g=torch.zeros(4, 3, HV, 16), + write_pos=torch.zeros(3, dtype=torch.int32), + cache_base=torch.zeros(3, dtype=torch.int32), + is_flush=torch.zeros(3, dtype=torch.int32), + ) + k.update({f"{n}2": k[n][2] for n in ("conv", "window", "temporal")}) + k.update({f"{n}2": k[n][2] for n in ("ring_d", "ring_k", "ring_g")}) + attn = SimpleNamespace( + layer_id=2, + conv_weights=k["conv_w"], + q_dim=H * K, + k_dim=H * K, + v_dim=HV * V, + num_k_heads=H, + num_v_heads=HV, + head_k_dim=K, + head_v_dim=V, + A_log=torch.zeros(HV), + dt_bias=torch.zeros(HV), + ) + gdn = SimpleNamespace( + attn=attn, + norm=SimpleNamespace(weight=k["norm_w"], eps=1e-6), + in_proj_qkvz=SimpleNamespace(input_scale=k["in_s"], weight_scale=k["in_ws"]), + in_proj_ba=SimpleNamespace(weight=k["ba_w"]), + out_proj=SimpleNamespace(input_scale=k["out_s"], weight_scale=k["out_ws"]), + ) + pools = SimpleNamespace( + layer_ids=(0, 1, 2, 4), + conv=(k["conv"],), + intermediate_conv_window=(k["window"],), + temporal=k["temporal"], + ring_d=k["ring_d"], + ring_k=k["ring_k"], + ring_g=k["ring_g"], + ring_rawv=None, + ring_rawk=None, + ring_beta=None, + ring_write_pos=k["write_pos"], + ring_cache_base=k["cache_base"], + ring_is_flush=k["is_flush"], + ) + meta = SimpleNamespace( + state_indices=k["states"], + intermediate_indices=k["inter"], + query_start_loc=k["qsl"], + pad_slot_id=-1, + ) + st = SimpleNamespace( + fb=SimpleNamespace(req_pool_indices=k["req"]), + view=SimpleNamespace(gdn=meta, mamba=pools), + bound={ + "gdn": SimpleNamespace( + front={2: k["front2"]}, packed={2: k["packed2"]}, error=k["error"] + ) + }, + ) + return gdn, st + + +@pytest.mark.parametrize("push", [False, True]) +def test_the_gdn_front_slot_runs_the_front_the_verify_the_norm_and_out_proj(push): + """The branch's fused path in its order: the front (projections, conv, b / a) on + the layer's pools at the step's slots, the ReplaySSM verify on its conv rows waiting + for it before the gates, the norm into out_proj's FP8 rows, then out_proj (with + slot gdn_push, the hop's push GEMM).""" + import msgspec + + from sgl_nvidia_mega.qwen3_5.gdn import front, front_norm, replayssm, verify + from sgl_nvidia_mega.qwen3_5.sm103.gdn import front as kernel + + log = _log() + gdn, st = _gdn_front_setup(log) + k = log.known + hop = SimpleNamespace(out_proj_push=log.op("out_proj_push", "pushed")) + st.bound["boundary"] = SimpleNamespace(hop=hop) + seen = {} + + def front_launch(*args, **kw): + seen["projected"], seen["conv"], seen["ba"] = args[9], args[10], kw["ba_out"] + log.op("front")(*args, **kw) + + def spec_decode(q, k_, v, a, b, *args, **kw): + seen["core"] = args[2] + seen["qkv"] = (q, k_, v) + seen["ab"] = (a, b) + log.append(("verify", tuple(map(log.tag, args[3:])), kw)) + + def norm(x, z, weight, scale, quant, rstd, eps): + seen["norm"] = (x, z, quant, rstd) + log.append(("norm", log.tag(weight), log.tag(scale), eps)) + + opts = msgspec.structs.replace(verify.SLOT_PDL, push=push) + with ( + patch.object(kernel, "front", front_launch), + patch.object(replayssm, "verify", spec_decode), + patch.object(front_norm, "norm", norm), + patch.object( + verify, + "_out_proj", + lambda linear, fp8, dtype: ( + log.append(("out_proj", linear, fp8, dtype)) or "partial" + ), + ), + ): + out = front.step(gdn, k["x"], st, opts) + assert [e[0] for e in log] == [ + "front", + "verify", + "norm", + "out_proj_push" if push else "out_proj", + ] + _, _, args, kw = log[0] + assert args == ( + "x(7, 4096)", + "front2(16,)", + "in_s(1,)", + "in_ws(1,)", + "conv_w(1536, 4)", + "conv2(3, 1536, 3)", + "window2(3, 7, 1536, 3)", + "states(1,)", + "inter(1,)", + "?(7, 2560)", + "?(7, 1536)", + "error(1,)", + ) + assert dict(kw) == { + "ba_weight": "ba_w(16, 4096)", + "ba_out": "?(7, 16)", + "checkpoint": "temporal2(3, 8, 128, 128)", + "history_d": "ring_d2(3, 8, 16, 128)", + "history_k": "ring_k2(3, 2, 16, 128)", + "history_g": "ring_g2(3, 8, 16)", + "ring_indices": "req(1,)", + "pad_slot": -1, + } + # q / k / v are the conv rows' views; a / b the front's b8 / a8 halves + q, k_, v = seen["qkv"] + conv, ba = seen["conv"], seen["ba"] + assert q.data_ptr() == conv.data_ptr() and q.shape == (7, 2, 128) + assert v.shape == (7, 8, 128) and v.data_ptr() == conv[:, 512:].data_ptr() + a, b = seen["ab"] + assert a.data_ptr() == ba[:, 8:].data_ptr() and b.data_ptr() == ba.data_ptr() + assert a.shape == b.shape == (7, 8) and a.stride() == (16, 1) + tags, options = log[1][1], log[1][2] + assert tags[:4] == ( + "temporal2(3, 8, 128, 128)", + "ring_d2(3, 8, 16, 128)", + "ring_k2(3, 2, 16, 128)", + "ring_g2(3, 8, 16)", + ) + assert tags[4:10] == (None, None, None, "qsl(2,)", "states(1,)", "req(1,)") + assert options == dict( + max_cache_len=16, + max_spec_len=7, + scale=128**-0.5, + null_block_id=-1, + block_v=32, + num_warps=2, + pdl=True, + wait_for_gate_producer=True, + ) + # the norm reads the verify's output and the front's z, into FP8 [56, 128] + x_norm, z, quant, rstd = seen["norm"] + assert x_norm.data_ptr() == seen["core"].data_ptr() and x_norm.shape == (56, 128) + assert z.data_ptr() == seen["projected"][:, 1536:].data_ptr() + assert z.shape == (7, 8, 128) and z.stride() == (2560, 128, 1) + assert quant.dtype == torch.float8_e4m3fn and quant.shape == (56, 128) + assert rstd.shape == (56,) and log[2][1:] == ("norm_w(128,)", "out_s(1,)", 1e-6) + if push: + assert out == "pushed" + assert dict(log[3][3]) == { + "input_scale": "out_s(1,)", + "packed_w": "packed2(16,)", + "weight_scale": "out_ws(1,)", + "x_fp8": "?(7, 1024)", + } + else: + assert out == "partial" + _, linear, fp8, dtype = log[3] + assert linear is gdn.out_proj and dtype == torch.bfloat16 + assert fp8.data_ptr() == quant.data_ptr() and fp8.shape == (7, 1024) + + +@pytest.mark.parametrize("front_slot", [False, True]) +def test_the_gdn_step_runs_the_front_where_the_plan_gives_it(monkeypatch, front_slot): + from sgl_nvidia_mega.qwen3_5 import gdn + from sgl_nvidia_mega.qwen3_5.gdn import front, verify + + seen = [] + monkeypatch.setattr(verify, "step", lambda mod, x, st, opts: seen.append("verify")) + monkeypatch.setattr(front, "step", lambda mod, x, st, opts: seen.append(opts)) + st = SimpleNamespace( + slots=plan.Slots(gdn_verify=True, gdn_pdl=True, gdn_front=front_slot) + ) + gdn.step(SimpleNamespace(linear_attn="gdn"), torch.zeros(7, 4), st) + assert seen == ([verify.SLOT_PDL] if front_slot else ["verify"]) + + +def _front_gdn(**change): + """A Qwen3.5 TP8 GatedDeltaNet as SGLang loads the NVFP4 V2 checkpoint (static FP8 + in_proj_qkvz and out_proj, BF16 in_proj_ba), with ``change`` applied.""" + from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp8LinearMethod + from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod + + def fp8(shape): + method = object.__new__(ModelOptFp8LinearMethod) + method.use_marlin = method.use_sm120_fp8 = False + return SimpleNamespace( + quant_method=method, + weight=torch.zeros(shape, dtype=torch.float8_e4m3fn), + input_scale=torch.ones(1), + weight_scale=torch.ones(1), + ) + + gdn = SimpleNamespace( + attn_tp_size=8, + head_k_dim=128, + head_v_dim=128, + conv_kernel_size=4, + activation="silu", + attn=SimpleNamespace( + num_k_heads=2, + num_v_heads=8, + bias=None, + conv_weights=torch.zeros(1536, 4, dtype=torch.bfloat16), + ), + norm=SimpleNamespace( + activation="silu", norm_before_gate=True, bias=None, group_size=None + ), + in_proj_qkvz=fp8((4096, 2560)), + out_proj=fp8((1024, 4096)), + in_proj_ba=SimpleNamespace( + quant_method=object.__new__(UnquantizedLinearMethod), + bias=None, + weight=torch.zeros(16, 4096, dtype=torch.bfloat16), + ), + ) + for path, value in change.items(): + obj, *names = path.split("__") + target = gdn if obj == "gdn" else getattr(gdn, obj) + for name in names[:-1]: + target = getattr(target, name) + setattr(target, names[-1], value) + return gdn + + +@pytest.mark.parametrize( + "change, want", + [ + ({}, ""), + ({"gdn__attn_tp_size": 4}, "TP 4"), + ({"norm__group_size": 2}, "a grouped, biased or gate-first norm"), + ({"attn__bias": torch.zeros(1536)}, "a conv bias"), + ({"in_proj_qkvz__weight": torch.zeros(4096, 2560)}, "in_proj_qkvz is not"), + ({"in_proj_ba__bias": torch.zeros(16)}, "in_proj_ba is not"), + ({"out_proj__input_scale": torch.ones(2)}, "out_proj is not"), + ], +) +def test_the_gdn_front_slot_names_what_a_layer_lacks(change, want): + from sgl_nvidia_mega.qwen3_5.gdn import front + + got = front.unsupported(_front_gdn(**change)) + assert (want in got) if want else got == "" + + +def test_the_gdn_front_slot_names_the_pools_it_cannot_serve(): + from sgl_nvidia_mega.qwen3_5.gdn import front + + def pools(window): + return SimpleNamespace( + conv=(torch.zeros(2, 3, 1536, 3, dtype=torch.bfloat16),), + intermediate_conv_window=(window,), + ) + + phys = torch.zeros(2, 3, 1536, 9, dtype=torch.bfloat16) + dedup = phys.as_strided((2, 3, 7, 1536, 3), (*phys.stride()[:2], 1, 9, 1)) + assert front.pool_reasons(pools(dedup)) == [] + dense = torch.zeros(2, 3, 7, 1536, 3, dtype=torch.bfloat16) + assert "deduplicated" in front.pool_reasons(pools(dense))[0] + + +def test_in_proj_qkvz_packs_into_the_fronts_fragments(): + from sgl_nvidia_mega.qwen3_5.sm103.gdn import front + + w = torch.arange(2560 * 4096, dtype=torch.int64).remainder(251).to(torch.uint8) + packed = front.pack(w.view(2560, 4096).view(torch.float8_e4m3fn)) + assert packed.dtype == torch.uint8 and packed.numel() == 2560 * 4096 + # unit 0, rank 0, warp 0, step 0, block 0, lane 0: the first 16 bytes the CTA loads + want = w.view(2560, 4096) + first = packed[:16].view(4, 4) + assert torch.equal(first[0], want[0, 0:4]) and torch.equal(first[1], want[8, 0:4]) + assert torch.equal(first[2], want[0, 16:20]) and torch.equal( + first[3], want[8, 16:20] + ) + with pytest.raises(ValueError, match="the GDN front packs"): + front.pack(torch.zeros(2560, 4096)) + + +def test_the_fronts_launch_arguments_have_the_kernels_layout(): + import ctypes + + from sgl_nvidia_mega.qwen3_5.sm103.gdn import front + + # gdn_pdl_front_ba_mkm_detail::Params and gdn_ba::Params on a 64-bit target + assert ctypes.sizeof(front.Prefetch) == 32 + assert ctypes.sizeof(front.BaParams) == 208 and front.BaParams.prefetch.offset == 40 + assert front.BaParams.pad_slot.offset == 200 + assert ctypes.sizeof(front.Params) == 376 and front.Params.ba.offset == 168 + assert ( + front.Params.state_stride.offset == 112 and front.Params.pad_slot.offset == 160 + ) + + +def test_the_fronts_l2_prefetch_takes_contiguous_slots_only(): + from sgl_nvidia_mega.qwen3_5.sm103.gdn import front + + t = torch.zeros(3, 8, 16, 128, dtype=torch.bfloat16) + region = front._prefetch(t, 1) + assert (region.slot_stride, region.bytes, region.ring) == ( + 8 * 16 * 128 * 2, + 8 * 16 * 128 * 2, + 1, + ) + assert front._prefetch(t.transpose(2, 3), 0) is None + assert front._prefetch(torch.zeros(3, 7, dtype=torch.bfloat16), 0) is None + + @pytest.mark.parametrize( "kw, want", [ From d147d3d7c1fb5169ede3025324553cbc20d356cb Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:41:00 -0700 Subject: [PATCH 04/27] [Perf] [GB300] qwen3_5: Slot route_shared: the shared expert's gate/up in the router front's launch (numeric) The router front's launch also quantizes the rows to the shared expert's static FP8 and computes its gate/up, so the shared expert on the alternate stream waits for that launch only. Slot route_shared (with route, up to 8 rows); numeric: the gate/up sums in another order than the FP8 linear's GEMM. --- README.md | 1 + sgl_nvidia_mega/qwen3_5/moe/__init__.py | 22 ++- sgl_nvidia_mega/qwen3_5/moe/route.py | 133 ++++++++++++++--- sgl_nvidia_mega/qwen3_5/plan.py | 12 +- sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 6 + sgl_nvidia_mega/qwen3_5/sm103/moe/router.py | 102 +++++++++++-- .../qwen3_5/sm103/moe/src/router_front.cu | 14 +- sgl_nvidia_mega/qwen3_5/spec.py | 4 + test/kernels/test_qwen3_5_moe.py | 68 ++++++++- test/unit/test_qwen3_5_spec.py | 7 + test/unit/test_qwen3_5_steps.py | 141 ++++++++++++++++++ 11 files changed, 478 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index e0710675..1f0dcfe9 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ These slots change the arithmetic (numeric: each sums in another order than SGLa | --- | --- | --- | | `gdn_push` (with `gdn_verify`, `hop`) | 7 | The GatedDeltaNet `out_proj` as the hop's GEMM, whose epilogue pushes the partial into the hop's mailbox | | `gdn_front` (with `gdn_verify`, `gdn_pdl`) | 7 (one request) | The GatedDeltaNet `in_proj_qkvz` (static FP8) and `in_proj_ba` (BF16) projections, the conv update and an L2 prefetch of the verify's state in one launch, then the ReplaySSM verify and the gated norm writing `out_proj`'s FP8 input, each launched dependent on the one before | +| `route_shared` (with `route`) | 1–8 | The shared expert's static-FP8 gate/up in the router front's launch, so the shared expert on the alternate stream waits for that launch only | The sampling and acceptance after the verify, and the GatedDeltaNet extend of prefill, stay SGLang's own code. In an SGLang that has the switches, `SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH=1` runs the sampling and acceptance in CUDA graphs, and `SGLANG_ENABLE_GDN_PREFILL_GRAPH_EXTEND=1` captures the GatedDeltaNet extend inside the breakable prefill CUDA graphs. diff --git a/sgl_nvidia_mega/qwen3_5/moe/__init__.py b/sgl_nvidia_mega/qwen3_5/moe/__init__.py index fdae24a1..abf28068 100644 --- a/sgl_nvidia_mega/qwen3_5/moe/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/moe/__init__.py @@ -12,7 +12,10 @@ alternate stream), otherwise the eager order. With ``defer`` (the boundary takes the finalize) it returns SGLang's ``MoeDeferredFinalize`` handoff, otherwise the summed rows. Slot ``route`` (``route.py``) runs the deferred routed experts on the router -front, and slot ``mkm`` (``mkm.py``) runs them in one launch on its outputs. +front, and slot ``mkm`` (``mkm.py``) runs them in one launch on its outputs. In the +dual-stream order of up to 8 rows, slot ``route_shared`` computes the shared expert's +gate/up in the router front's launch; the alternate stream waits for that launch only and +runs the rest of the shared expert (numeric). """ from __future__ import annotations @@ -72,7 +75,9 @@ def build(model, ctx) -> route.Bound: if "route" not in SLOTS: return route.Bound(reason="stock (SGLANG_NVIDIA_MEGAKERNEL_SLOTS)") - return route.build(model, max(ctx.widths), fused="mkm" in SLOTS) + return route.build( + model, max(ctx.widths), fused="mkm" in SLOTS, shared="route_shared" in SLOTS + ) def step(layer, x: torch.Tensor, st, defer: bool): @@ -125,6 +130,19 @@ def _dual_stream(mlp, x, use_fused_gate: bool, defer: bool, st): shared).""" current = torch.cuda.current_stream() mlp.alt_stream.wait_stream(current) + if defer and st.slots.route_shared: + # the router launch also computes the shared gate/up; the alternate stream + # waits for that launch only + routed, gate_up, ready = route.routed( + mlp, x, st.bound["moe"] if st.slots.mkm else None, shared=True + ) + gate_up.record_stream(mlp.alt_stream) + with torch.cuda.stream(mlp.alt_stream): + mlp.alt_stream.wait_event(ready) + shared = route.shared_expert_tail(mlp.shared_expert, gate_up) + shared = mlp._gate_shared_output_out_of_place(x, shared) + current.wait_stream(mlp.alt_stream) + return routed, shared if defer: # routed FC2 stays on the current stream, the finalize's PDL dependency routed = _router_experts(mlp, x, True, st) diff --git a/sgl_nvidia_mega/qwen3_5/moe/route.py b/sgl_nvidia_mega/qwen3_5/moe/route.py index 3315cec8..c8446476 100644 --- a/sgl_nvidia_mega/qwen3_5/moe/route.py +++ b/sgl_nvidia_mega/qwen3_5/moe/route.py @@ -10,6 +10,12 @@ trtllm runner's own quantizer, so the step's output is the stock one's. The runner takes the quantized input through its pre-quantized (``hidden_states_scale``) path, or slot ``mkm`` (``mkm.py``) takes the router front's outputs in its place. + +Slot ``route_shared`` (numeric) also computes the shared expert's static-FP8 gate/up +projection in the router front's launch (SGLANG_Q35MF_ROUTE_SHARED on SGLang branch +nvidia/qwen35-agentx-gb300), so the shared expert on the alternate stream starts at its +activation, after that launch only; the gate/up sums in another order than the FP8 +linear's GEMM. """ from __future__ import annotations @@ -30,6 +36,15 @@ "experts", "supports_deferred_finalize", "layer_id", + # the shared expert after its gate/up projection (slot route_shared) + "Qwen2MoeMLP", + "shared_expert", + "shared_expert_gate", + "gate_up_proj", + "down_proj", + "act_fn", + "_enable_silu_fp4_quant_fusion", + "_silu_fp4_quant_fused", ), "srt/models/qwen3_5.py": ("layers", "start_layer", "end_layer", "mlp"), "srt/layers/moe/topk.py": ("TopKOutputChecker", "format_is_bypassed"), @@ -53,7 +68,13 @@ '"w13_input_scale_quant"', "quant_config", "use_per_token_activation", + "ModelOptFp8LinearMethod", + "use_marlin", + "use_sm120_fp8", + '"input_scale"', + '"weight_scale"', ), + "srt/layers/activation.py": ("SiluAndMul",), } # The SGLang functions whose body this module mirrors, by the digest of their code @@ -61,6 +82,7 @@ SGLANG_MIRRORS = { "srt/layers/moe/fused_moe_triton/layer.py:FusedMoE.forward_deferred_finalize": "af4b474cbaf33493", "srt/layers/moe/fused_moe_triton/layer.py:FusedMoE._dispatch_with_pre_quant": "dcdefc28cf643081", + "srt/models/qwen2_moe.py:Qwen2MoeMLP.forward": "6091ffb613568a10", } @@ -74,12 +96,14 @@ class Bound(msgspec.Struct, frozen=True): route: bool = False # the router front stands in for every layer's gate mkm: dict = msgspec.field(default_factory=dict) # layer id -> its weight TMA maps identity: object = None # [70] int32 0, 1, ...: slot mkm's rows are in expert order + shared: bool = False # slot route_shared: the shared gate/up in the router launch reason: str = "" # why a requested slot is not built -def build(model, max_rows: int, fused: bool) -> Bound: - """The router front for every MoE layer, compiled for 1..max_rows rows, and with - ``fused`` slot mkm's kernel and weight maps; or why a layer cannot take them.""" +def build(model, max_rows: int, fused: bool, shared: bool = False) -> Bound: + """The router front for every MoE layer, compiled for 1..max_rows rows, with + ``fused`` slot mkm's kernel and weight maps, and with ``shared`` slot route_shared's + launch; or why a layer cannot take them.""" from sgl_nvidia_mega.qwen3_5.moe import mkm from sgl_nvidia_mega.qwen3_5.sm103.moe import mkm as mkm_kernel from sgl_nvidia_mega.qwen3_5.sm103.moe import router @@ -90,10 +114,15 @@ def build(model, max_rows: int, fused: bool) -> Bound: reason = unsupported(mlp) if reason: return Bound(reason=f"layer {i}: {reason}") + if shared: + for i, mlp in zip(range(m.start_layer, m.end_layer), layers): + reason = shared_unsupported(mlp) + if reason: + return Bound(reason=f"route_shared: layer {i}: {reason}") device = torch.cuda.current_device() - router.build(device, max_rows) + router.build(device, max_rows, shared=shared) if not fused: - return Bound(route=True) + return Bound(route=True, shared=shared) for i, mlp in zip(range(m.start_layer, m.end_layer), layers): reason = mkm.unsupported(mlp) if reason: @@ -110,6 +139,7 @@ def build(model, max_rows: int, fused: bool) -> Bound: identity=torch.arange( mkm_kernel.MAX_TOKENS * mkm_kernel.TOP_K, dtype=torch.int32, device="cuda" ), + shared=shared, ) @@ -138,28 +168,97 @@ def unsupported(mlp) -> str: return "" -def routed(mlp, x: torch.Tensor, fused: Bound | None = None): +def shared_unsupported(mlp) -> str: + """Why the router front cannot compute ``mlp``'s shared gate/up ("" when it can): + a static per-tensor FP8 projection of at most 512 rows, then SiLU-and-mul.""" + from sglang.srt.layers.activation import SiluAndMul + from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp8LinearMethod + from sglang.srt.models.qwen2_moe import Qwen2MoeMLP + + from sgl_nvidia_mega.qwen3_5.sm103.moe import router + + expert = mlp.shared_expert + if not isinstance(expert, Qwen2MoeMLP) or mlp.shared_expert_gate is None: + return "no gated Qwen2MoeMLP shared expert" + gu = expert.gate_up_proj + method = gu.quant_method + if ( + not isinstance(method, ModelOptFp8LinearMethod) + or method.use_marlin + or method.use_sm120_fp8 + or gu.bias is not None + or gu.weight.dtype != torch.float8_e4m3fn + or gu.weight.shape[0] != router.HIDDEN + or gu.weight.shape[1] % 8 + or gu.weight.shape[1] > router.SHARED_MAX_N + or not gu.weight.t().is_contiguous() + or gu.input_scale.numel() != 1 + or gu.weight_scale.numel() != 1 + ): + return "the shared gate/up is not a static per-tensor FP8 linear of <= 512 rows" + if not isinstance(expert.act_fn, SiluAndMul): + return f"shared expert activation {type(expert.act_fn).__name__}" + return "" + + +def shared_expert_tail(expert, gate_up: torch.Tensor) -> torch.Tensor: + """``Qwen2MoeMLP.forward`` after its gate_up projection.""" + if expert._enable_silu_fp4_quant_fusion and not isinstance(gate_up, tuple): + x, _ = expert.down_proj(expert._silu_fp4_quant_fused(gate_up)) + return x + x = expert.act_fn(gate_up) + x, _ = expert.down_proj(x) + return x + + +def routed(mlp, x: torch.Tensor, fused: Bound | None = None, shared: bool = False): """``FusedMoE.forward_deferred_finalize`` of ``mlp``'s routed experts on ``x`` [rows, hidden], with the router front's logits and quantized input (``fused``: on - slot mkm's kernel).""" + slot mkm's kernel). With ``shared`` (at most 8 rows) the router launch also computes + the shared expert's gate/up: returns (routed, gate_up, the event recorded on the + current stream right after that launch).""" + from sgl_nvidia_mega.qwen3_5.sm103.moe import router + + experts = mlp.experts + for slot in ("route" if fused is None else "mkm",) + ( + ("route_shared",) if shared else () + ): + if (slot, x.shape[0]) not in _ENGAGED: + _ENGAGED.add((slot, x.shape[0])) + logger.info( + "NVIDIA megakernel: Qwen3.5 slot %s runs at %d rows", slot, x.shape[0] + ) + if shared: + gu = mlp.shared_expert.gate_up_proj + logits, x_fp4, x_sf, gate_up = router.router_gemm_quant_shared( + x, + mlp.gate.weight, + experts.w13_input_scale_quant.reshape(1), + gu.weight.t(), + gu.input_scale.reshape(1), + gu.weight_scale.reshape(1), + ) + ready = torch.cuda.Event() + ready.record() + out = _routed_experts(mlp, x, fused, logits, x_fp4, x_sf) + return out, gate_up, ready + logits, x_fp4, x_sf = router.router_gemm_quant( + x, mlp.gate.weight, experts.w13_input_scale_quant.reshape(1) + ) + return _routed_experts(mlp, x, fused, logits, x_fp4, x_sf) + + +def _routed_experts(mlp, x, fused, logits, x_fp4, x_sf): + """The routed experts on the router front's outputs (``fused``: slot mkm's kernel), + deferred.""" from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( flashinfer_trtllm_deferred_finalize_context, ) from sglang.srt.layers.moe.topk import TopKOutputChecker from sgl_nvidia_mega.qwen3_5.moe import mkm - from sgl_nvidia_mega.qwen3_5.sm103.moe import router experts = mlp.experts - slot = "route" if fused is None else "mkm" - if (slot, x.shape[0]) not in _ENGAGED: - _ENGAGED.add((slot, x.shape[0])) - logger.info( - "NVIDIA megakernel: Qwen3.5 slot %s runs at %d rows", slot, x.shape[0] - ) - logits, x_fp4, x_sf = router.router_gemm_quant( - x, mlp.gate.weight, experts.w13_input_scale_quant.reshape(1) - ) if fused is not None: maps = fused.mkm[mlp.layer_id] return mkm.routed(mlp, maps, fused.identity, logits, x_fp4, x_sf) diff --git a/sgl_nvidia_mega/qwen3_5/plan.py b/sgl_nvidia_mega/qwen3_5/plan.py index 15e30aff..aab3bc9c 100644 --- a/sgl_nvidia_mega/qwen3_5/plan.py +++ b/sgl_nvidia_mega/qwen3_5/plan.py @@ -10,7 +10,8 @@ ``hop`` (the boundary's fused all-reduce points on the decode hop) and ``gdn_push`` (numeric: out_proj as the hop's push GEMM, on gdn_verify's FP8 norm). ``gdn_front`` (numeric: the GDN projections and conv in one launch, then gdn_pdl's verify) covers one -request of 7 rows. +request of 7 rows, ``route_shared`` (numeric: the shared expert's gate/up in route's +launch) up to 8 rows. """ from __future__ import annotations @@ -19,6 +20,9 @@ MAX_ROWS = 7 # the widest covered step: one request of 7 verify rows (TP8 bs1) GDN_FRONT_ROWS = 7 # the rows slot gdn_front's front is built for +ROUTE_SHARED_ROWS = ( + 8 # the rows slot route_shared's shared stage fits shared memory for +) class Slots(msgspec.Struct, frozen=True): @@ -35,6 +39,8 @@ class Slots(msgspec.Struct, frozen=True): gdn_push: bool = False # the projections and the conv in one launch (gdn/front.py; numeric) gdn_front: bool = False + # the shared expert's gate/up in the router launch (moe/route.py; numeric) + route_shared: bool = False class Facts(msgspec.Struct, frozen=True): @@ -48,6 +54,9 @@ class Facts(msgspec.Struct, frozen=True): hop: bool = False # the decode hop, built for the widest covered step gdn_push: bool = False # out_proj packed for the push GEMM on every GDN layer gdn_front: bool = False # in_proj_qkvz packed for the front on every GDN layer + route_shared: bool = ( + False # the router front with the shared stage, every MoE layer + ) def slots(rows: int, facts: Facts) -> Slots: @@ -64,6 +73,7 @@ def slots(rows: int, facts: Facts) -> Slots: and facts.gdn_pdl and facts.gdn_front and rows == GDN_FRONT_ROWS, + route_shared=facts.route and facts.route_shared and rows <= ROUTE_SHARED_ROWS, ) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index fa0e0cb3..6157f989 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -22,6 +22,7 @@ "hop", "gdn_push", "gdn_front", + "route_shared", ) SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" @@ -53,6 +54,10 @@ def slot_reasons() -> list[str]: ) if "mkm" in SLOTS and "route" not in SLOTS: reasons.append(f"{SLOTS_ENV}: slot 'mkm' runs on slot 'route''s outputs") + if "route_shared" in SLOTS and "route" not in SLOTS: + reasons.append( + f"{SLOTS_ENV}: slot 'route_shared' runs in slot 'route''s launch" + ) if "gdn_push" in SLOTS and not {"gdn_verify", "hop"} <= SLOTS: reasons.append( f"{SLOTS_ENV}: slot 'gdn_push' pushes slot 'gdn_verify''s out_proj into " @@ -121,6 +126,7 @@ def build(model, ctx): hop=bound["boundary"].hop is not None, gdn_push=bool(bound["gdn"].packed), gdn_front=bool(bound["gdn"].front), + route_shared=bound["moe"].shared, ) slots = slot_plan.plan(pctx.widths, facts) built = msgspec.structs.asdict(facts) diff --git a/sgl_nvidia_mega/qwen3_5/sm103/moe/router.py b/sgl_nvidia_mega/qwen3_5/sm103/moe/router.py index e99b8a66..c9f035a8 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/moe/router.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/moe/router.py @@ -5,6 +5,10 @@ The logits equal SGLang's cuBLAS router GEMM bit for bit, and the quantization equals the runner's own NVFP4 input quantizer (16-element blocks, e4m3 scales, linear layout). +``router_gemm_quant_shared`` also computes the shared expert's static-FP8 gate/up +projection of up to 8 rows in the same launch (SGLANG_Q35MF_ROUTE_SHARED on SGLang branch +nvidia/qwen35-agentx-gb300); it sums in another order than the FP8 linear's GEMM +(numeric). """ from __future__ import annotations @@ -22,6 +26,11 @@ THREADS, CTAS = 256, 64 # 8 experts a CTA _ROW_BYTES = HIDDEN * 2 + 16 _PART_BYTES = 2 * 16 * 8 * 4 +_SH_ROW_BYTES = ( + HIDDEN + 16 +) # an e4m3 row of the shared stage, padded like the bf16 ones +_SH_PART_BYTES = 4 * 16 * 8 * 4 # fp32 [kShSplits 4][16][8] +SHARED_MAX_M, SHARED_MAX_N = 8, CTAS * 8 # 8 shared rows a CTA MODE_QUANT = 1 @@ -30,10 +39,14 @@ def max_m_bucket(m: int) -> int: return 1 if m <= 1 else 8 if m <= 8 else 16 -def shared_bytes(max_m: int) -> int: - """GemvSmem::kBytes: 8 weight rows, max_m input rows, the fp32 partials and - two mbarriers.""" - return 8 * _ROW_BYTES + max_m * _ROW_BYTES + _PART_BYTES + 16 +def shared_bytes(max_m: int, shared: bool = False) -> int: + """GemvSmem::kBytes: 8 weight rows, max_m input rows, the fp32 partials + and two mbarriers; with the shared stage also 8 FP8 weight rows, max_m FP8 input rows, + its partials and a third mbarrier.""" + plain = 8 * _ROW_BYTES + max_m * _ROW_BYTES + _PART_BYTES + if not shared: + return plain + 16 + return plain + (8 + max_m) * _SH_ROW_BYTES + _SH_PART_BYTES + 24 class Params(ctypes.Structure): @@ -72,23 +85,43 @@ class Params(ctypes.Structure): @cache -def _kernel(device_index: int, max_m: int) -> jit.Kernel: +def _cubin(device_index: int, max_m: int) -> bytes: major, minor = torch.cuda.get_device_capability(device_index) - cubin = jit.compile_source( + return jit.compile_source( SOURCE.read_text(), f"q35_router_front_m{max_m}", f"sm_{major}{minor}a", ("-std=c++17", "-default-device", f"-DQ35_ROUTER_MAX_M={max_m}"), ) + + +@cache +def _kernel(device_index: int, max_m: int) -> jit.Kernel: return jit.load( - cubin, "q35_router_gemm_quant", (THREADS, 1, 1), shared_bytes(max_m) + _cubin(device_index, max_m), + "q35_router_gemm_quant", + (THREADS, 1, 1), + shared_bytes(max_m), ) -def build(device_index: int, max_rows: int) -> None: - """Compile every row capacity up to max_rows (no compile inside a step).""" +@cache +def _shared_kernel(device_index: int, max_m: int) -> jit.Kernel: + return jit.load( + _cubin(device_index, max_m), + "q35_router_gemm_quant_shared", + (THREADS, 1, 1), + shared_bytes(max_m, shared=True), + ) + + +def build(device_index: int, max_rows: int, shared: bool = False) -> None: + """Compile every row capacity up to max_rows, with ``shared`` the shared gate/up + ones too (no compile inside a step).""" for m in sorted({max_m_bucket(r) for r in range(1, max_rows + 1)}): _kernel(device_index, m) + if shared and m <= SHARED_MAX_M: + _shared_kernel(device_index, m) def router_gemm_quant( @@ -120,3 +153,54 @@ def router_gemm_quant( kernel = _kernel(x.device.index, max_m_bucket(m)) jit.launch(kernel, (CTAS, 1, 1), [p], torch.cuda.current_stream().cuda_stream) return logits, x_fp4, x_sf + + +def router_gemm_quant_shared( + x: torch.Tensor, + weight: torch.Tensor, + global_scale: torch.Tensor, + sh_weight: torch.Tensor, + sh_in_scale: torch.Tensor, + sh_w_scale: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """``router_gemm_quant`` of x [M, 4096] (M <= 8) plus the shared expert's gate/up + projection in the same launch: sh_weight [N, 4096] e4m3 row-major (N % 8 == 0, + N <= 512), static scales sh_in_scale / sh_w_scale [1] fp32. Returns (logits, x_fp4, + x_sf, gate_up bf16 [M, N]); gate_up = (q(x) @ sh_weight.T) * sh_in_scale * sh_w_scale + with q the static e4m3 quantization of x, summed in fp32.""" + m, n = x.shape[0], sh_weight.shape[0] + if x.dtype != torch.bfloat16 or x.shape[1:] != (HIDDEN,) or not x.is_contiguous(): + raise ValueError( + f"router front: x {x.dtype} {tuple(x.stride())} {tuple(x.shape)}" + ) + if not 1 <= m <= SHARED_MAX_M: + raise ValueError(f"router front with the shared stage: {m} rows (1-8)") + if ( + sh_weight.shape[1:] != (HIDDEN,) + or n % 8 + or n > SHARED_MAX_N + or not sh_weight.is_contiguous() + ): + raise ValueError(f"router front: shared weight {tuple(sh_weight.shape)}") + logits = torch.empty((m, EXPERTS), dtype=torch.bfloat16, device=x.device) + x_fp4 = torch.empty((m, HIDDEN // 2), dtype=torch.uint8, device=x.device) + x_sf = torch.empty((m, HIDDEN // 16), dtype=torch.uint8, device=x.device) + gate_up = torch.empty((m, n), dtype=torch.bfloat16, device=x.device) + p = Params( + x=x.data_ptr(), + weight=weight.data_ptr(), + global_scale=global_scale.data_ptr(), + logits=logits.data_ptr(), + x_fp4=x_fp4.data_ptr(), + x_sf=x_sf.data_ptr(), + num_tokens=m, + mode=MODE_QUANT, + sh_weight=sh_weight.data_ptr(), + sh_in_scale=sh_in_scale.data_ptr(), + sh_w_scale=sh_w_scale.data_ptr(), + sh_out=gate_up.data_ptr(), + sh_n=n, + ) + kernel = _shared_kernel(x.device.index, max_m_bucket(m)) + jit.launch(kernel, (CTAS, 1, 1), [p], torch.cuda.current_stream().cuda_stream) + return logits, x_fp4, x_sf, gate_up diff --git a/sgl_nvidia_mega/qwen3_5/sm103/moe/src/router_front.cu b/sgl_nvidia_mega/qwen3_5/sm103/moe/src/router_front.cu index 24040b14..41bdce53 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/moe/src/router_front.cu +++ b/sgl_nvidia_mega/qwen3_5/sm103/moe/src/router_front.cu @@ -20,8 +20,9 @@ // routing tables equal flashinfer moe_sort's output on every entry the GEMMs // read. // -// Built with NVRTC: the kernel is a device function, and the extern "C" entry at the -// end instantiates only its router GEMM + NVFP4 input quantization (mode kModeQuant). +// Built with NVRTC: the kernel is a device function, and the extern "C" entries at the +// end instantiate its router GEMM + NVFP4 input quantization (mode kModeQuant), without and +// (up to 8 rows) with the shared expert's gate/up projection. #include @@ -1041,3 +1042,12 @@ extern "C" __global__ __launch_bounds__(sglang::qwen_router_front::kThreads, 1) const __grid_constant__ sglang::qwen_router_front::Params p) { sglang::qwen_router_front::router_front_kernel(p); } + +#if Q35_ROUTER_MAX_M <= 8 +// The same plus the shared expert's static-FP8 gate/up projection of the rows (sh_* in Params) +// in the same launch: CTA c computes its 8 rows while warps 0 and 1 run the router GEMV. +extern "C" __global__ __launch_bounds__(sglang::qwen_router_front::kThreads, 1) void q35_router_gemm_quant_shared( + const __grid_constant__ sglang::qwen_router_front::Params p) { + sglang::qwen_router_front::router_front_kernel(p); +} +#endif diff --git a/sgl_nvidia_mega/qwen3_5/spec.py b/sgl_nvidia_mega/qwen3_5/spec.py index 95ff334d..361e3eb1 100644 --- a/sgl_nvidia_mega/qwen3_5/spec.py +++ b/sgl_nvidia_mega/qwen3_5/spec.py @@ -107,6 +107,10 @@ "gdn_front.in_proj_ba.rel_l2_vs_stock": 1.05, "gdn_front.in_proj_ba.max_abs_vs_stock": 2.0, }, + "route_shared": { + "route_shared.gate_up.rel_l2_vs_stock": 1.05, + "route_shared.gate_up.max_abs_vs_stock": 2.0, + }, } diff --git a/test/kernels/test_qwen3_5_moe.py b/test/kernels/test_qwen3_5_moe.py index 5136d93e..feed304a 100644 --- a/test/kernels/test_qwen3_5_moe.py +++ b/test/kernels/test_qwen3_5_moe.py @@ -7,7 +7,11 @@ runner's quantizer (flashinfer.fp4_quantize, linear scale layout), at 1-16 rows; - the fused routed experts (slot mkm), against FlashInfer's trtllm-gen NVFP4 MoE as SGLang serves Qwen3.5 at TP8 (routing from the logits, RenormalizeNaive, deferred - finalize): each token's 10 expert rows and their weights, at 1, 4 and 7 rows.""" + finalize): each token's 10 expert rows and their weights, at 1, 4 and 7 rows; +- the router front with the shared gate/up stage (slot route_shared, numeric): its + logits and NVFP4 input bit for bit the plain launch's, and its gate/up rows against + fp64 on SGLang's static FP8 quantization of the rows, at most 1.05 x the mean relative + L2 and 2 x the max abs error of FlashInfer's FP8 BMM, at 1-8 rows; graph replay.""" import pytest import torch @@ -224,3 +228,65 @@ def test_router_gemm_quant_replays_from_a_cuda_graph(): graph.replay() torch.cuda.synchronize() assert all(torch.equal(a, b) for a, b in zip(eager, captured)) + + +def _shared_weights(seed: int, n: int = 256): + """The shared expert's static-FP8 gate/up at TP8 (2 x 128 rows a rank): [N, 4096] + e4m3 row-major, and its input and weight scales.""" + g = torch.Generator(device="cuda").manual_seed(1000 + seed) + w = torch.randn(n, router.HIDDEN, device="cuda", generator=g) * 0.05 + return ( + w.clamp(-440, 440).to(torch.float8_e4m3fn), + torch.tensor([0.01], device="cuda"), + torch.tensor([0.002], device="cuda"), + ) + + +@pytest.mark.parametrize("m", list(range(1, router.SHARED_MAX_M + 1))) +def test_router_gemm_quant_shared_holds_its_gate_up_to_fp64(m): + from sglang.kernels.ops.quantization.fp8_kernel import static_quant_fp8 + from sglang.srt.layers.quantization.fp8_utils import flashinfer_bmm_fp8 + + errors = {"got": [], "stock": []} + for seed in range(3): + x, w, scale = _router_inputs(m, seed=100 * m + seed) + x[-1, 32:48] /= 1e4 # keep the shared projection's input in e4m3's range + sh_w, sh_in, sh_ws = _shared_weights(seed) + *routed, gate_up = router.router_gemm_quant_shared( + x, w, scale, sh_w, sh_in, sh_ws + ) + for name, a, b in zip( + ("logits", "x_fp4", "x_sf"), routed, router.router_gemm_quant(x, w, scale) + ): + assert torch.equal(a, b), name + xq, _ = static_quant_fp8(x, sh_in, repeat_scale=False) + ref = (xq.double() @ sh_w.double().t()) * (sh_in.double() * sh_ws.double()) + stock = flashinfer_bmm_fp8(xq, sh_w.t(), sh_in, sh_ws, torch.bfloat16) + for name, t in (("got", gate_up), ("stock", stock)): + diff = t.double() - ref + errors[name].append( + ((diff.norm() / ref.norm()).item(), diff.abs().max().item()) + ) + got, stock = ( + torch.tensor(errors[n], dtype=torch.float64) for n in ("got", "stock") + ) + print( + f"T0 route_shared m={m}: mean rel-L2 {got[:, 0].mean():.4g} vs stock " + f"{stock[:, 0].mean():.4g}, max abs {got[:, 1].max():.4g} vs {stock[:, 1].max():.4g}", + flush=True, + ) + assert got[:, 0].mean() <= 1.05 * stock[:, 0].mean() + assert got[:, 1].max() <= 2 * stock[:, 1].max() + + +def test_router_gemm_quant_shared_replays_from_a_cuda_graph(): + x, w, scale = _router_inputs(7, seed=0) + sh_w, sh_in, sh_ws = _shared_weights(0) + router.build(x.device.index, 7, shared=True) + eager = router.router_gemm_quant_shared(x, w, scale, sh_w, sh_in, sh_ws) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = router.router_gemm_quant_shared(x, w, scale, sh_w, sh_in, sh_ws) + graph.replay() + torch.cuda.synchronize() + assert all(torch.equal(a, b) for a, b in zip(eager, captured)) diff --git a/test/unit/test_qwen3_5_spec.py b/test/unit/test_qwen3_5_spec.py index 3c4b653d..bb0ff975 100644 --- a/test/unit/test_qwen3_5_spec.py +++ b/test/unit/test_qwen3_5_spec.py @@ -138,6 +138,7 @@ def test_an_sglang_without_the_eagle_forward_view_is_refused(monkeypatch): {"gdn_verify", "gdn_front"}, "slot 'gdn_front' runs slot 'gdn_pdl''s dependent verify", ), + ({"route_shared"}, "slot 'route_shared' runs in slot 'route''s launch"), ], ) def test_a_slot_set_that_cannot_run_is_refused_before_the_load( @@ -170,6 +171,10 @@ def test_the_slot_plan_covers_one_request_of_7_rows(): assert plan.plan((7,), plan.Facts(**front)) == {7: plan.Slots(**front)} assert not plan.slots(4, plan.Facts(**front)).gdn_front assert not plan.slots(7, plan.Facts(gdn_verify=True, gdn_front=True)).gdn_front + # route_shared rides route's launch, up to 8 rows + shared = dict(route=True, route_shared=True) + assert plan.plan((7,), plan.Facts(**shared)) == {7: plan.Slots(**shared)} + assert not plan.slots(7, plan.Facts(route_shared=True)).route_shared with pytest.raises(ValueError): plan.slots(plan.MAX_ROWS + 1, plan.Facts()) @@ -181,6 +186,8 @@ def test_a_numeric_slot_turns_the_claim_into_its_tolerances(): claim = spec.numerics(frozenset({"gdn_verify", "hop", "gdn_push"})) assert claim.kind is NumericsKind.TOLERANCE assert dict(claim.tolerances) == spec.NUMERIC_TOLERANCES["gdn_push"] + shared = spec.numerics(frozenset({"route", "route_shared"})) + assert dict(shared.tolerances) == spec.NUMERIC_TOLERANCES["route_shared"] both = spec.numerics(frozenset({"gdn_verify", "gdn_pdl", "gdn_front", "gdn_push"})) assert dict(both.tolerances) == { **spec.NUMERIC_TOLERANCES["gdn_push"], diff --git a/test/unit/test_qwen3_5_steps.py b/test/unit/test_qwen3_5_steps.py index 7a050c40..425a9a08 100644 --- a/test/unit/test_qwen3_5_steps.py +++ b/test/unit/test_qwen3_5_steps.py @@ -302,6 +302,147 @@ def test_the_route_slot_replaces_only_the_gate_and_the_routed_experts(capture): assert got_out == stock_out == "handoff" +class _GateUp: + """The shared gate/up rows the router front returns, logging the stream they are + recorded on.""" + + def __init__(self, log): + self.log = log + + def record_stream(self, stream): + self.log.append(("record_stream", stream.name)) + + +def test_the_route_shared_slot_starts_the_shared_expert_after_the_router_launch(): + """Slot route_shared at graph capture with the deferred finalize: the router launch + also computes the shared gate/up; the alternate stream waits for that launch only (an + event recorded right after it), then runs the activation, down_proj and the gate, + while the routed experts run on the main stream.""" + from sgl_nvidia_mega.qwen3_5.sm103.moe import router + + x = torch.zeros(7, 4) + log = _log(x=x) + mlp = _moe_block(log, x) + _, front, context = _router_front(log, mlp) + k = log.known + k.update( + # ModelOpt keeps the weight as the [K, N] transpose of a row-major [N, K] + sh_w=torch.zeros(256, 4096, dtype=torch.float8_e4m3fn).t(), + sh_in=torch.ones(()), + sh_ws=torch.ones(()), + ) + gate_up = _GateUp(log) + k["gate_up"] = gate_up + mlp.shared_expert = SimpleNamespace( + gate_up_proj=SimpleNamespace( + weight=k["sh_w"], input_scale=k["sh_in"], weight_scale=k["sh_ws"] + ), + _enable_silu_fp4_quant_fusion=False, + act_fn=log.op("act", "act_out"), + down_proj=log.op("down_proj", ("down_out", None)), + ) + k["act_out"] = "act_out" + + def front_shared(x_, w, scale, sh_w, sh_in, sh_ws): + log.op("router_front_shared")(x_, w, scale, sh_w, sh_in, sh_ws) + assert sh_w.shape == (256, 4096) and sh_w.is_contiguous() + assert sh_in.shape == sh_ws.shape == (1,) + return k["logits"], k["x_fp4"], k["x_sf"], gate_up + + class _Event: + def record(self): + log.append(("event_record", log.stream)) + + def wait_event(self, event): + self.log.append(("wait_event", self.name)) + + with ( + _streams(log, True), + _moe_kernels(log), + front, + context, + patch.object(router, "router_gemm_quant_shared", front_shared), + patch.object(torch.cuda, "Event", _Event), + patch.object(_Stream, "wait_event", wait_event, create=True), + ): + st = SimpleNamespace(slots=plan.Slots(route=True, route_shared=True)) + out = moe.step(SimpleNamespace(mlp=mlp), x, st, defer=True) + assert log.tag(out) == "handoff" + ops = [(e[0], e[1]) for e in log] + assert ops == [ + ("wait", "alt"), + ("router_front_shared", "main"), + ("event_record", "main"), + ("topk", "main"), + ("dispatch", "main"), + ("replace", "main"), + ("deferred_context", "main"), + ("run_moe_core", "main"), + ("combine", "main"), + ("record_stream", "alt"), + ("wait_event", "alt"), + ("act", "alt"), + ("down_proj", "alt"), + ("gate_mul", "alt"), + ("wait", "main"), + ("deferred_finalize", "main"), + ] + assert log[1][2] == ( + "x(7, 4)", + "gate_w(512, 4096)", + "scale(1,)", + "sh_w(256, 4096)", + "sh_in(1,)", + "sh_ws(1,)", + ) + assert log[11][2] == ("gate_up",) and log[12][2] == ("act_out",) + + +def test_the_router_fronts_shared_stage_fits_shared_memory(): + from sgl_nvidia_mega.qwen3_5.sm103.moe import router + + # GemvSmem::kBytes: + 8 and max_m e4m3 rows of 4112 B, 2 KiB of + # partials and a third mbarrier + assert router.shared_bytes(8) == 131328 + 1024 + 16 + assert router.shared_bytes(8, shared=True) == 200216 + assert router.shared_bytes(1, shared=True) == 113976 + + +@pytest.mark.parametrize( + "change, want", + [ + ({}, ""), + ({"act_fn": object()}, "shared expert activation object"), + ({"bias": torch.zeros(256)}, "not a static per-tensor FP8 linear"), + ({"rows": 1024}, "not a static per-tensor FP8 linear"), + ], +) +def test_the_route_shared_slot_names_what_a_layer_lacks(change, want): + from sglang.srt.layers.activation import SiluAndMul + from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp8LinearMethod + + from sgl_nvidia_mega.qwen3_5.moe import route + + method = object.__new__(ModelOptFp8LinearMethod) + method.use_marlin = method.use_sm120_fp8 = False + expert = qwen2_moe.Qwen2MoeMLP.__new__(qwen2_moe.Qwen2MoeMLP) + torch.nn.Module.__init__(expert) + rows = change.get("rows", 256) + expert.__dict__.update( + gate_up_proj=SimpleNamespace( + quant_method=method, + bias=change.get("bias"), + weight=torch.zeros(rows, 4096, dtype=torch.float8_e4m3fn).t(), + input_scale=torch.ones(1), + weight_scale=torch.ones(1), + ), + act_fn=change.get("act_fn", object.__new__(SiluAndMul)), + ) + mlp = SimpleNamespace(shared_expert=expert, shared_expert_gate="gate") + got = route.shared_unsupported(mlp) + assert (want in got) if want else got == "" + + @pytest.mark.parametrize("capture", [False, True]) def test_the_mkm_slot_runs_the_routed_experts_on_the_router_fronts_outputs(capture): """Slot mkm: the router front, then one fused launch whose rows and weights reach From f8b4b09da3495924d1d562b41dc5ab132f8f7e94 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:42:00 -0700 Subject: [PATCH 05/27] [Perf] [GB300] qwen3_5: Slot skv2: the full attention's core on the split-KV attention (numeric) After the fused FP8 K / V write, the full attention's core runs SKV2, the split-KV attention of SGLang branch nvidia/qwen35-agentx-gb300 (a CuTe DSL producer over 148 page splits and a Triton merge); its KV-length graph variants keep SGLang's attention below 8192 tokens. Slot skv2; numeric: SKV2 splits the KV and sums in another order than trtllm-gen. --- sgl_nvidia_mega/qwen3_5/attention/__init__.py | 52 +- sgl_nvidia_mega/qwen3_5/attention/route.py | 88 +++ sgl_nvidia_mega/qwen3_5/attention/split_kv.py | 204 +++++ sgl_nvidia_mega/qwen3_5/plan.py | 7 +- sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 8 +- sgl_nvidia_mega/qwen3_5/runtime/executor.py | 14 + .../qwen3_5/sm103/attention/__init__.py | 5 + .../qwen3_5/sm103/attention/skv2.py | 704 ++++++++++++++++++ sgl_nvidia_mega/qwen3_5/spec.py | 2 + test/kernels/test_qwen3_5_attention.py | 145 ++++ test/unit/test_qwen3_5_spec.py | 5 + test/unit/test_qwen3_5_steps.py | 306 +++++++- 12 files changed, 1534 insertions(+), 6 deletions(-) create mode 100644 sgl_nvidia_mega/qwen3_5/attention/route.py create mode 100644 sgl_nvidia_mega/qwen3_5/attention/split_kv.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/attention/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/attention/skv2.py create mode 100644 test/kernels/test_qwen3_5_attention.py diff --git a/sgl_nvidia_mega/qwen3_5/attention/__init__.py b/sgl_nvidia_mega/qwen3_5/attention/__init__.py index 3ac82765..e392aea1 100644 --- a/sgl_nvidia_mega/qwen3_5/attention/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/attention/__init__.py @@ -9,14 +9,21 @@ ops in its order: the qkv projection with the fused q / k Gemma RMSNorm + RoPE + gate split, the attention through the backend (which writes this step's KV rows), the sigmoid output gate and ``o_proj``. ``step`` returns the TP-partial output, whose -all-reduce the boundary runs. +all-reduce the boundary runs. Slot ``skv2`` (``split_kv.py``) runs the attention's core +through the split-KV attention of SGLang branch nvidia/qwen35-agentx-gb300. """ from __future__ import annotations +import msgspec + # The SGLang names this module reads (test/unit/test_contract.py pins them) SGLANG_READS = { "srt/models/qwen3_5.py": ( + "Qwen3_5AttentionDecoderLayer", + "layers", + "start_layer", + "end_layer", "_prepare_qkv_gate", "_prepare_qkv_gate(positions)", "_prepare_qkv_gate(hidden_states)", @@ -37,6 +44,42 @@ "srt/models/qwen3_5.py:Qwen3_5AttentionDecoderLayer.self_attention": "db0f4d98e0fb1abf", } +ATTENTION_SLOTS = ("skv2",) + + +class Bound(msgspec.Struct, frozen=True): + """The attention package's build result (``st.bound["attention"]``).""" + + skv2: bool = False # slot skv2 serves every full-attention layer + reason: str = "" # why a requested slot is not built + + +def build(model, ctx) -> Bound: + """The requested attention slots for every full-attention layer, or why a layer + cannot take them.""" + from sglang.srt.models.qwen3_5 import Qwen3_5AttentionDecoderLayer + + from sgl_nvidia_mega.qwen3_5.attention import split_kv + from sgl_nvidia_mega.qwen3_5.runtime import SLOTS + + if not SLOTS & set(ATTENTION_SLOTS): + return Bound(reason="stock (SGLANG_NVIDIA_MEGAKERNEL_SLOTS)") + m = model.model + for i in range(m.start_layer, m.end_layer): + layer = m.layers[i] + if isinstance(layer, Qwen3_5AttentionDecoderLayer) and "skv2" in SLOTS: + reason = split_kv.unsupported(layer.attn, ctx.tokens_per_request) + if reason: + return Bound(reason=f"layer {i}: {reason}") + return Bound(skv2="skv2" in SLOTS) + + +def pool_reasons(view, binding) -> list[str]: + """Why the attention slots cannot read this forward's KV pools ([] when they can).""" + from sgl_nvidia_mega.qwen3_5.attention import split_kv + + return split_kv.pool_reasons(view.kv, binding.page_size) + def step(layer, x, st): """``layer``'s attention on the normed input ``x`` [rows, hidden]: the TP-partial @@ -46,7 +89,12 @@ def step(layer, x, st): q, k, v, gate = layer._prepare_qkv_gate( positions=st.positions, hidden_states=x, forward_batch=st.fb ) - o = layer.attn(q, k, v, st.fb) + if st.slots.skv2: + from sgl_nvidia_mega.qwen3_5.attention import split_kv + + o = split_kv.attend(layer.attn, q, k, v, st) + else: + o = layer.attn(q, k, v, st.fb) if layer.attn_output_gate: o = fused_sigmoid_mul(o, gate, inplace=True) out, _ = layer.o_proj(o) diff --git a/sgl_nvidia_mega/qwen3_5/attention/route.py b/sgl_nvidia_mega/qwen3_5/attention/route.py new file mode 100644 index 00000000..bf7595f1 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/attention/route.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang branch nvidia/qwen35-agentx-gb300: +# python/sglang/srt/layers/attention/graph_variants.py (SplitKVGraphVariants, +# _split_kv_min_length, use_split_kv_attention). +"""Which verify steps take split-KV attention: one request whose KV holds at least +``min_length`` tokens, by a host-side lower bound of its length, so a captured graph is +picked without reading the device. + +``VERIFY`` (8192 tokens, as the TP8 serving configuration on SGLang branch +nvidia/qwen35-agentx-gb300 sets ``SGLANG_Q35MF_MKA_SKV2_VERIFY_MIN_LENGTH``) routes the +target verify and ``EXTEND`` (32768, that branch's ``SPLIT_KV_MIN_LENGTH``) the draft +extend. A built megakernel offers its variants per SGLang runner (``graph_variants``: +"target_verify", "draft_loop", "draft_extend"); a runner captures each of +``capture_labels`` in order and replays the one ``select`` picks. A step reads the label +being captured; where SGLang captures none (no runner takes the variants yet), every +covered step takes split-KV attention. +""" + +from __future__ import annotations + +from typing import ClassVar + +import msgspec + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/model_executor/runner_utils/capture_mode.py": ( + "get_capture_attention_variant", + "get_is_capture_mode", + ), + "srt/model_executor/forward_batch_info.py": ("batch_size", "seq_lens_cpu"), +} +# select also reads ForwardBatch.attention_kv_len_lower_bound (the request's committed +# KV length), which SGLang adds with the runners that take these variants + +SPLIT_KV_STOCK = "split_kv_stock" +SPLIT_KV_LONG = "split_kv_long" + + +class SplitKVGraphVariants(msgspec.Struct, frozen=True): + """SGLang's ``AttentionGraphVariants``: short sequences stay on SGLang's attention + without a device-to-host copy.""" + + min_length: int + # capture order matters where the variants share a graph memory pool + capture_labels: ClassVar[tuple[str, ...]] = (SPLIT_KV_STOCK, SPLIT_KV_LONG) + + def select(self, forward_batch) -> str: + if forward_batch.batch_size != 1: + return SPLIT_KV_STOCK + length = forward_batch.attention_kv_len_lower_bound + if length is None: + lengths = forward_batch.seq_lens_cpu + if ( + lengths is not None + and lengths.device.type == "cpu" + and lengths.numel() == 1 + ): + length = int(lengths[0]) + return ( + SPLIT_KV_LONG + if length is not None and length >= self.min_length + else SPLIT_KV_STOCK + ) + + +VERIFY = SplitKVGraphVariants(min_length=8192) +EXTEND = SplitKVGraphVariants(min_length=32768) + + +def split_kv(variants: SplitKVGraphVariants, forward_batch) -> bool: + """Whether this step takes split-KV attention: the label being captured, else (an + eager step) ``variants.select``; every step where SGLang captures no label of + ours or keeps no host lower bound.""" + from sglang.srt.model_executor.runner_utils.capture_mode import ( + get_capture_attention_variant, + get_is_capture_mode, + ) + + label = get_capture_attention_variant() + if label in SplitKVGraphVariants.capture_labels: + return label == SPLIT_KV_LONG + if get_is_capture_mode() or not hasattr( + forward_batch, "attention_kv_len_lower_bound" + ): + return True + return variants.select(forward_batch) == SPLIT_KV_LONG diff --git a/sgl_nvidia_mega/qwen3_5/attention/split_kv.py b/sgl_nvidia_mega/qwen3_5/attention/split_kv.py new file mode 100644 index 00000000..c61e5bbb --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/attention/split_kv.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/srt/layers/attention/trtllm_mha_backend.py (TRTLLMHAAttnBackend.forward_extend, +# _fused_fp8_qkv_kv_cache, _get_bmm_scales, _reshape_paged_kv_cache), and from SGLang branch +# nvidia/qwen35-agentx-gb300: the same file's _run_fixed_q_len_decode and _run_skv2 +# (use_split_kv with use_skv2). +"""Slot ``skv2``: the full attention of a covered target-verify step with its core +through SKV2 (``sm103/attention/skv2.py``), the split-KV attention of SGLang branch +nvidia/qwen35-agentx-gb300: a CuTe DSL producer over 148 page splits and the Triton +merge, bit for bit the branch's. + +``attend`` stands in for ``layer.attn(q, k, v, forward_batch)``: as TRT-LLM MHA's +``forward_extend`` does for a target verify on an FP8 KV cache, it writes the step's +K / V rows and quantizes Q in one ``fused_fp8_qkv_kv_cache``, then runs ``core``. +``core`` takes an FP8 Q whose K / V rows are written (slot ``mka``'s prologue writes +them itself) and runs SKV2 where the step takes split-KV attention (``route``), +SGLang's attention otherwise. It reads the page table, the KV lengths and the pool from +the forward's view (``Step.view``). +""" + +from __future__ import annotations + +import logging + +import torch + +logger = logging.getLogger(__name__) + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/layers/radix_attention.py": ( + "layer_id", + "k_scale", + "v_scale", + "tp_q_head_num", + "tp_k_head_num", + "tp_v_head_num", + "head_dim", + "scaling", + "sliding_window_size", + ), + "kernels/ops/kvcache/fused_fp8_qkv_kv_cache.py": ( + "fused_fp8_qkv_kv_cache", + "fused_fp8_qkv_kv_cache(q)", + "fused_fp8_qkv_kv_cache(k)", + "fused_fp8_qkv_kv_cache(v)", + "fused_fp8_qkv_kv_cache(k_cache)", + "fused_fp8_qkv_kv_cache(v_cache)", + "fused_fp8_qkv_kv_cache(cache_loc)", + "fused_fp8_qkv_kv_cache(k_scale)", + "fused_fp8_qkv_kv_cache(v_scale)", + ), + "kernels/ops/attention/utils.py": ("canonicalize_stride",), + "srt/megakernels/eagle_binding.py": ( + "attn", + "kv", + "page_table", + "cache_seqlens", + "layer_ids", + "k_scale", + "v_scale", + "page_size", + "tokens_per_request", + ), + "srt/model_executor/forward_batch_info.py": ("out_cache_loc",), + "srt/environ.py": ("envs", "SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR"), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + "srt/layers/attention/trtllm_mha_backend.py:TRTLLMHAAttnBackend.forward_extend": "8e36076dafabff66", + "srt/layers/attention/trtllm_mha_backend.py:TRTLLMHAAttnBackend._should_use_fused_fp8_path": "e0b14791d523a019", + "srt/layers/attention/trtllm_mha_backend.py:TRTLLMHAAttnBackend._fused_fp8_qkv_kv_cache": "363661c3585e6a0f", + "srt/layers/attention/trtllm_mha_backend.py:TRTLLMHAAttnBackend._get_bmm_scales": "90a1387e8f561611", + "srt/layers/attention/trtllm_mha_backend.py:TRTLLMHAAttnBackend._reshape_paged_kv_cache": "53ae20937769b99a", +} + +PAGE_SIZE, HEAD_DIM, ROWS = 64, 256, 7 +# the producer's compiled plans and storage, per input layout (as the branch keeps them) +_PLANS: dict = {} +# the FP8 KV scale of a layer without checkpoint scales (TRT-LLM MHA's default) +_ONES: dict = {} +# the rows the slot has run at, by route (each logged once, as engagement evidence) +_ENGAGED: set = set() + + +def unsupported(attn, tokens_per_request: int) -> str: + """Why SKV2 cannot run this RadixAttention ("" when it can): the branch's + ``_run_fixed_q_len_decode`` support check, on what build knows.""" + from sglang.srt.environ import envs + + if tokens_per_request != ROWS: + return f"{tokens_per_request} rows a request (SKV2 runs 7)" + if attn.tp_q_head_num not in (4, 8) or attn.tp_k_head_num != 1: + return ( + f"{attn.tp_q_head_num} query / {attn.tp_k_head_num} KV heads a rank " + "(SKV2 runs 4 or 8 / 1)" + ) + if attn.head_dim != HEAD_DIM: + return f"head_dim {attn.head_dim} (SKV2 runs 256)" + if attn.sliding_window_size >= 0: + return f"a sliding window of {attn.sliding_window_size}" + if envs.SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR.get() not in (None, 0.0): + return "SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR is set" + if torch.cuda.get_device_capability()[0] != 10: + return "SKV2 runs on SM10x" + return "" + + +def pool_reasons(kv, page_size: int) -> list[str]: + """Why SKV2 cannot read these KV pools ([] when it can).""" + reasons = [] + if page_size != PAGE_SIZE: + reasons.append(f"page size {page_size} (SKV2 reads pages of 64)") + for buf in (*kv.k, *kv.v): + if buf.dtype != torch.float8_e4m3fn or not buf.is_contiguous(): + reasons.append(f"a {buf.dtype} KV pool buffer that is not contiguous FP8") + break + if buf.numel() % (PAGE_SIZE * HEAD_DIM): + reasons.append(f"a KV pool buffer of shape {tuple(buf.shape)}") + break + return reasons + + +def attend(attn, q, k, v, st): + """``attn(q, k, v, st.fb)`` for a covered verify step: the attention output + [rows, heads * head_dim].""" + from sglang.kernels.ops.kvcache.fused_fp8_qkv_kv_cache import ( + fused_fp8_qkv_kv_cache, + ) + + i = st.view.kv.layer_ids.index(attn.layer_id) + ones = _ONES.get(q.device) + if ones is None: + ones = _ONES[q.device] = torch.ones(1, dtype=torch.float32, device=q.device) + q = fused_fp8_qkv_kv_cache( + q=q, + k=k, + v=v, + k_cache=st.view.kv.k[i], + v_cache=st.view.kv.v[i], + cache_loc=st.fb.out_cache_loc, + k_scale=attn.k_scale if attn.k_scale is not None else ones, + v_scale=attn.v_scale if attn.v_scale is not None else ones, + ) + return core(attn, q, st) + + +def core(attn, q, st): + """The attention of the FP8 query rows ``q`` over the KV cache, whose rows for this + step are written: [rows, heads * head_dim] in bf16.""" + from sgl_nvidia_mega.qwen3_5.attention import route + + split = route.split_kv(route.VERIFY, st.fb) + if (split, q.shape[0]) not in _ENGAGED: + _ENGAGED.add((split, q.shape[0])) + logger.info( + "NVIDIA megakernel: Qwen3.5 slot skv2 runs at %d rows (%s)", + q.shape[0], + "SKV2" if split else "SGLang's attention below the split-KV length", + ) + if not split: + return attn(q, None, None, st.fb) + return skv2(attn, q, st.view.kv, st.view.attn) + + +def skv2(attn, q, kv, meta): + """SKV2 on one request's 7 FP8 query rows ``q`` of ``attn`` over the pool ``kv`` + (KvPoolBuffers) at ``meta``'s pages and lengths (AttnStepMetadata), whose rows for + this step are written: [rows, heads * head_dim] in bf16. The draft extend's + attention (``EXTEND``'s route) runs it on the draft's pool too.""" + from sglang.kernels.ops.attention.utils import canonicalize_stride + + from sgl_nvidia_mega.qwen3_5.sm103.attention.skv2 import StockSplitPlan + + i = kv.layer_ids.index(attn.layer_id) + # the pool's NHD rows as HND pages, as TRT-LLM MHA hands them to its decode kernel + caches = tuple( + canonicalize_stride( + buf.view(-1, PAGE_SIZE, heads, attn.head_dim).permute(0, 2, 1, 3) + ) + for buf, heads in ((kv.k[i], attn.tp_k_head_num), (kv.v[i], attn.tp_v_head_num)) + ) + q = q.reshape(-1, attn.tp_q_head_num, attn.head_dim) + table, lengths = meta.page_table, meta.cache_seqlens + key = (q.shape, caches[0].shape, table.shape, table.stride(), lengths.shape) + plan = _PLANS.get(key) + if plan is None: + # compiles the producer: the first covered step runs eagerly, before capture + plan = _PLANS[key] = StockSplitPlan(q, caches, table, lengths) + out = torch.empty_like(q, dtype=torch.bfloat16) + # TRT-LLM MHA's BMM scales with Q unscaled: q_scale * k_scale * scaling, v_scale + plan.run( + q, + caches, + table, + lengths, + qk_scale=float(1.0 * kv.k_scale[i] * attn.scaling), + v_scale=float(kv.v_scale[i]), + out=out, + ) + return out.view(-1, attn.tp_q_head_num * attn.head_dim) diff --git a/sgl_nvidia_mega/qwen3_5/plan.py b/sgl_nvidia_mega/qwen3_5/plan.py index aab3bc9c..9fe476cf 100644 --- a/sgl_nvidia_mega/qwen3_5/plan.py +++ b/sgl_nvidia_mega/qwen3_5/plan.py @@ -11,7 +11,9 @@ (numeric: out_proj as the hop's push GEMM, on gdn_verify's FP8 norm). ``gdn_front`` (numeric: the GDN projections and conv in one launch, then gdn_pdl's verify) covers one request of 7 rows, ``route_shared`` (numeric: the shared expert's gate/up in route's -launch) up to 8 rows. +launch) up to 8 rows. ``skv2`` runs the +full attention's core through the split-KV attention of SGLang branch +nvidia/qwen35-agentx-gb300. """ from __future__ import annotations @@ -41,6 +43,7 @@ class Slots(msgspec.Struct, frozen=True): gdn_front: bool = False # the shared expert's gate/up in the router launch (moe/route.py; numeric) route_shared: bool = False + skv2: bool = False # the full attention's core through SKV2 (attention/split_kv.py) class Facts(msgspec.Struct, frozen=True): @@ -57,6 +60,7 @@ class Facts(msgspec.Struct, frozen=True): route_shared: bool = ( False # the router front with the shared stage, every MoE layer ) + skv2: bool = False # SKV2 for every full-attention layer def slots(rows: int, facts: Facts) -> Slots: @@ -74,6 +78,7 @@ def slots(rows: int, facts: Facts) -> Slots: and facts.gdn_front and rows == GDN_FRONT_ROWS, route_shared=facts.route and facts.route_shared and rows <= ROUTE_SHARED_ROWS, + skv2=facts.skv2, ) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index 6157f989..81cf38b6 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -23,6 +23,7 @@ "gdn_push", "gdn_front", "route_shared", + "skv2", ) SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" @@ -98,7 +99,7 @@ def build(model, ctx): SGLang's BuildContext.""" import torch - from sgl_nvidia_mega.qwen3_5 import boundary, gdn, moe + from sgl_nvidia_mega.qwen3_5 import attention, boundary, gdn, moe from sgl_nvidia_mega.qwen3_5 import plan as slot_plan from sgl_nvidia_mega.qwen3_5.runtime import parity from sgl_nvidia_mega.qwen3_5.runtime.executor import Qwen35Verify @@ -117,6 +118,7 @@ def build(model, ctx): "moe": moe.build(model, pctx), "gdn": gdn.build(model, pctx), "boundary": boundary.build(model, pctx), + "attention": attention.build(model, pctx), } facts = slot_plan.Facts( route=bound["moe"].route, @@ -127,6 +129,7 @@ def build(model, ctx): gdn_push=bool(bound["gdn"].packed), gdn_front=bool(bound["gdn"].front), route_shared=bound["moe"].shared, + skv2=bound["attention"].skv2, ) slots = slot_plan.plan(pctx.widths, facts) built = msgspec.structs.asdict(facts) @@ -136,7 +139,8 @@ def build(model, ctx): raise RuntimeError( f"NVIDIA megakernel: requested slots {missing} built no width (MoE: " f"{bound['moe'].reason or 'built'}; GDN: {bound['gdn'].reason or 'built'}; " - f"boundary: {bound['boundary'].reason or 'built'})" + f"boundary: {bound['boundary'].reason or 'built'}; " + f"attention: {bound['attention'].reason or 'built'})" ) checker = ( parity.Parity(pctx.widths, torch.cuda.current_device(), model) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/executor.py b/sgl_nvidia_mega/qwen3_5/runtime/executor.py index 34174dca..10db108f 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/executor.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/executor.py @@ -32,6 +32,7 @@ from sglang.srt.megakernels.spec import Step from sgl_nvidia_mega.qwen3_5 import attention, boundary, gdn, logits, moe +from sgl_nvidia_mega.qwen3_5.attention import route as _attention_route # noqa: F401 from sgl_nvidia_mega.qwen3_5.gdn import verify as _gdn_verify # noqa: F401 from sgl_nvidia_mega.qwen3_5.runtime import embedding from sgl_nvidia_mega.qwen3_5.runtime.state import StepState @@ -111,6 +112,13 @@ def __init__( self.binding = binding # SGLang's EagleBinding (BuildContext.binding) self._summary = summary self.parity = parity # runtime.parity.Parity under the test-only switch + # the attention graph variants per SGLang runner ("target_verify"), which it + # captures once it takes a megakernel's (attention/route.py) + self.graph_variants = ( + {"target_verify": attention.route.VERIFY} + if any(s.skv2 for s in slots.values()) + else {} + ) self._pools_checked = False m = lm.model self.mixers = { @@ -179,6 +187,12 @@ def _check_pools(self, view): "NVIDIA megakernel: slot gdn_front cannot run: " + "; ".join(reasons) ) + if any(s.skv2 for s in self.slots.values()): + reasons = attention.pool_reasons(view, self.binding) + if reasons: + raise RuntimeError( + "NVIDIA megakernel: slot skv2 cannot run: " + "; ".join(reasons) + ) self._pools_checked = True @torch.no_grad() diff --git a/sgl_nvidia_mega/qwen3_5/sm103/attention/__init__.py b/sgl_nvidia_mega/qwen3_5/sm103/attention/__init__.py new file mode 100644 index 00000000..c42f68ac --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/attention/__init__.py @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The full-attention kernels: SKV2 (``skv2``), the paged FP8 split-KV attention of +one request's 7 verify rows (a CuTe DSL producer over 148 page splits and a Triton +merge).""" diff --git a/sgl_nvidia_mega/qwen3_5/sm103/attention/skv2.py b/sgl_nvidia_mega/qwen3_5/sm103/attention/skv2.py new file mode 100644 index 00000000..16f1026a --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/attention/skv2.py @@ -0,0 +1,704 @@ +# Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# Derived from NVIDIA/cutlass v4.8.0 examples/python/CuTeDSL/cute_ext/blackwell/attention/ +# (gqa_decode_simple.py, gqa_decode_opt.py; BSD-3-Clause): warp-specialized split-KV decode structure, +# rewritten for paged FP8 KV. + +# Copied from SGLang branch nvidia/qwen35-agentx-gb300 (BSD-3-Clause as above): +# python/sglang/kernels/ops/attention/skv2_split_kv.py (the producer and its plan) and +# python/sglang/kernels/ops/attention/split_kv_verify.py (_merge_tile, _merge_attention), +# unchanged, so a slot run is bit for bit the branch's. + +# Modified by NVIDIA: paged FP8 verify with explicit local reduction. + +# Modified by NVIDIA: separate softmax and value consumer groups. +# Modified by NVIDIA: separate key and value pipeline handoffs. + +# Modified by NVIDIA: runtime attention scales, page-tail bounds and reusable buffers. +# Modified by NVIDIA: stock split-KV rounding (page splits, one FP8 P rounding, bf16 partials). + +"""SKV2: paged FP8 q7 split attention with the stock split-KV rounding structure. + +Same splits (page-based, 148), 128-token blocks, running-max sequence, single e4m3 rounding of P x 256 and bf16 +partials x (V_SCALE / 256) as sglang split_kv_verify, so the stock merge kernel reduces the partials unchanged. +Only fp32 association differs: per-lane denominators and O = O * alpha + O_block. +""" + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass.cute import experimental as ex +from cutlass.cute.nvgpu import OperandMajorMode, tcgen05 +from cutlass.experimental import primitives as prim + + +@cute.experimental.jit +def load_keys( + k: cute.Tensor, + table: cute.Tensor, + sk: cute.Tensor, + barrier: cute.Pointer, + first_page: cutlass.Int32, + stage: cutlass.Int32, + tile: cutlass.Constexpr[int], + length: cutlass.Int32, +): + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive_and_expect_tx(barrier, tile * 256) + for half in cutlass.range_constexpr(tile // 64): + page_index = first_page + half + page = cutlass.Int32(0) + if page_index < cute.ceil_div(length, 64): + page = table[0, page_index] + source = cute.local_tile(k, (64, 256), (page, 0)) + dest = cute.local_tile(sk[None, None, stage], (64, 256), (half, 0)) + ex.tma_load(source, dest, barrier.value, update_expect_tx=False) + + +@cute.experimental.jit +def load_values( + v: cute.Tensor, + table: cute.Tensor, + sv: cute.Tensor, + barrier: cute.Pointer, + first_page: cutlass.Int32, + stage: cutlass.Int32, + tile: cutlass.Constexpr[int], + length: cutlass.Int32, +): + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive_and_expect_tx(barrier, tile * 256) + for half in cutlass.range_constexpr(tile // 64): + page_index = first_page + half + page = cutlass.Int32(0) + if page_index < cute.ceil_div(length, 64): + page = table[0, page_index] + source = cute.local_tile(v, (256, 64), (0, page)) + dest = cute.local_tile(sv[None, None, stage], (256, 64), (0, half)) + ex.tma_load(source, dest, barrier.value, update_expect_tx=False) + + +class StockSplitDecode: + def __init__(self, heads=4, qlen=7, splits=148, stages=2, slots=2): + assert stages >= 1 and slots in (2, 3, 4) + self.heads, self.qlen, self.splits, self.stages, self.slots = ( + heads, + qlen, + splits, + stages, + slots, + ) + self.rows = heads * qlen + self.width = 32 if self.rows <= 32 else 64 + self.tile = 128 + assert self.width * slots * 3 <= 512 + + @cute.experimental.jit + def __call__( + self, + q: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + table: cute.Tensor, + lengths: cute.Tensor, + partial: cute.Tensor, + maxima: cute.Tensor, + denom: cute.Tensor, + qk_scale: cutlass.Float32, + out_scale: cutlass.Float32, + stream: cuda.CUstream, + ): + dtype = cutlass.Float8E4M3FN + kq = cute.make_tiled_mma( + tcgen05.MmaFP8Op( + dtype, + cutlass.Float32, + (128, self.width), + tcgen05.CtaGroup.ONE, + tcgen05.OperandSource.SMEM, + OperandMajorMode.K, + OperandMajorMode.K, + ) + ) + vp = cute.make_tiled_mma( + tcgen05.MmaFP8Op( + dtype, + cutlass.Float32, + (128, self.width), + tcgen05.CtaGroup.ONE, + tcgen05.OperandSource.SMEM, + OperandMajorMode.MN, + OperandMajorMode.MN, + ) + ) + self.kernel( + q, k, v, table, lengths, partial, maxima, denom, qk_scale, out_scale, kq, vp + ).launch(grid=(self.splits, 1, 1), block=(384, 1, 1), stream=stream) + + @cute.experimental.kernel + def kernel( + self, + q: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + table: cute.Tensor, + lengths: cute.Tensor, + partial: cute.Tensor, + maxima: cute.Tensor, + denom: cute.Tensor, + qk_scale: cutlass.Float32, + out_scale: cutlass.Float32, + kq: cute.TiledMma, + vp: cute.TiledMma, + ): + tid, _, _ = cute.arch.thread_idx() + split, _, _ = cute.arch.block_idx() + warp = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + lane = cute.arch.lane_idx() + local_tid = tid % 128 + width = self.width + dtype, f32 = cutlass.Float8E4M3FN, cutlass.Float32 + k_atom = tcgen05.make_smem_layout_atom( + tcgen05.SmemLayoutAtomKind.K_SW128, dtype + ) + v_atom = tcgen05.make_smem_layout_atom( + tcgen05.SmemLayoutAtomKind.MN_SW128, dtype + ) + p_kind = ( + tcgen05.SmemLayoutAtomKind.MN_SW32 + if width == 32 + else tcgen05.SmemLayoutAtomKind.MN_SW64 + ) + p_atom = tcgen05.make_smem_layout_atom(p_kind, dtype) + sq = ex.allocate( + dtype, + cutlass.AddressSpace.smem, + cute.tile_to_shape(k_atom, (width, 256), order=(1, 0)), + alignment=128, + ) + sk = ex.allocate( + dtype, + cutlass.AddressSpace.smem, + cute.tile_to_shape(k_atom, (self.tile, 256, self.stages), order=(1, 0, 2)), + alignment=128, + ) + sv = ex.allocate( + dtype, + cutlass.AddressSpace.smem, + cute.tile_to_shape(v_atom, (256, self.tile, self.stages), order=(0, 1, 2)), + alignment=128, + ) + sp = ex.allocate( + dtype, + cutlass.AddressSpace.smem, + cute.tile_to_shape(p_atom, (width, self.tile, self.slots), order=(0, 1, 2)), + alignment=128, + ) + shared_l = ex.allocate( + f32, + cutlass.AddressSpace.smem, + cute.make_layout((128, width), stride=(width + 1, 1)), + alignment=128, + ) + shared = ex.allocate( + f32, + cutlass.AddressSpace.smem, + cute.make_layout((4, width), stride=(width, 1)), + alignment=128, + ) + alphas = ex.allocate( + f32, + cutlass.AddressSpace.smem, + cute.make_layout((width, self.slots), stride=(1, width)), + alignment=128, + ) + barriers = ex.allocate( + cutlass.Int64, + cutlass.AddressSpace.smem, + cute.make_layout(4 * self.stages + 5 * self.slots + 2), + alignment=8, + ) + bq = barriers.iterator + bk = bq + 1 + bv = bk + self.stages + bkfree = bv + self.stages + bvfree = bkfree + self.stages + bs = bvfree + self.stages + bsfree = bs + self.slots + bp = bsfree + self.slots + bo = bp + self.slots + bfree = bo + self.slots + bdenom = bfree + self.slots + if tid < 4 * self.stages + 5 * self.slots + 2: + count = cutlass.Int32(1) + if ( + tid >= 4 * self.stages + 1 + 2 * self.slots + and tid < 4 * self.stages + 1 + 3 * self.slots + ): + count = cutlass.Int32(128) + if tid == 4 * self.stages + 5 * self.slots + 1: + count = cutlass.Int32(128) + cute.arch.mbarrier_init(barriers.iterator + tid, count) + cute.arch.mbarrier_init_fence() + cute.arch.sync_threads() + ts = ex.allocate( + f32, + cutlass.AddressSpace.tmem, + ex.make_tmem_layout_acc(kq, (128, width, 256), self.slots), + alignment=32, + ) + to = ex.allocate( + f32, + cutlass.AddressSpace.tmem, + ex.make_tmem_layout_acc(vp, (128, width, 128), 2 * self.slots), + alignment=32, + ) + copy_atom = cute.make_copy_atom( + tcgen05.Ld32x32bOp(tcgen05.Repetition(width)), f32 + ) + s_plane = ts[(None, None), 0, 0, 0] + copy_s = tcgen05.make_tmem_copy(copy_atom, s_plane).get_slice(local_tid) + rlayout = copy_s.partition_D(s_plane).layout + # Stock split rule: pages_per_split = cdiv(cdiv(length, 64), SPLITS); split covers whole pages. + length = lengths[0] + pps = cute.ceil_div(cute.ceil_div(length, 64), self.splits) + begin_tok = split * pps * 64 + end_tok = begin_tok + pps * 64 + if end_tok > length: + end_tok = length + nblk = cutlass.Int32(0) + if begin_tok < end_tok: + nblk = cute.ceil_div(end_tok - begin_tok, self.tile) + begin_page = split * pps + cute.arch.sync_threads() + if warp < 4: + scores = cute.make_rmem_tensor((width, 1), f32) + running_m = cute.make_rmem_tensor((width // 32,), f32) + lane_maxima = cute.make_rmem_tensor((width // 32,), f32) + lane_alphas = cute.make_rmem_tensor((width // 32,), f32) + running_l = cute.make_rmem_tensor((self.rows,), f32) + packed = cute.make_rmem_tensor((width, 1), dtype) + running_m.fill(-f32.inf) + running_l.fill(0) + for it in cutlass.range(0, nblk): + slot = it % self.slots + cute.arch.mbarrier_wait(bs + slot, phase=(it // self.slots) % 2) + current_s = ts[(None, None), 0, 0, slot] + current_copy = tcgen05.make_tmem_copy(copy_atom, current_s).get_slice( + local_tid + ) + sub_scores = cute.make_tensor(scores.iterator, rlayout) + ex.partition_and_copy(current_copy, current_s, sub_scores) + cute.arch.fence_view_async_tmem_load() + prim.barrier_cta_sync(1, thread_count=128) + if warp == 0: + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive(bsfree + slot) + token = begin_tok + it * self.tile + local_tid + for j in cutlass.range_constexpr(self.rows): + score = scores[j, 0] * qk_scale + if token >= end_tok: + score = -f32.inf + if token > length - self.qlen + j // self.heads: + score = -f32.inf + scores[j, 0] = score + maximum = cute.arch.warp_redux_sync(score, kind="fmax", nan=True) + if lane == j % 32: + shared[warp, j] = maximum + prim.barrier_cta_sync(1, thread_count=128) + # The value consumer releases both the probability and alpha slot. + cute.arch.mbarrier_wait(bfree + slot, phase=(it // self.slots + 1) % 2) + for chunk in cutlass.range_constexpr(width // 32): + column = lane + 32 * chunk + raw = running_m[chunk] + if column < self.rows: + for w in cutlass.range_constexpr(4): + raw = cute.arch.fmax(raw, shared[w, column]) + else: + raw = f32(0) + # Stock keeps the raw running max (-inf for a row with no valid token yet) and + # exponentiates against 0 instead. + safe = raw + if raw == -f32.inf: + safe = f32(0) + lane_maxima[chunk] = safe + lane_alphas[chunk] = cute.math.exp2( + running_m[chunk] - safe, fastmath=True + ) + running_m[chunk] = raw + if warp == 0: + alphas[column, slot] = lane_alphas[chunk] + for j in cutlass.range_constexpr(self.rows): + maximum = cute.arch.shuffle_sync(lane_maxima[j // 32], j % 32) + alpha = cute.arch.shuffle_sync(lane_alphas[j // 32], j % 32) + total = running_l[j] * alpha + probability = cute.math.exp2(scores[j, 0] - maximum, fastmath=True) + total += probability + scores[j, 0] = probability * 256 + running_l[j] = total + for j in cutlass.range_constexpr(self.rows, width): + scores[j, 0] = f32(0) + packed[None, 0].store(scores[None, 0].load().to(dtype)) + cute.autovec_copy(packed[None, 0], sp[None, local_tid, slot]) + cute.arch.fence_view_async_shared() + cute.arch.mbarrier_arrive(bp + slot) + for j in cutlass.range_constexpr(self.rows): + shared_l[local_tid, j] = running_l[j] + cute.arch.mbarrier_arrive(bdenom) + if warp == 0: + for chunk in cutlass.range_constexpr(width // 32): + column = lane + 32 * chunk + if column < self.rows: + maxima[split, column] = running_m[chunk] + elif warp < 8: + value = cute.make_rmem_tensor(rlayout, f32) + accum = cute.make_rmem_tensor((self.rows, 2), f32) + accum.fill(0) + for it in cutlass.range(0, nblk): + slot = it % self.slots + cute.arch.mbarrier_wait(bo + slot, phase=(it // self.slots) % 2) + for dm in cutlass.range_constexpr(2): + o_plane = to[(None, None), 0, 0, 2 * slot + dm] + copy_o = tcgen05.make_tmem_copy(copy_atom, o_plane).get_slice( + local_tid + ) + ex.partition_and_copy(copy_o, o_plane, value) + cute.arch.fence_view_async_tmem_load() + for j in cutlass.range_constexpr(self.rows): + accum[j, dm] = accum[j, dm] * alphas[j, slot] + value[j] + prim.barrier_cta_sync(2, thread_count=128) + if warp == 4: + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive(bfree + slot) + for j in cutlass.range_constexpr(self.rows): + for dm in cutlass.range_constexpr(2): + partial[split, j, local_tid + dm * 128] = cutlass.BFloat16( + accum[j, dm] * out_scale + ) + else: + if warp == 8: + q_frag = kq.get_slice(0).partition_B(sq) + atom_kq = cute.make_mma_atom(kq.op) + cute.arch.mbarrier_wait(bq, phase=0) + for it in cutlass.range(0, nblk): + slot, stage = it % self.slots, it % self.stages + cute.arch.mbarrier_wait( + bsfree + slot, phase=(it // self.slots + 1) % 2 + ) + cute.arch.mbarrier_wait(bk + stage, phase=(it // self.stages) % 2) + ktile = cute.local_tile(sk[None, None, stage], (128, 256), (0, 0)) + k_frag = kq.get_slice(0).partition_A(ktile) + atom_kq.set(tcgen05.Field.ACCUMULATE, False) + for kk in cutlass.range_constexpr(cute.size(k_frag, mode=[2])): + ex.dot( + atom_kq, + k_frag[None, None, kk], + q_frag[None, None, kk], + ts[None, None, None, slot], + ) + atom_kq.set(tcgen05.Field.ACCUMULATE, True) + with cute.arch.elect_one(): + tcgen05.commit(bs + slot, None, tcgen05.CtaGroup.ONE) + tcgen05.commit(bkfree + stage, None, tcgen05.CtaGroup.ONE) + elif warp == 9: + atom_vp = cute.make_mma_atom(vp.op) + for it in cutlass.range(0, nblk): + slot, stage = it % self.slots, it % self.stages + cute.arch.mbarrier_wait(bp + slot, phase=(it // self.slots) % 2) + cute.arch.mbarrier_wait(bv + stage, phase=(it // self.stages) % 2) + p_frag = vp.get_slice(0).partition_B(sp[None, None, slot]) + for dm in cutlass.range_constexpr(2): + vtile = cute.local_tile( + sv[None, None, stage], (128, self.tile), (dm, 0) + ) + v_frag = vp.get_slice(0).partition_A(vtile) + atom_vp.set(tcgen05.Field.ACCUMULATE, False) + for kk in cutlass.range_constexpr(cute.size(v_frag, mode=[2])): + ex.dot( + atom_vp, + v_frag[None, None, kk], + p_frag[None, None, kk], + to[None, None, None, 2 * slot + dm], + ) + atom_vp.set(tcgen05.Field.ACCUMULATE, True) + with cute.arch.elect_one(): + tcgen05.commit(bo + slot, None, tcgen05.CtaGroup.ONE) + tcgen05.commit(bvfree + stage, None, tcgen05.CtaGroup.ONE) + elif warp == 10: + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive_and_expect_tx(bq, width * 256) + ex.tma_load( + cute.local_tile(q, (width, 256), (0, 0)), + sq, + bq.value, + update_expect_tx=False, + ) + for stage in cutlass.range_constexpr(self.stages): + if stage < nblk: + load_keys( + k, + table, + sk, + bk + stage, + begin_page + stage * (self.tile // 64), + stage, + self.tile, + length, + ) + for it in cutlass.range(0, nblk): + stage = it % self.stages + cute.arch.mbarrier_wait( + bvfree + stage, phase=(it // self.stages + 1) % 2 + ) + load_values( + v, + table, + sv, + bv + stage, + begin_page + it * (self.tile // 64), + stage, + self.tile, + length, + ) + if it + self.stages < nblk: + cute.arch.mbarrier_wait( + bkfree + stage, phase=(it // self.stages) % 2 + ) + load_keys( + k, + table, + sk, + bk + stage, + begin_page + (it + self.stages) * (self.tile // 64), + stage, + self.tile, + length, + ) + cute.arch.mbarrier_wait(bdenom, phase=0) + for chunk in cutlass.range_constexpr(cute.ceil_div(self.rows, 12)): + column = warp + 12 * chunk + if column < self.rows: + total = (shared_l[lane, column] + shared_l[lane + 32, column]) + ( + shared_l[lane + 64, column] + shared_l[lane + 96, column] + ) + for offset in cutlass.range_constexpr(5): + total += cute.arch.shuffle_sync_bfly(total, 1 << offset) + if lane == 0: + denom[split, column] = total + + +import torch +import triton +import triton.language as tl +from cutlass.cute.runtime import from_dlpack + + +@triton.jit +def _merge_tile( + Partial, + Maxima, + Denom, + Lengths, + ROWS: tl.constexpr, + SPLITS: tl.constexpr, + BS: tl.constexpr, + BD: tl.constexpr, + BN: tl.constexpr, + BALANCED: tl.constexpr, + SKIP_EMPTY: tl.constexpr, +): + row, dim_tile, batch = tl.program_id(0), tl.program_id(1), tl.program_id(2) + splits = tl.arange(0, BS) + dims = dim_tile * BD + tl.arange(0, BD) + base = (batch * SPLITS + splits) * ROWS + row + active = splits < SPLITS + if SKIP_EMPTY: + length = tl.load(Lengths + batch) + if BALANCED: + blocks = tl.cdiv(length, BN) + begin = (splits * blocks // SPLITS) * BN + end = tl.minimum(((splits + 1) * blocks // SPLITS) * BN, length) + else: + pages_per_split = tl.cdiv(tl.cdiv(length, 64), SPLITS) + begin = splits * pages_per_split * 64 + end = tl.minimum(begin + pages_per_split * 64, length) + active = active & (begin < end) + maximum = tl.load(Maxima + base, active, -float("inf")) + denom = tl.load(Denom + base, active, 0.0) + max_all = tl.max(maximum, 0) + max_all = tl.where(max_all == -float("inf"), 0.0, max_all) + weights = tl.exp2(maximum - max_all) + total = tl.sum(denom * weights, 0) + partial = tl.load( + Partial + base[:, None] * 256 + dims[None, :], + active[:, None] & (dims[None, :] < 256), + 0.0, + ) + result = tl.sum(partial * weights[:, None], 0) / tl.where(total > 0.0, total, 1.0) + return result, dims + + +@triton.jit +def _merge_attention( + Partial, + Maxima, + Denom, + Lengths, + Out, + ROWS: tl.constexpr, + SPLITS: tl.constexpr, + BS: tl.constexpr, + BD: tl.constexpr, + BN: tl.constexpr, + BALANCED: tl.constexpr, + SKIP_EMPTY: tl.constexpr, + SHORT_MERGE: tl.constexpr, +): + if SHORT_MERGE and not BALANCED: + length = tl.load(Lengths + tl.program_id(2)) + if length <= 2048: + result, dims = _merge_tile( + Partial, + Maxima, + Denom, + Lengths, + ROWS, + SPLITS, + 32, + BD, + BN, + BALANCED, + SKIP_EMPTY, + ) + else: + result, dims = _merge_tile( + Partial, + Maxima, + Denom, + Lengths, + ROWS, + SPLITS, + BS, + BD, + BN, + BALANCED, + SKIP_EMPTY, + ) + else: + result, dims = _merge_tile( + Partial, + Maxima, + Denom, + Lengths, + ROWS, + SPLITS, + BS, + BD, + BN, + BALANCED, + SKIP_EMPTY, + ) + row, batch = tl.program_id(0), tl.program_id(2) + tl.store(Out + (batch * ROWS + row) * 256 + dims, result, dims < 256) + + +class StockSplitPlan: + """Reusable storage and a compiled producer for one fixed input layout; merge is the stock kernel.""" + + def __init__(self, query, kv, table, lengths, *, splits=148, stages=2, slots=2): + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError("SKV2 storage must be prepared before graph capture") + self.qlen, self.heads, _ = query.shape + self.splits = splits + rows = self.qlen * self.heads + self.partial = torch.empty( + (splits, rows, 256), dtype=torch.bfloat16, device=query.device + ) + self.maximum = torch.empty( + (splits, rows), dtype=torch.float32, device=query.device + ) + self.denom = torch.empty_like(self.maximum) + kernel = StockSplitDecode( + self.heads, self.qlen, splits, stages=stages, slots=slots + ) + views = self._views(query, kv, table, lengths) + stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) + self.compiled = cute.compile( + kernel, *views, cutlass.Float32(1.0), cutlass.Float32(1.0), stream + ) + + def _views(self, query, kv, table, lengths): + k, v = kv + tensors = ( + query.view(-1, 256), + k.view(-1, 256), + v.view(-1, 256).T, + table, + lengths, + self.partial, + self.maximum, + self.denom, + ) + return tuple(from_dlpack(t) for t in tensors) + + def run(self, query, kv, table, lengths, *, qk_scale, v_scale, out): + views = self._views(query, kv, table, lengths) + stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) + # Same fp32 constants as the stock kernel's constexpr products. + self.compiled( + *views, + cutlass.Float32(qk_scale * 1.4426950408889634), + cutlass.Float32(v_scale / 256.0), + stream, + ) + rows = self.qlen * self.heads + _merge_attention[(rows, triton.cdiv(256, 64), 1)]( + self.partial, + self.maximum, + self.denom, + lengths, + out, + rows, + self.splits, + triton.next_power_of_2(self.splits), + 64, + 128, + False, + True, + True, + num_warps=4, + enable_fp_fusion=False, + ) + return out + + +def skv2_split_kv(query, kv, table, lengths, plan, *, qk_scale, v_scale, out): + """q7 split attention for one request through a prepared plan; same contract as split_kv_verify.""" + return plan.run( + query, kv, table, lengths, qk_scale=qk_scale, v_scale=v_scale, out=out + ) diff --git a/sgl_nvidia_mega/qwen3_5/spec.py b/sgl_nvidia_mega/qwen3_5/spec.py index 361e3eb1..1f741b95 100644 --- a/sgl_nvidia_mega/qwen3_5/spec.py +++ b/sgl_nvidia_mega/qwen3_5/spec.py @@ -111,6 +111,8 @@ "route_shared.gate_up.rel_l2_vs_stock": 1.05, "route_shared.gate_up.max_abs_vs_stock": 2.0, }, + # SKV2 against SGLang's TRT-LLM MHA decode (test_skv2_is_within_twice_sglangs_error) + "skv2": {"skv2.attention.rel_l2_vs_stock": 2.0}, } diff --git a/test/kernels/test_qwen3_5_attention.py b/test/kernels/test_qwen3_5_attention.py new file mode 100644 index 00000000..bca0cca8 --- /dev/null +++ b/test/kernels/test_qwen3_5_attention.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The full-attention kernels of slot skv2, on one SM100-family GPU (GB300), at Qwen3.5's +TP8 verify shape (7 rows of 4 query heads over one KV head of 256, FP8 pages of 64): + +- SKV2 against an FP64 reference of the masked attention: at most twice the error of + SGLang's attention (TRT-LLM MHA's decode kernel) on the same FP8 inputs, at KV lengths + from one page to past 128K; +- SKV2 replayed in a CUDA graph on new inputs, bit for bit its eager result. + +Its bits against the SKV2 of SGLang branch nvidia/qwen35-agentx-gb300 are checked +with that branch installed as ``sglang``, not here. +""" + +import math + +import pytest +import torch + +ROWS, HEADS, DIM, PAGE = 7, 4, 256, 64 +SCALE = DIM**-0.5 + + +def _gpu_ready() -> bool: + if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10: + return False + try: + import cutlass # noqa: F401 + import triton # noqa: F401 + except ImportError: + return False + return True + + +pytestmark = pytest.mark.skipif(not _gpu_ready(), reason="needs an SM100-family GPU") + + +def _hnd(pool): + """The pool's NHD rows as HND pages of one KV head, with TRT-LLM MHA's canonical + strides (as its decode kernel and SKV2 get them).""" + pages = pool.shape[0] // PAGE + return pool.as_strided((pages, 1, PAGE, DIM), (PAGE * DIM, PAGE * DIM, DIM, 1)) + + +def _fill(q, pools, table, lengths, length, g): + """New FP8 query rows and pool contents, the request's pages scattered over the + pool, and its length, written in place.""" + q.copy_((torch.randn(q.shape, device="cuda", generator=g) * 2).to(q.dtype)) + for pool in pools: + pool.copy_( + (torch.randn(pool.shape, device="cuda", generator=g) * 2).to(q.dtype) + ) + pages = math.ceil(length / PAGE) + total = pools[0].shape[0] // PAGE + table.zero_() + table[0, :pages] = torch.randperm(total, device="cuda", generator=g)[:pages].int() + lengths.fill_(length) + + +def _inputs(length, g, spare_pages=8): + """FP8 query rows, a paged FP8 KV pool with the request's pages scattered, its page + table and its length.""" + total = math.ceil(length / PAGE) + spare_pages + q = torch.empty(ROWS, HEADS, DIM, dtype=torch.float8_e4m3fn, device="cuda") + pools = [ + torch.empty(total * PAGE, 1, DIM, dtype=torch.float8_e4m3fn, device="cuda") + for _ in range(2) + ] + table = torch.zeros(1, total + 4, dtype=torch.int32, device="cuda") + lengths = torch.zeros(1, dtype=torch.int32, device="cuda") + _fill(q, pools, table, lengths, length, g) + return q, pools, table, lengths + + +def _reference(q, kv, table, length, qk_scale, v_scale): + """FP64 attention of the 7 verify rows, row t seeing the first length - 7 + t + 1 + tokens.""" + k, v = kv + pages = table[0, : math.ceil(length / PAGE)].long() + keys = k[pages, 0].reshape(-1, DIM)[:length].double() + values = v[pages, 0].reshape(-1, DIM)[:length].double() + scores = q.double() @ keys.T * qk_scale # [rows, heads, length] + tokens = torch.arange(length, device="cuda") + for t in range(ROWS): + scores[t, :, tokens > length - ROWS + t] = -math.inf + return torch.softmax(scores, dim=-1) @ values * v_scale + + +def _sglang(q, kv, table, lengths, qk_scale, v_scale): + """SGLang's attention on the same inputs: TRT-LLM MHA's decode kernel at 7 rows.""" + import flashinfer + + workspace = torch.zeros(512 * 1024 * 1024, dtype=torch.uint8, device="cuda") + return flashinfer.decode.trtllm_batch_decode_with_kv_cache( + query=q, + kv_cache=kv, + workspace_buffer=workspace, + block_tables=table, + seq_lens=lengths, + max_seq_len=table.shape[1] * PAGE, + bmm1_scale=qk_scale, + bmm2_scale=v_scale, + out_dtype=torch.bfloat16, + q_len_per_req=ROWS, + ) + + +@pytest.mark.parametrize("length", [7, 64, 2048, 8192, 32769, 131072, 150007]) +def test_skv2_is_within_twice_sglangs_error(length): + from sgl_nvidia_mega.qwen3_5.sm103.attention.skv2 import StockSplitPlan + + g = torch.Generator(device="cuda").manual_seed(length) + q, pools, table, lengths = _inputs(length, g) + kv = tuple(map(_hnd, pools)) + qk_scale, v_scale = 0.75 * SCALE, 0.5 + out = torch.empty(ROWS, HEADS, DIM, dtype=torch.bfloat16, device="cuda") + plan = StockSplitPlan(q, kv, table, lengths) + plan.run(q, kv, table, lengths, qk_scale=qk_scale, v_scale=v_scale, out=out) + want = _reference(q, kv, table, length, qk_scale, v_scale) + assert torch.isfinite(out).all() + err = (out.double() - want).square().mean().sqrt().item() + stock = _sglang(q, kv, table, lengths, qk_scale, v_scale) + stock_err = (stock.double() - want).square().mean().sqrt().item() + assert err <= 2 * stock_err + 1e-6, (length, err, stock_err) + + +def test_skv2_replays_in_a_cuda_graph_bit_for_bit(): + from sgl_nvidia_mega.qwen3_5.sm103.attention.skv2 import StockSplitPlan + + g = torch.Generator(device="cuda").manual_seed(1) + q, pools, table, lengths = _inputs(150007, g) + kv = tuple(map(_hnd, pools)) + qk_scale, v_scale = 0.75 * SCALE, 0.5 + out = torch.empty(ROWS, HEADS, DIM, dtype=torch.bfloat16, device="cuda") + plan = StockSplitPlan(q, kv, table, lengths) + plan.run(q, kv, table, lengths, qk_scale=qk_scale, v_scale=v_scale, out=out) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + plan.run(q, kv, table, lengths, qk_scale=qk_scale, v_scale=v_scale, out=out) + for length in (131072, 9000, 7, 150007): + _fill(q, pools, table, lengths, length, g) + graph.replay() + want = torch.empty_like(out) + plan.run(q, kv, table, lengths, qk_scale=qk_scale, v_scale=v_scale, out=want) + assert torch.equal(out.view(torch.int16), want.view(torch.int16)), length diff --git a/test/unit/test_qwen3_5_spec.py b/test/unit/test_qwen3_5_spec.py index bb0ff975..5eb21780 100644 --- a/test/unit/test_qwen3_5_spec.py +++ b/test/unit/test_qwen3_5_spec.py @@ -193,3 +193,8 @@ def test_a_numeric_slot_turns_the_claim_into_its_tolerances(): **spec.NUMERIC_TOLERANCES["gdn_push"], **spec.NUMERIC_TOLERANCES["gdn_front"], } + attention = spec.numerics(frozenset({"skv2"})) + assert attention.kind is NumericsKind.TOLERANCE + assert dict(attention.tolerances) == { + **spec.NUMERIC_TOLERANCES["skv2"], + } diff --git a/test/unit/test_qwen3_5_steps.py b/test/unit/test_qwen3_5_steps.py index 425a9a08..4b6529ab 100644 --- a/test/unit/test_qwen3_5_steps.py +++ b/test/unit/test_qwen3_5_steps.py @@ -1376,13 +1376,317 @@ def test_the_attention_step_issues_what_sglang_issues(): positions="pos", hidden_states=x, forward_batch="fb" ) else: - st = SimpleNamespace(positions="pos", fb="fb") + st = SimpleNamespace(positions="pos", fb="fb", slots=plan.Slots()) out = attention.step(layer, x, st) runs.append((log, out)) (want_log, want), (got_log, got) = runs assert got_log == want_log and want == got == "out" +# ------------------------------------------------------------------- slot skv2 + + +class _Plan: + """The SKV2 plan's stand-in: logs its build and its runs.""" + + log = None + + def __init__(self, query, kv, table, lengths): + self.log.append(("plan", self.log.tag(query), *map(self.log.tag, kv))) + self.log.append(("strides", *(c.stride() for c in kv))) + self.log.append(("plan_inputs", self.log.tag(table), self.log.tag(lengths))) + + def run(self, query, kv, table, lengths, *, qk_scale, v_scale, out): + self.log.append( + ("skv2", self.log.tag(query), self.log.tag(table), qk_scale, v_scale) + ) + self.log.known["skv2_out"] = out + return out + + +@contextlib.contextmanager +def _skv2_kernels(log, label=None, capture=True): + import sys + + from sglang.kernels.ops.kvcache import fused_fp8_qkv_kv_cache as fused_module + + q8 = log.known.setdefault("q8", torch.zeros(7, 1024, dtype=torch.float8_e4m3fn)) + + def fused(**kw): + log.append( + ( + "fused_fp8_qkv_kv_cache", + tuple(sorted((k, log.tag(v)) for k, v in kw.items())), + ) + ) + return q8 + + _Plan.log = log + with ( + patch.object(fused_module, "fused_fp8_qkv_kv_cache", fused), + patch.dict( + sys.modules, + { + "sgl_nvidia_mega.qwen3_5.sm103.attention.skv2": SimpleNamespace( + StockSplitPlan=_Plan + ) + }, + ), + patch( + "sglang.srt.model_executor.runner_utils.capture_mode." + "get_capture_attention_variant", + return_value=label, + ), + patch( + "sglang.srt.model_executor.runner_utils.capture_mode.get_is_capture_mode", + return_value=capture, + ), + ): + yield + + +def _skv2_setup(log, k_scale=None, **fb): + """Layer 7's RadixAttention (TP8: 4 query heads, 1 KV head of 256), the forward's + view of a pool of 3 pages for layers 3 and 7, and the verify step's state.""" + from sgl_nvidia_mega.qwen3_5.attention import split_kv + + split_kv._PLANS.clear() + pool = { + name: torch.zeros(3 * 64, 1, 256, dtype=torch.float8_e4m3fn) + for name in ("k3", "v3", "k7", "v7") + } + attn = _Callable( + log.op("attention", "stock_o"), + layer_id=7, + k_scale=k_scale, + v_scale=torch.full((1,), 0.5), + tp_q_head_num=4, + tp_k_head_num=1, + tp_v_head_num=1, + head_dim=256, + scaling=0.0625, + sliding_window_size=-1, + ) + view = SimpleNamespace( + kv=SimpleNamespace( + layer_ids=(3, 7), + k=(pool["k3"], pool["k7"]), + v=(pool["v3"], pool["v7"]), + k_scale=(1.0, 0.75), + v_scale=(1.0, 0.5), + ), + attn=SimpleNamespace( + page_table=torch.zeros(1, 4, dtype=torch.int32), + cache_seqlens=torch.full((1,), 130, dtype=torch.int32), + ), + ) + batch = SimpleNamespace( + out_cache_loc=torch.zeros(7, dtype=torch.int64), batch_size=1, **fb + ) + log.known.update( + pool, + loc=batch.out_cache_loc, + table=view.attn.page_table, + lengths=view.attn.cache_seqlens, + v_scale=attn.v_scale, + fb=batch, + ) + return attn, SimpleNamespace(view=view, fb=batch) + + +def test_the_skv2_slot_writes_kv_as_the_backend_does_then_runs_skv2_on_the_view(): + from sgl_nvidia_mega.qwen3_5.attention import split_kv + + log = _log() + attn, st = _skv2_setup(log) + q, k, v = (torch.zeros(7, n, dtype=torch.bfloat16) for n in (1024, 256, 256)) + log.known.update(q=q, k=k, v=v) + with _skv2_kernels(log): + outs = [split_kv.attend(attn, q, k, v, st) for _ in range(2)] # two steps + fused = log[0] + assert fused[0] == "fused_fp8_qkv_kv_cache" + kw = dict(fused[1]) + # TRT-LLM MHA's fused FP8 write of this layer's pool rows, its default scale for + # a missing checkpoint scale + assert (kw["q"], kw["k"], kw["v"]) == ("q(7, 1024)", "k(7, 256)", "v(7, 256)") + assert (kw["k_cache"], kw["v_cache"], kw["cache_loc"]) == ( + "k7(192, 1, 256)", + "v7(192, 1, 256)", + "loc(7,)", + ) + assert kw["k_scale"] == "?(1,)" and kw["v_scale"] == "v_scale(1,)" + # the query as (rows, heads, head_dim) and the pool as HND pages; one plan + assert log[1] == ("plan", "q8(7, 4, 256)", "k7(3, 1, 64, 256)", "v7(3, 1, 64, 256)") + assert log[2] == ("strides", (16384, 16384, 256, 1), (16384, 16384, 256, 1)) + assert log[3] == ("plan_inputs", "table(1, 4)", "lengths(1,)") + # the BMM scales TRT-LLM MHA passes: k_scale * scaling and v_scale + assert log[4] == ("skv2", "q8(7, 4, 256)", "table(1, 4)", 0.75 * 0.0625, 0.5) + assert [e[0] for e in log[5:]] == ["fused_fp8_qkv_kv_cache", "skv2"] + assert all(o.shape == (7, 1024) and o.dtype == torch.bfloat16 for o in outs) + + +@pytest.mark.parametrize( + "label, capture, fb, want", + [ + ("split_kv_long", True, {}, "skv2"), + ("split_kv_stock", True, {}, "attention"), + # SGLang captures no label of ours (no runner takes the variants yet) + (None, True, {}, "skv2"), + (None, False, {}, "skv2"), + # an eager step routed by the request's committed KV length + (None, False, dict(attention_kv_len_lower_bound=8192), "skv2"), + (None, False, dict(attention_kv_len_lower_bound=8191), "attention"), + ( + None, + False, + dict(attention_kv_len_lower_bound=None, seq_lens_cpu=torch.tensor([9000])), + "skv2", + ), + ( + None, + False, + dict(attention_kv_len_lower_bound=None, seq_lens_cpu=None), + "attention", + ), + ], +) +def test_the_skv2_slot_takes_split_kv_where_the_branch_routes_it( + label, capture, fb, want +): + from sgl_nvidia_mega.qwen3_5.attention import split_kv + + log = _log() + attn, st = _skv2_setup(log, **{"seq_lens_cpu": None, **fb}) + q8 = torch.zeros(7, 1024, dtype=torch.float8_e4m3fn) + log.known["q8"] = q8 + with _skv2_kernels(log, label=label, capture=capture): + split_kv.core(attn, q8, st) + assert log[-1][0] == want + if want == "attention": + # SGLang's attention on the written rows: no K / V to write + assert log[-1][2:] == (("q8(7, 1024)", None, None, "fb"), ()) + + +def test_the_split_kv_variants_select_on_host_state_only(): + from sgl_nvidia_mega.qwen3_5.attention import route + + assert route.SplitKVGraphVariants.capture_labels == ( + "split_kv_stock", + "split_kv_long", + ) + assert (route.VERIFY.min_length, route.EXTEND.min_length) == (8192, 32768) + + def batch(bs=1, bound=None, lens=None): + return SimpleNamespace( + batch_size=bs, attention_kv_len_lower_bound=bound, seq_lens_cpu=lens + ) + + assert route.VERIFY.select(batch(bound=8192)) == "split_kv_long" + assert route.VERIFY.select(batch(bound=8191)) == "split_kv_stock" + assert route.EXTEND.select(batch(bound=32767)) == "split_kv_stock" + assert route.VERIFY.select(batch(bs=2, bound=10**6)) == "split_kv_stock" + assert route.VERIFY.select(batch(lens=torch.tensor([8192]))) == "split_kv_long" + assert route.VERIFY.select(batch(lens=torch.tensor([9000, 9000]))) == ( + "split_kv_stock" + ) + assert route.VERIFY.select(batch()) == "split_kv_stock" + + +@pytest.mark.parametrize( + "kw, want", + [ + (dict(), ""), + (dict(rows=4), "4 rows a request (SKV2 runs 7)"), + (dict(tp_q_head_num=2), "2 query / 1 KV heads a rank"), + (dict(tp_k_head_num=2), "4 query / 2 KV heads a rank"), + (dict(head_dim=128), "head_dim 128"), + (dict(sliding_window_size=4096), "a sliding window of 4096"), + ], +) +def test_the_skv2_slot_names_what_a_layer_lacks(kw, want): + from sgl_nvidia_mega.qwen3_5.attention import split_kv + + rows = kw.pop("rows", 7) + attn = SimpleNamespace( + **{ + "tp_q_head_num": 4, + "tp_k_head_num": 1, + "head_dim": 256, + "sliding_window_size": -1, + **kw, + } + ) + with patch.object(torch.cuda, "get_device_capability", return_value=(10, 3)): + got = split_kv.unsupported(attn, rows) + assert want in got if want else got == "" + with patch.object(torch.cuda, "get_device_capability", return_value=(9, 0)): + assert ( + split_kv.unsupported( + SimpleNamespace( + tp_q_head_num=4, + tp_k_head_num=1, + head_dim=256, + sliding_window_size=-1, + ), + 7, + ) + == "SKV2 runs on SM10x" + ) + + +def test_the_skv2_slot_names_the_pools_it_cannot_read(): + from sgl_nvidia_mega.qwen3_5.attention import split_kv + + def kv(*bufs): + return SimpleNamespace(k=bufs, v=bufs) + + fp8 = torch.zeros(3 * 64, 1, 256, dtype=torch.float8_e4m3fn) + assert split_kv.pool_reasons(kv(fp8), 64) == [] + assert split_kv.pool_reasons(kv(fp8), 32) == [ + "page size 32 (SKV2 reads pages of 64)" + ] + bf16 = torch.zeros(3 * 64, 1, 256, dtype=torch.bfloat16) + assert "not contiguous FP8" in split_kv.pool_reasons(kv(bf16), 64)[0] + odd = torch.zeros(65, 1, 256, dtype=torch.float8_e4m3fn) + assert split_kv.pool_reasons(kv(odd), 64) == [ + "a KV pool buffer of shape (65, 1, 256)" + ] + + +def test_the_skv2_slot_runs_the_attention_step_through_the_slot(): + runs = [] + for slots in (plan.Slots(), plan.Slots(skv2=True)): + x = torch.zeros(7, 4) + log = _log(x=x, positions="pos", batch="fb") + layer = _attention_layer(log, x) + mul = log.op("sigmoid_mul", "gated") + st = SimpleNamespace(positions="pos", fb="fb", slots=slots) + with ( + patch.object(qwen3_5, "_is_cuda", True), + patch("sglang.kernels.ops.elementwise.elementwise.fused_sigmoid_mul", mul), + patch( + "sgl_nvidia_mega.qwen3_5.attention.split_kv.attend", + log.op("skv2_attend", log.known["o"]), + ), + ): + attention.step(layer, x, st) + runs.append([e[0] for e in log]) + stock, slot = runs + assert stock == ["prepare", "attention", "sigmoid_mul", "o_proj"] + assert slot == ["prepare", "skv2_attend", "sigmoid_mul", "o_proj"] + + +def test_the_executor_offers_the_verify_split_kv_variants_with_slot_skv2(): + from sgl_nvidia_mega.qwen3_5.attention import route + + lm = SimpleNamespace(model=SimpleNamespace(start_layer=0, end_layer=0, layers=[])) + stock = executor.Qwen35Verify(lm, {7: plan.Slots()}, {}, (7,)) + slot = executor.Qwen35Verify(lm, {7: plan.Slots(skv2=True)}, {}, (7,)) + # keyed by the runner that captures them; a runner without a key captures one graph + assert stock.graph_variants == {} + assert slot.graph_variants == {"target_verify": route.VERIFY} + + class _Exit: def __init__(self, log, defer): self.log, self.defer_moe_finalize = log, defer From 6c6142e8b62102d8f243bc41e44c1aac593b796d Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:43:00 -0700 Subject: [PATCH 06/27] [Perf] [GB300] qwen3_5: Slot mka: the full attention's fused prologue and epilogue (numeric) MKA's prologue writes the FP8 query, the bf16 gate and the step's FP8 K / V rows into the pool, and its epilogue gates, quantizes and runs o_proj into the TP-partial output (the kernels of SGLang branch nvidia/qwen35-agentx-gb300, built with NVRTC). Slot mka; numeric: its qkv and o_proj GEMMs sum in another order than SGLang's. --- sgl_nvidia_mega/qwen3_5/attention/__init__.py | 50 +- sgl_nvidia_mega/qwen3_5/attention/mka.py | 189 ++++++ sgl_nvidia_mega/qwen3_5/attention/split_kv.py | 19 +- sgl_nvidia_mega/qwen3_5/plan.py | 6 +- sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 2 + sgl_nvidia_mega/qwen3_5/runtime/executor.py | 9 +- .../qwen3_5/sm103/attention/mka.py | 264 +++++++++ .../qwen3_5/sm103/attention/src/mka.cu | 538 ++++++++++++++++++ sgl_nvidia_mega/qwen3_5/spec.py | 7 + test/kernels/test_qwen3_5_attention.py | 137 ++++- test/unit/test_qwen3_5_spec.py | 3 +- test/unit/test_qwen3_5_steps.py | 238 ++++++++ 12 files changed, 1435 insertions(+), 27 deletions(-) create mode 100644 sgl_nvidia_mega/qwen3_5/attention/mka.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/attention/mka.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/attention/src/mka.cu diff --git a/sgl_nvidia_mega/qwen3_5/attention/__init__.py b/sgl_nvidia_mega/qwen3_5/attention/__init__.py index e392aea1..74a491c6 100644 --- a/sgl_nvidia_mega/qwen3_5/attention/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/attention/__init__.py @@ -10,7 +10,9 @@ split, the attention through the backend (which writes this step's KV rows), the sigmoid output gate and ``o_proj``. ``step`` returns the TP-partial output, whose all-reduce the boundary runs. Slot ``skv2`` (``split_kv.py``) runs the attention's core -through the split-KV attention of SGLang branch nvidia/qwen35-agentx-gb300. +through the split-KV attention of SGLang branch nvidia/qwen35-agentx-gb300, and slot +``mka`` (``mka.py``) the rest of the layer through that branch's fused prologue and +epilogue. """ from __future__ import annotations @@ -19,6 +21,7 @@ # The SGLang names this module reads (test/unit/test_contract.py pins them) SGLANG_READS = { + "srt/models/qwen3_vl.py": ("is_mrope_enabled",), "srt/models/qwen3_5.py": ( "Qwen3_5AttentionDecoderLayer", "layers", @@ -44,13 +47,14 @@ "srt/models/qwen3_5.py:Qwen3_5AttentionDecoderLayer.self_attention": "db0f4d98e0fb1abf", } -ATTENTION_SLOTS = ("skv2",) +ATTENTION_SLOTS = ("skv2", "mka") class Bound(msgspec.Struct, frozen=True): """The attention package's build result (``st.bound["attention"]``).""" skv2: bool = False # slot skv2 serves every full-attention layer + mka: bool = False reason: str = "" # why a requested slot is not built @@ -59,26 +63,42 @@ def build(model, ctx) -> Bound: cannot take them.""" from sglang.srt.models.qwen3_5 import Qwen3_5AttentionDecoderLayer - from sgl_nvidia_mega.qwen3_5.attention import split_kv + from sgl_nvidia_mega.qwen3_5.attention import mka, split_kv from sgl_nvidia_mega.qwen3_5.runtime import SLOTS if not SLOTS & set(ATTENTION_SLOTS): return Bound(reason="stock (SGLANG_NVIDIA_MEGAKERNEL_SLOTS)") m = model.model - for i in range(m.start_layer, m.end_layer): - layer = m.layers[i] - if isinstance(layer, Qwen3_5AttentionDecoderLayer) and "skv2" in SLOTS: + layers = [ + (i, m.layers[i]) + for i in range(m.start_layer, m.end_layer) + if isinstance(m.layers[i], Qwen3_5AttentionDecoderLayer) + ] + for i, layer in layers: + reason = "" + if "skv2" in SLOTS: reason = split_kv.unsupported(layer.attn, ctx.tokens_per_request) - if reason: - return Bound(reason=f"layer {i}: {reason}") - return Bound(skv2="skv2" in SLOTS) - - -def pool_reasons(view, binding) -> list[str]: + if not reason and "mka" in SLOTS: + reason = ( + mka.unsupported(layer) + if model.is_mrope_enabled + else "no M-RoPE positions (MKA reads them)" + ) + if reason: + return Bound(reason=f"layer {i}: {reason}") + if "mka" in SLOTS: + for _, layer in layers: + mka.pack(layer) + return Bound(skv2="skv2" in SLOTS, mka="mka" in SLOTS) + + +def pool_reasons(view, binding, skv2: bool = True, mka: bool = False) -> list[str]: """Why the attention slots cannot read this forward's KV pools ([] when they can).""" + from sgl_nvidia_mega.qwen3_5.attention import mka as fused from sgl_nvidia_mega.qwen3_5.attention import split_kv - return split_kv.pool_reasons(view.kv, binding.page_size) + reasons = split_kv.pool_reasons(view.kv, binding.page_size) if skv2 else [] + return reasons + (fused.pool_reasons(view.kv) if mka else []) def step(layer, x, st): @@ -86,6 +106,10 @@ def step(layer, x, st): output [rows, hidden].""" from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul + if st.slots.mka: + from sgl_nvidia_mega.qwen3_5.attention import mka + + return mka.step(layer, x, st) q, k, v, gate = layer._prepare_qkv_gate( positions=st.positions, hidden_states=x, forward_batch=st.fb ) diff --git a/sgl_nvidia_mega/qwen3_5/attention/mka.py b/sgl_nvidia_mega/qwen3_5/attention/mka.py new file mode 100644 index 00000000..458d074a --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/attention/mka.py @@ -0,0 +1,189 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang branch nvidia/qwen35-agentx-gb300: +# python/sglang/srt/models/qwen3_5.py (Qwen3_5AttentionDecoderLayer._mka_static_eligible, +# _mka_eligible and _mka_self_attention). +"""Slot ``mka``: the full attention of a covered target-verify step through the fused +prologue and epilogue of SGLang branch nvidia/qwen35-agentx-gb300 +(``sm103/attention/mka.py``) around the attention. + +The prologue takes the normed rows and writes the FP8 query rows, the bf16 gate and this +step's FP8 K / V rows into the forward view's pool; the attention reads them (slot +``skv2``'s core when it runs, else SGLang's attention with no K / V to write); the +epilogue gates, quantizes and runs ``o_proj`` into the TP-partial output, in SGLang's +symmetric pool as ``RowParallelLinear``'s output. The weights are packed at build. +""" + +from __future__ import annotations + +import logging + +import torch + +logger = logging.getLogger(__name__) + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen3_5.py": ( + "attn_output_gate", + "num_heads", + "num_kv_heads", + "head_dim", + "hidden_size", + "rotary_emb", + "qkv_proj", + "o_proj", + "q_norm", + "k_norm", + "attn", + ), + "srt/layers/rotary_embedding/mrope.py": ( + "MRotaryEmbedding", + '"axis_map"', + "cos_sin_cache", + ), + "srt/layers/rotary_embedding/base.py": ("rotary_dim", "cos_sin_cache"), + "srt/layers/layernorm.py": ("variance_epsilon",), + "srt/layers/quantization/modelopt_quant.py": ( + "ModelOptFp8LinearMethod", + "use_marlin", + '"input_scale"', + '"weight_scale"', + ), + "srt/layers/radix_attention.py": ("layer_id",), + "srt/distributed/device_communicators/pynccl_allocator.py": ( + "use_symmetric_memory", + ), + "srt/layers/dp_attention.py": ("is_allocation_symmetric",), + "srt/runtime_context.py": ("get_parallel",), + "srt/model_executor/forward_batch_info.py": ("out_cache_loc",), + "srt/megakernels/eagle_binding.py": ("kv", "layer_ids", "k", "v"), +} + +# the packed (qkv, o_proj) weights per full-attention layer id, filled at build +WEIGHTS: dict = {} +# the rows the slot has run at (each logged once, as engagement evidence) +_ENGAGED: set = set() + + +def _per_tensor_fp8(linear) -> bool: + method = linear.quant_method + return ( + type(method).__name__ == "ModelOptFp8LinearMethod" + and not method.use_marlin + and linear.weight.dtype == torch.float8_e4m3fn + and linear.input_scale is not None + and linear.input_scale.numel() == 1 + and linear.weight_scale.numel() == 1 + ) + + +def unsupported(layer) -> str: + """Why the slot cannot serve ``layer`` (a Qwen3_5AttentionDecoderLayer; "" when it + can): the branch's ``_mka_static_eligible`` and its M-RoPE positions.""" + from sglang.srt.layers.rotary_embedding.mrope import MRotaryEmbedding + + rope = layer.rotary_emb + if not layer.attn_output_gate: + return "no attention output gate" + if layer.num_heads not in (4, 8) or layer.num_kv_heads != 1: + return ( + f"{layer.num_heads} query / {layer.num_kv_heads} KV heads a rank " + "(MKA runs 4 or 8 / 1)" + ) + if layer.head_dim != 256 or layer.hidden_size != 4096: + return f"head_dim {layer.head_dim}, hidden {layer.hidden_size} (MKA runs 256, 4096)" + if not isinstance(rope, MRotaryEmbedding) or rope.axis_map is None: + return "no M-RoPE axis map" + if rope.rotary_dim != 64 or rope.cos_sin_cache.dtype != torch.float32: + return f"rotary_dim {rope.rotary_dim} with a {rope.cos_sin_cache.dtype} cos/sin cache" + if not (_per_tensor_fp8(layer.qkv_proj) and _per_tensor_fp8(layer.o_proj)): + return "qkv_proj or o_proj is not a static per-tensor FP8 linear" + return "" + + +def pool_reasons(kv) -> list[str]: + """Why the prologue cannot write these KV pools ([] when it can): one FP8 KV head + of 256 per slot.""" + for buf in (*kv.k, *kv.v): + if buf.dtype != torch.float8_e4m3fn or buf.stride(0) != 256: + return [f"a {buf.dtype} KV pool buffer of strides {buf.stride()}"] + return [] + + +def pack(layer) -> None: + """Pack ``layer``'s qkv (+ gate) and o_proj weights for the kernels (at build).""" + from sgl_nvidia_mega.qwen3_5.sm103.attention import mka as kernels + + kernels.build(layer.qkv_proj.weight.device) + WEIGHTS[layer.attn.layer_id] = ( + kernels.pack_qkv(layer.qkv_proj.weight.t(), layer.num_heads), + kernels.pack_o(layer.o_proj.weight.t()), + ) + + +def step(layer, x, st): + """``layer``'s attention on the normed rows ``x`` [rows, hidden]: the TP-partial + output [rows, hidden].""" + from sglang.srt.distributed.device_communicators.pynccl_allocator import ( + use_symmetric_memory, + ) + from sglang.srt.layers.dp_attention import is_allocation_symmetric + from sglang.srt.runtime_context import get_parallel + + from sgl_nvidia_mega.qwen3_5.attention import split_kv + from sgl_nvidia_mega.qwen3_5.sm103.attention import mka as kernels + + m = x.shape[0] + if m not in _ENGAGED: + _ENGAGED.add(m) + logger.info("NVIDIA megakernel: Qwen3.5 slot mka runs at %d rows", m) + attn = layer.attn + width = layer.num_heads * layer.head_dim + q = torch.empty((m, width), dtype=torch.float8_e4m3fn, device=x.device) + gate = torch.empty((m, width), dtype=torch.bfloat16, device=x.device) + i = st.view.kv.layer_ids.index(attn.layer_id) + k_scale, v_scale = split_kv.kv_scales(attn, x.device) + w_qkv, w_o = WEIGHTS[attn.layer_id] + kernels.prologue( + x=x.contiguous(), + w_packed=w_qkv, + sx=layer.qkv_proj.input_scale, + sw=layer.qkv_proj.weight_scale, + q_norm_w=layer.q_norm.weight.data, + k_norm_w=layer.k_norm.weight.data, + cos_sin=layer.rotary_emb.cos_sin_cache, + positions=st.positions, + axis_map=layer.rotary_emb.axis_map, + cache_loc=st.fb.out_cache_loc, + k_cache=st.view.kv.k[i], + v_cache=st.view.kv.v[i], + k_scale=k_scale, + v_scale=v_scale, + q_out=q, + gate_out=gate, + nq=layer.num_heads, + eps=layer.q_norm.variance_epsilon, + ) + if st.slots.skv2: + o = split_kv.core(attn, q, st) + else: + o = attn(q, None, None, st.fb) + o = o.reshape(m, width).contiguous() + # the same allocation as RowParallelLinear's output, which the boundary all-reduces + with use_symmetric_memory( + get_parallel().tp_group, disabled=not is_allocation_symmetric() + ): + out = torch.empty((m, layer.hidden_size), dtype=torch.bfloat16, device=x.device) + # slot mka_push hands the hop's site here and marks the rows pushed + kernels.epilogue( + attn=o, + gate=gate, + w_packed=w_o, + sx=layer.o_proj.input_scale, + sw=layer.o_proj.weight_scale, + out=out, + nq=layer.num_heads, + site=None, + ) + return out diff --git a/sgl_nvidia_mega/qwen3_5/attention/split_kv.py b/sgl_nvidia_mega/qwen3_5/attention/split_kv.py index c61e5bbb..e17335e9 100644 --- a/sgl_nvidia_mega/qwen3_5/attention/split_kv.py +++ b/sgl_nvidia_mega/qwen3_5/attention/split_kv.py @@ -124,6 +124,17 @@ def pool_reasons(kv, page_size: int) -> list[str]: return reasons +def kv_scales(attn, device): + """The FP8 KV write's scale tensors TRT-LLM MHA passes: the layer's, else ones.""" + ones = _ONES.get(device) + if ones is None: + ones = _ONES[device] = torch.ones(1, dtype=torch.float32, device=device) + return ( + attn.k_scale if attn.k_scale is not None else ones, + attn.v_scale if attn.v_scale is not None else ones, + ) + + def attend(attn, q, k, v, st): """``attn(q, k, v, st.fb)`` for a covered verify step: the attention output [rows, heads * head_dim].""" @@ -132,9 +143,7 @@ def attend(attn, q, k, v, st): ) i = st.view.kv.layer_ids.index(attn.layer_id) - ones = _ONES.get(q.device) - if ones is None: - ones = _ONES[q.device] = torch.ones(1, dtype=torch.float32, device=q.device) + k_scale, v_scale = kv_scales(attn, q.device) q = fused_fp8_qkv_kv_cache( q=q, k=k, @@ -142,8 +151,8 @@ def attend(attn, q, k, v, st): k_cache=st.view.kv.k[i], v_cache=st.view.kv.v[i], cache_loc=st.fb.out_cache_loc, - k_scale=attn.k_scale if attn.k_scale is not None else ones, - v_scale=attn.v_scale if attn.v_scale is not None else ones, + k_scale=k_scale, + v_scale=v_scale, ) return core(attn, q, st) diff --git a/sgl_nvidia_mega/qwen3_5/plan.py b/sgl_nvidia_mega/qwen3_5/plan.py index 9fe476cf..c7cfe5c0 100644 --- a/sgl_nvidia_mega/qwen3_5/plan.py +++ b/sgl_nvidia_mega/qwen3_5/plan.py @@ -13,7 +13,8 @@ request of 7 rows, ``route_shared`` (numeric: the shared expert's gate/up in route's launch) up to 8 rows. ``skv2`` runs the full attention's core through the split-KV attention of SGLang branch -nvidia/qwen35-agentx-gb300. +nvidia/qwen35-agentx-gb300, and ``mka`` the fused attention prologue and epilogue +around it. """ from __future__ import annotations @@ -44,6 +45,7 @@ class Slots(msgspec.Struct, frozen=True): # the shared expert's gate/up in the router launch (moe/route.py; numeric) route_shared: bool = False skv2: bool = False # the full attention's core through SKV2 (attention/split_kv.py) + mka: bool = False # the fused attention prologue and epilogue (attention/mka.py) class Facts(msgspec.Struct, frozen=True): @@ -61,6 +63,7 @@ class Facts(msgspec.Struct, frozen=True): False # the router front with the shared stage, every MoE layer ) skv2: bool = False # SKV2 for every full-attention layer + mka: bool = False # the fused prologue and epilogue for every full-attention layer def slots(rows: int, facts: Facts) -> Slots: @@ -79,6 +82,7 @@ def slots(rows: int, facts: Facts) -> Slots: and rows == GDN_FRONT_ROWS, route_shared=facts.route and facts.route_shared and rows <= ROUTE_SHARED_ROWS, skv2=facts.skv2, + mka=facts.mka, ) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index 81cf38b6..299f101f 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -24,6 +24,7 @@ "gdn_front", "route_shared", "skv2", + "mka", ) SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" @@ -130,6 +131,7 @@ def build(model, ctx): gdn_front=bool(bound["gdn"].front), route_shared=bound["moe"].shared, skv2=bound["attention"].skv2, + mka=bound["attention"].mka, ) slots = slot_plan.plan(pctx.widths, facts) built = msgspec.structs.asdict(facts) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/executor.py b/sgl_nvidia_mega/qwen3_5/runtime/executor.py index 10db108f..e9e21c8c 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/executor.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/executor.py @@ -187,11 +187,14 @@ def _check_pools(self, view): "NVIDIA megakernel: slot gdn_front cannot run: " + "; ".join(reasons) ) - if any(s.skv2 for s in self.slots.values()): - reasons = attention.pool_reasons(view, self.binding) + skv2 = any(s.skv2 for s in self.slots.values()) + mka = any(s.mka for s in self.slots.values()) + if skv2 or mka: + reasons = attention.pool_reasons(view, self.binding, skv2=skv2, mka=mka) if reasons: raise RuntimeError( - "NVIDIA megakernel: slot skv2 cannot run: " + "; ".join(reasons) + "NVIDIA megakernel: the attention slots cannot run: " + + "; ".join(reasons) ) self._pools_checked = True diff --git a/sgl_nvidia_mega/qwen3_5/sm103/attention/mka.py b/sgl_nvidia_mega/qwen3_5/sm103/attention/mka.py new file mode 100644 index 00000000..e98af0d7 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/attention/mka.py @@ -0,0 +1,264 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang branch nvidia/qwen35-agentx-gb300: +# python/sglang/kernels/ops/attention/mka_fused.py and mka_fused.cu (its host side). +"""MKA (``src/mka.cu``): the fused full-attention prologue and epilogue of one request's +verify rows (at most 8) at hidden 4096 and head_dim 256, the kernels of SGLang branch +nvidia/qwen35-agentx-gb300 built with NVRTC. + +``prologue``: the normed hidden rows' static FP8 quantization, the qkv (+ gate) GEMM, the +q / k Gemma RMSNorm, the partial NeoX M-RoPE, the FP8 query rows, the FP8 K / V rows +written into the paged cache (one KV head a rank) and the bf16 gate. ``epilogue``: the +attention output times sigmoid(gate), its static FP8 quantization and the ``o_proj`` +GEMM to bf16, optionally pushed into the decode hop's mailbox. Each launches clusters of +8 CTAs with the dependent launch; the weights come packed by ``pack_qkv`` / ``pack_o``. +""" + +from __future__ import annotations + +import ctypes +from functools import cache +from pathlib import Path +from typing import Optional + +import torch + +from sgl_nvidia_mega.common import jit +from sgl_nvidia_mega.qwen3_5.sm103 import hop +from sgl_nvidia_mega.qwen3_5.sm103.hop import Site + +SOURCE = Path(__file__).parent / "src" / "mka.cu" +THREADS, CLUSTER, HEAD_DIM, HIDDEN, MAX_ROWS = 256, 8, 256, 4096, 8 +ENTRIES = ( + "q35mka_prologue", + "q35mka_prologue_late", + "q35mka_gate", + "q35mka_epilogue_q4", + "q35mka_epilogue_q4_late", + "q35mka_epilogue_q8", + "q35mka_epilogue_q8_late", +) + + +class ProParams(ctypes.Structure): + _fields_ = [ + (name, ctypes.c_void_p) + for name in ( + "x", + "w", + "sx", + "sw", + "qnw", + "knw", + "cos_sin", + "pos", + "axis", + "loc", + ) + ] + [ + ("loc_i32", ctypes.c_int), + ("k_cache", ctypes.c_void_p), + ("v_cache", ctypes.c_void_p), + ("k_scale", ctypes.c_void_p), + ("v_scale", ctypes.c_void_p), + ("q_out", ctypes.c_void_p), + ("gate_out", ctypes.c_void_p), + ("m", ctypes.c_int), + ("nq", ctypes.c_int), + ("pos_stride", ctypes.c_int), + ("cs_stride", ctypes.c_int), + ("eps", ctypes.c_float), + ("split", ctypes.c_int), + ("wg", ctypes.c_void_p), + ] + + +class EpiParams(ctypes.Structure): + _fields_ = [ + (name, ctypes.c_void_p) for name in ("attn", "gate", "w", "sx", "sw", "out") + ] + [("m", ctypes.c_int), ("push", ctypes.c_int), ("site", Site)] + + +@cache +def _kernels(device_index: int) -> dict[str, jit.Kernel]: + major, minor = torch.cuda.get_device_capability(device_index) + cubin = jit.compile_source( + SOURCE.read_text(), + "q35_mka", + f"sm_{major}{minor}a", + ("-std=c++17", "-default-device"), + headers=hop.headers(), + ) + return {n: jit.load(cubin, n, (THREADS, 1, 1), 0) for n in ENTRIES} + + +def build(device) -> None: + """Compile (or load) the kernels now, before any capture.""" + _kernels(torch.device(device).index) + + +def _launch(name: str, grid: int, params, device, cluster: bool = True) -> None: + jit.launch( + _kernels(device.index)[name], + (grid, 1, 1), + [params], + torch.cuda.current_stream(device).cuda_stream, + (CLUSTER, 1, 1) if cluster else None, + ) + + +def _pack_fragments(w_nk: torch.Tensor, units: int, steps: int) -> torch.Tensor: + # [units*256, K] e4m3 -> [unit][rank 8][warp 8][step][m16 block 2][lane 32][16 B] mma.m16n8k32 A + # fragments; rank r covers K [r*steps*32, (r+1)*steps*32), warp w rows [32w, 32w + 32) of the unit. + w = w_nk.contiguous().view(torch.uint8) + assert w.shape[0] == units * 256 and w.shape[1] == 8 * steps * 32, w.shape + v = w.view(units, 8, 2, 2, 8, 8, steps, 2, 4, 4) + return v.permute(0, 5, 1, 6, 2, 4, 8, 7, 3, 9).contiguous().view(-1) + + +def _pack_rows32(w_nk: torch.Tensor) -> torch.Tensor: + # [R, 4096] e4m3 -> [cta = R/32][warp 8][step 16][m16 block 2][lane 32][16 B]; warp w covers K + # [512w, 512w + 512) of the CTA's 32 rows (mka_gate_kernel). + w = w_nk.contiguous().view(torch.uint8) + v = w.view(w.shape[0] // 32, 2, 2, 8, 8, 16, 2, 4, 4) + return v.permute(0, 4, 5, 1, 3, 7, 6, 2, 8).contiguous().view(-1) + + +def split_gate(nq: int) -> bool: + """Whether the prologue hands the gate rows to mka_gate_kernel. + + 2*nq + 2 clusters of 8 CTAs exceed the 15 that co-reside on GB300 once nq = 8 (TP4). + """ + return nq >= 8 + + +def pack_qkv(w_nk: torch.Tensor, nq: int) -> tuple[torch.Tensor, torch.Tensor]: + """qkv(+gate) weight [N, 4096] e4m3 -> (prologue layout, gate-kernel layout or empty).""" + main = _pack_fragments(w_nk, w_nk.shape[0] // 256, 16) + if not split_gate(nq): + return main, main.new_empty(0) + gate_rows = ( + w_nk[: nq * 512] + .reshape(nq, 2, 256, w_nk.shape[1])[:, 1] + .reshape(nq * 256, w_nk.shape[1]) + ) + return main, _pack_rows32(gate_rows) + + +def pack_o(w_nk: torch.Tensor) -> torch.Tensor: + """o_proj weight [4096, nq*256] e4m3 -> epilogue layout (K chunk nq*32 per cluster rank).""" + return _pack_fragments(w_nk, 16, w_nk.shape[1] // 256) + + +def prologue( + x, + w_packed, + sx, + sw, + q_norm_w, + k_norm_w, + cos_sin, + positions, + axis_map, + cache_loc, + k_cache, + v_cache, + k_scale, + v_scale, + q_out, + gate_out, + nq: int, + eps: float, + early_trigger: bool = True, +) -> None: + """The branch's ``prologue`` (its host function's checks and launches).""" + w_main, w_gate = w_packed + if ( + not (x.dim() == 2 and x.shape[0] <= MAX_ROWS and x.shape[1] == HIDDEN) + or not x.is_contiguous() + ): + raise ValueError( + f"MKA prologue: x {tuple(x.shape)} is not [<= 8, 4096] contiguous" + ) + if positions.dim() != 2 or positions.shape[0] != 3 or positions.stride(1) != 1: + raise ValueError( + "MKA prologue: positions must be M-RoPE [3, rows], rows contiguous" + ) + if cos_sin.dtype != torch.float32 or axis_map.dtype != torch.int64: + raise ValueError("MKA prologue: cos_sin must be float32 and axis_map int64") + if positions.dtype != torch.int64: + raise ValueError("MKA prologue: positions must be int64") + if k_cache.stride(0) != HEAD_DIM or v_cache.stride(0) != HEAD_DIM: + raise ValueError("MKA prologue: one KV head of 256 per slot") + if cache_loc.dtype not in (torch.int64, torch.int32): + raise ValueError(f"MKA prologue: cache_loc of {cache_loc.dtype}") + split = w_gate.numel() > 0 + p = ProParams( + x=x.data_ptr(), + w=w_main.data_ptr(), + sx=sx.data_ptr(), + sw=sw.data_ptr(), + qnw=q_norm_w.data_ptr(), + knw=k_norm_w.data_ptr(), + cos_sin=cos_sin.data_ptr(), + pos=positions.data_ptr(), + axis=axis_map.data_ptr(), + loc=cache_loc.data_ptr(), + loc_i32=int(cache_loc.dtype == torch.int32), + k_cache=k_cache.data_ptr(), + v_cache=v_cache.data_ptr(), + k_scale=k_scale.data_ptr(), + v_scale=v_scale.data_ptr(), + q_out=q_out.data_ptr(), + gate_out=gate_out.data_ptr(), + m=int(x.shape[0]), + nq=int(nq), + pos_stride=int(positions.stride(0)), + cs_stride=int(cos_sin.stride(0)), + eps=float(eps), + split=int(split), + wg=w_gate.data_ptr() if split else None, + ) + if not split: + name = "q35mka_prologue" if early_trigger else "q35mka_prologue_late" + _launch(name, (2 * nq + 2) * CLUSTER, p, x.device) + return + # The gate launch must start while the prologue runs, so the prologue triggers right after its wait. + _launch("q35mka_prologue", (nq + 2) * CLUSTER, p, x.device) + _launch("q35mka_gate", nq * HEAD_DIM // 32, p, x.device, cluster=False) + + +def epilogue( + attn, + gate, + w_packed, + sx, + sw, + out, + nq: int, + early_trigger: bool = True, + site: Optional[Site] = None, +) -> None: + """The branch's ``epilogue``; ``site``: the hop mailbox to push the rows into.""" + if not (attn.is_contiguous() and gate.is_contiguous() and out.is_contiguous()): + raise ValueError("MKA epilogue: attn, gate and out must be contiguous") + if attn.shape[0] > MAX_ROWS or attn.shape[1] != nq * HEAD_DIM: + raise ValueError(f"MKA epilogue: attn {tuple(attn.shape)} for {nq} heads") + if nq not in (4, 8): + raise ValueError("MKA epilogue: nq must be 4 (TP8) or 8 (TP4)") + m = int(attn.shape[0]) + if site is not None and m > site.m_cap: + raise ValueError(f"MKA epilogue: {m} rows past the hop's {site.m_cap}") + p = EpiParams( + attn=attn.data_ptr(), + gate=gate.data_ptr(), + w=w_packed.data_ptr(), + sx=sx.data_ptr(), + sw=sw.data_ptr(), + out=out.data_ptr(), + m=m, + push=int(site is not None), + site=site if site is not None else Site(), + ) + name = f"q35mka_epilogue_q{nq}" + ("" if early_trigger else "_late") + _launch(name, 16 * CLUSTER, p, attn.device) diff --git a/sgl_nvidia_mega/qwen3_5/sm103/attention/src/mka.cu b/sgl_nvidia_mega/qwen3_5/sm103/attention/src/mka.cu new file mode 100644 index 00000000..5531d774 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/attention/src/mka.cu @@ -0,0 +1,538 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copied from SGLang branch nvidia/qwen35-agentx-gb300: +// python/sglang/kernels/ops/attention/mka_fused.cu, device code unchanged. Built with NVRTC: the +// kernels are device bodies under C-linkage entries (below), and the push reads the plugin's hop. +// +// Fused full-attention prologue / epilogue for Qwen3.5 bs1 verify (M <= 8 tokens), SM100+. +// +// prologue: normed hidden (bf16) -> static FP8 quant -> qkv(+gate) GEMM -> q/k Gemma RMSNorm -> partial +// NeoX M-RoPE -> FP8 Q, FP8 K/V into the paged cache (one KV head per rank), bf16 gate. +// epilogue: attention output (bf16) * sigmoid(gate) -> bf16 -> static FP8 -> o_proj GEMM -> bf16. +// +// Both kernels give one 8-CTA cluster 256 output rows and split K over the cluster ranks. Weights are +// pre-packed as mma.m16n8k32 e4m3 A fragments and loaded into registers before griddepcontrol.wait, so +// the weight stream overlaps the predecessor. Rank r owns token r: every rank pushes its partial rows of +// token r into rank r's inbox (st.async + mbarrier), which keeps a whole head local for the q/k norm. +// Rounding steps copy the stock Triton kernels (static quant, ATTN_FRONT, ATTN_GATE_QUANT) instruction +// for instruction; only the FP32 summation order of the two GEMMs and of the norm differs. +#include + +// the hop's device protocol (sm103/hop.headers()): q35hop::Site, push_frag, stage_peek; it also +// declares the fixed-width integers NVRTC lacks, but for int64_t +#include "q35_hop.cuh" + +typedef long long int64_t; + +namespace q35mka { + +constexpr int kThreads = 256; +constexpr int kCluster = 8; +constexpr int kHeadDim = 256; +constexpr int kHalfRot = 32; // rotary_dim 64 + +__device__ __forceinline__ void pdl_wait() { asm volatile("griddepcontrol.wait;" ::: "memory"); } +__device__ __forceinline__ void pdl_trigger() { asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); } +__device__ __forceinline__ void cluster_arrive_relaxed() { + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); +} +__device__ __forceinline__ void cluster_wait() { asm volatile("barrier.cluster.wait.aligned;" ::: "memory"); } +__device__ __forceinline__ uint32_t cluster_ctarank() { + uint32_t r; + asm("mov.u32 %0, %%cluster_ctarank;" : "=r"(r)); + return r; +} +__device__ __forceinline__ uint32_t smem_u32(const void* ptr) { + return static_cast(__cvta_generic_to_shared(ptr)); +} +__device__ __forceinline__ uint32_t mapa(uint32_t addr, uint32_t rank) { + uint32_t out; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(out) : "r"(addr), "r"(rank)); + return out; +} +__device__ __forceinline__ void mbar_init(uint64_t* bar, uint32_t count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" ::"r"(smem_u32(bar)), "r"(count) : "memory"); +} +// Publishes the mbarrier init to the cluster before peers' st.async can credit it. +__device__ __forceinline__ void mbar_init_fence() { + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); +} +__device__ __forceinline__ void mbar_expect_tx(uint64_t* bar, uint32_t bytes) { + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(smem_u32(bar)), "r"(bytes) + : "memory"); +} +__device__ __forceinline__ void mbar_wait(uint64_t* bar, uint32_t parity) { + asm volatile( + "{\n .reg .pred p;\n WAIT_%=:\n mbarrier.try_wait.parity.shared::cta.b64 p, [%0], %1;\n @!p bra WAIT_%=;\n}" ::"r"( + smem_u32(bar)), + "r"(parity) + : "memory"); +} +__device__ __forceinline__ void st_async_v4(uint32_t dst, float a, float b, float c, float d, uint32_t bar) { + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.f32 [%0], {%1, %2, %3, %4}, [%5];" ::"r"(dst), + "f"(a), "f"(b), "f"(c), "f"(d), "r"(bar) + : "memory"); +} +__device__ __forceinline__ uint4 ldg_nc_v4(const uint4* p) { + uint4 r; + asm volatile("ld.global.nc.L1::no_allocate.v4.u32 {%0,%1,%2,%3}, [%4];" + : "=r"(r.x), "=r"(r.y), "=r"(r.z), "=r"(r.w) + : "l"(p)); + return r; +} +__device__ __forceinline__ float div_full(float a, float b) { + float r; + asm("div.full.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} +__device__ __forceinline__ float rsqrt_approx_ftz(float a) { + float r; + asm("rsqrt.approx.ftz.f32 %0, %1;" : "=f"(r) : "f"(a)); + return r; +} +__device__ __forceinline__ float ex2_approx(float a) { + float r; + asm("ex2.approx.f32 %0, %1;" : "=f"(r) : "f"(a)); + return r; +} +__device__ __forceinline__ float bf16_round(float a) { return __bfloat162float(__float2bfloat16_rn(a)); } +// low byte = e4m3(lo), high byte = e4m3(hi) +__device__ __forceinline__ uint32_t e4m3x2(float lo, float hi) { + uint16_t r; + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(r) : "f"(hi), "f"(lo)); + return r; +} +__device__ __forceinline__ uint8_t e4m3(float v) { return (uint8_t)(e4m3x2(v, 0.f) & 0xff); } + +__device__ __forceinline__ void mma_e4m3(float (&d)[4], const uint4& a, uint32_t b0, uint32_t b1) { + asm volatile( + "mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, " + "{%0,%1,%2,%3};" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a.x), "r"(a.y), "r"(a.z), "r"(a.w), "r"(b0), "r"(b1)); +} + +// _static_quant_fp8: y * div.full(1, s), clamp +-448, cvt.rn.satfinite.e4m3 +__device__ __forceinline__ uint32_t quant4_static(const __nv_bfloat16* p, float inv) { + const uint2 raw = *reinterpret_cast(p); + const __nv_bfloat162 v01 = *reinterpret_cast(&raw.x); + const __nv_bfloat162 v23 = *reinterpret_cast(&raw.y); + float f[4] = {__low2float(v01), __high2float(v01), __low2float(v23), __high2float(v23)}; +#pragma unroll + for (int i = 0; i < 4; ++i) f[i] = fminf(fmaxf(__fmul_rn(f[i], inv), -448.f), 448.f); + return e4m3x2(f[0], f[1]) | (e4m3x2(f[2], f[3]) << 16); +} + +// ATTN_GATE_QUANT: sig = div.full(1, 1 + ex2.approx(g * -log2e)); bf16(sig * o) * div.full(1, s); clamp +__device__ __forceinline__ float gate_quant1(float o, float g, float inv) { + const float e = ex2_approx(__fmul_rn(g, __int_as_float(0xBFB8AA3B))); + const float sig = div_full(1.0f, __fadd_rn(e, 1.0f)); + const float y = __fmul_rn(inv, bf16_round(__fmul_rn(sig, o))); + return copysignf(fminf(fabsf(y), 448.f), y); +} +__device__ __forceinline__ uint32_t gate_quant4(const __nv_bfloat16* o, const __nv_bfloat16* g, float inv) { + const uint2 ro = *reinterpret_cast(o); + const uint2 rg = *reinterpret_cast(g); + const __nv_bfloat162 o01 = *reinterpret_cast(&ro.x); + const __nv_bfloat162 o23 = *reinterpret_cast(&ro.y); + const __nv_bfloat162 g01 = *reinterpret_cast(&rg.x); + const __nv_bfloat162 g23 = *reinterpret_cast(&rg.y); + return e4m3x2(gate_quant1(__low2float(o01), __low2float(g01), inv), + gate_quant1(__high2float(o01), __high2float(g01), inv)) | + (e4m3x2(gate_quant1(__low2float(o23), __low2float(g23), inv), + gate_quant1(__high2float(o23), __high2float(g23), inv)) + << 16); +} + +// Inbox slot s of a 256-row tile -> row: warp w = s/32 holds rows 32w + {g, g+8, g+16, g+24}. +__device__ __forceinline__ int slot_row(int s) { return 32 * (s >> 5) + ((s >> 2) & 7) + 8 * (s & 3); } + +struct ProParams { + const __nv_bfloat16* x; // [M, 4096] + const uint4* w; // packed qkv(+gate) weights + const float* sx; + const float* sw; + const __nv_bfloat16* qnw; + const __nv_bfloat16* knw; + const float* cos_sin; // [max_pos, 64] fp32 + const int64_t* pos; // [3, M] + const int64_t* axis; // [32] + const void* loc; // [M] cache slots, int64 or int32 + int loc_i32; + uint8_t* k_cache; // [slots, 256] e4m3 + uint8_t* v_cache; + const float* k_scale; + const float* v_scale; + uint8_t* q_out; // [M, nq*256] e4m3 + __nv_bfloat16* gate_out; // [M, nq*256] + int m; + int nq; + int pos_stride; + int cs_stride; + float eps; + int split; // 1: this launch skips the gate units, which mka_gate_kernel computes + const uint4* wg; // split: gate rows packed 32 rows x full K per CTA +}; + +// grid = (2*nq + 2) units x 8 ranks; unit u = qkv rows [256u, 256u + 256) (q_h, gate_h per head, then k, +// v); rank r = K chunk [512r, 512r + 512) for the GEMM and token r for the epilogue. +template +__device__ __forceinline__ void mka_prologue_kernel(const ProParams p) { + __shared__ __align__(16) float inbox[kCluster][256]; + __shared__ __align__(16) uint2 bfr[16][32]; + __shared__ float red8[8]; + __shared__ float nrm[256]; + __shared__ __align__(8) uint64_t ibar; + const int rank = (int)cluster_ctarank(); + const int nq = p.nq; + // split launches cover q_0..q_{nq-1}, k, v; natural units interleave q_h and gate_h, then k, v + const int u = blockIdx.x / kCluster; + const int unit = !p.split ? u : (u < nq ? 2 * u : 2 * nq + (u - nq)); + const int tid = threadIdx.x, warp = tid >> 5, lane = tid & 31; + const int g = lane >> 2, t = lane & 3; + if (tid == 0) { + mbar_init(&ibar, 1); + mbar_init_fence(); + } + __syncthreads(); + cluster_arrive_relaxed(); + + uint4 a[16][2]; + { + const uint4* wb = p.w + (size_t)(unit * kCluster + rank) * (8 * 16 * 2 * 32) + warp * 1024 + lane; +#pragma unroll + for (int s = 0; s < 16; ++s) +#pragma unroll + for (int b = 0; b < 2; ++b) a[s][b] = ldg_nc_v4(wb + (s * 2 + b) * 32); + } + // Static operands only before the wait, and nothing consumes them there. + const bool is_q = unit < 2 * nq && (unit & 1) == 0; + const bool is_gate = unit < 2 * nq && (unit & 1) == 1; + const bool is_k = unit == 2 * nq; + const bool needs_norm = is_q || is_k; + const int f = slot_row(tid); + const bool rot = needs_norm && f < 2 * kHalfRot; + const int rl = f & (kHalfRot - 1); + const float sx = __ldg(p.sx), sw = __ldg(p.sw); + __nv_bfloat16 nwr = __float2bfloat16_rn(0.f); + if (needs_norm) nwr = (is_q ? p.qnw : p.knw)[f]; + int64_t ax = 0; + if (rot) ax = __ldg(p.axis + rl); + float kvs = 1.f; + if (is_k) kvs = __ldg(p.k_scale); + if (!needs_norm && !is_gate) kvs = __ldg(p.v_scale); + + pdl_wait(); + if (kEarlyTrigger) pdl_trigger(); + const float inv_sx = div_full(1.0f, sx); + const float alpha = __fmul_rn(sx, sw); + const int m = p.m; + const bool owner = rank < m; + int64_t slot = 0; + float cs_c = 0.f, cs_s = 0.f; + if (owner) { + slot = p.loc_i32 ? (int64_t)__ldg(static_cast(p.loc) + rank) + : __ldg(static_cast(p.loc) + rank); + if (rot) { + const int64_t ps = __ldg(p.pos + ax * p.pos_stride + rank); + cs_c = __ldg(p.cos_sin + ps * p.cs_stride + rl); + cs_s = __ldg(p.cos_sin + ps * p.cs_stride + kHalfRot + rl); + } + } + { + const bool tok_ok = g < m; + const __nv_bfloat16* xr = p.x + (size_t)g * 4096 + rank * 512 + 4 * t; +#pragma unroll + for (int i = 0; i < 2; ++i) { + const int s = 2 * warp + i; + bfr[s][lane] = tok_ok ? make_uint2(quant4_static(xr + s * 32, inv_sx), quant4_static(xr + s * 32 + 16, inv_sx)) + : make_uint2(0u, 0u); + } + } + __syncthreads(); + float ch[2][4][4]; +#pragma unroll + for (int b = 0; b < 2; ++b) +#pragma unroll + for (int c = 0; c < 4; ++c) +#pragma unroll + for (int i = 0; i < 4; ++i) ch[b][c][i] = 0.f; +#pragma unroll + for (int s = 0; s < 16; ++s) { + const uint2 bb = bfr[s][lane]; + mma_e4m3(ch[0][s & 3], a[s][0], bb.x, bb.y); + mma_e4m3(ch[1][s & 3], a[s][1], bb.x, bb.y); + } + float acc[2][4]; +#pragma unroll + for (int b = 0; b < 2; ++b) +#pragma unroll + for (int i = 0; i < 4; ++i) + acc[b][i] = __fadd_rn(__fadd_rn(ch[b][0][i], ch[b][1][i]), __fadd_rn(ch[b][2][i], ch[b][3][i])); + + if (tid == 0 && owner) mbar_expect_tx(&ibar, kCluster * 256 * 4); + cluster_wait(); + { + const uint32_t src = smem_u32(&inbox[rank][warp * 32 + g * 4]); + const uint32_t bar = smem_u32(&ibar); +#pragma unroll + for (int j = 0; j < 2; ++j) { + const int tok = 2 * t + j; + if (tok < m) st_async_v4(mapa(src, tok), acc[0][j], acc[0][2 + j], acc[1][j], acc[1][2 + j], mapa(bar, tok)); + } + } + if (!owner) { + if (!kEarlyTrigger) pdl_trigger(); + return; + } + mbar_wait(&ibar, 0); // also keeps this CTA alive until every peer's push has landed + + float v = 0.f; +#pragma unroll + for (int r = 0; r < kCluster; ++r) v = __fadd_rn(v, inbox[r][tid]); + v = bf16_round(__fmul_rn(v, alpha)); + const int tok = rank; + if (is_gate) { + p.gate_out[(size_t)tok * nq * kHeadDim + (unit >> 1) * kHeadDim + f] = __float2bfloat16_rn(v); + } else if (!needs_norm) { + p.v_cache[(size_t)slot * kHeadDim + f] = e4m3(__fmul_rn(__fdiv_rn(1.0f, kvs), v)); + } else { + float ss = __fmul_rn(v, v); +#pragma unroll + for (int o = 16; o > 0; o >>= 1) ss = __fadd_rn(ss, __shfl_xor_sync(0xffffffffu, ss, o)); + if (lane == 0) red8[warp] = ss; + __syncthreads(); + float tot = 0.f; +#pragma unroll + for (int i = 0; i < 8; ++i) tot = __fadd_rn(tot, red8[i]); + const float inv = rsqrt_approx_ftz(__fadd_rn(div_full(tot, 256.0f), p.eps)); + float n = bf16_round(__fmul_rn(__fadd_rn(__bfloat162float(nwr), 1.0f), __fmul_rn(inv, v))); + nrm[f] = n; + __syncthreads(); + if (rot) { + // first half: fma(cos, x1, -(sin * x2)); second half: fma(x1, sin, cos * x2) + const float x1 = f < kHalfRot ? n : nrm[f - kHalfRot]; + const float x2 = f < kHalfRot ? nrm[f + kHalfRot] : n; + n = f < kHalfRot ? bf16_round(__fmaf_rn(cs_c, x1, -__fmul_rn(cs_s, x2))) + : bf16_round(__fmaf_rn(x1, cs_s, __fmul_rn(cs_c, x2))); + } + if (is_k) + p.k_cache[(size_t)slot * kHeadDim + f] = e4m3(__fmul_rn(__fdiv_rn(1.0f, kvs), n)); + else + p.q_out[(size_t)tok * nq * kHeadDim + (unit >> 1) * kHeadDim + f] = e4m3(n); + } + if (!kEarlyTrigger) pdl_trigger(); +} + +// Gate rows for split launches: no cluster, CTA c owns gate rows [32c, 32c + 32) (head c/8) over the +// full K = 4096; warp w takes K [512w, 512w + 512) and the 8 warp partials are summed in warp order. +// It runs beside the prologue: launched when the prologue triggers (after the hop's output is visible), +// it reads x through L2 without its own griddepcontrol.wait and waits for the prologue only at the end, +// so a consumer of this launch also sees the prologue's Q / KV. +__device__ __forceinline__ void mka_gate_kernel(const ProParams p) { + __shared__ __align__(16) float red[8][32][8]; + const int cta = blockIdx.x; + const int tid = threadIdx.x, warp = tid >> 5, lane = tid & 31; + const int g = lane >> 2, t = lane & 3; + pdl_trigger(); + uint4 a[16][2]; + { + const uint4* wb = p.wg + ((size_t)cta * 8 + warp) * (16 * 2 * 32) + lane; +#pragma unroll + for (int s = 0; s < 16; ++s) +#pragma unroll + for (int b = 0; b < 2; ++b) a[s][b] = ldg_nc_v4(wb + (s * 2 + b) * 32); + } + const float sx = __ldg(p.sx), sw = __ldg(p.sw); + const float inv_sx = div_full(1.0f, sx); + const float alpha = __fmul_rn(sx, sw); + const int m = p.m; + const bool tok_ok = g < m; + const __nv_bfloat16* xr = p.x + (size_t)g * 4096 + warp * 512 + 4 * t; + uint2 raw[16][2]; +#pragma unroll + for (int s = 0; s < 16; ++s) +#pragma unroll + for (int h = 0; h < 2; ++h) + raw[s][h] = tok_ok ? __ldcg(reinterpret_cast(xr + s * 32 + h * 16)) : make_uint2(0u, 0u); + float ch[2][4][4]; +#pragma unroll + for (int b = 0; b < 2; ++b) +#pragma unroll + for (int c = 0; c < 4; ++c) +#pragma unroll + for (int i = 0; i < 4; ++i) ch[b][c][i] = 0.f; +#pragma unroll + for (int s = 0; s < 16; ++s) { + const uint32_t b0 = quant4_static(reinterpret_cast(&raw[s][0]), inv_sx); + const uint32_t b1 = quant4_static(reinterpret_cast(&raw[s][1]), inv_sx); + mma_e4m3(ch[0][s & 3], a[s][0], b0, b1); + mma_e4m3(ch[1][s & 3], a[s][1], b0, b1); + } + { + float4* dst = reinterpret_cast(&red[warp][lane][0]); +#pragma unroll + for (int b = 0; b < 2; ++b) + dst[b] = make_float4(__fadd_rn(__fadd_rn(ch[b][0][0], ch[b][1][0]), __fadd_rn(ch[b][2][0], ch[b][3][0])), + __fadd_rn(__fadd_rn(ch[b][0][1], ch[b][1][1]), __fadd_rn(ch[b][2][1], ch[b][3][1])), + __fadd_rn(__fadd_rn(ch[b][0][2], ch[b][1][2]), __fadd_rn(ch[b][2][2], ch[b][3][2])), + __fadd_rn(__fadd_rn(ch[b][0][3], ch[b][1][3]), __fadd_rn(ch[b][2][3], ch[b][3][3]))); + } + __syncthreads(); + { + // thread -> fragment lane l = tid/8, element e = tid%8: block e/4, row 16*(e/4) + g' + 8*((e%4)/2), + // token 2t' + (e%2) + const int l = tid >> 3, e = tid & 7; + float v = red[0][l][e]; +#pragma unroll + for (int w = 1; w < 8; ++w) v = __fadd_rn(v, red[w][l][e]); + const int row = 16 * (e >> 2) + (l >> 2) + 8 * ((e & 3) >> 1); + const int tok = 2 * (l & 3) + (e & 1); + const int h = cta >> 3, r = (cta & 7) * 32 + row; + if (tok < m) + p.gate_out[(size_t)tok * p.nq * kHeadDim + h * kHeadDim + r] = __float2bfloat16_rn(__fmul_rn(v, alpha)); + } + pdl_wait(); // completes only after the prologue grid has completed +} + +struct EpiParams { + const __nv_bfloat16* attn; // [M, nq*256] + const __nv_bfloat16* gate; // [M, nq*256] + const uint4* w; // packed o_proj weights + const float* sx; + const float* sw; + __nv_bfloat16* out; // [M, 4096] + int m; + int push; // 1: also push the bf16 partial into the q35 hop mailbox (MKH interface v1) + q35hop::Site site; +}; + +// grid = 16 units x 8 ranks; NQ = q heads per rank (K = NQ*256, K chunk NQ*32 per rank). +template +__device__ __forceinline__ void mka_epilogue_kernel(const EpiParams p) { + constexpr int kSteps = NQ; + constexpr int kK = NQ * kHeadDim; + constexpr int kWarpU4 = kSteps * 2 * 32; + __shared__ __align__(16) float inbox[kCluster][256]; + __shared__ __align__(16) uint2 bfr[kSteps][32]; + __shared__ __align__(16) __nv_bfloat16 ob[256]; + __shared__ __align__(8) uint64_t ibar; + const int rank = (int)cluster_ctarank(); + const int unit = blockIdx.x / kCluster; + const int tid = threadIdx.x, warp = tid >> 5, lane = tid & 31; + const int g = lane >> 2, t = lane & 3; + if (tid == 0) { + mbar_init(&ibar, 1); + mbar_init_fence(); + } + __syncthreads(); + cluster_arrive_relaxed(); + + uint4 a[kSteps][2]; + { + const uint4* wb = p.w + (size_t)(unit * kCluster + rank) * (8 * kWarpU4) + warp * kWarpU4 + lane; +#pragma unroll + for (int s = 0; s < kSteps; ++s) +#pragma unroll + for (int b = 0; b < 2; ++b) a[s][b] = ldg_nc_v4(wb + (s * 2 + b) * 32); + } + const float sx = __ldg(p.sx), sw = __ldg(p.sw); + + pdl_wait(); + if (kEarlyTrigger) pdl_trigger(); + const float inv_sx = div_full(1.0f, sx); + const float alpha = __fmul_rn(sx, sw); + const int m = p.m; + { + const bool tok_ok = g < m; + const size_t off = (size_t)g * kK + rank * (kSteps * 32) + 4 * t; + uint32_t* bw = reinterpret_cast(&bfr[0][0]); +#pragma unroll + for (int i = warp; i < 2 * kSteps; i += 8) { + const int s = i >> 1, h = i & 1; + bw[(s * 32 + lane) * 2 + h] = + tok_ok ? gate_quant4(p.attn + off + s * 32 + h * 16, p.gate + off + s * 32 + h * 16, inv_sx) : 0u; + } + } + __syncthreads(); + constexpr int kCh = kSteps < 4 ? kSteps : 4; + float ch[2][kCh][4]; +#pragma unroll + for (int b = 0; b < 2; ++b) +#pragma unroll + for (int c = 0; c < kCh; ++c) +#pragma unroll + for (int i = 0; i < 4; ++i) ch[b][c][i] = 0.f; +#pragma unroll + for (int s = 0; s < kSteps; ++s) { + const uint2 bb = bfr[s][lane]; + mma_e4m3(ch[0][s % kCh], a[s][0], bb.x, bb.y); + mma_e4m3(ch[1][s % kCh], a[s][1], bb.x, bb.y); + } + float acc[2][4]; +#pragma unroll + for (int b = 0; b < 2; ++b) +#pragma unroll + for (int i = 0; i < 4; ++i) { + float x = ch[b][0][i]; +#pragma unroll + for (int c = 1; c < kCh; ++c) x = __fadd_rn(x, ch[b][c][i]); + acc[b][i] = x; + } + const bool owner = rank < m; + if (tid == 0 && owner) mbar_expect_tx(&ibar, kCluster * 256 * 4); + cluster_wait(); + { + const uint32_t src = smem_u32(&inbox[rank][warp * 32 + g * 4]); + const uint32_t bar = smem_u32(&ibar); +#pragma unroll + for (int j = 0; j < 2; ++j) { + const int tok = 2 * t + j; + if (tok < m) st_async_v4(mapa(src, tok), acc[0][j], acc[0][2 + j], acc[1][j], acc[1][2 + j], mapa(bar, tok)); + } + } + if (!owner) { + if (!kEarlyTrigger) pdl_trigger(); + return; + } + mbar_wait(&ibar, 0); + float v = 0.f; +#pragma unroll + for (int r = 0; r < kCluster; ++r) v = __fadd_rn(v, inbox[r][tid]); + const __nv_bfloat16 o = __float2bfloat16_rn(__fmul_rn(v, alpha)); + if (p.push) { + // fragments are 8 consecutive columns, so regroup the row through smem; push before the store + ob[slot_row(tid)] = o; + __syncthreads(); + if (tid < 32) + q35hop::push_frag(p.site, q35hop::stage_peek(p.site), rank, unit * 32 + tid, + *reinterpret_cast(&ob[8 * tid])); + } + p.out[(size_t)rank * 4096 + unit * kHeadDim + slot_row(tid)] = o; + if (!kEarlyTrigger) pdl_trigger(); +} + +} // namespace q35mka + +// C-linkage entries, one per instantiation the launcher uses (mka.py) +extern "C" __global__ void __launch_bounds__(q35mka::kThreads, 1) q35mka_prologue(const q35mka::ProParams p) { + q35mka::mka_prologue_kernel(p); +} +extern "C" __global__ void __launch_bounds__(q35mka::kThreads, 1) q35mka_prologue_late(const q35mka::ProParams p) { + q35mka::mka_prologue_kernel(p); +} +extern "C" __global__ void __launch_bounds__(q35mka::kThreads, 1) q35mka_gate(const q35mka::ProParams p) { + q35mka::mka_gate_kernel(p); +} +extern "C" __global__ void __launch_bounds__(q35mka::kThreads, 1) q35mka_epilogue_q4(const q35mka::EpiParams p) { + q35mka::mka_epilogue_kernel<4, true>(p); +} +extern "C" __global__ void __launch_bounds__(q35mka::kThreads, 1) q35mka_epilogue_q4_late(const q35mka::EpiParams p) { + q35mka::mka_epilogue_kernel<4, false>(p); +} +extern "C" __global__ void __launch_bounds__(q35mka::kThreads, 1) q35mka_epilogue_q8(const q35mka::EpiParams p) { + q35mka::mka_epilogue_kernel<8, true>(p); +} +extern "C" __global__ void __launch_bounds__(q35mka::kThreads, 1) q35mka_epilogue_q8_late(const q35mka::EpiParams p) { + q35mka::mka_epilogue_kernel<8, false>(p); +} diff --git a/sgl_nvidia_mega/qwen3_5/spec.py b/sgl_nvidia_mega/qwen3_5/spec.py index 1f741b95..78861069 100644 --- a/sgl_nvidia_mega/qwen3_5/spec.py +++ b/sgl_nvidia_mega/qwen3_5/spec.py @@ -113,6 +113,13 @@ }, # SKV2 against SGLang's TRT-LLM MHA decode (test_skv2_is_within_twice_sglangs_error) "skv2": {"skv2.attention.rel_l2_vs_stock": 2.0}, + # MKA's own qkv / o_proj GEMMs; the GEMM slots' bounds, no fp64 kernel test yet + "mka": { + "mka.qkv_proj.rel_l2_vs_stock": 1.05, + "mka.qkv_proj.max_abs_vs_stock": 2.0, + "mka.o_proj.rel_l2_vs_stock": 1.05, + "mka.o_proj.max_abs_vs_stock": 2.0, + }, } diff --git a/test/kernels/test_qwen3_5_attention.py b/test/kernels/test_qwen3_5_attention.py index bca0cca8..2158b9c4 100644 --- a/test/kernels/test_qwen3_5_attention.py +++ b/test/kernels/test_qwen3_5_attention.py @@ -1,14 +1,18 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -"""The full-attention kernels of slot skv2, on one SM100-family GPU (GB300), at Qwen3.5's -TP8 verify shape (7 rows of 4 query heads over one KV head of 256, FP8 pages of 64): +"""The full-attention kernels of slots skv2 and mka, on one SM100-family GPU (GB300), at +Qwen3.5's TP8 verify shape (7 rows of 4 query heads over one KV head of 256, FP8 pages of +64): - SKV2 against an FP64 reference of the masked attention: at most twice the error of SGLang's attention (TRT-LLM MHA's decode kernel) on the same FP8 inputs, at KV lengths from one page to past 128K; -- SKV2 replayed in a CUDA graph on new inputs, bit for bit its eager result. +- SKV2 replayed in a CUDA graph on new inputs, bit for bit its eager result; +- the MKA prologue and epilogue twice on the same inputs, and replayed in a CUDA graph on + new ones, bit for bit their eager result (FP8 Q, bf16 gate, the written K / V rows and + the o_proj output). -Its bits against the SKV2 of SGLang branch nvidia/qwen35-agentx-gb300 are checked +Their bits against the kernels of SGLang branch nvidia/qwen35-agentx-gb300 are checked with that branch installed as ``sglang``, not here. """ @@ -143,3 +147,128 @@ def test_skv2_replays_in_a_cuda_graph_bit_for_bit(): want = torch.empty_like(out) plan.run(q, kv, table, lengths, qk_scale=qk_scale, v_scale=v_scale, out=want) assert torch.equal(out.view(torch.int16), want.view(torch.int16)), length + + +HIDDEN, QKV_ROWS, POOL_SLOTS = 4096, (2 * HEADS + 2) * DIM, 1024 + + +def _fp8(shape, g, scale=1.0): + return (torch.randn(shape, device="cuda", generator=g) * scale).to( + torch.float8_e4m3fn + ) + + +def _mka_operands(g): + """The TP8 layer's static operands: packed FP8 qkv (+ gate) and o_proj weights and + their static scales, the q / k norm weights, an fp32 M-RoPE cos / sin cache, its axis + map and FP8 KV scales.""" + from sgl_nvidia_mega.qwen3_5.sm103.attention import mka + + def scale(v): + return torch.full((1,), v, dtype=torch.float32, device="cuda") + + angles = torch.rand(4096, 32, device="cuda", generator=g) * 6.3 + return dict( + w_qkv=mka.pack_qkv(_fp8((QKV_ROWS, HIDDEN), g), HEADS), + w_o=mka.pack_o(_fp8((HIDDEN, HEADS * DIM), g)), + qkv_sx=scale(0.02), + qkv_sw=scale(0.003), + o_sx=scale(0.05), + o_sw=scale(0.002), + qnw=(torch.randn(DIM, device="cuda", generator=g) * 0.1).to(torch.bfloat16), + knw=(torch.randn(DIM, device="cuda", generator=g) * 0.1).to(torch.bfloat16), + cos_sin=torch.cat([angles.cos(), angles.sin()], dim=1).contiguous(), + axis=torch.randint(0, 3, (32,), device="cuda", generator=g), + k_scale=scale(0.8), + v_scale=scale(1.3), + ) + + +class _MkaStep: + """One step's buffers, refilled in place: the normed rows, positions, cache slots and + the attention output the epilogue reads; the FP8 pool and every output.""" + + def __init__(self, m, g): + self.x = torch.empty(m, HIDDEN, dtype=torch.bfloat16, device="cuda") + self.pos = torch.empty(3, m, dtype=torch.int64, device="cuda") + self.loc = torch.empty(m, dtype=torch.int64, device="cuda") + self.attn = torch.empty(m, HEADS * DIM, dtype=torch.bfloat16, device="cuda") + self.k = _fp8((POOL_SLOTS, 1, DIM), g) + self.v = _fp8((POOL_SLOTS, 1, DIM), g) + self.q = torch.empty(m, HEADS * DIM, dtype=torch.float8_e4m3fn, device="cuda") + self.gate = torch.empty(m, HEADS * DIM, dtype=torch.bfloat16, device="cuda") + self.out = torch.empty(m, HIDDEN, dtype=torch.bfloat16, device="cuda") + self.fill(g) + + def fill(self, g): + m = self.x.shape[0] + self.x.copy_(torch.randn(self.x.shape, device="cuda", generator=g) * 3) + self.pos.copy_( + torch.randint(0, 4096, self.pos.shape, device="cuda", generator=g) + ) + self.loc.copy_(torch.randperm(POOL_SLOTS, device="cuda", generator=g)[:m]) + self.attn.copy_(torch.randn(self.attn.shape, device="cuda", generator=g)) + + def run(self, ops): + from sgl_nvidia_mega.qwen3_5.sm103.attention import mka + + mka.prologue( + x=self.x, + w_packed=ops["w_qkv"], + sx=ops["qkv_sx"], + sw=ops["qkv_sw"], + q_norm_w=ops["qnw"], + k_norm_w=ops["knw"], + cos_sin=ops["cos_sin"], + positions=self.pos, + axis_map=ops["axis"], + cache_loc=self.loc, + k_cache=self.k, + v_cache=self.v, + k_scale=ops["k_scale"], + v_scale=ops["v_scale"], + q_out=self.q, + gate_out=self.gate, + nq=HEADS, + eps=1e-6, + ) + mka.epilogue( + attn=self.attn, + gate=self.gate, + w_packed=ops["w_o"], + sx=ops["o_sx"], + sw=ops["o_sw"], + out=self.out, + nq=HEADS, + ) + + def bits(self): + torch.cuda.synchronize() + rows = (self.k[self.loc], self.v[self.loc]) + return [ + t.clone().view(torch.uint8) for t in (self.q, self.gate, *rows, self.out) + ] + + +def _same(a, b): + return all(torch.equal(x, y) for x, y in zip(a, b)) + + +def test_mka_replays_in_a_cuda_graph_bit_for_bit(): + g = torch.Generator(device="cuda").manual_seed(5) + ops = _mka_operands(g) + step = _MkaStep(7, g) + step.run(ops) + first = step.bits() + step.run(ops) + assert _same(first, step.bits()) # deterministic + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + step.run(ops) + for _ in range(3): + step.fill(g) + graph.replay() + replayed = step.bits() + step.run(ops) + assert _same(replayed, step.bits()) + assert torch.isfinite(step.out.float()).all() diff --git a/test/unit/test_qwen3_5_spec.py b/test/unit/test_qwen3_5_spec.py index 5eb21780..3852a9b6 100644 --- a/test/unit/test_qwen3_5_spec.py +++ b/test/unit/test_qwen3_5_spec.py @@ -193,8 +193,9 @@ def test_a_numeric_slot_turns_the_claim_into_its_tolerances(): **spec.NUMERIC_TOLERANCES["gdn_push"], **spec.NUMERIC_TOLERANCES["gdn_front"], } - attention = spec.numerics(frozenset({"skv2"})) + attention = spec.numerics(frozenset({"skv2", "mka"})) assert attention.kind is NumericsKind.TOLERANCE assert dict(attention.tolerances) == { **spec.NUMERIC_TOLERANCES["skv2"], + **spec.NUMERIC_TOLERANCES["mka"], } diff --git a/test/unit/test_qwen3_5_steps.py b/test/unit/test_qwen3_5_steps.py index 4b6529ab..a2bb4dd6 100644 --- a/test/unit/test_qwen3_5_steps.py +++ b/test/unit/test_qwen3_5_steps.py @@ -1687,6 +1687,244 @@ def test_the_executor_offers_the_verify_split_kv_variants_with_slot_skv2(): assert slot.graph_variants == {"target_verify": route.VERIFY} +# -------------------------------------------------------------------- slot mka + + +def _fp8_linear(log, name): + """A static per-tensor FP8 ModelOpt linear (its method matched by name, as the branch does).""" + method = type("ModelOptFp8LinearMethod", (), {"use_marlin": False})() + linear = SimpleNamespace( + quant_method=method, + weight=torch.zeros(8, 4, dtype=torch.float8_e4m3fn), + input_scale=torch.ones(1), + weight_scale=torch.ones(1), + ) + log.known.update( + {f"{name}_sx": linear.input_scale, f"{name}_sw": linear.weight_scale} + ) + return linear + + +def _mka_layer(log, attn, **kw): + """A Qwen3.5 TP8 full-attention layer as slot mka reads it.""" + from sglang.srt.layers.rotary_embedding.mrope import MRotaryEmbedding + + rope = MRotaryEmbedding.__new__(MRotaryEmbedding) + torch.nn.Module.__init__(rope) + rope.__dict__.update( + rotary_dim=64, + cos_sin_cache=torch.zeros(16, 64), + axis_map=torch.zeros(32, dtype=torch.int64), + ) + layer = SimpleNamespace( + attn=attn, + attn_output_gate=True, + num_heads=4, + num_kv_heads=1, + head_dim=256, + hidden_size=4096, + rotary_emb=rope, + qkv_proj=_fp8_linear(log, "qkv"), + o_proj=_fp8_linear(log, "o"), + q_norm=SimpleNamespace(weight=torch.zeros(256), variance_epsilon=1e-6), + k_norm=SimpleNamespace(weight=torch.zeros(256)), + ) + layer.__dict__.update(kw) + log.known.update( + cos_sin=rope.cos_sin_cache, + axis=rope.axis_map, + qnw=layer.q_norm.weight, + knw=layer.k_norm.weight, + ) + return layer + + +@pytest.mark.parametrize("skv2", [False, True]) +def test_the_mka_slot_runs_the_prologue_the_attention_and_the_epilogue(skv2): + from sgl_nvidia_mega.qwen3_5.attention import mka, split_kv + from sgl_nvidia_mega.qwen3_5.sm103.attention import mka as kernels + + log = _log() + attn, st = _skv2_setup(log, k_scale=torch.full((1,), 0.75)) + log.known["k_scale"] = attn.k_scale + layer = _mka_layer(log, attn) + x = torch.zeros(7, 4096, dtype=torch.bfloat16) + positions = torch.zeros(3, 7, dtype=torch.int64) + st.positions, st.slots = positions, plan.Slots(skv2=skv2, mka=True) + log.known.update(x=x, positions=positions, w_qkv="w_qkv", w_o="w_o") + mka.WEIGHTS[7] = ("w_qkv", "w_o") + attn._fn = lambda *a, **k: log.op("attention")(*a, **k) or torch.zeros(7, 4, 256) + + def prologue(**kw): + log.known.update(q=kw["q_out"], gate=kw["gate_out"]) + log.append(("prologue", tuple(sorted((k, log.tag(v)) for k, v in kw.items())))) + log.known["eps"] = kw["eps"] + + def epilogue(**kw): + log.append(("epilogue", tuple(sorted((k, log.tag(v)) for k, v in kw.items())))) + + def core(attn_, q, st_): + log.append(("skv2_core", log.tag(q))) + return torch.zeros(7, 1024, dtype=torch.bfloat16) + + @contextlib.contextmanager + def symmetric(group, disabled=False): + log.append(("symmetric", group, disabled)) + yield + + with ( + patch.object(kernels, "prologue", prologue), + patch.object(kernels, "epilogue", epilogue), + patch.object(split_kv, "core", core), + patch( + "sglang.srt.distributed.device_communicators.pynccl_allocator.use_symmetric_memory", + symmetric, + ), + patch("sglang.srt.layers.dp_attention.is_allocation_symmetric", lambda: True), + patch( + "sglang.srt.runtime_context.get_parallel", + lambda: SimpleNamespace(tp_group="tp"), + ), + ): + out = attention.step(layer, x, st) + del mka.WEIGHTS[7] + assert [e[0] for e in log] == [ + "prologue", + "skv2_core" if skv2 else "attention", + "symmetric", + "epilogue", + ] + assert dict(log[0][1]) == { + "x": "x(7, 4096)", + "w_packed": "w_qkv", + "sx": "qkv_sx(1,)", + "sw": "qkv_sw(1,)", + "q_norm_w": "qnw(256,)", + "k_norm_w": "knw(256,)", + "cos_sin": "cos_sin(16, 64)", + "positions": "positions(3, 7)", + "axis_map": "axis(32,)", + # this layer's pool rows of the forward's view, at the step's slots + "cache_loc": "loc(7,)", + "k_cache": "k7(192, 1, 256)", + "v_cache": "v7(192, 1, 256)", + "k_scale": "k_scale(1,)", + "v_scale": "v_scale(1,)", + "q_out": "q(7, 1024)", + "gate_out": "gate(7, 1024)", + "nq": 4, + "eps": "?", + } + assert log.known["eps"] == 1e-6 # the q norm's epsilon + if skv2: + assert log[1] == ("skv2_core", "q(7, 1024)") + else: + # SGLang's attention on the rows the prologue wrote: no K / V to write + assert log[1][2:] == (("q(7, 1024)", None, None, "fb"), ()) + assert log[2] == ("symmetric", "tp", False) + kw = dict(log[3][1]) + assert (kw["gate"], kw["w_packed"], kw["sx"], kw["sw"]) == ( + "gate(7, 1024)", + "w_o", + "o_sx(1,)", + "o_sw(1,)", + ) + assert kw["attn"] == "?(7, 1024)" and kw["nq"] == 4 and kw["site"] is None + assert out.shape == (7, 4096) and out.dtype == torch.bfloat16 + + +@pytest.mark.parametrize( + "kw, want", + [ + (dict(), ""), + (dict(attn_output_gate=False), "no attention output gate"), + (dict(num_heads=2), "2 query / 1 KV heads a rank"), + (dict(num_kv_heads=2), "4 query / 2 KV heads a rank"), + (dict(head_dim=128), "head_dim 128, hidden 4096"), + (dict(rotary_emb=SimpleNamespace(rotary_dim=64)), "no M-RoPE axis map"), + (dict(rotary_dim=128), "rotary_dim 128 with a torch.float32 cos/sin cache"), + (dict(cos_sin_dtype=torch.bfloat16), "rotary_dim 64 with a torch.bfloat16"), + (dict(qkv_marlin=True), "not a static per-tensor FP8 linear"), + (dict(o_scales=2), "not a static per-tensor FP8 linear"), + ], +) +def test_the_mka_slot_names_what_a_layer_lacks(kw, want): + from sgl_nvidia_mega.qwen3_5.attention import mka + + log = _log() + special = { + k: kw.pop(k) + for k in list(kw) + if k in ("rotary_dim", "cos_sin_dtype", "qkv_marlin", "o_scales") + } + layer = _mka_layer(log, None, **kw) + if "rotary_dim" in special: + layer.rotary_emb.rotary_dim = special["rotary_dim"] + if "cos_sin_dtype" in special: + layer.rotary_emb.cos_sin_cache = torch.zeros( + 16, 64, dtype=special["cos_sin_dtype"] + ) + if "qkv_marlin" in special: + layer.qkv_proj.quant_method.use_marlin = True + if "o_scales" in special: + layer.o_proj.weight_scale = torch.ones(special["o_scales"]) + got = mka.unsupported(layer) + assert (want in got) if want else got == "" + + +def test_the_mka_slot_is_built_with_its_weights_packed_for_every_attention_layer( + monkeypatch, +): + from sgl_nvidia_mega.qwen3_5.attention import mka + + def layer(kind): + made = kind.__new__(kind) + torch.nn.Module.__init__(made) + return made + + layers = [ + layer(qwen3_5.Qwen3_5LinearDecoderLayer), + layer(qwen3_5.Qwen3_5AttentionDecoderLayer), + layer(qwen3_5.Qwen3_5AttentionDecoderLayer), + ] + model = SimpleNamespace( + is_mrope_enabled=True, + model=SimpleNamespace(start_layer=0, end_layer=3, layers=layers), + ) + packed, why = [], {} + monkeypatch.setattr(runtime, "SLOTS", frozenset({"mka"})) + monkeypatch.setattr(mka, "unsupported", lambda l: why.get(id(l), "")) + monkeypatch.setattr(mka, "pack", packed.append) + ctx = SimpleNamespace(tokens_per_request=7) + assert attention.build(model, ctx) == attention.Bound(mka=True) + assert packed == layers[1:] + # a layer the slot cannot serve leaves the slot unbuilt, packs nothing more + why[id(layers[2])] = "no attention output gate" + assert attention.build(model, ctx) == attention.Bound( + reason="layer 2: no attention output gate" + ) + assert len(packed) == 2 + model.is_mrope_enabled = False + assert attention.build(model, ctx).reason == ( + "layer 1: no M-RoPE positions (MKA reads them)" + ) + + +def test_the_mka_slot_names_the_pools_it_cannot_write(): + from sgl_nvidia_mega.qwen3_5.attention import mka + + def kv(*bufs): + return SimpleNamespace(k=bufs, v=bufs) + + assert ( + mka.pool_reasons(kv(torch.zeros(64, 1, 256, dtype=torch.float8_e4m3fn))) == [] + ) + assert mka.pool_reasons(kv(torch.zeros(64, 2, 256, dtype=torch.float8_e4m3fn))) == [ + "a torch.float8_e4m3fn KV pool buffer of strides (512, 256, 1)" + ] + assert "torch.float32" in mka.pool_reasons(kv(torch.zeros(64, 1, 256)))[0] + + class _Exit: def __init__(self, log, defer): self.log, self.defer_moe_finalize = log, defer From 5508ef9e756677e9c18f4b37e2ef66cec1a4ce39 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:44:00 -0700 Subject: [PATCH 07/27] [Perf] [GB300] qwen3_5: Slot mka_push: the attention epilogue pushes its rows into the hop Where the MoE input's all-reduce runs the hop, MKA's epilogue pushes its output fragments into the hop's mailbox and that all-reduce only consumes them. Slot mka_push (with mka and hop); the pushed values are mka's own, so it adds no tolerance. --- sgl_nvidia_mega/qwen3_5/attention/__init__.py | 5 +- sgl_nvidia_mega/qwen3_5/attention/mka.py | 25 +++++++- sgl_nvidia_mega/qwen3_5/plan.py | 7 ++- sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 7 +++ test/unit/test_qwen3_5_spec.py | 2 +- test/unit/test_qwen3_5_steps.py | 59 +++++++++++++++++++ 6 files changed, 97 insertions(+), 8 deletions(-) diff --git a/sgl_nvidia_mega/qwen3_5/attention/__init__.py b/sgl_nvidia_mega/qwen3_5/attention/__init__.py index 74a491c6..52b66d69 100644 --- a/sgl_nvidia_mega/qwen3_5/attention/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/attention/__init__.py @@ -47,7 +47,7 @@ "srt/models/qwen3_5.py:Qwen3_5AttentionDecoderLayer.self_attention": "db0f4d98e0fb1abf", } -ATTENTION_SLOTS = ("skv2", "mka") +ATTENTION_SLOTS = ("skv2", "mka", "mka_push") class Bound(msgspec.Struct, frozen=True): @@ -55,6 +55,7 @@ class Bound(msgspec.Struct, frozen=True): skv2: bool = False # slot skv2 serves every full-attention layer mka: bool = False + mka_push: bool = False reason: str = "" # why a requested slot is not built @@ -89,7 +90,7 @@ def build(model, ctx) -> Bound: if "mka" in SLOTS: for _, layer in layers: mka.pack(layer) - return Bound(skv2="skv2" in SLOTS, mka="mka" in SLOTS) + return Bound(skv2="skv2" in SLOTS, mka="mka" in SLOTS, mka_push="mka_push" in SLOTS) def pool_reasons(view, binding, skv2: bool = True, mka: bool = False) -> list[str]: diff --git a/sgl_nvidia_mega/qwen3_5/attention/mka.py b/sgl_nvidia_mega/qwen3_5/attention/mka.py index 458d074a..c4a22067 100644 --- a/sgl_nvidia_mega/qwen3_5/attention/mka.py +++ b/sgl_nvidia_mega/qwen3_5/attention/mka.py @@ -11,7 +11,10 @@ step's FP8 K / V rows into the forward view's pool; the attention reads them (slot ``skv2``'s core when it runs, else SGLang's attention with no K / V to write); the epilogue gates, quantizes and runs ``o_proj`` into the TP-partial output, in SGLang's -symmetric pool as ``RowParallelLinear``'s output. The weights are packed at build. +symmetric pool as ``RowParallelLinear``'s output. The weights are packed at build. With +slot ``mka_push`` the epilogue also pushes its rows into the decode hop's mailbox where +the MoE input's all-reduce will run the hop for them, and that all-reduce only consumes +(the branch's MKA_PUSH). """ from __future__ import annotations @@ -175,7 +178,7 @@ def step(layer, x, st): get_parallel().tp_group, disabled=not is_allocation_symmetric() ): out = torch.empty((m, layer.hidden_size), dtype=torch.bfloat16, device=x.device) - # slot mka_push hands the hop's site here and marks the rows pushed + hop = _push_hop(layer, st, m) if st.slots.mka_push else None kernels.epilogue( attn=o, gate=gate, @@ -184,6 +187,22 @@ def step(layer, x, st): sw=layer.o_proj.weight_scale, out=out, nq=layer.num_heads, - site=None, + site=None if hop is None else hop.site, ) + if hop is not None: + hop.mark_pushed(out) return out + + +def _push_hop(layer, st, m: int): + """Slot mka_push: the hop the epilogue pushes its rows into, where the MoE input's + all-reduce runs the hop for them (it then only consumes); else None.""" + from sgl_nvidia_mega.qwen3_5.boundary import hop as boundary_hop + + hop = st.bound["boundary"].hop + if not boundary_hop.takes_moe_input(hop, layer, st, m): + return None + if ("push", m) not in _ENGAGED: + _ENGAGED.add(("push", m)) + logger.info("NVIDIA megakernel: Qwen3.5 slot mka_push runs at %d rows", m) + return hop diff --git a/sgl_nvidia_mega/qwen3_5/plan.py b/sgl_nvidia_mega/qwen3_5/plan.py index c7cfe5c0..c10d2c9f 100644 --- a/sgl_nvidia_mega/qwen3_5/plan.py +++ b/sgl_nvidia_mega/qwen3_5/plan.py @@ -13,8 +13,8 @@ request of 7 rows, ``route_shared`` (numeric: the shared expert's gate/up in route's launch) up to 8 rows. ``skv2`` runs the full attention's core through the split-KV attention of SGLang branch -nvidia/qwen35-agentx-gb300, and ``mka`` the fused attention prologue and epilogue -around it. +nvidia/qwen35-agentx-gb300, ``mka`` the fused attention prologue and epilogue around it, +and ``mka_push`` (with ``hop``) has the epilogue push its rows into the hop. """ from __future__ import annotations @@ -46,6 +46,7 @@ class Slots(msgspec.Struct, frozen=True): route_shared: bool = False skv2: bool = False # the full attention's core through SKV2 (attention/split_kv.py) mka: bool = False # the fused attention prologue and epilogue (attention/mka.py) + mka_push: bool = False # the epilogue pushes its rows into the hop class Facts(msgspec.Struct, frozen=True): @@ -64,6 +65,7 @@ class Facts(msgspec.Struct, frozen=True): ) skv2: bool = False # SKV2 for every full-attention layer mka: bool = False # the fused prologue and epilogue for every full-attention layer + mka_push: bool = False # with the epilogue's push into the hop def slots(rows: int, facts: Facts) -> Slots: @@ -83,6 +85,7 @@ def slots(rows: int, facts: Facts) -> Slots: route_shared=facts.route and facts.route_shared and rows <= ROUTE_SHARED_ROWS, skv2=facts.skv2, mka=facts.mka, + mka_push=facts.mka and facts.mka_push and facts.hop, ) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index 299f101f..369e0de5 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -25,6 +25,7 @@ "route_shared", "skv2", "mka", + "mka_push", ) SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" @@ -73,6 +74,11 @@ def slot_reasons() -> list[str]: reasons.append( f"{SLOTS_ENV}: slot 'gdn_pdl' launches slot 'gdn_verify''s verify" ) + if "mka_push" in SLOTS and not {"mka", "hop"} <= SLOTS: + reasons.append( + f"{SLOTS_ENV}: slot 'mka_push' pushes slot 'mka''s epilogue into slot " + "'hop''s mailbox" + ) return reasons @@ -132,6 +138,7 @@ def build(model, ctx): route_shared=bound["moe"].shared, skv2=bound["attention"].skv2, mka=bound["attention"].mka, + mka_push=bound["attention"].mka_push, ) slots = slot_plan.plan(pctx.widths, facts) built = msgspec.structs.asdict(facts) diff --git a/test/unit/test_qwen3_5_spec.py b/test/unit/test_qwen3_5_spec.py index 3852a9b6..b5abc9f5 100644 --- a/test/unit/test_qwen3_5_spec.py +++ b/test/unit/test_qwen3_5_spec.py @@ -193,7 +193,7 @@ def test_a_numeric_slot_turns_the_claim_into_its_tolerances(): **spec.NUMERIC_TOLERANCES["gdn_push"], **spec.NUMERIC_TOLERANCES["gdn_front"], } - attention = spec.numerics(frozenset({"skv2", "mka"})) + attention = spec.numerics(frozenset({"skv2", "mka", "mka_push", "hop"})) assert attention.kind is NumericsKind.TOLERANCE assert dict(attention.tolerances) == { **spec.NUMERIC_TOLERANCES["skv2"], diff --git a/test/unit/test_qwen3_5_steps.py b/test/unit/test_qwen3_5_steps.py index a2bb4dd6..4f70ea05 100644 --- a/test/unit/test_qwen3_5_steps.py +++ b/test/unit/test_qwen3_5_steps.py @@ -1833,6 +1833,65 @@ def symmetric(group, disabled=False): assert out.shape == (7, 4096) and out.dtype == torch.bfloat16 +@pytest.mark.parametrize("consumes", [False, True]) +def test_the_mka_push_slot_pushes_where_the_moe_input_only_consumes(consumes): + from sgl_nvidia_mega.qwen3_5.attention import mka + from sgl_nvidia_mega.qwen3_5.boundary import hop as boundary_hop + from sgl_nvidia_mega.qwen3_5.sm103.attention import mka as kernels + + log = _log() + attn, st = _skv2_setup(log, k_scale=torch.full((1,), 0.75)) + layer = _mka_layer(log, attn) + x = torch.zeros(7, 4096, dtype=torch.bfloat16) + st.positions = torch.zeros(3, 7, dtype=torch.int64) + st.slots = plan.Slots(skv2=True, mka=True, mka_push=True, hop=True) + hop = SimpleNamespace( + site="site", mark_pushed=lambda t: log.append(("mark_pushed", log.tag(t))) + ) + st.bound = {"boundary": SimpleNamespace(hop=hop)} + mka.WEIGHTS[7] = ("w_qkv", "w_o") + + def epilogue(**kw): + log.known["out"] = kw["out"] + log.append(("epilogue", kw["site"])) + + def takes(hop_, layer_, st_, rows): + log.append(("takes_moe_input", hop_ is hop, layer_ is layer, rows)) + return consumes + + @contextlib.contextmanager + def symmetric(group, disabled=False): + yield + + with ( + patch.object(kernels, "prologue", lambda **kw: None), + patch.object(kernels, "epilogue", epilogue), + patch( + "sgl_nvidia_mega.qwen3_5.attention.split_kv.core", + lambda *a: torch.zeros(7, 1024, dtype=torch.bfloat16), + ), + patch.object(boundary_hop, "takes_moe_input", takes), + patch( + "sglang.srt.distributed.device_communicators.pynccl_allocator.use_symmetric_memory", + symmetric, + ), + patch("sglang.srt.layers.dp_attention.is_allocation_symmetric", lambda: True), + patch( + "sglang.srt.runtime_context.get_parallel", + lambda: SimpleNamespace(tp_group="tp"), + ), + ): + out = attention.step(layer, x, st) + del mka.WEIGHTS[7] + assert log[0] == ("takes_moe_input", True, True, 7) + if consumes: + # the epilogue pushes through the hop's site, and its all-reduce only consumes + assert log[1:] == [("epilogue", "site"), ("mark_pushed", "out(7, 4096)")] + else: + assert log[1:] == [("epilogue", None)] + assert out is log.known["out"] + + @pytest.mark.parametrize( "kw, want", [ From 258cbe28991d49f774d684ff7d19952bc18391eb Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:45:00 -0700 Subject: [PATCH 08/27] [Feat] qwen3_5: Entries draft_loop and draft_extend: the MTP draft's EAGLE entries on SGLang's own ops The draft spec offers draft_loop (with begin_draft_round) and draft_extend for one request when they are requested in SGLANG_NVIDIA_MEGAKERNEL_SLOTS. Each runs inside SGLang's draft graphs over SGLang's loaded draft modules with SGLang's own ops (bitwise), the base the draft slots replace. --- sgl_nvidia_mega/qwen3_5/draft.py | 67 ------- sgl_nvidia_mega/qwen3_5/draft/__init__.py | 168 +++++++++++++++++ sgl_nvidia_mega/qwen3_5/draft/attention.py | 196 ++++++++++++++++++++ sgl_nvidia_mega/qwen3_5/draft/extend.py | 73 ++++++++ sgl_nvidia_mega/qwen3_5/draft/layer.py | 92 +++++++++ sgl_nvidia_mega/qwen3_5/draft/loop.py | 86 +++++++++ sgl_nvidia_mega/qwen3_5/draft/parts.py | 90 +++++++++ sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 8 +- test/unit/test_qwen3_5_runtime.py | 145 ++++++++++++++- 9 files changed, 854 insertions(+), 71 deletions(-) delete mode 100644 sgl_nvidia_mega/qwen3_5/draft.py create mode 100644 sgl_nvidia_mega/qwen3_5/draft/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/draft/attention.py create mode 100644 sgl_nvidia_mega/qwen3_5/draft/extend.py create mode 100644 sgl_nvidia_mega/qwen3_5/draft/layer.py create mode 100644 sgl_nvidia_mega/qwen3_5/draft/loop.py create mode 100644 sgl_nvidia_mega/qwen3_5/draft/parts.py diff --git a/sgl_nvidia_mega/qwen3_5/draft.py b/sgl_nvidia_mega/qwen3_5/draft.py deleted file mode 100644 index 95c6a7ce..00000000 --- a/sgl_nvidia_mega/qwen3_5/draft.py +++ /dev/null @@ -1,67 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -"""Qwen3.5's MTP draft on SM103 (GB300): the megakernel draft spec. - -The draft's forward stays SGLang's (the built object covers no step width), and the -built object offers no EAGLE entry yet (``draft_loop``, ``draft_extend``, -``begin_draft_round``): every draft phase runs SGLang's own path. -""" - -from __future__ import annotations - -from sglang.srt.megakernels.spec import Feature, MegakernelSpec, Requirements - -from sgl_nvidia_mega.qwen3_5 import spec as target - -# The SGLang names this module reads (test/unit/test_contract.py pins them) -SGLANG_READS = { - "srt/megakernels/spec.py": ( - "Feature", - "SPECULATIVE_DECODING", - "MegakernelSpec", - "Requirements", - ), -} - -ARCHITECTURE = "Qwen3_5ForCausalLMMTP" - - -class MtpDraft: - """The draft spec's built object: SGLang's forward on every step.""" - - widths: tuple[int, ...] = () - draft_sampler = None # SGLang's draft proposal - - def __init__(self, lm): - self.lm = lm - - def summary(self) -> str: - return "Qwen3.5 MTP draft: SGLang's forward (no draft phase ported yet)" - - def plan(self, input_ids, forward_batch, *args, **kwargs): - return None - - def forward(self, step, original, *args, **kwargs): - raise AssertionError("the MTP draft covers no step width") - - -def _build(model, hf_config, ctx) -> MtpDraft: - return MtpDraft(model) - - -SPEC = MegakernelSpec( - name="Qwen3.5 MTP draft", - # the target's devices, TP size, dtype and decode graphs: the draft shares its GPUs - requires=Requirements( - isas=target.REQUIREMENTS.isas, - tp_sizes=target.REQUIREMENTS.tp_sizes, - dtypes=target.REQUIREMENTS.dtypes, - decode_graph_backends=target.REQUIREMENTS.decode_graph_backends, - ), - features=frozenset({Feature.SPECULATIVE_DECODING}), - numerics=target.NUMERICS, - # the draft's forward stays SGLang's: no width is built - max_rows=0, - build=_build, - api_version=target.API_VERSION, -) diff --git a/sgl_nvidia_mega/qwen3_5/draft/__init__.py b/sgl_nvidia_mega/qwen3_5/draft/__init__.py new file mode 100644 index 00000000..fdb2820c --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/draft/__init__.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen3.5's MTP draft on SM103 (GB300): the megakernel draft spec and its built +object. + +The draft's forward stays SGLang's (the built object covers no step width). Requested +in ``SGLANG_NVIDIA_MEGAKERNEL_SLOTS``, the built object offers SGLang's EAGLE draft +entries for one request (bs 1), which run inside SGLang's draft graphs: + +- ``draft_loop`` (with ``begin_draft_round``): the whole draft loop of a served draft + graph (``loop``); +- ``draft_extend``: the model step and top-1 of a served draft-extend graph + (``extend``). + +Inside them every op is SGLang's own. +""" + +from __future__ import annotations + +import logging +from typing import Callable, Optional + +from sglang.srt.megakernels.spec import Feature, MegakernelSpec, Requirements + +from sgl_nvidia_mega.qwen3_5 import spec as target + +logger = logging.getLogger(__name__) + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/megakernels/spec.py": ( + "Feature", + "SPECULATIVE_DECODING", + "MegakernelSpec", + "Requirements", + '"draft_loop"', + '"draft_extend"', + '"begin_draft_round"', + "binding", + "deadline_s", + ), + # where SGLang calls the entries (their keyword arguments are pinned by the GPU tests) + "srt/speculative/eagle_megakernel.py": ( + "_draft_loop", + "_draft_extend", + "_begin_draft_round", + "bind_draft_graphs", + "bind_extend_graphs", + "begin_draft_round", + ), +} + +ARCHITECTURE = "Qwen3_5ForCausalLMMTP" +# the one request a draft entry serves: 2.2's draft kernels are single-row +SERVED_BS = 1 + + +class MtpDraft: + """The draft spec's built object: SGLang's forward on every step, and the EAGLE + draft entries requested.""" + + widths: tuple[int, ...] = () + draft_sampler = None # SGLang's draft proposal + + def __init__( + self, + lm, + *, + binding, + deadline_s: float, + slots: frozenset[str], + context_len=None, + ): + self.lm = lm + # the draft's ModelConfig.context_len (trtllm-gen's max_seq_len) + self.context_len = context_len + self.binding = binding # SGLang's EagleBinding (BuildContext.binding) + self.deadline_s = deadline_s + self.slots = slots + self.parts = None # draft.parts.Parts, built by the first factory call + self._loop = None + # the entries offered (spec.ENTRIES); None leaves SGLang's own code + self.draft_loop: Optional[Callable] = ( + self._draft_loop if "draft_loop" in slots else None + ) + self.begin_draft_round: Optional[Callable] = ( + self._begin_draft_round if "draft_loop" in slots else None + ) + self.draft_extend: Optional[Callable] = ( + self._draft_extend if "draft_extend" in slots else None + ) + + def summary(self) -> str: + entries = [ + name + for name in ("draft_loop", "draft_extend") + if getattr(self, name) is not None + ] + return f"Qwen3.5 MTP draft: entries {entries or 'none'}" + + def plan(self, input_ids, forward_batch, *args, **kwargs): + return None + + def forward(self, step, original, *args, **kwargs): + raise AssertionError("the MTP draft covers no step width") + + def _parts(self): + """What the entries share, built once the draft's weights are final: the + factories run before SGLang captures, after the EAGLE worker gave the draft the + target's embedding and LM head.""" + if self.parts is None: + from sgl_nvidia_mega.qwen3_5.draft.parts import Parts + + self.parts = Parts.build(self) + return self.parts + + def _draft_loop(self, *, num_steps: int, max_bs: int, draft_tokens): + from sgl_nvidia_mega.qwen3_5.draft.loop import DraftLoop + + self._loop = DraftLoop(self._parts(), num_steps=num_steps) + return self._loop.step_for + + def _begin_draft_round(self, io) -> None: + if self._loop is not None: + self._loop.begin_round(io) + + def _draft_extend( + self, *, tokens_per_request: int, max_bs: int, topk_index, topk_p, hidden + ): + from sgl_nvidia_mega.qwen3_5.draft.extend import DraftExtend + + extend = DraftExtend(self._parts(), tokens_per_request=tokens_per_request) + return extend.step_for + + +def _requested() -> frozenset: + """The draft's entries in SGLANG_NVIDIA_MEGAKERNEL_SLOTS.""" + from sgl_nvidia_mega.qwen3_5 import runtime + + return runtime.SLOTS & set(runtime.DRAFT_ENTRIES) + + +def _build(model, hf_config, ctx) -> MtpDraft: + return MtpDraft( + model, + binding=ctx.binding, + deadline_s=ctx.deadline_s, + slots=frozenset(_requested()), + context_len=getattr(hf_config, "context_len", None), + ) + + +SPEC = MegakernelSpec( + name="Qwen3.5 MTP draft", + # the target's devices, TP size, dtype and decode graphs: the draft shares its GPUs + requires=Requirements( + isas=target.REQUIREMENTS.isas, + tp_sizes=target.REQUIREMENTS.tp_sizes, + dtypes=target.REQUIREMENTS.dtypes, + decode_graph_backends=target.REQUIREMENTS.decode_graph_backends, + ), + features=frozenset({Feature.SPECULATIVE_DECODING}), + numerics=target.NUMERICS, + # the draft's forward stays SGLang's: no width is built + max_rows=0, + build=_build, + api_version=target.API_VERSION, +) diff --git a/sgl_nvidia_mega/qwen3_5/draft/attention.py b/sgl_nvidia_mega/qwen3_5/draft/attention.py new file mode 100644 index 00000000..985f9730 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/draft/attention.py @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/srt/models/qwen3_5.py (Qwen3_5AttentionDecoderLayer.self_attention) and +# python/sglang/srt/layers/attention/trtllm_mha_backend.py (TRTLLMHAAttnBackend +# forward_decode / forward_extend on the fused FP8 KV path, _run_fixed_q_len_decode). +"""The MTP draft layer's gated full attention inside a draft entry, which gets the +step's attention metadata and the draft's KV pool from SGLang's binding +(``AttnStepMetadata``, ``KvPoolBuffers``) instead of a forward batch. + +Stock: SGLang's ops in the order the TRT-LLM MHA backend runs them on an FP8 KV cache: +the qkv projection with the fused q / k Gemma RMSNorm + RoPE + gate split, the fused +FP8 quantization of q / k / v with the paged KV write, trtllm-gen's decode with +``q_len_per_req`` rows a request (1 for a draft step, the verify width for the draft +extend), the sigmoid output gate and ``o_proj``. +""" + +from __future__ import annotations + +import torch + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen3_5.py": ( + "forward_prepare_cuda_fused", + "attn_output_gate", + "o_proj", + ), + "srt/layers/radix_attention.py": ( + "tp_q_head_num", + "tp_k_head_num", + "tp_v_head_num", + "head_dim", + "scaling", + "sliding_window_size", + "k_scale", + "v_scale", + ), + "kernels/ops/kvcache/fused_fp8_qkv_kv_cache.py": ("fused_fp8_qkv_kv_cache",), + "kernels/ops/elementwise/elementwise.py": ( + "fused_sigmoid_mul", + "fused_sigmoid_mul(inplace)", + ), + "kernels/ops/attention/utils.py": ("canonicalize_stride",), + "srt/mem_cache/layout/paged_view.py": ("paged_kv_view",), + # TRTLLMHAAttnBackend: its shared workspace, its counters and max_context_len, and + # _run_fixed_q_len_decode's trtllm-gen call + "srt/layers/attention/trtllm_mha_backend.py": ( + "DEFAULT_WORKSPACE_SIZE_MB", + '"trtllm_mha_zero_workspace"', + "max_context_len", + "_multi_ctas_kv_counter_buffer", + "_run_fixed_q_len_decode", + "flashinfer", + ), + "srt/layers/attention/trtllm_mla_backend.py": ( + "make_persistent_multi_ctas_kv_counter_buffer", + ), + "srt/environ.py": ("envs", "SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR"), + "srt/runtime_context.py": ("get_buffer",), + # ModelConfig._derive_context_length writes the backend's max_context_len into the + # HF config the draft's build gets + "srt/configs/model_config.py": ("context_len",), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + "srt/models/qwen3_5.py:Qwen3_5AttentionDecoderLayer.self_attention": "db0f4d98e0fb1abf", +} + + +def _context_len(draft) -> int: + """The max_seq_len SGLang's TRT-LLM MHA backend gives trtllm-gen: the draft's + ModelConfig.context_len, which it writes into the HF config the build gets.""" + if draft.context_len is None: + raise RuntimeError( + "NVIDIA megakernel: the draft's HF config has no context_len" + ) + return int(draft.context_len) + + +class Attention: + """The draft layer's attention buffers: trtllm-gen's zeroed workspace (SGLang's + shared one) and multi-CTA counters, and the FP8 KV scales.""" + + def __init__(self, parts): + from sglang.srt.layers.attention.trtllm_mha_backend import ( + DEFAULT_WORKSPACE_SIZE_MB, + ) + from sglang.srt.layers.attention.trtllm_mla_backend import ( + make_persistent_multi_ctas_kv_counter_buffer, + ) + from sglang.srt.runtime_context import get_buffer + + layer, device = parts.layer, parts.device + attn = layer.attn + self.layer = layer + self.heads = attn.tp_q_head_num + self.head_dim = attn.head_dim + self.page_size = parts.binding.page_size + self.max_context_len = _context_len(parts.draft) + self.workspace = get_buffer( + "trtllm_mha_zero_workspace", + lambda: torch.zeros( + DEFAULT_WORKSPACE_SIZE_MB * 1024 * 1024, + dtype=torch.uint8, + device=device, + ), + ) + # the draft's rows a call at most: the verify width of the draft extend + width = int(parts.binding.tokens_per_request) + self.counters = make_persistent_multi_ctas_kv_counter_buffer( + device, + num_q_heads=parts.lm.config.num_attention_heads, + max_batch_size=width, + ) + ones = torch.ones(1, dtype=torch.float32, device=device) + self.k_scale = attn.k_scale if attn.k_scale is not None else ones + self.v_scale = attn.v_scale if attn.v_scale is not None else ones + + def __call__(self, x, positions, cache_loc, meta, kv, *, rows_per_request: int): + """The TP-partial attention output [rows, hidden] of the normed input ``x`` + [rows, hidden]; writes this step's KV rows at ``cache_loc``.""" + from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul + + layer = self.layer + q, k, v, gate = layer.forward_prepare_cuda_fused( + positions=positions, hidden_states=x + ) + o = self.core(q, k, v, cache_loc, meta, kv, rows_per_request) + if layer.attn_output_gate: + o = fused_sigmoid_mul(o, gate, inplace=True) + out, _ = layer.o_proj(o) + return out + + def core(self, q, k, v, cache_loc, meta, kv, rows_per_request: int): + """FP8 q / k / v with the KV write, then the decode: [rows, heads * head_dim].""" + import flashinfer + from sglang.kernels.ops.kvcache.fused_fp8_qkv_kv_cache import ( + fused_fp8_qkv_kv_cache, + ) + from sglang.srt.environ import envs + + k_cache, v_cache = kv.k[0], kv.v[0] + q8 = fused_fp8_qkv_kv_cache( + q=q, + k=k, + v=v, + k_cache=k_cache, + v_cache=v_cache, + cache_loc=cache_loc, + k_scale=self.k_scale, + v_scale=self.v_scale, + ) + q8 = q8.reshape(-1, self.heads, self.head_dim) + attn = self.layer.attn + bmm1_scale = 1.0 * kv.k_scale[0] * attn.scaling + bmm2_scale = kv.v_scale[0] + kwargs = {} + if rows_per_request != 1: + kwargs["q_len_per_req"] = rows_per_request + o = flashinfer.decode.trtllm_batch_decode_with_kv_cache( + query=q8, + kv_cache=self._paged(k_cache, v_cache), + workspace_buffer=self.workspace, + block_tables=meta.page_table, + seq_lens=meta.cache_seqlens, + max_seq_len=self.max_context_len, + bmm1_scale=bmm1_scale, + bmm2_scale=bmm2_scale, + window_left=attn.sliding_window_size, + sinks=None, + skip_softmax_threshold_scale_factor=envs.SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR.get(), + out=None, + out_dtype=torch.bfloat16, + kv_cache_sf=None, + multi_ctas_kv_counter_buffer=self.counters, + **kwargs, + ) + return o.view(-1, self.heads * self.head_dim) + + def _paged(self, k_cache, v_cache): + """The pool buffers as trtllm-gen's HND pages, as the backend views them.""" + from sglang.kernels.ops.attention.utils import canonicalize_stride + from sglang.srt.mem_cache.layout.paged_view import paged_kv_view + + attn = self.layer.attn + k = paged_kv_view(k_cache, self.page_size, attn.tp_k_head_num, self.head_dim) + v = paged_kv_view(v_cache, self.page_size, attn.tp_v_head_num, self.head_dim) + k, v = k.permute(0, 2, 1, 3), v.permute(0, 2, 1, 3) + if attn.tp_k_head_num == 1: + k = canonicalize_stride(k) + if attn.tp_v_head_num == 1: + v = canonicalize_stride(v) + return k, v diff --git a/sgl_nvidia_mega/qwen3_5/draft/extend.py b/sgl_nvidia_mega/qwen3_5/draft/extend.py new file mode 100644 index 00000000..255a967e --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/draft/extend.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py (the captured +# draft extend with the row selection before the LM head) and +# python/sglang/srt/speculative/eagle_worker_v2.py (EagleDraftWorker +# ._draft_extend_for_decode: the selected rows' top-1). +"""Entry ``draft_extend``: a served draft-extend graph's model step and top-1 for one +request. + +SGLang calls the factory once before it captures its draft-extend graphs; the step it +returns for bs 1 runs inside the capture in place of the model's forward. It runs the +model step on the verify's rows (writing their draft KV), and writes the selected +row's top-1 (``torch.argmax``'s choice), 1.0 and final-norm hidden state into the +SGLang-owned outputs. + +Stock: SGLang's model step and top-1 (``layer``). +""" + +from __future__ import annotations + +import torch + +from sgl_nvidia_mega.qwen3_5.draft import SERVED_BS, layer + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/megakernels/eagle_binding.py": ( + "EagleDraftExtendIO", + "num_requests", + "tokens_per_request", + "input_ids", + "positions", + "hidden_states", + "out_cache_loc", + "select_index", + "attn", + "kv", + "topk_index_out", + "topk_p_out", + "hidden_out", + ), +} + + +class DraftExtend: + """The draft_extend factory's result: bs -> the step.""" + + def __init__(self, parts, *, tokens_per_request: int): + self.parts = parts + self.width = tokens_per_request + + def step_for(self, bs: int): + return self.step if bs == SERVED_BS else None + + @torch.no_grad() + def step(self, io) -> None: + parts = self.parts + parts.note_engaged("draft_extend", io.num_requests) + x = layer.fc_input(parts, io.input_ids, io.hidden_states) + hidden = layer.layer( + parts, + x, + io.positions, + io.out_cache_loc, + io.attn, + io.kv, + rows_per_request=io.tokens_per_request, + ) + selected = hidden.index_select(0, io.select_index) + io.topk_index_out.copy_(layer.top1(parts, selected).view(-1, 1)) + io.topk_p_out.fill_(1.0) + io.hidden_out.copy_(selected) diff --git a/sgl_nvidia_mega/qwen3_5/draft/layer.py b/sgl_nvidia_mega/qwen3_5/draft/layer.py new file mode 100644 index 00000000..e0aa058a --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/draft/layer.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/srt/models/qwen3_5_mtp.py (Qwen3_5ForCausalLMMTP.forward), +# python/sglang/srt/models/qwen3_5.py (Qwen3_5ForCausalLM.forward and +# Qwen3_5AttentionDecoderLayer.forward) and +# python/sglang/srt/layers/logits_processor.py (LogitsProcessor._get_logits). +"""The MTP draft's model step inside a draft entry, over SGLang's loaded draft modules: + +- ``fc_input``: the embedding, the two pre-fc Gemma RMSNorms and ``fc``; +- ``layer``: the decoder layer (input norm, attention, the attention output's + all-reduce + residual add + post-attention norm, the MoE, the MoE output's + all-reduce + residual add + final norm), returning the final-norm rows; +- ``top1``: the LM head on this rank's vocabulary shard, the all-gather of the logits + and their arg-max (``torch.argmax``'s choice, as SGLang's top-1). + +Stock: the ops SGLang's draft runs for these rows, in its order. +""" + +from __future__ import annotations + +import torch + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen3_5_mtp.py": ( + "embed_tokens", + "pre_fc_norm_embedding", + "pre_fc_norm_hidden", + "fc", + "logits_processor", + "lm_head", + ), + "srt/layers/logits_processor.py": ( + "_compute_lm_head", + "_compute_lm_head(embedding_bias)", + "vocab_size", + ), + "srt/distributed/communication_op.py": ( + "tensor_model_parallel_all_reduce", + "tensor_model_parallel_all_gather", + ), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang) +SGLANG_MIRRORS = { + "srt/models/qwen3_5_mtp.py:Qwen3_5ForCausalLMMTP.forward": "6d038e59d55980a6", + "srt/models/qwen3_5.py:Qwen3_5AttentionDecoderLayer.forward": "6473f673a1a0796b", +} + + +def fc_input(parts, token_ids, hidden): + """The fc output [rows, hidden] of the tokens ``token_ids`` [rows] and the hidden + rows ``hidden`` [rows, hidden] they follow.""" + lm = parts.lm + embeds = lm.pre_fc_norm_embedding(parts.model.embed_tokens(token_ids)) + hidden = lm.pre_fc_norm_hidden(hidden) + return lm.fc(torch.cat([embeds, hidden], dim=-1)) + + +def layer(parts, x, positions, cache_loc, meta, kv, *, rows_per_request: int): + """The decoder layer on the fc output ``x`` [rows, hidden]: the final-norm rows.""" + from sglang.srt.distributed import tensor_model_parallel_all_reduce + + lay = parts.layer + residual = x + attn_out = parts.attention( + lay.input_layernorm(x), + positions, + cache_loc, + meta, + kv, + rows_per_request=rows_per_request, + ) + moe_in, residual = lay.post_attention_layernorm( + tensor_model_parallel_all_reduce(attn_out), residual + ) + moe_out = lay.mlp(moe_in) + out, _ = parts.model.norm(tensor_model_parallel_all_reduce(moe_out), residual) + return out + + +def top1(parts, hidden): + """SGLang's draft top-1 [rows] int64 of the final-norm rows ``hidden``.""" + from sglang.srt.distributed import tensor_model_parallel_all_gather + + lm = parts.lm + logits = lm.logits_processor._compute_lm_head(hidden, lm.lm_head, None) + logits = tensor_model_parallel_all_gather(logits) + logits = logits[:, : lm.logits_processor.vocab_size] + return torch.argmax(logits, dim=-1) diff --git a/sgl_nvidia_mega/qwen3_5/draft/loop.py b/sgl_nvidia_mega/qwen3_5/draft/loop.py new file mode 100644 index 00000000..487a9013 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/draft/loop.py @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/srt/speculative/eagle_worker_v2.py (EagleDraftWorker.draft_forward, the +# topk-1 chain with draft_topk1_postprocess). +"""Entry ``draft_loop``: a served draft graph's whole draft loop for one request. + +SGLang calls the factory once before it captures its draft graphs; the step it returns +for bs 1 runs inside the capture in place of ``draft_forward``. Draft step i (of +``num_steps - 1``) runs the model step at ``positions + i`` on the KV slot +``out_cache_loc[i]`` and writes its top-1 into ``draft_tokens_out[:, i + 1]`` +(column 0 is the round's first token); the step advances the positions in place and +restores them before it returns, as the binding asks. + +Stock: SGLang's model step and top-1 (``layer``). +""" + +from __future__ import annotations + +import torch + +from sgl_nvidia_mega.qwen3_5.draft import SERVED_BS, layer + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/megakernels/eagle_binding.py": ( + "EagleDraftLoopIO", + "num_requests", + "num_steps", + "topk_index", + "hidden_states", + "positions", + "mrope_positions", + "out_cache_loc", + "steps", + "kv", + "draft_tokens_out", + ), +} + + +class DraftLoop: + """The draft_loop factory's result: bs -> the step.""" + + def __init__(self, parts, *, num_steps: int): + self.parts = parts + self.num_steps = num_steps + + def step_for(self, bs: int): + return self.step if bs == SERVED_BS else None + + def begin_round(self, io) -> None: + """On the host before a served draft graph replays: nothing to prepare while + every op is SGLang's.""" + + @torch.no_grad() + def step(self, io) -> None: + parts = self.parts + parts.note_engaged("draft_loop", io.num_requests) + steps = io.num_steps - 1 + tokens = io.draft_tokens_out + tokens[:, :1].copy_(io.topk_index) + positions, mrope = io.positions, io.mrope_positions + token, hidden = io.topk_index.view(-1), io.hidden_states + for i in range(steps): + hidden = self._model_step(io, i, layer.fc_input(parts, token, hidden)) + token = layer.top1(parts, hidden) + tokens[:, i + 1].copy_(token) + positions.add_(1) + if mrope is not None: + mrope.add_(1) + positions.sub_(steps) + if mrope is not None: + mrope.sub_(steps) + + def _model_step(self, io, i: int, x): + """Draft step i's decoder layer on its fc output ``x``: the final-norm rows.""" + return layer.layer( + self.parts, + x, + io.positions, + io.out_cache_loc[i :: io.num_steps], + io.steps[i], + io.kv, + rows_per_request=1, + ) diff --git a/sgl_nvidia_mega/qwen3_5/draft/parts.py b/sgl_nvidia_mega/qwen3_5/draft/parts.py new file mode 100644 index 00000000..718582ec --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/draft/parts.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""What the draft entries share, built once by the first entry factory SGLang calls +(before it captures the draft graphs, every rank in the same order): the draft layer, +the attention's buffers, and each requested draft slot's build.""" + +from __future__ import annotations + +import logging + +import torch + +logger = logging.getLogger(__name__) + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen3_5_mtp.py": ( + "model", + "fc", + "pre_fc_norm_embedding", + "pre_fc_norm_hidden", + "lm_head", + "logits_processor", + ), + "srt/models/qwen3_5.py": ( + "layers", + "start_layer", + "end_layer", + "norm", + "embed_tokens", + "input_layernorm", + "post_attention_layernorm", + "mlp", + "attn", + ), + "srt/megakernels/eagle_binding.py": ( + "tp_rank", + "tp_size", + "tp_device_group", + "tp_cpu_group", + "page_size", + ), +} + + +class Parts: + """The draft model's modules the entries run, and the slot builds.""" + + def __init__(self, draft, layer): + self.draft = draft + self.lm = draft.lm # Qwen3_5ForCausalLMMTP + self.model = draft.lm.model # its one-layer Qwen3_5ForCausalLM + self.layer = layer # the MTP decoder layer (full attention + MoE) + self.binding = draft.binding + self.slots = draft.slots + self.device = torch.device("cuda", torch.cuda.current_device()) + self.attention = None # draft.attention.Attention + # (entry, bs) pairs whose capture logged the engagement line + self.engaged: set = set() + + @classmethod + def build(cls, draft) -> Parts: + from sgl_nvidia_mega.qwen3_5.draft.attention import Attention + + m = draft.lm.model + layers = [m.layers[i] for i in range(m.start_layer, m.end_layer)] + if len(layers) != 1: + raise RuntimeError( + f"NVIDIA megakernel: the MTP draft has {len(layers)} layers, not 1" + ) + parts = cls(draft, layers[0]) + parts.attention = Attention(parts) + slots = parts.slots + logger.info( + "NVIDIA megakernel: Qwen3.5 MTP draft parts built (slots %s)", + sorted(slots), + ) + return parts + + def note_engaged(self, entry: str, bs: int) -> None: + """Log once per (entry, bs) captured: the engagement line every rank prints.""" + if torch.cuda.is_current_stream_capturing() and (entry, bs) not in self.engaged: + self.engaged.add((entry, bs)) + logger.info( + "NVIDIA megakernel: Qwen3.5 MTP draft %s captured for batch size %d " + "(slots %s)", + entry, + bs, + sorted(self.slots), + ) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index 369e0de5..b419a56a 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -27,6 +27,9 @@ "mka", "mka_push", ) +# The MTP draft's EAGLE entries (qwen3_5/draft), requested in the same variable; the +# draft spec's build reads them, the target's ignores them +DRAFT_ENTRIES: tuple[str, ...] = ("draft_loop", "draft_extend") SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" @@ -44,7 +47,7 @@ def _requested_slots() -> frozenset: def slot_reasons() -> list[str]: """Why the requested slots cannot be served ([] when they can).""" - known = (*SHIPPED_SLOTS, *OPT_IN_SLOTS) + known = (*SHIPPED_SLOTS, *OPT_IN_SLOTS, *DRAFT_ENTRIES) reasons = [ f"{SLOTS_ENV}: unknown slot {name!r} (known: {', '.join(known) or 'none'})" for name in sorted(SLOTS - set(known) - {"verify_sample"}) @@ -143,7 +146,8 @@ def build(model, ctx): slots = slot_plan.plan(pctx.widths, facts) built = msgspec.structs.asdict(facts) # every package's build agrees across ranks, so this raises on every rank alike - missing = sorted(name for name in SLOTS if not built[name]) + target_slots = SLOTS - set(DRAFT_ENTRIES) + missing = sorted(name for name in target_slots if not built[name]) if missing: raise RuntimeError( f"NVIDIA megakernel: requested slots {missing} built no width (MoE: " diff --git a/test/unit/test_qwen3_5_runtime.py b/test/unit/test_qwen3_5_runtime.py index c30ad842..9bd91d2a 100644 --- a/test/unit/test_qwen3_5_runtime.py +++ b/test/unit/test_qwen3_5_runtime.py @@ -66,7 +66,7 @@ def test_the_build_hands_over_the_binding_and_every_entry_stays_sglangs(): # covered from the first step on: the binding comes with the build ids = torch.zeros(7, dtype=torch.int64) assert built.plan(ids, _batch()) == Step(mode="target_verify", rows=7) - target, mtp = built, draft.SPEC.build("lm", None, None) + target, mtp = built, draft.SPEC.build("lm", None, _draft_ctx()) assert [n for n in ENTRIES if getattr(target, n, None) is not None] == [] assert [n for n in ENTRIES if getattr(mtp, n, None) is not None] == [] @@ -91,15 +91,156 @@ def test_the_runtime_gives_the_executor_the_build_contexts_binding(monkeypatch): assert built.binding == "EagleBinding" and built.summary() == "slots none (stock)" +def _draft_ctx(): + return SimpleNamespace( + widths=(), tokens_per_request=7, deadline_s=60.0, binding="EagleBinding" + ) + + def test_the_mtp_draft_covers_no_step(): + from sglang.srt.megakernels.spec import ENTRIES + from sgl_nvidia_mega.qwen3_5 import draft - built = draft.SPEC.build("lm", None, None) + built = draft.SPEC.build("lm", SimpleNamespace(context_len=4096), _draft_ctx()) assert ( draft.SPEC.max_rows == 0 and built.widths == () and built.draft_sampler is None ) assert built.plan(torch.zeros(7, dtype=torch.int64), _batch()) is None assert (draft.ARCHITECTURE, "sgl_nvidia_mega.qwen3_5.draft:SPEC") in plugin.SPECS + # without a request every draft phase stays SGLang's + assert [n for n in ENTRIES if getattr(built, n, None) is not None] == [] + + +def test_the_draft_entries_are_offered_only_on_request(monkeypatch): + from sglang.srt.megakernels.spec import ENTRIES + + from sgl_nvidia_mega.qwen3_5 import draft + + monkeypatch.setattr(runtime, "SLOTS", frozenset({"draft_loop", "draft_extend"})) + assert runtime.slot_reasons() == [] + built = draft.SPEC.build("lm", SimpleNamespace(context_len=4096), _draft_ctx()) + assert [n for n in ENTRIES if getattr(built, n, None) is not None] == [ + "draft_loop", + "draft_extend", + "begin_draft_round", + ] + assert built.context_len == 4096 and built.binding == "EagleBinding" + # the target's build ignores the draft's names + model = SimpleNamespace( + model=SimpleNamespace(start_layer=0, end_layer=0, layers=[]) + ) + target_ctx = SimpleNamespace( + widths=(7,), tokens_per_request=7, deadline_s=60.0, binding="EagleBinding" + ) + assert runtime.build(model, target_ctx).summary() == "slots none (stock)" + + +class _DraftParts: + """Parts with no slot built: the stock model step.""" + + def __init__(self, slots=frozenset()): + self.slots, self.engaged = slots, [] + + def note_engaged(self, entry, bs): + self.engaged.append((entry, bs)) + + +def _stub_model_step(monkeypatch, calls): + """The draft's model step as a log: fc_input -> x, layer -> hidden, top1 -> token.""" + from sgl_nvidia_mega.qwen3_5.draft import layer + + def fc_input(parts, token, hidden): + calls.append(("fc", token.tolist(), hidden.tolist())) + return hidden + 1 + + def run_layer(parts, x, positions, cache_loc, meta, kv, *, rows_per_request): + calls.append(("layer", positions.tolist(), cache_loc.tolist(), meta, kv)) + return x * 10 + + def top1(parts, hidden): + calls.append(("top1", hidden.tolist())) + return hidden[:, 0].to(torch.int64) % 1000 + + monkeypatch.setattr(layer, "fc_input", fc_input) + monkeypatch.setattr(layer, "layer", run_layer) + monkeypatch.setattr(layer, "top1", top1) + + +def test_the_draft_loop_writes_each_steps_token_and_restores_the_positions( + monkeypatch, +): + from sgl_nvidia_mega.qwen3_5.draft import SERVED_BS + from sgl_nvidia_mega.qwen3_5.draft.loop import DraftLoop + + calls = [] + _stub_model_step(monkeypatch, calls) + parts = _DraftParts() + loop = DraftLoop(parts, num_steps=4) + assert loop.step_for(SERVED_BS) == loop.step and loop.step_for(2) is None + io = SimpleNamespace( + num_requests=1, + num_steps=4, + topk_index=torch.tensor([[7]]), + hidden_states=torch.tensor([[1.0, 2.0]]), + positions=torch.tensor([100]), + mrope_positions=torch.tensor([[100], [100], [100]]), + # request-major: step i writes slot 40 + i + out_cache_loc=torch.tensor([40, 41, 42, 43]), + steps=("m0", "m1", "m2"), + kv="kv", + draft_tokens_out=torch.full((1, 4), -1, dtype=torch.int64), + ) + loop.step(io) + assert parts.engaged == [("draft_loop", 1)] + # three draft steps at positions 100..102 on slots 40..42 with step i's metadata + layers = [c for c in calls if c[0] == "layer"] + assert [c[1:] for c in layers] == [ + ([100], [40], "m0", "kv"), + ([101], [41], "m1", "kv"), + ([102], [42], "m2", "kv"), + ] + # step i + 1 embeds step i's token after its final-norm hidden state + fcs = [c for c in calls if c[0] == "fc"] + assert [c[1] for c in fcs] == [[7], [20], [210]] + assert io.draft_tokens_out.tolist() == [[7, 20, 210, 110]] + assert io.positions.tolist() == [100] and io.mrope_positions.tolist() == [ + [100], + [100], + [100], + ] + + +def test_the_draft_extend_writes_the_selected_rows_outputs(monkeypatch): + from sgl_nvidia_mega.qwen3_5.draft.extend import DraftExtend + + calls = [] + _stub_model_step(monkeypatch, calls) + parts = _DraftParts() + extend = DraftExtend(parts, tokens_per_request=3) + assert extend.step_for(1) == extend.step and extend.step_for(4) is None + io = SimpleNamespace( + num_requests=1, + tokens_per_request=3, + input_ids=torch.tensor([5, 6, 7]), + positions=torch.tensor([10, 11, 12]), + hidden_states=torch.tensor([[1.0], [2.0], [3.0]]), + out_cache_loc=torch.tensor([20, 21, 22]), + select_index=torch.tensor([1]), + attn="meta", + kv="kv", + topk_index_out=torch.zeros(1, 1, dtype=torch.int64), + topk_p_out=torch.zeros(1, 1), + hidden_out=torch.zeros(1, 1), + ) + extend.step(io) + assert parts.engaged == [("draft_extend", 1)] + assert [c[1:] for c in calls if c[0] == "layer"] == [ + ([10, 11, 12], [20, 21, 22], "meta", "kv") + ] + # the selected row 1: (2 + 1) * 10 + assert io.hidden_out.tolist() == [[30.0]] + assert io.topk_index_out.tolist() == [[30]] and io.topk_p_out.tolist() == [[1.0]] # ------------------------------------------------------------------- the parity check From 7a839b93a9892ef20b32d69e9003748830c28677 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:46:00 -0700 Subject: [PATCH 09/27] [Perf] [GB300] qwen3_5: Draft slot draft_moe: the MTP draft's MoE in one launch (numeric) Inside draft_loop, a one-row draft step's MoE (router GEMV, softmax top-10, routed experts and gated shared expert) runs as one launch of one CTA per SM on the bf16 weights SGLang loaded. Draft slot draft_moe; numeric: the dot products sum in another fp32 order, so the draft's claim turns into TOLERANCE. --- sgl_nvidia_mega/qwen3_5/draft/__init__.py | 32 +- sgl_nvidia_mega/qwen3_5/draft/layer.py | 6 +- sgl_nvidia_mega/qwen3_5/draft/moe.py | 190 ++++ sgl_nvidia_mega/qwen3_5/draft/parts.py | 18 +- sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 17 +- .../qwen3_5/sm103/draft/__init__.py | 3 + sgl_nvidia_mega/qwen3_5/sm103/draft/moe.py | 286 ++++++ .../qwen3_5/sm103/draft/src/draft_moe.cu | 843 ++++++++++++++++++ test/kernels/test_qwen3_5_draft_moe.py | 233 +++++ test/unit/test_qwen3_5_runtime.py | 11 + 10 files changed, 1627 insertions(+), 12 deletions(-) create mode 100644 sgl_nvidia_mega/qwen3_5/draft/moe.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/moe.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_moe.cu create mode 100644 test/kernels/test_qwen3_5_draft_moe.py diff --git a/sgl_nvidia_mega/qwen3_5/draft/__init__.py b/sgl_nvidia_mega/qwen3_5/draft/__init__.py index fdb2820c..e55b5043 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/draft/__init__.py @@ -12,7 +12,8 @@ - ``draft_extend``: the model step and top-1 of a served draft-extend graph (``extend``). -Inside them every op is SGLang's own. +Inside them every op is SGLang's own unless a draft slot replaces it: ``draft_moe`` +(the single-launch MoE). """ from __future__ import annotations @@ -20,7 +21,13 @@ import logging from typing import Callable, Optional -from sglang.srt.megakernels.spec import Feature, MegakernelSpec, Requirements +from sglang.srt.megakernels.spec import ( + Feature, + MegakernelSpec, + Numerics, + NumericsKind, + Requirements, +) from sgl_nvidia_mega.qwen3_5 import spec as target @@ -32,6 +39,9 @@ "Feature", "SPECULATIVE_DECODING", "MegakernelSpec", + "Numerics", + "NumericsKind", + "TOLERANCE", "Requirements", '"draft_loop"', '"draft_extend"', @@ -51,6 +61,18 @@ } ARCHITECTURE = "Qwen3_5ForCausalLMMTP" + + +def numerics(slots: frozenset) -> Numerics: + """SGLang's draft bit for bit with no draft slot; with one, 2.2's draft kernels, + which differ from SGLang's ops only in fp32 summation order (q1 split-KV, the + o_proj / fc / MoE GEMVs, the draft head), so a draft token can differ where two + logits are within a bf16 rounding.""" + if not slots & {"draft_moe"}: + return target.NUMERICS + return Numerics(NumericsKind.TOLERANCE, {"draft_logits.rel": 2**-7}) + + # the one request a draft entry serves: 2.2's draft kernels are single-row SERVED_BS = 1 @@ -134,10 +156,10 @@ def _draft_extend( def _requested() -> frozenset: - """The draft's entries in SGLANG_NVIDIA_MEGAKERNEL_SLOTS.""" + """The draft's entries and slots in SGLANG_NVIDIA_MEGAKERNEL_SLOTS.""" from sgl_nvidia_mega.qwen3_5 import runtime - return runtime.SLOTS & set(runtime.DRAFT_ENTRIES) + return runtime.SLOTS & (set(runtime.DRAFT_ENTRIES) | set(runtime.DRAFT_SLOTS)) def _build(model, hf_config, ctx) -> MtpDraft: @@ -160,7 +182,7 @@ def _build(model, hf_config, ctx) -> MtpDraft: decode_graph_backends=target.REQUIREMENTS.decode_graph_backends, ), features=frozenset({Feature.SPECULATIVE_DECODING}), - numerics=target.NUMERICS, + numerics=numerics(_requested()), # the draft's forward stays SGLang's: no width is built max_rows=0, build=_build, diff --git a/sgl_nvidia_mega/qwen3_5/draft/layer.py b/sgl_nvidia_mega/qwen3_5/draft/layer.py index e0aa058a..285cf4a1 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/layer.py +++ b/sgl_nvidia_mega/qwen3_5/draft/layer.py @@ -14,7 +14,8 @@ - ``top1``: the LM head on this rank's vocabulary shard, the all-gather of the logits and their arg-max (``torch.argmax``'s choice, as SGLang's top-1). -Stock: the ops SGLang's draft runs for these rows, in its order. +Stock: the ops SGLang's draft runs for these rows, in its order. Slot ``draft_moe`` +replaces the MoE. """ from __future__ import annotations @@ -76,7 +77,8 @@ def layer(parts, x, positions, cache_loc, meta, kv, *, rows_per_request: int): moe_in, residual = lay.post_attention_layernorm( tensor_model_parallel_all_reduce(attn_out), residual ) - moe_out = lay.mlp(moe_in) + moe = parts.moe if rows_per_request == 1 else None + moe_out = moe(moe_in) if moe is not None else lay.mlp(moe_in) out, _ = parts.model.norm(tensor_model_parallel_all_reduce(moe_out), residual) return out diff --git a/sgl_nvidia_mega/qwen3_5/draft/moe.py b/sgl_nvidia_mega/qwen3_5/draft/moe.py new file mode 100644 index 00000000..2fcd4069 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/draft/moe.py @@ -0,0 +1,190 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0): python/sglang/srt/layers/moe/single_token_draft_moe.py +# (SingleTokenDraftMoe: its eligibility checks and single-launch forward), on branch +# nvidia/qwen35-agentx-gb300. +"""The MTP draft layer's MoE as one launch per token (``sm103/draft/moe.py``): router +GEMV + softmax top-k, the routed experts and the gated shared expert, from the bf16 +weights SGLang loaded for the draft's ``Qwen2MoeSparseMoeBlock``. + +``DraftMoe(x)`` returns this rank's unreduced output (routed and gated shared summed), +whose TP sum the draft boundary completes. Against SGLang's stock draft MoE only the +fp32 dot-product order differs. +""" + +from __future__ import annotations + +import logging +from typing import Optional + +import torch + +from sgl_nvidia_mega.qwen3_5.moe.mkm import shared_topk_slots +from sgl_nvidia_mega.qwen3_5.sm103.draft import moe as kernels + +logger = logging.getLogger(__name__) + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/models/qwen2_moe.py": ( + "gate", + "experts", + "shared_expert", + "shared_expert_gate", + "topk", + "gate_up_proj", + "down_proj", + "layer_id", + ), + "srt/layers/moe/fused_moe_triton/layer.py": ("quant_method",), + "srt/layers/quantization/unquant.py": ( + "UnquantizedFusedMoEMethod", + "use_flashinfer_trtllm_moe", + "w13_swiglu_interleaved", + '"w13_weight"', + '"w2_weight"', + ), + "srt/layers/moe/topk.py": ( + "topk_config", + "top_k", + "enable_waterfill", + "scoring_func", + "renormalize", + "use_grouped_topk", + "correction_bias", + "custom_routing_function", + "routed_scaling_factor", + ), + "srt/runtime_context.py": ("get_parallel", '"moe_ep_size"'), +} + + +def _router_reason(mlp) -> Optional[str]: + cfg = mlp.topk.topk_config + if ( + cfg.scoring_func != "softmax" + or not cfg.renormalize + or cfg.use_grouped_topk + or cfg.correction_bias is not None + or cfg.custom_routing_function is not None + or shared_topk_slots(cfg) != 0 + or cfg.routed_scaling_factor not in (None, 1.0) + or mlp.topk.enable_waterfill + ): + return "router is not plain softmax top-k with renormalization" + w = mlp.gate.weight + if w.dtype != torch.bfloat16 or w.shape[0] % 32 != 0 or not w.is_contiguous(): + return f"router weight {w.dtype} {tuple(w.shape)}" + return None + + +def unsupported_reason(mlp) -> Optional[str]: + """Why the single launch cannot serve this block (None when it can): the checks of + SingleTokenDraftMoe._unsupported_reason and _router_unsupported_reason.""" + from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod + from sglang.srt.runtime_context import get_parallel + + experts, shared = mlp.experts, mlp.shared_expert + if shared is None or mlp.shared_expert_gate is None: + return "no separate shared expert" + # The kernel reads W13 as [gate rows | up rows] in checkpoint order. + method = experts.quant_method + if not isinstance(method, UnquantizedFusedMoEMethod): + return f"expert weights handled by {type(method).__name__}" + if method.use_flashinfer_trtllm_moe or method.w13_swiglu_interleaved: + return "expert weights in a kernel-specific layout" + if get_parallel().moe_ep_size != 1: + return "expert parallelism" + w13, w2 = experts.w13_weight, experts.w2_weight + ws13, ws2 = shared.gate_up_proj.weight, shared.down_proj.weight + hidden, inter = w13.shape[-1], w2.shape[-1] + tensors = (w13, w2, ws13, ws2, mlp.shared_expert_gate.weight) + if any(t.dtype != torch.bfloat16 for t in tensors): + return "non-bf16 operands" + if w13.dim() != 3 or tuple(w13.shape[1:]) != (2 * inter, hidden): + return f"w13 shape {tuple(w13.shape)}" + if tuple(ws13.shape) != (2 * inter, hidden) or tuple(ws2.shape) != (hidden, inter): + return f"shared expert shapes {tuple(ws13.shape)} {tuple(ws2.shape)}" + if hidden % 1024 != 0 or inter % 32 != 0: + return f"hidden {hidden} / intermediate {inter} not tileable" + if not all(t.is_contiguous() for t in tensors): + return "non-contiguous operands" + reason = _router_reason(mlp) + if reason is not None: + return reason + # the single launch's own tiling (static_asserts of src/draft_moe.cu) + num_experts, topk = mlp.gate.weight.shape[0], mlp.topk.topk_config.top_k + per_lane = num_experts // 32 + if inter % 64 != 0 or per_lane & (per_lane - 1) or not 1 <= topk <= kernels.STAGES: + return f"{num_experts} experts / top-{topk} / intermediate {inter} not tileable" + return None + + +class DraftMoe: + """The single-launch draft MoE of one block: its scratch and the per-token launch.""" + + MAX_TOKENS = 2 + + def __init__(self, mlp, device: torch.device): + self.mlp = mlp + num_experts = mlp.gate.weight.shape[0] + topk = mlp.topk.topk_config.top_k + inter = mlp.experts.w2_weight.shape[-1] + self.scratch = kernels.draft_moe_fused_scratch(num_experts, topk, inter, device) + self.topk_weights = torch.empty(1, topk, dtype=torch.float32, device=device) + self.topk_ids = torch.empty(1, topk, dtype=torch.int32, device=device) + + @classmethod + def build(cls, mlp, device_index: int) -> Optional[DraftMoe]: + """The block's single-launch MoE, compiled and checked to fit this device, or + None (logged) when it cannot serve the block.""" + reason = unsupported_reason(mlp) + if reason is None: + w13 = mlp.experts.w13_weight + reason = "; ".join( + kernels.build( + device_index, + hidden=w13.shape[-1], + inter=mlp.experts.w2_weight.shape[-1], + experts=mlp.gate.weight.shape[0], + topk=mlp.topk.topk_config.top_k, + ) + ) + if reason: + logger.warning( + "Single-launch draft MoE off for layer %d (%s); SGLang's MoE runs.", + mlp.layer_id, + reason, + ) + return None + logger.info("Single-launch draft MoE active for layer %d.", mlp.layer_id) + return cls(mlp, torch.device("cuda", device_index)) + + def ready(self, num_tokens: int) -> bool: + """Whether it runs a batch of this many tokens.""" + return 0 < num_tokens <= self.MAX_TOKENS + + def single_launch(self, num_tokens: int) -> bool: + """Whether that batch runs as single launches (always, once ready).""" + return self.ready(num_tokens) + + def __call__(self, hidden_states: torch.Tensor) -> torch.Tensor: + """[m, H] -> this rank's MoE output [m, H] (m <= MAX_TOKENS), unreduced; the + tokens run one launch each, in order.""" + mlp = self.mlp + out = torch.empty_like(hidden_states) + for m in range(hidden_states.shape[0]): + kernels.draft_moe_fused( + hidden_states[m : m + 1], + mlp.gate.weight, + mlp.experts.w13_weight, + mlp.experts.w2_weight, + mlp.shared_expert.gate_up_proj.weight, + mlp.shared_expert.down_proj.weight, + mlp.shared_expert_gate.weight.view(-1), + self.scratch, + self.topk_weights, + self.topk_ids, + out[m : m + 1], + ) + return out diff --git a/sgl_nvidia_mega/qwen3_5/draft/parts.py b/sgl_nvidia_mega/qwen3_5/draft/parts.py index 718582ec..07704e50 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/parts.py +++ b/sgl_nvidia_mega/qwen3_5/draft/parts.py @@ -7,6 +7,7 @@ from __future__ import annotations import logging +from typing import Any, Optional import torch @@ -55,6 +56,7 @@ def __init__(self, draft, layer): self.slots = draft.slots self.device = torch.device("cuda", torch.cuda.current_device()) self.attention = None # draft.attention.Attention + self.moe: Optional[Any] = None # draft.moe.DraftMoe (slot draft_moe) # (entry, bs) pairs whose capture logged the engagement line self.engaged: set = set() @@ -71,12 +73,26 @@ def build(cls, draft) -> Parts: parts = cls(draft, layers[0]) parts.attention = Attention(parts) slots = parts.slots + if "draft_moe" in slots: + parts.moe = parts._build_moe() logger.info( - "NVIDIA megakernel: Qwen3.5 MTP draft parts built (slots %s)", + "NVIDIA megakernel: Qwen3.5 MTP draft parts built (slots %s; MoE %s)", sorted(slots), + "single launch" if parts.moe is not None else "SGLang's", ) return parts + def _build_moe(self): + from sgl_nvidia_mega.qwen3_5.draft.moe import DraftMoe + + moe = DraftMoe.build(self.layer.mlp, self.device.index) + if moe is None: + raise RuntimeError( + "NVIDIA megakernel: slot draft_moe cannot run: " + f"{getattr(DraftMoe, 'last_reason', 'see the log')}" + ) + return moe + def note_engaged(self, entry: str, bs: int) -> None: """Log once per (entry, bs) captured: the engagement line every rank prints.""" if torch.cuda.is_current_stream_capturing() and (entry, bs) not in self.engaged: diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index b419a56a..bd7aaa68 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -27,9 +27,10 @@ "mka", "mka_push", ) -# The MTP draft's EAGLE entries (qwen3_5/draft), requested in the same variable; the -# draft spec's build reads them, the target's ignores them +# The MTP draft's EAGLE entries and kernel groups (qwen3_5/draft), requested in the same +# variable; the draft spec's build reads them, the target's ignores them DRAFT_ENTRIES: tuple[str, ...] = ("draft_loop", "draft_extend") +DRAFT_SLOTS: tuple[str, ...] = ("draft_moe",) SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" @@ -47,7 +48,12 @@ def _requested_slots() -> frozenset: def slot_reasons() -> list[str]: """Why the requested slots cannot be served ([] when they can).""" - known = (*SHIPPED_SLOTS, *OPT_IN_SLOTS, *DRAFT_ENTRIES) + known = ( + *SHIPPED_SLOTS, + *OPT_IN_SLOTS, + *DRAFT_ENTRIES, + *DRAFT_SLOTS, + ) reasons = [ f"{SLOTS_ENV}: unknown slot {name!r} (known: {', '.join(known) or 'none'})" for name in sorted(SLOTS - set(known) - {"verify_sample"}) @@ -77,6 +83,9 @@ def slot_reasons() -> list[str]: reasons.append( f"{SLOTS_ENV}: slot 'gdn_pdl' launches slot 'gdn_verify''s verify" ) + for name in sorted(SLOTS & set(DRAFT_SLOTS)): + if "draft_loop" not in SLOTS: + reasons.append(f"{SLOTS_ENV}: slot {name!r} runs inside entry 'draft_loop'") if "mka_push" in SLOTS and not {"mka", "hop"} <= SLOTS: reasons.append( f"{SLOTS_ENV}: slot 'mka_push' pushes slot 'mka''s epilogue into slot " @@ -146,7 +155,7 @@ def build(model, ctx): slots = slot_plan.plan(pctx.widths, facts) built = msgspec.structs.asdict(facts) # every package's build agrees across ranks, so this raises on every rank alike - target_slots = SLOTS - set(DRAFT_ENTRIES) + target_slots = SLOTS - set(DRAFT_ENTRIES) - set(DRAFT_SLOTS) missing = sorted(name for name in target_slots if not built[name]) if missing: raise RuntimeError( diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py new file mode 100644 index 00000000..df0d5ba8 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The MTP draft's SM103 (GB300) kernels: the single-launch draft MoE (``moe``).""" diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/moe.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/moe.py new file mode 100644 index 00000000..413e4f4e --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/moe.py @@ -0,0 +1,286 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0): python/sglang/kernels/ops/speculative/draft_moe_gemv.py +# (draft_moe_fused_scratch, draft_moe_fused), on branch nvidia/qwen35-agentx-gb300. +"""The single-launch M = 1 bf16 draft MoE (``src/draft_moe.cu``): router GEMV, +softmax top-k with renormalized weights, the routed and gated shared expert, one grid +of one CTA per SM whose CTAs hand over through tagged words in global scratch. + +Its ids, weights and output equal the router + two-pass GEMV path of the same kernels +bit for bit; against SGLang's stock draft MoE only the fp32 dot-product order differs. +""" + +from __future__ import annotations + +import ctypes +from functools import cache +from pathlib import Path +from typing import Optional, Tuple + +import torch + +from sgl_nvidia_mega.common import jit + +SOURCE = Path(__file__).parent / "src" / "draft_moe.cu" +THREADS = 288 # 8 consumer warps and the copy / selection warp +# Ring stages (16 KB each) of the fused kernel; at least topk, since every routed +# down slice holds a stage until the end. +STAGES = 11 +# Copies of every tagged handoff word (kFusedReplicas in src/draft_moe.cu). +REPLICAS = 152 +TRACE_SLOTS = 32 +# Routed pairs a CTA keeps in flight (I = 128: 4, -0.35 us per call on GB300). +WINDOW_SMALL_I = 4 + + +def fused_window(inter: int) -> int: + return WINDOW_SMALL_I if inter <= 128 else STAGES + + +def logit_words(experts: int) -> int: + """E logits and the gate, padded to whole 128-byte lines per replica.""" + return (experts + 1 + 15) // 16 * 16 + + +@cache +def sm_count(device_index: int) -> int: + return torch.cuda.get_device_properties(device_index).multi_processor_count + + +def columns(hidden: int, device_index: int) -> int: + """Output columns per CTA: one CTA per SM.""" + return -(-hidden // sm_count(device_index)) + + +def shared_bytes(hidden: int, inter: int, topk: int, cols: int, stages: int) -> int: + """DraftMoeFused::kSmem: the ring, the shared down slice, x and the act values.""" + slice_alloc = (cols * inter * 2 + 127) // 128 * 128 + return stages * 4 * hidden + slice_alloc + 2 * hidden + 2 * (topk + 1) * inter + + +class Params(ctypes.Structure): + """DraftMoeFusedParams.""" + + _fields_ = [ + (name, ctypes.c_void_p) + for name in ( + "x", + "router_w", + "w13", + "w2", + "w13_shared", + "w2_shared", + "gate_w", + "logit_buf", + "act_buf", + "epoch", + "topk_weights", + "topk_ids", + "out", + "trace", + ) + ] + + +@cache +def _kernel( + device_index: int, + hidden: int, + inter: int, + experts: int, + topk: int, + cols: int, + stages: int, + window: int, + trace: bool, +) -> jit.Kernel: + major, minor = torch.cuda.get_device_capability(device_index) + defines = ( + f"-DQ35_DMOE_H={hidden}", + f"-DQ35_DMOE_I={inter}", + f"-DQ35_DMOE_E={experts}", + f"-DQ35_DMOE_TOPK={topk}", + f"-DQ35_DMOE_COLS={cols}", + f"-DQ35_DMOE_STAGES={stages}", + f"-DQ35_DMOE_WINDOW={window}", + f"-DQ35_DMOE_TRACE={int(trace)}", + ) + cubin = jit.compile_source( + SOURCE.read_text(), + f"q35_draft_moe_h{hidden}_i{inter}_e{experts}_k{topk}_c{cols}_s{stages}_w{window}_t{int(trace)}", + f"sm_{major}{minor}a", + # the flags of SGLang's JIT build of this kernel (draft_moe_gemv.py) + ("-std=c++17", "-default-device", "--use_fast_math", *defines), + ) + return jit.load( + cubin, + "q35_draft_moe_fused", + (THREADS, 1, 1), + shared_bytes(hidden, inter, topk, cols, stages), + ) + + +def _fit_reasons( + kernel: jit.Kernel, *, grid: int, experts: int, inter: int, topk: int, stages: int +) -> list[str]: + """DraftMoeFused::run's launch checks: the whole grid co-resident (the CTAs spin + on each other) and every CTA's static work within the ring.""" + cu = jit.driver() + (per_sm,) = jit.check( + cu.cuOccupancyMaxActiveBlocksPerMultiprocessor( + kernel.function, THREADS, kernel.shared_memory_bytes + ), + "cuOccupancyMaxActiveBlocksPerMultiprocessor", + ) + reasons = [] + device = torch.cuda.current_device() + if per_sm < 1 or grid > per_sm * sm_count(device): + reasons.append( + f"fused draft MoE grid {grid} does not fit: {per_sm} CTA/SM x " + f"{sm_count(device)} SMs" + ) + max_static = ((experts + grid - 1) // grid + 1) // 2 + (inter + grid - 1) // grid + max_pairs = (inter + grid - 1) // grid + (topk * inter + grid - 1) // grid + if max_static > stages or max_pairs > 32 or (experts + grid - 1) // grid > 8: + reasons.append(f"too much work per CTA for grid {grid}") + return reasons + + +def build( + device_index: int, + *, + hidden: int, + inter: int, + experts: int, + topk: int, + stages: int = STAGES, + window: Optional[int] = None, +) -> list[str]: + """Compile the kernel for this shape (no compile inside a step); the reasons it + cannot launch on this device ([] when it can).""" + cols = columns(hidden, device_index) + window = fused_window(inter) if window is None else window + if not 1 <= window <= stages or stages < topk: + return [f"window {window} / stages {stages} for top-{topk}"] + with torch.cuda.device(device_index): + kernel = _kernel( + device_index, hidden, inter, experts, topk, cols, stages, window, False + ) + return _fit_reasons( + kernel, + grid=-(-hidden // cols), + experts=experts, + inter=inter, + topk=topk, + stages=stages, + ) + + +def draft_moe_fused_scratch( + experts: int, topk: int, inter: int, device: torch.device +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Zeroed scratch of draft_moe_fused, reused across calls: tagged logits and + gate, tagged act values (both replicated), and the call epoch.""" + return ( + torch.zeros(REPLICAS * logit_words(experts), dtype=torch.int64, device=device), + torch.zeros(REPLICAS * (topk + 1) * inter, dtype=torch.int32, device=device), + torch.zeros(1, dtype=torch.int32, device=device), + ) + + +def _check(name: str, t: torch.Tensor, shape: tuple, dtype, device, align: int = 0): + if ( + tuple(t.shape) != shape + or t.dtype != dtype + or t.device != device + or not t.is_contiguous() + or (align and t.data_ptr() % align) + ): + raise ValueError( + f"draft_moe_fused: {name} {tuple(t.shape)} {t.dtype} {t.device} " + f"(contiguous {t.is_contiguous()}), expected {shape} {dtype} {device}" + ) + + +def draft_moe_fused( + x: torch.Tensor, + router_w: torch.Tensor, + w13: torch.Tensor, + w2: torch.Tensor, + w13_shared: torch.Tensor, + w2_shared: torch.Tensor, + shared_gate_w: torch.Tensor, + scratch: Tuple[torch.Tensor, torch.Tensor, torch.Tensor], + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + out: torch.Tensor, + trace: Optional[torch.Tensor] = None, + stages: int = STAGES, + window: Optional[int] = None, + hop_site: Optional[torch.Tensor] = None, + hop_token: int = 0, +) -> torch.Tensor: + """Router top-k + the routed and gated shared expert of one token in one launch. + + x [1, H]; router_w [E, H]; w13 [E, 2I, H] (gate rows, then up rows); w2 [E, H, I]; + w13_shared [2I, H]; w2_shared [H, I]; shared_gate_w [H] (all bf16); scratch from + draft_moe_fused_scratch; outputs topk_weights fp32 [1, K], topk_ids int32 [1, K] + (lowest id on ties) and out bf16 [1, H]. trace: int64 [grid * 32] stamps, or None. + The hop push (hop_site) is not part of this port.""" + if hop_site is not None: + raise ValueError( + "draft_moe_fused: the hop push is not ported (TP8 runs it off)" + ) + hidden, inter = x.shape[-1], w2.shape[-1] + experts, topk = router_w.shape[0], topk_ids.shape[-1] + dev = x.device + bf16 = torch.bfloat16 + _check("x", x, (1, hidden), bf16, dev, 16) + _check("router_w", router_w, (experts, hidden), bf16, dev, 16) + _check("w13", w13, (experts, 2 * inter, hidden), bf16, dev, 16) + _check("w2", w2, (experts, hidden, inter), bf16, dev, 16) + _check("w13_shared", w13_shared, (2 * inter, hidden), bf16, dev, 16) + _check("w2_shared", w2_shared, (hidden, inter), bf16, dev, 16) + _check("shared_gate_w", shared_gate_w, (hidden,), bf16, dev, 16) + logit_buf, act_buf, epoch = scratch + _check("logit_buf", logit_buf, (REPLICAS * logit_words(experts),), torch.int64, dev) + _check("act_buf", act_buf, (REPLICAS * (topk + 1) * inter,), torch.int32, dev, 16) + _check("epoch", epoch, (1,), torch.int32, dev) + _check("topk_weights", topk_weights, (1, topk), torch.float32, dev) + _check("topk_ids", topk_ids, (1, topk), torch.int32, dev) + _check("out", out, (1, hidden), bf16, dev) + cols = columns(hidden, dev.index) + grid = -(-hidden // cols) + if trace is not None: + _check("trace", trace, (grid * TRACE_SLOTS,), torch.int64, dev) + window = fused_window(inter) if window is None else window + kernel = _kernel( + dev.index, + hidden, + inter, + experts, + topk, + cols, + stages, + window, + trace is not None, + ) + p = Params( + x=x.data_ptr(), + router_w=router_w.data_ptr(), + w13=w13.data_ptr(), + w2=w2.data_ptr(), + w13_shared=w13_shared.data_ptr(), + w2_shared=w2_shared.data_ptr(), + gate_w=shared_gate_w.data_ptr(), + logit_buf=logit_buf.data_ptr(), + act_buf=act_buf.data_ptr(), + epoch=epoch.data_ptr(), + topk_weights=topk_weights.data_ptr(), + topk_ids=topk_ids.data_ptr(), + out=out.data_ptr(), + # the kernel forms its trace row pointer even when it records nothing + trace=(epoch if trace is None else trace).data_ptr(), + ) + jit.launch(kernel, (grid, 1, 1), [p], torch.cuda.current_stream(dev).cuda_stream) + return out diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_moe.cu b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_moe.cu new file mode 100644 index 00000000..57d18148 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_moe.cu @@ -0,0 +1,843 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Adapted from SGLang (Apache-2.0): python/sglang/kernels/jit/csrc/speculative/draft_moe_fused.cuh, +// the draft_moe helpers of draft_moe_gemv.cuh, and the device helpers of +// python/sglang/kernels/jit/include/sgl_kernel/{utils,vec,type,warp,mbarrier}.cuh it uses +// (branch nvidia/qwen35-agentx-gb300). +// +// Single-launch M = 1 bf16 draft MoE: router logits, softmax top-k, routed and +// shared up pass, act, down pass and output in one grid of one CTA per SM. +// +// Every dot product keeps the lane partition and reduction order of +// draft_router_kernel, draft_moe_up_tma_kernel and draft_moe_down_v3_kernel, +// and every rounding point of that path, so ids, weights and the output are +// bitwise equal to it; only the schedule differs. +// +// Phases hand over through tagged words in global scratch. A word carries its +// value and the call tag (epoch + 1): logits and the shared-expert gate as +// tag << 32 | fp32 bits, act values as (tag & 0xffff) << 16 | bf16 bits. Every +// call rewrites every word, so a reader polls until each word it needs carries +// this call's tag and nothing is ever reset. CTA 0 advances the epoch once it +// has seen every CTA's logits, which is after every CTA has read the old epoch. +// Writers store every word into kFusedReplicas copies and CTA c polls copy +// c % kFusedReplicas, so no cache line is polled by the whole grid. +// +// Built with NVRTC and --use_fast_math (the flags of the SGLang JIT build): the kernel is a +// device function and the extern "C" entry at the end instantiates it from the Q35_DMOE_* +// defines. The hop push (kPush, off at TP8) is not part of this port. + +#include + +// NVRTC has no +typedef int int32_t; +typedef long long int64_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; + +#define SGL_DEVICE __forceinline__ __device__ +#define SGL_DEVICE_HOST __forceinline__ __device__ __host__ + +namespace sglang { + +using fp32_t = float; +using bf16_t = __nv_bfloat16; + +namespace device { + +// SGL_ARCH_BLACKWELL_OR_GREATER (sm_103, CUDA >= 12.9): 256-bit vectors +inline constexpr unsigned long long kMaxVecBytes = 32; +inline constexpr uint32_t kWarpThreads = 32u; + +SGL_DEVICE uint32_t get_lane_id() { + uint32_t lane_id; + asm volatile("mov.u32 %0, %%laneid;" : "=r"(lane_id)); + return lane_id; +} + +template +SGL_DEVICE void PDLWaitPrimary() { + if constexpr (kUsePDL) { + asm volatile("griddepcontrol.wait;" ::: "memory"); + } +} + +template +SGL_DEVICE void PDLTriggerSecondary() { + if constexpr (kUsePDL) { + asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); + } +} + +template +struct alignas(sizeof(T) * N) AlignedStorage { + T data[N]; +}; + +// AlignedVector of 16-bit elements (the only width this kernel uses). +template +struct AlignedVector { + private: + static_assert((N > 0 && (N & (N - 1)) == 0) && sizeof(T) * N <= kMaxVecBytes, "vector size"); + static_assert(sizeof(T) == 2, "16-bit elements"); + using element_t = uint16_t; + using storage_t = AlignedStorage; + + public: + SGL_DEVICE void load(const void* ptr, int64_t offset = 0) { + m_storage = reinterpret_cast(ptr)[offset]; + } + SGL_DEVICE void store(void* ptr, int64_t offset = 0) const { + reinterpret_cast(ptr)[offset] = m_storage; + } + SGL_DEVICE T& operator[](unsigned idx) { + return reinterpret_cast(&m_storage)[idx]; + } + SGL_DEVICE T operator[](unsigned idx) const { + return reinterpret_cast(&m_storage)[idx]; + } + + private: + storage_t m_storage; +}; + +// DTypeTrait::from as sgl_kernel/type.cuh registers it for these two conversions. +template +SGL_DEVICE To cast(const From& value); + +template <> +SGL_DEVICE fp32_t cast(const bf16_t& value) { + return __bfloat162float(value); +} + +template <> +SGL_DEVICE bf16_t cast(const fp32_t& value) { + return __float2bfloat16_rn(value); +} + +namespace warp { + +// sgl_kernel/warp.cuh reduce for float: xor butterfly 16 .. 1. +SGL_DEVICE float reduce_sum(float value) { +#pragma unroll + for (uint32_t mask = kWarpThreads / 2; mask >= 1; mask >>= 1) { + value = static_cast(value + __shfl_xor_sync(0xffffffffu, value, mask, kWarpThreads)); + } + return value; +} + +// sgl_kernel/warp.cuh reduce for float on SGL_CUDA_ARCH 1000-1099: redux.sync. +SGL_DEVICE float reduce_max(float value) { + float result; + asm volatile("redux.sync.max.f32 %0, %1, %2;" : "=f"(result) : "f"(value), "r"(0xffffffffu)); + return result; +} + +} // namespace warp + +namespace ptx { + +template +SGL_DEVICE uint32_t to_shared(T* ptr) { + return static_cast(__cvta_generic_to_shared(ptr)); +} + +SGL_DEVICE void mbar_init(uint64_t* bar, uint32_t count) { + asm volatile("mbarrier.init.shared.b64 [%0], %1;" ::"r"(to_shared(bar)), "r"(count)); +} + +SGL_DEVICE uint64_t mbar_arrive(uint64_t* bar) { + uint64_t state; + asm volatile("mbarrier.arrive.shared.b64 %0, [%1];" : "=l"(state) : "r"(to_shared(bar))); + return state; +} + +SGL_DEVICE void mbar_arrive_expect_tx(uint64_t* bar, uint32_t bytes) { + asm volatile("mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;" ::"r"(to_shared(bar)), "r"(bytes)); +} + +SGL_DEVICE void mbar_wait_parity(uint64_t* bar, uint32_t parity) { + asm volatile( + "{\n\t.reg .pred p;\n\t" + "WAIT_%=: mbarrier.try_wait.parity.shared.b64 p, [%0], %1;\n\t" + "@!p bra WAIT_%=;\n\t}\n" ::"r"(to_shared(bar)), + "r"(parity)); +} + +} // namespace ptx + +} // namespace device + +namespace draft_moe { + +/// \brief Same expression as the SiLU of the act-and-mul kernel. +SGL_DEVICE float silu_f32(float x) { + return x / (1.0f + expf(-x)); +} + +// -inf (NVRTC has no __builtin_huge_valf) +SGL_DEVICE float neg_inf() { + return __uint_as_float(0xff800000u); +} + +SGL_DEVICE float round_bf16(float x) { + return device::cast(device::cast(x)); +} + +// global -> shared bulk copy completed on an smem mbarrier (PTX ISA cp.async.bulk); +// size and both addresses must be 16-byte aligned. +SGL_DEVICE void bulk_g2s(void* smem_dst, const void* gmem_src, uint32_t bytes, uint64_t* bar) { + asm volatile( + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"( + device::ptx::to_shared(smem_dst)), + "l"(gmem_src), + "r"(bytes), + "r"(device::ptx::to_shared(bar)) + : "memory"); +} + +SGL_DEVICE void fence_mbarrier_init() { + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); +} + +} // namespace draft_moe + +namespace draft_moe_fused { + +SGL_DEVICE void st_relaxed(uint64_t* ptr, uint64_t v) { + asm volatile("st.relaxed.gpu.global.b64 [%0], %1;" ::"l"(ptr), "l"(v) : "memory"); +} + +SGL_DEVICE void st_relaxed(uint32_t* ptr, uint32_t v) { + asm volatile("st.relaxed.gpu.global.b32 [%0], %1;" ::"l"(ptr), "r"(v) : "memory"); +} + +SGL_DEVICE uint64_t ld_relaxed(const uint64_t* ptr) { + uint64_t v; + asm volatile("ld.relaxed.gpu.global.b64 %0, [%1];" : "=l"(v) : "l"(ptr) : "memory"); + return v; +} + +// Four tagged 32-bit words; each word is single-copy atomic on its own. +SGL_DEVICE uint4 ld_relaxed_v4(const uint32_t* ptr) { + uint4 v; + asm volatile("ld.relaxed.gpu.global.v4.b32 {%0, %1, %2, %3}, [%4];" + : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) + : "l"(ptr) + : "memory"); + return v; +} + +SGL_DEVICE uint64_t globaltimer() { + uint64_t t; + asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t)); + return t; +} + +// A cross-CTA wait this long means the grid was not co-resident; trap rather +// than wedge the device. +inline constexpr uint64_t kSpinLimitNs = 2'000'000'000ull; + +SGL_DEVICE void check_spin(uint64_t t0) { + if (globaltimer() - t0 > kSpinLimitNs) __trap(); +} + +SGL_DEVICE void split(int total, int parts, int idx, int& begin, int& count) { + const int base = total / parts; + const int rem = total % parts; + begin = idx * base + min(idx, rem); + count = base + (idx < rem ? 1 : 0); +} + +// mbarrier wait that sleeps between tries, so idle warps leave the issue slots to the +// warps of the same SM that are computing. +SGL_DEVICE void mbar_wait_sleep(uint64_t* bar, uint32_t parity) { + uint32_t done; + while (true) { + asm volatile( + "{\n\t.reg .pred p;\n\tmbarrier.try_wait.parity.shared.b64 p, [%1], %2;\n\tselp.b32 %0, 1, 0, p;\n\t}\n" + : "=r"(done) + : "r"(device::ptx::to_shared(bar)), "r"(parity) + : "memory"); + if (done) return; + __nanosleep(64); + } +} + +SGL_DEVICE void named_sync(uint32_t id, uint32_t threads) { + asm volatile("bar.sync %0, %1;" ::"r"(id), "r"(threads) : "memory"); +} + +// Bitonic sort of one lane's keys, descending. +template +SGL_DEVICE void sort_desc(uint32_t (&key)[kN]) { + static_assert((kN & (kN - 1)) == 0); +#pragma unroll + for (int size = 2; size <= kN; size <<= 1) { +#pragma unroll + for (int stride = size / 2; stride > 0; stride >>= 1) { +#pragma unroll + for (int i = 0; i < kN; ++i) { + const int j = i ^ stride; + if (j > i) { + const uint32_t a = key[i], b = key[j]; + const bool desc = (i & size) == 0; + key[i] = desc ? max(a, b) : min(a, b); + key[j] = desc ? min(a, b) : max(a, b); + } + } + } + } +} + +} // namespace draft_moe_fused + +struct DraftMoeFusedParams { + const bf16_t* x; // [H], written by an earlier grid + const bf16_t* __restrict__ router_w; // [E, H] + const bf16_t* __restrict__ w13; // [E, 2I, H] + const bf16_t* __restrict__ w2; // [E, H, I] + const bf16_t* __restrict__ w13_shared; // [2I, H] + const bf16_t* __restrict__ w2_shared; // [H, I] + const bf16_t* __restrict__ gate_w; // [H] + uint64_t* logit_buf; // [kFusedReplicas, kFusedLogitWords(E)] tagged logits, then the gate + uint32_t* act_buf; // [kFusedReplicas, (K + 1) * I] tagged act values + uint32_t* epoch; // [1] completed calls + float* topk_weights; // [K] + int32_t* topk_ids; // [K] + bf16_t* out; // [H] + int64_t* trace; // [grid, kFusedTraceSlots] clock64 stamps, globaltimer at 0 and 9 (kTrace only) +}; + +inline constexpr int kFusedTraceSlots = 32; +inline constexpr int kFusedReplicas = 152; + +// E logits and the gate, padded to whole 128-byte lines per replica. +SGL_DEVICE_HOST constexpr int kFusedLogitWords(int experts) { + return (experts + 1 + 15) / 16 * 16; +} + +/** + * \brief Fused M = 1 draft MoE, one CTA per SM, all CTAs co-resident. + * + * Warps 0-7 consume a shared-memory ring of 16 KB stages (two router rows, or + * one gate/up row pair); warp 8 issues the bulk copies and, once every CTA's + * logits are published, repeats the top-k selection itself. Router rows and + * shared-expert weights are requested before the PDL wait; the routed up rows + * and routed down slices right after the selection. Down slices are the last + * ring items and stay resident until every act value of the call is published. + * + * \tparam kH hidden size + * \tparam kI per-rank intermediate size of one expert + * \tparam kE routed experts + * \tparam kTopK routed experts per token + * \tparam kCols output columns per CTA (grid = ceil(kH / kCols)) + * \tparam kStages ring stages of 16 KB; at least kTopK + * \tparam kWindow routed pairs a CTA keeps in flight (at most kStages) + * \tparam kTrace record per-phase clock64 stamps + */ +template +SGL_DEVICE void draft_moe_fused_kernel(const DraftMoeFusedParams& p) { + using namespace device; + namespace fz = draft_moe_fused; + constexpr int kConsumers = 8; + constexpr int kConsumerThreads = kConsumers * static_cast(kWarpThreads); + constexpr int kSlots = kTopK + 1; + constexpr int kRowBytes = kH * static_cast(sizeof(bf16_t)); + constexpr int kStageBytes = 2 * kRowBytes; + constexpr int kQuarter = kH / 4; + using cvec_t = AlignedVector; + static_assert(kQuarter % (8 * kWarpThreads) == 0, "hidden size must tile 4 warps x 16 bytes"); + constexpr int kChunksPerLane = kQuarter / (8 * kWarpThreads); + constexpr int kVecN = static_cast(kMaxVecBytes / sizeof(bf16_t)); + using vec_t = AlignedVector; + static_assert(kH % (kVecN * kWarpThreads) == 0 && kE % kWarpThreads == 0); + constexpr int kRouterPerLane = kH / kVecN / static_cast(kWarpThreads); + constexpr int kPerLaneE = kE / static_cast(kWarpThreads); + static_assert(kI % 64 == 0, "intermediate size must tile 8 lanes x 16 bytes"); + constexpr int kDownPerLane = kI / 64; + constexpr int kColsCap = kCols; + constexpr int kSliceBytes = kColsCap * kI * static_cast(sizeof(bf16_t)); + constexpr int kSliceAlloc = (kSliceBytes + 127) / 128 * 128; + static_assert(kSliceBytes <= kStageBytes && kStages >= kTopK, "every down slice must own a ring stage"); + constexpr int kMaxPairs = 32; + constexpr int kActWords = kSlots * kI; + constexpr int kLogitWords = kFusedLogitWords(kE); + + extern __shared__ __align__(128) uint8_t smem[]; + uint8_t* const ring = smem; + uint8_t* const s_wsd = smem + kStages * kStageBytes; + bf16_t* const s_x = reinterpret_cast(s_wsd + kSliceAlloc); + bf16_t* const s_act = s_x + kH; + __shared__ uint64_t full_bar[kStages]; + __shared__ uint64_t empty_bar[kStages]; + __shared__ uint64_t wsd_bar; + __shared__ uint64_t sel_bar; + __shared__ float s_part[kMaxPairs][kConsumers]; + __shared__ float s_d[kSlots][kColsCap]; + __shared__ float s_gate_part[4]; + __shared__ float s_v[kE]; + __shared__ float s_w[kTopK]; + __shared__ int32_t s_ids[kTopK]; + __shared__ float s_gate; + __shared__ uint32_t s_tag; + __shared__ uint32_t s_word[kMaxPairs]; + + const int warp = static_cast(threadIdx.x / kWarpThreads); + const int lane = static_cast(get_lane_id()); + const int G = static_cast(gridDim.x); + const int c = static_cast(blockIdx.x); + int row0, nrows, sh0, nsh, rq0, nrq; + fz::split(kE, G, c, row0, nrows); + fz::split(kI, G, c, sh0, nsh); + // Reversed, so the CTAs with an extra router row do not also get an extra routed pair. + fz::split(kTopK * kI, G, G - 1 - c, rq0, nrq); + const int nrouter = (nrows + 1) / 2; + const int nstatic = nrouter + nsh; + const int nup = nstatic + nrq; + const int npairs = nsh + nrq; + int j0, ncols; + j0 = c * kCols; + ncols = min(kCols, kH - j0); + const uint32_t slice_bytes = static_cast(ncols) * kI * sizeof(bf16_t); + const bool gate_cta = c == G - 1; + const uint64_t* const my_logits = p.logit_buf + (c % kFusedReplicas) * kLogitWords; + const uint32_t* const my_acts = p.act_buf + (c % kFusedReplicas) * kActWords; + int64_t* const trace = p.trace + c * kFusedTraceSlots; + // After a CTA barrier the clock is read behind a dependent shared load: the barrier does + // not hold back the clock read itself. + auto stamp_after_sync = [&](int slot) { + if constexpr (kTrace) { + if (threadIdx.x == 0 && *reinterpret_cast(&s_tag) != 0xffffffffu) trace[slot] = clock64(); + } + }; + + if (threadIdx.x == 0) { + if constexpr (kTrace) { + trace[0] = static_cast(fz::globaltimer()); + trace[23] = clock64(); + } +#pragma unroll + for (int s = 0; s < kStages; ++s) { + ptx::mbar_init(&full_bar[s], 1); + ptx::mbar_init(&empty_bar[s], kConsumers); + } + ptx::mbar_init(&wsd_bar, 1); + ptx::mbar_init(&sel_bar, 1); + draft_moe::fence_mbarrier_init(); + } + __syncthreads(); + PDLTriggerSecondary(); + + // Ring item n: two router rows, then shared-expert pairs, routed pairs, routed down slices. + auto issue = [&](int n) { + const int stage = n % kStages; + uint8_t* dst = ring + stage * kStageBytes; + uint64_t* bar = &full_bar[stage]; + if (n < nrouter) { + const int r = row0 + 2 * n; + const uint32_t bytes = static_cast(min(2, row0 + nrows - r)) * kRowBytes; + ptx::mbar_arrive_expect_tx(bar, bytes); + draft_moe::bulk_g2s(dst, p.router_w + static_cast(r) * kH, bytes, bar); + return; + } + if (n >= nup) { + const int64_t e = s_ids[n - nup]; + ptx::mbar_arrive_expect_tx(bar, slice_bytes); + draft_moe::bulk_g2s(dst, p.w2 + (e * kH + j0) * kI, slice_bytes, bar); + return; + } + const bf16_t* src; + if (n < nstatic) { + src = p.w13_shared + static_cast(sh0 + n - nrouter) * kH; + } else { + const int rq = rq0 + n - nstatic; + src = p.w13 + (static_cast(s_ids[rq / kI]) * 2 * kI + rq % kI) * kH; + } + ptx::mbar_arrive_expect_tx(bar, kStageBytes); + draft_moe::bulk_g2s(dst, src, kRowBytes, bar); + draft_moe::bulk_g2s(dst + kRowBytes, src + static_cast(kI) * kH, kRowBytes, bar); + }; + + if (warp == kConsumers && lane == 0) { + for (int n = 0; n < nstatic; ++n) issue(n); + ptx::mbar_arrive_expect_tx(&wsd_bar, slice_bytes); + draft_moe::bulk_g2s(s_wsd, p.w2_shared + static_cast(j0) * kI, slice_bytes, &wsd_bar); + if constexpr (kTrace) trace[19] = clock64(); + } + // Warp w < nrows computes router row row0 + w out of ring item w / 2. + const bool router_warp = warp < nrows; + cvec_t gw[kChunksPerLane]; + if (gate_cta && warp < 4) { +#pragma unroll + for (int k = 0; k < kChunksPerLane; ++k) gw[k].load(p.gate_w + warp * kQuarter, k * kWarpThreads + lane); + } + + PDLWaitPrimary(); + for (int i = static_cast(threadIdx.x); i < kH / 8; i += static_cast(blockDim.x)) { + cvec_t v; + v.load(p.x, i); + v.store(s_x, i); + } + if (threadIdx.x == 0) { + s_tag = *p.epoch + 1u; + } + __syncthreads(); + const uint32_t tag = s_tag; + stamp_after_sync(1); + + if (warp == kConsumers) { + // Every CTA's logits, in the lane layout of the router's selection warp. + uint64_t lw[kPerLaneE]; +#pragma unroll + for (int i = 0; i < kPerLaneE; ++i) lw[i] = fz::ld_relaxed(my_logits + i * kWarpThreads + lane); + const uint64_t t0 = fz::globaltimer(); + while (true) { + bool ready = true; +#pragma unroll + for (int i = 0; i < kPerLaneE; ++i) ready &= static_cast(lw[i] >> 32) == tag; + if (__all_sync(0xffffffffu, ready)) break; + fz::check_spin(t0); +#pragma unroll + for (int i = 0; i < kPerLaneE; ++i) { + if (static_cast(lw[i] >> 32) != tag) lw[i] = fz::ld_relaxed(my_logits + i * kWarpThreads + lane); + } + } + if constexpr (kTrace) { + if (lane == 0) { + trace[2] = clock64(); + trace[28] = static_cast(fz::globaltimer()); + } + } + + // Keys, max and sum exactly as in the selection warp of draft_router_kernel. + float v[kPerLaneE]; + uint32_t key[kPerLaneE]; + float row_max = draft_moe::neg_inf(); +#pragma unroll + for (int i = 0; i < kPerLaneE; ++i) { + const float l = __uint_as_float(static_cast(lw[i])); + v[i] = l == l ? l : -1e30f; + s_v[i * kWarpThreads + lane] = v[i]; + row_max = fmaxf(row_max, v[i]); + uint32_t b = __float_as_uint(cast(cast(v[i]))) >> 16; + b = b == 0x8000u ? 0u : b; + const uint32_t ord = (b & 0x8000u) ? (~b & 0xffffu) : (b | 0x8000u); + key[i] = (ord << 16) | (0xffffu - static_cast(i * kWarpThreads + lane)); + } + // Round k of the router's warp max picks the k-th largest (unique) key; with + // each lane's keys sorted, a round is one warp max and a shift. + fz::sort_desc(key); + uint32_t my_key = 0; +#pragma unroll + for (int k = 0; k < kTopK; ++k) { + const uint32_t best = __reduce_max_sync(0xffffffffu, key[0]); + if (lane == k) my_key = best; + const bool pop = key[0] == best; +#pragma unroll + for (int i = 0; i + 1 < kPerLaneE; ++i) key[i] = pop ? key[i + 1] : key[i]; + key[kPerLaneE - 1] = pop ? 0u : key[kPerLaneE - 1]; + } + const int32_t id = lane < kTopK ? static_cast(0xffffu - (my_key & 0xffffu)) : 0; + if (lane < kTopK) s_ids[lane] = id; + __syncwarp(); + // The routed copies need only the ids; the weights are only read by the final combine. + int next = nstatic; + if (lane == 0) { + for (; next < min(nstatic + kWindow, nup); ++next) { + ptx::mbar_wait_parity(&empty_bar[next % kStages], ((next / kStages) & 1) ^ 1); + issue(next); + } + } + row_max = warp::reduce_max(row_max); + float lsum = 0.0f; +#pragma unroll + for (int i = 0; i < kPerLaneE; ++i) lsum += expf(v[i] - row_max); + const float row_sum = warp::reduce_sum(lsum); + const float pw = expf(s_v[id] - row_max) / row_sum; + float routed_sum = 0.0f; +#pragma unroll + for (int k = 0; k < kTopK; ++k) routed_sum += __shfl_sync(0xffffffffu, pw, k); + const float norm = routed_sum > 0.0f ? routed_sum : 1.0f; + if (lane < kTopK) { + const float w = pw / norm; + s_w[lane] = w; + if (c == 0) { + p.topk_weights[lane] = w; + p.topk_ids[lane] = id; + } + } + __syncwarp(); + if (lane == 0) { + if (c == 0) *p.epoch = tag; + ptx::mbar_arrive(&sel_bar); + if constexpr (kTrace) trace[3] = clock64(); + for (int n = next; n < nup + kTopK; ++n) { + // A short window keeps every CTA's rows arriving at the same pace. + const int m = n - kWindow; + if (n < nup && m >= nstatic) ptx::mbar_wait_parity(&empty_bar[m % kStages], (m / kStages) & 1); + ptx::mbar_wait_parity(&empty_bar[n % kStages], ((n / kStages) & 1) ^ 1); + issue(n); + if constexpr (kTrace) { + if (n == nup - 1) trace[12] = clock64(); + } + } + if constexpr (kTrace) trace[11] = clock64(); + } + return; + } + + if (router_warp) { + const int item = warp / 2; + fz::mbar_wait_sleep(&full_bar[item], 0); + const auto* wrow = reinterpret_cast(ring + item * kStageBytes + (warp & 1) * kRowBytes); + float acc = 0.0f; +#pragma unroll + for (int k = 0; k < kRouterPerLane; ++k) { + vec_t w, xv; + w.load(wrow, k * kWarpThreads + lane); + xv.load(s_x, k * kWarpThreads + lane); +#pragma unroll + for (int j = 0; j < kVecN; ++j) acc = fmaf(cast(w[j]), cast(xv[j]), acc); + } + acc = warp::reduce_sum(acc); + if constexpr (kTrace) { + if (threadIdx.x == 0) trace[20] = clock64(); + } + const uint64_t word = (static_cast(tag) << 32) | __float_as_uint(draft_moe::round_bf16(acc)); + for (int r = lane; r < kFusedReplicas; r += kWarpThreads) { + fz::st_relaxed(p.logit_buf + r * kLogitWords + row0 + warp, word); + } + if constexpr (kTrace) { + if (lane == 0) trace[24 + warp] = static_cast(fz::globaltimer()); + if (threadIdx.x == 0) trace[4] = clock64(); + } + } + + // Consumers: x quarter in registers, as in the up_tma consumers. + const int quarter = warp & 3; + const bool is_up = warp >= 4; + cvec_t xr[kChunksPerLane]; +#pragma unroll + for (int k = 0; k < kChunksPerLane; ++k) xr[k].load(s_x + quarter * kQuarter, k * kWarpThreads + lane); + if (gate_cta && warp < 4) { + float acc = 0.0f; +#pragma unroll + for (int k = 0; k < kChunksPerLane; ++k) { +#pragma unroll + for (int j = 0; j < 8; ++j) acc = fmaf(cast(xr[k][j]), cast(gw[k][j]), acc); + } + acc = warp::reduce_sum(acc); + if (lane == 0) s_gate_part[warp] = acc; + } + + for (int n = 0; n < nup; ++n) { + const int stage = n % kStages; + fz::mbar_wait_sleep(&full_bar[stage], (n / kStages) & 1); + if (n < nrouter) { + __syncwarp(); + if (lane == 0) ptx::mbar_arrive(&empty_bar[stage]); + continue; + } + if constexpr (kTrace) { + if (n == nstatic && threadIdx.x == 0) trace[5] = clock64(); + } + const auto* row = + reinterpret_cast(ring + stage * kStageBytes + (is_up ? kRowBytes : 0)) + quarter * kQuarter; + float acc = 0.0f; +#pragma unroll + for (int k = 0; k < kChunksPerLane; ++k) { + cvec_t w; + w.load(row, k * kWarpThreads + lane); +#pragma unroll + for (int j = 0; j < 8; ++j) acc = fmaf(cast(w[j]), cast(xr[k][j]), acc); + } + acc = warp::reduce_sum(acc); + if (lane == 0) { + s_part[n - nrouter][warp] = acc; + ptx::mbar_arrive(&empty_bar[stage]); + } + } + + fz::named_sync(1, kConsumerThreads); + stamp_after_sync(6); + if (static_cast(threadIdx.x) < npairs) { + const int t = static_cast(threadIdx.x); + const float g = draft_moe::round_bf16(((s_part[t][0] + s_part[t][1]) + s_part[t][2]) + s_part[t][3]); + const float u = draft_moe::round_bf16(((s_part[t][4] + s_part[t][5]) + s_part[t][6]) + s_part[t][7]); + const bf16_t a = cast(draft_moe::silu_f32(g) * u); + s_word[t] = ((tag & 0xffffu) << 16) | static_cast(__bfloat16_as_ushort(a)); + } + fz::named_sync(1, kConsumerThreads); + // One warp store per replica: the routed words of a CTA are contiguous, so a replica + // costs one or two lines instead of one line per word. + if (lane < npairs) { + const int idx = lane < nsh ? kTopK * kI + sh0 + lane : rq0 + lane - nsh; + const uint32_t word = s_word[lane]; + for (int r = warp; r < kFusedReplicas; r += kConsumers) fz::st_relaxed(p.act_buf + r * kActWords + idx, word); + } + if constexpr (kTrace) { + if (threadIdx.x == 0) trace[29] = static_cast(fz::globaltimer()); + } + if (gate_cta) { + const float t = ((s_gate_part[0] + s_gate_part[1]) + s_gate_part[2]) + s_gate_part[3]; + const float gate = 1.0f / (1.0f + expf(-t)); + const uint64_t word = (static_cast(tag) << 32) | __float_as_uint(gate); + for (int r = static_cast(threadIdx.x); r < kFusedReplicas; r += kConsumerThreads) { + fz::st_relaxed(p.logit_buf + r * kLogitWords + kE, word); + } + } + + // Act values of the whole call: the first loads go out before the down slices are waited + // on, and with few enough registers each group lifts its down weights out of shared memory + // while the act values are still on the way. + constexpr int kGroups = kConsumers * 4; + constexpr int kRounds = (kColsCap + kGroups - 1) / kGroups; + constexpr int kActVecs = kActWords / 4; + constexpr int kActVecsPerThread = (kActVecs + kConsumerThreads - 1) / kConsumerThreads; + const uint32_t tag16 = tag & 0xffffu; + uint4 av[kActVecsPerThread]; +#pragma unroll + for (int i = 0; i < kActVecsPerThread; ++i) { + const int vi = static_cast(threadIdx.x) + i * kConsumerThreads; + if (vi < kActVecs) av[i] = fz::ld_relaxed_v4(my_acts + 4 * vi); + } + const int grp = warp * 4 + lane / 8; + const int gl = lane % 8; + const bf16_t* slices[kSlots]; + fz::mbar_wait_sleep(&wsd_bar, 0); + slices[kTopK] = reinterpret_cast(s_wsd); +#pragma unroll + for (int s = 0; s < kTopK; ++s) { + const int n = nup + s; + fz::mbar_wait_sleep(&full_bar[n % kStages], (n / kStages) & 1); + slices[s] = reinterpret_cast(ring + (n % kStages) * kStageBytes); + } + constexpr bool kPreload = kRounds * kSlots * kDownPerLane <= 24; + cvec_t wpre[kPreload ? kRounds : 1][kPreload ? kSlots : 1][kPreload ? kDownPerLane : 1]; + if constexpr (kPreload) { +#pragma unroll + for (int r = 0; r < kRounds; ++r) { + const int jr = min(r * kGroups + grp, ncols - 1); +#pragma unroll + for (int s = 0; s < kSlots; ++s) { +#pragma unroll + for (int k = 0; k < kDownPerLane; ++k) wpre[r][s][k].load(slices[s] + jr * kI, gl + 8 * k); + } + } + } + { + const uint64_t t0 = fz::globaltimer(); + while (true) { + bool ready = true; +#pragma unroll + for (int i = 0; i < kActVecsPerThread; ++i) { + const int vi = static_cast(threadIdx.x) + i * kConsumerThreads; + if (vi >= kActVecs) continue; + const bool ok = (av[i].x >> 16) == tag16 && (av[i].y >> 16) == tag16 && (av[i].z >> 16) == tag16 && + (av[i].w >> 16) == tag16; + if (!ok) { + ready = false; + av[i] = fz::ld_relaxed_v4(my_acts + 4 * vi); + } + } + if (ready) break; + fz::check_spin(t0); + } +#pragma unroll + for (int i = 0; i < kActVecsPerThread; ++i) { + const int vi = static_cast(threadIdx.x) + i * kConsumerThreads; + if (vi < kActVecs) { + uint2 packed; + packed.x = (av[i].x & 0xffffu) | (av[i].y << 16); + packed.y = (av[i].z & 0xffffu) | (av[i].w << 16); + *reinterpret_cast(s_act + 4 * vi) = packed; + } + } + } + if (threadIdx.x == 0) { + uint64_t gword = fz::ld_relaxed(my_logits + kE); + const uint64_t t0 = fz::globaltimer(); + while (static_cast(gword >> 32) != tag) { + fz::check_spin(t0); + gword = fz::ld_relaxed(my_logits + kE); + } + s_gate = __uint_as_float(static_cast(gword)); + if constexpr (kTrace) trace[30] = static_cast(fz::globaltimer()); + } + fz::mbar_wait_sleep(&sel_bar, 0); + fz::named_sync(1, kConsumerThreads); + stamp_after_sync(7); + + // Down pass of down_v3: one 8-lane group per (slot, column), the kSlots independent + // dots of a group interleaved. +#pragma unroll + for (int r = 0; r < kRounds; ++r) { + const int jj = r * kGroups + grp; + const int jr = min(jj, ncols - 1); + float d[kSlots]; +#pragma unroll + for (int s = 0; s < kSlots; ++s) { + float acc = 0.0f; +#pragma unroll + for (int k = 0; k < kDownPerLane; ++k) { + cvec_t w, a; + if constexpr (kPreload) { + w = wpre[r][s][k]; + } else { + w.load(slices[s] + jr * kI, gl + 8 * k); + } + a.load(s_act + s * kI, gl + 8 * k); +#pragma unroll + for (int e = 0; e < 8; ++e) acc = fmaf(cast(w[e]), cast(a[e]), acc); + } + d[s] = acc; + } +#pragma unroll + for (int s = 0; s < kSlots; ++s) d[s] += __shfl_xor_sync(0xffffffffu, d[s], 4); +#pragma unroll + for (int s = 0; s < kSlots; ++s) d[s] += __shfl_xor_sync(0xffffffffu, d[s], 2); +#pragma unroll + for (int s = 0; s < kSlots; ++s) d[s] += __shfl_xor_sync(0xffffffffu, d[s], 1); + if (gl == 0 && jj < ncols) { +#pragma unroll + for (int s = 0; s < kSlots; ++s) s_d[s][jj] = d[s]; + } + } + fz::named_sync(1, kConsumerThreads); + stamp_after_sync(8); + auto column = [&](int jj) { + float sum = 0.0f; +#pragma unroll + for (int s = 0; s < kTopK; ++s) sum += draft_moe::round_bf16(s_d[s][jj] * s_w[s]); + const float routed = draft_moe::round_bf16(sum); + const float shared = draft_moe::round_bf16(s_d[kTopK][jj]); + float r = s_gate * shared; + r += routed; + return cast(r); + }; + if (static_cast(threadIdx.x) < ncols) { + const int jj = static_cast(threadIdx.x); + p.out[j0 + jj] = column(jj); + } + if constexpr (kTrace) { + if (threadIdx.x == 0) { + trace[15] = clock64(); + trace[9] = static_cast(fz::globaltimer()); + } + } +} + +} // namespace sglang + +#if !defined(Q35_DMOE_H) || !defined(Q35_DMOE_I) || !defined(Q35_DMOE_E) || !defined(Q35_DMOE_TOPK) || \ + !defined(Q35_DMOE_COLS) || !defined(Q35_DMOE_STAGES) || !defined(Q35_DMOE_WINDOW) || !defined(Q35_DMOE_TRACE) +#error "the launcher defines Q35_DMOE_{H,I,E,TOPK,COLS,STAGES,WINDOW,TRACE}" +#endif + +extern "C" __global__ void __launch_bounds__(288, 1) + q35_draft_moe_fused(const sglang::DraftMoeFusedParams __grid_constant__ p) { + sglang::draft_moe_fused_kernel(p); +} diff --git a/test/kernels/test_qwen3_5_draft_moe.py b/test/kernels/test_qwen3_5_draft_moe.py new file mode 100644 index 00000000..e547bf1e --- /dev/null +++ b/test/kernels/test_qwen3_5_draft_moe.py @@ -0,0 +1,233 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The draft package's single-launch MoE (sm103/draft/moe.py) on one SM100-family GPU +(GB300), at the TP8 draft shape (hidden 4096, 512 experts of intermediate 128, top-10): +- the top-10 is exactly the kernel's own selection rule on the logits it published + (bf16 order keys, ties to the lowest id), and the weights its renormalized softmax; +- logits, weights and output match a float64 reference with the kernel's rounding + points (only the fp32 summation order differs); +- repeated calls and CUDA-graph replays give the same bytes; +- DraftMoe runs a two-token batch as two single launches. +Bitwise equality with the SGLang kernel it is ported from (on SGLang branch +nvidia/qwen35-agentx-gb300) is checked with that branch installed as ``sglang``, not here.""" + +from types import SimpleNamespace + +import pytest +import torch + +H, I, E, K = 4096, 128, 512, 10 + + +def _gpu_ready() -> bool: + if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10: + return False + try: + from cuda.bindings import nvrtc # noqa: F401 + except ImportError: + return False + return True + + +pytestmark = pytest.mark.skipif(not _gpu_ready(), reason="needs an SM100-family GPU") + + +@pytest.fixture(scope="module") +def weights(): + """One TP8 draft MoE block's bf16 weights in checkpoint layout.""" + g = torch.Generator(device="cuda").manual_seed(5) + + def randn(*shape, std): + return (torch.randn(*shape, device="cuda", generator=g) * std).to( + torch.bfloat16 + ) + + return { + "router_w": randn(E, H, std=0.02), + "w13": randn(E, 2 * I, H, std=0.02), + "w2": randn(E, H, I, std=0.05), + "w13_shared": randn(2 * I, H, std=0.02), + "w2_shared": randn(H, I, std=0.05), + "gate_w": randn(H, std=0.02), + } + + +def _x(seed: int) -> torch.Tensor: + g = torch.Generator(device="cuda").manual_seed(1000 + seed) + return torch.randn(1, H, device="cuda", generator=g).to(torch.bfloat16) + + +def _outputs(): + return ( + torch.empty(1, K, dtype=torch.float32, device="cuda"), + torch.empty(1, K, dtype=torch.int32, device="cuda"), + torch.empty(1, H, dtype=torch.bfloat16, device="cuda"), + ) + + +def _run(w, x, scratch, outs): + from sgl_nvidia_mega.qwen3_5.sm103.draft import moe + + topk_weights, topk_ids, out = outs + moe.draft_moe_fused( + x, + w["router_w"], + w["w13"], + w["w2"], + w["w13_shared"], + w["w2_shared"], + w["gate_w"], + scratch, + topk_weights, + topk_ids, + out, + ) + return outs + + +def _kernel_logits(scratch) -> torch.Tensor: + """The fp32 logits the call published (replica 0 of the tagged words).""" + words = scratch[0][:E] + return (words & 0xFFFFFFFF).to(torch.int32).view(torch.float32) + + +def _order_keys(logits: torch.Tensor) -> torch.Tensor: + """draft_router_kernel's selection keys: bf16 order key, then the lower id.""" + v = torch.where(logits == logits, logits, torch.full_like(logits, -1e30)) + b = (v.to(torch.bfloat16).view(torch.int16).to(torch.int64)) & 0xFFFF + b = torch.where(b == 0x8000, torch.zeros_like(b), b) + ordered = torch.where((b & 0x8000) != 0, (~b) & 0xFFFF, b | 0x8000) + ids = torch.arange(E, device=logits.device, dtype=torch.int64) + return (ordered << 16) | (0xFFFF - ids) + + +def _r(t: torch.Tensor) -> torch.Tensor: + return t.to(torch.bfloat16).double() + + +def _reference(w, x, ids, weights): + """float64 dots with the kernel's bf16 rounding points.""" + xd = x.double().view(-1) + acts = [] + for s in range(K + 1): + rows = w["w13_shared"] if s == K else w["w13"][int(ids[s])] + gate, up = _r(rows[:I].double() @ xd), _r(rows[I:].double() @ xd) + acts.append(_r(gate / (1.0 + torch.exp(-gate)) * up)) + routed = torch.zeros(H, dtype=torch.float64, device="cuda") + for s in range(K): + routed = routed + _r( + (w["w2"][int(ids[s])].double() @ acts[s]) * float(weights[s]) + ) + shared = _r(w["w2_shared"].double() @ acts[K]) + gate = torch.sigmoid(w["gate_w"].double() @ xd) + return _r(gate * shared + _r(routed)) + + +def test_selection_weights_and_output(weights): + from sgl_nvidia_mega.qwen3_5.sm103.draft import moe + + scratch = moe.draft_moe_fused_scratch(E, K, I, torch.device("cuda")) + for seed in range(4): + x = _x(seed) + topk_weights, topk_ids, out = _run(weights, x, scratch, _outputs()) + torch.cuda.synchronize() + assert int(scratch[2]) == seed + 1 # the call epoch + logits = _kernel_logits(scratch) + assert torch.equal( + (scratch[0][:E] >> 32).to(torch.int32), + torch.full_like(logits, seed + 1).int(), + ) + # the logits are bf16 values within a bf16 step of the float64 dot + ref_logits = weights["router_w"].double() @ x.double().view(-1) + assert torch.equal(logits.to(torch.bfloat16).float(), logits) + assert ( + logits.double() - ref_logits + ).abs().max() <= 2**-7 * ref_logits.abs().max() + # the top-10 is the kernel's own rule on its own logits, exactly + want = torch.topk(_order_keys(logits), K).values + assert torch.equal(topk_ids.view(-1).long(), 0xFFFF - (want & 0xFFFF)) + probs = torch.softmax(logits.double(), dim=0)[topk_ids.view(-1).long()] + assert torch.allclose( + topk_weights.view(-1).double(), probs / probs.sum(), rtol=1e-5 + ) + ref = _reference(weights, x, topk_ids.view(-1), topk_weights.view(-1)) + err = (out.view(-1).double() - ref).abs() + assert ( + err.max() <= 2e-2 * ref.abs().max() + and err.mean() <= 2e-3 * ref.abs().mean() + ) + + +def test_repeat_and_graph_replay(weights): + from sgl_nvidia_mega.qwen3_5.sm103.draft import moe + + scratch = moe.draft_moe_fused_scratch(E, K, I, torch.device("cuda")) + eager = {} + for seed in range(3): + a = [t.clone() for t in _run(weights, _x(seed), scratch, _outputs())] + b = [t.clone() for t in _run(weights, _x(seed), scratch, _outputs())] + assert all(torch.equal(p, q) for p, q in zip(a, b)) + eager[seed] = a + x = _x(0).clone() + outs = _outputs() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + _run(weights, x, scratch, outs) + for _ in range(2): + for seed in range(3): + x.copy_(_x(seed)) + graph.replay() + torch.cuda.synchronize() + assert all(torch.equal(p, q) for p, q in zip(outs, eager[seed])) + + +def test_draft_moe_two_tokens(weights): + """DraftMoe on a loaded block's weights: two tokens are two launches in order.""" + # the draft package's spec needs an SGLang with sglang.srt.megakernels + pytest.importorskip("sglang.srt.megakernels.spec") + from sgl_nvidia_mega.qwen3_5.draft.moe import DraftMoe + + mlp = SimpleNamespace( + gate=SimpleNamespace(weight=weights["router_w"]), + experts=SimpleNamespace(w13_weight=weights["w13"], w2_weight=weights["w2"]), + shared_expert=SimpleNamespace( + gate_up_proj=SimpleNamespace(weight=weights["w13_shared"]), + down_proj=SimpleNamespace(weight=weights["w2_shared"]), + ), + shared_expert_gate=SimpleNamespace(weight=weights["gate_w"].view(1, H)), + topk=SimpleNamespace(topk_config=SimpleNamespace(top_k=K)), + ) + block = DraftMoe(mlp, torch.device("cuda")) + assert block.ready(1) and block.ready(2) and block.single_launch(2) + assert not block.ready(0) and not block.ready(3) + x = torch.cat([_x(7), _x(8)]) + got = block(x) + from sgl_nvidia_mega.qwen3_5.sm103.draft import moe + + scratch = moe.draft_moe_fused_scratch(E, K, I, torch.device("cuda")) + for m in range(2): + *_, out = _run(weights, x[m : m + 1], scratch, _outputs()) + assert torch.equal(got[m : m + 1], out) + + +def test_rejects_bad_operands(weights): + from sgl_nvidia_mega.qwen3_5.sm103.draft import moe + + scratch = moe.draft_moe_fused_scratch(E, K, I, torch.device("cuda")) + with pytest.raises(ValueError): + _run(weights, _x(0).float(), scratch, _outputs()) + with pytest.raises(ValueError): + _run(weights, torch.cat([_x(0), _x(1)]), scratch, _outputs()) + with pytest.raises(ValueError): + moe.draft_moe_fused( + _x(0), + weights["router_w"], + weights["w13"], + weights["w2"], + weights["w13_shared"], + weights["w2_shared"], + weights["gate_w"], + scratch, + *_outputs(), + hop_site=torch.zeros(7, dtype=torch.int64), + ) diff --git a/test/unit/test_qwen3_5_runtime.py b/test/unit/test_qwen3_5_runtime.py index 9bd91d2a..b026a461 100644 --- a/test/unit/test_qwen3_5_runtime.py +++ b/test/unit/test_qwen3_5_runtime.py @@ -136,6 +136,17 @@ def test_the_draft_entries_are_offered_only_on_request(monkeypatch): assert runtime.build(model, target_ctx).summary() == "slots none (stock)" +@pytest.mark.parametrize( + "slots, want", + [ + ({"draft_moe"}, "slot 'draft_moe' runs inside entry 'draft_loop'"), + ], +) +def test_the_draft_slots_name_what_they_need(monkeypatch, slots, want): + monkeypatch.setattr(runtime, "SLOTS", frozenset(slots)) + assert [r.split(": ", 1)[1] for r in runtime.slot_reasons()] == [want] + + class _DraftParts: """Parts with no slot built: the stock model step.""" From 256406dcc027ebc8d96ebf7caab088bc489ea359 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:47:00 -0700 Subject: [PATCH 10/27] [Perf] [GB300] qwen3_5: Draft slot draft_q1: the draft steps' q1 split-KV decode (numeric) Inside draft_loop, a one-row draft step's decode attention runs SKV2-Q1: trtllm-gen's KV split and rounding points, the CTAs merged in fp32 by a small Triton kernel. The draft loop's graph variants keep trtllm-gen below 16384 tokens of KV. Draft slot draft_q1; numeric: only the fp32 association differs from trtllm-gen. --- sgl_nvidia_mega/qwen3_5/draft/__init__.py | 11 +- sgl_nvidia_mega/qwen3_5/draft/attention.py | 74 +- sgl_nvidia_mega/qwen3_5/draft/loop.py | 4 + sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 2 +- .../qwen3_5/sm103/draft/__init__.py | 3 +- sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py | 366 ++++++++++ .../qwen3_5/sm103/draft/q1_kernel.py | 667 ++++++++++++++++++ test/kernels/test_qwen3_5_draft_q1.py | 239 +++++++ test/unit/test_qwen3_5_runtime.py | 28 + 9 files changed, 1389 insertions(+), 5 deletions(-) create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/q1_kernel.py create mode 100644 test/kernels/test_qwen3_5_draft_q1.py diff --git a/sgl_nvidia_mega/qwen3_5/draft/__init__.py b/sgl_nvidia_mega/qwen3_5/draft/__init__.py index e55b5043..86ba2502 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/draft/__init__.py @@ -13,7 +13,7 @@ (``extend``). Inside them every op is SGLang's own unless a draft slot replaces it: ``draft_moe`` -(the single-launch MoE). +(the single-launch MoE) and ``draft_q1`` (the q1 split-KV decode). """ from __future__ import annotations @@ -68,7 +68,7 @@ def numerics(slots: frozenset) -> Numerics: which differ from SGLang's ops only in fp32 summation order (q1 split-KV, the o_proj / fc / MoE GEMVs, the draft head), so a draft token can differ where two logits are within a bf16 rounding.""" - if not slots & {"draft_moe"}: + if not slots & {"draft_moe", "draft_q1"}: return target.NUMERICS return Numerics(NumericsKind.TOLERANCE, {"draft_logits.rel": 2**-7}) @@ -111,6 +111,13 @@ def __init__( self.draft_extend: Optional[Callable] = ( self._draft_extend if "draft_extend" in slots else None ) + # the KV-length graph variants by runner kind, for an SGLang that captures them + # (2.2 routes the draft steps' q1 decode by KV length) + self.graph_variants = None + if "draft_loop" in slots and "draft_q1" in slots: + from sgl_nvidia_mega.qwen3_5.draft.attention import Q1Variants + + self.graph_variants = {"draft_loop": Q1Variants()} def summary(self) -> str: entries = [ diff --git a/sgl_nvidia_mega/qwen3_5/draft/attention.py b/sgl_nvidia_mega/qwen3_5/draft/attention.py index 985f9730..f03ed596 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/attention.py +++ b/sgl_nvidia_mega/qwen3_5/draft/attention.py @@ -12,13 +12,46 @@ the qkv projection with the fused q / k Gemma RMSNorm + RoPE + gate split, the fused FP8 quantization of q / k / v with the paged KV write, trtllm-gen's decode with ``q_len_per_req`` rows a request (1 for a draft step, the verify width for the draft -extend), the sigmoid output gate and ``o_proj``. +extend), the sigmoid output gate and ``o_proj``. Slot ``draft_q1`` runs a draft step's +decode as the q1 split-KV kernel (``sm103/draft/q1``). """ from __future__ import annotations import torch +# the KV-length graph variants of the draft loop (2.2's graph_variants.Q1_STOCK / +# Q1_LONG): trtllm-gen below Q1_MIN_LENGTH, SKV2-Q1 from it (2.2's +# SGLANG_Q35MF_MKA_Q1_MIN_LENGTH default, which its TP8 serving configuration keeps) +Q1_STOCK, Q1_LONG = "q1_stock", "q1_long" +Q1_MIN_LENGTH = 16384 + + +class Q1Variants: + """The draft loop's graph variants (SGLang's AttentionGraphVariants), as 2.2's + Q1GraphVariants: SKV2-Q1 from a host-only KV length lower bound.""" + + # stock first, as 2.2 captures them (variants share a graph memory pool) + capture_labels = (Q1_STOCK, Q1_LONG) + + def __init__(self, min_length: int = Q1_MIN_LENGTH): + self.min_length = min_length + + def select(self, forward_batch) -> str: + if forward_batch.batch_size != 1: + return Q1_STOCK + length = getattr(forward_batch, "attention_kv_len_lower_bound", None) + if length is None: + lengths = forward_batch.seq_lens_cpu + if ( + lengths is not None + and lengths.device.type == "cpu" + and lengths.numel() == 1 + ): + length = int(lengths[0]) + return Q1_LONG if length is not None and length >= self.min_length else Q1_STOCK + + # The SGLang names this module reads (test/unit/test_contract.py pins them) SGLANG_READS = { "srt/models/qwen3_5.py": ( @@ -58,6 +91,9 @@ ), "srt/environ.py": ("envs", "SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR"), "srt/runtime_context.py": ("get_buffer",), + "srt/model_executor/runner_utils/capture_mode.py": ( + "get_capture_attention_variant", + ), # ModelConfig._derive_context_length writes the backend's max_context_len into the # HF config the draft's build gets "srt/configs/model_config.py": ("context_len",), @@ -118,6 +154,7 @@ def __init__(self, parts): ones = torch.ones(1, dtype=torch.float32, device=device) self.k_scale = attn.k_scale if attn.k_scale is not None else ones self.v_scale = attn.v_scale if attn.v_scale is not None else ones + self.q1 = None # sm103.draft.q1 decode (slot draft_q1), set by DraftLoop def __call__(self, x, positions, cache_loc, meta, kv, *, rows_per_request: int): """The TP-partial attention output [rows, hidden] of the normed input ``x`` @@ -157,6 +194,8 @@ def core(self, q, k, v, cache_loc, meta, kv, rows_per_request: int): attn = self.layer.attn bmm1_scale = 1.0 * kv.k_scale[0] * attn.scaling bmm2_scale = kv.v_scale[0] + if self.q1 is not None and rows_per_request == 1 and _q1_label() != Q1_STOCK: + return self.q1(q8, kv, meta).view(-1, self.heads * self.head_dim) kwargs = {} if rows_per_request != 1: kwargs["q_len_per_req"] = rows_per_request @@ -194,3 +233,36 @@ def _paged(self, k_cache, v_cache): if attn.tp_v_head_num == 1: v = canonicalize_stride(v) return k, v + + +def _q1_label(): + """The KV-length graph variant SGLang is capturing ("q1_stock" / "q1_long"), or + None where its draft runner captures one graph per batch size (then Q1 serves + every length; 2.2 switched at 16384 by graph variant).""" + from sglang.srt.model_executor.runner_utils.capture_mode import ( + get_capture_attention_variant, + ) + + return get_capture_attention_variant() + + +def build_q1(parts): + """Slot draft_q1: the draft steps' q1 decode as 2.2's SKV2-Q1 (``sm103/draft/q1``), + a callable (q8, kv, meta) -> [1, heads, head_dim] bf16; each layout's producer + compiles at the graph warmup's first call, outside capture, as in 2.2.""" + from sgl_nvidia_mega.qwen3_5.sm103.draft import q1 + + attention, device = parts.attention, parts.device + why = q1.enabled_reason( + device_index=device.index, max_context_len=attention.max_context_len + ) + if why is not None: + raise RuntimeError(f"NVIDIA megakernel: slot draft_q1 cannot run: {why}") + q1.build(device.index, heads=attention.heads) + decode = q1.Q1Decode() + attn = parts.layer.attn + + def run(q8, kv, meta): + return decode.step(q8, kv, attn.layer_id, meta, scaling=attn.scaling) + + return run diff --git a/sgl_nvidia_mega/qwen3_5/draft/loop.py b/sgl_nvidia_mega/qwen3_5/draft/loop.py index 487a9013..8533673c 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/loop.py +++ b/sgl_nvidia_mega/qwen3_5/draft/loop.py @@ -45,6 +45,10 @@ class DraftLoop: def __init__(self, parts, *, num_steps: int): self.parts = parts self.num_steps = num_steps + if "draft_q1" in parts.slots and parts.attention.q1 is None: + from sgl_nvidia_mega.qwen3_5.draft.attention import build_q1 + + parts.attention.q1 = build_q1(parts) def step_for(self, bs: int): return self.step if bs == SERVED_BS else None diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index bd7aaa68..bc448d38 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -30,7 +30,7 @@ # The MTP draft's EAGLE entries and kernel groups (qwen3_5/draft), requested in the same # variable; the draft spec's build reads them, the target's ignores them DRAFT_ENTRIES: tuple[str, ...] = ("draft_loop", "draft_extend") -DRAFT_SLOTS: tuple[str, ...] = ("draft_moe",) +DRAFT_SLOTS: tuple[str, ...] = ("draft_moe", "draft_q1") SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py index df0d5ba8..d298d829 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py @@ -1,3 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -"""The MTP draft's SM103 (GB300) kernels: the single-launch draft MoE (``moe``).""" +"""The MTP draft's SM103 (GB300) kernels: the single-launch draft MoE (``moe``) and +the q1 split-KV decode attention (``q1``).""" diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py new file mode 100644 index 00000000..d62e4b3d --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py @@ -0,0 +1,366 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang's python/sglang/srt/layers/attention/trtllm_mha_backend.py (branch +# nvidia/qwen35-agentx-gb300): the SKV2-Q1 arm of TRTLLMHAAttnBackend._run_fixed_q_len_decode, +# _run_q1 and the _q1_enabled check, with _get_bmm_scales, _reshape_paged_kv_cache +# and SGLang's paged_kv_view (srt/mem_cache/layout/paged_view.py). +"""The MTP draft's q1 decode attention on SKV2-Q1 (``q1_kernel``): FP8 paged KV, +trtllm-gen's KV split (one CTA per SM, contiguous 256-token steps) and its rounding +points, the CTAs merged in fp32 by a small Triton kernel. Only the fp32 association +differs from trtllm-gen; SGLang branch nvidia/qwen35-agentx-gb300 serves it at KV >= 16384. + +The draft step has no forward batch or attention backend. ``Q1Decode.step`` takes the +step's ``AttnStepMetadata`` and the draft's ``KvPoolBuffers`` +(``sglang.srt.megakernels.eagle_binding``), forms the paged cache view and the BMM +scales the way the backend does, and runs the plan of that input layout. The query +is the step's FP8 query as the backend hands it to the kernel: the attention front's +FP8 output, with no query scale (q_scale 1.0). +""" + +from __future__ import annotations + +from functools import cache +from typing import Optional + +import torch + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "kernels/ops/attention/utils.py": ("canonicalize_stride",), + "srt/environ.py": ("envs", "SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR"), + "srt/megakernels/eagle_binding.py": ( + "AttnStepMetadata", + "KvPoolBuffers", + "page_table", + "cache_seqlens", + "layer_ids", + "k_scale", + "v_scale", + ), +} + +# The SGLang functions whose body this module mirrors, by the digest of their code +# (test/unit/test_contract.py recomputes them on the installed SGLang); SGLang branch +# nvidia/qwen35-agentx-gb300 has the same digests +SGLANG_MIRRORS = { + "srt/layers/attention/trtllm_mha_backend.py:TRTLLMHAAttnBackend._get_bmm_scales": "90a1387e8f561611", + "srt/layers/attention/trtllm_mha_backend.py:TRTLLMHAAttnBackend._reshape_paged_kv_cache": "53ae20937769b99a", + "srt/mem_cache/layout/paged_view.py:paged_kv_view": "e73ce9ff261c4a9b", + "srt/mem_cache/layout/paged_view.py:paged_view": "5fd2565ec50e2620", +} + +PAGE_SIZE = 64 +HEAD_DIM = 256 +KV_HEADS = 1 # the draft's KV heads on a rank: 2 KV heads over TP8, replicated +QUERY_HEADS = (4, 8) +# the KV length from which the branch serves the draft's q1 decode with SKV2-Q1 +# (SGLANG_Q35MF_MKA_Q1_MIN_LENGTH, unset in its TP8 serving configuration) +MIN_LENGTH = 16384 +# the merge's head-dim block and warps (Q1SplitPlan.run) +MERGE_BLOCK_D, MERGE_WARPS = 32, 4 + + +def enabled_reason(*, device_index: int, max_context_len: int) -> Optional[str]: + """None where the branch's backend enables SKV2-Q1 (``_q1_enabled``), else why + not: an SM10x GPU, and a context of at least one 256-token step per SM, the + lengths at which trtllm-gen splits the KV one CTA per SM.""" + major, _ = torch.cuda.get_device_capability(device_index) + if major != 10: + return f"SKV2-Q1 runs on SM10x, not SM{major}x" + sms = torch.cuda.get_device_properties(device_index).multi_processor_count + if -(-max_context_len // 256) < sms: + return ( + f"a {max_context_len}-token context spans fewer 256-token steps than the " + f"{sms} SMs trtllm-gen splits over" + ) + return None + + +def max_context_len_bound(page_table: torch.Tensor, page_size: int = PAGE_SIZE) -> int: + """The backend's max_context_len rounded up to whole pages: its graph page tables + are ``ceil(max_context_len / page_size)`` wide.""" + return int(page_table.shape[1]) * page_size + + +def paged_view(flat: torch.Tensor, page_size: int) -> torch.Tensor: + """SGLang's ``paged_view``: ``[slots, ...]`` -> ``[slots // page_size, page_size, + ...]`` from the buffer's own strides.""" + num_slots = int(flat.shape[0]) + assert num_slots % page_size == 0, (num_slots, page_size) + slot_stride = flat.stride(0) + return flat.as_strided( + (num_slots // page_size, page_size, *flat.shape[1:]), + (slot_stride * page_size, slot_stride, *flat.stride()[1:]), + ) + + +def paged_kv_view( + buf: torch.Tensor, page_size: int, head_num: int, head_dim: int +) -> torch.Tensor: + """SGLang's ``paged_kv_view``: an MHA pool buffer (slot-major or HND) as + ``[num_pages, page_size, head_num, head_dim]``.""" + if buf.dim() <= 3: + return paged_view(buf.view(buf.shape[0], head_num, head_dim), page_size) + assert tuple(buf.shape[1:]) == (head_num, page_size, head_dim), ( + f"expected an HND buffer [pages, {head_num}, {page_size}, {head_dim}], " + f"got {tuple(buf.shape)}" + ) + return buf.permute(0, 2, 1, 3) + + +def paged_kv_cache( + k_buffer: torch.Tensor, + v_buffer: torch.Tensor, + *, + page_size: int = PAGE_SIZE, + head_num: int = KV_HEADS, + head_dim: int = HEAD_DIM, +) -> tuple[torch.Tensor, torch.Tensor]: + """The backend's ``_reshape_paged_kv_cache`` on one layer's pool buffers: + ``[pages, heads, page_size, head_dim]`` views, a single head's stride made + canonical.""" + from sglang.kernels.ops.attention.utils import canonicalize_stride + + k_cache = paged_kv_view(k_buffer, page_size, head_num, head_dim).permute(0, 2, 1, 3) + v_cache = paged_kv_view(v_buffer, page_size, head_num, head_dim).permute(0, 2, 1, 3) + if head_num == 1: + k_cache = canonicalize_stride(k_cache) + v_cache = canonicalize_stride(v_cache) + return k_cache, v_cache + + +def bmm_scales( + *, k_scale: float, v_scale: float, scaling: float, q_scale: float = 1.0 +) -> tuple[float, float]: + """The backend's ``_get_bmm_scales`` for an FP8 KV cache: (bmm1, bmm2) = + (q_scale * k_scale * scaling, v_scale), in Python floats as there.""" + return q_scale * k_scale * scaling, v_scale + + +def _skip_softmax_factor(): + from sglang.srt.environ import envs + + return envs.SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR.get() + + +def unsupported_reason( + query: torch.Tensor, + kv_cache, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + *, + bmm1_scale, + bmm2_scale, + out: Optional[torch.Tensor] = None, + out_dtype: torch.dtype = torch.bfloat16, + page_size: int = PAGE_SIZE, + q_len_per_req: int = 1, + attn_dcp_size: int = 1, + window_left: int = -1, + sinks: Optional[torch.Tensor] = None, + mask: Optional[torch.Tensor] = None, + kv_cache_sf=None, +) -> Optional[str]: + """None when the SKV2-Q1 arm of the branch's ``_run_fixed_q_len_decode`` takes + the call, else the first condition it fails (the branch then runs trtllm-gen). + The branch also requires trtllm-gen rather than XQA, which SM10x always is.""" + checks = ( + (attn_dcp_size == 1, f"decode context parallel size {attn_dcp_size}"), + (page_size == PAGE_SIZE, f"page size {page_size}"), + (q_len_per_req == 1, f"{q_len_per_req} query rows a request"), + (seq_lens.numel() == 1, f"{seq_lens.numel()} requests"), + ( + query.shape == (1, query.shape[-2], HEAD_DIM) + and query.shape[-2] in QUERY_HEADS, + f"query shape {tuple(query.shape)}", + ), + (query.dtype == torch.float8_e4m3fn, f"query dtype {query.dtype}"), + (query.is_contiguous(), "a non-contiguous query"), + ( + isinstance(kv_cache, tuple) and len(kv_cache) == 2, + "a KV cache that is not a (k, v) pair", + ), + ) + for ok, why in checks: + if not ok: + return why + for x in kv_cache: + if not ( + x.dtype == query.dtype + and x.ndim == 4 + and x.shape[1:] == (1, PAGE_SIZE, HEAD_DIM) + and x.is_contiguous() + ): + return f"KV cache view {x.dtype} {tuple(x.shape)} {tuple(x.stride())}" + rest = ( + (out_dtype == torch.bfloat16, f"output dtype {out_dtype}"), + ( + out is None + or ( + out.is_contiguous() + and out.shape == query.shape + and out.dtype == out_dtype + ), + "an output buffer of another shape or layout", + ), + (isinstance(bmm1_scale, (int, float)), "a tensor bmm1 scale"), + # where trtllm-gen applies bmm2_scale relative to its bf16 partials is not + # pinned down, so the branch takes only 1.0 + ( + isinstance(bmm2_scale, (int, float)) and bmm2_scale == 1.0, + f"bmm2 scale {bmm2_scale!r}", + ), + (window_left < 0, f"a sliding window ({window_left})"), + (sinks is None and mask is None, "attention sinks or a mask"), + (kv_cache_sf is None, "KV cache block scales"), + ( + _skip_softmax_factor() in (None, 0.0), + "SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR is set", + ), + ) + for ok, why in rest: + if not ok: + return why + return None + + +class Q1Decode: + """One rank's SKV2-Q1 plans by input layout (the backend's ``_q1_plans``): a + plan owns the CTA partials and the producer compiled for that layout, so the + first call of a layout must not be under graph capture (graph warmup runs it + eagerly, as in the branch).""" + + def __init__(self): + self._plans = {} + + @staticmethod + def key(query, kv_cache, block_tables, seq_lens) -> tuple: + return ( + query.shape, + kv_cache[0].shape, + block_tables.shape, + block_tables.stride(), + seq_lens.shape, + ) + + def plan(self, query, kv_cache, block_tables, seq_lens): + """The layout's plan, compiled on its first use.""" + from sgl_nvidia_mega.qwen3_5.sm103.draft.q1_kernel import Q1SplitPlan + + key = self.key(query, kv_cache, block_tables, seq_lens) + plan = self._plans.get(key) + if plan is None: + plan = Q1SplitPlan(query, kv_cache, block_tables, seq_lens) + self._plans[key] = plan + return plan + + def __call__( + self, query, kv_cache, block_tables, seq_lens, *, out, qk_scale, v_scale + ) -> torch.Tensor: + """``_run_q1``: the merged output [1, heads, 256] bf16 in ``out``.""" + from sgl_nvidia_mega.qwen3_5.sm103.draft.q1_kernel import skv2_q1_decode + + plan = self.plan(query, kv_cache, block_tables, seq_lens) + return skv2_q1_decode( + query, + kv_cache, + block_tables, + seq_lens, + plan, + qk_scale=qk_scale, + v_scale=v_scale, + out=out, + ) + + def step( + self, + query: torch.Tensor, + kv, + layer_id: int, + step, + *, + scaling: float, + out: Optional[torch.Tensor] = None, + q_scale: float = 1.0, + ) -> torch.Tensor: + """One draft step's attention for ``layer_id`` on the draft's pool ``kv`` + (``KvPoolBuffers``) and the step's ``AttnStepMetadata``: [1, heads, 256] + bf16. Raises where the branch would run trtllm-gen instead.""" + i = kv.layer_ids.index(layer_id) + kv_cache = paged_kv_cache(kv.k[i], kv.v[i]) + bmm1, bmm2 = bmm_scales( + k_scale=kv.k_scale[i], + v_scale=kv.v_scale[i], + scaling=scaling, + q_scale=q_scale, + ) + why = unsupported_reason( + query, + kv_cache, + step.page_table, + step.cache_seqlens, + bmm1_scale=bmm1, + bmm2_scale=bmm2, + out=out, + ) + if why is not None: + raise ValueError(f"SKV2-Q1 cannot serve this step: {why}") + if out is None: + out = torch.empty_like(query, dtype=torch.bfloat16) + return self( + query, + kv_cache, + step.page_table, + step.cache_seqlens, + out=out, + qk_scale=float(bmm1), + v_scale=float(bmm2), + ) + + def prepare(self, heads: int, kv, layer_id: int, step) -> None: + """Compile the plan of a step's layout (an FP8 query of ``heads`` heads) + without running it; the producer is specialized to the layout, so this needs + the step's own pool and metadata tensors.""" + i = kv.layer_ids.index(layer_id) + query = torch.empty( + (1, heads, HEAD_DIM), dtype=torch.float8_e4m3fn, device=kv.k[i].device + ) + self.plan( + query, + paged_kv_cache(kv.k[i], kv.v[i]), + step.page_table, + step.cache_seqlens, + ) + + +@cache +def build(device_index: int, heads: int = 4) -> None: + """Import CuTe DSL and Triton and compile the fp32 merge for ``heads`` rows and + this GPU's SM count; each layout's producer compiles at its plan's first use.""" + import triton + + from sgl_nvidia_mega.qwen3_5.sm103.draft.q1_kernel import _q1_merge + + device = torch.device("cuda", device_index) + nsplit = torch.cuda.get_device_properties(device_index).multi_processor_count + partial = torch.zeros( + (nsplit, heads, HEAD_DIM), dtype=torch.bfloat16, device=device + ) + maximum = torch.zeros((nsplit, heads), dtype=torch.float32, device=device) + denom = torch.ones_like(maximum) + lengths = torch.full((1,), 256, dtype=torch.int32, device=device) + out = torch.empty((1, heads, HEAD_DIM), dtype=torch.bfloat16, device=device) + with torch.cuda.device(device): + _q1_merge[(heads, HEAD_DIM // MERGE_BLOCK_D, 1)]( + partial, + maximum, + denom, + lengths, + out, + heads, + nsplit, + triton.next_power_of_2(nsplit), + MERGE_BLOCK_D, + num_warps=MERGE_WARPS, + ) + torch.cuda.synchronize(device) diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/q1_kernel.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/q1_kernel.py new file mode 100644 index 00000000..6c5e84b6 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/q1_kernel.py @@ -0,0 +1,667 @@ +# SPDX-License-Identifier: BSD-3-Clause +# SPDX-FileCopyrightText: Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Adapted from SGLang's python/sglang/kernels/ops/attention/skv2_q1.py (branch +# nvidia/qwen35-agentx-gb300), which is distributed under this notice: +# +# Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# Derived from NVIDIA/cutlass v4.8.0 examples/python/CuTeDSL/cute_ext/blackwell/attention/ +# (gqa_decode_simple.py, gqa_decode_opt.py; BSD-3-Clause): warp-specialized split-KV decode structure, +# rewritten for paged FP8 KV. +# +# Modified by NVIDIA: paged FP8 verify with explicit local reduction. +# +# Modified by NVIDIA: separate softmax and value consumer groups. +# Modified by NVIDIA: separate key and value pipeline handoffs. +# +# Modified by NVIDIA: runtime attention scales, page-tail bounds and reusable buffers. +# Modified by NVIDIA: stock split-KV rounding (page splits, one FP8 P rounding, bf16 partials). +# Modified by NVIDIA: q1 decode with the trtllm-gen generation kernel's KV split and rounding (two streams per CTA). +# +# The kernel, the merge and the plan are that file's as they are: only the imports +# moved to the top, and the layout follows ruff format. + +"""SKV2-Q1: paged FP8 q1 decode (draft model) with trtllm-gen's rounding structure. + +Same KV split as trtllm-gen's multi-CTA generation kernel: one CTA per SM, ceil(steps / CTAs) contiguous 256-token +steps per CTA. Each CTA keeps two softmax streams (the two 128-token halves of every step) with a running max of the +raw scores; P x 448 = ex2(fma(s, c, -fma(m, c, -log2 448))) is rounded once to e4m3, the CTA writes bf16(O / 448) +with fp32 maxima and denominators, and a small kernel merges the CTAs in fp32. Only fp32 association differs. +""" + +import math + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import torch +import triton +import triton.language as tl +from cutlass.cute import experimental as ex +from cutlass.cute.nvgpu import OperandMajorMode, tcgen05 +from cutlass.cute.runtime import from_dlpack +from cutlass.experimental import primitives as prim + +LOG2_448 = math.log2(448.0) + + +@cute.experimental.jit +def load_keys( + k: cute.Tensor, + table: cute.Tensor, + sk: cute.Tensor, + barrier: cute.Pointer, + first_page: cutlass.Int32, + stage: cutlass.Int32, + tile: cutlass.Constexpr[int], + length: cutlass.Int32, +): + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive_and_expect_tx(barrier, tile * 256) + for half in cutlass.range_constexpr(tile // 64): + page_index = first_page + half + page = cutlass.Int32(0) + if page_index < cute.ceil_div(length, 64): + page = table[0, page_index] + source = cute.local_tile(k, (64, 256), (page, 0)) + dest = cute.local_tile(sk[None, None, stage], (64, 256), (half, 0)) + ex.tma_load(source, dest, barrier.value, update_expect_tx=False) + + +@cute.experimental.jit +def load_values( + v: cute.Tensor, + table: cute.Tensor, + sv: cute.Tensor, + barrier: cute.Pointer, + first_page: cutlass.Int32, + stage: cutlass.Int32, + tile: cutlass.Constexpr[int], + length: cutlass.Int32, +): + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive_and_expect_tx(barrier, tile * 256) + for half in cutlass.range_constexpr(tile // 64): + page_index = first_page + half + page = cutlass.Int32(0) + if page_index < cute.ceil_div(length, 64): + page = table[0, page_index] + source = cute.local_tile(v, (256, 64), (0, page)) + dest = cute.local_tile(sv[None, None, stage], (256, 64), (0, half)) + ex.tma_load(source, dest, barrier.value, update_expect_tx=False) + + +class Q1SplitDecode: + def __init__(self, heads=4, nsplit=152, stages=2, slots=2): + assert stages >= 1 and slots in (2, 3, 4) and 1 <= heads <= 12 + self.heads, self.nsplit, self.stages, self.slots = heads, nsplit, stages, slots + self.rows = heads + self.width = 32 + self.tile = 128 + assert self.width * slots * 3 <= 512 + + @cute.experimental.jit + def __call__( + self, + q: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + table: cute.Tensor, + lengths: cute.Tensor, + partial: cute.Tensor, + maxima: cute.Tensor, + denom: cute.Tensor, + qk_scale: cutlass.Float32, + out_scale: cutlass.Float32, + stream: cuda.CUstream, + ): + dtype = cutlass.Float8E4M3FN + kq = cute.make_tiled_mma( + tcgen05.MmaFP8Op( + dtype, + cutlass.Float32, + (128, self.width), + tcgen05.CtaGroup.ONE, + tcgen05.OperandSource.SMEM, + OperandMajorMode.K, + OperandMajorMode.K, + ) + ) + vp = cute.make_tiled_mma( + tcgen05.MmaFP8Op( + dtype, + cutlass.Float32, + (128, self.width), + tcgen05.CtaGroup.ONE, + tcgen05.OperandSource.SMEM, + OperandMajorMode.MN, + OperandMajorMode.MN, + ) + ) + self.kernel( + q, k, v, table, lengths, partial, maxima, denom, qk_scale, out_scale, kq, vp + ).launch(grid=(self.nsplit, 1, 1), block=(384, 1, 1), stream=stream) + + @cute.experimental.kernel + def kernel( + self, + q: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + table: cute.Tensor, + lengths: cute.Tensor, + partial: cute.Tensor, + maxima: cute.Tensor, + denom: cute.Tensor, + qk_scale: cutlass.Float32, + out_scale: cutlass.Float32, + kq: cute.TiledMma, + vp: cute.TiledMma, + ): + tid, _, _ = cute.arch.thread_idx() + split, _, _ = cute.arch.block_idx() + warp = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + lane = cute.arch.lane_idx() + local_tid = tid % 128 + width, rows = self.width, self.rows + dtype, f32 = cutlass.Float8E4M3FN, cutlass.Float32 + k_atom = tcgen05.make_smem_layout_atom( + tcgen05.SmemLayoutAtomKind.K_SW128, dtype + ) + v_atom = tcgen05.make_smem_layout_atom( + tcgen05.SmemLayoutAtomKind.MN_SW128, dtype + ) + p_atom = tcgen05.make_smem_layout_atom( + tcgen05.SmemLayoutAtomKind.MN_SW32, dtype + ) + sq = ex.allocate( + dtype, + cutlass.AddressSpace.smem, + cute.tile_to_shape(k_atom, (width, 256), order=(1, 0)), + alignment=128, + ) + sk = ex.allocate( + dtype, + cutlass.AddressSpace.smem, + cute.tile_to_shape(k_atom, (self.tile, 256, self.stages), order=(1, 0, 2)), + alignment=128, + ) + sv = ex.allocate( + dtype, + cutlass.AddressSpace.smem, + cute.tile_to_shape(v_atom, (256, self.tile, self.stages), order=(0, 1, 2)), + alignment=128, + ) + sp = ex.allocate( + dtype, + cutlass.AddressSpace.smem, + cute.tile_to_shape(p_atom, (width, self.tile, self.slots), order=(0, 1, 2)), + alignment=128, + ) + # Per-thread denominators of both streams, padded against bank conflicts. + shared_l = ex.allocate( + f32, + cutlass.AddressSpace.smem, + cute.make_layout((2, 128, rows), stride=(128 * (rows + 1), rows + 1, 1)), + alignment=128, + ) + shared = ex.allocate( + f32, + cutlass.AddressSpace.smem, + cute.make_layout((4, width), stride=(width, 1)), + alignment=128, + ) + alphas = ex.allocate( + f32, + cutlass.AddressSpace.smem, + cute.make_layout((width, self.slots), stride=(1, width)), + alignment=128, + ) + stream_off = ex.allocate( + f32, + cutlass.AddressSpace.smem, + cute.make_layout((2, width), stride=(width, 1)), + alignment=128, + ) + barriers = ex.allocate( + cutlass.Int64, + cutlass.AddressSpace.smem, + cute.make_layout(4 * self.stages + 5 * self.slots + 2), + alignment=8, + ) + bq = barriers.iterator + bk = bq + 1 + bv = bk + self.stages + bkfree = bv + self.stages + bvfree = bkfree + self.stages + bs = bvfree + self.stages + bsfree = bs + self.slots + bp = bsfree + self.slots + bo = bp + self.slots + bfree = bo + self.slots + bdenom = bfree + self.slots + if tid < 4 * self.stages + 5 * self.slots + 2: + count = cutlass.Int32(1) + if ( + tid >= 4 * self.stages + 1 + 2 * self.slots + and tid < 4 * self.stages + 1 + 3 * self.slots + ): + count = cutlass.Int32(128) + if tid == 4 * self.stages + 5 * self.slots + 1: + count = cutlass.Int32(128) + cute.arch.mbarrier_init(barriers.iterator + tid, count) + cute.arch.mbarrier_init_fence() + cute.arch.sync_threads() + ts = ex.allocate( + f32, + cutlass.AddressSpace.tmem, + ex.make_tmem_layout_acc(kq, (128, width, 256), self.slots), + alignment=32, + ) + to = ex.allocate( + f32, + cutlass.AddressSpace.tmem, + ex.make_tmem_layout_acc(vp, (128, width, 128), 2 * self.slots), + alignment=32, + ) + copy_atom = cute.make_copy_atom( + tcgen05.Ld32x32bOp(tcgen05.Repetition(width)), f32 + ) + s_plane = ts[(None, None), 0, 0, 0] + copy_s = tcgen05.make_tmem_copy(copy_atom, s_plane).get_slice(local_tid) + rlayout = copy_s.partition_D(s_plane).layout + # trtllm-gen split: one CTA per SM, ceil(steps / CTAs) contiguous 256-token steps each. + length = lengths[0] + spc = cute.ceil_div(cute.ceil_div(length, 256), self.nsplit) + begin_tok = split * spc * 256 + end_tok = begin_tok + spc * 256 + if end_tok > length: + end_tok = length + nblk = cutlass.Int32(0) + if begin_tok < end_tok: + nblk = cute.ceil_div(end_tok - begin_tok, self.tile) + npair = cute.ceil_div(nblk, 2) + begin_page = split * spc * 4 + cute.arch.sync_threads() + if warp < 4: + scores = cute.make_rmem_tensor((width, 1), f32) + running_m = cute.make_rmem_tensor((2,), f32) + running_off = cute.make_rmem_tensor((2,), f32) + lane_off = cute.make_rmem_tensor((1,), f32) + lane_alpha = cute.make_rmem_tensor((1,), f32) + running_l = cute.make_rmem_tensor((2, rows), f32) + packed = cute.make_rmem_tensor((width, 1), dtype) + running_m.fill(-f32.inf) + running_off.fill(-f32.inf) + running_l.fill(0) + for pair in cutlass.range(0, npair): + for st in cutlass.range_constexpr(2): + it = pair * 2 + st + if it < nblk: + slot = it % self.slots + cute.arch.mbarrier_wait(bs + slot, phase=(it // self.slots) % 2) + current_s = ts[(None, None), 0, 0, slot] + current_copy = tcgen05.make_tmem_copy( + copy_atom, current_s + ).get_slice(local_tid) + sub_scores = cute.make_tensor(scores.iterator, rlayout) + ex.partition_and_copy(current_copy, current_s, sub_scores) + cute.arch.fence_view_async_tmem_load() + prim.barrier_cta_sync(1, thread_count=128) + if warp == 0: + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive(bsfree + slot) + token = begin_tok + it * self.tile + local_tid + for j in cutlass.range_constexpr(rows): + score = scores[j, 0] + if token >= end_tok: + score = -f32.inf + scores[j, 0] = score + maximum = cute.arch.warp_redux_sync( + score, kind="fmax", nan=True + ) + if lane == j % 32: + shared[warp, j] = maximum + prim.barrier_cta_sync(1, thread_count=128) + # The value consumer releases both the probability and alpha slot. + cute.arch.mbarrier_wait( + bfree + slot, phase=(it // self.slots + 1) % 2 + ) + raw = running_m[st] + if lane < rows: + for w in cutlass.range_constexpr(4): + raw = cute.arch.fmax(raw, shared[w, lane]) + else: + raw = f32(0) + # trtllm-gen exponent: off = fma(m, c, -log2 448), P448 = ex2(fma(s, c, -off)). + off = cute.math.fma(raw, qk_scale, f32(-LOG2_448)) + safe = off + if raw == -f32.inf: + safe = f32(0) + lane_off[0] = safe + lane_alpha[0] = cute.math.exp2( + running_off[st] - safe, fastmath=True + ) + running_m[st] = raw + running_off[st] = off + if warp == 0: + alphas[lane, slot] = lane_alpha[0] + for j in cutlass.range_constexpr(rows): + offset = cute.arch.shuffle_sync(lane_off[0], j) + alpha = cute.arch.shuffle_sync(lane_alpha[0], j) + total = running_l[st, j] * alpha + probability = cute.math.exp2( + cute.math.fma(scores[j, 0], qk_scale, -offset), + fastmath=True, + ) + total += probability + scores[j, 0] = probability + running_l[st, j] = total + for j in cutlass.range_constexpr(rows, width): + scores[j, 0] = f32(0) + packed[None, 0].store(scores[None, 0].load().to(dtype)) + cute.autovec_copy(packed[None, 0], sp[None, local_tid, slot]) + cute.arch.fence_view_async_shared() + cute.arch.mbarrier_arrive(bp + slot) + for st in cutlass.range_constexpr(2): + for j in cutlass.range_constexpr(rows): + shared_l[st, local_tid, j] = running_l[st, j] + if warp == 0: + if lane < rows: + stream_off[0, lane] = running_off[0] + stream_off[1, lane] = running_off[1] + cute.arch.mbarrier_arrive(bdenom) + elif warp < 8: + value = cute.make_rmem_tensor(rlayout, f32) + accum = cute.make_rmem_tensor((2, rows, 2), f32) + accum.fill(0) + for pair in cutlass.range(0, npair): + for st in cutlass.range_constexpr(2): + it = pair * 2 + st + if it < nblk: + slot = it % self.slots + cute.arch.mbarrier_wait(bo + slot, phase=(it // self.slots) % 2) + for dm in cutlass.range_constexpr(2): + o_plane = to[(None, None), 0, 0, 2 * slot + dm] + copy_o = tcgen05.make_tmem_copy( + copy_atom, o_plane + ).get_slice(local_tid) + ex.partition_and_copy(copy_o, o_plane, value) + cute.arch.fence_view_async_tmem_load() + for j in cutlass.range_constexpr(rows): + accum[st, j, dm] = ( + accum[st, j, dm] * alphas[j, slot] + value[j] + ) + prim.barrier_cta_sync(2, thread_count=128) + if warp == 4: + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive(bfree + slot) + cute.arch.mbarrier_wait(bdenom, phase=0) + for j in cutlass.range_constexpr(rows): + o0 = stream_off[0, j] + o1 = stream_off[1, j] + top = cute.arch.fmax(o0, o1) + w0 = f32(0) + w1 = f32(0) + if o0 != -f32.inf: + w0 = cute.math.exp2(o0 - top, fastmath=True) + if o1 != -f32.inf: + w1 = cute.math.exp2(o1 - top, fastmath=True) + for dm in cutlass.range_constexpr(2): + merged = accum[0, j, dm] * w0 + accum[1, j, dm] * w1 + partial[split, j, local_tid + dm * 128] = cutlass.BFloat16( + merged * out_scale + ) + else: + if warp == 8: + q_frag = kq.get_slice(0).partition_B(sq) + atom_kq = cute.make_mma_atom(kq.op) + cute.arch.mbarrier_wait(bq, phase=0) + for it in cutlass.range(0, nblk): + slot, stage = it % self.slots, it % self.stages + cute.arch.mbarrier_wait( + bsfree + slot, phase=(it // self.slots + 1) % 2 + ) + cute.arch.mbarrier_wait(bk + stage, phase=(it // self.stages) % 2) + ktile = cute.local_tile(sk[None, None, stage], (128, 256), (0, 0)) + k_frag = kq.get_slice(0).partition_A(ktile) + atom_kq.set(tcgen05.Field.ACCUMULATE, False) + for kk in cutlass.range_constexpr(cute.size(k_frag, mode=[2])): + ex.dot( + atom_kq, + k_frag[None, None, kk], + q_frag[None, None, kk], + ts[None, None, None, slot], + ) + atom_kq.set(tcgen05.Field.ACCUMULATE, True) + with cute.arch.elect_one(): + tcgen05.commit(bs + slot, None, tcgen05.CtaGroup.ONE) + tcgen05.commit(bkfree + stage, None, tcgen05.CtaGroup.ONE) + elif warp == 9: + atom_vp = cute.make_mma_atom(vp.op) + for it in cutlass.range(0, nblk): + slot, stage = it % self.slots, it % self.stages + cute.arch.mbarrier_wait(bp + slot, phase=(it // self.slots) % 2) + cute.arch.mbarrier_wait(bv + stage, phase=(it // self.stages) % 2) + p_frag = vp.get_slice(0).partition_B(sp[None, None, slot]) + for dm in cutlass.range_constexpr(2): + vtile = cute.local_tile( + sv[None, None, stage], (128, self.tile), (dm, 0) + ) + v_frag = vp.get_slice(0).partition_A(vtile) + atom_vp.set(tcgen05.Field.ACCUMULATE, False) + for kk in cutlass.range_constexpr(cute.size(v_frag, mode=[2])): + ex.dot( + atom_vp, + v_frag[None, None, kk], + p_frag[None, None, kk], + to[None, None, None, 2 * slot + dm], + ) + atom_vp.set(tcgen05.Field.ACCUMULATE, True) + with cute.arch.elect_one(): + tcgen05.commit(bo + slot, None, tcgen05.CtaGroup.ONE) + tcgen05.commit(bvfree + stage, None, tcgen05.CtaGroup.ONE) + elif warp == 10: + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive_and_expect_tx(bq, width * 256) + ex.tma_load( + cute.local_tile(q, (width, 256), (0, 0)), + sq, + bq.value, + update_expect_tx=False, + ) + for stage in cutlass.range_constexpr(self.stages): + if stage < nblk: + load_keys( + k, + table, + sk, + bk + stage, + begin_page + stage * (self.tile // 64), + stage, + self.tile, + length, + ) + for it in cutlass.range(0, nblk): + stage = it % self.stages + cute.arch.mbarrier_wait( + bvfree + stage, phase=(it // self.stages + 1) % 2 + ) + load_values( + v, + table, + sv, + bv + stage, + begin_page + it * (self.tile // 64), + stage, + self.tile, + length, + ) + if it + self.stages < nblk: + cute.arch.mbarrier_wait( + bkfree + stage, phase=(it // self.stages) % 2 + ) + load_keys( + k, + table, + sk, + bk + stage, + begin_page + (it + self.stages) * (self.tile // 64), + stage, + self.tile, + length, + ) + cute.arch.mbarrier_wait(bdenom, phase=0) + for chunk in cutlass.range_constexpr(cute.ceil_div(rows, 12)): + column = warp + 12 * chunk + if column < rows: + o0 = stream_off[0, column] + o1 = stream_off[1, column] + top = cute.arch.fmax(o0, o1) + total = f32(0) + for st in cutlass.range_constexpr(2): + part = ( + shared_l[st, lane, column] + shared_l[st, lane + 32, column] + ) + ( + shared_l[st, lane + 64, column] + + shared_l[st, lane + 96, column] + ) + for offset in cutlass.range_constexpr(5): + part += cute.arch.shuffle_sync_bfly(part, 1 << offset) + stream_top = o0 if st == 0 else o1 + if stream_top != -f32.inf: + total += part * cute.math.exp2(stream_top - top, fastmath=True) + if lane == 0: + denom[split, column] = total + maxima[split, column] = top + + +@triton.jit +def _q1_merge( + partial, + maxima, + denom, + lengths, + out, + ROWS: tl.constexpr, + NSPLIT: tl.constexpr, + NP2: tl.constexpr, + BLOCK_D: tl.constexpr, +): + row = tl.program_id(0) + dblk = tl.program_id(1) + length = tl.load(lengths) + steps = tl.cdiv(length, 256) + spc = tl.cdiv(steps, NSPLIT) + active = tl.cdiv(steps, spc) + c = tl.arange(0, NP2) + valid = c < active + m = tl.load(maxima + c * ROWS + row, mask=valid, other=float("-inf")) + w = tl.where(valid, tl.exp2(m - tl.max(m, 0)), 0.0) + total = tl.sum(tl.load(denom + c * ROWS + row, mask=valid, other=0.0) * w, 0) + d = dblk * BLOCK_D + tl.arange(0, BLOCK_D) + part = tl.load( + partial + (c[:, None] * ROWS + row) * 256 + d[None, :], + mask=valid[:, None], + other=0.0, + ) + o = tl.sum(part.to(tl.float32) * w[:, None], 0) + # Denominators are sums of P x 448; partials are O / 448. + tl.store(out + row * 256 + d, (o / (total * (1.0 / 448.0))).to(tl.bfloat16)) + + +class Q1SplitPlan: + """Reusable storage and a compiled producer for one fixed input layout, plus the fp32 merge.""" + + def __init__(self, query, kv, table, lengths, *, nsplit=None, stages=2, slots=2): + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError("SKV2-Q1 storage must be prepared before graph capture") + qlen, self.heads, _ = query.shape + assert qlen == 1 + self.nsplit = ( + nsplit + or torch.cuda.get_device_properties(query.device).multi_processor_count + ) + self.partial = torch.empty( + (self.nsplit, self.heads, 256), dtype=torch.bfloat16, device=query.device + ) + self.maximum = torch.empty( + (self.nsplit, self.heads), dtype=torch.float32, device=query.device + ) + self.denom = torch.empty_like(self.maximum) + kernel = Q1SplitDecode(self.heads, self.nsplit, stages=stages, slots=slots) + views = self._views(query, kv, table, lengths) + stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) + self.compiled = cute.compile( + kernel, *views, cutlass.Float32(1.0), cutlass.Float32(1.0), stream + ) + + def _views(self, query, kv, table, lengths): + k, v = kv + tensors = ( + query.view(-1, 256), + k.view(-1, 256), + v.view(-1, 256).T, + table, + lengths, + self.partial, + self.maximum, + self.denom, + ) + return tuple(from_dlpack(t) for t in tensors) + + def run(self, query, kv, table, lengths, *, qk_scale, v_scale, out): + views = self._views(query, kv, table, lengths) + stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) + self.compiled( + *views, + cutlass.Float32(qk_scale * 1.4426950408889634), + cutlass.Float32(v_scale / 448.0), + stream, + ) + _q1_merge[(self.heads, 256 // 32, 1)]( + self.partial, + self.maximum, + self.denom, + lengths, + out, + self.heads, + self.nsplit, + triton.next_power_of_2(self.nsplit), + 32, + num_warps=4, + ) + return out + + +def skv2_q1_decode(query, kv, table, lengths, plan, *, qk_scale, v_scale, out): + """q1 decode for one request through a prepared plan; same contract as trtllm_batch_decode_with_kv_cache.""" + return plan.run( + query, kv, table, lengths, qk_scale=qk_scale, v_scale=v_scale, out=out + ) diff --git a/test/kernels/test_qwen3_5_draft_q1.py b/test/kernels/test_qwen3_5_draft_q1.py new file mode 100644 index 00000000..873badf6 --- /dev/null +++ b/test/kernels/test_qwen3_5_draft_q1.py @@ -0,0 +1,239 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The MTP draft's q1 decode attention (SKV2-Q1, sm103/draft/q1.py) on one SM100-family +GPU (GB300): FP8 paged KV of one KV head, 4 query heads (TP8) and 8 (TP4), 64-token +pages, at KV lengths from 1 to 150K: +- against a float64 attention over the same FP8 values, no farther than trtllm-gen + (FlashInfer's decode as SGLang serves the draft) is, and close to trtllm-gen + itself (only the fp32 association differs); +- the same bits eager and replayed from a CUDA graph whose KV length changes on the + device between replays; +- the cache view and BMM scales formed from a draft step's KvPoolBuffers and + AttnStepMetadata are SGLang's own (paged_kv_view, canonicalize_stride). +The kernel of SGLang branch nvidia/qwen35-agentx-gb300 against this copy, bit for bit, is +checked with that branch installed as ``sglang``, not here.""" + +from types import SimpleNamespace + +import pytest +import torch + +from sgl_nvidia_mega.qwen3_5.sm103.draft import q1 + +PAGE, D = q1.PAGE_SIZE, q1.HEAD_DIM +SCALING = D**-0.5 # Qwen3.5's attention scaling +LENGTHS = (1, 255, 256, 4096, 16384, 32768, 131072, 150000) +MAX_LEN = max(LENGTHS) + + +def _gpu_ready() -> bool: + if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10: + return False + try: + import cutlass # noqa: F401 + import triton # noqa: F401 + except ImportError: + return False + return True + + +pytestmark = pytest.mark.skipif(not _gpu_ready(), reason="needs an SM100-family GPU") + + +@pytest.fixture(scope="module") +def decode(): + """One rank's plans: every case below shares a layout per head count, so the + producer compiles once for each (it compiles on a plan's first, eager, use).""" + return q1.Q1Decode() + + +def _pool(g, pages: int): + """K and V pool buffers of one KV head, slot-major [slots, 1, D] FP8, as the + draft's MHATokenToKVPool keeps them (slot 0's page absorbs padding writes).""" + slots = (pages + 1) * PAGE + k = (torch.randn(slots, 1, D, device="cuda", generator=g) * 2.0).to( + torch.float8_e4m3fn + ) + v = (torch.randn(slots, 1, D, device="cuda", generator=g) * 2.0).to( + torch.float8_e4m3fn + ) + return k, v + + +def _case(seed: int, heads: int, length: int, k_scale: float = 1.0): + g = torch.Generator(device="cuda").manual_seed(seed) + width = -(-MAX_LEN // PAGE) # the graph page table's static width + k, v = _pool(g, width) + pages = -(-length // PAGE) + table = torch.zeros(1, width, dtype=torch.int32, device="cuda") + # pages scattered over the pool, never page 0 + table[0, :pages] = ( + torch.randperm(width, device="cuda", generator=g)[:pages].int() + 1 + ) + query = (torch.randn(1, heads, D, device="cuda", generator=g) * 2.0).to( + torch.float8_e4m3fn + ) + lengths = torch.tensor([length], dtype=torch.int32, device="cuda") + kv = SimpleNamespace( + layer_ids=(3,), k=(k,), v=(v,), k_scale=(k_scale,), v_scale=(1.0,) + ) + step = SimpleNamespace(page_table=table, cache_seqlens=lengths) + return query, kv, step + + +def _reference(query, kv, step, scale): + """float64 attention over the FP8 values: softmax(q k^T * scale) v.""" + length = int(step.cache_seqlens[0]) + pages = step.page_table[0, : -(-length // PAGE)].long() + slots = (pages[:, None] * PAGE + torch.arange(PAGE, device="cuda")).flatten() + slots = slots[:length] + + def rows(buf): # gather as bytes: FP8 tensors do not index on every torch + raw = buf.view(torch.uint8).view(-1, D)[slots] + return raw.view(torch.float8_e4m3fn).float().double() + + kk, vv = rows(kv.k[0]), rows(kv.v[0]) + s = query[0].float().double() @ kk.T * scale + return (torch.softmax(s, dim=-1) @ vv).unsqueeze(0) + + +def _trtllm_gen(query, kv_cache, step, bmm1, bmm2): + """FlashInfer's trtllm-gen decode with SGLang's arguments (the draft's stock arm).""" + import flashinfer + from sglang.srt.layers.attention.trtllm_mla_backend import ( + make_persistent_multi_ctas_kv_counter_buffer, + ) + + workspace = torch.zeros(128 * 2**20, dtype=torch.uint8, device="cuda") + counter = make_persistent_multi_ctas_kv_counter_buffer( + torch.device("cuda"), num_q_heads=32, max_batch_size=8 + ) + kwargs = dict( + query=query, + kv_cache=kv_cache, + workspace_buffer=workspace, + block_tables=step.page_table, + seq_lens=step.cache_seqlens, + max_seq_len=step.page_table.shape[1] * PAGE, + bmm1_scale=bmm1, + bmm2_scale=bmm2, + window_left=-1, + sinks=None, + out_dtype=torch.bfloat16, + ) + try: + return flashinfer.decode.trtllm_batch_decode_with_kv_cache( + **kwargs, multi_ctas_kv_counter_buffer=counter + ) + except TypeError: + return flashinfer.decode.trtllm_batch_decode_with_kv_cache(**kwargs) + + +@pytest.mark.parametrize("heads", [4, 8]) +@pytest.mark.parametrize("length", LENGTHS) +def test_q1_against_float64_and_trtllm_gen(decode, heads, length): + query, kv, step = _case(100 * heads + length % 97, heads, length, k_scale=0.75) + got = decode.step(query, kv, 3, step, scaling=SCALING) + assert got.dtype == torch.bfloat16 and got.shape == (1, heads, D) + assert torch.isfinite(got).all() + bmm1, bmm2 = q1.bmm_scales(k_scale=0.75, v_scale=1.0, scaling=SCALING) + ref = _reference(query, kv, step, bmm1) + stock = _trtllm_gen(query, q1.paged_kv_cache(kv.k[0], kv.v[0]), step, bmm1, bmm2) + err = (got.double() - ref).abs().max().item() + err_stock = (stock.double() - ref).abs().max().item() + # the FP8 rounding of P dominates both errors; Q1 rounds where trtllm-gen does + assert err <= 1.5 * err_stock + 1e-3, (err, err_stock) + gap = (got.float() - stock.float()).abs().max().item() + assert gap <= 2**-6 * max(1.0, ref.abs().max().item()), gap + + +def test_q1_graph_replay_follows_the_device_length(decode): + """One capture, the KV length and pages rewritten on the device before each + replay.""" + query, kv, step = _case(7, 4, 20000) + out = torch.empty(1, 4, D, dtype=torch.bfloat16, device="cuda") + decode.step(query, kv, 3, step, scaling=SCALING, out=out) # compiles, eagerly + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + decode.step(query, kv, 3, step, scaling=SCALING, out=out) + for length in (20000, 131072, 70001, 16385, 150000, 300): + pages = -(-length // PAGE) + g = torch.Generator(device="cuda").manual_seed(length) + order = torch.randperm(step.page_table.shape[1], device="cuda", generator=g) + step.page_table[0, :pages] = order[:pages].int() + 1 + step.cache_seqlens.fill_(length) + graph.replay() + torch.cuda.synchronize() + replayed = out.clone() + eager = decode.step(query, kv, 3, step, scaling=SCALING) + assert torch.equal(replayed.view(torch.int16), eager.view(torch.int16)), length + + +def test_q1_repeats_bit_for_bit(decode): + query, kv, step = _case(9, 4, 131072) + a = decode.step(query, kv, 3, step, scaling=SCALING) + b = decode.step(query, kv, 3, step, scaling=SCALING) + assert torch.equal(a.view(torch.int16), b.view(torch.int16)) + + +@pytest.mark.parametrize("hnd", [False, True]) +def test_cache_view_is_sglangs(hnd): + """The view from a pool buffer equals the backend's _reshape_paged_kv_cache + (SGLang's paged_kv_view, permute, canonicalize_stride), slot-major or HND.""" + from sglang.kernels.ops.attention.utils import canonicalize_stride + from sglang.srt.mem_cache.layout.paged_view import paged_kv_view + + pages = 33 + if hnd: + buf = torch.zeros(pages, 1, PAGE, D, dtype=torch.float8_e4m3fn, device="cuda") + else: + buf = torch.zeros(pages * PAGE, 1, D, dtype=torch.float8_e4m3fn, device="cuda") + want = canonicalize_stride(paged_kv_view(buf, PAGE, 1, D).permute(0, 2, 1, 3)) + got, _ = q1.paged_kv_cache(buf, buf) + assert (got.shape, got.stride(), got.data_ptr()) == ( + want.shape, + want.stride(), + want.data_ptr(), + ) + assert ( + q1.unsupported_reason( + torch.zeros(1, 4, D, dtype=torch.float8_e4m3fn, device="cuda"), + (got, got), + torch.zeros(1, 4, dtype=torch.int32, device="cuda"), + torch.ones(1, dtype=torch.int32, device="cuda"), + bmm1_scale=SCALING, + bmm2_scale=1.0, + ) + is None + ) + + +def test_unsupported_reasons(decode): + query, kv, step = _case(3, 4, 4096) + kv_cache = q1.paged_kv_cache(kv.k[0], kv.v[0]) + ok = dict(bmm1_scale=SCALING, bmm2_scale=1.0) + args = (query, kv_cache, step.page_table, step.cache_seqlens) + assert q1.unsupported_reason(*args, **ok) is None + assert q1.unsupported_reason(*args, bmm1_scale=SCALING, bmm2_scale=0.5) + assert q1.unsupported_reason(*args, **ok, page_size=32) + assert q1.unsupported_reason(*args, **ok, window_left=128) + assert q1.unsupported_reason(*args, **ok, attn_dcp_size=2) + assert q1.unsupported_reason(query.view(torch.uint8), *args[1:], **ok) + two = torch.ones(2, dtype=torch.int32, device="cuda") + assert q1.unsupported_reason(query, kv_cache, step.page_table, two, **ok) + with pytest.raises(ValueError): + decode.step( + query, + SimpleNamespace(**{**vars(kv), "v_scale": (0.5,)}), + 3, + step, + scaling=SCALING, + ) + + +def test_build_and_enabled(): + device = torch.cuda.current_device() + q1.build(device, 4) + assert q1.enabled_reason(device_index=device, max_context_len=262144) is None + assert q1.enabled_reason(device_index=device, max_context_len=4096) diff --git a/test/unit/test_qwen3_5_runtime.py b/test/unit/test_qwen3_5_runtime.py index b026a461..726b28b6 100644 --- a/test/unit/test_qwen3_5_runtime.py +++ b/test/unit/test_qwen3_5_runtime.py @@ -136,6 +136,34 @@ def test_the_draft_entries_are_offered_only_on_request(monkeypatch): assert runtime.build(model, target_ctx).summary() == "slots none (stock)" +@pytest.mark.parametrize( + "batch_size, lower_bound, seq_lens, want", + [ + (1, 16384, None, "q1_long"), + (1, 16383, None, "q1_stock"), + (1, None, [20000], "q1_long"), + (1, None, [100], "q1_stock"), + (1, None, None, "q1_stock"), + (2, 50000, None, "q1_stock"), + ], +) +def test_the_draft_loop_routes_q1_by_kv_length_as_2_2( + monkeypatch, batch_size, lower_bound, seq_lens, want +): + from sgl_nvidia_mega.qwen3_5 import draft + + monkeypatch.setattr(runtime, "SLOTS", frozenset({"draft_loop", "draft_q1"})) + built = draft.SPEC.build("lm", SimpleNamespace(context_len=4096), _draft_ctx()) + variants = built.graph_variants["draft_loop"] + assert variants.capture_labels == ("q1_stock", "q1_long") + batch = SimpleNamespace( + batch_size=batch_size, + attention_kv_len_lower_bound=lower_bound, + seq_lens_cpu=None if seq_lens is None else torch.tensor(seq_lens), + ) + assert variants.select(batch) == want + + @pytest.mark.parametrize( "slots, want", [ From 4b56c1ff67cfc5f625a82a1c9e166a6d52dd3b72 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:48:00 -0700 Subject: [PATCH 11/27] [Perf] [GB300] qwen3_5: Draft slot draft_xchg: the draft step boundary's fc split, exchange and head (numeric) Inside draft_loop, the greedy one-row draft step boundary runs in two launches instead of seven: the fc as a row split, the LM head as a GEMV on the vocabulary shard, and one multicast exchange for the argmax, the positions and the next step's fc. The hidden-half fc partial declares one CTA per SM, as it launches, so NVRTC's build issues its weight loads together. Draft slot draft_xchg; numeric: only the fp32 summation order differs from SGLang's fc and LM head. --- sgl_nvidia_mega/qwen3_5/draft/__init__.py | 5 +- sgl_nvidia_mega/qwen3_5/draft/extend.py | 22 +- sgl_nvidia_mega/qwen3_5/draft/loop.py | 59 +- sgl_nvidia_mega/qwen3_5/draft/parts.py | 55 +- sgl_nvidia_mega/qwen3_5/draft/xchg.py | 637 +++++++++++++++ sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 2 +- .../qwen3_5/sm103/draft/__init__.py | 5 +- .../qwen3_5/sm103/draft/src/draft_fc.cu | 246 ++++++ .../qwen3_5/sm103/draft/src/mkd_common.cuh | 152 ++++ .../qwen3_5/sm103/draft/src/mkd_fc.cu | 323 ++++++++ .../qwen3_5/sm103/draft/src/mkd_head.cu | 114 +++ .../qwen3_5/sm103/draft/src/mkd_xchg.cu | 482 ++++++++++++ sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py | 738 ++++++++++++++++++ test/kernels/test_qwen3_5_draft_xchg.py | 303 +++++++ test/unit/test_qwen3_5_runtime.py | 2 +- 15 files changed, 3123 insertions(+), 22 deletions(-) create mode 100644 sgl_nvidia_mega/qwen3_5/draft/xchg.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_fc.cu create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_common.cuh create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_fc.cu create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_head.cu create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_xchg.cu create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py create mode 100644 test/kernels/test_qwen3_5_draft_xchg.py diff --git a/sgl_nvidia_mega/qwen3_5/draft/__init__.py b/sgl_nvidia_mega/qwen3_5/draft/__init__.py index 86ba2502..74b83d5f 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/draft/__init__.py @@ -13,7 +13,8 @@ (``extend``). Inside them every op is SGLang's own unless a draft slot replaces it: ``draft_moe`` -(the single-launch MoE) and ``draft_q1`` (the q1 split-KV decode). +(the single-launch MoE), ``draft_q1`` (the q1 split-KV decode) and ``draft_xchg`` (the +step boundary's fc split, exchange and head). """ from __future__ import annotations @@ -68,7 +69,7 @@ def numerics(slots: frozenset) -> Numerics: which differ from SGLang's ops only in fp32 summation order (q1 split-KV, the o_proj / fc / MoE GEMVs, the draft head), so a draft token can differ where two logits are within a bf16 rounding.""" - if not slots & {"draft_moe", "draft_q1"}: + if not slots & {"draft_moe", "draft_q1", "draft_xchg"}: return target.NUMERICS return Numerics(NumericsKind.TOLERANCE, {"draft_logits.rel": 2**-7}) diff --git a/sgl_nvidia_mega/qwen3_5/draft/extend.py b/sgl_nvidia_mega/qwen3_5/draft/extend.py index 255a967e..27591343 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/extend.py +++ b/sgl_nvidia_mega/qwen3_5/draft/extend.py @@ -14,7 +14,9 @@ row's top-1 (``torch.argmax``'s choice), 1.0 and final-norm hidden state into the SGLang-owned outputs. -Stock: SGLang's model step and top-1 (``layer``). +Stock: SGLang's model step and top-1 (``layer``). Slot ``draft_xchg``: the fc split +of 2.2 and the exchange's top-1, which also writes the next draft round's first fc +(2.2's MKD extend fc). """ from __future__ import annotations @@ -57,7 +59,12 @@ def step_for(self, bs: int): def step(self, io) -> None: parts = self.parts parts.note_engaged("draft_extend", io.num_requests) - x = layer.fc_input(parts, io.input_ids, io.hidden_states) + mkd = parts.mkd + if mkd is not None: + embeds = parts.model.embed_tokens(io.input_ids) + x = mkd.split_fc(embeds, io.hidden_states) + else: + x = layer.fc_input(parts, io.input_ids, io.hidden_states) hidden = layer.layer( parts, x, @@ -68,6 +75,13 @@ def step(self, io) -> None: rows_per_request=io.tokens_per_request, ) selected = hidden.index_select(0, io.select_index) - io.topk_index_out.copy_(layer.top1(parts, selected).view(-1, 1)) - io.topk_p_out.fill_(1.0) + if mkd is not None: + lm = parts.lm + local = lm.logits_processor._compute_lm_head(selected, lm.lm_head, None) + topk_p, topk_index = mkd.exchange_extend(local, selected) + io.topk_index_out.copy_(topk_index) + io.topk_p_out.copy_(topk_p) + else: + io.topk_index_out.copy_(layer.top1(parts, selected).view(-1, 1)) + io.topk_p_out.fill_(1.0) io.hidden_out.copy_(selected) diff --git a/sgl_nvidia_mega/qwen3_5/draft/loop.py b/sgl_nvidia_mega/qwen3_5/draft/loop.py index 8533673c..dae86bb2 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/loop.py +++ b/sgl_nvidia_mega/qwen3_5/draft/loop.py @@ -12,7 +12,11 @@ (column 0 is the round's first token); the step advances the positions in place and restores them before it returns, as the binding asks. -Stock: SGLang's model step and top-1 (``layer``). +Stock: SGLang's model step and top-1 (``layer``). Slot ``draft_xchg`` ends each step +in the exchange: the next step's fc comes from it, the LM head is the shard GEMV, and +the token is chosen across the shards in the same launch (``draft/xchg``). +``begin_draft_round`` then fills the first step's fc when the round's input did not +come from a served draft extend (2.2's take_fc_buf / fill_fc decision). """ from __future__ import annotations @@ -35,6 +39,8 @@ "steps", "kv", "draft_tokens_out", + "EagleDraftRoundIO", + "from_served_extend", ), } @@ -54,8 +60,16 @@ def step_for(self, bs: int): return self.step if bs == SERVED_BS else None def begin_round(self, io) -> None: - """On the host before a served draft graph replays: nothing to prepare while - every op is SGLang's.""" + """On the host before a served draft graph replays: the first step's fc when + no served draft extend of these requests produced the round's input.""" + mkd = self.parts.mkd + if ( + mkd is not None + and mkd.extend_fc + and io.num_requests == SERVED_BS + and not io.from_served_extend + ): + mkd.fill_fc(io.topk_index, io.hidden_states) @torch.no_grad() def step(self, io) -> None: @@ -65,14 +79,17 @@ def step(self, io) -> None: tokens = io.draft_tokens_out tokens[:, :1].copy_(io.topk_index) positions, mrope = io.positions, io.mrope_positions - token, hidden = io.topk_index.view(-1), io.hidden_states - for i in range(steps): - hidden = self._model_step(io, i, layer.fc_input(parts, token, hidden)) - token = layer.top1(parts, hidden) - tokens[:, i + 1].copy_(token) - positions.add_(1) - if mrope is not None: - mrope.add_(1) + if parts.mkd is not None: + self._mkd_steps(io, steps) + else: + token, hidden = io.topk_index.view(-1), io.hidden_states + for i in range(steps): + hidden = self._model_step(io, i, layer.fc_input(parts, token, hidden)) + token = layer.top1(parts, hidden) + tokens[:, i + 1].copy_(token) + positions.add_(1) + if mrope is not None: + mrope.add_(1) positions.sub_(steps) if mrope is not None: mrope.sub_(steps) @@ -88,3 +105,23 @@ def _model_step(self, io, i: int, x): io.kv, rows_per_request=1, ) + + def _mkd_steps(self, io, steps: int) -> None: + """The steps with slot draft_xchg, in 2.2's order: fc from the exchange, the + layer, the hidden half of the next fc on a side stream, the shard head, the + exchange (token, positions and the next fc).""" + mkd = self.parts.mkd + for i in range(steps): + mkd.begin_step(i, io.num_requests) + hidden = self._model_step(io, i, mkd.fc()) + if mkd.armed: + mkd.hidden_partial(hidden) + mkd.finish_step( + mkd.head(hidden), + io.positions, + io.draft_tokens_out, + i, + i + 1, + io.mrope_positions, + ) + mkd.end() diff --git a/sgl_nvidia_mega/qwen3_5/draft/parts.py b/sgl_nvidia_mega/qwen3_5/draft/parts.py index 07704e50..936d5cf9 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/parts.py +++ b/sgl_nvidia_mega/qwen3_5/draft/parts.py @@ -23,6 +23,11 @@ "lm_head", "logits_processor", ), + "srt/layers/logits_processor.py": ("logit_scale", "vocab_size"), + "srt/layers/vocab_parallel_embedding.py": ( + "shard_indices", + "org_vocab_start_index", + ), "srt/models/qwen3_5.py": ( "layers", "start_layer", @@ -57,6 +62,7 @@ def __init__(self, draft, layer): self.device = torch.device("cuda", torch.cuda.current_device()) self.attention = None # draft.attention.Attention self.moe: Optional[Any] = None # draft.moe.DraftMoe (slot draft_moe) + self.mkd: Optional[Any] = None # draft.xchg.MkdBoundary (slot draft_xchg) # (entry, bs) pairs whose capture logged the engagement line self.engaged: set = set() @@ -75,10 +81,14 @@ def build(cls, draft) -> Parts: slots = parts.slots if "draft_moe" in slots: parts.moe = parts._build_moe() + if "draft_xchg" in slots: + parts.mkd = parts._build_mkd() logger.info( - "NVIDIA megakernel: Qwen3.5 MTP draft parts built (slots %s; MoE %s)", + "NVIDIA megakernel: Qwen3.5 MTP draft parts built (slots %s; MoE %s, " + "exchange %s)", sorted(slots), "single launch" if parts.moe is not None else "SGLang's", + "MKD" if parts.mkd is not None else "SGLang's", ) return parts @@ -93,6 +103,49 @@ def _build_moe(self): ) return moe + def _build_mkd(self): + from sgl_nvidia_mega.qwen3_5.draft.xchg import MkdBoundary + + lm, b = self.lm, self.binding + embed = self.model.embed_tokens + processor = lm.logits_processor + # what 2.2's boundary assumed of the draft head and MkdBoundary cannot see + refused = [ + why + for bad, why in ( + (lm.config.tie_word_embeddings, "tied embeddings"), + (processor.logit_scale is not None, "logit scaling"), + ( + getattr(processor, "final_logit_softcapping", None) is not None, + "logit softcapping", + ), + ) + if bad + ] + mkd = None + if not refused: + mkd = MkdBoundary.maybe_create( + vocab_size=processor.vocab_size, + fc_weight=lm.fc.weight, + norm_embedding=lm.pre_fc_norm_embedding, + norm_hidden=lm.pre_fc_norm_hidden, + embed_shard=embed.weight, + vocab_start=embed.shard_indices.org_vocab_start_index, + lm_head_shard=lm.lm_head.weight, + embed=embed, + rank=b.tp_rank, + world=b.tp_size, + device_group=b.tp_device_group, + cpu_group=b.tp_cpu_group, + num_steps=b.num_steps, + ) + if mkd is None: + raise RuntimeError( + "NVIDIA megakernel: slot draft_xchg cannot run: " + + ("; ".join(refused) or "see the MKD draft boundary warning") + ) + return mkd + def note_engaged(self, entry: str, bs: int) -> None: """Log once per (entry, bs) captured: the engagement line every rank prints.""" if torch.cuda.is_current_stream_capturing() and (entry, bs) not in self.engaged: diff --git a/sgl_nvidia_mega/qwen3_5/draft/xchg.py b/sgl_nvidia_mega/qwen3_5/draft/xchg.py new file mode 100644 index 00000000..8cfcc241 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/draft/xchg.py @@ -0,0 +1,637 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0): python/sglang/srt/speculative/mkd_draft_boundary.py +# (MkdDraftBoundary), draft_vocab_exchange.py (the one-launch boundary's path) and +# python/sglang/srt/models/qwen3_5_mtp.py (the row-split fc), on branch +# nvidia/qwen35-agentx-gb300. +"""The greedy one-row MTP draft step boundary (MKD) in two launches instead of seven. + +A draft step's lm_head runs on this rank's vocabulary shard (``head``, which also leaves +each CTA's best row), and one exchange launch (``exchange``) takes the shard argmax, +multicasts each rank's candidate together with its hidden-half fc partial (computed +beside the lm_head on a side stream, ``hidden_partial``), selects the same winner on every +rank and writes the next step's fc output (``fc``). The candidates are memo rows (the +T-table): per vocabulary row of the shard, the embedding-half fc partials already summed +in rank order, built at construction with the split fc's arithmetic and checked against +it on every row. The draft extend's exchange (``exchange_extend``) also writes the next +draft's first fc (extend fc); ``fill_fc`` writes it when no extend did. + +Every value equals the row-split fc path bit for bit: the same argmax keys and ties, the +same per-slice dot arithmetic, the same rank-order fp32 sum and bf16 rounding. The step +state (``armed``, ``pending``, ``last_step``, ``step``) follows 2.2's: ``begin_step`` +arms a one-row step, the exchange leaves ``pending`` for the next step's ``fc``. +""" + +from __future__ import annotations + +import hashlib +import logging +from typing import Callable, Optional + +import torch + +from sgl_nvidia_mega.qwen3_5.sm103.draft import xchg as kernels + +logger = logging.getLogger(__name__) + +# Memo rows built per launch at construction +_MEMO_BLOCK = 4096 +# CTAs of mkd_draft_head per SM (T1, TP8 shard: 1 -> 39.3 us, 2 -> 40.5 us, 3 -> 41.6 us) +_HEAD_CTAS_PER_SM = 1 +# Draft and draft-extend widths that take the row-split fc (2.2 _FC_SPLIT_MAX_TOKENS) +FC_SPLIT_MAX_TOKENS = kernels.GATHER_MAX_TOKENS +# Rows of a draft step the vocabulary exchange serves (2.2 DraftVocabExchange.MAX_ROWS) +MAX_ROWS = 2 + + +def fc_split_slice( + hidden: int, rank: int, world: int +) -> Optional[tuple[bool, int, int]]: + """(uses the hidden half, start inside the half, width) of this rank's fc input + columns, or None when TP ``world`` does not split them.""" + width = 2 * hidden // world + if world == 1 or (2 * hidden) % world != 0 or hidden % width != 0: + return None + start = rank * width + return (start >= hidden, start % hidden, width) + + +def unsupported_reason( + *, + hidden: int, + world: int, + rank: int, + vocab_size: int, + embed_shard: torch.Tensor, + lm_head_shard: torch.Tensor, +) -> Optional[str]: + """Why the boundary cannot serve this draft, or None (2.2's MKD and vocabulary-exchange + checks that the tensors decide; the caller checks the logits processor: untied + embeddings, an unquantized lm_head, no logit scaling or softcapping).""" + if world not in kernels.WORLDS: + return f"tp_size {world}" + if hidden != kernels.HIDDEN: + return f"hidden size {hidden}" + if fc_split_slice(hidden, rank, world) is None: + return f"the fc does not split at TP {world}" + sms = torch.cuda.get_device_properties(embed_shard.device).multi_processor_count + if kernels.draft_fc_partial_rows(hidden, 2 * hidden // world, sms) is None: + return "the fused fc norm + slice GEMV does not fit in shared memory" + for name, w in (("embedding", embed_shard), ("lm_head", lm_head_shard)): + if w.dtype != torch.bfloat16: + return f"{name} dtype {w.dtype}" + if embed_shard.shape != lm_head_shard.shape: + return "embedding and lm_head shards differ" + if embed_shard.shape[0] % 8 != 0 or embed_shard.shape[1] % 8 != 0: + return "shard shape not 16-byte tileable" + if embed_shard.shape[0] * world != vocab_size: + return "full-vocabulary shards with padding rows" + return None + + +def _clear_negative_zero(rows: torch.Tensor) -> torch.Tensor: + # The exchanged embedding row has -0.0 cleared (vocab_argmax_pack); memo from the same row. + bits = rows.view(torch.int16) + return torch.where(bits == -32768, torch.zeros_like(bits), bits).view( + torch.bfloat16 + ) + + +def _source_digests() -> str: + return " ".join( + f"{path.name}={hashlib.sha256(path.read_bytes()).hexdigest()[:16]}" + for path in sorted(kernels.SRC.iterdir()) + if path.suffix in (".cu", ".cuh") and path.stem.startswith(("mkd_", "draft_fc")) + ) + + +class MkdBoundary: + """Exchange buffers, the memo table, the split fc and the per-step launches of one + rank. Construct on every rank of the TP group (collectives) after the draft shares the + target's embedding and lm_head, before any capture. + + ``norm_embedding`` / ``norm_hidden``: the MTP model's pre-fc Gemma RMSNorms (their + ``weight``, ``variance_epsilon``, and the module itself for splits over 2 rows); + ``embed``: the vocabulary-parallel embedding lookup ([n] ids -> [n, hidden], with its + all-reduce), which ``fill_fc`` runs.""" + + def __init__( + self, + *, + fc_weight: torch.Tensor, + norm_embedding, + norm_hidden, + embed_shard: torch.Tensor, + vocab_start: int, + lm_head_shard: torch.Tensor, + embed: Callable[[torch.Tensor], torch.Tensor], + rank: int, + world: int, + device_group, + cpu_group, + num_steps: int, + memo: bool = True, + extend_fc: bool = True, + head: bool = True, + ) -> None: + import torch.distributed as dist + + dev = fc_weight.device + hidden = int(fc_weight.shape[0]) + self.hidden, self.world, self.rank, self.num_steps = ( + hidden, + world, + rank, + num_steps, + ) + self.fc_weight, self.norm_embedding, self.norm_hidden = ( + fc_weight, + norm_embedding, + norm_hidden, + ) + self._embed = embed + self._device_group = device_group + sms = torch.cuda.get_device_properties(dev).multi_processor_count + # the MTP model's row-split fc: this rank's input-column slice and its GEMV rows + self.fc_split_slice = fc_split_slice(hidden, rank, world) + use_hidden, start, width = self.fc_split_slice + col = start + (hidden if use_hidden else 0) + self._fc_shard = fc_weight[:, col : col + width].contiguous() + self._fc_gemv_rows = kernels.draft_fc_partial_rows(hidden, width, sms) + self._fc_gather = kernels.FcGather( + world=world, rank=rank, group=device_group, cpu_group=cpu_group, device=dev + ) + self.is_hidden_rank = use_hidden + self.slice_start, self.width = start, width + self.n_embed = hidden // width + self.memo = memo + rows = int(embed_shard.shape[0]) + self.embed_weight, self.local_vocab = embed_shard, rows + if vocab_start != rank * rows: + raise RuntimeError( + "MKD draft boundary: vocabulary shard is not the rank-th equal slice" + ) + # Full vocabulary: candidate position p (rank-major) is vocabulary id p. + self.hot_ids = torch.arange(rows * world, dtype=torch.int64).to(dev) + self.cand = self._build_memo() if memo else embed_shard + + words = kernels.mkd_xchg_words(hidden, world, memo) + self._buf, self._handle, self.mc = kernels.symmetric_words( + words, device_group, dev + ) + self.uc = self._buf.data_ptr() + torch.cuda.synchronize(dev) + dist.barrier(group=cpu_group) + + self.epoch_done = torch.tensor([1, 0], dtype=torch.int32, device=dev) + self.own_partial = torch.zeros(1, hidden, dtype=torch.float32, device=dev) + # [CTA counter, epoch flag] of the side-stream hidden partial (mkd_hidden_partial). + self.partial_sync = torch.zeros(2, dtype=torch.int32, device=dev) + # Per-CTA shard-argmax partials for large (full-vocabulary) shards. + self.scan_part = torch.zeros( + kernels.SLOTS * kernels.CHUNKS, dtype=torch.int64, device=dev + ) + self.out_partials = torch.zeros( + world - self.n_embed, hidden, dtype=torch.float32, device=dev + ) + self.out_cand = torch.empty( + 1, hidden, dtype=torch.float32 if memo else torch.bfloat16, device=dev + ) + self.fc_out = torch.empty(1, hidden, dtype=torch.bfloat16, device=dev) + # Memo mode: the exchange writes the next step's fc output here directly. + # Zeros: graph-capture warmups read it before any exchange has written it. + self.fc_buf = ( + torch.zeros(1, hidden, dtype=torch.bfloat16, device=dev) if memo else None + ) + # The draft extend's exchange only needs the token; its payload lands here unused. + self.ext_cand = torch.empty_like(self.out_cand) + self.ext_partials = torch.empty_like(self.out_partials) + self.side_stream = torch.cuda.Stream(device=dev) + self.lm_head_weight = lm_head_shard + self._install_head(head, dev) + # extend_fc: the draft extend's exchange writes fc_buf for the next draft's first + # step. input_seq counts calls that produce a draft input; fc_buf_key is the + # input_seq whose extend wrote fc_buf. + self.extend_fc = memo and extend_fc + self.input_seq = 0 + self.fc_buf_key: Optional[int] = None + # armed: this draft step ends in the exchange; pending: the next step's fc reads it; + # last_step: no draft step follows, so the exchange carries the token alone. + self.armed = False + self.pending = False + self.step = 0 + self.last_step = False + self.partial_pending = False + # the vocabulary exchange's per-step outputs (DraftVocabExchange) + self.num_rows = 1 + self.topk_index = [ + torch.empty(MAX_ROWS, 1, dtype=torch.int64, device=dev) + for _ in range(num_steps) + ] + self.topk_p = [ + torch.empty(MAX_ROWS, 1, dtype=torch.float32, device=dev) + for _ in range(num_steps) + ] + logger.info( + "MKD draft boundary active (tp=%d, %s candidates, %.1f MB exchange buffer); " + "sources %s.", + world, + "memo" if memo else "embedding", + words * 8 / 2**20, + _source_digests(), + ) + + @classmethod + def maybe_create(cls, *, vocab_size: int, **kwargs) -> Optional[MkdBoundary]: + """The boundary, or None (logged) when ``unsupported_reason`` refuses it; every + rank decides alike.""" + fc_weight = kwargs["fc_weight"] + reason = unsupported_reason( + hidden=int(fc_weight.shape[0]), + world=kwargs["world"], + rank=kwargs["rank"], + vocab_size=vocab_size, + embed_shard=kwargs["embed_shard"], + lm_head_shard=kwargs["lm_head_shard"], + ) + if reason is not None: + logger.warning( + "MKD draft boundary disabled (%s); using the stock path.", reason + ) + return None + return cls(**kwargs) + + # ---------------------------------------------------------------- memo table + + def _build_memo(self) -> torch.Tensor: + import torch.distributed as dist + + emb = self.embed_weight + rows = emb.shape[0] + memo = torch.empty(rows, self.hidden, dtype=torch.float32, device=emb.device) + norm = self.norm_embedding + for i in range(0, rows, _MEMO_BLOCK): + block = _clear_negative_zero(emb[i : i + _MEMO_BLOCK].clone()) + kernels.mkd_fc_memo( + block, + self.fc_weight, + norm.weight.data, + norm.variance_epsilon, + self.world, + memo[i : i + _MEMO_BLOCK], + ) + bad = self._check_memo(emb, memo) + bad_t = torch.tensor([bad, rows], dtype=torch.int64, device=emb.device) + dist.all_reduce(bad_t, group=self._device_group) + if int(bad_t[0]) != 0: + raise RuntimeError( + f"MKD memo table differs from the split fc on {int(bad_t[0])} of " + f"{int(bad_t[1])} rows" + ) + logger.info( + "MKD memo table: %d rows on this rank (%.1f MB), equal to the split fc on all %d rows.", + rows, + memo.numel() * 4 / 2**20, + int(bad_t[1]), + ) + return memo + + def _check_memo(self, emb: torch.Tensor, memo: torch.Tensor) -> int: + """Rows whose memo differs from the split-fc kernels (every row, all slices).""" + norm = self.norm_embedding + sms = torch.cuda.get_device_properties(emb.device).multi_processor_count + width = self.width + gemv_rows = kernels.draft_fc_partial_rows(self.hidden, width, sms) + shards = [ + self.fc_weight[:, s * width : (s + 1) * width].contiguous() + for s in range(self.n_embed) + ] + part = torch.empty(1, self.hidden, dtype=torch.float32, device=emb.device) + bad = torch.zeros((), dtype=torch.int64, device=emb.device) + for v in range(emb.shape[0]): + x = _clear_negative_zero(emb[v : v + 1].clone()) + acc = None + for s in range(self.n_embed): + kernels.draft_fc_partial( + x, + norm.weight.data, + norm.variance_epsilon, + shards[s], + s * width, + part, + gemv_rows, + ) + acc = part.clone() if acc is None else acc + part + bad += (acc.view(torch.int32) != memo[v : v + 1].view(torch.int32)).any() + return int(bad) + + # ---------------------------------------------------------------- draft lm_head + + def _install_head(self, enabled: bool, dev: torch.device) -> None: + self.head_part = None + self.head_used = False + if not enabled: + return + head = self.lm_head_weight + if ( + head.dtype != torch.bfloat16 + or head.dim() != 2 + or head.shape[1] != self.hidden + or head.shape[0] > 65536 + ): + logger.warning( + "MKD draft head disabled (lm_head %s %s).", + head.dtype, + tuple(head.shape), + ) + return + sms = torch.cuda.get_device_properties(dev).multi_processor_count + self.head_part = torch.zeros( + _HEAD_CTAS_PER_SM * sms, dtype=torch.int32, device=dev + ) + logger.info( + "MKD draft head active: %d x %d shard, %d CTAs.", + head.shape[0], + head.shape[1], + self.head_part.numel(), + ) + + def head(self, hidden_states: torch.Tensor) -> torch.Tensor: + """The one-row draft logits [1, N] bf16 on this rank's shard (the exchange then + reads the per-CTA maxima). For an armed step with the head installed.""" + x = hidden_states.to(torch.bfloat16).contiguous() + logits = torch.empty( + 1, self.lm_head_weight.shape[0], dtype=torch.bfloat16, device=x.device + ) + # Alternate the row walk per draft step: the next step re-reads first what is still in L2 + # (T1: -1.9 to -2.4 us per call on the TP8 shard). + kernels.mkd_draft_head( + x, self.lm_head_weight, logits, self.head_part, reverse=self.step % 2 == 1 + ) + self.head_used = True + return logits + + # ---------------------------------------------------------------- per step + + def begin_step(self, step: int, num_rows: int) -> None: + """A draft step of ``num_rows`` rows starts (the draft loop forwards steps + 0 .. num_steps - 2); with extend fc, step 0's fc is ``fc_buf``.""" + self.num_rows = num_rows + self.armed = num_rows == 1 + self.step = step + self.last_step = step == self.num_steps - 2 + if step == 0: + # With extend_fc the draft caller guarantees fc_buf holds this step's fc. + self.pending = self.extend_fc and num_rows == 1 + + def hidden_partial(self, hidden_states: torch.Tensor) -> None: + """This rank's hidden-slice fc partial for the next draft step, on a side stream; + the exchange waits on its flag, not on a stream join.""" + if not self.is_hidden_rank or self.last_step: + return + norm = self.norm_hidden + self.side_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(self.side_stream): + kernels.mkd_hidden_partial( + hidden_states, + self._fc_shard, + norm.weight.data, + norm.variance_epsilon, + self.slice_start, + self.own_partial, + self.epoch_done, + self.partial_sync, + self.world, + ) + self.partial_pending = True + + def _join_side_streams(self) -> None: + # Joins only end graph capture cleanly: the exchange syncs on flags; a join before + # the exchange would cost its early launch. + if self.partial_pending: + torch.cuda.current_stream().wait_stream(self.side_stream) + self.partial_pending = False + + def exchange( + self, + local_logits: torch.Tensor, + topk_index: torch.Tensor, + topk_p: torch.Tensor, + draft_tokens: Optional[torch.Tensor], + positions: torch.Tensor, + draft_column: int, + mrope_positions: Optional[torch.Tensor], + ) -> None: + """The step-boundary exchange: the token into ``topk_index`` / ``topk_p`` and + ``draft_tokens[:, draft_column]``, positions (and M-RoPE positions) + 1, and unless + this is the last step the next step's fc into ``fc_buf``.""" + partial_flag = ( + self.partial_sync[1:] + if self.partial_pending and not self.last_step + else None + ) + kernels.mkd_xchg( + local_logits, + self.cand, + self.hot_ids, + self.own_partial, + self.epoch_done, + self.out_cand, + self.out_partials, + topk_index, + topk_p, + draft_tokens, + positions, + mrope_positions, + self.mc, + self.uc, + self.rank, + self.world, + draft_column, + kernels.NAN_KEY_SPLIT_ARGMAX, + fc_out=self.fc_buf, + token_only=self.last_step, + partial_flag=partial_flag, + scan_part=self.scan_part, + head_part=self.head_part if self.head_used else None, + ) + self.head_used = False + self.pending = not self.last_step + if self.last_step: + self._join_side_streams() + + def finish_step( + self, + local_logits: torch.Tensor, + positions: torch.Tensor, + draft_tokens: Optional[torch.Tensor], + step: int, + draft_column: int, + mrope_positions: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """(topk_p, topk_index) of an armed step from its shard-local logits, through the + exchange (DraftVocabExchange.finish_step on the one-launch boundary).""" + m = self.num_rows + topk_index, topk_p = self.topk_index[step][:m], self.topk_p[step][:m] + if not self.armed: + raise RuntimeError( + "MKD draft boundary: only armed (one-row) steps take the exchange" + ) + self.exchange( + local_logits, + topk_index, + topk_p, + draft_tokens, + positions, + draft_column, + mrope_positions, + ) + self.armed = False + return topk_p, topk_index + + def end(self) -> None: + """The draft loop's steps are over (DraftVocabExchange.end: the model's handoffs, + which the caller owns here).""" + + def fc(self) -> torch.Tensor: + """The draft fc output [1, H] of the step after an exchange.""" + self.pending = False + if self.fc_buf is not None: + return self.fc_buf + norm = self.norm_embedding + out = torch.empty_like(self.fc_out) + kernels.mkd_fc_sum( + self.out_cand, + self.out_cand if self.memo else None, + self.out_partials, + self.fc_weight, + norm.weight.data, + norm.variance_epsilon, + out, + self.world, + ) + return out + + # ---------------------------------------------------------------- the draft extend + + def is_local(self, logits: torch.Tensor) -> bool: + """Whether ``logits`` are this rank's shard of at most MAX_ROWS rows.""" + return logits.shape[-1] == self.local_vocab and 0 < logits.shape[0] <= MAX_ROWS + + def select_extend( + self, local_logits: torch.Tensor, hidden_states: Optional[torch.Tensor] = None + ) -> tuple[torch.Tensor, torch.Tensor]: + """Draft-extend argmax (torch.argmax semantics) of one row's shard-local logits.""" + if local_logits.shape[0] != 1: + raise RuntimeError("MKD draft boundary: the extend exchange serves one row") + return self.exchange_extend(local_logits, hidden_states) + + def exchange_extend( + self, local_logits: torch.Tensor, hidden_states: Optional[torch.Tensor] + ) -> tuple[torch.Tensor, torch.Tensor]: + """Draft-extend token (torch.argmax NaN rule) through the same one-launch exchange; + with extend_fc it also writes the next draft's first fc into fc_buf.""" + topk_index = torch.empty((1, 1), dtype=torch.int64, device=local_logits.device) + topk_p = torch.empty((1, 1), dtype=torch.float32, device=local_logits.device) + payload = ( + self.extend_fc and hidden_states is not None and hidden_states.shape[0] == 1 + ) + if payload and self.is_hidden_rank: + norm = self.norm_hidden + kernels.draft_fc_partial( + hidden_states.contiguous(), + norm.weight.data, + norm.variance_epsilon, + self._fc_shard, + self.slice_start, + self.own_partial, + self._fc_gemv_rows, + ) + kernels.mkd_xchg( + local_logits, + self.cand, + self.hot_ids, + self.own_partial, + self.epoch_done, + self.ext_cand, + self.ext_partials, + topk_index, + topk_p, + None, + None, + None, + self.mc, + self.uc, + self.rank, + self.world, + 0, + kernels.NAN_KEY_TORCH_ARGMAX, + fc_out=self.fc_buf if payload else None, + token_only=not payload, + scan_part=self.scan_part, + ) + self.fc_buf_key = self.input_seq if payload else None + return topk_p, topk_index + + def note_draft_input(self) -> None: + """A draft extend (after prefill or decode) is about to produce the next draft input.""" + self.input_seq += 1 + + def take_fc_buf(self) -> bool: + """True when fc_buf holds the first fc of the coming draft: its extend produced the + current draft input. Consumes it (the draft's own exchanges overwrite fc_buf).""" + holds = self.fc_buf_key is not None and self.fc_buf_key == self.input_seq + self.fc_buf_key = None + return holds + + def fill_fc(self, token_ids: torch.Tensor, hidden_states: torch.Tensor) -> None: + """The fc of a draft's first step into fc_buf, when no extend exchange produced it.""" + self.pending = False + embeds = self._embed(token_ids.flatten()) + self.fc_buf.copy_(self.split_fc(embeds, hidden_states)) + + # ---------------------------------------------------------------- the row-split fc + + def split_fc( + self, input_embeds: torch.Tensor, hidden_states: torch.Tensor + ) -> torch.Tensor: + """The MTP fc [n, H] bf16 of n <= FC_SPLIT_MAX_TOKENS rows: the fp32 partial of + this rank's input-column slice, all-gathered and summed in rank order, rounded once + to bf16 (the replicated fc's rounding points).""" + if not 0 < input_embeds.shape[0] <= FC_SPLIT_MAX_TOKENS: + raise ValueError( + f"split fc: {input_embeds.shape[0]} rows (1-{FC_SPLIT_MAX_TOKENS})" + ) + use_hidden, start, width = self.fc_split_slice + x = hidden_states if use_hidden else input_embeds + norm = self.norm_hidden if use_hidden else self.norm_embedding + if ( + self._fc_gemv_rows is not None + and 0 < x.shape[0] <= kernels.MAX_TOKENS + and x.shape[0] == input_embeds.shape[0] + ): + partial = kernels.draft_fc_partial( + x.contiguous(), + norm.weight.data, + norm.variance_epsilon, + self._fc_shard, + start, + x.new_empty((x.shape[0], self.hidden), dtype=torch.float32), + self._fc_gemv_rows, + ) + return self._gather_partials(partial) + x = norm(x) + if use_hidden and x.shape[0] != input_embeds.shape[0]: + rows = min(x.shape[0], input_embeds.shape[0]) + slot = x.new_zeros((input_embeds.shape[0], x.shape[1])) + slot[:rows] = x[:rows] + x = slot + partial = torch.mm( + x[:, start : start + width], self._fc_shard.T, out_dtype=torch.float32 + ) + return self._gather_partials(partial) + + def _gather_partials(self, partial: torch.Tensor) -> torch.Tensor: + gathered = self._fc_gather(partial) + out = partial.new_empty(partial.shape, dtype=torch.bfloat16) + return kernels.ordered_partial_sum(gathered, self.world, out) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index bc448d38..4ac0b4c8 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -30,7 +30,7 @@ # The MTP draft's EAGLE entries and kernel groups (qwen3_5/draft), requested in the same # variable; the draft spec's build reads them, the target's ignores them DRAFT_ENTRIES: tuple[str, ...] = ("draft_loop", "draft_extend") -DRAFT_SLOTS: tuple[str, ...] = ("draft_moe", "draft_q1") +DRAFT_SLOTS: tuple[str, ...] = ("draft_moe", "draft_q1", "draft_xchg") SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py index d298d829..2ef16834 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -"""The MTP draft's SM103 (GB300) kernels: the single-launch draft MoE (``moe``) and -the q1 split-KV decode attention (``q1``).""" +"""The MTP draft's SM103 (GB300) kernels: the single-launch draft MoE (``moe``), the +q1 split-KV decode attention (``q1``) and the draft step boundary's exchange, fc and +head (``xchg``).""" diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_fc.cu b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_fc.cu new file mode 100644 index 00000000..ce142c10 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_fc.cu @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Adapted from SGLang (Apache-2.0): python/sglang/kernels/jit/csrc/speculative/draft_fc_partial.cuh +// (branch nvidia/qwen35-agentx-gb300), device code unchanged; fc_gather is new (see below). +// +// Row-parallel MTP fc for draft-width batches: per token, the Gemma RMSNorm of +// one input half (fp32 statistics, bf16 output as the stock norm stores it), +// then this rank's slice of the fc GEMV with an fp32 result per output row. +// One CTA per SM owns a contiguous block of weight rows; the static rows are +// copied into shared memory before the PDL wait, so they stream in while the +// previous grid finishes. +// +// fc_gather moves the fp32 partials of every rank into the layout ordered_partial_sum reads, in +// place of SGLang's multimem all-gather that 2.2 used (bytes only, no arithmetic). It uses the +// mkd_xchg transport: epoch-tagged 64-bit words through the multicast mapping, three slots +// rotating by epoch, arrival read from the data, bounded waits that trap. +// +// Built with NVRTC (MKD_WORLD = the TP size, 4 or 8; DRAFT_FC_MAX_ROWS = weight rows a CTA, +// draft_fc_partial_rows of 2.2): the kernels are device functions, and the extern "C" entries +// at the end instantiate them for hidden 4096, 4096 fc rows, 1 or 2 tokens, PDL on. + +#include "mkd_common.cuh" + +#ifndef MKD_WORLD +#define MKD_WORLD 8 +#endif +#ifndef DRAFT_FC_MAX_ROWS +#define DRAFT_FC_MAX_ROWS 26 +#endif + +namespace sglang { + +namespace draft_fc { + +// global -> shared bulk copy completed on an smem mbarrier (PTX ISA cp.async.bulk); +// size and both addresses must be 16-byte aligned. +SGL_DEVICE void bulk_g2s(void* smem_dst, const void* gmem_src, uint32_t bytes, uint64_t* bar) { + asm volatile( + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"( + device::ptx::to_shared(smem_dst)), + "l"(gmem_src), + "r"(bytes), + "r"(device::ptx::to_shared(bar)) + : "memory"); +} + +} // namespace draft_fc + +struct DraftFcPartialParams { + const bf16_t* x; // [M, H] input half, written by an earlier grid + const bf16_t* __restrict__ norm_w; // [H] Gemma norm weight (scale = 1 + w) + const bf16_t* __restrict__ w; // [N, W] this rank's fc input-column slice + float* out; // [M, N] fp32 partial + float eps; + int32_t col_start; // first column of the slice inside the half + int32_t rows; // M +}; + +/** + * \brief Fused RMSNorm + slice GEMV. \tparam kH half width (norm length), + * \tparam kW slice width, \tparam kN output rows, \tparam kMaxRows weight rows per CTA, + * \tparam kTokens rows of x (exact). The whole CTA computes the norm statistics of + * every token in one pass; each warp then holds one weight row in registers and + * dots it with every normalized token. + */ +template +SGL_DEVICE void draft_fc_partial_kernel(const DraftFcPartialParams& p) { + using namespace device; + constexpr int kThreads = 256; + constexpr int kWarps = kThreads / 32; + using vec_t = AlignedVector; + static_assert(kH % (8 * kThreads) == 0, "norm length must tile the CTA in 16-byte vectors"); + static_assert(kW % (8 * kWarpThreads) == 0, "slice width must tile a warp in 16-byte vectors"); + constexpr int kNormVecs = kH / (8 * kThreads); + constexpr int kDotVecs = kW / (8 * kWarpThreads); + constexpr int kRowBytes = kW * static_cast(sizeof(bf16_t)); + + extern __shared__ __align__(128) uint8_t smem[]; // [kMaxRows, kW] weight rows + __shared__ alignas(16) bf16_t s_x[kTokens][kW]; + __shared__ float s_red[kTokens][kWarps]; + __shared__ uint64_t bar; + + const int warp = static_cast(threadIdx.x / kWarpThreads); + const int lane = static_cast(get_lane_id()); + const int G = static_cast(gridDim.x); + const int c = static_cast(blockIdx.x); + const int base = kN / G, rem = kN % G; + const int r0 = c * base + min(c, rem); + const int nrows = base + (c < rem ? 1 : 0); + + if (threadIdx.x == 0) { + ptx::mbar_init(&bar, 1); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); + if (nrows > 0) { + ptx::mbar_arrive_expect_tx(&bar, static_cast(nrows) * kRowBytes); + draft_fc::bulk_g2s(smem, p.w + static_cast(r0) * kW, static_cast(nrows) * kRowBytes, &bar); + } + } + __syncthreads(); + PDLWaitPrimary(); + + // Norm statistics of every token over the whole half. +#pragma unroll + for (int m = 0; m < kTokens; ++m) { + const bf16_t* xm = p.x + static_cast(m) * kH; + float ss = 0.0f; +#pragma unroll + for (int k = 0; k < kNormVecs; ++k) { + vec_t v; + v.load(xm, k * kThreads + static_cast(threadIdx.x)); +#pragma unroll + for (int j = 0; j < 8; ++j) { + const float f = cast(v[j]); + ss = fmaf(f, f, ss); + } + } + ss = warp::reduce_sum(ss); + if (lane == 0) s_red[m][warp] = ss; + } + __syncthreads(); + // Normalized slices, rounded to bf16 as the norm output is stored. +#pragma unroll + for (int m = 0; m < kTokens; ++m) { + float tot = 0.0f; +#pragma unroll + for (int w = 0; w < kWarps; ++w) tot += s_red[m][w]; + const float inv = rsqrtf(tot / static_cast(kH) + p.eps); + const bf16_t* xm = p.x + static_cast(m) * kH + p.col_start; + for (int k = static_cast(threadIdx.x); k < kW / 8; k += kThreads) { + vec_t v, g, o; + v.load(xm, k); + g.load(p.norm_w + p.col_start, k); +#pragma unroll + for (int j = 0; j < 8; ++j) { + o[j] = cast(cast(v[j]) * inv * (1.0f + cast(g[j]))); + } + o.store(s_x[m], k); + } + } + __syncthreads(); + if (nrows > 0) ptx::mbar_wait_parity(&bar, 0); + for (int r = warp; r < nrows; r += kWarps) { + const auto* row = reinterpret_cast(smem + static_cast(r) * kRowBytes); + vec_t wv[kDotVecs]; +#pragma unroll + for (int k = 0; k < kDotVecs; ++k) wv[k].load(row, k * kWarpThreads + lane); +#pragma unroll + for (int m = 0; m < kTokens; ++m) { + float acc = 0.0f; +#pragma unroll + for (int k = 0; k < kDotVecs; ++k) { + vec_t xv; + xv.load(s_x[m], k * kWarpThreads + lane); +#pragma unroll + for (int j = 0; j < 8; ++j) acc = fmaf(cast(wv[k][j]), cast(xv[j]), acc); + } + acc = warp::reduce_sum(acc); + if (lane == 0) p.out[static_cast(m) * kN + r0 + r] = acc; + } + } + PDLTriggerSecondary(); +} + +namespace fc_gather { + +inline constexpr int kSlots = 3; +inline constexpr uint32_t kSpinLimit = 1u << 27; +inline constexpr int kMaxTokens = 16; // the fc split's widest batch (2.2 _FC_SPLIT_MAX_TOKENS) + +SGL_DEVICE void mc_store(uint64_t* addr, uint32_t payload, uint32_t tag) { + const uint64_t w = static_cast(payload) | (static_cast(tag) << 32); + asm volatile("multimem.st.relaxed.sys.global.b64 [%0], %1;" ::"l"(addr), "l"(w) : "memory"); +} + +} // namespace fc_gather + +struct FcGatherParams { + const float* partial; // [m, kH] this rank's fp32 partial, written by an earlier grid + float* out; // [m, world * kH]: rank r's row i at columns [r * kH, (r + 1) * kH) + uint64_t* mc; // multicast base of the gather buffer [kSlots, world, kMaxTokens, kH] + const uint64_t* uc; // this rank's own copy of it + uint32_t* epoch; // [1] call counter shared by all ranks, starts at 1 + uint32_t* done; // [1] CTA arrival counter, zero between calls + int32_t rank; + int32_t m; +}; + +// Every CTA pushes its share of this rank's packet, then polls its share of all packets; no CTA +// waits on another CTA of its rank, so the launch needs no grid barrier (keep the grid resident). +template +SGL_DEVICE void fc_gather_kernel(const FcGatherParams& p) { + constexpr int64_t kPacket = static_cast(fc_gather::kMaxTokens) * kH; + __shared__ uint32_t s_tag; + if (threadIdx.x == 0) { + uint32_t e; + asm volatile("ld.relaxed.gpu.global.u32 %0, [%1];" : "=r"(e) : "l"(p.epoch) : "memory"); + s_tag = e; + } + __syncthreads(); + const uint32_t tag = s_tag; + const int64_t slot = static_cast(tag % fc_gather::kSlots); + const int64_t n = static_cast(p.m) * kH; + const int64_t stride = static_cast(gridDim.x) * blockDim.x; + const int64_t first = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + uint64_t* dst = p.mc + (slot * kWorld + p.rank) * kPacket; + for (int64_t i = first; i < n; i += stride) { + fc_gather::mc_store(dst + i, __float_as_uint(p.partial[i]), tag); + } + for (int64_t i = first; i < kWorld * n; i += stride) { + const int64_t r = i / n, j = i % n; + const uint64_t* src = p.uc + (slot * kWorld + r) * kPacket + j; + uint64_t w; + uint32_t spins = 0; + do { + asm volatile("ld.relaxed.sys.global.u64 %0, [%1];" : "=l"(w) : "l"(src) : "memory"); + if (++spins > fc_gather::kSpinLimit) __trap(); + } while (static_cast(w >> 32) != tag); + p.out[(j / kH) * kWorld * kH + r * kH + j % kH] = __uint_as_float(static_cast(w)); + } + __syncthreads(); + if (threadIdx.x == 0) { + __threadfence(); + if (atomicAdd(p.done, 1u) == gridDim.x - 1) { + *p.done = 0u; + *p.epoch = tag + 1; + } + } +} + + +} // namespace sglang + +using sglang::DraftFcPartialParams; +using sglang::FcGatherParams; + +extern "C" __global__ void __launch_bounds__(256, 1) draft_fc_partial_m1(const DraftFcPartialParams __grid_constant__ p) { + sglang::draft_fc_partial_kernel<4096, 2 * 4096 / MKD_WORLD, 4096, DRAFT_FC_MAX_ROWS, 1, true>(p); +} + +extern "C" __global__ void __launch_bounds__(256, 1) draft_fc_partial_m2(const DraftFcPartialParams __grid_constant__ p) { + sglang::draft_fc_partial_kernel<4096, 2 * 4096 / MKD_WORLD, 4096, DRAFT_FC_MAX_ROWS, 2, true>(p); +} + +extern "C" __global__ void __launch_bounds__(512) fc_gather(const FcGatherParams __grid_constant__ p) { + sglang::fc_gather_kernel<4096, MKD_WORLD>(p); +} diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_common.cuh b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_common.cuh new file mode 100644 index 00000000..293f72c6 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_common.cuh @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Adapted from SGLang (Apache-2.0): python/sglang/kernels/jit/include/sgl_kernel/{utils,type,vec, +// warp,mbarrier}.cuh (branch nvidia/qwen35-agentx-gb300), the subset the MKD draft-boundary +// kernels use. +// +// NVRTC prelude of the MTP draft step-boundary kernels (mkd_xchg.cu, mkd_fc.cu, mkd_head.cu, +// draft_fc.cu): the same definitions as SGLang's JIT headers for sm_103 (PDL on, 32-lane warps), +// so the kernel bodies compile unchanged. + +#pragma once + +#include + +// NVRTC has no +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef long long int64_t; +typedef unsigned long long uint64_t; + +#define SGL_DEVICE __forceinline__ __device__ + +namespace sglang { + +using fp32_t = float; +using bf16_t = __nv_bfloat16; + +namespace device { + +inline constexpr uint32_t kWarpThreads = 32u; + +SGL_DEVICE uint32_t get_lane_id() { + uint32_t lane_id; + asm volatile("mov.u32 %0, %%laneid;" : "=r"(lane_id)); + return lane_id; +} + +// griddepcontrol.wait / launch_dependents (sm_90+), as SGLang's PDLWaitPrimary / PDLTriggerSecondary +template +SGL_DEVICE void PDLWaitPrimary() { + if constexpr (kUsePDL) { + asm volatile("griddepcontrol.wait;" ::: "memory"); + } +} + +template +SGL_DEVICE void PDLTriggerSecondary() { + if constexpr (kUsePDL) { + asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); + } +} + +namespace details { + +template +struct sized_int {}; + +template <> +struct sized_int { + using type = uint16_t; +}; + +} // namespace details + +template +struct alignas(sizeof(T) * N) AlignedStorage { + T data[N]; +}; + +// SGLang's AlignedVector: N elements loaded / stored as one aligned vector +template +struct AlignedVector { + private: + using element_t = typename details::sized_int::type; + using storage_t = AlignedStorage; + + public: + SGL_DEVICE void load(const void* ptr, int64_t offset = 0) { + m_storage = reinterpret_cast(ptr)[offset]; + } + SGL_DEVICE void store(void* ptr, int64_t offset = 0) const { + reinterpret_cast(ptr)[offset] = m_storage; + } + SGL_DEVICE T& operator[](int idx) { + return reinterpret_cast(&m_storage)[idx]; + } + SGL_DEVICE T operator[](int idx) const { + return reinterpret_cast(&m_storage)[idx]; + } + + private: + storage_t m_storage; +}; + +// SGLang's DTypeTrait conversions: bf16 -> fp32 exact, fp32 -> bf16 round to nearest even +template +SGL_DEVICE To cast(const From& value); + +template <> +SGL_DEVICE fp32_t cast(const bf16_t& value) { + return __bfloat162float(value); +} + +template <> +SGL_DEVICE bf16_t cast(const fp32_t& value) { + return __float2bfloat16_rn(value); +} + +namespace warp { + +// SGLang's warp::reduce_sum<32, 1> for fp32: xor butterfly 16, 8, 4, 2, 1, value + other +template +SGL_DEVICE T reduce_sum(T value) { +#pragma unroll + for (uint32_t mask = kWarpThreads / 2; mask >= 1; mask >>= 1) { + value = static_cast(value + __shfl_xor_sync(0xffffffffu, value, mask, kWarpThreads)); + } + return value; +} + +} // namespace warp + +namespace ptx { + +template +SGL_DEVICE uint32_t to_shared(T* ptr) { + return static_cast(__cvta_generic_to_shared(ptr)); +} + +SGL_DEVICE void mbar_init(uint64_t* bar, uint32_t count) { + asm volatile("mbarrier.init.shared.b64 [%0], %1;" ::"r"(to_shared(bar)), "r"(count)); +} + +SGL_DEVICE void mbar_arrive_expect_tx(uint64_t* bar, uint32_t bytes) { + asm volatile("mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;" ::"r"(to_shared(bar)), "r"(bytes)); +} + +SGL_DEVICE void mbar_wait_parity(uint64_t* bar, uint32_t parity) { + asm volatile( + "{\n\t.reg .pred p;\n\t" + "WAIT_%=: mbarrier.try_wait.parity.shared.b64 p, [%0], %1;\n\t" + "@!p bra WAIT_%=;\n\t}\n" ::"r"(to_shared(bar)), + "r"(parity)); +} + +} // namespace ptx + +} // namespace device + +} // namespace sglang diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_fc.cu b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_fc.cu new file mode 100644 index 00000000..b7856bea --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_fc.cu @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Adapted from SGLang (Apache-2.0): python/sglang/kernels/jit/csrc/speculative/mkd_draft_fc.cuh +// (branch nvidia/qwen35-agentx-gb300), device code unchanged. +// +// MTP draft fc for one token from an exchanged input, bitwise equal to the row-split fc +// (draft_fc_partial on each rank's input-column slice, fp32 partials summed in rank order, +// one bf16 rounding). +// +// sum: every rank computes the partials of the first kEmbed slices (the embedding half) +// itself from the exchanged embedding row, with the weight rows bulk-copied into +// shared memory before the PDL wait, then adds the received partials of the hidden +// slices in rank order. With a memo row (T = the embedding-half partials already +// summed in rank order) the embedding-half dots are skipped and the sum starts at T. +// memo: builds those T rows for a list of embedding rows with the same arithmetic. +// +// Per partial: Gemma RMSNorm of the half (256-thread statistics, bf16 output), then one warp +// per weight row over the slice in 16-byte lane-strided vectors, warp::reduce_sum; this is +// the arithmetic of draft_fc_partial_kernel. +// +// Built with NVRTC (MKD_WORLD = the TP size, 4 or 8): the kernels are device functions, and the +// extern "C" entries at the end instantiate them for hidden 4096 (MKD_FC_MAX_ROWS = 27 fc rows a +// CTA, the 2.2 MAX_ROWS; memo blocks of 8 rows), PDL on where 2.2 launches with PDL. + +#include "mkd_common.cuh" + +#ifndef MKD_WORLD +#define MKD_WORLD 8 +#endif +#ifndef MKD_FC_MAX_ROWS +#define MKD_FC_MAX_ROWS 27 +#endif + +namespace sglang { + +namespace mkd_fc { + +SGL_DEVICE void bulk_g2s(void* smem_dst, const void* gmem_src, uint32_t bytes, uint64_t* bar) { + asm volatile( + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"( + device::ptx::to_shared(smem_dst)), + "l"(gmem_src), + "r"(bytes), + "r"(device::ptx::to_shared(bar)) + : "memory"); +} + +/// \brief Gemma RMSNorm of one row x[kH] into s_x (bf16), with draft_fc_partial's +/// statistics order: 256 threads, thread t owns 16-byte vectors t, t + 256, ... +template +SGL_DEVICE void gemma_norm_row( + const bf16_t* x, const bf16_t* norm_w, float eps, bf16_t* s_x, float* s_red) { + using namespace device; + constexpr int kThreads = 256; + constexpr int kWarps = kThreads / 32; + using vec_t = AlignedVector; + constexpr int kNormVecs = kH / (8 * kThreads); + const int warp = static_cast(threadIdx.x / kWarpThreads); + const int lane = static_cast(get_lane_id()); + float ss = 0.0f; +#pragma unroll + for (int k = 0; k < kNormVecs; ++k) { + vec_t v; + v.load(x, k * kThreads + static_cast(threadIdx.x)); +#pragma unroll + for (int j = 0; j < 8; ++j) { + const float f = cast(v[j]); + ss = fmaf(f, f, ss); + } + } + ss = warp::reduce_sum(ss); + if (lane == 0) s_red[warp] = ss; + __syncthreads(); + float tot = 0.0f; +#pragma unroll + for (int w = 0; w < kWarps; ++w) tot += s_red[w]; + const float inv = rsqrtf(tot / static_cast(kH) + eps); + for (int k = static_cast(threadIdx.x); k < kH / 8; k += kThreads) { + vec_t v, g, o; + v.load(x, k); + g.load(norm_w, k); +#pragma unroll + for (int j = 0; j < 8; ++j) { + o[j] = cast(cast(v[j]) * inv * (1.0f + cast(g[j]))); + } + o.store(s_x, k); + } + __syncthreads(); +} + +/// \brief One slice partial: warp dot of a weight row segment [kW] with s_x[kW]. +template +SGL_DEVICE float slice_dot(const bf16_t* w_row, const bf16_t* s_x) { + using namespace device; + using vec_t = AlignedVector; + constexpr int kDotVecs = kW / (8 * kWarpThreads); + const int lane = static_cast(get_lane_id()); + vec_t wv[kDotVecs]; +#pragma unroll + for (int k = 0; k < kDotVecs; ++k) wv[k].load(w_row, k * kWarpThreads + lane); + float acc = 0.0f; +#pragma unroll + for (int k = 0; k < kDotVecs; ++k) { + vec_t xv; + xv.load(s_x, k * kWarpThreads + lane); +#pragma unroll + for (int j = 0; j < 8; ++j) acc = fmaf(cast(wv[k][j]), cast(xv[j]), acc); + } + return warp::reduce_sum(acc); +} + +} // namespace mkd_fc + +struct MkdFcSumParams { + const bf16_t* emb; // [kH] exchanged embedding row (null with a memo row) + const float* memo; // [kH] memo row T (null without) + const float* hidden_partials; // [kWorld - kEmbed, kH] fp32 partials of the hidden slices + const bf16_t* __restrict__ fc_w; // [kH, 2 * kH] fc weight + const bf16_t* __restrict__ norm_w; // [kH] pre-fc embedding norm weight + bf16_t* out; // [kH] + float eps; +}; + +/** + * \brief fc output of one token. Grid = one CTA per SM, 256 threads; CTA c owns a + * contiguous block of output rows. \tparam kMemo selects the memo-row path. + */ +template +SGL_DEVICE void mkd_fc_sum_kernel(const MkdFcSumParams& p) { + using namespace device; + constexpr int kThreads = 256; + constexpr int kWarps = kThreads / 32; + constexpr int kW = 2 * kH / kWorld; // slice width + constexpr int kEmbed = kH / kW; // embedding-half slices + constexpr int kRowBytes = kH * static_cast(sizeof(bf16_t)); // embedding half of one fc row + static_assert(kW % (8 * 32) == 0 && kH % kW == 0); + + extern __shared__ __align__(128) uint8_t smem[]; // [kMaxRows, kH] embedding-half weight rows + __shared__ alignas(16) bf16_t s_x[kMemo ? 8 : kH]; + __shared__ float s_red[kWarps]; + __shared__ uint64_t bar; + + const int warp = static_cast(threadIdx.x / kWarpThreads); + const int lane = static_cast(get_lane_id()); + const int G = static_cast(gridDim.x); + const int c = static_cast(blockIdx.x); + const int base = kH / G, rem = kH % G; + const int r0 = c * base + min(c, rem); + const int nrows = base + (c < rem ? 1 : 0); + + if constexpr (!kMemo) { + if (threadIdx.x == 0) { + ptx::mbar_init(&bar, 1); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); + if (nrows > 0) { + ptx::mbar_arrive_expect_tx(&bar, static_cast(nrows) * kRowBytes); + // fc_w rows are [embedding half | hidden half]; copy the embedding half of each row. + for (int r = 0; r < nrows; ++r) { + mkd_fc::bulk_g2s( + smem + static_cast(r) * kRowBytes, + p.fc_w + static_cast(r0 + r) * 2 * kH, + kRowBytes, + &bar); + } + } + } + __syncthreads(); + } + PDLWaitPrimary(); + PDLTriggerSecondary(); + if constexpr (!kMemo) { + mkd_fc::gemma_norm_row(p.emb, p.norm_w, p.eps, s_x, s_red); + if (nrows > 0) ptx::mbar_wait_parity(&bar, 0); + } + for (int r = warp; r < nrows; r += kWarps) { + const int j = r0 + r; + float acc; + if constexpr (kMemo) { + acc = p.memo[j]; + } else { + const auto* row = reinterpret_cast(smem + static_cast(r) * kRowBytes); + acc = mkd_fc::slice_dot(row, s_x); +#pragma unroll + for (int s = 1; s < kEmbed; ++s) acc += mkd_fc::slice_dot(row + s * kW, s_x + s * kW); + } + if (lane == 0) { +#pragma unroll + for (int s = 0; s < kWorld - kEmbed; ++s) acc += p.hidden_partials[static_cast(s) * kH + j]; + p.out[j] = cast(acc); + } + } +} + +struct MkdFcMemoParams { + const bf16_t* __restrict__ emb; // [N, kH] embedding rows + const bf16_t* __restrict__ fc_w; // [kH, 2 * kH] + const bf16_t* __restrict__ norm_w; // [kH] + float* memo; // [N, kH] + float eps; + int32_t n; +}; + +/** + * \brief Memo rows T[i] = embedding-half partials of emb[i] summed in rank order. + * A CTA normalizes kB rows into shared memory, then each warp loads one weight-row slice + * into registers and dots it with all kB rows (the per-row arithmetic is unchanged). + */ +template +SGL_DEVICE void mkd_fc_memo_kernel(const MkdFcMemoParams& p) { + using namespace device; + using vec_t = AlignedVector; + constexpr int kWarps = 8; + constexpr int kW = 2 * kH / kWorld; + constexpr int kEmbed = kH / kW; + constexpr int kDotVecs = kW / (8 * kWarpThreads); + extern __shared__ __align__(128) uint8_t smem[]; // [kB, kH] normalized rows + __shared__ float s_red[kWarps]; + auto* s_x = reinterpret_cast(smem); + const int warp = static_cast(threadIdx.x / kWarpThreads); + const int lane = static_cast(get_lane_id()); + for (int i0 = static_cast(blockIdx.x) * kB; i0 < p.n; i0 += static_cast(gridDim.x) * kB) { + const int nb = min(kB, p.n - i0); + for (int b = 0; b < nb; ++b) { + mkd_fc::gemma_norm_row(p.emb + static_cast(i0 + b) * kH, p.norm_w, p.eps, s_x + b * kH, s_red); + } + for (int j = warp; j < kH; j += kWarps) { + const bf16_t* row = p.fc_w + static_cast(j) * 2 * kH; + float tot[kB]; +#pragma unroll + for (int s = 0; s < kEmbed; ++s) { + vec_t wv[kDotVecs]; +#pragma unroll + for (int k = 0; k < kDotVecs; ++k) wv[k].load(row + s * kW, k * kWarpThreads + lane); +#pragma unroll + for (int b = 0; b < kB; ++b) { + float acc = 0.0f; + if (b < nb) { +#pragma unroll + for (int k = 0; k < kDotVecs; ++k) { + vec_t xv; + xv.load(s_x + b * kH + s * kW, k * kWarpThreads + lane); +#pragma unroll + for (int e = 0; e < 8; ++e) acc = fmaf(cast(wv[k][e]), cast(xv[e]), acc); + } + } + acc = warp::reduce_sum(acc); + tot[b] = s == 0 ? acc : tot[b] + acc; + } + } + if (lane == 0) { + for (int b = 0; b < nb; ++b) p.memo[static_cast(i0 + b) * kH + j] = tot[b]; + } + } + __syncthreads(); + } +} + +struct MkdHiddenPartialParams { + const bf16_t* __restrict__ h; // [kH] hidden state row + const bf16_t* __restrict__ w; // [kH, kW] this rank's fc input-column slice (contiguous) + const bf16_t* __restrict__ norm_w; // [kH] pre_fc_norm_hidden + float* out; // [kH] fp32 partial + const uint32_t* epoch; // tag of the exchange that sends out + uint32_t* done; // [1] CTA arrival counter, zero between calls + uint32_t* flag; // [1] set to the tag once out is complete + float eps; + int32_t col; // slice start inside the hidden half +}; + +/** + * \brief The hidden-slice fc partial of one row (draft_fc_partial arithmetic), then a flag: + * the step-boundary exchange waits on the flag instead of a stream join, so it keeps its + * programmatic early launch behind the lm_head. + */ +template +SGL_DEVICE void mkd_hidden_partial_kernel(const MkdHiddenPartialParams& p) { + using namespace device; + constexpr int kW = 2 * kH / kWorld; + constexpr int kWarps = 8; + __shared__ alignas(16) bf16_t s_x[kH]; + __shared__ float s_red[kWarps]; + const int warp = static_cast(threadIdx.x / kWarpThreads); + const int lane = static_cast(get_lane_id()); + mkd_fc::gemma_norm_row(p.h, p.norm_w, p.eps, s_x, s_red); + for (int j = static_cast(blockIdx.x) * kWarps + warp; j < kH; j += static_cast(gridDim.x) * kWarps) { + const float acc = mkd_fc::slice_dot(p.w + static_cast(j) * kW, s_x + p.col); + if (lane == 0) p.out[j] = acc; + } + __syncthreads(); + if (threadIdx.x == 0) { + __threadfence(); + if (atomicAdd(p.done, 1u) == gridDim.x - 1) { + *p.done = 0u; + uint32_t e; + asm volatile("ld.relaxed.gpu.global.u32 %0, [%1];" : "=r"(e) : "l"(p.epoch) : "memory"); + asm volatile("st.release.gpu.global.u32 [%0], %1;" ::"l"(p.flag), "r"(e) : "memory"); + } + } +} + +} // namespace sglang + +using sglang::MkdFcMemoParams; +using sglang::MkdFcSumParams; +using sglang::MkdHiddenPartialParams; + +extern "C" __global__ void __launch_bounds__(256, 1) mkd_fc_sum_memo(const MkdFcSumParams __grid_constant__ p) { + sglang::mkd_fc_sum_kernel<4096, MKD_WORLD, MKD_FC_MAX_ROWS, true, true>(p); +} + +extern "C" __global__ void __launch_bounds__(256, 1) mkd_fc_sum_emb(const MkdFcSumParams __grid_constant__ p) { + sglang::mkd_fc_sum_kernel<4096, MKD_WORLD, MKD_FC_MAX_ROWS, false, true>(p); +} + +extern "C" __global__ void __launch_bounds__(256) mkd_fc_memo(const MkdFcMemoParams __grid_constant__ p) { + sglang::mkd_fc_memo_kernel<4096, MKD_WORLD, 8>(p); +} + +// One CTA per SM (the launch is SM-count CTAs). With 2.2's __launch_bounds__(256) ptxas holds NVRTC's +// build to 32 registers and splits each row's four weight loads into two batches (+24%). +extern "C" __global__ void __launch_bounds__(256, 1) mkd_hidden_partial(const MkdHiddenPartialParams __grid_constant__ p) { + sglang::mkd_hidden_partial_kernel<4096, MKD_WORLD>(p); +} diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_head.cu b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_head.cu new file mode 100644 index 00000000..66d963c8 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_head.cu @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Adapted from SGLang (Apache-2.0): python/sglang/kernels/jit/csrc/speculative/mkd_draft_head.cuh +// (branch nvidia/qwen35-agentx-gb300), device code unchanged. +// +// Draft lm_head of one token on this rank's vocabulary shard, with the shard argmax folded in: +// logits[n] = bf16(sum_k W[n, k] * x[k]) with fp32 accumulation (bf16 inputs and weights, logits +// rounded to bf16 as the served GEMM writes them; only the fp32 summation order differs), and per +// CTA the packed order key of its best row (vocab_argmax order keys, -0.0 == +0.0, ties to the +// lowest row) for the step-boundary exchange, which then skips its own scan. +// +// Built with NVRTC: the kernel is a device function, and the extern "C" entry instantiates it +// for hidden 4096, 16 warps a CTA, PDL on (MkdHead of 2.2). + +#include "mkd_common.cuh" + +namespace sglang { + +namespace mkd_head { + +// Packed partial of a CTA whose rows hold a NaN logit (no NaN-free packed key reaches it). +inline constexpr uint32_t kNanPart = 0xffffffffu; + +/// \brief 16-bit order key of one bf16 value (-0.0 as +0.0); sets nan for a NaN. +SGL_DEVICE uint32_t key16(bf16_t v, bool& nan) { + uint32_t b = static_cast(__bfloat16_as_ushort(v)); + nan |= (b & 0x7fffu) > 0x7f80u; + b = b == 0x8000u ? 0u : b; + return (b & 0x8000u) ? (~b & 0xffffu) : (b | 0x8000u); +} + +} // namespace mkd_head + +struct MkdHeadParams { + const bf16_t* x; // [K] final-norm hidden state, written by the previous grid + const bf16_t* __restrict__ w; // [N, K] lm_head shard + bf16_t* logits; // [N] + uint32_t* part; // [gridDim.x] packed (key16 << 16) | (0xffff - row), or kNanPart + int32_t n; + int32_t reverse; // 1: walk the rows from the end (alternate steps, see the caller) +}; + +/** + * \brief One warp per weight row at a time, all 16 lane vectors of the row in flight; x in shared + * memory. The first row of every warp is loaded before the PDL wait (weights are static). + */ +template +SGL_DEVICE void mkd_head_kernel(const MkdHeadParams& p) { + using namespace device; + constexpr int kVecs = kK / 8 / 32; + __shared__ alignas(16) bf16_t s_x[kK]; + __shared__ uint32_t s_best[kWarps]; + const int warp = static_cast(threadIdx.x / 32); + const int lane = static_cast(threadIdx.x % 32); + const int stride = static_cast(gridDim.x) * kWarps; + int row = static_cast(blockIdx.x) * kWarps + warp; + // Walk position i maps to weight row i, or n - 1 - i on reversed calls. + auto wrow = [&](int i) { return p.reverse ? p.n - 1 - i : i; }; + uint4 wv[kVecs]; + if (row < p.n) { + const uint4* wr = reinterpret_cast(p.w + static_cast(wrow(row)) * kK); +#pragma unroll + for (int j = 0; j < kVecs; ++j) wv[j] = __ldg(wr + j * 32 + lane); + } + PDLWaitPrimary(); + PDLTriggerSecondary(); + for (int i = static_cast(threadIdx.x); i < kK / 8; i += kWarps * 32) { + reinterpret_cast(s_x)[i] = reinterpret_cast(p.x)[i]; + } + __syncthreads(); + uint32_t best = 0u; + bool nan = false; + for (; row < p.n; row += stride) { + float acc = 0.0f; +#pragma unroll + for (int j = 0; j < kVecs; ++j) { + const uint4 xv = reinterpret_cast(s_x)[j * 32 + lane]; + const bf16_t* a = reinterpret_cast(&wv[j]); + const bf16_t* b = reinterpret_cast(&xv); +#pragma unroll + for (int e = 0; e < 8; ++e) acc = fmaf(cast(a[e]), cast(b[e]), acc); + } + const int next = row + stride; + if (next < p.n) { + const uint4* wr = reinterpret_cast(p.w + static_cast(wrow(next)) * kK); +#pragma unroll + for (int j = 0; j < kVecs; ++j) wv[j] = __ldg(wr + j * 32 + lane); + } + acc = warp::reduce_sum(acc); + if (lane == 0) { + const int r = wrow(row); + const bf16_t v = cast(acc); + p.logits[r] = v; + const uint32_t k = mkd_head::key16(v, nan); + best = max(best, (k << 16) | (0xffffu - static_cast(r))); + } + } + if (lane == 0) s_best[warp] = nan ? mkd_head::kNanPart : best; + __syncthreads(); + if (threadIdx.x == 0) { + uint32_t b = s_best[0]; +#pragma unroll + for (int i = 1; i < kWarps; ++i) b = max(b, s_best[i]); + p.part[blockIdx.x] = b; + } +} + +} // namespace sglang + +using sglang::MkdHeadParams; + +extern "C" __global__ void __launch_bounds__(16 * 32) mkd_head(const MkdHeadParams __grid_constant__ p) { + sglang::mkd_head_kernel<4096, 16, true>(p); +} diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_xchg.cu b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_xchg.cu new file mode 100644 index 00000000..001cce50 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_xchg.cu @@ -0,0 +1,482 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Adapted from SGLang (Apache-2.0): python/sglang/kernels/jit/csrc/speculative/mkd_draft_xchg.cuh +// (branch nvidia/qwen35-agentx-gb300), device code unchanged. +// +// Greedy MTP draft step boundary in one launch: shard argmax of the hot-vocab logits, a +// multicast push of each rank's candidate (its embedding row, or its fc memo row) plus the +// rank's hidden-half fc partial, and the selection, replacing pack + all-gather + select and +// the fc partial all-gather; with memo rows it also writes the next step's fc output (the +// mkd_fc_sum memo arithmetic). Selection rules are those of vocab_argmax_pack / _select +// (order key, -0.0 == +0.0, NaN key from the caller, lowest vocabulary id on ties). +// +// Transport: a symmetric buffer with a multicast mapping (multi-node NVLink). Every 32-bit +// payload word travels in a 64-bit word whose high half is the call's epoch, stored with +// multimem.st.relaxed.sys.b64 and polled with ld.relaxed.sys.u64, so arrival is read from +// the data itself (single-copy-atomic 8-byte accesses). Three slots rotate by epoch: a slot +// is rewritten only three calls later, after two more exchanges that each need every rank's +// packet. Waits are bounded and trap. +// +// Built with NVRTC (MKD_WORLD = the TP size, 4 or 8): the kernel is a device function, and +// the extern "C" entries at the end instantiate it for hidden 4096, 16 chunks of 1024 threads, +// PDL on, with memo (fp32) or embedding (bf16) candidate rows. + +#include "mkd_common.cuh" + +#ifndef MKD_WORLD +#define MKD_WORLD 8 +#endif + +namespace sglang { + +namespace mkd_xchg { + +inline constexpr int kSlots = 3; +inline constexpr uint32_t kSpinLimit = 1u << 27; +// Shards of at least this many 16-byte vectors are scanned in 1/kChunks parts and combined +// through scan_part: the redundant full scan per CTA is instruction-bound (2.8 us at 31,040 +// rows), the combine costs one L2 round trip. +inline constexpr int kSplitScanMinVecs = 2048; +// Packed partial of a CTA that saw a NaN (no NaN-free packed key reaches it). +inline constexpr uint32_t kNanPart = 0xffffffffu; + +SGL_DEVICE uint32_t order_key(float f) { + uint32_t b = __float_as_uint(f); + b = b == 0x80000000u ? 0u : b; + return (b & 0x80000000u) ? ~b : (b | 0x80000000u); +} + +/// \brief Order keys of two packed bf16 (16 bits each): -0.0 as +0.0, negatives inverted. +/// For non-NaN bf16 the 32-bit key of order_key(float) is (key16 << 16) | (neg ? 0xffff : 0), +/// so comparing key16 orders like the 32-bit keys; NaN halves are flagged in nan_any. +SGL_DEVICE uint32_t packed_key(uint32_t w, uint32_t& nan_any) { + w &= ~__vcmpeq2(w, 0x80008000u); + nan_any |= __vcmpgtu2(w & 0x7fff7fffu, 0x7f807f80u); + const uint32_t neg = __vcmpgtu2(w, 0x7fff7fffu); + return (w ^ neg) | (~neg & 0x80008000u); +} + +SGL_DEVICE bool better(uint32_t v, int64_t i, uint32_t bv, int64_t bi) { + return v > bv || (v == bv && i < bi); +} + +SGL_DEVICE uint32_t clear_negative_zero(uint32_t two_bf16) { + uint32_t lo = two_bf16 & 0xffffu; + uint32_t hi = two_bf16 >> 16; + lo = lo == 0x8000u ? 0u : lo; + hi = hi == 0x8000u ? 0u : hi; + return lo | (hi << 16); +} + +SGL_DEVICE void mc_store(uint64_t* addr, uint32_t payload, uint32_t tag) { + const uint64_t w = static_cast(payload) | (static_cast(tag) << 32); + asm volatile("multimem.st.relaxed.sys.global.b64 [%0], %1;" ::"l"(addr), "l"(w) : "memory"); +} + +SGL_DEVICE uint64_t now_ns() { + uint64_t t; + asm volatile("mov.u64 %0, %globaltimer;" : "=l"(t)); + return t; +} + +} // namespace mkd_xchg + +struct MkdXchgParams { + const bf16_t* logits; // [V_local] hot-vocab shard logits, written by the lm_head + const void* __restrict__ cand; // [V_local, H] embedding rows (bf16) or memo rows (fp32) + const int64_t* hot_ids; // [world * V_local] vocabulary ids in hot order (sorted, rank-major) + const float* own_partial; // [H] this rank's hidden-slice fc partial (hidden ranks only) + const uint32_t* partial_flag; // equals the epoch once own_partial is written, or null + uint64_t* scan_part; // [kSlots, kChunks] per-CTA shard-argmax partials, or null + const uint32_t* head_part; // [head_parts] packed partials of mkd_head (no scan), or null + int32_t head_parts; + uint64_t* mc; // multicast base of the exchange buffer + const uint64_t* uc; // this rank's own copy of the exchange buffer + uint32_t* epoch; // [1] call counter shared by all ranks, starts at 1 + uint32_t* done; // [1] CTA arrival counter, zero between calls + void* out_cand; // [H] winner's embedding row (bf16) or memo row (fp32) + float* out_partials; // [world - embed, H] hidden-slice partials in rank order + bf16_t* fc_out; // [H] memo only: the next step's fc output, else null + int32_t token_only; // 1: exchange keys only (no next step reads the payload) + int64_t* topk_index; // [1] + float* topk_p; // [1] + int64_t* draft_tokens; // [1, S] or null + int64_t* positions; // [1] or null + int64_t* mrope; // [3, >= 1] or null + int32_t vocab; // V_local, multiple of 8 + uint32_t nan_key; + int32_t rank; + int32_t draft_column; + int32_t mrope_stride; + uint64_t* trace; // [trace_slots, kChunks, 9] %globaltimer per phase, or null + int32_t trace_slots; // calls kept in the trace ring (slot = epoch % trace_slots) + const char* pf; // static weight range prefetched into L2 for a later kernel, or null + int64_t pf_bytes; +}; + +/** + * \brief Step-boundary exchange for one token. Grid = kChunks CTAs; CTA c owns columns + * [c * H / kChunks, ...) of the candidate and partial rows. Every CTA redoes the shard + * argmax (a few tens of KB from L2) so no CTA waits for another before sending. + */ +template +SGL_DEVICE void mkd_xchg_kernel(const MkdXchgParams& p) { + using namespace device; + using vec_t = AlignedVector; + constexpr int kW = 2 * kH / kWorld; + constexpr int kEmbed = kH / kW; + constexpr int kCols = kH / kChunks; + constexpr int kCandWords = kMemo ? kCols : kCols / 2; + constexpr int kPartWords = kCols; + constexpr int kRec = 2 + kCandWords + kPartWords; + constexpr int kWarps = kThreads / 32; + static_assert(kH % (kChunks * 2) == 0); + + __shared__ uint32_t s_v[kWarps]; + __shared__ int32_t s_i[kWarps]; + __shared__ uint32_t s_key[kWorld]; + __shared__ uint32_t s_cand[kWorld][kCandWords]; + __shared__ float s_part[kWorld - kEmbed][kPartWords]; + __shared__ int64_t s_gid[kWorld]; + __shared__ int s_win; + __shared__ uint32_t s_tag; + __shared__ uint32_t s_best; + + const int c = static_cast(blockIdx.x); + const int warp = static_cast(threadIdx.x / 32); + const int lane = static_cast(get_lane_id()); + + const uint64_t t_start = mkd_xchg::now_ns(); + if (p.pf != nullptr) { + // Each CTA hints its share of the range in 64 KB bulk prefetches (static weights only). + constexpr int64_t kPfChunk = 64 * 1024; + const int64_t share = (p.pf_bytes + kChunks - 1) / kChunks; + const int64_t beg = c * share, end = min(p.pf_bytes, beg + share); + for (int64_t off = beg + static_cast(threadIdx.x) * kPfChunk; off < end; + off += static_cast(kThreads) * kPfChunk) { + const uint32_t size = static_cast(min(kPfChunk, end - off)) & ~15u; + if (size > 0) asm volatile("cp.async.bulk.prefetch.L2.global [%0], %1;" ::"l"(p.pf + off), "r"(size) : "memory"); + } + } + PDLWaitPrimary(); + // The fc launch behind us only prefetches static weights before its own wait. + PDLTriggerSecondary(); + const uint64_t t_wait = mkd_xchg::now_ns(); + // Read the epoch now, publish it after the scan: the load overlaps the logits loads. + uint32_t epoch = 0; + if (threadIdx.x == 0) asm volatile("ld.relaxed.gpu.global.u32 %0, [%1];" : "=r"(epoch) : "l"(p.epoch) : "memory"); + + // Shard argmax, ties to the lowest local row; hot positions order like vocabulary ids. + // Fast pass on packed 16-bit order keys (exact for NaN-free shards, see packed_key); a shard + // holding a NaN is rescanned with the 32-bit keys and the caller's NaN key. + uint32_t best = 0u; // (key16 << 16) | (0xffff - row) + uint32_t nan_any = 0u; + uint64_t t_ld = 0, t_cmp = 0, t_warp = 0; + constexpr int kScanUnroll = 4; + const int nvec = p.vocab / 8; + const bool from_head = p.head_part != nullptr; + const bool split = !from_head && p.scan_part != nullptr && nvec >= mkd_xchg::kSplitScanMinVecs; + const int lo = split ? static_cast((static_cast(nvec) * c) / kChunks) : 0; + const int hi = from_head ? 0 : split ? static_cast((static_cast(nvec) * (c + 1)) / kChunks) : nvec; + for (int v0 = lo + static_cast(threadIdx.x); v0 < hi; v0 += kThreads * kScanUnroll) { + uint4 x[kScanUnroll]; +#pragma unroll + for (int u = 0; u < kScanUnroll; ++u) { + if (v0 + u * kThreads < hi) x[u] = reinterpret_cast(p.logits)[v0 + u * kThreads]; + } + if (v0 == lo + static_cast(threadIdx.x)) t_ld = mkd_xchg::now_ns(); +#pragma unroll + for (int u = 0; u < kScanUnroll; ++u) { + const int v = v0 + u * kThreads; + if (v < hi) { + const uint32_t w4[4] = {x[u].x, x[u].y, x[u].z, x[u].w}; +#pragma unroll + for (int q = 0; q < 4; ++q) { + const uint32_t key = mkd_xchg::packed_key(w4[q], nan_any); + const uint32_t row = static_cast(v * 8 + 2 * q); + best = max(best, ((key & 0xffffu) << 16) | (0xffffu - row)); + best = max(best, (key & 0xffff0000u) | (0xffffu - row - 1)); + } + } + } + } + if (from_head && static_cast(threadIdx.x) < p.head_parts) { + // The lm_head kernel (complete after the PDL wait) left one packed maximum per CTA; every + // thread takes one, so the block reduce below combines them in one L2 round trip. + best = p.head_part[threadIdx.x]; + } + t_cmp = mkd_xchg::now_ns(); + best = __reduce_max_sync(0xffffffffu, best); + nan_any = __reduce_or_sync(0xffffffffu, nan_any); + t_warp = mkd_xchg::now_ns(); + if (lane == 0) { + s_v[warp] = best; + s_i[warp] = static_cast(nan_any); + } + if (threadIdx.x == 0) s_tag = epoch; + __syncthreads(); + static_assert(kWarps <= 32); + best = __reduce_max_sync(0xffffffffu, lane < kWarps ? s_v[lane] : 0u); + nan_any = __reduce_or_sync(0xffffffffu, lane < kWarps ? static_cast(s_i[lane]) : 0u); + if (from_head) { + // kNanPart (all ones) won the max: a head CTA saw a NaN logit, so take the exact rescan. + nan_any = best == mkd_xchg::kNanPart ? 1u : 0u; + } else if (split) { + // Every CTA publishes its sub-shard maximum (tagged with the epoch) and takes the max of all. + if (warp == 0) { + const uint32_t tg = __shfl_sync(0xffffffffu, epoch, 0); // only thread 0 loaded it + const int part_slot = static_cast(tg % mkd_xchg::kSlots); + if (lane == 0) { + const uint64_t w = (static_cast(tg) << 32) | (nan_any != 0u ? mkd_xchg::kNanPart : best); + asm volatile("st.relaxed.gpu.global.u64 [%0], %1;" ::"l"(p.scan_part + part_slot * kChunks + c), "l"(w) : "memory"); + } + uint32_t v = 0u; + if (lane < kChunks) { + uint64_t w; + uint32_t n = 0; + do { + asm volatile("ld.relaxed.gpu.global.u64 %0, [%1];" : "=l"(w) : "l"(p.scan_part + part_slot * kChunks + lane) : "memory"); + if (++n > mkd_xchg::kSpinLimit) __trap(); + } while (static_cast(w >> 32) != tg); + v = static_cast(w); + } + v = __reduce_max_sync(0xffffffffu, v); + if (lane == 0) s_best = v; + } + __syncthreads(); + best = s_best; + nan_any = best == mkd_xchg::kNanPart ? 1u : 0u; + } + int32_t bidx = static_cast(0xffffu - (best & 0xffffu)); + const uint32_t k16 = best >> 16; + best = (k16 << 16) | (k16 < 0x8000u ? 0xffffu : 0u); + if (nan_any != 0u) { + __syncthreads(); + best = 0u; + bidx = 0x7fffffff; + for (int v = static_cast(threadIdx.x); v < nvec; v += kThreads) { + vec_t xv; + xv.load(p.logits, v); +#pragma unroll + for (int j = 0; j < 8; ++j) { + const float f = cast(xv[j]); + const uint32_t key = f == f ? mkd_xchg::order_key(f) : p.nan_key; + const int32_t idx = v * 8 + j; + if (key > best || (key == best && idx < bidx)) { + best = key; + bidx = idx; + } + } + } +#pragma unroll + for (int off = 16; off > 0; off >>= 1) { + const uint32_t ov = __shfl_xor_sync(0xffffffffu, best, off); + const int32_t oi = __shfl_xor_sync(0xffffffffu, bidx, off); + if (ov > best || (ov == best && oi < bidx)) { + best = ov; + bidx = oi; + } + } + if (lane == 0) { + s_v[warp] = best; + s_i[warp] = bidx; + } + __syncthreads(); + best = s_v[0]; + bidx = s_i[0]; +#pragma unroll + for (int w = 1; w < kWarps; ++w) { + if (s_v[w] > best || (s_v[w] == best && s_i[w] < bidx)) { + best = s_v[w]; + bidx = s_i[w]; + } + } + } + const uint32_t tag = s_tag; + const int slot = static_cast(tag % mkd_xchg::kSlots); + const uint64_t t_scan = mkd_xchg::now_ns(); + const uint32_t hot_pos = static_cast(p.rank * p.vocab + bidx); + const int64_t brow = bidx; + + // Send this CTA's chunk of the packet to every rank (self included). + uint64_t* dst = p.mc + ((static_cast(slot) * kWorld + p.rank) * kChunks + c) * kRec; + if (threadIdx.x == 0) { + mkd_xchg::mc_store(dst, best, tag); + mkd_xchg::mc_store(dst + 1, hot_pos, tag); + } + const bool payload = p.token_only == 0; + if (payload) { + if constexpr (kMemo) { + const float* row = static_cast(p.cand) + brow * kH + c * kCols; + for (int w = static_cast(threadIdx.x); w < kCandWords; w += kThreads) { + mkd_xchg::mc_store(dst + 2 + w, __float_as_uint(row[w]), tag); + } + } else { + const uint32_t* row = reinterpret_cast(static_cast(p.cand) + brow * kH + c * kCols); + for (int w = static_cast(threadIdx.x); w < kCandWords; w += kThreads) { + mkd_xchg::mc_store(dst + 2 + w, mkd_xchg::clear_negative_zero(row[w]), tag); + } + } + if (p.rank >= kEmbed) { + const float* part = p.own_partial + c * kCols; + for (int w = static_cast(threadIdx.x); w < kPartWords; w += kThreads) { + if (p.partial_flag != nullptr) { + // The partial comes from a side-stream kernel that is not a stream predecessor. + uint32_t f, n = 0; + do { + asm volatile("ld.acquire.gpu.global.u32 %0, [%1];" : "=r"(f) : "l"(p.partial_flag) : "memory"); + if (++n > mkd_xchg::kSpinLimit) __trap(); + } while (f != tag); + } + mkd_xchg::mc_store(dst + 2 + kCandWords + w, __float_as_uint(part[w]), tag); + } + } + } + + const uint64_t t_sent = mkd_xchg::now_ns(); + // Receive in one batch: every rank's key and candidate chunk, and the hidden ranks' + // partial chunks; the winner's candidate is then picked from shared memory. + auto rec = [&](int s) { return p.uc + ((static_cast(slot) * kWorld + s) * kChunks + c) * kRec; }; + constexpr int kHead = 2 + kCandWords; + const int head = payload ? kHead : 2; + const int heads = kWorld * head; + const int total = payload ? heads + (kWorld - kEmbed) * kPartWords : heads; + constexpr int kRecvUnroll = 4; + for (int w0 = static_cast(threadIdx.x); w0 < total; w0 += kThreads * kRecvUnroll) { + const uint64_t* addr[kRecvUnroll]; + uint64_t v[kRecvUnroll]; +#pragma unroll + for (int u = 0; u < kRecvUnroll; ++u) { + const int i = w0 + u * kThreads; + addr[u] = nullptr; + if (i < total) { + if (i < heads) { + addr[u] = rec(i / head) + i % head; + } else { + const int j = i - heads; + addr[u] = rec(kEmbed + j / kPartWords) + kHead + j % kPartWords; + } + asm volatile("ld.relaxed.sys.global.u64 %0, [%1];" : "=l"(v[u]) : "l"(addr[u]) : "memory"); + } + } + // Re-poll every word still missing in one batch, so the last arrivals cost one round trip. + uint32_t want = 0, got = 0; +#pragma unroll + for (int u = 0; u < kRecvUnroll; ++u) { + if (w0 + u * kThreads < total) { + want |= 1u << u; + if (static_cast(v[u] >> 32) == tag) got |= 1u << u; + } + } + for (uint32_t n = 0; got != want;) { + if (++n > mkd_xchg::kSpinLimit) __trap(); +#pragma unroll + for (int u = 0; u < kRecvUnroll; ++u) { + if ((want & ~got) >> u & 1u) asm volatile("ld.relaxed.sys.global.u64 %0, [%1];" : "=l"(v[u]) : "l"(addr[u]) : "memory"); + } +#pragma unroll + for (int u = 0; u < kRecvUnroll; ++u) { + if (((want & ~got) >> u & 1u) && static_cast(v[u] >> 32) == tag) got |= 1u << u; + } + } +#pragma unroll + for (int u = 0; u < kRecvUnroll; ++u) { + const int i = w0 + u * kThreads; + if (i < total) { + const uint32_t x = static_cast(v[u]); + if (i < heads) { + const int s = i / head, w = i % head; + if (w == 0) { + s_key[s] = x; + } else if (w == 1) { + s_gid[s] = static_cast(x); + } else { + s_cand[s][w - 2] = x; + } + } else { + const int j = i - heads; + s_part[j / kPartWords][j % kPartWords] = __uint_as_float(x); + } + } + } + } + __syncthreads(); + const uint64_t t_recv = mkd_xchg::now_ns(); + if (threadIdx.x == 0) { + int win = 0; + uint32_t bv = s_key[0]; + int64_t bi = s_gid[0]; +#pragma unroll + for (int s = 1; s < kWorld; ++s) { + if (mkd_xchg::better(s_key[s], s_gid[s], bv, bi)) { + bv = s_key[s]; + bi = s_gid[s]; + win = s; + } + } + s_win = win; + if (c == 0) { + const int64_t gid = p.hot_ids[bi]; + p.topk_index[0] = gid; + p.topk_p[0] = 1.0f; + if (p.draft_tokens != nullptr) p.draft_tokens[p.draft_column] = gid; + if (p.positions != nullptr) p.positions[0] = p.positions[0] + 1; + if (p.mrope != nullptr) { +#pragma unroll + for (int d = 0; d < 3; ++d) p.mrope[d * p.mrope_stride] += 1; + } + } + } + __syncthreads(); + bool fold = false; + if constexpr (kMemo) fold = p.fc_out != nullptr; + if (payload && fold) { + // The fc output (mkd_fc_sum, memo path): memo row, then the hidden partials in rank order. + for (int w = static_cast(threadIdx.x); w < kCandWords; w += kThreads) { + float acc = __uint_as_float(s_cand[s_win][w]); +#pragma unroll + for (int h = 0; h < kWorld - kEmbed; ++h) acc += s_part[h][w]; + p.fc_out[c * kCols + w] = cast(acc); + } + } else if (payload) { + for (int w = static_cast(threadIdx.x); w < kCandWords; w += kThreads) { + static_cast(p.out_cand)[(kMemo ? c * kCols : c * kCols / 2) + w] = s_cand[s_win][w]; + } + for (int j = static_cast(threadIdx.x); j < (kWorld - kEmbed) * kPartWords; j += kThreads) { + p.out_partials[static_cast(j / kPartWords) * kH + c * kCols + j % kPartWords] = s_part[j / kPartWords][j % kPartWords]; + } + } + + __syncthreads(); + if (threadIdx.x == 0 && p.trace != nullptr) { + uint64_t* t = p.trace + (static_cast(tag % static_cast(p.trace_slots)) * kChunks + c) * 9; + t[0] = t_start; + t[1] = t_wait; + t[2] = t_ld; + t[3] = t_cmp; + t[4] = t_warp; + t[5] = t_scan; + t[6] = t_sent; + t[7] = t_recv; + t[8] = mkd_xchg::now_ns(); + } + if (threadIdx.x == 0) { + __threadfence(); + if (atomicAdd(p.done, 1u) == kChunks - 1) { + *p.done = 0u; + *p.epoch = tag + 1; + } + } +} + +} // namespace sglang + +using sglang::MkdXchgParams; + +extern "C" __global__ void __launch_bounds__(1024) mkd_xchg_memo(const MkdXchgParams __grid_constant__ p) { + sglang::mkd_xchg_kernel<4096, MKD_WORLD, true, 16, 1024, true>(p); +} + +extern "C" __global__ void __launch_bounds__(1024) mkd_xchg_emb(const MkdXchgParams __grid_constant__ p) { + sglang::mkd_xchg_kernel<4096, MKD_WORLD, false, 16, 1024, true>(p); +} diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py new file mode 100644 index 00000000..82c0b274 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py @@ -0,0 +1,738 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0): python/sglang/kernels/ops/speculative/{mkd_draft_xchg, +# mkd_draft_fc, mkd_draft_head, draft_fc_partial, ordered_partial_sum, +# draft_vocab_argmax}.py (branch nvidia/qwen35-agentx-gb300). +"""The MTP draft step boundary's kernels (``src/mkd_xchg.cu``, ``src/mkd_fc.cu``, +``src/mkd_head.cu``, ``src/draft_fc.cu``) with the launch signatures of the 2.2 ops: + +- ``mkd_draft_head``: the one-row draft lm_head on this rank's vocabulary shard, with the + per-CTA packed maxima the exchange reads; +- ``mkd_xchg``: the shard argmax, the multicast exchange of each rank's candidate and + hidden-half fc partial, the selection, and with memo rows the next step's fc output; +- ``mkd_fc_sum`` / ``mkd_fc_memo`` / ``mkd_hidden_partial``: the fc of an exchanged input, + the memo (T-table) rows, and the hidden-slice partial behind a flag; +- ``draft_fc_partial`` + ``FcGather`` + ``ordered_partial_sum``: the row-split fc of up to + 16 tokens (fused norm + slice GEMV, all-gather of the fp32 partials, rank-order sum). + +Every launch keeps the 2.2 grid, block, shared memory and PDL setting. ``FcGather`` moves +bytes only; the arithmetic of the split fc is ``draft_fc_partial`` and +``ordered_partial_sum``, copied unchanged. +""" + +from __future__ import annotations + +import ctypes +import struct +from functools import cache +from pathlib import Path +from typing import Optional + +import torch +import triton +import triton.language as tl + +from sgl_nvidia_mega.common import jit + +SRC = Path(__file__).parent / "src" +HIDDEN = 4096 +WORLDS = (4, 8) +# CTAs per exchange; each owns HIDDEN / CHUNKS columns of the candidate and partial rows +CHUNKS = 16 +XCHG_THREADS = 1024 +# Rotating epoch slots of the exchange buffer (mkd_xchg::kSlots) +SLOTS = 3 +# fc output rows per CTA held in shared memory by mkd_fc_sum (2.2 mkd_draft_fc.MAX_ROWS) +FC_MAX_ROWS = 27 +MEMO_BLOCK_ROWS = 8 # embedding rows a mkd_fc_memo CTA normalizes at once (kB) +HEAD_WARPS = 16 +# Rows of x per draft_fc_partial launch (2.2 draft_fc_partial.MAX_TOKENS) +MAX_TOKENS = 2 +_MAX_SMEM_BYTES = 216 * 1024 +# fc_gather: the split fc's widest batch (2.2 _FC_SPLIT_MAX_TOKENS) and its grid +GATHER_MAX_TOKENS = 16 +GATHER_CTAS, GATHER_THREADS = 32, 512 + + +def _order_key(value: float) -> int: + bits = struct.unpack(" int: + return torch.cuda.get_device_properties(device_index).multi_processor_count + + +def _compile(device_index: int, name: str, defines: tuple[str, ...]) -> bytes: + major, minor = torch.cuda.get_device_capability(device_index) + return jit.compile_source( + (SRC / f"{name}.cu").read_text(), + "q35_" + "_".join([name, *(d.split("=")[-1] for d in defines)]), + f"sm_{major}{minor}a", + ("-std=c++17", "-default-device", *(f"-D{d}" for d in defines)), + headers={"mkd_common.cuh": (SRC / "mkd_common.cuh").read_text()}, + ) + + +@cache +def _xchg_kernels(device_index: int, world: int) -> dict[str, jit.Kernel]: + cubin = _compile(device_index, "mkd_xchg", (f"MKD_WORLD={world}",)) + return { + name: jit.load(cubin, name, (XCHG_THREADS, 1, 1), 0) + for name in ("mkd_xchg_memo", "mkd_xchg_emb") + } + + +@cache +def _fc_kernels(device_index: int, world: int) -> dict[str, jit.Kernel]: + cubin = _compile( + device_index, "mkd_fc", (f"MKD_WORLD={world}", f"MKD_FC_MAX_ROWS={FC_MAX_ROWS}") + ) + return { + "mkd_fc_sum_memo": jit.load(cubin, "mkd_fc_sum_memo", (256, 1, 1), 0), + "mkd_fc_sum_emb": jit.load( + cubin, "mkd_fc_sum_emb", (256, 1, 1), FC_MAX_ROWS * HIDDEN * 2 + ), + "mkd_fc_memo": jit.load( + cubin, "mkd_fc_memo", (256, 1, 1), MEMO_BLOCK_ROWS * HIDDEN * 2 + ), + "mkd_hidden_partial": jit.load(cubin, "mkd_hidden_partial", (256, 1, 1), 0), + } + + +@cache +def _head_kernel(device_index: int) -> jit.Kernel: + cubin = _compile(device_index, "mkd_head", ()) + return jit.load(cubin, "mkd_head", (HEAD_WARPS * 32, 1, 1), 0) + + +@cache +def _draft_fc_kernels(device_index: int, world: int, max_rows: int) -> dict: + cubin = _compile( + device_index, + "draft_fc", + (f"MKD_WORLD={world}", f"DRAFT_FC_MAX_ROWS={max_rows}"), + ) + width = 2 * HIDDEN // world + smem = max_rows * width * 2 + kernels = { + f"draft_fc_partial_m{m}": jit.load( + cubin, f"draft_fc_partial_m{m}", (256, 1, 1), smem + ) + for m in range(1, MAX_TOKENS + 1) + } + kernels["fc_gather"] = jit.load(cubin, "fc_gather", (GATHER_THREADS, 1, 1), 0) + return kernels + + +def build(device_index: int, world: int) -> None: + """Compile every kernel of the draft step boundary at TP ``world`` (no compile + inside a capture).""" + if world not in WORLDS: + raise ValueError(f"the draft step boundary serves TP {WORLDS}, not {world}") + _xchg_kernels(device_index, world) + _fc_kernels(device_index, world) + _head_kernel(device_index) + rows = draft_fc_partial_rows(HIDDEN, 2 * HIDDEN // world, _sm_count(device_index)) + if rows is not None: + _draft_fc_kernels(device_index, world, rows) + + +def _launch(kernel: jit.Kernel, grid: int, params, device, pdl: bool) -> None: + """``jit.launch`` on the current stream; ``pdl`` False launches without programmatic + stream serialization, as 2.2 launches its kernels that never wait on a primary.""" + stream = torch.cuda.current_stream(device).cuda_stream + if pdl: + jit.launch(kernel, (grid, 1, 1), [params], stream) + return + cu = jit.driver() + pointer_array = (ctypes.c_void_p * 1)(ctypes.addressof(params)) + config = cu.CUlaunchConfig() + config.gridDimX, config.gridDimY, config.gridDimZ = grid, 1, 1 + config.blockDimX, config.blockDimY, config.blockDimZ = kernel.block + config.sharedMemBytes = kernel.shared_memory_bytes + config.hStream = cu.CUstream(stream) + config.attrs = [] + config.numAttrs = 0 + jit.check( + cu.cuLaunchKernelEx( + config, kernel.function, ctypes.addressof(pointer_array), 0 + ), + "cuLaunchKernelEx", + ) + + +def _check(t: torch.Tensor, what: str, shape, dtype, aligned: bool = False) -> None: + ok = ( + tuple(t.shape) == tuple(shape) + and t.dtype == dtype + and t.is_cuda + and t.is_contiguous() + and (not aligned or t.data_ptr() % 16 == 0) + ) + if not ok: + raise ValueError( + f"{what}: {t.dtype} {tuple(t.shape)} strides {tuple(t.stride())} " + f"(want {dtype} {tuple(shape)}{', 16-byte aligned' if aligned else ''})" + ) + + +def _embed_slices(world: int) -> int: + """Slices of the fc input's embedding half (kEmbed): one per rank of the first half.""" + return HIDDEN // (2 * HIDDEN // world) + + +# ---------------------------------------------------------------- the exchange + + +def mkd_xchg_words(hidden: int, world: int, memo: bool) -> int: + """64-bit words of the symmetric exchange buffer (mirrors MkdXchg::kWords).""" + cols = hidden // CHUNKS + rec = 2 + (cols if memo else cols // 2) + cols + return SLOTS * world * CHUNKS * rec + + +def mkd_xchg( + logits: torch.Tensor, + cand: torch.Tensor, + hot_ids: torch.Tensor, + own_partial: torch.Tensor, + epoch_done: torch.Tensor, + out_cand: torch.Tensor, + out_partials: torch.Tensor, + topk_index: torch.Tensor, + topk_p: torch.Tensor, + draft_tokens: Optional[torch.Tensor], + positions: Optional[torch.Tensor], + mrope_positions: Optional[torch.Tensor], + mc_ptr: int, + uc_ptr: int, + rank: int, + world: int, + draft_column: int, + nan_key: int, + trace: Optional[torch.Tensor] = None, + prefetch: Optional[torch.Tensor] = None, + prefetch_bytes: int = 0, + fc_out: Optional[torch.Tensor] = None, + token_only: bool = False, + partial_flag: Optional[torch.Tensor] = None, + scan_part: Optional[torch.Tensor] = None, + head_part: Optional[torch.Tensor] = None, +) -> None: + """Shard argmax + multicast exchange + selection for one draft token (2.2's + ``mkd_xchg``, same arguments). ``cand`` holds bf16 embedding rows or fp32 memo rows of + the local shard (memo selects the kernel); the winner's row lands in ``out_cand`` and + the hidden-slice partials in ``out_partials``, or with memo rows and ``fc_out`` the + next step's fc output. ``token_only`` exchanges the keys alone; ``partial_flag`` makes + a hidden rank wait for the side-stream partial; ``scan_part`` splits large shard + scans; ``head_part`` (the head's per-CTA maxima) replaces the scan.""" + memo = cand.dtype == torch.float32 + device = logits.device + v = int(logits.shape[-1]) + if cand.shape[-1] != HIDDEN or world not in WORLDS: + raise ValueError(f"mkd_xchg: hidden {cand.shape[-1]} at TP {world}") + _check(logits, "logits", (1, v), torch.bfloat16, aligned=True) + row_dtype = torch.float32 if memo else torch.bfloat16 + _check(cand, "cand", (v, HIDDEN), row_dtype, aligned=True) + _check(out_cand, "out_cand", (1, HIDDEN), row_dtype) + _check(hot_ids, "hot_ids", (world * v,), torch.int64) + _check(own_partial, "own_partial", (1, HIDDEN), torch.float32) + _check(epoch_done, "epoch_done", (2,), torch.int32) + _check( + out_partials, + "out_partials", + (world - _embed_slices(world), HIDDEN), + torch.float32, + ) + if fc_out is not None and fc_out.numel() > 0: + if not memo: + raise ValueError("mkd_xchg: fc_out needs memo candidates") + _check(fc_out, "fc_out", (1, HIDDEN), torch.bfloat16) + else: + fc_out = None + _check(topk_index, "topk_index", (1, 1), torch.int64) + _check(topk_p, "topk_p", (1, 1), torch.float32) + if v % 8 != 0 or v > 65536: + raise ValueError("mkd_xchg: local vocab must be a multiple of 8 and <= 65536") + head_parts = head_part.numel() if head_part is not None else 0 + if head_parts > XCHG_THREADS: + raise ValueError("mkd_xchg: at most one head partial per exchange thread") + trace_slots = trace.numel() // (CHUNKS * 9) if trace is not None else 1 + if trace_slots < 1: + raise ValueError("mkd_xchg: trace_slots must be >= 1") + tokens = None + if draft_tokens is not None and draft_tokens.numel() > 0: + _check(draft_tokens, "draft_tokens", (1, draft_tokens.shape[1]), torch.int64) + if not 0 <= draft_column < draft_tokens.shape[1]: + raise ValueError("mkd_xchg: draft column out of range") + tokens = draft_tokens + pos = None + if positions is not None and positions.numel() > 0: + _check(positions, "positions", (1,), torch.int64) + pos = positions + mrope, mrope_stride = None, 0 + if mrope_positions is not None and mrope_positions.numel() > 0: + if ( + tuple(mrope_positions.shape) != (3, 1) + or mrope_positions.stride(1) != 1 + or mrope_positions.dtype != torch.int64 + ): + raise ValueError( + f"mkd_xchg: mrope_positions {mrope_positions.dtype} " + f"{tuple(mrope_positions.shape)} strides {tuple(mrope_positions.stride())}" + ) + mrope, mrope_stride = mrope_positions, int(mrope_positions.stride(0)) + pf_bytes = ( + min(prefetch_bytes, prefetch.numel() * prefetch.element_size()) + if prefetch is not None + else 0 + ) + p = MkdXchgParams( + logits=logits.data_ptr(), + cand=cand.data_ptr(), + hot_ids=hot_ids.data_ptr(), + own_partial=own_partial.data_ptr(), + partial_flag=jit.null_or_pointer(partial_flag).value, + scan_part=jit.null_or_pointer(scan_part).value, + head_part=jit.null_or_pointer(head_part).value, + head_parts=head_parts, + mc=mc_ptr, + uc=uc_ptr, + epoch=epoch_done.data_ptr(), + done=epoch_done.data_ptr() + 4, + out_cand=out_cand.data_ptr(), + out_partials=out_partials.data_ptr(), + fc_out=jit.null_or_pointer(fc_out).value, + token_only=int(bool(token_only)), + topk_index=topk_index.data_ptr(), + topk_p=topk_p.data_ptr(), + draft_tokens=jit.null_or_pointer(tokens).value, + positions=jit.null_or_pointer(pos).value, + mrope=jit.null_or_pointer(mrope).value, + vocab=v, + nan_key=nan_key & 0xFFFFFFFF, + rank=rank, + draft_column=draft_column, + mrope_stride=mrope_stride, + trace=jit.null_or_pointer(trace).value, + trace_slots=trace_slots, + pf=jit.null_or_pointer(prefetch).value, + pf_bytes=pf_bytes, + ) + kernels = _xchg_kernels(device.index, world) + _launch( + kernels["mkd_xchg_memo" if memo else "mkd_xchg_emb"], + CHUNKS, + p, + device, + pdl=True, + ) + + +# ---------------------------------------------------------------- fc of an exchanged input + + +def mkd_fc_sum( + emb: torch.Tensor, + memo: Optional[torch.Tensor], + hidden_partials: torch.Tensor, + fc_weight: torch.Tensor, + norm_weight: torch.Tensor, + eps: float, + out: torch.Tensor, + world: int, +) -> torch.Tensor: + """fc output [1, H] of one token: rank-order sum of the embedding-half partials + (computed here from ``emb`` [1, H], or the memo row [1, H] fp32) and the received + hidden-half partials [world / 2, H] fp32; bitwise equal to the row-split fc.""" + device = out.device + _check( + hidden_partials, + "hidden_partials", + (world - _embed_slices(world), HIDDEN), + torch.float32, + ) + _check(fc_weight, "fc_weight", (HIDDEN, 2 * HIDDEN), torch.bfloat16, aligned=True) + _check(norm_weight, "norm_weight", (HIDDEN,), torch.bfloat16, aligned=True) + _check(out, "out", (1, HIDDEN), torch.bfloat16) + use_memo = memo is not None and memo.numel() > 0 + if use_memo: + _check(memo, "memo", (1, HIDDEN), torch.float32) + else: + _check(emb, "emb", (1, HIDDEN), torch.bfloat16, aligned=True) + grid = _sm_count(device.index) + if (HIDDEN + grid - 1) // grid > FC_MAX_ROWS: + raise ValueError("mkd_fc_sum: more fc rows per CTA than kMaxRows") + p = MkdFcSumParams( + emb=None if use_memo else emb.data_ptr(), + memo=memo.data_ptr() if use_memo else None, + hidden_partials=hidden_partials.data_ptr(), + fc_w=fc_weight.data_ptr(), + norm_w=norm_weight.data_ptr(), + out=out.data_ptr(), + eps=eps, + ) + kernels = _fc_kernels(device.index, world) + name = "mkd_fc_sum_memo" if use_memo else "mkd_fc_sum_emb" + _launch(kernels[name], grid, p, device, pdl=True) + return out + + +def mkd_fc_memo( + emb: torch.Tensor, + fc_weight: torch.Tensor, + norm_weight: torch.Tensor, + eps: float, + world: int, + out: torch.Tensor, +) -> torch.Tensor: + """Memo rows [N, H] fp32 for embedding rows [N, H]: the rank-order sum of the + embedding-half partials the row-split fc would compute for each row.""" + device = out.device + n = int(emb.shape[0]) + _check(emb, "emb", (n, HIDDEN), torch.bfloat16, aligned=True) + _check(fc_weight, "fc_weight", (HIDDEN, 2 * HIDDEN), torch.bfloat16, aligned=True) + _check(norm_weight, "norm_weight", (HIDDEN,), torch.bfloat16, aligned=True) + _check(out, "out", (n, HIDDEN), torch.float32) + p = MkdFcMemoParams( + emb=emb.data_ptr(), + fc_w=fc_weight.data_ptr(), + norm_w=norm_weight.data_ptr(), + memo=out.data_ptr(), + eps=eps, + n=n, + ) + kernels = _fc_kernels(device.index, world) + _launch(kernels["mkd_fc_memo"], _sm_count(device.index) * 2, p, device, pdl=False) + return out + + +def mkd_hidden_partial( + h: torch.Tensor, + shard: torch.Tensor, + norm_weight: torch.Tensor, + eps: float, + col: int, + out: torch.Tensor, + epoch_done: torch.Tensor, + sync: torch.Tensor, + world: int, +) -> None: + """Hidden-slice fc partial [1, H] fp32 of ``h`` [1, H] (draft_fc_partial arithmetic + over ``shard`` [H, W] at hidden-half column ``col``); the last CTA then sets + ``sync[1]`` to the exchange epoch ``epoch_done[0]`` (``sync[0]`` counts CTAs, zero + between calls).""" + device = out.device + width = 2 * HIDDEN // world + _check(h, "h", (1, HIDDEN), torch.bfloat16, aligned=True) + _check(shard, "shard", (HIDDEN, width), torch.bfloat16, aligned=True) + _check(norm_weight, "norm_weight", (HIDDEN,), torch.bfloat16, aligned=True) + _check(out, "out", (1, HIDDEN), torch.float32) + _check(epoch_done, "epoch_done", (2,), torch.int32) + _check(sync, "sync", (2,), torch.int32) + if not (col >= 0 and col % width == 0 and col + width <= HIDDEN): + raise ValueError("mkd_hidden_partial: slice start must be a slice boundary") + p = MkdHiddenPartialParams( + h=h.data_ptr(), + w=shard.data_ptr(), + norm_w=norm_weight.data_ptr(), + out=out.data_ptr(), + epoch=epoch_done.data_ptr(), + done=sync.data_ptr(), + flag=sync.data_ptr() + 4, + eps=eps, + col=col, + ) + kernels = _fc_kernels(device.index, world) + _launch( + kernels["mkd_hidden_partial"], _sm_count(device.index), p, device, pdl=False + ) + + +# ---------------------------------------------------------------- draft lm_head + + +def mkd_draft_head( + x: torch.Tensor, + weight: torch.Tensor, + logits: torch.Tensor, + part: torch.Tensor, + reverse: bool = False, +) -> None: + """Draft lm_head of one token, ``logits`` [1, N] bf16 = ``x`` [1, 4096] @ ``weight`` + [N, 4096]^T with fp32 accumulation, and per CTA (``part.numel()`` CTAs) the packed + maximum of its rows for mkd_xchg. ``reverse`` walks the rows from the end (same + values).""" + device = logits.device + n = int(weight.shape[0]) + _check(x, "x", (1, HIDDEN), torch.bfloat16, aligned=True) + _check(weight, "weight", (n, HIDDEN), torch.bfloat16, aligned=True) + _check(logits, "logits", (1, n), torch.bfloat16) + _check(part, "part", (part.numel(),), torch.int32) + if n > 65536: + raise ValueError( + "mkd_draft_head: rows must fit the 16-bit row of the order key" + ) + p = MkdHeadParams( + x=x.data_ptr(), + w=weight.data_ptr(), + logits=logits.data_ptr(), + part=part.data_ptr(), + n=n, + reverse=int(bool(reverse)), + ) + _launch(_head_kernel(device.index), part.numel(), p, device, pdl=True) + + +# ---------------------------------------------------------------- the row-split fc + + +def draft_fc_partial_rows(out_rows: int, width: int, num_sms: int) -> Optional[int]: + """Weight rows per CTA for the fused kernel, or None if they do not fit in shared + memory.""" + rows = -(-out_rows // num_sms) + static = MAX_TOKENS * width * 2 + return rows if rows * width * 2 + static <= _MAX_SMEM_BYTES else None + + +def draft_fc_partial( + x: torch.Tensor, + norm_weight: torch.Tensor, + eps: float, + weight_slice: torch.Tensor, + col_start: int, + out: torch.Tensor, + max_rows: int, +) -> torch.Tensor: + """Gemma RMSNorm of the input half x [M, H] (bf16 output; M <= MAX_TOKENS), then + this rank's fc slice: out [M, N] fp32 = norm(x)[:, col_start : col_start + W] @ + weight_slice [N, W]^T.""" + device = out.device + m, width = int(x.shape[0]), int(weight_slice.shape[-1]) + world = 2 * HIDDEN // width + if ( + world not in WORLDS + or weight_slice.shape[0] != HIDDEN + or not 1 <= m <= MAX_TOKENS + ): + raise ValueError( + f"draft_fc_partial: {m} rows of a {tuple(weight_slice.shape)} slice " + f"(built for hidden {HIDDEN} at TP {WORLDS}, 1-{MAX_TOKENS} rows)" + ) + _check(x, "x", (m, HIDDEN), torch.bfloat16, aligned=True) + _check(norm_weight, "norm_weight", (HIDDEN,), torch.bfloat16, aligned=True) + _check(weight_slice, "weight_slice", (HIDDEN, width), torch.bfloat16, aligned=True) + _check(out, "out", (m, HIDDEN), torch.float32) + if not (col_start >= 0 and col_start + width <= HIDDEN and col_start % 8 == 0): + raise ValueError("draft_fc_partial: slice out of range") + grid = _sm_count(device.index) + if (HIDDEN + grid - 1) // grid > max_rows: + raise ValueError("draft_fc_partial: more weight rows per CTA than kMaxRows") + p = DraftFcPartialParams( + x=x.data_ptr(), + norm_w=norm_weight.data_ptr(), + w=weight_slice.data_ptr(), + out=out.data_ptr(), + eps=eps, + col_start=col_start, + rows=m, + ) + kernels = _draft_fc_kernels(device.index, world, max_rows) + _launch(kernels[f"draft_fc_partial_m{m}"], grid, p, device, pdl=True) + return out + + +@triton.jit +def _ordered_partial_sum_kernel( + src_ptr, # [M, world * N] fp32: rank r's partial at columns [r * N, (r + 1) * N) + dst_ptr, # [M, N] bf16 + N, + WORLD: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + offs = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK) + mask = offs < N + base = src_ptr + row * WORLD * N + offs + acc = tl.load(base, mask=mask, other=0.0) + for r in tl.static_range(1, WORLD): + acc += tl.load(base + r * N, mask=mask, other=0.0) + tl.store(dst_ptr + row * N + offs, acc.to(tl.bfloat16), mask=mask) + + +def ordered_partial_sum( + gathered: torch.Tensor, world: int, out: torch.Tensor +) -> torch.Tensor: + """Sum ``world`` fp32 partials in rank order, rounding once to bf16 into ``out``.""" + m, n = out.shape + block = 1024 + _ordered_partial_sum_kernel[(m, triton.cdiv(n, block))]( + gathered, out, n, WORLD=world, BLOCK=block, num_warps=4 + ) + return out + + +def symmetric_words(words: int, group, device: torch.device): + """(tensor, handle, multicast pointer) of a zeroed symmetric int64 buffer on + ``group``; a collective on every rank of it.""" + import torch.distributed._symmetric_memory as symm_mem + + if symm_mem.get_backend(device) is None: + # the backend FlashInfer's fusion workspace would select later in this process + symm_mem.set_backend("NCCL") + buf = symm_mem.empty((words,), dtype=torch.int64, device=device) + handle = symm_mem.rendezvous(buf, group) + multicast = int(handle.multicast_ptr or 0) + if not multicast or multicast % 16: + raise RuntimeError("the draft step boundary needs an NVLink multicast mapping") + buf.zero_() + return buf, handle, multicast + + +class FcGather: + """The split fc's all-gather of fp32 partials, [m, H] on every rank -> [m, world * H] + with rank r's partial at columns [r * H, (r + 1) * H) (the layout + ``ordered_partial_sum`` reads), for m <= GATHER_MAX_TOKENS; graph-capturable. Build on + every rank of ``group`` (a collective).""" + + def __init__( + self, *, world: int, rank: int, group, cpu_group, device: torch.device + ): + import torch.distributed as dist + + if world not in WORLDS: + raise ValueError(f"the fc gather serves TP {WORLDS}, not {world}") + self.world, self.rank, self.device = world, rank, device + words = SLOTS * world * GATHER_MAX_TOKENS * HIDDEN + self._buf, self._handle, self.mc = symmetric_words(words, group, device) + self.uc = self._buf.data_ptr() + # [epoch, CTA arrivals]: the epoch tags the call's words, from 1 + self.epoch_done = torch.tensor([1, 0], dtype=torch.int32, device=device) + sms = _sm_count(device.index) + self._kernels = _draft_fc_kernels( + device.index, world, draft_fc_partial_rows(HIDDEN, 2 * HIDDEN // world, sms) + ) + torch.cuda.synchronize(device) + dist.barrier(group=cpu_group) + + def __call__(self, partial: torch.Tensor) -> torch.Tensor: + m = int(partial.shape[0]) + if not 1 <= m <= GATHER_MAX_TOKENS: + raise ValueError(f"fc gather: {m} rows (1-{GATHER_MAX_TOKENS})") + _check(partial, "partial", (m, HIDDEN), torch.float32) + out = torch.empty( + (m, self.world * HIDDEN), dtype=torch.float32, device=partial.device + ) + p = FcGatherParams( + partial=partial.data_ptr(), + out=out.data_ptr(), + mc=self.mc, + uc=self.uc, + epoch=self.epoch_done.data_ptr(), + done=self.epoch_done.data_ptr() + 4, + rank=self.rank, + m=m, + ) + _launch(self._kernels["fc_gather"], GATHER_CTAS, p, self.device, pdl=False) + return out diff --git a/test/kernels/test_qwen3_5_draft_xchg.py b/test/kernels/test_qwen3_5_draft_xchg.py new file mode 100644 index 00000000..943e3035 --- /dev/null +++ b/test/kernels/test_qwen3_5_draft_xchg.py @@ -0,0 +1,303 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The MTP draft step boundary (``qwen3_5/draft/xchg.py`` on ``qwen3_5/sm103/draft``) on 4 +SM100-family GPUs (GB300) with NVLink multicast, against torch on the same inputs: + +- the draft head + exchange pick the global argmax of the ranks' shard logits (order keys: + -0.0 as +0.0, ties to the lowest id; NaN as -1e30 in draft steps, highest after the draft + extend), write the draft column, advance the positions and the M-RoPE positions, and the + next step's fc equals the memo row plus the hidden ranks' partials summed in rank order, + rounded once to bf16, bit for bit; +- the memo rows equal the split fc's partials summed in rank order (checked at + construction), and the split fc's gather and rank-order sum are exact; +- a whole draft round (five steps, the extend exchange with extend fc) replays from a CUDA + graph with fresh inputs, equal to its eager run. + +The bitwise comparison with the kernels of SGLang branch nvidia/qwen35-agentx-gb300 needs that +branch installed as ``sglang`` and is not part of this test. +Run with ``python test/kernels/test_qwen3_5_draft_xchg.py --num-gpu 4``. +""" + +import os + +import pytest +import torch +import torch.distributed as dist +from sglang.test.kernels.utils import multigpu_pytest_main + +H, EPS, NUM_STEPS, VOCAB = 4096, 1e-6, 6, 1024 # VOCAB rows a rank: a short memo check + +pytestmark = pytest.mark.skipif( + not (torch.cuda.is_available() and torch.cuda.get_device_capability()[0] == 10), + reason="Requires SM100/SM103", +) + + +class _Norm: + """A Gemma RMSNorm's weight and epsilon, and its forward (x * rsqrt(mean x^2 + eps) * + (1 + w), fp32 statistics, bf16 output) for splits over two rows.""" + + def __init__(self, weight): + self.weight, self.variance_epsilon = weight, EPS + + def __call__(self, x): + xf = x.float() + inv = torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + EPS) + return (xf * inv * (1.0 + self.weight.float())).to(torch.bfloat16) + + +@pytest.fixture(scope="module") +def ranks(): + local = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local) + dist.init_process_group("nccl", device_id=torch.device("cuda", local)) + cpu_group = dist.new_group(backend="gloo") + yield dist.get_rank(), dist.get_world_size(), dist.group.WORLD, cpu_group + dist.destroy_process_group() + + +def _gen(seed): + return torch.Generator(device="cuda").manual_seed(seed) + + +def _randn(*shape, g, scale=1.0, dtype=torch.bfloat16): + return (torch.randn(*shape, device="cuda", generator=g) * scale).to(dtype) + + +@pytest.fixture(scope="module") +def boundary(ranks): + from sgl_nvidia_mega.qwen3_5.draft.xchg import MkdBoundary + + rank, world, group, cpu_group = ranks + g = _gen(2026) + full_embed = _randn(world * VOCAB, H, g=g) + weights = dict( + fc=_randn(H, 2 * H, g=g, scale=0.02), + norm_emb=_randn(H, g=g, scale=0.1), + norm_hid=_randn(H, g=g, scale=0.1), + embed=full_embed[rank * VOCAB : (rank + 1) * VOCAB].contiguous(), + head=_randn(world * VOCAB, H, g=g, scale=0.02)[ + rank * VOCAB : (rank + 1) * VOCAB + ].contiguous(), + ) + mkd = MkdBoundary( + fc_weight=weights["fc"], + norm_embedding=_Norm(weights["norm_emb"]), + norm_hidden=_Norm(weights["norm_hid"]), + embed_shard=weights["embed"], + vocab_start=rank * VOCAB, + lm_head_shard=weights["head"], + embed=lambda ids: full_embed[ids], + rank=rank, + world=world, + device_group=group, + cpu_group=cpu_group, + num_steps=NUM_STEPS, + ) + return mkd, weights + + +def _all_gather(t, group): + world = dist.get_world_size(group) + out = torch.empty((world, *t.shape), dtype=t.dtype, device=t.device) + dist.all_gather_into_tensor(out, t.contiguous(), group=group) + return out + + +def _want_token(local_logits, group, nan_as): + """The argmax over the ranks' logits concatenated in rank order, ties to the lowest id; + a NaN counts as ``nan_as``.""" + full = _all_gather(local_logits[0], group).reshape(-1).float() + full = torch.where(torch.isnan(full), torch.full_like(full, nan_as), full) + return int(torch.argmax(full)) # the first maximum: the lowest id + + +def test_step_exchange_picks_the_global_argmax(ranks, boundary): + rank, world, group, _ = ranks + mkd, _ = boundary + n_embed = world // 2 + with torch.inference_mode(): + for seed in range(4): + g = _gen(100 + seed) + hidden = _randn(1, H, g=g) + if seed == 1: # ties between rows and ranks + mkd.lm_head_weight.copy_(torch.round(mkd.lm_head_weight * 64) / 64) + draft_tokens = torch.zeros(1, NUM_STEPS, dtype=torch.int64, device="cuda") + positions = torch.tensor([77], dtype=torch.int64, device="cuda") + mrope = torch.arange(6, dtype=torch.int64, device="cuda").view(3, 2).clone() + step = seed % (NUM_STEPS - 2) # not the last step: the payload travels + mkd.begin_step(step, 1) + mkd.fc() + mkd.hidden_partial(hidden) + logits = mkd.head(hidden) + topk_p, topk_index = mkd.finish_step( + logits, positions, draft_tokens, step, step + 1, mrope[:, :1] + ) + torch.cuda.synchronize() + want = _want_token(logits, group, -1e30) + assert int(topk_index) == want and float(topk_p) == 1.0 + assert int(draft_tokens[0, step + 1]) == want + assert int(positions) == 78 + assert mrope[:, 0].tolist() == [1, 3, 5] and mrope[:, 1].tolist() == [ + 1, + 3, + 5, + ] + # the next fc: the winner's memo row, then the hidden ranks' partials in rank order + owner, row = divmod(want, VOCAB) + memo_row = _all_gather(mkd.cand[min(row, VOCAB - 1)], group)[owner].float() + partials = _all_gather(mkd.own_partial[0], group)[n_embed:] + acc = memo_row.clone() + for h in range(world - n_embed): + acc = acc + partials[h] + assert torch.equal( + mkd.fc_buf[0].view(torch.int16), acc.bfloat16().view(torch.int16) + ) + assert mkd.pending + # the head's logits against torch (only the fp32 summation order differs) + ref = (hidden.float() @ mkd.lm_head_weight.float().T).bfloat16() + torch.testing.assert_close( + logits.float(), ref.float(), atol=2e-2, rtol=2e-2 + ) + + +@pytest.mark.parametrize("nan_key", ["split", "torch"]) +def test_extend_exchange_nan_rules(ranks, boundary, nan_key): + from sgl_nvidia_mega.qwen3_5.sm103.draft import xchg as kernels + + rank, world, group, _ = ranks + mkd, _ = boundary + with torch.inference_mode(): + g = _gen(500 + rank) + logits = _randn(1, VOCAB, g=g) + if rank == 1: + logits[0, 17] = float("nan") + topk_index = torch.empty(1, 1, dtype=torch.int64, device="cuda") + topk_p = torch.empty(1, 1, dtype=torch.float32, device="cuda") + key = ( + kernels.NAN_KEY_SPLIT_ARGMAX + if nan_key == "split" + else kernels.NAN_KEY_TORCH_ARGMAX + ) + kernels.mkd_xchg( + logits, + mkd.cand, + mkd.hot_ids, + mkd.own_partial, + mkd.epoch_done, + mkd.ext_cand, + mkd.ext_partials, + topk_index, + topk_p, + None, + None, + None, + mkd.mc, + mkd.uc, + rank, + world, + 0, + key, + token_only=True, + scan_part=mkd.scan_part, + ) + torch.cuda.synchronize() + want = VOCAB + 17 if nan_key == "torch" else _want_token(logits, group, -1e30) + assert int(topk_index) == want + + +@pytest.mark.parametrize("m", [1, 2, 7]) +def test_split_fc_gathers_and_sums_in_rank_order(ranks, boundary, m): + from sgl_nvidia_mega.qwen3_5.sm103.draft import xchg as kernels + + rank, world, group, _ = ranks + mkd, weights = boundary + with torch.inference_mode(): + g = _gen(900 + m) + embeds, hidden = _randn(m, H, g=g), _randn(m, H, g=g) + partial = _randn(m, H, g=_gen(950 + 7 * rank + m), dtype=torch.float32) + gathered = mkd._fc_gather(partial) + want = _all_gather(partial, group).permute(1, 0, 2).reshape(m, world * H) + assert torch.equal(gathered, want) + acc = want[:, :H].clone() + for r in range(1, world): + acc = acc + want[:, r * H : (r + 1) * H] + summed = kernels.ordered_partial_sum( + gathered, world, torch.empty(m, H, dtype=torch.bfloat16, device="cuda") + ) + assert torch.equal(summed.view(torch.int16), acc.bfloat16().view(torch.int16)) + # the split fc against the replicated fc in fp32 (only the summation order differs) + got = mkd.split_fc(embeds, hidden) + x = torch.cat( + [_Norm(weights["norm_emb"])(embeds), _Norm(weights["norm_hid"])(hidden)], -1 + ) + ref = (x.float() @ weights["fc"].float().T).bfloat16() + torch.testing.assert_close(got.float(), ref.float(), atol=3e-2, rtol=3e-2) + + +def test_round_replays_from_a_cuda_graph(ranks, boundary): + rank, world, group, cpu_group = ranks + mkd, _ = boundary + + def inputs(seed): + g = _gen(3000 + seed) + return dict( + token=torch.randint(0, world * VOCAB, (1, 1), device="cuda", generator=g), + hidden0=_randn(1, H, g=g), + hidden=[_randn(1, H, g=g) for _ in range(NUM_STEPS - 1)], + ext_logits=_randn(1, VOCAB, g=_gen(4000 + 10 * seed + rank)), + tokens=torch.zeros(1, NUM_STEPS, dtype=torch.int64, device="cuda"), + pos=torch.tensor([9], dtype=torch.int64, device="cuda"), + ) + + def run_round(st): + out = [] + for i in range(NUM_STEPS - 1): + mkd.begin_step(i, 1) + out.append(mkd.fc().clone()) + mkd.hidden_partial(st["hidden"][i]) + logits = mkd.head(st["hidden"][i]) + mkd.finish_step(logits, st["pos"], st["tokens"], i, i + 1) + mkd.end() + mkd.note_draft_input() + _, ti = mkd.exchange_extend(st["ext_logits"], st["hidden0"]) + return out + [ti.clone(), mkd.fc_buf.clone(), st["tokens"], st["pos"]] + + with torch.inference_mode(): + eager = [] + for seed in range(3): + st = inputs(seed) + mkd.fill_fc(st["token"], st["hidden0"]) + eager.append([t.clone() for t in run_round(st)]) + static = inputs(0) + mkd.fill_fc(static["token"], static["hidden0"]) + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + run_round(static) + torch.cuda.current_stream().wait_stream(stream) + dist.barrier(group=cpu_group) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = run_round(static) + torch.cuda.synchronize() + dist.barrier(group=cpu_group) + for seed in range(3): + fresh = inputs(seed) + for key in ("token", "hidden0", "ext_logits", "tokens", "pos"): + static[key].copy_(fresh[key]) + for i in range(NUM_STEPS - 1): + static["hidden"][i].copy_(fresh["hidden"][i]) + mkd.fill_fc(static["token"], static["hidden0"]) + graph.replay() + torch.cuda.synchronize() + for got, want in zip(captured, eager[seed]): + assert torch.equal( + got.view(torch.int16) if got.dtype == torch.bfloat16 else got, + want.view(torch.int16) if want.dtype == torch.bfloat16 else want, + ) + dist.barrier(group=cpu_group) + + +if __name__ == "__main__": + multigpu_pytest_main(__name__, __file__, num_gpus=(4,)) diff --git a/test/unit/test_qwen3_5_runtime.py b/test/unit/test_qwen3_5_runtime.py index 726b28b6..e624f174 100644 --- a/test/unit/test_qwen3_5_runtime.py +++ b/test/unit/test_qwen3_5_runtime.py @@ -179,7 +179,7 @@ class _DraftParts: """Parts with no slot built: the stock model step.""" def __init__(self, slots=frozenset()): - self.slots, self.engaged = slots, [] + self.slots, self.mkd, self.engaged = slots, None, [] def note_engaged(self, entry, bs): self.engaged.append((entry, bs)) From 3c99a88c397fadb9cc93f66346da2f6a6527ed99 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:49:00 -0700 Subject: [PATCH 12/27] [Perf] [GB300] qwen3_5: Draft slot draft_hop: the draft's all-reduce points and o_proj push on the decode hop (numeric) Inside draft_loop, the draft step's two all-reduce + residual + norm points run as hops on the target's decode hop, in the rounding of the kernels they replace, and o_proj runs as a one-row bf16 GEMV whose epilogue pushes into the FFN input's hop. Draft slot draft_hop (with hop); numeric: the GEMV sums in another fp32 order than SGLang's GEMM. --- sgl_nvidia_mega/qwen3_5/boundary/hop.py | 10 + sgl_nvidia_mega/qwen3_5/draft/__init__.py | 7 +- sgl_nvidia_mega/qwen3_5/draft/attention.py | 9 +- sgl_nvidia_mega/qwen3_5/draft/hop.py | 168 +++++ sgl_nvidia_mega/qwen3_5/draft/layer.py | 34 +- sgl_nvidia_mega/qwen3_5/draft/parts.py | 22 +- sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 4 +- .../qwen3_5/sm103/draft/__init__.py | 4 +- sgl_nvidia_mega/qwen3_5/sm103/draft/hop.py | 248 +++++++ .../qwen3_5/sm103/draft/src/draft_gemv.cu | 450 +++++++++++++ .../qwen3_5/sm103/draft/src/draft_hop.cu | 621 ++++++++++++++++++ test/kernels/test_qwen3_5_draft_hop.py | 283 ++++++++ test/unit/test_qwen3_5_runtime.py | 1 + 13 files changed, 1847 insertions(+), 14 deletions(-) create mode 100644 sgl_nvidia_mega/qwen3_5/draft/hop.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/hop.py create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_gemv.cu create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_hop.cu create mode 100644 test/kernels/test_qwen3_5_draft_hop.py diff --git a/sgl_nvidia_mega/qwen3_5/boundary/hop.py b/sgl_nvidia_mega/qwen3_5/boundary/hop.py index 94083545..de081384 100644 --- a/sgl_nvidia_mega/qwen3_5/boundary/hop.py +++ b/sgl_nvidia_mega/qwen3_5/boundary/hop.py @@ -128,6 +128,8 @@ # the (point, rows) the slot has run at (each logged once, as engagement evidence) _ENGAGED: set = set() +# the target's hop once built (active()) +_ACTIVE: Optional[object] = None class Bound(msgspec.Struct, frozen=True): @@ -162,9 +164,17 @@ def build(model, ctx) -> Bound: device=torch.device("cuda", torch.cuda.current_device()), deadline_s=ctx.deadline_s, ) + global _ACTIVE + _ACTIVE = hop return Bound(hop=hop) +def active() -> Optional[object]: + """The target's built hop (sm103.hop.Hop), or None: the MTP draft's hops share its + mailbox and stage order (built before the draft, which loads after the target).""" + return _ACTIVE + + def refusals(m, world: int) -> list[str]: """Why the hop cannot stand in for the CuTe DSL kernels of ``m`` (Qwen3_5ForCausalLM) at TP ``world``; [] when it can.""" diff --git a/sgl_nvidia_mega/qwen3_5/draft/__init__.py b/sgl_nvidia_mega/qwen3_5/draft/__init__.py index 74b83d5f..1a953f7e 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/draft/__init__.py @@ -13,8 +13,9 @@ (``extend``). Inside them every op is SGLang's own unless a draft slot replaces it: ``draft_moe`` -(the single-launch MoE), ``draft_q1`` (the q1 split-KV decode) and ``draft_xchg`` (the -step boundary's fc split, exchange and head). +(the single-launch MoE), ``draft_q1`` (the q1 split-KV decode), ``draft_xchg`` (the +step boundary's fc split, exchange and head) and ``draft_hop`` (the draft hops and the +o_proj GEMV). """ from __future__ import annotations @@ -69,7 +70,7 @@ def numerics(slots: frozenset) -> Numerics: which differ from SGLang's ops only in fp32 summation order (q1 split-KV, the o_proj / fc / MoE GEMVs, the draft head), so a draft token can differ where two logits are within a bf16 rounding.""" - if not slots & {"draft_moe", "draft_q1", "draft_xchg"}: + if not slots & {"draft_moe", "draft_q1", "draft_xchg", "draft_hop"}: return target.NUMERICS return Numerics(NumericsKind.TOLERANCE, {"draft_logits.rel": 2**-7}) diff --git a/sgl_nvidia_mega/qwen3_5/draft/attention.py b/sgl_nvidia_mega/qwen3_5/draft/attention.py index f03ed596..63fbaa2f 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/attention.py +++ b/sgl_nvidia_mega/qwen3_5/draft/attention.py @@ -156,9 +156,12 @@ def __init__(self, parts): self.v_scale = attn.v_scale if attn.v_scale is not None else ones self.q1 = None # sm103.draft.q1 decode (slot draft_q1), set by DraftLoop - def __call__(self, x, positions, cache_loc, meta, kv, *, rows_per_request: int): + def __call__( + self, x, positions, cache_loc, meta, kv, *, rows_per_request: int, o_proj=None + ): """The TP-partial attention output [rows, hidden] of the normed input ``x`` - [rows, hidden]; writes this step's KV rows at ``cache_loc``.""" + [rows, hidden]; writes this step's KV rows at ``cache_loc``. ``o_proj`` (the + draft hop's GEMV) stands in for SGLang's when given.""" from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul layer = self.layer @@ -168,6 +171,8 @@ def __call__(self, x, positions, cache_loc, meta, kv, *, rows_per_request: int): o = self.core(q, k, v, cache_loc, meta, kv, rows_per_request) if layer.attn_output_gate: o = fused_sigmoid_mul(o, gate, inplace=True) + if o_proj is not None: + return o_proj(o) out, _ = layer.o_proj(o) return out diff --git a/sgl_nvidia_mega/qwen3_5/draft/hop.py b/sgl_nvidia_mega/qwen3_5/draft/hop.py new file mode 100644 index 00000000..7daba97c --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/draft/hop.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0): python/sglang/srt/layers/layer_boundary/fusions/q35_draft_hop.py +# (DraftHopFusion) and python/sglang/srt/models/qwen3_5.py +# (Qwen3_5AttentionDecoderLayer._draft_o_proj), on branch nvidia/qwen35-agentx-gb300. +"""The one-row MTP draft step's two all-reduce + residual + norm points as hops on the +target's decode hop, and the draft o_proj GEMV that pushes into the first one. + +- ``ffn_input``: the attention output's sum, the residual add and the post-attention + norm, with FlashInfer's legacy MNNVL one-shot rounding (the norm's folded Gemma + multiplier ``gemma_weight``); +- ``final_norm``: the MoE output's sum, the residual add and the final norm, with the + rounding of SGLang's one-shot push all-reduce and FlashInfer's CuTe DSL + FusedAddRMSNorm (multiplier 1 + the raw weight, in fp32); +- ``o_proj``: the attention output's one-row o_proj (fp32 accumulation), pushed into + ``ffn_input``'s hop, which then only consumes. + +The draft shares the target's hop (``boundary.hop.active()``): one mailbox and one +stage order for every hop of the process, as SGLang runs the draft after the target's +build on every rank alike. +""" + +from __future__ import annotations + +from typing import Optional + +import torch + +from sgl_nvidia_mega.qwen3_5.sm103.draft import hop as kernels + +# The SGLang names this module reads (test/unit/test_contract.py pins them) +SGLANG_READS = { + "srt/layers/layernorm.py": ('"gemma_weight"', "weight", "variance_epsilon"), +} + +# Draft steps carry one row; the verify and extend widths keep their own kernels. +MAX_M = 1 + + +class DraftHop: + """The draft layer's hop points on the target's hop. ``early``: the FFN input's + consume releases its dependents at entry when the next launch is the single-launch + draft MoE; ``early_head``: the final norm's does when the next launch is the draft + head (both read only static data before their PDL wait).""" + + def __init__( + self, hop, *, early: bool = True, early_head: bool = False, delay_ns: int = 0 + ): + self.hop = hop # the target's sm103.hop.Hop + self.early = early + self.early_head = early and early_head + self.delay_ns = delay_ns # diagnostic: hold the consume's output stores + # (data_ptr, shape) of a partial a producer already pushed; its all-reduce + # only consumes + self._pushed: Optional[tuple[int, torch.Size]] = None + + @classmethod + def build( + cls, + *, + tp_size: int, + deadline_s: float, + o_proj_k: Optional[int] = None, + early: bool = True, + early_head: bool = False, + ) -> tuple[Optional[DraftHop], str]: + """(the draft hop, "") on the target's hop, compiling its kernels (and the o_proj + GEMV for reduction length ``o_proj_k``), or (None, why not).""" + from sgl_nvidia_mega.qwen3_5.boundary.hop import active + + hop = active() + if hop is None: + return None, "the target built no decode hop (slot hop off)" + if hop.world != tp_size or not hop.supports(MAX_M): + return None, ( + f"the target's hop serves TP {hop.world}, 1-{hop.max_m} rows, " + f"not TP {tp_size}, {MAX_M} row" + ) + kernels.build( + hop.device.index, + int(deadline_s * 1e9), + gemv_ks=() if o_proj_k is None else (o_proj_k,), + ) + return cls(hop, early=early, early_head=early_head), "" + + def supports(self, m: int) -> bool: + return 1 <= m <= MAX_M and self.hop.supports(m) + + def attn_push_target(self, m: int) -> Optional[DraftHop]: + """This hop when the attention output's producer may push its m rows into it, + else None.""" + return self if self.supports(m) else None + + def mark_pushed(self, partial: torch.Tensor) -> None: + """A producer kernel already pushed this bf16 partial; its all-reduce only + consumes.""" + if self._pushed is not None: + raise RuntimeError("draft hop: a pushed partial was never consumed") + self._pushed = (partial.data_ptr(), partial.shape) + + def has_pushed(self, partial: torch.Tensor) -> bool: + return self._pushed == (partial.data_ptr(), partial.shape) + + def _take_pushed(self, partial: torch.Tensor) -> bool: + if self.has_pushed(partial): + self._pushed = None + return True + if self._pushed is not None: + raise RuntimeError("draft hop: a pushed partial was never consumed") + return False + + def o_proj(self, attn_output: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: + """The one-row o_proj partial [1, N] bf16 (``weight`` [N, K] bf16), pushed into + the FFN input's hop when that hop takes the row.""" + out = torch.empty( + 1, weight.shape[0], dtype=torch.bfloat16, device=weight.device + ) + if self.attn_push_target(int(attn_output.shape[0])) is None: + return kernels.draft_gemv(attn_output, weight, out) + kernels.draft_gemv(attn_output, weight, out, hop_site=self.hop.site) + self.mark_pushed(out) + return out + + def ffn_input( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + norm, + *, + moe_single_launch: bool, + ) -> tuple[torch.Tensor, torch.Tensor]: + """(the MoE input, residual) of the attention output's partial + ``hidden_states``; ``norm`` is the layer's post-attention GemmaRMSNorm.""" + return kernels.draft_all_reduce_rms_norm( + self.hop, + local_contribution=hidden_states, + residual=residual, + gamma=norm.gemma_weight, + eps=norm.variance_epsilon, + weight_bias=0.0, + add_norm=False, + early_release=self.early and moe_single_launch, + delay_ns=self.delay_ns, + pushed=self._take_pushed(hidden_states), + ) + + def final_norm( + self, + partial: torch.Tensor, + residual: torch.Tensor, + norm, + *, + head_follows: bool = False, + ) -> tuple[torch.Tensor, torch.Tensor]: + """(the final hidden state, residual) of the MoE output's unreduced + ``partial``; ``norm`` is the model's final GemmaRMSNorm.""" + return kernels.draft_all_reduce_rms_norm( + self.hop, + local_contribution=partial, + residual=residual, + gamma=norm.weight, + eps=norm.variance_epsilon, + weight_bias=1.0, + add_norm=True, + early_release=self.early_head and head_follows, + delay_ns=self.delay_ns, + pushed=self._take_pushed(partial), + ) diff --git a/sgl_nvidia_mega/qwen3_5/draft/layer.py b/sgl_nvidia_mega/qwen3_5/draft/layer.py index 285cf4a1..7c7e73bd 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/layer.py +++ b/sgl_nvidia_mega/qwen3_5/draft/layer.py @@ -15,11 +15,13 @@ and their arg-max (``torch.argmax``'s choice, as SGLang's top-1). Stock: the ops SGLang's draft runs for these rows, in its order. Slot ``draft_moe`` -replaces the MoE. +replaces the MoE, slot ``draft_hop`` the two all-reduce points and ``o_proj``. """ from __future__ import annotations +from functools import partial + import torch # The SGLang names this module reads (test/unit/test_contract.py pins them) @@ -60,12 +62,20 @@ def fc_input(parts, token_ids, hidden): return lm.fc(torch.cat([embeds, hidden], dim=-1)) +# Slot draft_hop on the draft extend's rows too: their FFN input all-reduce + residual + +# norm in the hop's MNNVL rounding, which equals what 2.2 runs there (FlashInfer's MNNVL +# fusion under DRAFT_MNNVL_AR) bit for bit at 1-7 rows +EXTEND_MNNVL = True + + def layer(parts, x, positions, cache_loc, meta, kv, *, rows_per_request: int): """The decoder layer on the fc output ``x`` [rows, hidden]: the final-norm rows.""" from sglang.srt.distributed import tensor_model_parallel_all_reduce lay = parts.layer residual = x + hop = parts.hop if rows_per_request == 1 else None + extend_hop = parts.hop if rows_per_request > 1 and EXTEND_MNNVL else None attn_out = parts.attention( lay.input_layernorm(x), positions, @@ -73,12 +83,28 @@ def layer(parts, x, positions, cache_loc, meta, kv, *, rows_per_request: int): meta, kv, rows_per_request=rows_per_request, - ) - moe_in, residual = lay.post_attention_layernorm( - tensor_model_parallel_all_reduce(attn_out), residual + o_proj=None if hop is None else partial(hop.o_proj, weight=lay.o_proj.weight), ) moe = parts.moe if rows_per_request == 1 else None + if hop is not None: + moe_in, residual = hop.ffn_input( + attn_out, + residual, + lay.post_attention_layernorm, + moe_single_launch=moe is not None and moe.single_launch(1), + ) + elif extend_hop is not None: + moe_in, residual = extend_hop.ffn_input( + attn_out, residual, lay.post_attention_layernorm, moe_single_launch=False + ) + else: + moe_in, residual = lay.post_attention_layernorm( + tensor_model_parallel_all_reduce(attn_out), residual + ) moe_out = moe(moe_in) if moe is not None else lay.mlp(moe_in) + if hop is not None: + out, _ = hop.final_norm(moe_out, residual, parts.model.norm) + return out out, _ = parts.model.norm(tensor_model_parallel_all_reduce(moe_out), residual) return out diff --git a/sgl_nvidia_mega/qwen3_5/draft/parts.py b/sgl_nvidia_mega/qwen3_5/draft/parts.py index 936d5cf9..c6f9a241 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/parts.py +++ b/sgl_nvidia_mega/qwen3_5/draft/parts.py @@ -62,6 +62,7 @@ def __init__(self, draft, layer): self.device = torch.device("cuda", torch.cuda.current_device()) self.attention = None # draft.attention.Attention self.moe: Optional[Any] = None # draft.moe.DraftMoe (slot draft_moe) + self.hop: Optional[Any] = None # draft.hop.DraftHop (slot draft_hop) self.mkd: Optional[Any] = None # draft.xchg.MkdBoundary (slot draft_xchg) # (entry, bs) pairs whose capture logged the engagement line self.engaged: set = set() @@ -81,13 +82,16 @@ def build(cls, draft) -> Parts: slots = parts.slots if "draft_moe" in slots: parts.moe = parts._build_moe() + if "draft_hop" in slots: + parts.hop = parts._build_hop() if "draft_xchg" in slots: parts.mkd = parts._build_mkd() logger.info( - "NVIDIA megakernel: Qwen3.5 MTP draft parts built (slots %s; MoE %s, " - "exchange %s)", + "NVIDIA megakernel: Qwen3.5 MTP draft parts built (slots %s; MoE %s, hop " + "%s, exchange %s)", sorted(slots), "single launch" if parts.moe is not None else "SGLang's", + "draft hop" if parts.hop is not None else "SGLang's", "MKD" if parts.mkd is not None else "SGLang's", ) return parts @@ -103,6 +107,20 @@ def _build_moe(self): ) return moe + def _build_hop(self): + from sgl_nvidia_mega.qwen3_5.draft.hop import DraftHop + + hop, why = DraftHop.build( + tp_size=self.binding.tp_size, + deadline_s=self.draft.deadline_s, + o_proj_k=int(self.layer.o_proj.weight.shape[1]), + early=True, + early_head=False, + ) + if hop is None: + raise RuntimeError(f"NVIDIA megakernel: slot draft_hop cannot run: {why}") + return hop + def _build_mkd(self): from sgl_nvidia_mega.qwen3_5.draft.xchg import MkdBoundary diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index 4ac0b4c8..e46d0d65 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -30,7 +30,7 @@ # The MTP draft's EAGLE entries and kernel groups (qwen3_5/draft), requested in the same # variable; the draft spec's build reads them, the target's ignores them DRAFT_ENTRIES: tuple[str, ...] = ("draft_loop", "draft_extend") -DRAFT_SLOTS: tuple[str, ...] = ("draft_moe", "draft_q1", "draft_xchg") +DRAFT_SLOTS: tuple[str, ...] = ("draft_moe", "draft_q1", "draft_xchg", "draft_hop") SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" @@ -86,6 +86,8 @@ def slot_reasons() -> list[str]: for name in sorted(SLOTS & set(DRAFT_SLOTS)): if "draft_loop" not in SLOTS: reasons.append(f"{SLOTS_ENV}: slot {name!r} runs inside entry 'draft_loop'") + if "draft_hop" in SLOTS and "hop" not in SLOTS: + reasons.append(f"{SLOTS_ENV}: slot 'draft_hop' runs on slot 'hop''s mailbox") if "mka_push" in SLOTS and not {"mka", "hop"} <= SLOTS: reasons.append( f"{SLOTS_ENV}: slot 'mka_push' pushes slot 'mka''s epilogue into slot " diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py index 2ef16834..549b5c62 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/__init__.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. """The MTP draft's SM103 (GB300) kernels: the single-launch draft MoE (``moe``), the -q1 split-KV decode attention (``q1``) and the draft step boundary's exchange, fc and -head (``xchg``).""" +q1 split-KV decode attention (``q1``), the draft step boundary's exchange, fc and head +(``xchg``), and the draft hop consume and o_proj GEMV (``hop``).""" diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/hop.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/hop.py new file mode 100644 index 00000000..8d5c4723 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/hop.py @@ -0,0 +1,248 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0): python/sglang/srt/layers/q35_hop.py +# (Q35Hop.draft_all_reduce_rms_norm) and python/sglang/kernels/ops/speculative/mkd_draft_gemv.py, +# on branch nvidia/qwen35-agentx-gb300. +"""The MTP draft's hops on the decode hop's mailbox (``sm103/hop``), and its o_proj GEMV. + +``draft_all_reduce_rms_norm`` pushes this rank's partial with the decode hop's +``q35hop_publish`` (skipped when a producer already pushed it) and consumes with +``src/draft_hop.cu``, which keeps the rounding of the kernel the draft step replaces: +``add_norm=False`` FlashInfer's legacy MNNVL one-shot all-reduce + residual + RMSNorm +(``gamma`` the applied multiplier), ``add_norm=True`` SGLang's one-shot push all-reduce, +then FlashInfer's CuTe DSL FusedAddRMSNorm (multiplier ``weight_bias + gamma``). +``draft_gemv`` (``src/draft_gemv.cu``) is the one-row bf16 GEMV with fp32 accumulation; +with ``hop_site`` it also pushes its output into the hop, whose all-reduce then only +consumes. + +Kernels compile in ``build`` (never under CUDA-graph capture); the consume's wait +deadline is compiled in, as the decode hop's. +""" + +from __future__ import annotations + +import ctypes +from functools import cache +from pathlib import Path +from typing import Optional + +import torch + +from sgl_nvidia_mega.common import jit +from sgl_nvidia_mega.qwen3_5.sm103 import hop as decode_hop + +SOURCE = Path(__file__).parent / "src" / "draft_hop.cu" +GEMV_SOURCE = Path(__file__).parent / "src" / "draft_gemv.cu" +HIDDEN = decode_hop.HIDDEN +GEMV_ROWS, GEMV_PUSH_ROWS = 4, 8 # warps (rows in flight) per CTA + + +class Args(ctypes.Structure): + """q35hop_draft::Args: the decode hop's Args with the draft consume's fields.""" + + _fields_ = list(decode_hop.Args._fields_) + [ + ("weight_bias", ctypes.c_float), + ("delay_ns", ctypes.c_int), + ] + + +class GemvParams(ctypes.Structure): + """q35draft_gemv::Params.""" + + _fields_ = [ + ("x", ctypes.c_void_p), + ("w", ctypes.c_void_p), + ("y", ctypes.c_void_p), + ("n", ctypes.c_int), + ("hop", decode_hop.Site), + ("hop_token", ctypes.c_int), + ] + + +def consume_name(world: int, add_norm: bool, early: bool) -> str: + mode = "add_norm" if add_norm else "mnnvl" + return f"q35hop_draft_{mode}{'_early' if early else ''}_w{world}" + + +@cache +def _consume_kernels(device_index: int, deadline_ns: int) -> dict[str, jit.Kernel]: + major, minor = torch.cuda.get_device_capability(device_index) + cubin = jit.compile_source( + SOURCE.read_text(), + "q35_draft_hop", + f"sm_{major}{minor}a", + ("-std=c++17", "-default-device", f"-DQ35_HOP_DEADLINE_NS={deadline_ns}ull"), + ) + names = [ + consume_name(w, add_norm, early) + for w in decode_hop.WORLDS + for add_norm in (False, True) + for early in (False, True) + ] + return {n: jit.load(cubin, n, (decode_hop.ROW_THREADS, 1, 1), 0) for n in names} + + +@cache +def _gemv_kernels(device_index: int, k: int) -> tuple[jit.Kernel, jit.Kernel, int]: + """(plain, pushing, the plain kernel's grid cap: resident CTAs on the device).""" + if k % 256: + raise ValueError(f"draft GEMV: K {k} is not a multiple of 256") + major, minor = torch.cuda.get_device_capability(device_index) + cubin = jit.compile_source( + GEMV_SOURCE.read_text(), + f"q35_draft_gemv_k{k}", + f"sm_{major}{minor}a", + ("-std=c++17", "-default-device", f"-DQ35_DRAFT_GEMV_K={k}"), + ) + plain = jit.load(cubin, "q35_draft_gemv", (GEMV_ROWS * 32, 1, 1), 0) + push = jit.load(cubin, "q35_draft_gemv_push", (GEMV_PUSH_ROWS * 32, 1, 1), 0) + cu = jit.driver() + (per_sm,) = jit.check( + cu.cuOccupancyMaxActiveBlocksPerMultiprocessor( + plain.function, GEMV_ROWS * 32, 0 + ), + "cuOccupancyMaxActiveBlocksPerMultiprocessor", + ) + sms = torch.cuda.get_device_properties(device_index).multi_processor_count + return plain, push, int(per_sm) * sms + + +# device index -> the consume kernels built for it +_CONSUME: dict[int, dict[str, jit.Kernel]] = {} + + +def build(device_index: int, deadline_ns: int, gemv_ks: tuple[int, ...] = ()) -> None: + """Compile the consume kernels (with the decode hop's wait deadline) and the GEMV + for each reduction length in ``gemv_ks``.""" + _CONSUME[device_index] = _consume_kernels(device_index, int(deadline_ns)) + for k in gemv_ks: + _gemv_kernels(device_index, int(k)) + + +def draft_all_reduce_rms_norm( + hop, + *, + local_contribution: torch.Tensor, + residual: torch.Tensor, + gamma: torch.Tensor, + eps: float, + weight_bias: float, + add_norm: bool, + early_release: bool = False, + delay_ns: int = 0, + pushed: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + """(norm, residual_out) [M, 4096] bf16 of sum_ranks(partial) + residual on ``hop`` + (an ``sm103.hop.Hop``). ``pushed``: a producer already pushed this partial, so only + the consume runs. ``early_release``: the consume releases its dependents at entry + (they must read only static data before their PDL wait).""" + m = int(local_contribution.shape[0]) + shape = (m, HIDDEN) + if ( + not hop.supports(m) + or tuple(residual.shape) != shape + or tuple(gamma.shape) != (HIDDEN,) + or residual.dtype != torch.bfloat16 + or gamma.dtype != torch.bfloat16 + or not (residual.is_contiguous() and gamma.is_contiguous()) + ): + raise ValueError( + f"draft hop: rows {m} (1-{hop.max_m}), residual {residual.dtype} " + f"{tuple(residual.shape)}, gamma {gamma.dtype} {tuple(gamma.shape)}" + ) + stream = torch.cuda.current_stream(hop.device).cuda_stream + norm = torch.empty(shape, dtype=torch.bfloat16, device=hop.device) + residual_out = torch.empty(shape, dtype=torch.bfloat16, device=hop.device) + if not pushed: + if ( + tuple(local_contribution.shape) != shape + or local_contribution.dtype != torch.bfloat16 + or not local_contribution.is_contiguous() + ): + raise ValueError( + f"draft hop: partial {local_contribution.dtype} " + f"{tuple(local_contribution.shape)}" + ) + publish = decode_hop.Args( + site=hop.site, partial=local_contribution.data_ptr(), m=m, eps=eps + ) + jit.launch( + hop.kernels["q35hop_publish"], + (m, decode_hop.ROW_CTAS, 1), + [publish], + stream, + ) + args = Args( + site=hop.site, + residual_in=residual.data_ptr(), + gamma=gamma.data_ptr(), + residual_out=residual_out.data_ptr(), + norm_out=norm.data_ptr(), + m=m, + eps=eps, + weight_bias=weight_bias, + delay_ns=delay_ns, + ) + kernel = _CONSUME[hop.device.index][ + consume_name(hop.world, add_norm, early_release) + ] + jit.launch( + kernel, + (m, decode_hop.ROW_CTAS, 1), + [args], + stream, + cluster=(1, decode_hop.ROW_CTAS, 1), + ) + return norm, residual_out + + +def draft_gemv( + x: torch.Tensor, + weight: torch.Tensor, + out: torch.Tensor, + hop_site: Optional[decode_hop.Site] = None, + hop_token: int = 0, +) -> torch.Tensor: + """``out`` [1, N] bf16 = ``x`` [1, K] @ ``weight`` [N, K]^T with fp32 accumulation, + one warp per row. ``hop_site``: the decode hop's ``Site``; the output (N = 4096) is + then also pushed into that hop as row ``hop_token``, and its all-reduce only + consumes.""" + k, n = int(x.shape[-1]), int(weight.shape[0]) + if ( + x.dtype != torch.bfloat16 + or weight.dtype != torch.bfloat16 + or out.dtype != torch.bfloat16 + or tuple(x.shape) != (1, k) + or tuple(weight.shape) != (n, k) + or tuple(out.shape) != (1, n) + or not (x.is_contiguous() and weight.is_contiguous() and out.is_contiguous()) + or x.data_ptr() % 16 + or weight.data_ptr() % 16 + ): + raise ValueError( + f"draft GEMV: x {x.dtype} {tuple(x.shape)}, weight {weight.dtype} " + f"{tuple(weight.shape)}, out {out.dtype} {tuple(out.shape)}" + ) + plain, push, resident = _gemv_kernels(x.device.index, k) + params = GemvParams( + x=x.data_ptr(), + w=weight.data_ptr(), + y=out.data_ptr(), + n=n, + hop=hop_site if hop_site is not None else decode_hop.Site(), + hop_token=hop_token, + ) + if hop_site is not None: + if n != HIDDEN or not 0 <= hop_token < hop_site.m_cap: + raise ValueError( + f"draft GEMV: a pushed partial is one hidden row of {HIDDEN} at a " + f"hop row below {hop_site.m_cap}, not {n} at row {hop_token}" + ) + kernel, grid = push, (n + GEMV_PUSH_ROWS - 1) // GEMV_PUSH_ROWS + else: + # rows beyond one resident wave loop inside the warps + kernel, grid = plain, min((n + GEMV_ROWS - 1) // GEMV_ROWS, resident) + jit.launch( + kernel, (grid, 1, 1), [params], torch.cuda.current_stream(x.device).cuda_stream + ) + return out diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_gemv.cu b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_gemv.cu new file mode 100644 index 00000000..5712ba33 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_gemv.cu @@ -0,0 +1,450 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Adapted from SGLang (Apache-2.0): python/sglang/kernels/jit/csrc/speculative/mkd_draft_gemv.cuh +// (branch nvidia/qwen35-agentx-gb300). +// One-row draft GEMV y[n] = bf16(sum_k W[n, k] * x[k]): bf16 operands, fp32 accumulation, output rounded +// to bf16 as the served GEMM writes it (only the fp32 summation order differs). One warp per row with the +// whole row in flight, the row loaded before the PDL wait (weights are static). With a hop site, every CTA +// owns one 8-row fragment of the output and pushes it into the decode hop (sm103/hop) before storing it. +// The q35hop namespace below is hop.cu's, copied verbatim (the push must follow its mailbox protocol; +// test/kernels/test_qwen3_5_draft_hop.py compares the text). +// +// Built with NVRTC per reduction length Q35_DRAFT_GEMV_K: C-linkage entries q35_draft_gemv (4 rows a +// CTA) and q35_draft_gemv_push (8 rows a CTA, pushing). + +#include + +// NVRTC has no +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; + +#ifndef Q35_HOP_DEADLINE_NS +#define Q35_HOP_DEADLINE_NS 0ull +#endif + +namespace q35hop { + + +constexpr int kHidden = 4096; +constexpr int kVec = 8; +constexpr int kFrags = kHidden / kVec; +constexpr int kStages = 3; +constexpr int kRowCtas = 4; +constexpr int kRowThreads = 128; +constexpr int kRowWarps = kRowThreads / 32; +constexpr uint32_t kNegZeroPair = 0x80008000u; +static_assert(kRowCtas * kRowThreads == kFrags, "a row is one fragment per thread of the cluster"); +static_assert(kRowWarps == 4, "row_sumsq_fold folds exactly four warp sums per CTA"); + +// Per-process control block in device memory. +struct Ctrl { + uint32_t stage; // stage of the next hop + uint32_t arrive; // CTA tickets of the running consumer launch + uint32_t seq; // hops completed + uint32_t pad; +}; + +// Passed by value as a kernel parameter. +struct Site { + void* mc; // multicast base of the mailbox [kStages][world][m_cap][kHidden] bf16 + void* uc; // this rank's unicast view of the same mailbox + Ctrl* ctrl; + int rank, world, m_cap; +}; + +// ---------------------------------------------------------------- PTX + +__device__ __forceinline__ void pdl_wait() { asm volatile("griddepcontrol.wait;" ::: "memory"); } +__device__ __forceinline__ void pdl_trigger() { asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); } +__device__ __forceinline__ unsigned long long gtimer() { + unsigned long long t; + asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t)); + return t; +} +__device__ __forceinline__ uint32_t ld_volatile_u32(const uint32_t* p) { + uint32_t v; + asm volatile("ld.volatile.global.u32 %0, [%1];" : "=r"(v) : "l"(p) : "memory"); + return v; +} +__device__ __forceinline__ uint4 ld_volatile_v4(const void* p) { + uint4 v; + asm volatile("ld.volatile.global.v4.u32 {%0,%1,%2,%3}, [%4];" + : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) + : "l"(p) + : "memory"); + return v; +} +__device__ __forceinline__ void st_v4(void* p, uint4 v) { + asm volatile("st.global.v4.u32 [%0], {%1,%2,%3,%4};" ::"l"(p), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w) + : "memory"); +} +__device__ __forceinline__ void mc_st_v4(void* mc, uint4 v) { + asm volatile("multimem.st.relaxed.sys.global.v4.bf16x2 [%0], {%1,%2,%3,%4};" ::"l"(mc), "r"(v.x), + "r"(v.y), "r"(v.z), "r"(v.w) + : "memory"); +} +__device__ __forceinline__ void mc_st_v2(void* mc, uint2 v) { + asm volatile("multimem.st.relaxed.sys.global.v2.bf16x2 [%0], {%1,%2};" ::"l"(mc), "r"(v.x), "r"(v.y) : "memory"); +} +__device__ __forceinline__ uint32_t cluster_ctarank() { + uint32_t r; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(r)); + return r; +} +__device__ __forceinline__ float div_full(float a, float b) { + float r; + asm("div.full.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} +// Two f32 to an e4m3 pair; lo lands in the low byte. +__device__ __forceinline__ uint16_t cvt_e4m3x2(float lo, float hi) { + uint16_t r; + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(r) : "f"(hi), "f"(lo)); + return r; +} + +// ---------------------------------------------------------------- bf16 + +__device__ __forceinline__ float bf_lo(uint32_t w) { return __uint_as_float(w << 16); } +__device__ __forceinline__ float bf_hi(uint32_t w) { return __uint_as_float(w & 0xFFFF0000u); } +__device__ __forceinline__ uint32_t pack_bf16x2(float lo, float hi) { + __nv_bfloat162 v = __floats2bfloat162_rn(lo, hi); + return *reinterpret_cast(&v); +} +__device__ __forceinline__ void unpack8(uint4 v, float f[8]) { + f[0] = bf_lo(v.x); f[1] = bf_hi(v.x); f[2] = bf_lo(v.y); f[3] = bf_hi(v.y); + f[4] = bf_lo(v.z); f[5] = bf_hi(v.z); f[6] = bf_lo(v.w); f[7] = bf_hi(v.w); +} +__device__ __forceinline__ uint4 pack8(const float f[8]) { + return make_uint4(pack_bf16x2(f[0], f[1]), pack_bf16x2(f[2], f[3]), pack_bf16x2(f[4], f[5]), + pack_bf16x2(f[6], f[7])); +} +__device__ __forceinline__ bool word_has_negzero(uint32_t w) { + return (w & 0xFFFFu) == 0x8000u || (w >> 16) == 0x8000u; +} +__device__ __forceinline__ bool frag_has_negzero(uint4 v) { + return word_has_negzero(v.x) | word_has_negzero(v.y) | word_has_negzero(v.z) | word_has_negzero(v.w); +} +__device__ __forceinline__ uint32_t sanitize_word(uint32_t w) { + if ((w & 0xFFFFu) == 0x8000u) w &= 0xFFFF0000u; + if ((w >> 16) == 0x8000u) w &= 0x0000FFFFu; + return w; +} + +// ---------------------------------------------------------------- stage protocol + +__device__ __forceinline__ size_t slot(const Site& s, uint32_t stage, int src, int token, int frag) { + return ((static_cast(stage) * s.world + src) * s.m_cap + token) * kHidden + + static_cast(frag) * kVec; +} + +struct Hop { + uint32_t stage; + uint32_t seq; + uint32_t ticket; // thread 0 only +}; + +// Consumer launch, every thread of every participating CTA, after griddepcontrol.wait. Thread +// 0 takes a ticket whose value is consumed only in hop_leave, so the atomic stays off the +// critical path. +__device__ __forceinline__ Hop hop_enter(const Site& s) { + Hop h; + h.stage = ld_volatile_u32(&s.ctrl->stage); + h.seq = ld_volatile_u32(&s.ctrl->seq); + h.ticket = threadIdx.x == 0 ? atomicAdd(&s.ctrl->arrive, 1u) : 0u; + return h; +} + +// Thread 0 of every participating CTA; the last arrival advances the stage. No CTA of this +// launch can read the advanced value, since every CTA read the stage before taking its ticket. +__device__ __forceinline__ void hop_leave(const Site& s, const Hop& h, uint32_t participants) { + if (threadIdx.x == 0 && h.ticket == participants - 1) { + s.ctrl->arrive = 0; + s.ctrl->stage = (h.stage + 1) % kStages; + s.ctrl->seq = h.seq + 1; + } +} + +// Producer-only launch, after griddepcontrol.wait. +__device__ __forceinline__ uint32_t stage_peek(const Site& s) { return ld_volatile_u32(&s.ctrl->stage); } + +// ---------------------------------------------------------------- producer + +// This rank's final bf16 partial for one fragment; exactly once per (hop, rank, token, frag). +// A launch that also stores its own outputs should push first: the push starts the flight. +__device__ __forceinline__ void push_frag(const Site& s, uint32_t stage, int token, int frag, uint4 v) { + v = make_uint4(sanitize_word(v.x), sanitize_word(v.y), sanitize_word(v.z), sanitize_word(v.w)); + mc_st_v4(static_cast(s.mc) + slot(s, stage, s.rank, token, frag), v); +} + +// Same for 4 consecutive bf16 (half a fragment, columns [4q, 4q + 4)); a fragment may be pushed +// as two halves by different threads. +__device__ __forceinline__ void push_quad(const Site& s, uint32_t stage, int token, int quad, uint2 v) { + v = make_uint2(sanitize_word(v.x), sanitize_word(v.y)); + mc_st_v2(static_cast(s.mc) + slot(s, stage, s.rank, token, quad >> 1) + (quad & 1) * 4, v); +} + +// ---------------------------------------------------------------- consumer + +// A peer that missed the wait deadline: store to address 0x0, then trap (fail-stop). +__device__ __forceinline__ void fault_at_null() { + asm volatile("{\n.reg .u64 a;\nmov.u64 a, 0;\nst.volatile.global.u32 [a], 0;\ntrap;\n}" ::: "memory"); +} + +// Sum over ranks 0..W-1 (fp32, from 0) of one fragment; spins on the sentinel. A wait past +// Q35_HOP_DEADLINE_NS (if not 0) stops the kernel. +template +__device__ __forceinline__ bool pull_frag(const Site& s, uint32_t stage, int token, int frag, float acc[8]) { + const uint16_t* uc = static_cast(s.uc); + uint4 v[W]; +#pragma unroll + for (int r = 0; r < W; ++r) v[r] = ld_volatile_v4(uc + slot(s, stage, r, token, frag)); + uint32_t pending = 0; +#pragma unroll + for (int r = 0; r < W; ++r) pending |= frag_has_negzero(v[r]) ? (1u << r) : 0u; + unsigned long long t0 = 0; + uint32_t spins = 0; + while (pending) { +#pragma unroll + for (int r = 0; r < W; ++r) + if (pending & (1u << r)) v[r] = ld_volatile_v4(uc + slot(s, stage, r, token, frag)); + uint32_t still = 0; +#pragma unroll + for (int r = 0; r < W; ++r) + if ((pending & (1u << r)) && frag_has_negzero(v[r])) still |= 1u << r; + pending = still; + if (Q35_HOP_DEADLINE_NS != 0 && pending && ((++spins & 255u) == 0)) { + const unsigned long long now = gtimer(); + if (t0 == 0) { + t0 = now; + } else if (now - t0 > Q35_HOP_DEADLINE_NS) { + fault_at_null(); + } + } + } +#pragma unroll + for (int e = 0; e < 8; ++e) acc[e] = 0.f; +#pragma unroll + for (int r = 0; r < W; ++r) { + float f[8]; + unpack8(v[r], f); +#pragma unroll + for (int e = 0; e < 8; ++e) acc[e] = __fadd_rn(acc[e], f[e]); + } + return true; +} + +// Write the sentinel back into the W slots this thread consumed (any time after pull_frag). +template +__device__ __forceinline__ void rearm_frag(const Site& s, uint32_t stage, int token, int frag) { + uint16_t* uc = static_cast(s.uc); + const uint4 z = make_uint4(kNegZeroPair, kNegZeroPair, kNegZeroPair, kNegZeroPair); +#pragma unroll + for (int r = 0; r < W; ++r) st_v4(uc + slot(s, stage, r, token, frag), z); +} + +// ---------------------------------------------------------------- row finish + +// prenorm = bf16(acc + residual); pre[] gets the bf16-rounded values the norm uses. +__device__ __forceinline__ uint4 residual_add(const float acc[8], uint4 residual, float pre[8]) { + float r[8]; + unpack8(residual, r); +#pragma unroll + for (int e = 0; e < 8; ++e) pre[e] = __fadd_rn(acc[e], r[e]); + const uint4 out = pack8(pre); + unpack8(out, pre); + return out; +} + +__device__ __forceinline__ float sumsq8(const float x[8]) { + float s = 0.f; +#pragma unroll + for (int e = 0; e < 8; ++e) s = __fadd_rn(s, __fmul_rn(x[e], x[e])); + return s; +} + +struct RowSmem { + float xs[kRowCtas][kRowWarps]; + uint64_t mbar; +}; + +// Prologue of every CTA of the row cluster, all threads, before griddepcontrol.wait: arm the +// exchange barrier and make it visible to the peers before any of them can send. +__device__ __forceinline__ void row_init(RowSmem& rs) { + if (threadIdx.x == 0) { + const uint32_t a = static_cast(__cvta_generic_to_shared(&rs.mbar)); + asm volatile("mbarrier.init.shared::cta.b64 [%0], 1;" ::"r"(a) : "memory"); + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(a), + "r"(static_cast(sizeof(rs.xs))) + : "memory"); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); + } + asm volatile("barrier.cluster.arrive.relaxed.aligned;\n\tbarrier.cluster.wait.acquire.aligned;" ::: "memory"); +} + +// Row total of the per-thread values: lane butterfly per warp, then each warp sends its sum to +// every CTA of the cluster with st.async; each CTA waits on its own mbarrier (no release fence, +// so this thread's earlier remote stores are not waited for) and folds per CTA as +// (w0 + w2) + (w1 + w3), then the CTAs in rank order from 0. +__device__ __forceinline__ float row_sumsq_fold(float v, RowSmem& rs) { + const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5; +#pragma unroll + for (int off = 1; off < 32; off <<= 1) v = __fadd_rn(v, __shfl_xor_sync(0xffffffffu, v, off)); + if (lane < kRowCtas) { + const uint32_t me = cluster_ctarank(); + const uint32_t slot_l = static_cast(__cvta_generic_to_shared(&rs.xs[me][warp])); + const uint32_t mbar_l = static_cast(__cvta_generic_to_shared(&rs.mbar)); + uint32_t slot_r, mbar_r; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(slot_r) : "r"(slot_l), "r"(lane)); + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(mbar_r) : "r"(mbar_l), "r"(lane)); + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.f32 [%0], %1, [%2];" ::"r"(slot_r), + "f"(v), "r"(mbar_r) + : "memory"); + } + { + const uint32_t a = static_cast(__cvta_generic_to_shared(&rs.mbar)); + asm volatile( + "{\n\t.reg .pred P;\n\tWAIT_ROW:\n\t" + "mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P, [%0], 0;\n\t" + "@!P bra WAIT_ROW;\n\t}" ::"r"(a) + : "memory"); + } + float total = 0.f; +#pragma unroll + for (int c = 0; c < kRowCtas; ++c) + total = __fadd_rn(total, __fadd_rn(__fadd_rn(rs.xs[c][0], rs.xs[c][2]), __fadd_rn(rs.xs[c][1], rs.xs[c][3]))); + return total; +} + +__device__ __forceinline__ float inv_rms(float row_sumsq, float eps) { + return rsqrtf(__fadd_rn(row_sumsq / static_cast(kHidden), eps)); +} + +// bf16((x * inv) * gamma), gamma being the applied multiplier. +__device__ __forceinline__ uint4 norm_apply(const float pre[8], float inv, const float gamma[8]) { + float out[8]; +#pragma unroll + for (int e = 0; e < 8; ++e) out[e] = __fmul_rn(__fmul_rn(pre[e], inv), gamma[e]); + return pack8(out); +} + +// The reciprocal the static per-tensor FP8 quant multiplies by. +__device__ __forceinline__ float inv_scale_of(float scale) { return div_full(1.f, scale); } + +// e4m3(clamp(x * inv_scale, +-448)) of the bf16 norm output, 8 bytes. +__device__ __forceinline__ uint2 quant_fp8(uint4 norm_bf16, float inv_scale) { + float q[8]; + unpack8(norm_bf16, q); + uint32_t w[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const float a = fminf(fmaxf(__fmul_rn(q[2 * j], inv_scale), -448.f), 448.f); + const float b = fminf(fmaxf(__fmul_rn(q[2 * j + 1], inv_scale), -448.f), 448.f); + w[j] = cvt_e4m3x2(a, b); + } + return make_uint2(w[0] | (w[1] << 16), w[2] | (w[3] << 16)); +} + +} // namespace q35hop + +typedef long long int64_t; + +#ifndef Q35_DRAFT_GEMV_K +#define Q35_DRAFT_GEMV_K 1024 +#endif + +namespace q35draft_gemv { + +struct Params { + const __nv_bfloat16* x; // [K], written by the previous grid + const __nv_bfloat16* __restrict__ w; // [N, K] + __nv_bfloat16* y; // [N] + int32_t n; + q35hop::Site hop; // kPush: the hop the fragments go to + int32_t hop_token; // kPush: the output's row in the hop +}; + +// SGLang's device::warp::reduce_sum over the full warp: lane masks 16 down to 1, plain fp32 adds +__device__ __forceinline__ float warp_reduce_sum(float value) { +#pragma unroll + for (uint32_t mask = 16; mask >= 1; mask >>= 1) value = value + __shfl_xor_sync(0xffffffffu, value, mask, 32); + return value; +} + +/** + * \brief kRows warps per CTA; warp w of CTA c takes rows kRows * c + w, then a grid stride further on, the + * next row loaded while the current one is reduced (as the draft head does). + * \tparam kK reduction length (a multiple of 256) + * \tparam kRows warps per CTA; 8 when pushing, with one row per warp so a CTA owns one hop fragment + */ +template +__device__ __forceinline__ void mkd_gemv_kernel(const Params& p) { + static_assert(kK % 256 == 0, "K must tile 32 lanes x 8 bf16"); + static_assert(!kPush || kRows == 8, "a pushing CTA owns exactly one 8-row fragment"); + constexpr int kVecs = kK / 8 / 32; + __shared__ alignas(16) __nv_bfloat16 s_x[kK]; + __shared__ alignas(16) __nv_bfloat16 s_y[kRows]; + __shared__ uint32_t s_stage; + const int warp = static_cast(threadIdx.x / 32); + const int lane = static_cast(threadIdx.x % 32); + const int stride = static_cast(gridDim.x) * kRows; + int row = static_cast(blockIdx.x) * kRows + warp; + uint4 wv[kVecs]; + if (row < p.n) { + const uint4* wr = reinterpret_cast(p.w + static_cast(row) * kK); +#pragma unroll + for (int j = 0; j < kVecs; ++j) wv[j] = __ldg(wr + j * 32 + lane); + } + if (kUsePDL) q35hop::pdl_wait(); + for (int i = static_cast(threadIdx.x); i < kK / 8; i += kRows * 32) { + reinterpret_cast(s_x)[i] = reinterpret_cast(p.x)[i]; + } + if constexpr (kPush) { + if (threadIdx.x == 0) s_stage = q35hop::stage_peek(p.hop); + } + __syncthreads(); + for (; row < p.n; row += stride) { + float acc = 0.0f; +#pragma unroll + for (int j = 0; j < kVecs; ++j) { + const uint4 xv = reinterpret_cast(s_x)[j * 32 + lane]; + const __nv_bfloat16* a = reinterpret_cast(&wv[j]); + const __nv_bfloat16* b = reinterpret_cast(&xv); +#pragma unroll + for (int e = 0; e < 8; ++e) acc = fmaf(__bfloat162float(a[e]), __bfloat162float(b[e]), acc); + } + const int next = row + stride; + if (next < p.n) { + const uint4* wr = reinterpret_cast(p.w + static_cast(next) * kK); +#pragma unroll + for (int j = 0; j < kVecs; ++j) wv[j] = __ldg(wr + j * 32 + lane); + } + acc = warp_reduce_sum(acc); + const __nv_bfloat16 v = __float2bfloat16_rn(acc); + if constexpr (kPush) { + if (lane == 0) s_y[warp] = v; + __syncthreads(); + // Push before the store: the push starts the flight to every rank. + if (threadIdx.x == 0) + q35hop::push_frag(p.hop, s_stage, p.hop_token, static_cast(blockIdx.x), + *reinterpret_cast(s_y)); + } + if (lane == 0) p.y[row] = v; + } + if (kUsePDL) q35hop::pdl_trigger(); +} + +} // namespace q35draft_gemv + +using q35draft_gemv::Params; + +extern "C" __global__ void __launch_bounds__(4 * 32) q35_draft_gemv(const Params __grid_constant__ p) { + q35draft_gemv::mkd_gemv_kernel(p); +} + +extern "C" __global__ void __launch_bounds__(8 * 32) q35_draft_gemv_push(const Params __grid_constant__ p) { + q35draft_gemv::mkd_gemv_kernel(p); +} diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_hop.cu b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_hop.cu new file mode 100644 index 00000000..93d06aff --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_hop.cu @@ -0,0 +1,621 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Adapted from SGLang (Apache-2.0): python/sglang/kernels/jit/csrc/distributed/q35_hop_kernels.cu +// (consume_draft_kernel and its helpers) on include/sgl_kernel/distributed/q35_hop.cuh +// (branch nvidia/qwen35-agentx-gb300). +// The draft hop: the consume of a one-row MTP draft step's all-reduce + residual + RMSNorm on the +// decode hop's mailbox (sm103/hop/src/hop.cu, whose producer q35hop_publish pushes the partials). The +// draft keeps the rounding of the kernels these replace; only the transport differs: +// mnnvl FlashInfer's legacy MNNVL one-shot all-reduce + residual + RMSNorm (fp32 rank sum +// rounded to bf16, bf16 residual add, bf16 squares, its warp / block / cluster order) +// add_norm SGLang's one-shot push all-reduce, then FlashInfer's CuTe DSL FusedAddRMSNorm (fp32 +// residual add, its per-thread fma order and butterfly) +// The q35hop namespace below is hop.cu's, copied verbatim (Site, Ctrl and the mailbox protocol +// must stay identical; test/kernels/test_qwen3_5_draft_hop.py compares the text). +// +// Built with NVRTC: C-linkage entries per instantiation; Q35_HOP_DEADLINE_NS as in hop.cu. + +#include + +// NVRTC has no +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; + +#ifndef Q35_HOP_DEADLINE_NS +#define Q35_HOP_DEADLINE_NS 0ull +#endif + +namespace q35hop { + + +constexpr int kHidden = 4096; +constexpr int kVec = 8; +constexpr int kFrags = kHidden / kVec; +constexpr int kStages = 3; +constexpr int kRowCtas = 4; +constexpr int kRowThreads = 128; +constexpr int kRowWarps = kRowThreads / 32; +constexpr uint32_t kNegZeroPair = 0x80008000u; +static_assert(kRowCtas * kRowThreads == kFrags, "a row is one fragment per thread of the cluster"); +static_assert(kRowWarps == 4, "row_sumsq_fold folds exactly four warp sums per CTA"); + +// Per-process control block in device memory. +struct Ctrl { + uint32_t stage; // stage of the next hop + uint32_t arrive; // CTA tickets of the running consumer launch + uint32_t seq; // hops completed + uint32_t pad; +}; + +// Passed by value as a kernel parameter. +struct Site { + void* mc; // multicast base of the mailbox [kStages][world][m_cap][kHidden] bf16 + void* uc; // this rank's unicast view of the same mailbox + Ctrl* ctrl; + int rank, world, m_cap; +}; + +// ---------------------------------------------------------------- PTX + +__device__ __forceinline__ void pdl_wait() { asm volatile("griddepcontrol.wait;" ::: "memory"); } +__device__ __forceinline__ void pdl_trigger() { asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); } +__device__ __forceinline__ unsigned long long gtimer() { + unsigned long long t; + asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t)); + return t; +} +__device__ __forceinline__ uint32_t ld_volatile_u32(const uint32_t* p) { + uint32_t v; + asm volatile("ld.volatile.global.u32 %0, [%1];" : "=r"(v) : "l"(p) : "memory"); + return v; +} +__device__ __forceinline__ uint4 ld_volatile_v4(const void* p) { + uint4 v; + asm volatile("ld.volatile.global.v4.u32 {%0,%1,%2,%3}, [%4];" + : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) + : "l"(p) + : "memory"); + return v; +} +__device__ __forceinline__ void st_v4(void* p, uint4 v) { + asm volatile("st.global.v4.u32 [%0], {%1,%2,%3,%4};" ::"l"(p), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w) + : "memory"); +} +__device__ __forceinline__ void mc_st_v4(void* mc, uint4 v) { + asm volatile("multimem.st.relaxed.sys.global.v4.bf16x2 [%0], {%1,%2,%3,%4};" ::"l"(mc), "r"(v.x), + "r"(v.y), "r"(v.z), "r"(v.w) + : "memory"); +} +__device__ __forceinline__ void mc_st_v2(void* mc, uint2 v) { + asm volatile("multimem.st.relaxed.sys.global.v2.bf16x2 [%0], {%1,%2};" ::"l"(mc), "r"(v.x), "r"(v.y) : "memory"); +} +__device__ __forceinline__ uint32_t cluster_ctarank() { + uint32_t r; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(r)); + return r; +} +__device__ __forceinline__ float div_full(float a, float b) { + float r; + asm("div.full.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} +// Two f32 to an e4m3 pair; lo lands in the low byte. +__device__ __forceinline__ uint16_t cvt_e4m3x2(float lo, float hi) { + uint16_t r; + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(r) : "f"(hi), "f"(lo)); + return r; +} + +// ---------------------------------------------------------------- bf16 + +__device__ __forceinline__ float bf_lo(uint32_t w) { return __uint_as_float(w << 16); } +__device__ __forceinline__ float bf_hi(uint32_t w) { return __uint_as_float(w & 0xFFFF0000u); } +__device__ __forceinline__ uint32_t pack_bf16x2(float lo, float hi) { + __nv_bfloat162 v = __floats2bfloat162_rn(lo, hi); + return *reinterpret_cast(&v); +} +__device__ __forceinline__ void unpack8(uint4 v, float f[8]) { + f[0] = bf_lo(v.x); f[1] = bf_hi(v.x); f[2] = bf_lo(v.y); f[3] = bf_hi(v.y); + f[4] = bf_lo(v.z); f[5] = bf_hi(v.z); f[6] = bf_lo(v.w); f[7] = bf_hi(v.w); +} +__device__ __forceinline__ uint4 pack8(const float f[8]) { + return make_uint4(pack_bf16x2(f[0], f[1]), pack_bf16x2(f[2], f[3]), pack_bf16x2(f[4], f[5]), + pack_bf16x2(f[6], f[7])); +} +__device__ __forceinline__ bool word_has_negzero(uint32_t w) { + return (w & 0xFFFFu) == 0x8000u || (w >> 16) == 0x8000u; +} +__device__ __forceinline__ bool frag_has_negzero(uint4 v) { + return word_has_negzero(v.x) | word_has_negzero(v.y) | word_has_negzero(v.z) | word_has_negzero(v.w); +} +__device__ __forceinline__ uint32_t sanitize_word(uint32_t w) { + if ((w & 0xFFFFu) == 0x8000u) w &= 0xFFFF0000u; + if ((w >> 16) == 0x8000u) w &= 0x0000FFFFu; + return w; +} + +// ---------------------------------------------------------------- stage protocol + +__device__ __forceinline__ size_t slot(const Site& s, uint32_t stage, int src, int token, int frag) { + return ((static_cast(stage) * s.world + src) * s.m_cap + token) * kHidden + + static_cast(frag) * kVec; +} + +struct Hop { + uint32_t stage; + uint32_t seq; + uint32_t ticket; // thread 0 only +}; + +// Consumer launch, every thread of every participating CTA, after griddepcontrol.wait. Thread +// 0 takes a ticket whose value is consumed only in hop_leave, so the atomic stays off the +// critical path. +__device__ __forceinline__ Hop hop_enter(const Site& s) { + Hop h; + h.stage = ld_volatile_u32(&s.ctrl->stage); + h.seq = ld_volatile_u32(&s.ctrl->seq); + h.ticket = threadIdx.x == 0 ? atomicAdd(&s.ctrl->arrive, 1u) : 0u; + return h; +} + +// Thread 0 of every participating CTA; the last arrival advances the stage. No CTA of this +// launch can read the advanced value, since every CTA read the stage before taking its ticket. +__device__ __forceinline__ void hop_leave(const Site& s, const Hop& h, uint32_t participants) { + if (threadIdx.x == 0 && h.ticket == participants - 1) { + s.ctrl->arrive = 0; + s.ctrl->stage = (h.stage + 1) % kStages; + s.ctrl->seq = h.seq + 1; + } +} + +// Producer-only launch, after griddepcontrol.wait. +__device__ __forceinline__ uint32_t stage_peek(const Site& s) { return ld_volatile_u32(&s.ctrl->stage); } + +// ---------------------------------------------------------------- producer + +// This rank's final bf16 partial for one fragment; exactly once per (hop, rank, token, frag). +// A launch that also stores its own outputs should push first: the push starts the flight. +__device__ __forceinline__ void push_frag(const Site& s, uint32_t stage, int token, int frag, uint4 v) { + v = make_uint4(sanitize_word(v.x), sanitize_word(v.y), sanitize_word(v.z), sanitize_word(v.w)); + mc_st_v4(static_cast(s.mc) + slot(s, stage, s.rank, token, frag), v); +} + +// Same for 4 consecutive bf16 (half a fragment, columns [4q, 4q + 4)); a fragment may be pushed +// as two halves by different threads. +__device__ __forceinline__ void push_quad(const Site& s, uint32_t stage, int token, int quad, uint2 v) { + v = make_uint2(sanitize_word(v.x), sanitize_word(v.y)); + mc_st_v2(static_cast(s.mc) + slot(s, stage, s.rank, token, quad >> 1) + (quad & 1) * 4, v); +} + +// ---------------------------------------------------------------- consumer + +// A peer that missed the wait deadline: store to address 0x0, then trap (fail-stop). +__device__ __forceinline__ void fault_at_null() { + asm volatile("{\n.reg .u64 a;\nmov.u64 a, 0;\nst.volatile.global.u32 [a], 0;\ntrap;\n}" ::: "memory"); +} + +// Sum over ranks 0..W-1 (fp32, from 0) of one fragment; spins on the sentinel. A wait past +// Q35_HOP_DEADLINE_NS (if not 0) stops the kernel. +template +__device__ __forceinline__ bool pull_frag(const Site& s, uint32_t stage, int token, int frag, float acc[8]) { + const uint16_t* uc = static_cast(s.uc); + uint4 v[W]; +#pragma unroll + for (int r = 0; r < W; ++r) v[r] = ld_volatile_v4(uc + slot(s, stage, r, token, frag)); + uint32_t pending = 0; +#pragma unroll + for (int r = 0; r < W; ++r) pending |= frag_has_negzero(v[r]) ? (1u << r) : 0u; + unsigned long long t0 = 0; + uint32_t spins = 0; + while (pending) { +#pragma unroll + for (int r = 0; r < W; ++r) + if (pending & (1u << r)) v[r] = ld_volatile_v4(uc + slot(s, stage, r, token, frag)); + uint32_t still = 0; +#pragma unroll + for (int r = 0; r < W; ++r) + if ((pending & (1u << r)) && frag_has_negzero(v[r])) still |= 1u << r; + pending = still; + if (Q35_HOP_DEADLINE_NS != 0 && pending && ((++spins & 255u) == 0)) { + const unsigned long long now = gtimer(); + if (t0 == 0) { + t0 = now; + } else if (now - t0 > Q35_HOP_DEADLINE_NS) { + fault_at_null(); + } + } + } +#pragma unroll + for (int e = 0; e < 8; ++e) acc[e] = 0.f; +#pragma unroll + for (int r = 0; r < W; ++r) { + float f[8]; + unpack8(v[r], f); +#pragma unroll + for (int e = 0; e < 8; ++e) acc[e] = __fadd_rn(acc[e], f[e]); + } + return true; +} + +// Write the sentinel back into the W slots this thread consumed (any time after pull_frag). +template +__device__ __forceinline__ void rearm_frag(const Site& s, uint32_t stage, int token, int frag) { + uint16_t* uc = static_cast(s.uc); + const uint4 z = make_uint4(kNegZeroPair, kNegZeroPair, kNegZeroPair, kNegZeroPair); +#pragma unroll + for (int r = 0; r < W; ++r) st_v4(uc + slot(s, stage, r, token, frag), z); +} + +// ---------------------------------------------------------------- row finish + +// prenorm = bf16(acc + residual); pre[] gets the bf16-rounded values the norm uses. +__device__ __forceinline__ uint4 residual_add(const float acc[8], uint4 residual, float pre[8]) { + float r[8]; + unpack8(residual, r); +#pragma unroll + for (int e = 0; e < 8; ++e) pre[e] = __fadd_rn(acc[e], r[e]); + const uint4 out = pack8(pre); + unpack8(out, pre); + return out; +} + +__device__ __forceinline__ float sumsq8(const float x[8]) { + float s = 0.f; +#pragma unroll + for (int e = 0; e < 8; ++e) s = __fadd_rn(s, __fmul_rn(x[e], x[e])); + return s; +} + +struct RowSmem { + float xs[kRowCtas][kRowWarps]; + uint64_t mbar; +}; + +// Prologue of every CTA of the row cluster, all threads, before griddepcontrol.wait: arm the +// exchange barrier and make it visible to the peers before any of them can send. +__device__ __forceinline__ void row_init(RowSmem& rs) { + if (threadIdx.x == 0) { + const uint32_t a = static_cast(__cvta_generic_to_shared(&rs.mbar)); + asm volatile("mbarrier.init.shared::cta.b64 [%0], 1;" ::"r"(a) : "memory"); + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(a), + "r"(static_cast(sizeof(rs.xs))) + : "memory"); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); + } + asm volatile("barrier.cluster.arrive.relaxed.aligned;\n\tbarrier.cluster.wait.acquire.aligned;" ::: "memory"); +} + +// Row total of the per-thread values: lane butterfly per warp, then each warp sends its sum to +// every CTA of the cluster with st.async; each CTA waits on its own mbarrier (no release fence, +// so this thread's earlier remote stores are not waited for) and folds per CTA as +// (w0 + w2) + (w1 + w3), then the CTAs in rank order from 0. +__device__ __forceinline__ float row_sumsq_fold(float v, RowSmem& rs) { + const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5; +#pragma unroll + for (int off = 1; off < 32; off <<= 1) v = __fadd_rn(v, __shfl_xor_sync(0xffffffffu, v, off)); + if (lane < kRowCtas) { + const uint32_t me = cluster_ctarank(); + const uint32_t slot_l = static_cast(__cvta_generic_to_shared(&rs.xs[me][warp])); + const uint32_t mbar_l = static_cast(__cvta_generic_to_shared(&rs.mbar)); + uint32_t slot_r, mbar_r; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(slot_r) : "r"(slot_l), "r"(lane)); + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(mbar_r) : "r"(mbar_l), "r"(lane)); + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.f32 [%0], %1, [%2];" ::"r"(slot_r), + "f"(v), "r"(mbar_r) + : "memory"); + } + { + const uint32_t a = static_cast(__cvta_generic_to_shared(&rs.mbar)); + asm volatile( + "{\n\t.reg .pred P;\n\tWAIT_ROW:\n\t" + "mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P, [%0], 0;\n\t" + "@!P bra WAIT_ROW;\n\t}" ::"r"(a) + : "memory"); + } + float total = 0.f; +#pragma unroll + for (int c = 0; c < kRowCtas; ++c) + total = __fadd_rn(total, __fadd_rn(__fadd_rn(rs.xs[c][0], rs.xs[c][2]), __fadd_rn(rs.xs[c][1], rs.xs[c][3]))); + return total; +} + +__device__ __forceinline__ float inv_rms(float row_sumsq, float eps) { + return rsqrtf(__fadd_rn(row_sumsq / static_cast(kHidden), eps)); +} + +// bf16((x * inv) * gamma), gamma being the applied multiplier. +__device__ __forceinline__ uint4 norm_apply(const float pre[8], float inv, const float gamma[8]) { + float out[8]; +#pragma unroll + for (int e = 0; e < 8; ++e) out[e] = __fmul_rn(__fmul_rn(pre[e], inv), gamma[e]); + return pack8(out); +} + +// The reciprocal the static per-tensor FP8 quant multiplies by. +__device__ __forceinline__ float inv_scale_of(float scale) { return div_full(1.f, scale); } + +// e4m3(clamp(x * inv_scale, +-448)) of the bf16 norm output, 8 bytes. +__device__ __forceinline__ uint2 quant_fp8(uint4 norm_bf16, float inv_scale) { + float q[8]; + unpack8(norm_bf16, q); + uint32_t w[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const float a = fminf(fmaxf(__fmul_rn(q[2 * j], inv_scale), -448.f), 448.f); + const float b = fminf(fmaxf(__fmul_rn(q[2 * j + 1], inv_scale), -448.f), 448.f); + w[j] = cvt_e4m3x2(a, b); + } + return make_uint2(w[0] | (w[1] << 16), w[2] | (w[3] << 16)); +} + +} // namespace q35hop + +// a named namespace: the C entries below name Args in their signatures +namespace q35hop_draft { + +using namespace q35hop; + +// hop.cu's Args with the draft consume's two fields appended (2.2's Args carries them too) +struct Args { + Site site; + const uint16_t* partial; // [M, H] bf16 (publish) + const uint16_t* routed; // [M*K, H] bf16 (publish_fin), rows indexed by permuted + const uint16_t* expert_w; // [M, K] bf16 + const int32_t* permuted; // [M, K]; -1 skips a slot + const uint16_t* shared; // [M, H] bf16 gated shared expert output, or null + const uint16_t* residual_in; // [M, H] bf16 + const uint16_t* gamma; // [H] bf16, the applied multiplier + const float* fp8_scale; // [1], or null + uint16_t* residual_out; // [M, H] bf16 + uint16_t* norm_out; // [M, H] bf16, or null + uint8_t* fp8_out; // [M, H] e4m3, or null + int m, top_k; + float eps; + float weight_bias; // draft consume modes only: multiplier = weight_bias + gamma + int32_t delay_ns; // draft consume modes only, diagnostic: hold the output stores this long +}; + +// a pull past the deadline faults (hop.cu's pull_frag), so the NaN poisoning below never runs +constexpr uint32_t kNanPair = 0x7FC07FC0u; + +__device__ __forceinline__ uint4 ld_v4(const void* p) { return *static_cast(p); } + +__device__ __forceinline__ size_t row_elem(int token, int frag) { + return static_cast(token) * kHidden + static_cast(frag) * kVec; +} + +// ---------------------------------------------------------------- draft-side consume +// The draft keeps the rounding of the stock kernels these replace; only the transport differs. +// kDraftMnnvl: FlashInfer's legacy MNNVL one-shot all-reduce + residual + RMSNorm as it launches for +// M <= 16 (clusters of 8 x 64 threads, thread t of a row owning columns [8t, 8t + 8)): fp32 rank sum +// rounded to bf16, bf16 residual add, bf16 squares summed in fp32 per thread, a 16 -> 1 butterfly per +// warp, the two warps of each 64-thread block, then the 8 blocks in order from 0. +// kDraftAddNorm: sglang's one-shot push all-reduce (fp32 rank sum rounded to bf16), then FlashInfer's +// CuTe DSL FusedAddRMSNorm for hidden 4096 (64 threads per row, thread t owning the 8-column chunks +// 64b + t, b = 0..7): fp32 residual add, squares accumulated by fma per thread in (b, column) order +// from 0, a 1 -> 16 butterfly per 32 threads, then the two warps. +constexpr int kDraftMnnvl = 1; +constexpr int kDraftAddNorm = 2; +// Test variant: FusedAddRMSNorm in FlashInfer's CUDA kernel order (512 threads x 8 columns). +constexpr int kDraftAddNormCuda = 3; + +__device__ __forceinline__ float add_ftz(float a, float b) { + float r; + asm("add.ftz.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} +__device__ __forceinline__ float mul_ftz(float a, float b) { + float r; + asm("mul.ftz.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} +__device__ __forceinline__ float rsqrt_ftz(float a) { + float r; + asm("rsqrt.approx.ftz.f32 %0, %1;" : "=f"(r) : "f"(a)); + return r; +} + +struct DraftSmem { + // Add-norm: the fp32 values of this CTA's 16 CuTe threads, [chunk b][thread t % 16][column]. + alignas(16) float xv[8][16][kVec]; + float xs[kRowCtas][kRowWarps]; // one value per warp of the row, from every CTA + float hs[kRowCtas]; // add-norm: every CTA's tree over its 16 CuTe threads + uint64_t mbar, mbar_x, mbar_h; +}; + +__device__ __forceinline__ uint32_t smem_u32(const void* p) { + return static_cast(__cvta_generic_to_shared(p)); +} +__device__ __forceinline__ uint32_t peer_addr(uint32_t local, uint32_t cta) { + uint32_t r; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(r) : "r"(local), "r"(cta)); + return r; +} +__device__ __forceinline__ void st_async_f32(uint32_t addr, float v, uint32_t mbar) { + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.f32 [%0], %1, [%2];" ::"r"(addr), "f"(v), + "r"(mbar) + : "memory"); +} +__device__ __forceinline__ void st_async_v4(uint32_t addr, float a, float b, float c, float d, uint32_t mbar) { + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.f32 [%0], {%1, %2, %3, %4}, [%5];" ::"r"(addr), + "f"(a), "f"(b), "f"(c), "f"(d), "r"(mbar) + : "memory"); +} +__device__ __forceinline__ void mbar_wait(const uint64_t* mbar) { + asm volatile( + "{\n\t.reg .pred P;\n\tWAIT_DRAFT:\n\t" + "mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P, [%0], 0;\n\t" + "@!P bra WAIT_DRAFT;\n\t}" ::"r"(smem_u32(mbar)) + : "memory"); +} + +// Prologue of every CTA of the row cluster, before griddepcontrol.wait: arm the barriers this mode +// receives on and make them visible to the peers before any of them can send. +template +__device__ __forceinline__ void draft_init(DraftSmem& ds) { + if (threadIdx.x == 0) { + const auto arm = [](uint64_t* mbar, uint32_t bytes) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], 1;" ::"r"(smem_u32(mbar)) : "memory"); + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(smem_u32(mbar)), "r"(bytes) + : "memory"); + }; + if (MODE == kDraftAddNorm) { + arm(&ds.mbar_x, static_cast(sizeof(ds.xv))); + arm(&ds.mbar_h, static_cast(sizeof(ds.hs))); + } else { + arm(&ds.mbar, static_cast(sizeof(ds.xs))); + } + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); + } + asm volatile("barrier.cluster.arrive.relaxed.aligned;\n\tbarrier.cluster.wait.acquire.aligned;" ::: "memory"); +} + +// Every CTA of the row cluster gets this warp's value v in xs[cta][warp]. +__device__ __forceinline__ void row_exchange(float v, DraftSmem& ds) { + const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5; + if (lane < kRowCtas) + st_async_f32(peer_addr(smem_u32(&ds.xs[cluster_ctarank()][warp]), lane), v, peer_addr(smem_u32(&ds.mbar), lane)); + mbar_wait(&ds.mbar); +} + +// Sum of squares in the CuTe DSL FusedAddRMSNorm order. Thread frag = 64b + t sends its 8 values to the +// CTA owning CuTe threads [16c, 16c + 16), where 16 lanes each accumulate one thread's 64 squares; the +// butterfly then pairs (0 + 1) + (2 + 3) ... within those 16, the CTAs as (0 + 1) + (2 + 3). +__device__ __forceinline__ float cute_sumsq(const float x[8], int frag, DraftSmem& ds) { + const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5; + const int b = frag >> 6, t = frag & 63; + const uint32_t dst = static_cast(t >> 4); + const uint32_t slot = peer_addr(smem_u32(&ds.xv[b][t & 15][0]), dst); + const uint32_t mbar = peer_addr(smem_u32(&ds.mbar_x), dst); + st_async_v4(slot, x[0], x[1], x[2], x[3], mbar); + st_async_v4(slot + 16, x[4], x[5], x[6], x[7], mbar); + mbar_wait(&ds.mbar_x); + if (warp == 0) { + float sum = 0.f; + if (lane < 16) { +#pragma unroll + for (int c = 0; c < 8; ++c) { + const float4 lo = *reinterpret_cast(&ds.xv[c][lane][0]); + const float4 hi = *reinterpret_cast(&ds.xv[c][lane][4]); + sum = __fmaf_rn(lo.x, lo.x, sum); + sum = __fmaf_rn(lo.y, lo.y, sum); + sum = __fmaf_rn(lo.z, lo.z, sum); + sum = __fmaf_rn(lo.w, lo.w, sum); + sum = __fmaf_rn(hi.x, hi.x, sum); + sum = __fmaf_rn(hi.y, hi.y, sum); + sum = __fmaf_rn(hi.z, hi.z, sum); + sum = __fmaf_rn(hi.w, hi.w, sum); + } + } +#pragma unroll + for (int off = 1; off < 16; off <<= 1) sum = __fadd_rn(sum, __shfl_xor_sync(0xffffffffu, sum, off)); + if (lane < kRowCtas) + st_async_f32(peer_addr(smem_u32(&ds.hs[cluster_ctarank()]), lane), sum, peer_addr(smem_u32(&ds.mbar_h), lane)); + } + mbar_wait(&ds.mbar_h); + return __fadd_rn(__fadd_rn(ds.hs[0], ds.hs[1]), __fadd_rn(ds.hs[2], ds.hs[3])); +} + +// EARLY releases the dependents at entry, as consume_kernel does; they read only static data before +// their own griddepcontrol.wait, which returns once this grid has completed. +template +__device__ __forceinline__ void consume_draft_kernel(const Args& a) { + __shared__ DraftSmem ds; + const int token = blockIdx.x; + const int frag = static_cast(blockIdx.y) * kRowThreads + threadIdx.x; + if (EARLY) pdl_trigger(); + float gam[8]; + unpack8(ld_v4(a.gamma + static_cast(frag) * kVec), gam); + draft_init(ds); + pdl_wait(); + const Hop h = hop_enter(a.site); + const uint4 res = ld_v4(a.residual_in + row_elem(token, frag)); + float acc[8]; + const bool ok = pull_frag(a.site, h.stage, token, frag, acc); + float res_f[8]; + unpack8(res, res_f); + float x[8]; + uint4 res_out; + float total; + if (MODE == kDraftMnnvl) { + float part = 0.f; +#pragma unroll + for (int e = 0; e < 8; ++e) { + const __nv_bfloat16 r = __hadd(__float2bfloat16_rn(acc[e]), __float2bfloat16_rn(res_f[e])); + x[e] = __bfloat162float(r); + part = add_ftz(part, __bfloat162float(__hmul(r, r))); + } + res_out = pack8(x); +#pragma unroll + for (int off = 16; off > 0; off >>= 1) part = add_ftz(part, __shfl_xor_sync(0xffffffffu, part, off)); + row_exchange(part, ds); + // Global warp 2k + i is xs[k / 2][(k % 2) * 2 + i]; 64-thread block k sums its two warps. + total = 0.f; +#pragma unroll + for (int k = 0; k < 8; ++k) + total = add_ftz(total, add_ftz(ds.xs[k >> 1][(k & 1) * 2], ds.xs[k >> 1][(k & 1) * 2 + 1])); + } else { +#pragma unroll + for (int e = 0; e < 8; ++e) x[e] = __fadd_rn(__bfloat162float(__float2bfloat16_rn(acc[e])), res_f[e]); + res_out = pack8(x); + if (MODE == kDraftAddNorm) { + total = cute_sumsq(x, frag, ds); + } else { + float part = 0.f; +#pragma unroll + for (int e = 0; e < 8; ++e) part = __fmaf_rn(x[e], x[e], part); +#pragma unroll + for (int off = 16; off > 0; off >>= 1) part = __fadd_rn(part, __shfl_xor_sync(0xffffffffu, part, off)); + row_exchange(part, ds); + float v[16]; +#pragma unroll + for (int i = 0; i < 16; ++i) v[i] = ds.xs[i >> 2][i & 3]; +#pragma unroll + for (int w = 8; w > 0; w >>= 1) +#pragma unroll + for (int i = 0; i < w; ++i) v[i] = __fadd_rn(v[i], v[i + w]); + total = v[0]; + } + } + const float inv = rsqrt_ftz(add_ftz(mul_ftz(total, 1.f / static_cast(kHidden)), a.eps)); + float out[8]; +#pragma unroll + for (int e = 0; e < 8; ++e) out[e] = __fmul_rn(__fmul_rn(x[e], inv), __fadd_rn(a.weight_bias, gam[e])); + uint4 norm = pack8(out); + if (!ok) norm = make_uint4(kNanPair, kNanPair, kNanPair, kNanPair); + if (a.delay_ns > 0) { + const unsigned long long t0 = gtimer(); + while (gtimer() - t0 < static_cast(a.delay_ns)) { + } + } + *reinterpret_cast(a.residual_out + row_elem(token, frag)) = res_out; + *reinterpret_cast(a.norm_out + row_elem(token, frag)) = norm; + if (!EARLY) pdl_trigger(); + rearm_frag(a.site, h.stage, token, frag); + hop_leave(a.site, h, gridDim.x * gridDim.y); +} + +} // namespace q35hop_draft + +using q35hop_draft::Args; + +#define Q35HOP_DRAFT_ENTRY(NAME, W, MODE, EARLY) \ + extern "C" __global__ void __launch_bounds__(q35hop::kRowThreads) NAME(Args a) { \ + q35hop_draft::consume_draft_kernel(a); \ + } + +Q35HOP_DRAFT_ENTRY(q35hop_draft_mnnvl_w2, 2, kDraftMnnvl, false) +Q35HOP_DRAFT_ENTRY(q35hop_draft_mnnvl_w4, 4, kDraftMnnvl, false) +Q35HOP_DRAFT_ENTRY(q35hop_draft_mnnvl_w8, 8, kDraftMnnvl, false) +Q35HOP_DRAFT_ENTRY(q35hop_draft_mnnvl_early_w2, 2, kDraftMnnvl, true) +Q35HOP_DRAFT_ENTRY(q35hop_draft_mnnvl_early_w4, 4, kDraftMnnvl, true) +Q35HOP_DRAFT_ENTRY(q35hop_draft_mnnvl_early_w8, 8, kDraftMnnvl, true) +Q35HOP_DRAFT_ENTRY(q35hop_draft_add_norm_w2, 2, kDraftAddNorm, false) +Q35HOP_DRAFT_ENTRY(q35hop_draft_add_norm_w4, 4, kDraftAddNorm, false) +Q35HOP_DRAFT_ENTRY(q35hop_draft_add_norm_w8, 8, kDraftAddNorm, false) +Q35HOP_DRAFT_ENTRY(q35hop_draft_add_norm_early_w2, 2, kDraftAddNorm, true) +Q35HOP_DRAFT_ENTRY(q35hop_draft_add_norm_early_w4, 4, kDraftAddNorm, true) +Q35HOP_DRAFT_ENTRY(q35hop_draft_add_norm_early_w8, 8, kDraftAddNorm, true) diff --git a/test/kernels/test_qwen3_5_draft_hop.py b/test/kernels/test_qwen3_5_draft_hop.py new file mode 100644 index 00000000..8ef13af8 --- /dev/null +++ b/test/kernels/test_qwen3_5_draft_hop.py @@ -0,0 +1,283 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""The MTP draft's hop (``qwen3_5/sm103/draft/hop.py``, ``qwen3_5/draft/hop.py``) on 4 +SM100-family GPUs (GB300) with NVLink multicast, on the decode hop's mailbox: + +- add_norm: the consume equals the fp32 rank-order sum rounded to bf16 followed by + SGLang's ``gemma_fused_add_rmsnorm`` (the stock draft's add-norm), bit for bit; +- mnnvl: its residual output equals bf16(bf16(sum) + residual) bit for bit, and its norm + stays within one bf16 ulp of an fp32 reference (its bits against FlashInfer's legacy + MNNVL kernel are checked with SGLang branch nvidia/qwen35-agentx-gb300, not here); +- rows 1 and 7, both entry releases, a late rank in turn, hops interleaved with the + decode hop's own, so the three mailbox stages cycle; +- the draft o_proj GEMV equals an fp32 emulation of its summation order, bit for bit, and + its pushed output followed by a consume-only all-reduce equals publish + consume; +- a draft step's hops replayed from a CUDA graph with fresh inputs equal the eager run. + +Run with ``python test/kernels/test_qwen3_5_draft_hop.py --num-gpu 4``. +``test_protocol_copies`` needs no GPU. +""" + +import os +import re +from pathlib import Path + +import pytest +import torch + +HIDDEN, EPS, MAX_M, O_PROJ_K = 4096, 1e-6, 7, 1024 +SM103 = Path(__file__).resolve().parents[2] / "sgl_nvidia_mega" / "qwen3_5" / "sm103" + +gpu = pytest.mark.skipif( + not (torch.cuda.is_available() and torch.cuda.get_device_capability()[0] == 10), + reason="Requires SM100/SM103", +) + + +def _protocol(path: Path) -> str: + text = path.read_text() + start = text.index("#include ") + end = text.index("} // namespace q35hop\n") + len("} // namespace q35hop\n") + return text[start:end] + + +def test_protocol_copies(): + """The draft kernels' copy of the decode hop's Site, Ctrl and mailbox protocol is + q35_hop.cuh's, verbatim.""" + want = _protocol(SM103 / "hop" / "src" / "q35_hop.cuh") + for name in ("draft_hop.cu", "draft_gemv.cu"): + assert _protocol(SM103 / "draft" / "src" / name) == want, name + # the draft consume's Args extends hop.cu's field by field + fields = re.compile(r"struct Args \{(.*?)\n\};", re.S) + hop_args = fields.search((SM103 / "hop" / "src" / "hop.cu").read_text()).group(1) + draft_args = fields.search((SM103 / "draft" / "src" / "draft_hop.cu").read_text()) + assert draft_args.group(1).startswith(hop_args) + + +@pytest.fixture(scope="module") +def rank(): + from sglang.srt.distributed import parallel_state as ps + from sglang.test.test_utils import publish_build_topology + + rank = int(os.environ["RANK"]) + world_size = int(os.environ["WORLD_SIZE"]) + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + ps.set_custom_all_reduce(False) + ps.init_distributed_environment( + world_size=world_size, + rank=rank, + local_rank=local_rank, + distributed_init_method="env://", + ) + publish_build_topology(tp_size=world_size, world_rank=rank) + ps.initialize_model_parallel() + yield rank + ps.destroy_model_parallel() + ps.destroy_distributed_environment() + + +@pytest.fixture(scope="module") +def draft(rank): + """The decode hop as the target builds it (7 rows), and the draft hop on it.""" + from sglang.srt.distributed import parallel_state as ps + + from sgl_nvidia_mega.qwen3_5.draft.hop import DraftHop + from sgl_nvidia_mega.qwen3_5.sm103.draft import hop as kernels + from sgl_nvidia_mega.qwen3_5.sm103.hop import Hop + + group = ps.get_tp_group().device_group + device = torch.device(f"cuda:{torch.cuda.current_device()}") + hop = Hop(max_m=MAX_M, eps=EPS, group=group, device=device, deadline_s=30.0) + kernels.build(device.index, int(30.0 * 1e9), gemv_ks=(O_PROJ_K,)) + return DraftHop(hop) + + +class _Norm: + """The GemmaRMSNorm fields the draft hop reads.""" + + def __init__(self, weight): + self.weight = weight + self.gemma_weight = torch.add(weight, 1.0) + self.variance_epsilon = EPS + + +def _operands(m, device, rank, seed): + own = torch.Generator(device=device).manual_seed(seed * 100 + rank) + common = torch.Generator(device=device).manual_seed(seed) + + def randn(*shape, gen=own): + return torch.randn(*shape, device=device, generator=gen).bfloat16() + + return dict( + partial=randn(m, HIDDEN), + attn=randn(1, O_PROJ_K), + o_proj=(0.03 * torch.randn(HIDDEN, O_PROJ_K, device=device, generator=own)) + .bfloat16() + .contiguous(), + residual=randn(m, HIDDEN, gen=common), + weight=(0.1 * randn(HIDDEN, gen=common).float()).bfloat16(), + ) + + +def _rank_sum(partial): + """The hop's pull: the ranks' partials (-0.0 pushed as +0.0) summed in fp32 from + rank 0, rounded to bf16.""" + import torch.distributed as dist + from sglang.srt.distributed import parallel_state as ps + + group = ps.get_tp_group().device_group + parts = [torch.empty_like(partial) for _ in range(dist.get_world_size(group))] + dist.all_gather(parts, partial.contiguous(), group=group) + acc = None + for p in parts: + p = torch.where(p.view(torch.int16) == -32768, torch.zeros_like(p), p) + acc = p.float() if acc is None else acc + p.float() + return acc.bfloat16() + + +def _same(got, want): + for a, b in zip(got, want): + assert torch.equal(a.view(torch.int16), b.view(torch.int16)) + + +def _gemv_ref(x, w): + """mkd_draft_gemv's arithmetic: lane l of a row's warp fma-accumulates columns + (32 j + l) * 8 + e in (j, e) order (a bf16 product is exact in fp32, so add after + multiply rounds once, as fma), then the xor butterfly 16 .. 1; lane 0 stores.""" + k, n = x.shape[1], w.shape[0] + xs = x.float().view(k // 256, 32, 8) + ws = w.float().view(n, k // 256, 32, 8) + acc = torch.zeros(n, 32, device=x.device) + for j in range(k // 256): + for e in range(8): + acc = acc + ws[:, j, :, e] * xs[j, :, e] + lanes = torch.arange(32, device=x.device) + for mask in (16, 8, 4, 2, 1): + acc = acc + acc[:, lanes ^ mask] + return acc[:, 0].bfloat16().view(1, n) + + +@gpu +def test_draft_hop_rounding(rank, draft): + from sgl_kernel import gemma_fused_add_rmsnorm + + from sgl_nvidia_mega.qwen3_5.sm103.draft import hop as kernels + + world = torch.distributed.get_world_size() + device = torch.device(f"cuda:{torch.cuda.current_device()}") + hop = draft.hop + calls = [ + (m, add_norm, early) + for m in (1, 7) + for add_norm in (False, True) + for early in (False, True) + ] + with torch.inference_mode(): + for i, (m, add_norm, early) in enumerate(calls): + x = _operands(m, device, rank, seed=i) + norm = _Norm(x["weight"]) + summed = _rank_sum(x["partial"]) + if i % (world + 1) == rank: + torch.cuda._sleep(2_000_000) # this rank publishes late + got = kernels.draft_all_reduce_rms_norm( + hop, + local_contribution=x["partial"], + residual=x["residual"], + gamma=norm.weight if add_norm else norm.gemma_weight, + eps=EPS, + weight_bias=1.0 if add_norm else 0.0, + add_norm=add_norm, + early_release=early, + ) + if add_norm: + want_x, want_r = summed.clone(), x["residual"].clone() + gemma_fused_add_rmsnorm(want_x, want_r, norm.weight, EPS) + _same(got, (want_x, want_r)) + else: + pre = summed + x["residual"] # bf16 add, as __hadd + _same(got[1:], (pre,)) + f = pre.float() + ref = f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + EPS) + ref = (ref * norm.gemma_weight.float()).bfloat16().float() + # bf16 squares and an approximate rsqrt: at most one bf16 ulp off + assert ((got[0].float() - ref).abs() <= ref.abs() * 2**-7).all() + # the decode hop's own hop in between keeps the stage order shared + y = _operands(m, device, rank, seed=1000 + i) + hop.all_reduce_residual_rms_norm( + local_contribution=y["partial"], + residual=y["residual"], + gamma=norm.gemma_weight, + ) + torch.cuda.synchronize() + + +@gpu +def test_draft_gemv_and_push(rank, draft): + from sgl_nvidia_mega.qwen3_5.sm103.draft import hop as kernels + + device = torch.device(f"cuda:{torch.cuda.current_device()}") + with torch.inference_mode(): + for seed in range(4): + x = _operands(1, device, rank, seed=200 + seed) + norm = _Norm(x["weight"]) + plain = torch.empty(1, HIDDEN, dtype=torch.bfloat16, device=device) + kernels.draft_gemv(x["attn"], x["o_proj"], plain) + _same((plain,), (_gemv_ref(x["attn"], x["o_proj"]),)) + want = kernels.draft_all_reduce_rms_norm( + draft.hop, + local_contribution=plain, + residual=x["residual"], + gamma=norm.gemma_weight, + eps=EPS, + weight_bias=0.0, + add_norm=False, + ) + pushed = draft.o_proj(x["attn"], x["o_proj"]) # pushes into the hop + assert draft.has_pushed(pushed) + got = draft.ffn_input( + pushed, x["residual"], norm, moe_single_launch=seed % 2 == 1 + ) + assert not draft.has_pushed(pushed) + _same((pushed,), (plain,)) + _same(got, want) + torch.cuda.synchronize() + + +@gpu +def test_draft_step_replays_from_a_cuda_graph(rank, draft): + device = torch.device(f"cuda:{torch.cuda.current_device()}") + + def step(x, norm): + part = draft.o_proj(x["attn"], x["o_proj"]) + h, res = draft.ffn_input(part, x["residual"], norm, moe_single_launch=True) + return draft.final_norm(x["partial"], res, norm) + (h,) + + with torch.inference_mode(): + static = [_operands(1, device, rank, seed=300 + i) for i in range(2)] + norms = [_Norm(x["weight"]) for x in static] + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for x, norm in zip(static, norms): + step(x, norm) # eager warmup on the capture stream + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outputs = [step(x, norm) for x, norm in zip(static, norms)] + for replay in range(3): + for i, x in enumerate(static): + fresh = _operands(1, device, rank, seed=400 + 10 * replay + i) + for name, tensor in x.items(): + tensor.copy_(fresh[name]) + norms[i].gemma_weight.copy_(torch.add(x["weight"], 1.0)) + graph.replay() + for x, norm, got in zip(static, norms, outputs): + _same(got, step(x, norm)) + torch.cuda.synchronize() + + +if __name__ == "__main__": + from sglang.test.kernels.utils import multigpu_pytest_main + + multigpu_pytest_main(__name__, __file__, num_gpus=(4,)) diff --git a/test/unit/test_qwen3_5_runtime.py b/test/unit/test_qwen3_5_runtime.py index e624f174..77363cd1 100644 --- a/test/unit/test_qwen3_5_runtime.py +++ b/test/unit/test_qwen3_5_runtime.py @@ -168,6 +168,7 @@ def test_the_draft_loop_routes_q1_by_kv_length_as_2_2( "slots, want", [ ({"draft_moe"}, "slot 'draft_moe' runs inside entry 'draft_loop'"), + ({"draft_loop", "draft_hop"}, "slot 'draft_hop' runs on slot 'hop''s mailbox"), ], ) def test_the_draft_slots_name_what_they_need(monkeypatch, slots, want): From 55b8ea0d901cc587c3e3a8e5c26d8b387168f89f Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:50:00 -0700 Subject: [PATCH 13/27] [Perf] [GB300] qwen3_5: Draft slot draft_front: the draft attention's front writes the FP8 KV in the same launch After the qkv projection, one Triton launch runs the q / k Gemma RMSNorm, the RoPE and the gate split, and also writes the FP8 query and the K / V rows into the FP8 KV cache, in place of SGLang's second launch. Draft slot draft_front; bitwise: the same bytes as SGLang's two launches. --- sgl_nvidia_mega/qwen3_5/draft/__init__.py | 6 +- sgl_nvidia_mega/qwen3_5/draft/attention.py | 76 ++++- sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 8 +- sgl_nvidia_mega/qwen3_5/sm103/draft/front.py | 309 +++++++++++++++++++ test/kernels/test_qwen3_5_attention.py | 91 +++++- test/kernels/test_qwen3_5_draft_front.py | 141 +++++++++ 6 files changed, 617 insertions(+), 14 deletions(-) create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/draft/front.py create mode 100644 test/kernels/test_qwen3_5_draft_front.py diff --git a/sgl_nvidia_mega/qwen3_5/draft/__init__.py b/sgl_nvidia_mega/qwen3_5/draft/__init__.py index 1a953f7e..e98302e5 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/draft/__init__.py @@ -14,8 +14,8 @@ Inside them every op is SGLang's own unless a draft slot replaces it: ``draft_moe`` (the single-launch MoE), ``draft_q1`` (the q1 split-KV decode), ``draft_xchg`` (the -step boundary's fc split, exchange and head) and ``draft_hop`` (the draft hops and the -o_proj GEMV). +step boundary's fc split, exchange and head), ``draft_hop`` (the draft hops and the +o_proj GEMV) and ``draft_front`` (the attention front writing the FP8 KV itself). """ from __future__ import annotations @@ -70,7 +70,7 @@ def numerics(slots: frozenset) -> Numerics: which differ from SGLang's ops only in fp32 summation order (q1 split-KV, the o_proj / fc / MoE GEMVs, the draft head), so a draft token can differ where two logits are within a bf16 rounding.""" - if not slots & {"draft_moe", "draft_q1", "draft_xchg", "draft_hop"}: + if not slots & {"draft_moe", "draft_q1", "draft_xchg", "draft_hop", "draft_front"}: return target.NUMERICS return Numerics(NumericsKind.TOLERANCE, {"draft_logits.rel": 2**-7}) diff --git a/sgl_nvidia_mega/qwen3_5/draft/attention.py b/sgl_nvidia_mega/qwen3_5/draft/attention.py index 63fbaa2f..2dc001dc 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/attention.py +++ b/sgl_nvidia_mega/qwen3_5/draft/attention.py @@ -58,6 +58,15 @@ def select(self, forward_batch) -> str: "forward_prepare_cuda_fused", "attn_output_gate", "o_proj", + "qkv_proj", + "q_size", + "kv_size", + "q_norm", + "k_norm", + "rotary_emb", + "num_heads", + "num_kv_heads", + "head_dim", ), "srt/layers/radix_attention.py": ( "tp_q_head_num", @@ -155,6 +164,8 @@ def __init__(self, parts): self.k_scale = attn.k_scale if attn.k_scale is not None else ones self.v_scale = attn.v_scale if attn.v_scale is not None else ones self.q1 = None # sm103.draft.q1 decode (slot draft_q1), set by DraftLoop + # slot draft_front: 2.2's attention front writes the KV and FP8 q in one launch + self.front = "draft_front" in parts.slots def __call__( self, x, positions, cache_loc, meta, kv, *, rows_per_request: int, o_proj=None @@ -165,10 +176,14 @@ def __call__( from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul layer = self.layer - q, k, v, gate = layer.forward_prepare_cuda_fused( - positions=positions, hidden_states=x - ) - o = self.core(q, k, v, cache_loc, meta, kv, rows_per_request) + if self.front: + q8, gate = self._front(x, positions, cache_loc, kv) + o = self.decode(q8, meta, kv, rows_per_request) + else: + q, k, v, gate = layer.forward_prepare_cuda_fused( + positions=positions, hidden_states=x + ) + o = self.core(q, k, v, cache_loc, meta, kv, rows_per_request) if layer.attn_output_gate: o = fused_sigmoid_mul(o, gate, inplace=True) if o_proj is not None: @@ -176,25 +191,68 @@ def __call__( out, _ = layer.o_proj(o) return out + def _front(self, x, positions, cache_loc, kv): + """Slot draft_front (2.2's ATTN_FRONT): the qkv projection, then one Triton launch for + the q / k Gemma RMSNorm, RoPE, gate split, FP8 q, and K / V into the FP8 cache + at ``cache_loc``: (q8 [rows, heads * head_dim], gate).""" + from sgl_nvidia_mega.qwen3_5.sm103.draft.front import ( + fused_qk_gemma_rmsnorm_rope_gate, + ) + + layer = self.layer + qkv, _ = layer.qkv_proj(x) + q_gate_size = layer.q_size * (2 if layer.attn_output_gate else 1) + q_gate, k, v = qkv.split([q_gate_size, layer.kv_size, layer.kv_size], dim=-1) + q8, _, gate = fused_qk_gemma_rmsnorm_rope_gate( + q_gate, + k, + layer.q_norm.weight.data, + layer.k_norm.weight.data, + layer.rotary_emb.cos_sin_cache, + positions, + layer.q_norm.variance_epsilon, + layer.num_heads, + layer.num_kv_heads, + layer.head_dim, + layer.rotary_emb.rotary_dim, + has_gate=layer.attn_output_gate, + mrope_axis_map=( + layer.rotary_emb.axis_map if positions.dim() == 2 else None + ), + v=v, + kv_cache=(kv.k[0], kv.v[0]), + cache_loc=cache_loc, + k_scale=self.k_scale, + v_scale=self.v_scale, + ) + rows = x.shape[0] + return q8.view(rows, -1), (gate.view(rows, -1) if gate is not None else None) + def core(self, q, k, v, cache_loc, meta, kv, rows_per_request: int): """FP8 q / k / v with the KV write, then the decode: [rows, heads * head_dim].""" - import flashinfer from sglang.kernels.ops.kvcache.fused_fp8_qkv_kv_cache import ( fused_fp8_qkv_kv_cache, ) - from sglang.srt.environ import envs - k_cache, v_cache = kv.k[0], kv.v[0] q8 = fused_fp8_qkv_kv_cache( q=q, k=k, v=v, - k_cache=k_cache, - v_cache=v_cache, + k_cache=kv.k[0], + v_cache=kv.v[0], cache_loc=cache_loc, k_scale=self.k_scale, v_scale=self.v_scale, ) + return self.decode(q8, meta, kv, rows_per_request) + + def decode(self, q8, meta, kv, rows_per_request: int): + """The decode of FP8 q rows over the cache (this step's rows written): + [rows, heads * head_dim].""" + import flashinfer + from sglang.srt.environ import envs + + k_cache, v_cache = kv.k[0], kv.v[0] q8 = q8.reshape(-1, self.heads, self.head_dim) attn = self.layer.attn bmm1_scale = 1.0 * kv.k_scale[0] * attn.scaling diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index e46d0d65..376c7ef3 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -30,7 +30,13 @@ # The MTP draft's EAGLE entries and kernel groups (qwen3_5/draft), requested in the same # variable; the draft spec's build reads them, the target's ignores them DRAFT_ENTRIES: tuple[str, ...] = ("draft_loop", "draft_extend") -DRAFT_SLOTS: tuple[str, ...] = ("draft_moe", "draft_q1", "draft_xchg", "draft_hop") +DRAFT_SLOTS: tuple[str, ...] = ( + "draft_moe", + "draft_q1", + "draft_xchg", + "draft_hop", + "draft_front", +) SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/front.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/front.py new file mode 100644 index 00000000..4b4d99b6 --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/front.py @@ -0,0 +1,309 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Adapted from SGLang (Apache-2.0, SGLang Team): +# python/sglang/kernels/ops/attention/fused_qk_rmsnorm_rope_gate.py, as SGLang branch +# nvidia/qwen35-agentx-gb300 extends it with the FP8 KV-cache write (SGLANG_Q35MF_ATTN_FRONT; +# 98573b3e69, 0d86b61136, 4115dd1469): copied verbatim. +"""Fused Q/K GemmaRMSNorm + NeoX RoPE + gate deinterleave (Triton), and with +``kv_cache`` 2.2's attention front (ATTN_FRONT): FP8 q, and K / V written straight +into the FP8 KV cache with the cache writer's round-to-nearest reciprocal scales, bit +for bit what SGLang's separate fused FP8 q/k/v + KV write produces. + +Single kernel launch fusing per-head GemmaRMSNorm, partial NeoX RoPE over 1-D or +mrope positions, and gate deinterleave for Qwen3.5's interleaved Q+Gate layout. + +2D grid (T, num_q_heads + num_kv_heads) — each program handles one +(token, head) pair. Q programs also copy the gate slice. +""" + +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl +from triton.language.extra.cuda import libdevice + + +def _pdl_supported() -> bool: + """Check if Programmatic Dependent Launch is supported (NVIDIA SM >= 90).""" + if not torch.cuda.is_available(): + return False + try: + major, _ = torch.cuda.get_device_capability() + return major >= 9 + except Exception: + return False + + +_ENABLE_PDL = _pdl_supported() + + +@triton.jit +def _fused_qk_rmsnorm_rope_gate_kernel( + q_gate_ptr, + k_ptr, + q_out_ptr, + k_out_ptr, + gate_out_ptr, + q_weight_ptr, + k_weight_ptr, + cos_sin_cache_ptr, + positions_ptr, + mrope_axis_map_ptr, + stride_qg_t, + stride_k_t, + stride_qo_t, + stride_ko_t, + stride_gate_t, + stride_cos_t, + stride_pos_axis, + NUM_Q_HEADS: tl.constexpr, + NUM_KV_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + ROTARY_DIM: tl.constexpr, + HALF_ROTARY: tl.constexpr, + HEAD_BLOCK: tl.constexpr, + ROT_HALF_BLOCK: tl.constexpr, + EPS: tl.constexpr, + FP16: tl.constexpr, + HAS_PASS: tl.constexpr, + HAS_GATE: tl.constexpr, + MROPE: tl.constexpr, + ENABLE_PDL: tl.constexpr, + WEIGHT_SHIFT: tl.constexpr = 1.0, + EXPLICIT_ROPE_FMA: tl.constexpr = False, + v_ptr=None, + v_cache_ptr=None, + cache_loc_ptr=None, + k_scale_ptr=None, + v_scale_ptr=None, + stride_v_t: tl.constexpr = 0, + stride_v_cache_t: tl.constexpr = 0, + FP8_CACHE: tl.constexpr = False, +): + token = tl.program_id(0) + head = tl.program_id(1) + is_k = head >= NUM_Q_HEADS + local_head = tl.where(is_k, head - NUM_Q_HEADS, head) + out_dtype = tl.float16 if FP16 else tl.bfloat16 + if FP8_CACHE: + slot = tl.load(cache_loc_ptr + token).to(tl.int64) + # Match the cache writer's round-to-nearest reciprocal. + inv_k = tl.div_rn(1.0, tl.load(k_scale_ptr).to(tl.float32)) + + if is_k: + in_base = k_ptr + token * stride_k_t + local_head * HEAD_DIM + w_ptr = k_weight_ptr + out_base = k_out_ptr + token * stride_ko_t + local_head * HEAD_DIM + if FP8_CACHE: + out_base = k_out_ptr + slot * stride_ko_t + local_head * HEAD_DIM + else: + if HAS_GATE: + in_base = q_gate_ptr + token * stride_qg_t + local_head * 2 * HEAD_DIM + else: + in_base = q_gate_ptr + token * stride_qg_t + local_head * HEAD_DIM + w_ptr = q_weight_ptr + out_base = q_out_ptr + token * stride_qo_t + local_head * HEAD_DIM + + # Full load -> RMSNorm variance + head_offs = tl.arange(0, HEAD_BLOCK) + head_mask = head_offs < HEAD_DIM + x = tl.load(in_base + head_offs, mask=head_mask, other=0.0).to(tl.float32) + w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32) + var = tl.sum(x * x, axis=0) / HEAD_DIM + inv_rms = tl.rsqrt(var + EPS) + x_norm = (x * inv_rms * (w + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32) + + # Pass-through tail [rotary_dim, head_dim) + if HAS_PASS: + pass_mask = head_mask & (head_offs >= ROTARY_DIM) + tail = x_norm + if FP8_CACHE and is_k: + tail *= inv_k + tl.store(out_base + head_offs, tail, mask=pass_mask) + + # Reload rotary portion from L1 -> re-norm -> RoPE + rot_offs = tl.arange(0, ROT_HALF_BLOCK) + rot_mask = rot_offs < HALF_ROTARY + xr1 = tl.load(in_base + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) + xr2 = tl.load(in_base + HALF_ROTARY + rot_offs, mask=rot_mask, other=0.0).to( + tl.float32 + ) + wr1 = tl.load(w_ptr + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) + wr2 = tl.load(w_ptr + HALF_ROTARY + rot_offs, mask=rot_mask, other=0.0).to( + tl.float32 + ) + xr1 = (xr1 * inv_rms * (wr1 + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32) + xr2 = (xr2 * inv_rms * (wr2 + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32) + + if MROPE: + axis = tl.load(mrope_axis_map_ptr + rot_offs, mask=rot_mask, other=0) + pos = tl.load( + positions_ptr + axis * stride_pos_axis + token, mask=rot_mask, other=0 + ) + else: + pos = tl.load(positions_ptr + token) + cache_off = pos.to(tl.int64) * stride_cos_t + cos = tl.load( + cos_sin_cache_ptr + cache_off + rot_offs, mask=rot_mask, other=0.0 + ).to(tl.float32) + sin = tl.load( + cos_sin_cache_ptr + cache_off + HALF_ROTARY + rot_offs, mask=rot_mask, other=0.0 + ).to(tl.float32) + if EXPLICIT_ROPE_FMA: + first = tl.fma(xr1, cos, -libdevice.mul_rn(xr2, sin)) + second = tl.fma(xr2, cos, libdevice.mul_rn(xr1, sin)) + else: + first = xr1 * cos - xr2 * sin + second = xr2 * cos + xr1 * sin + if FP8_CACHE: + first = first.to(out_dtype).to(tl.float32) + second = second.to(out_dtype).to(tl.float32) + if is_k: + first *= inv_k + second *= inv_k + tl.store(out_base + rot_offs, first, mask=rot_mask) + tl.store(out_base + HALF_ROTARY + rot_offs, second, mask=rot_mask) + + if FP8_CACHE and is_k: + value = tl.load( + v_ptr + token * stride_v_t + local_head * HEAD_DIM + head_offs, + head_mask, + 0.0, + ).to(tl.float32) + inv_v = tl.div_rn(1.0, tl.load(v_scale_ptr).to(tl.float32)) + value_out = v_cache_ptr + slot * stride_v_cache_t + local_head * HEAD_DIM + tl.store(value_out + head_offs, value * inv_v, head_mask) + + # Gate copy (Q heads only) + if HAS_GATE and not is_k: + gate_in = in_base + HEAD_DIM + gate_out = gate_out_ptr + token * stride_gate_t + local_head * HEAD_DIM + g = tl.load(gate_in + head_offs, mask=head_mask, other=0.0) + tl.store(gate_out + head_offs, g, mask=head_mask) + + # PDL: signal dependent kernels (attention/allreduce) can start early. + # Only available on NVIDIA Hopper+ (sm_90+); guarded for AMD/other backends. + if ENABLE_PDL: + tl.extra.cuda.gdc_launch_dependents() + + +def fused_qk_gemma_rmsnorm_rope_gate( + q_gate: torch.Tensor, + k: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + positions: torch.Tensor, + eps: float, + num_q_heads: int, + num_kv_heads: int, + head_dim: int, + rotary_dim: int, + has_gate: bool = True, + mrope_axis_map: Optional[torch.Tensor] = None, + *, + v: Optional[torch.Tensor] = None, + kv_cache: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + cache_loc: Optional[torch.Tensor] = None, + k_scale: Optional[torch.Tensor] = None, + v_scale: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: + """Fused QK GemmaRMSNorm + NeoX RoPE + gate deinterleave. + + Args: + q_gate: [T, num_q_heads * (1 + has_gate) * head_dim] — interleaved Q+Gate if has_gate + k: [T, num_kv_heads * head_dim] + q_weight, k_weight: [head_dim] — raw GemmaRMSNorm weights (kernel adds +1.0) + cos_sin_cache: [max_seq_len, rotary_dim] — [cos..., sin...] + positions: [T] token positions, or [3, T] mrope rows (temporal, height, width) + mrope_axis_map: [rotary_dim // 2] — the axis owning each rotary lane, from + MRotaryEmbedding + """ + assert positions.dim() in (1, 2), f"want [T] or [3, T], got {positions.shape}" + mrope = positions.dim() == 2 + assert mrope == (mrope_axis_map is not None), "mrope_axis_map needs [3, T]" + if mrope: + assert positions.shape[0] == 3 and positions.stride(1) == 1, ( + f"want [3, T] contiguous over T, got {positions.shape} " + f"stride {positions.stride()}" + ) + lanes = rotary_dim // 2 + assert mrope_axis_map.shape == (lanes,), f"want one axis per lane ({lanes})" + T = q_gate.shape[0] + q_size = num_q_heads * head_dim + kv_size = num_kv_heads * head_dim + + fp8_cache = kv_cache is not None + q_out = torch.empty( + T, + q_size, + dtype=torch.float8_e4m3fn if fp8_cache else q_gate.dtype, + device=q_gate.device, + ) + k_out = ( + kv_cache[0] + if fp8_cache + else torch.empty(T, kv_size, dtype=k.dtype, device=k.device) + ) + gate_out = ( + torch.empty(T, num_q_heads, head_dim, dtype=q_gate.dtype, device=q_gate.device) + if has_gate + else q_out + ) + + half_rotary = rotary_dim // 2 + head_block = triton.next_power_of_2(head_dim) + rot_half_block = triton.next_power_of_2(half_rotary) + + grid = (T, num_q_heads + num_kv_heads) + kwargs = {} + if fp8_cache: + assert v is not None and cache_loc is not None + assert k_scale is not None and v_scale is not None + assert kv_cache[0].dtype == kv_cache[1].dtype == torch.float8_e4m3fn + kwargs = dict( + v_ptr=v, + v_cache_ptr=kv_cache[1], + cache_loc_ptr=cache_loc, + k_scale_ptr=k_scale, + v_scale_ptr=v_scale, + stride_v_t=v.stride(0), + stride_v_cache_t=kv_cache[1].stride(0), + FP8_CACHE=True, + ) + _fused_qk_rmsnorm_rope_gate_kernel[grid]( + q_gate, + k, + q_out, + k_out, + gate_out, + q_weight, + k_weight, + cos_sin_cache, + positions, + mrope_axis_map, + q_gate.stride(0), + k.stride(0), + q_out.stride(0), + k_out.stride(0), + gate_out.stride(0), + cos_sin_cache.stride(0), + positions.stride(0), + NUM_Q_HEADS=num_q_heads, + NUM_KV_HEADS=num_kv_heads, + HEAD_DIM=head_dim, + ROTARY_DIM=rotary_dim, + HALF_ROTARY=half_rotary, + HEAD_BLOCK=head_block, + ROT_HALF_BLOCK=rot_half_block, + EPS=eps, + FP16=q_gate.dtype == torch.float16, + HAS_PASS=rotary_dim < head_dim, + HAS_GATE=has_gate, + MROPE=mrope, + ENABLE_PDL=_ENABLE_PDL, + **kwargs, + ) + + return q_out, None if fp8_cache else k_out, gate_out if has_gate else None diff --git a/test/kernels/test_qwen3_5_attention.py b/test/kernels/test_qwen3_5_attention.py index 2158b9c4..04bfe890 100644 --- a/test/kernels/test_qwen3_5_attention.py +++ b/test/kernels/test_qwen3_5_attention.py @@ -10,7 +10,10 @@ - SKV2 replayed in a CUDA graph on new inputs, bit for bit its eager result; - the MKA prologue and epilogue twice on the same inputs, and replayed in a CUDA graph on new ones, bit for bit their eager result (FP8 Q, bf16 gate, the written K / V rows and - the o_proj output). + the o_proj output); +- the ATTN_FRONT op (slot draft_front's kernel, sm103/draft/front.py) without its cache + write bit for bit SGLang's fused q / k norm + RoPE + gate split, and with it replayed + in a CUDA graph bit for bit. Their bits against the kernels of SGLang branch nvidia/qwen35-agentx-gb300 are checked with that branch installed as ``sglang``, not here. @@ -272,3 +275,89 @@ def test_mka_replays_in_a_cuda_graph_bit_for_bit(): step.run(ops) assert _same(replayed, step.bits()) assert torch.isfinite(step.out.float()).all() + + +def _bits(a, b): + """Whether two tensors (of any dtype, FP8 too) hold the same bytes.""" + if a is None or b is None: + return a is b + return a.shape == b.shape and torch.equal( + a.contiguous().view(torch.uint8), b.contiguous().view(torch.uint8) + ) + + +def _front_operands(rows, g, mrope=True, slots=64 * 16): + q_gate = (torch.randn(rows, HEADS * 2 * DIM, device="cuda", generator=g) * 3).to( + torch.bfloat16 + ) + k, v = ( + (torch.randn(rows, DIM, device="cuda", generator=g) * 3).to(torch.bfloat16) + for _ in range(2) + ) + norms = [ + (torch.randn(DIM, device="cuda", generator=g) * 0.1).to(torch.bfloat16) + for _ in range(2) + ] + angles = torch.rand(4096, 32, device="cuda", generator=g) * 6.3 + cos_sin = torch.cat([angles.cos(), angles.sin()], dim=1).contiguous() + if mrope: + pos = torch.randint(0, 4096, (3, rows), device="cuda", generator=g) + axis = torch.randint(0, 3, (32,), device="cuda", generator=g) + else: + pos, axis = torch.randint(0, 4096, (rows,), device="cuda", generator=g), None + pools = tuple( + torch.randn(slots, 1, DIM, device="cuda", generator=g).to(torch.float8_e4m3fn) + for _ in range(2) + ) + loc = torch.randperm(slots, device="cuda", generator=g)[:rows] + args = (q_gate, k, *norms, cos_sin, pos, 1e-6, HEADS, 1, DIM, 64) + return args, axis, v, pools, loc + + +@pytest.mark.parametrize("rows", [1, 7]) +@pytest.mark.parametrize("mrope", [False, True]) +def test_front_without_the_cache_write_is_sglangs(rows, mrope): + from sglang.kernels.ops.attention.fused_qk_rmsnorm_rope_gate import ( + fused_qk_gemma_rmsnorm_rope_gate as sglang_op, + ) + + from sgl_nvidia_mega.qwen3_5.sm103.draft.front import ( + fused_qk_gemma_rmsnorm_rope_gate, + ) + + g = torch.Generator(device="cuda").manual_seed(rows + 10 * mrope) + args, axis, *_ = _front_operands(rows, g, mrope) + want = sglang_op(*args, has_gate=True, mrope_axis_map=axis) + got = fused_qk_gemma_rmsnorm_rope_gate(*args, has_gate=True, mrope_axis_map=axis) + assert all(_bits(a, b) for a, b in zip(want, got)) + + +def test_front_with_the_cache_write_replays_in_a_cuda_graph_bit_for_bit(): + from sgl_nvidia_mega.qwen3_5.sm103.draft.front import ( + fused_qk_gemma_rmsnorm_rope_gate, + ) + + g = torch.Generator(device="cuda").manual_seed(5) + args, axis, v, pools, loc = _front_operands(7, g) + scales = [torch.full((1,), s, device="cuda") for s in (0.8, 1.3)] + kw = dict(has_gate=True, mrope_axis_map=axis, v=v, kv_cache=pools, cache_loc=loc) + + def run(): + return fused_qk_gemma_rmsnorm_rope_gate( + *args, **kw, k_scale=scales[0], v_scale=scales[1] + ) + + q8, _, gate = run() + eager = [q8.clone(), gate.clone(), *(p.clone() for p in pools)] + for p in pools: + p.zero_() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + q8, _, gate = run() + graph.replay() + torch.cuda.synchronize() + rows = loc.long() + for a, b in zip(eager[:2], (q8, gate)): + assert _bits(a, b) + for a, b in zip(eager[2:], pools): + assert _bits(a[rows], b[rows]) diff --git a/test/kernels/test_qwen3_5_draft_front.py b/test/kernels/test_qwen3_5_draft_front.py new file mode 100644 index 00000000..6d9c4059 --- /dev/null +++ b/test/kernels/test_qwen3_5_draft_front.py @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Slot draft_front's attention front (sm103/draft/front.py, 2.2's ATTN_FRONT) against SGLang's +two launches it replaces, on one GPU: the fused q / k Gemma RMSNorm + RoPE + gate split, +then the fused FP8 q / k / v + paged KV write. The FP8 q, the gate and the K / V cache +rows must be the same bytes, at the draft's widths (1 and 7 rows), in eager and in a +CUDA graph replayed with new positions and slots.""" + +import pytest +import torch + +cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") + +HEADS, KV_HEADS, HEAD_DIM, ROTARY = 4, 1, 256, 64 # the TP8 draft layer's shapes +SLOTS = 4096 + + +def _inputs(rows, seed): + g = torch.Generator(device="cuda").manual_seed(seed) + dev = "cuda" + + def randn(*shape, scale=1.0): + return (torch.randn(*shape, generator=g, device=dev) * scale).to(torch.bfloat16) + + q_gate = randn(rows, HEADS * 2 * HEAD_DIM, scale=2.0) + k = randn(rows, KV_HEADS * HEAD_DIM, scale=2.0) + v = randn(rows, KV_HEADS * HEAD_DIM, scale=3.0) + q_gate[0, :8] = -0.0 + k[-1, 16:32] *= 40 + q_w, k_w = randn(HEAD_DIM, scale=0.1), randn(HEAD_DIM, scale=0.1) + pos = torch.arange(4096, device=dev, dtype=torch.float32)[:, None] + freq = 1.0 / (10000 ** (torch.arange(0, ROTARY, 2, device=dev) / ROTARY)) + cos_sin = torch.cat([torch.cos(pos * freq), torch.sin(pos * freq)], dim=-1) + positions = torch.randint(0, 4000, (rows,), generator=g, device=dev) + cache_loc = torch.randperm(SLOTS, generator=g, device=dev)[:rows].to(torch.int64) + k_scale = torch.tensor([0.75 if seed % 2 else 1.0], device=dev) + v_scale = torch.tensor([1.0], device=dev) + return q_gate, k, v, q_w, k_w, cos_sin, positions, cache_loc, k_scale, v_scale + + +def _caches(): + shape = (SLOTS, KV_HEADS, HEAD_DIM) + return ( + torch.zeros(shape, dtype=torch.float8_e4m3fn, device="cuda"), + torch.zeros(shape, dtype=torch.float8_e4m3fn, device="cuda"), + ) + + +def _stock(inputs, kv): + from sglang.kernels.ops.attention.fused_qk_rmsnorm_rope_gate import ( + fused_qk_gemma_rmsnorm_rope_gate, + ) + from sglang.kernels.ops.kvcache.fused_fp8_qkv_kv_cache import ( + fused_fp8_qkv_kv_cache, + ) + + q_gate, k, v, q_w, k_w, cos_sin, positions, cache_loc, k_scale, v_scale = inputs + rows = q_gate.shape[0] + q, k_out, gate = fused_qk_gemma_rmsnorm_rope_gate( + q_gate, k, q_w, k_w, cos_sin, positions, 1e-6, HEADS, KV_HEADS, HEAD_DIM, ROTARY + ) + q8 = fused_fp8_qkv_kv_cache( + q=q.view(rows, -1), + k=k_out.view(rows, -1), + v=v, + k_cache=kv[0], + v_cache=kv[1], + cache_loc=cache_loc, + k_scale=k_scale, + v_scale=v_scale, + ) + return q8.view(rows, -1), gate.view(rows, -1) + + +def _plugin(inputs, kv): + from sgl_nvidia_mega.qwen3_5.sm103.draft.front import ( + fused_qk_gemma_rmsnorm_rope_gate, + ) + + q_gate, k, v, q_w, k_w, cos_sin, positions, cache_loc, k_scale, v_scale = inputs + rows = q_gate.shape[0] + q8, k_none, gate = fused_qk_gemma_rmsnorm_rope_gate( + q_gate, + k, + q_w, + k_w, + cos_sin, + positions, + 1e-6, + HEADS, + KV_HEADS, + HEAD_DIM, + ROTARY, + v=v, + kv_cache=kv, + cache_loc=cache_loc, + k_scale=k_scale, + v_scale=v_scale, + ) + assert k_none is None + return q8.view(rows, -1), gate.view(rows, -1) + + +def _same(a, b): + return torch.equal( + a.contiguous().view(torch.uint8), b.contiguous().view(torch.uint8) + ) + + +@cuda +@pytest.mark.parametrize("rows", [1, 7]) +@pytest.mark.parametrize("seed", [0, 1, 2]) +def test_the_front_writes_what_sglangs_two_launches_write(rows, seed): + inputs = _inputs(rows, seed) + kv_s, kv_p = _caches(), _caches() + q_s, gate_s = _stock(inputs, kv_s) + q_p, gate_p = _plugin(inputs, kv_p) + assert q_p.dtype == torch.float8_e4m3fn + assert _same(q_s, q_p) and _same(gate_s, gate_p) + assert _same(kv_s[0], kv_p[0]) and _same(kv_s[1], kv_p[1]) + + +@cuda +def test_the_front_replays_in_a_graph_with_new_positions_and_slots(): + rows = 1 + inputs = _inputs(rows, 5) + kv_s, kv_p = _caches(), _caches() + static = [t.clone() for t in inputs] + _plugin(static, kv_p) # compile outside capture + _stock(static, kv_s) # the same warm-up row in the stock caches + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + q_p, gate_p = _plugin(static, kv_p) + for seed in (11, 12, 13): + fresh = _inputs(rows, seed) + for dst, src in zip(static, fresh): + dst.copy_(src) + graph.replay() + q_s, gate_s = _stock(fresh, kv_s) + assert _same(q_s, q_p) and _same(gate_s, gate_p) + assert _same(kv_s[0], kv_p[0]) and _same(kv_s[1], kv_p[1]) From 71974f58b63414d61202f81e608b13daea3decba Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:51:00 -0700 Subject: [PATCH 14/27] [Perf] [GB300] qwen3_5: Draft slot draft_skv2: the draft extend's attention on the split-KV attention (numeric) Inside draft_extend, the draft extend's decode attention runs on slot skv2's split-KV attention over the draft's KV pool, with the extend's KV-length graph variants (SGLang's attention below 32768 tokens). Draft slot draft_skv2 (with draft_extend); numeric: SKV2 sums in another order than trtllm-gen. --- sgl_nvidia_mega/qwen3_5/draft/__init__.py | 19 +++++++- sgl_nvidia_mega/qwen3_5/draft/attention.py | 51 +++++++++++++++++++++ sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 7 ++- test/unit/test_qwen3_5_runtime.py | 27 +++++++++++ 4 files changed, 101 insertions(+), 3 deletions(-) diff --git a/sgl_nvidia_mega/qwen3_5/draft/__init__.py b/sgl_nvidia_mega/qwen3_5/draft/__init__.py index e98302e5..2b7457dd 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/draft/__init__.py @@ -15,7 +15,8 @@ Inside them every op is SGLang's own unless a draft slot replaces it: ``draft_moe`` (the single-launch MoE), ``draft_q1`` (the q1 split-KV decode), ``draft_xchg`` (the step boundary's fc split, exchange and head), ``draft_hop`` (the draft hops and the -o_proj GEMV) and ``draft_front`` (the attention front writing the FP8 KV itself). +o_proj GEMV), ``draft_front`` (the attention front writing the FP8 KV itself) and +``draft_skv2`` (the draft extend's split-KV decode, on the attention package's SKV2). """ from __future__ import annotations @@ -70,7 +71,14 @@ def numerics(slots: frozenset) -> Numerics: which differ from SGLang's ops only in fp32 summation order (q1 split-KV, the o_proj / fc / MoE GEMVs, the draft head), so a draft token can differ where two logits are within a bf16 rounding.""" - if not slots & {"draft_moe", "draft_q1", "draft_xchg", "draft_hop", "draft_front"}: + if not slots & { + "draft_moe", + "draft_q1", + "draft_xchg", + "draft_hop", + "draft_front", + "draft_skv2", + }: return target.NUMERICS return Numerics(NumericsKind.TOLERANCE, {"draft_logits.rel": 2**-7}) @@ -120,6 +128,13 @@ def __init__( from sgl_nvidia_mega.qwen3_5.draft.attention import Q1Variants self.graph_variants = {"draft_loop": Q1Variants()} + if "draft_extend" in slots and "draft_skv2" in slots: + from sgl_nvidia_mega.qwen3_5.attention import route + + self.graph_variants = { + **(self.graph_variants or {}), + "draft_extend": route.EXTEND, + } def summary(self) -> str: entries = [ diff --git a/sgl_nvidia_mega/qwen3_5/draft/attention.py b/sgl_nvidia_mega/qwen3_5/draft/attention.py index 2dc001dc..38f3fde8 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/attention.py +++ b/sgl_nvidia_mega/qwen3_5/draft/attention.py @@ -18,8 +18,12 @@ from __future__ import annotations +import logging + import torch +logger = logging.getLogger(__name__) + # the KV-length graph variants of the draft loop (2.2's graph_variants.Q1_STOCK / # Q1_LONG): trtllm-gen below Q1_MIN_LENGTH, SKV2-Q1 from it (2.2's # SGLANG_Q35MF_MKA_Q1_MIN_LENGTH default, which its TP8 serving configuration keeps) @@ -166,6 +170,10 @@ def __init__(self, parts): self.q1 = None # sm103.draft.q1 decode (slot draft_q1), set by DraftLoop # slot draft_front: 2.2's attention front writes the KV and FP8 q in one launch self.front = "draft_front" in parts.slots + # slot draft_skv2: the draft extend's split-KV decode (DraftExtendSplitKV) + self.skv2 = ( + DraftExtendSplitKV.build(parts) if "draft_skv2" in parts.slots else None + ) def __call__( self, x, positions, cache_loc, meta, kv, *, rows_per_request: int, o_proj=None @@ -259,6 +267,8 @@ def decode(self, q8, meta, kv, rows_per_request: int): bmm2_scale = kv.v_scale[0] if self.q1 is not None and rows_per_request == 1 and _q1_label() != Q1_STOCK: return self.q1(q8, kv, meta).view(-1, self.heads * self.head_dim) + if self.skv2 is not None and rows_per_request > 1 and self.skv2.long(): + return self.skv2.run(attn, q8, kv, meta) kwargs = {} if rows_per_request != 1: kwargs["q_len_per_req"] = rows_per_request @@ -298,6 +308,47 @@ def _paged(self, k_cache, v_cache): return k, v +class DraftExtendSplitKV: + """Slot draft_skv2: the draft extend's 7-row decode as 2.2's SKV2 (MKA_SKV2_DRAFT, + from 32768 tokens of KV), through the target attention package's SKV2 op + (``qwen3_5.attention.split_kv``, slot skv2's) on the draft's pool.""" + + def __init__(self, split_kv, route): + self.split_kv, self.route = split_kv, route + self._logged = False + + @classmethod + def build(cls, parts): + try: + from sgl_nvidia_mega.qwen3_5.attention import route, split_kv + except ImportError as e: + raise RuntimeError( + "NVIDIA megakernel: slot draft_skv2 needs the attention package's SKV2 " + f"(qwen3_5.attention.split_kv): {e}" + ) from e + width = int(parts.binding.tokens_per_request) + why = split_kv.unsupported(parts.layer.attn, width) + if why: + raise RuntimeError(f"NVIDIA megakernel: slot draft_skv2 cannot run: {why}") + return cls(split_kv, route) + + def long(self) -> bool: + """The draft-extend graph being captured is the split-KV one ("split_kv_long"), + or SGLang captures no label of ours (then every length, as for the verify).""" + label = _q1_label() + return label is None or label == self.route.SPLIT_KV_LONG + + def run(self, attn, q8, kv, meta): + if not self._logged and torch.cuda.is_current_stream_capturing(): + self._logged = True + logger.info( + "NVIDIA megakernel: Qwen3.5 MTP draft slot draft_skv2 runs the draft " + "extend's %d rows as SKV2", + q8.shape[0], + ) + return self.split_kv.skv2(attn, q8, kv, meta) + + def _q1_label(): """The KV-length graph variant SGLang is capturing ("q1_stock" / "q1_long"), or None where its draft runner captures one graph per batch size (then Q1 serves diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index 376c7ef3..f1a271b2 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -36,6 +36,7 @@ "draft_xchg", "draft_hop", "draft_front", + "draft_skv2", ) SLOTS_ENV = "SGLANG_NVIDIA_MEGAKERNEL_SLOTS" @@ -89,11 +90,15 @@ def slot_reasons() -> list[str]: reasons.append( f"{SLOTS_ENV}: slot 'gdn_pdl' launches slot 'gdn_verify''s verify" ) - for name in sorted(SLOTS & set(DRAFT_SLOTS)): + for name in sorted(SLOTS & set(DRAFT_SLOTS) - {"draft_skv2"}): if "draft_loop" not in SLOTS: reasons.append(f"{SLOTS_ENV}: slot {name!r} runs inside entry 'draft_loop'") if "draft_hop" in SLOTS and "hop" not in SLOTS: reasons.append(f"{SLOTS_ENV}: slot 'draft_hop' runs on slot 'hop''s mailbox") + if "draft_skv2" in SLOTS and "draft_extend" not in SLOTS: + reasons.append( + f"{SLOTS_ENV}: slot 'draft_skv2' runs inside entry 'draft_extend'" + ) if "mka_push" in SLOTS and not {"mka", "hop"} <= SLOTS: reasons.append( f"{SLOTS_ENV}: slot 'mka_push' pushes slot 'mka''s epilogue into slot " diff --git a/test/unit/test_qwen3_5_runtime.py b/test/unit/test_qwen3_5_runtime.py index 77363cd1..51290e22 100644 --- a/test/unit/test_qwen3_5_runtime.py +++ b/test/unit/test_qwen3_5_runtime.py @@ -164,11 +164,38 @@ def test_the_draft_loop_routes_q1_by_kv_length_as_2_2( assert variants.select(batch) == want +def test_the_draft_extend_takes_the_split_kv_variants_with_slot_draft_skv2( + monkeypatch, +): + from sgl_nvidia_mega.qwen3_5 import draft + from sgl_nvidia_mega.qwen3_5.attention import route + + # inside entry draft_extend only: the draft loop is not needed + monkeypatch.setattr(runtime, "SLOTS", frozenset({"draft_extend", "draft_skv2"})) + assert runtime.slot_reasons() == [] + built = draft.SPEC.build("lm", SimpleNamespace(context_len=4096), _draft_ctx()) + assert built.graph_variants == {"draft_extend": route.EXTEND} + assert route.EXTEND.min_length == 32768 + # with draft_q1 the draft offers each runner its own variants + monkeypatch.setattr( + runtime, + "SLOTS", + frozenset({"draft_loop", "draft_q1", "draft_extend", "draft_skv2"}), + ) + built = draft.SPEC.build("lm", SimpleNamespace(context_len=4096), _draft_ctx()) + assert sorted(built.graph_variants) == ["draft_extend", "draft_loop"] + assert built.graph_variants["draft_extend"] is route.EXTEND + + @pytest.mark.parametrize( "slots, want", [ ({"draft_moe"}, "slot 'draft_moe' runs inside entry 'draft_loop'"), ({"draft_loop", "draft_hop"}, "slot 'draft_hop' runs on slot 'hop''s mailbox"), + ( + {"draft_loop", "draft_skv2"}, + "slot 'draft_skv2' runs inside entry 'draft_extend'", + ), ], ) def test_the_draft_slots_name_what_they_need(monkeypatch, slots, want): From c679a3b9c69a51a969230c422fb132e96ca8d12c Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:52:00 -0700 Subject: [PATCH 15/27] [Perf] qwen3_5: The draft extend writes its token and selected rows straight into SGLang's outputs The served draft extend gathers its selected rows into hidden_out, and the exchange writes the token and its 1.0 into topk_index_out / topk_p_out, so the graph's three closing copies go. Entry draft_extend with slot draft_xchg; bitwise: the values are the same. --- sgl_nvidia_mega/qwen3_5/draft/extend.py | 9 ++++----- sgl_nvidia_mega/qwen3_5/draft/xchg.py | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/sgl_nvidia_mega/qwen3_5/draft/extend.py b/sgl_nvidia_mega/qwen3_5/draft/extend.py index 27591343..8a438cfa 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/extend.py +++ b/sgl_nvidia_mega/qwen3_5/draft/extend.py @@ -74,14 +74,13 @@ def step(self, io) -> None: io.kv, rows_per_request=io.tokens_per_request, ) - selected = hidden.index_select(0, io.select_index) + # the selected rows and the exchange's token go straight into SGLang's outputs: no + # copies at the end of the graph + selected = torch.index_select(hidden, 0, io.select_index, out=io.hidden_out) if mkd is not None: lm = parts.lm local = lm.logits_processor._compute_lm_head(selected, lm.lm_head, None) - topk_p, topk_index = mkd.exchange_extend(local, selected) - io.topk_index_out.copy_(topk_index) - io.topk_p_out.copy_(topk_p) + mkd.exchange_extend(local, selected, io.topk_index_out, io.topk_p_out) else: io.topk_index_out.copy_(layer.top1(parts, selected).view(-1, 1)) io.topk_p_out.fill_(1.0) - io.hidden_out.copy_(selected) diff --git a/sgl_nvidia_mega/qwen3_5/draft/xchg.py b/sgl_nvidia_mega/qwen3_5/draft/xchg.py index 8cfcc241..58352175 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/xchg.py +++ b/sgl_nvidia_mega/qwen3_5/draft/xchg.py @@ -527,12 +527,23 @@ def select_extend( return self.exchange_extend(local_logits, hidden_states) def exchange_extend( - self, local_logits: torch.Tensor, hidden_states: Optional[torch.Tensor] + self, + local_logits: torch.Tensor, + hidden_states: Optional[torch.Tensor], + topk_index: Optional[torch.Tensor] = None, + topk_p: Optional[torch.Tensor] = None, ) -> tuple[torch.Tensor, torch.Tensor]: """Draft-extend token (torch.argmax NaN rule) through the same one-launch exchange; - with extend_fc it also writes the next draft's first fc into fc_buf.""" - topk_index = torch.empty((1, 1), dtype=torch.int64, device=local_logits.device) - topk_p = torch.empty((1, 1), dtype=torch.float32, device=local_logits.device) + with extend_fc it also writes the next draft's first fc into fc_buf. The token and + its 1.0 land in ``topk_index`` / ``topk_p`` ([1, 1]) when given.""" + if topk_index is None: + topk_index = torch.empty( + (1, 1), dtype=torch.int64, device=local_logits.device + ) + if topk_p is None: + topk_p = torch.empty( + (1, 1), dtype=torch.float32, device=local_logits.device + ) payload = ( self.extend_fc and hidden_states is not None and hidden_states.shape[0] == 1 ) From a7b94636cde53063351892dfbfd07c3782e85689 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:53:00 -0700 Subject: [PATCH 16/27] [Perf] [GB300] qwen3_5: The GDN front takes each projection CTA's weights by one bulk copy into shared memory Slot gdn_front: each projection CTA brings its 128 KB of packed FP8 in_proj_qkvz weights into shared memory with one TMA bulk copy issued before its dependency wait, instead of 32 register loads per thread. The MMA sequence is unchanged, so every output is the same (bitwise). --- sgl_nvidia_mega/qwen3_5/sm103/gdn/front.py | 18 +++-- .../qwen3_5/sm103/gdn/src/front.cu | 70 ++++++++++++++++--- test/kernels/test_qwen3_5_gdn.py | 27 ++++++- 3 files changed, 101 insertions(+), 14 deletions(-) diff --git a/sgl_nvidia_mega/qwen3_5/sm103/gdn/front.py b/sgl_nvidia_mega/qwen3_5/sm103/gdn/front.py index ec5e748c..5b04a108 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/gdn/front.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/gdn/front.py @@ -10,7 +10,9 @@ It launches as a programmatic dependent and releases its own dependents at entry: the next launch must wait (``griddepcontrol.wait``) before it reads any output, b / a -included. +included. By default (``tma``) each projection CTA's packed weights arrive by one bulk +copy into shared memory issued before that wait, instead of register loads; same MMAs, +same bits. """ from __future__ import annotations @@ -33,6 +35,8 @@ TIMEOUT = ( 500_000_000 # clock cycles a CTA waits for its cluster's partials before trapping ) +# one projection CTA's packed weights (kWeightBytes) +WEIGHT_BYTES = 8 * 16 * 2 * 32 * 16 class Prefetch(ctypes.Structure): @@ -109,7 +113,7 @@ def pack(weight_nk: torch.Tensor) -> torch.Tensor: @cache -def _kernel(device_index: int) -> jit.Kernel: +def _kernel(device_index: int, tma: bool = True) -> jit.Kernel: major, minor = torch.cuda.get_device_capability(device_index) cubin = jit.compile_source( SOURCE.read_text(), @@ -117,12 +121,14 @@ def _kernel(device_index: int) -> jit.Kernel: f"sm_{major}{minor}a", ("-std=c++17", "-default-device"), ) + if tma: + return jit.load(cubin, "q35_gdn_front_tma", (THREADS, 1, 1), WEIGHT_BYTES) return jit.load(cubin, "q35_gdn_front", (THREADS, 1, 1), 0) def build(device_index: int) -> None: """Compile the front (no compile inside a step).""" - _kernel(device_index) + _kernel(device_index, True) def _prefetch(t: torch.Tensor, ring: int): @@ -162,6 +168,7 @@ def front( history_g: torch.Tensor, ring_indices: torch.Tensor, pad_slot: int = -1, + tma: bool = True, ) -> None: """The front of the BF16 rows ``x`` [7, 4096] (contiguous) into the caller's buffers: ``projected`` [7, 2560] BF16 (q / k / v / z), ``conv_out`` [7, 1536] BF16, @@ -171,7 +178,8 @@ def front( of ``pad_slot`` skips the conv writes). ``error`` is an int32 word, zero before capture, that a timed-out wait sets before trapping. The ReplaySSM verify's ``checkpoint`` (by state index) and ``history_*`` rings (by ``ring_indices[0]``) are - prefetched into L2: a hint, nothing is written.""" + prefetched into L2: a hint, nothing is written. ``tma`` False takes the weights by + register loads instead of the bulk copy (same values).""" if ( x.dtype != torch.bfloat16 or tuple(x.shape) != (ROWS, HIDDEN) @@ -230,7 +238,7 @@ def front( p.state_stride[:] = list(conv_state.stride()) p.window_stride[:] = [windows.stride(0), 9] jit.launch( - _kernel(x.device.index), + _kernel(x.device.index, tma), (CTAS, 1, 1), [p], torch.cuda.current_stream(x.device).cuda_stream, diff --git a/sgl_nvidia_mega/qwen3_5/sm103/gdn/src/front.cu b/sgl_nvidia_mega/qwen3_5/sm103/gdn/src/front.cu index 265e86c3..fb7abde4 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/gdn/src/front.cu +++ b/sgl_nvidia_mega/qwen3_5/sm103/gdn/src/front.cu @@ -8,8 +8,9 @@ // and history ring. Ported from gdn_pdl_front_ba_mkm.cuh of SGLang branch nvidia/qwen35-agentx-gb300 // (SGLANG_Q35MF_MKG_FUSED + MKG_BA_TC + MKM_GDN_FRONT); the device code is unchanged. // -// Built with NVRTC: front_kernel is a device function, and the extern "C" entry at the end -// instantiates its BF16-input form with the fused conv and the inline side projection. +// Built with NVRTC: front_kernel is a device function, and the extern "C" entries at the end +// instantiate its BF16-input form with the fused conv and the inline side projection, with the +// projection weights in registers (q35_gdn_front) or by bulk copy into shared memory (q35_gdn_front_tma). #include @@ -207,6 +208,21 @@ __device__ __forceinline__ void mbar_expect_tx(uint64_t* bar, uint32_t bytes) { asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(smem_u32(bar)), "r"(bytes) : "memory"); } +// Bulk copy of `bytes` from global memory into this CTA's shared memory, completing on `bar` (TMA engine). +__device__ __forceinline__ void bulk_g2s(void* dst, const void* src, uint32_t bytes, uint64_t* bar) { + asm volatile("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" + ::"r"(smem_u32(dst)), "l"(src), "r"(bytes), "r"(smem_u32(bar)) + : "memory"); +} +__device__ __forceinline__ void mbar_wait_parity0(uint64_t* bar) { + unsigned ready = 0; + do { + asm volatile("{ .reg .pred p; mbarrier.try_wait.parity.shared::cta.b64 p, [%1], 0; selp.u32 %0, 1, 0, p; }" + : "=r"(ready) + : "r"(smem_u32(bar)) + : "memory"); + } while (!ready); +} __device__ __forceinline__ float div_full(float a, float b) { float r; asm("div.full.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); @@ -289,13 +305,20 @@ struct Params { gdn_ba::Params ba; }; +// Weight bytes per projection CTA: [8 warps][16 steps][2 blocks][32 lanes] x 16 B, contiguous in the packed layout. +constexpr uint32_t kWeightBytes = 8 * 16 * 2 * 32 * 16; + // Each cluster reduces its split-K partials in fixed rank order. -template +// Tma: the CTA's weights arrive by one bulk copy into shared memory issued before the PDL wait (instead of 32 +// register loads per thread), then each thread reads the same A fragments; same MMAs, same bits. +template __device__ __forceinline__ void front_kernel(const Params& p) { if (threadIdx.x == 0) pdl_trigger(); __shared__ __align__(16) float inbox[kCluster][256]; __shared__ __align__(16) uint2 fragments[16][32]; __shared__ __align__(8) uint64_t barrier; + __shared__ __align__(8) uint64_t weight_barrier; + extern __shared__ __align__(128) uint4 weight_smem[]; // kWeightBytes when Tma __shared__ float conv_stage[32][8]; // per channel: state taps 0..2, weights 3..6 __shared__ int64_t conv_meta[3]; // valid, state slot, replay slot const int rank = cluster_rank(), unit = blockIdx.x / kCluster; @@ -310,9 +333,23 @@ __device__ __forceinline__ void front_kernel(const Params& p) { return; } } - if (tid == 0) { mbar_init(&barrier, 1); mbar_init_fence(); } + if (tid == 0) { + mbar_init(&barrier, 1); + if constexpr (Tma) mbar_init(&weight_barrier, 1); + mbar_init_fence(); + } __syncthreads(); cluster_arrive(); + const uint4* block_weights = p.weight + (size_t)(unit * kCluster + rank) * (8 * 16 * 2 * 32); + if constexpr (Tma) { + if (tid == 0) { + mbar_expect_tx(&weight_barrier, kWeightBytes); +#pragma unroll + for (int chunk = 0; chunk < 4; ++chunk) + bulk_g2s(weight_smem + chunk * (kWeightBytes / 64), block_weights + chunk * (kWeightBytes / 64), + kWeightBytes / 4, &weight_barrier); + } + } // Convolution owners: warp 0 reads the weights, indices and rolling state off the projection path. const bool conv_reader = FuseConv && unit < 6 && warp == 0; const int conv_channel = unit * 256 + rank * 32 + (tid >> 2) + 8 * (tid & 3); @@ -323,12 +360,14 @@ __device__ __forceinline__ void front_kernel(const Params& p) { for (int tap = 0; tap < 4; ++tap) conv_tap[tap] = p.conv_weight[conv_channel * 4 + tap]; } uint4 weights[16][2]; - const uint4* wp = p.weight + (size_t)(unit * kCluster + rank) * (8 * 16 * 2 * 32) + warp * 1024 + lane; + if constexpr (!Tma) { + const uint4* wp = block_weights + warp * 1024 + lane; #pragma unroll - for (int step = 0; step < 16; ++step) + for (int step = 0; step < 16; ++step) #pragma unroll - for (int block = 0; block < 2; ++block) - weights[step][block] = ldg_nc_v4(wp + (step * 2 + block) * 32); + for (int block = 0; block < 2; ++block) + weights[step][block] = ldg_nc_v4(wp + (step * 2 + block) * 32); + } const float sx = __ldg(p.sx), alpha = __fmul_rn(sx, __ldg(p.sw)); pdl_wait(); // Issued now, tested before the first push: no store precedes the check. @@ -360,6 +399,15 @@ __device__ __forceinline__ void front_kernel(const Params& p) { fragments[step][lane] = value; } __syncthreads(); + if constexpr (Tma) { + mbar_wait_parity0(&weight_barrier); + const uint4* ws = weight_smem + warp * 1024 + lane; +#pragma unroll + for (int step = 0; step < 16; ++step) +#pragma unroll + for (int block = 0; block < 2; ++block) + weights[step][block] = ws[(step * 2 + block) * 32]; + } float chains[2][4][4] = {}; #pragma unroll for (int step = 0; step < 16; ++step) { @@ -485,3 +533,9 @@ extern "C" __global__ __launch_bounds__(sglang::gdn_pdl_front_ba_mkm_detail::kTh const __grid_constant__ sglang::gdn_pdl_front_ba_mkm_detail::Params p) { sglang::gdn_pdl_front_ba_mkm_detail::front_kernel(p); } + +// The same with each projection CTA's weights bulk-copied into kWeightBytes of dynamic shared memory. +extern "C" __global__ __launch_bounds__(sglang::gdn_pdl_front_ba_mkm_detail::kThreads, 1) void q35_gdn_front_tma( + const __grid_constant__ sglang::gdn_pdl_front_ba_mkm_detail::Params p) { + sglang::gdn_pdl_front_ba_mkm_detail::front_kernel(p); +} diff --git a/test/kernels/test_qwen3_5_gdn.py b/test/kernels/test_qwen3_5_gdn.py index 2da8a4d4..bea4871d 100644 --- a/test/kernels/test_qwen3_5_gdn.py +++ b/test/kernels/test_qwen3_5_gdn.py @@ -284,7 +284,7 @@ def rand(*shape, scale=1.0): ) -def _run_front(o, packed, outs): +def _run_front(o, packed, outs, **kw): from sgl_nvidia_mega.qwen3_5.sm103.gdn import front front.front( @@ -307,6 +307,7 @@ def _run_front(o, packed, outs): history_k=o["ring_k"], history_g=o["ring_g"], ring_indices=o["req"], + **kw, ) @@ -393,6 +394,30 @@ def test_front_leaves_a_padded_requests_state_alone(): assert torch.equal(o["state"], state0) and torch.equal(o["phys"], phys0) +@pytest.mark.parametrize("pad", [False, True]) +def test_front_takes_its_weights_by_bulk_copy_with_the_same_bytes(pad): + """The default launch bulk-copies each projection CTA's weights into shared memory; + every output and pool equals the launch that loads them into registers.""" + from sgl_nvidia_mega.qwen3_5.sm103.gdn import front + + for seed in range(3): + o = _front_operands(20 + seed, pad=pad) + packed = front.pack(o["w"]) + start = {n: o[n].clone() for n in ("state", "phys")} + got = [] + for tma in (False, True): + for n, t in start.items(): + o[n].copy_(t) + outs = _front_outputs() + for t in outs.values(): # a padded request leaves the conv rows unwritten + t.zero_() + _run_front(o, packed, outs, tma=tma) + torch.cuda.synchronize() + got.append({**outs, "state": o["state"].clone(), "phys": o["phys"].clone()}) + for n, t in got[0].items(): + assert torch.equal(t, got[1][n]), n + + def test_front_replays_from_a_cuda_graph(): from sgl_nvidia_mega.qwen3_5.sm103.gdn import front From eb57195dc1deead3c0e0f1fd3060e9795321d057 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:54:00 -0700 Subject: [PATCH 17/27] [Perf] [GB300] qwen3_5: SKV2's producer and merge launch as programmatic dependents Slots skv2 and draft_skv2: the split-KV producer launches as a programmatic dependent of the attention's front and loads its leading K / V tiles before it waits, and the merge launches as the producer's dependent. Same arithmetic, same bytes (bitwise). --- .../qwen3_5/sm103/attention/skv2.py | 127 +++++++++++++++--- test/kernels/test_qwen3_5_attention.py | 20 +++ 2 files changed, 128 insertions(+), 19 deletions(-) diff --git a/sgl_nvidia_mega/qwen3_5/sm103/attention/skv2.py b/sgl_nvidia_mega/qwen3_5/sm103/attention/skv2.py index 16f1026a..5e2c0542 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/attention/skv2.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/attention/skv2.py @@ -41,6 +41,7 @@ # Modified by NVIDIA: runtime attention scales, page-tail bounds and reusable buffers. # Modified by NVIDIA: stock split-KV rounding (page splits, one FP8 P rounding, bf16 partials). +# Modified by NVIDIA: optional programmatic dependent launch with a pre-wait K/V prefetch. """SKV2: paged FP8 q7 split attention with the stock split-KV rounding structure. @@ -104,7 +105,17 @@ def load_values( class StockSplitDecode: - def __init__(self, heads=4, qlen=7, splits=148, stages=2, slots=2): + def __init__( + self, + heads=4, + qlen=7, + splits=148, + stages=2, + slots=2, + pdl=False, + prefetch=True, + trigger=False, + ): assert stages >= 1 and slots in (2, 3, 4) self.heads, self.qlen, self.splits, self.stages, self.slots = ( heads, @@ -113,6 +124,11 @@ def __init__(self, heads=4, qlen=7, splits=148, stages=2, slots=2): stages, slots, ) + # pdl: launched as a programmatic dependent; CTAs that land while the predecessor runs set up and (prefetch) + # load their first K / V tiles that hold no token written by this step, then wait for the predecessor. + # trigger: release the dependent (merge) after the wait; without it the merge launches at this grid's exit, + # which measured faster (an early-resident merge returns from its wait later than a fresh launch starts). + self.pdl, self.prefetch, self.trigger = pdl, prefetch, trigger self.rows = heads * qlen self.width = 32 if self.rows <= 32 else 64 self.tile = 128 @@ -158,7 +174,9 @@ def __call__( ) self.kernel( q, k, v, table, lengths, partial, maxima, denom, qk_scale, out_scale, kq, vp - ).launch(grid=(self.splits, 1, 1), block=(384, 1, 1), stream=stream) + ).launch( + grid=(self.splits, 1, 1), block=(384, 1, 1), stream=stream, use_pdl=self.pdl + ) @cute.experimental.kernel def kernel( @@ -296,6 +314,44 @@ def kernel( nblk = cute.ceil_div(end_tok - begin_tok, self.tile) begin_page = split * pps cute.arch.sync_threads() + # Leading tiles loaded before the dependency wait: whole tiles (2 pages) ending before the first page + # this step writes (the predecessor stores the new tokens' K / V at positions length - qlen and up). + npre = cutlass.Int32(0) + if cutlass.const_expr(self.pdl and self.prefetch): + safe_tiles = ((length - self.qlen) // 64 - begin_page) // (self.tile // 64) + if safe_tiles > 0: + npre = safe_tiles + if npre > self.stages: + npre = cutlass.Int32(self.stages) + if npre > nblk: + npre = nblk + if warp == 10: + for stage in cutlass.range_constexpr(self.stages): + if stage < npre: + load_keys( + k, + table, + sk, + bk + stage, + begin_page + stage * (self.tile // 64), + stage, + self.tile, + length, + ) + load_values( + v, + table, + sv, + bv + stage, + begin_page + stage * (self.tile // 64), + stage, + self.tile, + length, + ) + if cutlass.const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + if cutlass.const_expr(self.trigger): + cute.arch.griddepcontrol_launch_dependents() if warp < 4: scores = cute.make_rmem_tensor((width, 1), f32) running_m = cute.make_rmem_tensor((width // 32,), f32) @@ -459,7 +515,7 @@ def kernel( update_expect_tx=False, ) for stage in cutlass.range_constexpr(self.stages): - if stage < nblk: + if stage < nblk and stage >= npre: load_keys( k, table, @@ -472,19 +528,20 @@ def kernel( ) for it in cutlass.range(0, nblk): stage = it % self.stages - cute.arch.mbarrier_wait( - bvfree + stage, phase=(it // self.stages + 1) % 2 - ) - load_values( - v, - table, - sv, - bv + stage, - begin_page + it * (self.tile // 64), - stage, - self.tile, - length, - ) + if it >= npre: + cute.arch.mbarrier_wait( + bvfree + stage, phase=(it // self.stages + 1) % 2 + ) + load_values( + v, + table, + sv, + bv + stage, + begin_page + it * (self.tile // 64), + stage, + self.tile, + length, + ) if it + self.stages < nblk: cute.arch.mbarrier_wait( bkfree + stage, phase=(it // self.stages) % 2 @@ -578,7 +635,10 @@ def _merge_attention( BALANCED: tl.constexpr, SKIP_EMPTY: tl.constexpr, SHORT_MERGE: tl.constexpr, + USE_GDC: tl.constexpr = False, ): + if USE_GDC: # launched as a programmatic dependent of the split producer + tl.extra.cuda.gdc_wait() if SHORT_MERGE and not BALANCED: length = tl.load(Lengths + tl.program_id(2)) if length <= 2048: @@ -628,9 +688,27 @@ def _merge_attention( class StockSplitPlan: - """Reusable storage and a compiled producer for one fixed input layout; merge is the stock kernel.""" + """Reusable storage and a compiled producer for one fixed input layout; merge is the stock kernel. - def __init__(self, query, kv, table, lengths, *, splits=148, stages=2, slots=2): + ``pdl`` (the default): the producer launches as a programmatic dependent and loads its + leading K / V tiles that hold no token of this step before it waits; the merge launches + as its dependent at its exit and waits before reading the partials. Same arithmetic.""" + + def __init__( + self, + query, + kv, + table, + lengths, + *, + splits=148, + stages=2, + slots=2, + pdl=True, + prefetch=True, + merge_pdl=None, + trigger=False, + ): if torch.cuda.is_current_stream_capturing(): raise RuntimeError("SKV2 storage must be prepared before graph capture") self.qlen, self.heads, _ = query.shape @@ -643,8 +721,17 @@ def __init__(self, query, kv, table, lengths, *, splits=148, stages=2, slots=2): (splits, rows), dtype=torch.float32, device=query.device ) self.denom = torch.empty_like(self.maximum) + # merge_pdl: the merge is a programmatic dependent of the producer (waits before reading the partials). + self.merge_pdl = pdl if merge_pdl is None else merge_pdl kernel = StockSplitDecode( - self.heads, self.qlen, splits, stages=stages, slots=slots + self.heads, + self.qlen, + splits, + stages=stages, + slots=slots, + pdl=pdl, + prefetch=prefetch, + trigger=trigger, ) views = self._views(query, kv, table, lengths) stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) @@ -691,8 +778,10 @@ def run(self, query, kv, table, lengths, *, qk_scale, v_scale, out): False, True, True, + USE_GDC=self.merge_pdl, num_warps=4, enable_fp_fusion=False, + **({"launch_pdl": True} if self.merge_pdl else {}), ) return out diff --git a/test/kernels/test_qwen3_5_attention.py b/test/kernels/test_qwen3_5_attention.py index 04bfe890..14aaa4d5 100644 --- a/test/kernels/test_qwen3_5_attention.py +++ b/test/kernels/test_qwen3_5_attention.py @@ -152,6 +152,26 @@ def test_skv2_replays_in_a_cuda_graph_bit_for_bit(): assert torch.equal(out.view(torch.int16), want.view(torch.int16)), length +@pytest.mark.parametrize("length", [7, 2048, 32769, 131072, 150007]) +def test_skv2_as_a_programmatic_dependent_gives_the_same_bytes(length): + """The default plan launches the producer and the merge as programmatic dependents, + the producer loading its leading K / V tiles before it waits: same bytes as the plan + that launches them plainly.""" + from sgl_nvidia_mega.qwen3_5.sm103.attention.skv2 import StockSplitPlan + + g = torch.Generator(device="cuda").manual_seed(100 + length) + q, pools, table, lengths = _inputs(length, g) + kv = tuple(map(_hnd, pools)) + outs = [] + for pdl in (False, True): + out = torch.empty(ROWS, HEADS, DIM, dtype=torch.bfloat16, device="cuda") + plan = StockSplitPlan(q, kv, table, lengths, pdl=pdl) + plan.run(q, kv, table, lengths, qk_scale=0.75 * SCALE, v_scale=0.5, out=out) + outs.append(out) + torch.cuda.synchronize() + assert torch.equal(outs[0].view(torch.int16), outs[1].view(torch.int16)) + + HIDDEN, QKV_ROWS, POOL_SLOTS = 4096, (2 * HEADS + 2) * DIM, 1024 From a183562959d1536800271c366619891976ad703f Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:55:00 -0700 Subject: [PATCH 18/27] [Perf] [GB300] qwen3_5: The draft q1 decode and its merge launch as programmatic dependents Slot draft_q1: the SKV2-Q1 producer launches as a programmatic dependent and loads its leading K / V tiles before it waits, and the merge waits before it reads the partials; slot draft_front's one-row front releases its dependents at entry. The arithmetic is unchanged (bitwise). --- sgl_nvidia_mega/qwen3_5/draft/attention.py | 13 ++- sgl_nvidia_mega/qwen3_5/sm103/draft/front.py | 10 ++- sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py | 16 ++-- .../qwen3_5/sm103/draft/q1_kernel.py | 90 +++++++++++++++---- 4 files changed, 103 insertions(+), 26 deletions(-) diff --git a/sgl_nvidia_mega/qwen3_5/draft/attention.py b/sgl_nvidia_mega/qwen3_5/draft/attention.py index 38f3fde8..d336bc30 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/attention.py +++ b/sgl_nvidia_mega/qwen3_5/draft/attention.py @@ -29,6 +29,9 @@ # SGLANG_Q35MF_MKA_Q1_MIN_LENGTH default, which its TP8 serving configuration keeps) Q1_STOCK, Q1_LONG = "q1_stock", "q1_long" Q1_MIN_LENGTH = 16384 +# A draft step's q1 decode and its merge launch as programmatic dependents, the +# one-row front releasing them at entry (same arithmetic) +Q1_PDL = True class Q1Variants: @@ -232,6 +235,7 @@ def _front(self, x, positions, cache_loc, kv): cache_loc=cache_loc, k_scale=self.k_scale, v_scale=self.v_scale, + early_trigger=x.shape[0] == 1 and self.q1 is not None and Q1_PDL, ) rows = x.shape[0] return q8.view(rows, -1), (gate.view(rows, -1) if gate is not None else None) @@ -372,8 +376,13 @@ def build_q1(parts): ) if why is not None: raise RuntimeError(f"NVIDIA megakernel: slot draft_q1 cannot run: {why}") - q1.build(device.index, heads=attention.heads) - decode = q1.Q1Decode() + q1.build(device.index, heads=attention.heads, pdl=Q1_PDL) + decode = q1.Q1Decode(pdl=Q1_PDL) + if Q1_PDL: + logger.info( + "NVIDIA megakernel: Qwen3.5 MTP draft slot draft_q1 launches the q1 decode " + "and its merge as programmatic dependents" + ) attn = parts.layer.attn def run(q8, kv, meta): diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/front.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/front.py index 4b4d99b6..97b3637f 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/front.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/front.py @@ -71,6 +71,7 @@ def _fused_qk_rmsnorm_rope_gate_kernel( MROPE: tl.constexpr, ENABLE_PDL: tl.constexpr, WEIGHT_SHIFT: tl.constexpr = 1.0, + EARLY_TRIGGER: tl.constexpr = False, EXPLICIT_ROPE_FMA: tl.constexpr = False, v_ptr=None, v_cache_ptr=None, @@ -81,6 +82,9 @@ def _fused_qk_rmsnorm_rope_gate_kernel( stride_v_cache_t: tl.constexpr = 0, FP8_CACHE: tl.constexpr = False, ): + # EARLY_TRIGGER: release the dependents at entry; they must wait before reading this kernel's outputs. + if ENABLE_PDL and EARLY_TRIGGER: + tl.extra.cuda.gdc_launch_dependents() token = tl.program_id(0) head = tl.program_id(1) is_k = head >= NUM_Q_HEADS @@ -184,7 +188,7 @@ def _fused_qk_rmsnorm_rope_gate_kernel( # PDL: signal dependent kernels (attention/allreduce) can start early. # Only available on NVIDIA Hopper+ (sm_90+); guarded for AMD/other backends. - if ENABLE_PDL: + if ENABLE_PDL and not EARLY_TRIGGER: tl.extra.cuda.gdc_launch_dependents() @@ -208,9 +212,12 @@ def fused_qk_gemma_rmsnorm_rope_gate( cache_loc: Optional[torch.Tensor] = None, k_scale: Optional[torch.Tensor] = None, v_scale: Optional[torch.Tensor] = None, + early_trigger: bool = False, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: """Fused QK GemmaRMSNorm + NeoX RoPE + gate deinterleave. + early_trigger: release programmatic dependents at entry (they must wait before reading the outputs). + Args: q_gate: [T, num_q_heads * (1 + has_gate) * head_dim] — interleaved Q+Gate if has_gate k: [T, num_kv_heads * head_dim] @@ -303,6 +310,7 @@ def fused_qk_gemma_rmsnorm_rope_gate( HAS_GATE=has_gate, MROPE=mrope, ENABLE_PDL=_ENABLE_PDL, + EARLY_TRIGGER=early_trigger, **kwargs, ) diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py index d62e4b3d..26eba4ce 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py @@ -228,10 +228,13 @@ class Q1Decode: """One rank's SKV2-Q1 plans by input layout (the backend's ``_q1_plans``): a plan owns the CTA partials and the producer compiled for that layout, so the first call of a layout must not be under graph capture (graph warmup runs it - eagerly, as in the branch).""" + eagerly, as in the branch). ``pdl``: the producer and the merge launch as + programmatic dependents, the producer loading the leading K / V tiles this step + does not write before its wait (same arithmetic).""" - def __init__(self): + def __init__(self, pdl: bool = True): self._plans = {} + self.pdl = pdl @staticmethod def key(query, kv_cache, block_tables, seq_lens) -> tuple: @@ -250,7 +253,7 @@ def plan(self, query, kv_cache, block_tables, seq_lens): key = self.key(query, kv_cache, block_tables, seq_lens) plan = self._plans.get(key) if plan is None: - plan = Q1SplitPlan(query, kv_cache, block_tables, seq_lens) + plan = Q1SplitPlan(query, kv_cache, block_tables, seq_lens, pdl=self.pdl) self._plans[key] = plan return plan @@ -334,9 +337,10 @@ def prepare(self, heads: int, kv, layer_id: int, step) -> None: @cache -def build(device_index: int, heads: int = 4) -> None: +def build(device_index: int, heads: int = 4, pdl: bool = True) -> None: """Import CuTe DSL and Triton and compile the fp32 merge for ``heads`` rows and - this GPU's SM count; each layout's producer compiles at its plan's first use.""" + this GPU's SM count (as a programmatic dependent with ``pdl``); each layout's + producer compiles at its plan's first use.""" import triton from sgl_nvidia_mega.qwen3_5.sm103.draft.q1_kernel import _q1_merge @@ -361,6 +365,8 @@ def build(device_index: int, heads: int = 4) -> None: nsplit, triton.next_power_of_2(nsplit), MERGE_BLOCK_D, + USE_GDC=pdl, num_warps=MERGE_WARPS, + **({"launch_pdl": True} if pdl else {}), ) torch.cuda.synchronize(device) diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/q1_kernel.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/q1_kernel.py index 6c5e84b6..d8c871d1 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/q1_kernel.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/q1_kernel.py @@ -118,9 +118,14 @@ def load_values( class Q1SplitDecode: - def __init__(self, heads=4, nsplit=152, stages=2, slots=2): + def __init__( + self, heads=4, nsplit=152, stages=2, slots=2, pdl=False, prefetch=True + ): assert stages >= 1 and slots in (2, 3, 4) and 1 <= heads <= 12 self.heads, self.nsplit, self.stages, self.slots = heads, nsplit, stages, slots + # pdl: launched as a programmatic dependent; CTAs that land while the predecessor runs load their first + # K / V tiles that hold no token written by this step, then wait (the merge launches at this grid's exit). + self.pdl, self.prefetch = pdl, prefetch self.rows = heads self.width = 32 self.tile = 128 @@ -166,7 +171,9 @@ def __call__( ) self.kernel( q, k, v, table, lengths, partial, maxima, denom, qk_scale, out_scale, kq, vp - ).launch(grid=(self.nsplit, 1, 1), block=(384, 1, 1), stream=stream) + ).launch( + grid=(self.nsplit, 1, 1), block=(384, 1, 1), stream=stream, use_pdl=self.pdl + ) @cute.experimental.kernel def kernel( @@ -309,6 +316,42 @@ def kernel( npair = cute.ceil_div(nblk, 2) begin_page = split * spc * 4 cute.arch.sync_threads() + # Leading tiles loaded before the dependency wait: whole tiles ending before the page of the token this + # step writes (position length - 1). + npre = cutlass.Int32(0) + if cutlass.const_expr(self.pdl and self.prefetch): + safe_tiles = ((length - 1) // 64 - begin_page) // (self.tile // 64) + if safe_tiles > 0: + npre = safe_tiles + if npre > self.stages: + npre = cutlass.Int32(self.stages) + if npre > nblk: + npre = nblk + if warp == 10: + for stage in cutlass.range_constexpr(self.stages): + if stage < npre: + load_keys( + k, + table, + sk, + bk + stage, + begin_page + stage * (self.tile // 64), + stage, + self.tile, + length, + ) + load_values( + v, + table, + sv, + bv + stage, + begin_page + stage * (self.tile // 64), + stage, + self.tile, + length, + ) + if cutlass.const_expr(self.pdl): + cute.arch.griddepcontrol_wait() if warp < 4: scores = cute.make_rmem_tensor((width, 1), f32) running_m = cute.make_rmem_tensor((2,), f32) @@ -497,7 +540,7 @@ def kernel( update_expect_tx=False, ) for stage in cutlass.range_constexpr(self.stages): - if stage < nblk: + if stage < nblk and stage >= npre: load_keys( k, table, @@ -510,19 +553,20 @@ def kernel( ) for it in cutlass.range(0, nblk): stage = it % self.stages - cute.arch.mbarrier_wait( - bvfree + stage, phase=(it // self.stages + 1) % 2 - ) - load_values( - v, - table, - sv, - bv + stage, - begin_page + it * (self.tile // 64), - stage, - self.tile, - length, - ) + if it >= npre: + cute.arch.mbarrier_wait( + bvfree + stage, phase=(it // self.stages + 1) % 2 + ) + load_values( + v, + table, + sv, + bv + stage, + begin_page + it * (self.tile // 64), + stage, + self.tile, + length, + ) if it + self.stages < nblk: cute.arch.mbarrier_wait( bkfree + stage, phase=(it // self.stages) % 2 @@ -573,7 +617,10 @@ def _q1_merge( NSPLIT: tl.constexpr, NP2: tl.constexpr, BLOCK_D: tl.constexpr, + USE_GDC: tl.constexpr = False, ): + if USE_GDC: # launched as a programmatic dependent of the q1 producer + tl.extra.cuda.gdc_wait() row = tl.program_id(0) dblk = tl.program_id(1) length = tl.load(lengths) @@ -599,7 +646,9 @@ def _q1_merge( class Q1SplitPlan: """Reusable storage and a compiled producer for one fixed input layout, plus the fp32 merge.""" - def __init__(self, query, kv, table, lengths, *, nsplit=None, stages=2, slots=2): + def __init__( + self, query, kv, table, lengths, *, nsplit=None, stages=2, slots=2, pdl=False + ): if torch.cuda.is_current_stream_capturing(): raise RuntimeError("SKV2-Q1 storage must be prepared before graph capture") qlen, self.heads, _ = query.shape @@ -615,7 +664,10 @@ def __init__(self, query, kv, table, lengths, *, nsplit=None, stages=2, slots=2) (self.nsplit, self.heads), dtype=torch.float32, device=query.device ) self.denom = torch.empty_like(self.maximum) - kernel = Q1SplitDecode(self.heads, self.nsplit, stages=stages, slots=slots) + self.pdl = pdl + kernel = Q1SplitDecode( + self.heads, self.nsplit, stages=stages, slots=slots, pdl=pdl + ) views = self._views(query, kv, table, lengths) stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) self.compiled = cute.compile( @@ -655,7 +707,9 @@ def run(self, query, kv, table, lengths, *, qk_scale, v_scale, out): self.nsplit, triton.next_power_of_2(self.nsplit), 32, + USE_GDC=self.pdl, num_warps=4, + **({"launch_pdl": True} if self.pdl else {}), ) return out From 0b0edc26f5c0be38749f3ec7213c8a44bf971efc Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:56:00 -0700 Subject: [PATCH 19/27] [Perf] [GB300] qwen3_5: The final hop releases the draft head at its entry, with a shared-memory pad Slots draft_xchg and draft_hop: the draft step's final all-reduce + norm hop releases the draft head at its entry, and the head launches with 120 KB of unused shared memory per CTA so that it starts only on SMs the draft MoE has left. Each CTA's rows and arithmetic are unchanged (bitwise). --- sgl_nvidia_mega/qwen3_5/draft/layer.py | 7 ++++++- sgl_nvidia_mega/qwen3_5/draft/parts.py | 8 +++++++- sgl_nvidia_mega/qwen3_5/draft/xchg.py | 21 +++++++++++++++++++-- sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py | 16 +++++++++------- 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/sgl_nvidia_mega/qwen3_5/draft/layer.py b/sgl_nvidia_mega/qwen3_5/draft/layer.py index 7c7e73bd..f66e1db8 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/layer.py +++ b/sgl_nvidia_mega/qwen3_5/draft/layer.py @@ -103,7 +103,12 @@ def layer(parts, x, positions, cache_loc, meta, kv, *, rows_per_request: int): ) moe_out = moe(moe_in) if moe is not None else lay.mlp(moe_in) if hop is not None: - out, _ = hop.final_norm(moe_out, residual, parts.model.norm) + out, _ = hop.final_norm( + moe_out, + residual, + parts.model.norm, + head_follows=parts.mkd is not None and parts.mkd.head_follows(), + ) return out out, _ = parts.model.norm(tensor_model_parallel_all_reduce(moe_out), residual) return out diff --git a/sgl_nvidia_mega/qwen3_5/draft/parts.py b/sgl_nvidia_mega/qwen3_5/draft/parts.py index c6f9a241..780c06d5 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/parts.py +++ b/sgl_nvidia_mega/qwen3_5/draft/parts.py @@ -115,10 +115,16 @@ def _build_hop(self): deadline_s=self.draft.deadline_s, o_proj_k=int(self.layer.o_proj.weight.shape[1]), early=True, - early_head=False, + early_head=True, ) if hop is None: raise RuntimeError(f"NVIDIA megakernel: slot draft_hop cannot run: {why}") + logger.info( + "NVIDIA megakernel: Qwen3.5 MTP draft hop releases at entry: the " + "single-launch MoE %s, the draft head %s", + hop.early, + hop.early_head, + ) return hop def _build_mkd(self): diff --git a/sgl_nvidia_mega/qwen3_5/draft/xchg.py b/sgl_nvidia_mega/qwen3_5/draft/xchg.py index 58352175..b1bd8454 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/xchg.py +++ b/sgl_nvidia_mega/qwen3_5/draft/xchg.py @@ -38,6 +38,10 @@ _MEMO_BLOCK = 4096 # CTAs of mkd_draft_head per SM (T1, TP8 shard: 1 -> 39.3 us, 2 -> 40.5 us, 3 -> 41.6 us) _HEAD_CTAS_PER_SM = 1 +# Unused dynamic shared memory per head CTA: an early-released head CTA then starts only on +# an SM with 128 KB of shared memory free (its registers already hold the head at one CTA +# per SM) +HEAD_SMEM_PAD = 120 * 1024 # Draft and draft-extend widths that take the row-split fc (2.2 _FC_SPLIT_MAX_TOKENS) FC_SPLIT_MAX_TOKENS = kernels.GATHER_MAX_TOKENS # Rows of a draft step the vocabulary exchange serves (2.2 DraftVocabExchange.MAX_ROWS) @@ -133,6 +137,7 @@ def __init__( memo: bool = True, extend_fc: bool = True, head: bool = True, + head_smem_pad: int = HEAD_SMEM_PAD, ) -> None: import torch.distributed as dist @@ -208,6 +213,7 @@ def __init__( self.ext_partials = torch.empty_like(self.out_partials) self.side_stream = torch.cuda.Stream(device=dev) self.lm_head_weight = lm_head_shard + self.head_pad = head_smem_pad self._install_head(head, dev) # extend_fc: the draft extend's exchange writes fc_buf for the next draft's first # step. input_seq counts calls that produce a draft input; fc_buf_key is the @@ -350,10 +356,11 @@ def _install_head(self, enabled: bool, dev: torch.device) -> None: _HEAD_CTAS_PER_SM * sms, dtype=torch.int32, device=dev ) logger.info( - "MKD draft head active: %d x %d shard, %d CTAs.", + "MKD draft head active: %d x %d shard, %d CTAs, smem pad %d B.", head.shape[0], head.shape[1], self.head_part.numel(), + self.head_pad, ) def head(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -366,11 +373,21 @@ def head(self, hidden_states: torch.Tensor) -> torch.Tensor: # Alternate the row walk per draft step: the next step re-reads first what is still in L2 # (T1: -1.9 to -2.4 us per call on the TP8 shard). kernels.mkd_draft_head( - x, self.lm_head_weight, logits, self.head_part, reverse=self.step % 2 == 1 + x, + self.lm_head_weight, + logits, + self.head_part, + reverse=self.step % 2 == 1, + smem_pad=self.head_pad, ) self.head_used = True return logits + def head_follows(self) -> bool: + """Whether the draft head is the next launch after this step's final norm (an + armed step with the head installed).""" + return self.armed and self.head_part is not None + # ---------------------------------------------------------------- per step def begin_step(self, step: int, num_rows: int) -> None: diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py index 82c0b274..45998214 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py @@ -15,9 +15,9 @@ - ``draft_fc_partial`` + ``FcGather`` + ``ordered_partial_sum``: the row-split fc of up to 16 tokens (fused norm + slice GEMV, all-gather of the fp32 partials, rank-order sum). -Every launch keeps the 2.2 grid, block, shared memory and PDL setting. ``FcGather`` moves -bytes only; the arithmetic of the split fc is ``draft_fc_partial`` and -``ordered_partial_sum``, copied unchanged. +Every launch keeps the 2.2 grid, block, shared memory and PDL setting (the draft head may +add unused shared memory, ``smem_pad``). ``FcGather`` moves bytes only; the arithmetic of +the split fc is ``draft_fc_partial`` and ``ordered_partial_sum``, copied unchanged. """ from __future__ import annotations @@ -201,9 +201,9 @@ def _fc_kernels(device_index: int, world: int) -> dict[str, jit.Kernel]: @cache -def _head_kernel(device_index: int) -> jit.Kernel: +def _head_kernel(device_index: int, smem_pad: int = 0) -> jit.Kernel: cubin = _compile(device_index, "mkd_head", ()) - return jit.load(cubin, "mkd_head", (HEAD_WARPS * 32, 1, 1), 0) + return jit.load(cubin, "mkd_head", (HEAD_WARPS * 32, 1, 1), smem_pad) @cache @@ -559,11 +559,13 @@ def mkd_draft_head( logits: torch.Tensor, part: torch.Tensor, reverse: bool = False, + smem_pad: int = 0, ) -> None: """Draft lm_head of one token, ``logits`` [1, N] bf16 = ``x`` [1, 4096] @ ``weight`` [N, 4096]^T with fp32 accumulation, and per CTA (``part.numel()`` CTAs) the packed maximum of its rows for mkd_xchg. ``reverse`` walks the rows from the end (same - values).""" + values). ``smem_pad`` bytes of unused dynamic shared memory per CTA (same values; + above half an SM it allows one CTA per SM).""" device = logits.device n = int(weight.shape[0]) _check(x, "x", (1, HIDDEN), torch.bfloat16, aligned=True) @@ -582,7 +584,7 @@ def mkd_draft_head( n=n, reverse=int(bool(reverse)), ) - _launch(_head_kernel(device.index), part.numel(), p, device, pdl=True) + _launch(_head_kernel(device.index, smem_pad), part.numel(), p, device, pdl=True) # ---------------------------------------------------------------- the row-split fc From 90e5c479dd2868e8eef9f8061005ef203d1f92a7 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:57:00 -0700 Subject: [PATCH 20/27] [Perf] [GB300] qwen3_5: On hidden ranks the draft head leaves SMs to the side-stream fc partial Slot draft_xchg: on the ranks whose fc slice is in the hidden half, the next step's hidden-half fc partial launches on 24 CTAs and the draft head on the remaining SMs beside it. Each row is still one warp's in the same arithmetic, so the logits, the partial and the token are the same (bitwise). --- sgl_nvidia_mega/qwen3_5/draft/xchg.py | 27 ++++++++++++++++--- .../qwen3_5/sm103/draft/src/mkd_fc.cu | 4 +-- sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py | 14 +++++----- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/sgl_nvidia_mega/qwen3_5/draft/xchg.py b/sgl_nvidia_mega/qwen3_5/draft/xchg.py index b1bd8454..52430dc1 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/xchg.py +++ b/sgl_nvidia_mega/qwen3_5/draft/xchg.py @@ -42,6 +42,10 @@ # an SM with 128 KB of shared memory free (its registers already hold the head at one CTA # per SM) HEAD_SMEM_PAD = 120 * 1024 +# SMs a hidden rank leaves to the side-stream hidden partial: the early-released head +# would fill every SM, and the partial (which the exchange waits for) would start only as +# head CTAs retire; the head then runs on the other SMs +PARTIAL_SMS = 24 # Draft and draft-extend widths that take the row-split fc (2.2 _FC_SPLIT_MAX_TOKENS) FC_SPLIT_MAX_TOKENS = kernels.GATHER_MAX_TOKENS # Rows of a draft step the vocabulary exchange serves (2.2 DraftVocabExchange.MAX_ROWS) @@ -138,6 +142,7 @@ def __init__( extend_fc: bool = True, head: bool = True, head_smem_pad: int = HEAD_SMEM_PAD, + partial_sms: int = PARTIAL_SMS, ) -> None: import torch.distributed as dist @@ -214,6 +219,7 @@ def __init__( self.side_stream = torch.cuda.Stream(device=dev) self.lm_head_weight = lm_head_shard self.head_pad = head_smem_pad + self.partial_sms = partial_sms self._install_head(head, dev) # extend_fc: the draft extend's exchange writes fc_buf for the next draft's first # step. input_seq counts calls that produce a draft input; fc_buf_key is the @@ -335,6 +341,7 @@ def _check_memo(self, emb: torch.Tensor, memo: torch.Tensor) -> int: def _install_head(self, enabled: bool, dev: torch.device) -> None: self.head_part = None + self.head_part_cur = None self.head_used = False if not enabled: return @@ -356,13 +363,22 @@ def _install_head(self, enabled: bool, dev: torch.device) -> None: _HEAD_CTAS_PER_SM * sms, dtype=torch.int32, device=dev ) logger.info( - "MKD draft head active: %d x %d shard, %d CTAs, smem pad %d B.", + "MKD draft head active: %d x %d shard, %d CTAs, smem pad %d B, hidden ranks " + "leave %d SMs to the fc partial.", head.shape[0], head.shape[1], self.head_part.numel(), self.head_pad, + self.partial_sms, ) + def _partial_ctas(self) -> int: + """SMs left to the side-stream hidden partial on a hidden rank's step that sends + one; 0 keeps one partial CTA per SM, queued behind the head.""" + if not self.is_hidden_rank or self.last_step or self.head_part is None: + return 0 + return max(0, min(self.partial_sms, self.head_part.numel() - 1)) + def head(self, hidden_states: torch.Tensor) -> torch.Tensor: """The one-row draft logits [1, N] bf16 on this rank's shard (the exchange then reads the per-CTA maxima). For an armed step with the head installed.""" @@ -370,13 +386,17 @@ def head(self, hidden_states: torch.Tensor) -> torch.Tensor: logits = torch.empty( 1, self.lm_head_weight.shape[0], dtype=torch.bfloat16, device=x.device ) + # The head's grid is its part count: with SMs left to the hidden partial it runs on + # the rest + ctas = self.head_part.numel() - self._partial_ctas() + self.head_part_cur = self.head_part[:ctas] # Alternate the row walk per draft step: the next step re-reads first what is still in L2 # (T1: -1.9 to -2.4 us per call on the TP8 shard). kernels.mkd_draft_head( x, self.lm_head_weight, logits, - self.head_part, + self.head_part_cur, reverse=self.step % 2 == 1, smem_pad=self.head_pad, ) @@ -419,6 +439,7 @@ def hidden_partial(self, hidden_states: torch.Tensor) -> None: self.epoch_done, self.partial_sync, self.world, + ctas=self._partial_ctas(), ) self.partial_pending = True @@ -470,7 +491,7 @@ def exchange( token_only=self.last_step, partial_flag=partial_flag, scan_part=self.scan_part, - head_part=self.head_part if self.head_used else None, + head_part=self.head_part_cur if self.head_used else None, ) self.head_used = False self.pending = not self.last_step diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_fc.cu b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_fc.cu index b7856bea..f3a4c8cd 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_fc.cu +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/mkd_fc.cu @@ -316,8 +316,8 @@ extern "C" __global__ void __launch_bounds__(256) mkd_fc_memo(const MkdFcMemoPar sglang::mkd_fc_memo_kernel<4096, MKD_WORLD, 8>(p); } -// One CTA per SM (the launch is SM-count CTAs). With 2.2's __launch_bounds__(256) ptxas holds NVRTC's -// build to 32 registers and splits each row's four weight loads into two batches (+24%). +// One CTA per SM (SM-count CTAs, or the SMs the draft head leaves it). With 2.2's __launch_bounds__(256) +// ptxas holds NVRTC's build to 32 registers and splits each row's four weight loads into two batches (+24%). extern "C" __global__ void __launch_bounds__(256, 1) mkd_hidden_partial(const MkdHiddenPartialParams __grid_constant__ p) { sglang::mkd_hidden_partial_kernel<4096, MKD_WORLD>(p); } diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py index 45998214..144c5108 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/xchg.py @@ -16,8 +16,9 @@ 16 tokens (fused norm + slice GEMV, all-gather of the fp32 partials, rank-order sum). Every launch keeps the 2.2 grid, block, shared memory and PDL setting (the draft head may -add unused shared memory, ``smem_pad``). ``FcGather`` moves bytes only; the arithmetic of -the split fc is ``draft_fc_partial`` and ``ordered_partial_sum``, copied unchanged. +add unused shared memory, ``smem_pad``, and run beside a narrower hidden partial, +``ctas``). ``FcGather`` moves bytes only; the arithmetic of the split fc is +``draft_fc_partial`` and ``ordered_partial_sum``, copied unchanged. """ from __future__ import annotations @@ -518,11 +519,13 @@ def mkd_hidden_partial( epoch_done: torch.Tensor, sync: torch.Tensor, world: int, + ctas: int = 0, ) -> None: """Hidden-slice fc partial [1, H] fp32 of ``h`` [1, H] (draft_fc_partial arithmetic over ``shard`` [H, W] at hidden-half column ``col``); the last CTA then sets ``sync[1]`` to the exchange epoch ``epoch_done[0]`` (``sync[0]`` counts CTAs, zero - between calls).""" + between calls). ``ctas`` > 0 launches that many CTAs instead of one per SM (same + values: each row is one warp's).""" device = out.device width = 2 * HIDDEN // world _check(h, "h", (1, HIDDEN), torch.bfloat16, aligned=True) @@ -545,9 +548,8 @@ def mkd_hidden_partial( col=col, ) kernels = _fc_kernels(device.index, world) - _launch( - kernels["mkd_hidden_partial"], _sm_count(device.index), p, device, pdl=False - ) + grid = ctas if ctas > 0 else _sm_count(device.index) + _launch(kernels["mkd_hidden_partial"], grid, p, device, pdl=False) # ---------------------------------------------------------------- draft lm_head From 0d052c85a0dd6129346737898e2873990d66088f Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:58:00 -0700 Subject: [PATCH 21/27] [Perf] [GB300] qwen3_5: The draft step's front launches as a programmatic dependent of the qkv GEMM Slot draft_front: with one row, the front loads its static operands (the norm weights, the M-RoPE axis map, the cache scales) before its dependency wait and issues every other load right after it. The arithmetic is unchanged (bitwise). --- sgl_nvidia_mega/qwen3_5/draft/attention.py | 16 ++ sgl_nvidia_mega/qwen3_5/sm103/draft/front.py | 179 +++++++++++++++++++ 2 files changed, 195 insertions(+) diff --git a/sgl_nvidia_mega/qwen3_5/draft/attention.py b/sgl_nvidia_mega/qwen3_5/draft/attention.py index d336bc30..94c8515e 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/attention.py +++ b/sgl_nvidia_mega/qwen3_5/draft/attention.py @@ -32,6 +32,9 @@ # A draft step's q1 decode and its merge launch as programmatic dependents, the # one-row front releasing them at entry (same arithmetic) Q1_PDL = True +# The one-row front launches as a programmatic dependent of the qkv GEMM, loading its +# static operands before the wait (same arithmetic) +QKN_PDL = True class Q1Variants: @@ -171,6 +174,7 @@ def __init__(self, parts): self.k_scale = attn.k_scale if attn.k_scale is not None else ones self.v_scale = attn.v_scale if attn.v_scale is not None else ones self.q1 = None # sm103.draft.q1 decode (slot draft_q1), set by DraftLoop + self._noted: set = set() # engagement lines logged (once each, under capture) # slot draft_front: 2.2's attention front writes the KV and FP8 q in one launch self.front = "draft_front" in parts.slots # slot draft_skv2: the draft extend's split-KV decode (DraftExtendSplitKV) @@ -236,10 +240,22 @@ def _front(self, x, positions, cache_loc, kv): k_scale=self.k_scale, v_scale=self.v_scale, early_trigger=x.shape[0] == 1 and self.q1 is not None and Q1_PDL, + pdl_wait=x.shape[0] == 1 and QKN_PDL, ) + if x.shape[0] == 1 and QKN_PDL and layer.attn_output_gate: + self._note( + "slot draft_front launches the one-row front as a programmatic " + "dependent of the qkv GEMM" + ) rows = x.shape[0] return q8.view(rows, -1), (gate.view(rows, -1) if gate is not None else None) + def _note(self, what: str) -> None: + """Log ``what`` once, the first time a graph capture runs it.""" + if what not in self._noted and torch.cuda.is_current_stream_capturing(): + self._noted.add(what) + logger.info("NVIDIA megakernel: Qwen3.5 MTP draft %s", what) + def core(self, q, k, v, cache_loc, meta, kv, rows_per_request: int): """FP8 q / k / v with the KV write, then the decode: [rows, heads * head_dim].""" from sglang.kernels.ops.kvcache.fused_fp8_qkv_kv_cache import ( diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/front.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/front.py index 97b3637f..b2cc6551 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/front.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/front.py @@ -192,6 +192,142 @@ def _fused_qk_rmsnorm_rope_gate_kernel( tl.extra.cuda.gdc_launch_dependents() +@triton.jit +def _fused_qk_rmsnorm_rope_gate_pdl_kernel( + q_gate_ptr, + k_ptr, + q_out_ptr, + k_out_ptr, + gate_out_ptr, + q_weight_ptr, + k_weight_ptr, + cos_sin_cache_ptr, + positions_ptr, + mrope_axis_map_ptr, + v_ptr, + v_cache_ptr, + cache_loc_ptr, + k_scale_ptr, + v_scale_ptr, + stride_qg_t, + stride_k_t, + stride_qo_t, + stride_ko_t, + stride_gate_t, + stride_cos_t, + stride_pos_axis, + stride_v_t: tl.constexpr, + stride_v_cache_t: tl.constexpr, + NUM_Q_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + ROTARY_DIM: tl.constexpr, + HALF_ROTARY: tl.constexpr, + HEAD_BLOCK: tl.constexpr, + ROT_HALF_BLOCK: tl.constexpr, + EPS: tl.constexpr, + FP16: tl.constexpr, + HAS_PASS: tl.constexpr, + MROPE: tl.constexpr, + EARLY_TRIGGER: tl.constexpr, + WEIGHT_SHIFT: tl.constexpr = 1.0, +): + # The kernel above with an FP8 cache and a gate, launched as a programmatic dependent of the qkv GEMM: + # static operands load before griddepcontrol.wait, all other loads are issued right after it; same arithmetic. + if EARLY_TRIGGER: + tl.extra.cuda.gdc_launch_dependents() + token = tl.program_id(0) + head = tl.program_id(1) + is_k = head >= NUM_Q_HEADS + local_head = tl.where(is_k, head - NUM_Q_HEADS, head) + out_dtype = tl.float16 if FP16 else tl.bfloat16 + head_offs = tl.arange(0, HEAD_BLOCK) + head_mask = head_offs < HEAD_DIM + rot_offs = tl.arange(0, ROT_HALF_BLOCK) + rot_mask = rot_offs < HALF_ROTARY + + if is_k: + w_ptr = k_weight_ptr + else: + w_ptr = q_weight_ptr + w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32) + wr1 = tl.load(w_ptr + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) + wr2 = tl.load(w_ptr + HALF_ROTARY + rot_offs, mask=rot_mask, other=0.0).to( + tl.float32 + ) + # Match the cache writer's round-to-nearest reciprocal. + inv_k = tl.div_rn(1.0, tl.load(k_scale_ptr).to(tl.float32)) + inv_v = tl.div_rn(1.0, tl.load(v_scale_ptr).to(tl.float32)) + if MROPE: + axis = tl.load(mrope_axis_map_ptr + rot_offs, mask=rot_mask, other=0) + + tl.extra.cuda.gdc_wait() + slot = tl.load(cache_loc_ptr + token).to(tl.int64) + if MROPE: + pos = tl.load( + positions_ptr + axis * stride_pos_axis + token, mask=rot_mask, other=0 + ) + else: + pos = tl.load(positions_ptr + token) + if is_k: + in_base = k_ptr + token * stride_k_t + local_head * HEAD_DIM + out_base = k_out_ptr + slot * stride_ko_t + local_head * HEAD_DIM + else: + in_base = q_gate_ptr + token * stride_qg_t + local_head * 2 * HEAD_DIM + out_base = q_out_ptr + token * stride_qo_t + local_head * HEAD_DIM + x = tl.load(in_base + head_offs, mask=head_mask, other=0.0).to(tl.float32) + xr1 = tl.load(in_base + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) + xr2 = tl.load(in_base + HALF_ROTARY + rot_offs, mask=rot_mask, other=0.0).to( + tl.float32 + ) + value = tl.load( + v_ptr + token * stride_v_t + local_head * HEAD_DIM + head_offs, + head_mask & is_k, + 0.0, + ).to(tl.float32) + g = tl.load( + in_base + HEAD_DIM + head_offs, mask=head_mask & (head < NUM_Q_HEADS), other=0.0 + ) + cache_off = pos.to(tl.int64) * stride_cos_t + cos = tl.load( + cos_sin_cache_ptr + cache_off + rot_offs, mask=rot_mask, other=0.0 + ).to(tl.float32) + sin = tl.load( + cos_sin_cache_ptr + cache_off + HALF_ROTARY + rot_offs, mask=rot_mask, other=0.0 + ).to(tl.float32) + + var = tl.sum(x * x, axis=0) / HEAD_DIM + inv_rms = tl.rsqrt(var + EPS) + x_norm = (x * inv_rms * (w + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32) + + if HAS_PASS: + pass_mask = head_mask & (head_offs >= ROTARY_DIM) + tail = x_norm + if is_k: + tail *= inv_k + tl.store(out_base + head_offs, tail, mask=pass_mask) + + xr1 = (xr1 * inv_rms * (wr1 + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32) + xr2 = (xr2 * inv_rms * (wr2 + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32) + first = xr1 * cos - xr2 * sin + second = xr2 * cos + xr1 * sin + first = first.to(out_dtype).to(tl.float32) + second = second.to(out_dtype).to(tl.float32) + if is_k: + first *= inv_k + second *= inv_k + tl.store(out_base + rot_offs, first, mask=rot_mask) + tl.store(out_base + HALF_ROTARY + rot_offs, second, mask=rot_mask) + + if is_k: + value_out = v_cache_ptr + slot * stride_v_cache_t + local_head * HEAD_DIM + tl.store(value_out + head_offs, value * inv_v, head_mask) + else: + gate_out = gate_out_ptr + token * stride_gate_t + local_head * HEAD_DIM + tl.store(gate_out + head_offs, g, mask=head_mask) + if not EARLY_TRIGGER: + tl.extra.cuda.gdc_launch_dependents() + + def fused_qk_gemma_rmsnorm_rope_gate( q_gate: torch.Tensor, k: torch.Tensor, @@ -213,10 +349,13 @@ def fused_qk_gemma_rmsnorm_rope_gate( k_scale: Optional[torch.Tensor] = None, v_scale: Optional[torch.Tensor] = None, early_trigger: bool = False, + pdl_wait: bool = False, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: """Fused QK GemmaRMSNorm + NeoX RoPE + gate deinterleave. early_trigger: release programmatic dependents at entry (they must wait before reading the outputs). + pdl_wait: with an FP8 cache and a gate, launch as a programmatic dependent of the producer of q_gate / k / v + (static operands load before the wait); same outputs. Args: q_gate: [T, num_q_heads * (1 + has_gate) * head_dim] — interleaved Q+Gate if has_gate @@ -279,6 +418,46 @@ def fused_qk_gemma_rmsnorm_rope_gate( stride_v_cache_t=kv_cache[1].stride(0), FP8_CACHE=True, ) + if pdl_wait and fp8_cache and has_gate and _ENABLE_PDL: + _fused_qk_rmsnorm_rope_gate_pdl_kernel[grid]( + q_gate, + k, + q_out, + k_out, + gate_out, + q_weight, + k_weight, + cos_sin_cache, + positions, + mrope_axis_map, + v, + kv_cache[1], + cache_loc, + k_scale, + v_scale, + q_gate.stride(0), + k.stride(0), + q_out.stride(0), + k_out.stride(0), + gate_out.stride(0), + cos_sin_cache.stride(0), + positions.stride(0), + v.stride(0), + kv_cache[1].stride(0), + NUM_Q_HEADS=num_q_heads, + HEAD_DIM=head_dim, + ROTARY_DIM=rotary_dim, + HALF_ROTARY=half_rotary, + HEAD_BLOCK=head_block, + ROT_HALF_BLOCK=rot_half_block, + EPS=eps, + FP16=q_gate.dtype == torch.float16, + HAS_PASS=rotary_dim < head_dim, + MROPE=mrope, + EARLY_TRIGGER=early_trigger, + launch_pdl=True, + ) + return q_out, None, gate_out _fused_qk_rmsnorm_rope_gate_kernel[grid]( q_gate, k, From 6c2a69fef34deeb959521daa830ff5ed5a0b071e Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 09:59:00 -0700 Subject: [PATCH 22/27] [Perf] [GB300] qwen3_5: The draft o_proj GEMV applies the attention output gate as it stages its input Slot draft_hop: the one-row o_proj GEMV takes x * sigmoid(gate), computed while it stages x in the instruction sequence of SGLang's gate kernel, so that kernel no longer runs, and the q1 merge releases the GEMV at entry. The arithmetic is unchanged (bitwise). --- sgl_nvidia_mega/qwen3_5/draft/attention.py | 19 +++++++- sgl_nvidia_mega/qwen3_5/draft/hop.py | 17 +++++-- sgl_nvidia_mega/qwen3_5/sm103/draft/hop.py | 35 ++++++++++---- sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py | 23 ++++++--- .../qwen3_5/sm103/draft/q1_kernel.py | 17 ++++++- .../qwen3_5/sm103/draft/src/draft_gemv.cu | 48 ++++++++++++++++--- test/kernels/test_qwen3_5_draft_hop.py | 29 +++++++++++ 7 files changed, 160 insertions(+), 28 deletions(-) diff --git a/sgl_nvidia_mega/qwen3_5/draft/attention.py b/sgl_nvidia_mega/qwen3_5/draft/attention.py index 94c8515e..0c5e50fa 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/attention.py +++ b/sgl_nvidia_mega/qwen3_5/draft/attention.py @@ -35,6 +35,10 @@ # The one-row front launches as a programmatic dependent of the qkv GEMM, loading its # static operands before the wait (same arithmetic) QKN_PDL = True +# 1: slot draft_hop's one-row o_proj stages attn * sigmoid(gate) itself, in the gate +# kernel's arithmetic, in place of that kernel; 2: also, under Q1_PDL, the q1 merge +# releases that o_proj at entry +GATE_FOLD = 2 class Q1Variants: @@ -200,6 +204,16 @@ def __call__( ) o = self.core(q, k, v, cache_loc, meta, kv, rows_per_request) if layer.attn_output_gate: + if ( + o_proj is not None + and GATE_FOLD + and gate.shape == o.shape + and gate.is_contiguous() + ): + self._note( + "slot draft_hop's one-row o_proj applies the attention output gate" + ) + return o_proj(o, gate=gate) o = fused_sigmoid_mul(o, gate, inplace=True) if o_proj is not None: return o_proj(o) @@ -392,8 +406,9 @@ def build_q1(parts): ) if why is not None: raise RuntimeError(f"NVIDIA megakernel: slot draft_q1 cannot run: {why}") - q1.build(device.index, heads=attention.heads, pdl=Q1_PDL) - decode = q1.Q1Decode(pdl=Q1_PDL) + release = GATE_FOLD == 2 + q1.build(device.index, heads=attention.heads, pdl=Q1_PDL, release=release) + decode = q1.Q1Decode(pdl=Q1_PDL, release=release) if Q1_PDL: logger.info( "NVIDIA megakernel: Qwen3.5 MTP draft slot draft_q1 launches the q1 decode " diff --git a/sgl_nvidia_mega/qwen3_5/draft/hop.py b/sgl_nvidia_mega/qwen3_5/draft/hop.py index 7daba97c..9db1f519 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/hop.py +++ b/sgl_nvidia_mega/qwen3_5/draft/hop.py @@ -13,7 +13,8 @@ rounding of SGLang's one-shot push all-reduce and FlashInfer's CuTe DSL FusedAddRMSNorm (multiplier 1 + the raw weight, in fp32); - ``o_proj``: the attention output's one-row o_proj (fp32 accumulation), pushed into - ``ffn_input``'s hop, which then only consumes. + ``ffn_input``'s hop, which then only consumes; with the output gate it stages + ``attn * sigmoid(gate)`` itself, in place of SGLang's gate kernel. The draft shares the target's hop (``boundary.hop.active()``): one mailbox and one stage order for every hop of the process, as SGLang runs the draft after the target's @@ -109,15 +110,21 @@ def _take_pushed(self, partial: torch.Tensor) -> bool: raise RuntimeError("draft hop: a pushed partial was never consumed") return False - def o_proj(self, attn_output: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: + def o_proj( + self, + attn_output: torch.Tensor, + weight: torch.Tensor, + gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: """The one-row o_proj partial [1, N] bf16 (``weight`` [N, K] bf16), pushed into - the FFN input's hop when that hop takes the row.""" + the FFN input's hop when that hop takes the row; with ``gate`` [1, K] its input + is ``attn_output`` * sigmoid(``gate``).""" out = torch.empty( 1, weight.shape[0], dtype=torch.bfloat16, device=weight.device ) if self.attn_push_target(int(attn_output.shape[0])) is None: - return kernels.draft_gemv(attn_output, weight, out) - kernels.draft_gemv(attn_output, weight, out, hop_site=self.hop.site) + return kernels.draft_gemv(attn_output, weight, out, gate=gate) + kernels.draft_gemv(attn_output, weight, out, hop_site=self.hop.site, gate=gate) self.mark_pushed(out) return out diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/hop.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/hop.py index 8d5c4723..6c0fbbd9 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/hop.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/hop.py @@ -13,7 +13,8 @@ then FlashInfer's CuTe DSL FusedAddRMSNorm (multiplier ``weight_bias + gamma``). ``draft_gemv`` (``src/draft_gemv.cu``) is the one-row bf16 GEMV with fp32 accumulation; with ``hop_site`` it also pushes its output into the hop, whose all-reduce then only -consumes. +consumes; with ``gate`` it takes ``x * sigmoid(gate)`` as its input, in the arithmetic of +SGLang's attention output gate kernel. Kernels compile in ``build`` (never under CUDA-graph capture); the consume's wait deadline is compiled in, as the decode hop's. @@ -51,6 +52,7 @@ class GemvParams(ctypes.Structure): _fields_ = [ ("x", ctypes.c_void_p), + ("gate", ctypes.c_void_p), ("w", ctypes.c_void_p), ("y", ctypes.c_void_p), ("n", ctypes.c_int), @@ -83,8 +85,11 @@ def _consume_kernels(device_index: int, deadline_ns: int) -> dict[str, jit.Kerne @cache -def _gemv_kernels(device_index: int, k: int) -> tuple[jit.Kernel, jit.Kernel, int]: - """(plain, pushing, the plain kernel's grid cap: resident CTAs on the device).""" +def _gemv_kernels( + device_index: int, k: int, gate: bool = False +) -> tuple[jit.Kernel, jit.Kernel, int]: + """(plain, pushing, the plain kernel's grid cap: resident CTAs on the device), the + gated twins with ``gate``.""" if k % 256: raise ValueError(f"draft GEMV: K {k} is not a multiple of 256") major, minor = torch.cuda.get_device_capability(device_index) @@ -94,8 +99,11 @@ def _gemv_kernels(device_index: int, k: int) -> tuple[jit.Kernel, jit.Kernel, in f"sm_{major}{minor}a", ("-std=c++17", "-default-device", f"-DQ35_DRAFT_GEMV_K={k}"), ) - plain = jit.load(cubin, "q35_draft_gemv", (GEMV_ROWS * 32, 1, 1), 0) - push = jit.load(cubin, "q35_draft_gemv_push", (GEMV_PUSH_ROWS * 32, 1, 1), 0) + suffix = "_gate" if gate else "" + plain = jit.load(cubin, f"q35_draft_gemv{suffix}", (GEMV_ROWS * 32, 1, 1), 0) + push = jit.load( + cubin, f"q35_draft_gemv_push{suffix}", (GEMV_PUSH_ROWS * 32, 1, 1), 0 + ) cu = jit.driver() (per_sm,) = jit.check( cu.cuOccupancyMaxActiveBlocksPerMultiprocessor( @@ -116,7 +124,8 @@ def build(device_index: int, deadline_ns: int, gemv_ks: tuple[int, ...] = ()) -> for each reduction length in ``gemv_ks``.""" _CONSUME[device_index] = _consume_kernels(device_index, int(deadline_ns)) for k in gemv_ks: - _gemv_kernels(device_index, int(k)) + for gate in (False, True): + _gemv_kernels(device_index, int(k), gate) def draft_all_reduce_rms_norm( @@ -202,11 +211,13 @@ def draft_gemv( out: torch.Tensor, hop_site: Optional[decode_hop.Site] = None, hop_token: int = 0, + gate: Optional[torch.Tensor] = None, ) -> torch.Tensor: """``out`` [1, N] bf16 = ``x`` [1, K] @ ``weight`` [N, K]^T with fp32 accumulation, one warp per row. ``hop_site``: the decode hop's ``Site``; the output (N = 4096) is then also pushed into that hop as row ``hop_token``, and its all-reduce only - consumes.""" + consumes. ``gate`` [1, K] bf16: the input is then ``x`` * sigmoid(``gate``), + rounded to bf16.""" k, n = int(x.shape[-1]), int(weight.shape[0]) if ( x.dtype != torch.bfloat16 @@ -223,9 +234,17 @@ def draft_gemv( f"draft GEMV: x {x.dtype} {tuple(x.shape)}, weight {weight.dtype} " f"{tuple(weight.shape)}, out {out.dtype} {tuple(out.shape)}" ) - plain, push, resident = _gemv_kernels(x.device.index, k) + if gate is not None and ( + gate.dtype != torch.bfloat16 + or tuple(gate.shape) != (1, k) + or not gate.is_contiguous() + or gate.data_ptr() % 16 + ): + raise ValueError(f"draft GEMV: gate {gate.dtype} {tuple(gate.shape)}") + plain, push, resident = _gemv_kernels(x.device.index, k, gate is not None) params = GemvParams( x=x.data_ptr(), + gate=gate.data_ptr() if gate is not None else x.data_ptr(), w=weight.data_ptr(), y=out.data_ptr(), n=n, diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py index 26eba4ce..6d38e9ca 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/q1.py @@ -230,11 +230,12 @@ class Q1Decode: first call of a layout must not be under graph capture (graph warmup runs it eagerly, as in the branch). ``pdl``: the producer and the merge launch as programmatic dependents, the producer loading the leading K / V tiles this step - does not write before its wait (same arithmetic).""" + does not write before its wait; ``release``: with ``pdl``, the merge releases its + own dependent at entry (same arithmetic).""" - def __init__(self, pdl: bool = True): + def __init__(self, pdl: bool = True, release: bool = True): self._plans = {} - self.pdl = pdl + self.pdl, self.release = pdl, release @staticmethod def key(query, kv_cache, block_tables, seq_lens) -> tuple: @@ -253,7 +254,14 @@ def plan(self, query, kv_cache, block_tables, seq_lens): key = self.key(query, kv_cache, block_tables, seq_lens) plan = self._plans.get(key) if plan is None: - plan = Q1SplitPlan(query, kv_cache, block_tables, seq_lens, pdl=self.pdl) + plan = Q1SplitPlan( + query, + kv_cache, + block_tables, + seq_lens, + pdl=self.pdl, + release=self.release, + ) self._plans[key] = plan return plan @@ -337,9 +345,11 @@ def prepare(self, heads: int, kv, layer_id: int, step) -> None: @cache -def build(device_index: int, heads: int = 4, pdl: bool = True) -> None: +def build( + device_index: int, heads: int = 4, pdl: bool = True, release: bool = True +) -> None: """Import CuTe DSL and Triton and compile the fp32 merge for ``heads`` rows and - this GPU's SM count (as a programmatic dependent with ``pdl``); each layout's + this GPU's SM count (as ``Q1Decode(pdl, release)`` runs it); each layout's producer compiles at its plan's first use.""" import triton @@ -366,6 +376,7 @@ def build(device_index: int, heads: int = 4, pdl: bool = True) -> None: triton.next_power_of_2(nsplit), MERGE_BLOCK_D, USE_GDC=pdl, + RELEASE=pdl and release, num_warps=MERGE_WARPS, **({"launch_pdl": True} if pdl else {}), ) diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/q1_kernel.py b/sgl_nvidia_mega/qwen3_5/sm103/draft/q1_kernel.py index d8c871d1..47de7417 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/q1_kernel.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/q1_kernel.py @@ -618,7 +618,10 @@ def _q1_merge( NP2: tl.constexpr, BLOCK_D: tl.constexpr, USE_GDC: tl.constexpr = False, + RELEASE: tl.constexpr = False, ): + if RELEASE: # the consumer's own dependency wait still covers this grid's stores + tl.extra.cuda.gdc_launch_dependents() if USE_GDC: # launched as a programmatic dependent of the q1 producer tl.extra.cuda.gdc_wait() row = tl.program_id(0) @@ -647,7 +650,17 @@ class Q1SplitPlan: """Reusable storage and a compiled producer for one fixed input layout, plus the fp32 merge.""" def __init__( - self, query, kv, table, lengths, *, nsplit=None, stages=2, slots=2, pdl=False + self, + query, + kv, + table, + lengths, + *, + nsplit=None, + stages=2, + slots=2, + pdl=False, + release=False, ): if torch.cuda.is_current_stream_capturing(): raise RuntimeError("SKV2-Q1 storage must be prepared before graph capture") @@ -665,6 +678,7 @@ def __init__( ) self.denom = torch.empty_like(self.maximum) self.pdl = pdl + self.release = pdl and release kernel = Q1SplitDecode( self.heads, self.nsplit, stages=stages, slots=slots, pdl=pdl ) @@ -708,6 +722,7 @@ def run(self, query, kv, table, lengths, *, qk_scale, v_scale, out): triton.next_power_of_2(self.nsplit), 32, USE_GDC=self.pdl, + RELEASE=self.release, num_warps=4, **({"launch_pdl": True} if self.pdl else {}), ) diff --git a/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_gemv.cu b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_gemv.cu index 5712ba33..0ab64549 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_gemv.cu +++ b/sgl_nvidia_mega/qwen3_5/sm103/draft/src/draft_gemv.cu @@ -5,12 +5,14 @@ // One-row draft GEMV y[n] = bf16(sum_k W[n, k] * x[k]): bf16 operands, fp32 accumulation, output rounded // to bf16 as the served GEMM writes it (only the fp32 summation order differs). One warp per row with the // whole row in flight, the row loaded before the PDL wait (weights are static). With a hop site, every CTA -// owns one 8-row fragment of the output and pushes it into the decode hop (sm103/hop) before storing it. +// owns one 8-row fragment of the output and pushes it into the decode hop (sm103/hop) before storing it. With a +// gate, the input is x * sigmoid(gate) rounded to bf16, as SGLang's attention output gate kernel writes it. // The q35hop namespace below is hop.cu's, copied verbatim (the push must follow its mailbox protocol; // test/kernels/test_qwen3_5_draft_hop.py compares the text). // // Built with NVRTC per reduction length Q35_DRAFT_GEMV_K: C-linkage entries q35_draft_gemv (4 rows a -// CTA) and q35_draft_gemv_push (8 rows a CTA, pushing). +// CTA) and q35_draft_gemv_push (8 rows a CTA, pushing), and their gated twins q35_draft_gemv_gate and +// q35_draft_gemv_push_gate. #include @@ -360,6 +362,7 @@ namespace q35draft_gemv { struct Params { const __nv_bfloat16* x; // [K], written by the previous grid + const __nv_bfloat16* gate; // kGate: [K], applied to x as it is staged const __nv_bfloat16* __restrict__ w; // [N, K] __nv_bfloat16* y; // [N] int32_t n; @@ -374,13 +377,36 @@ __device__ __forceinline__ float warp_reduce_sum(float value) { return value; } +// Eight bf16(x * sigmoid(g)) in the instruction sequence Triton emits for _fused_sigmoid_mul_kernel (fp32, +// exp(-g) as ex2.approx of -g * log2(e), 1 / (1 + e) by div.full, product rounded to nearest-even bf16). +__device__ __forceinline__ uint4 gate_mul8(const uint4 xv, const uint4 gv) { + const __nv_bfloat16* x = reinterpret_cast(&xv); + const __nv_bfloat16* g = reinterpret_cast(&gv); + uint4 out; + __nv_bfloat16* o = reinterpret_cast<__nv_bfloat16*>(&out); +#pragma unroll + for (int e = 0; e < 8; ++e) { + float t; + asm("mul.rn.f32 %0, %1, 0fBFB8AA3B;\n\t" + "ex2.approx.f32 %0, %0;\n\t" + "add.rn.f32 %0, %0, 0f3F800000;\n\t" + "div.full.f32 %0, 0f3F800000, %0;\n\t" + "mul.rn.f32 %0, %2, %0;" + : "=&f"(t) + : "f"(__bfloat162float(g[e])), "f"(__bfloat162float(x[e]))); + o[e] = __float2bfloat16_rn(t); + } + return out; +} + /** * \brief kRows warps per CTA; warp w of CTA c takes rows kRows * c + w, then a grid stride further on, the * next row loaded while the current one is reduced (as the draft head does). * \tparam kK reduction length (a multiple of 256) * \tparam kRows warps per CTA; 8 when pushing, with one row per warp so a CTA owns one hop fragment + * \tparam kGate stage x * sigmoid(gate) instead of x */ -template +template __device__ __forceinline__ void mkd_gemv_kernel(const Params& p) { static_assert(kK % 256 == 0, "K must tile 32 lanes x 8 bf16"); static_assert(!kPush || kRows == 8, "a pushing CTA owns exactly one 8-row fragment"); @@ -400,7 +426,9 @@ __device__ __forceinline__ void mkd_gemv_kernel(const Params& p) { } if (kUsePDL) q35hop::pdl_wait(); for (int i = static_cast(threadIdx.x); i < kK / 8; i += kRows * 32) { - reinterpret_cast(s_x)[i] = reinterpret_cast(p.x)[i]; + uint4 xv = reinterpret_cast(p.x)[i]; + if constexpr (kGate) xv = gate_mul8(xv, reinterpret_cast(p.gate)[i]); + reinterpret_cast(s_x)[i] = xv; } if constexpr (kPush) { if (threadIdx.x == 0) s_stage = q35hop::stage_peek(p.hop); @@ -442,9 +470,17 @@ __device__ __forceinline__ void mkd_gemv_kernel(const Params& p) { using q35draft_gemv::Params; extern "C" __global__ void __launch_bounds__(4 * 32) q35_draft_gemv(const Params __grid_constant__ p) { - q35draft_gemv::mkd_gemv_kernel(p); + q35draft_gemv::mkd_gemv_kernel(p); } extern "C" __global__ void __launch_bounds__(8 * 32) q35_draft_gemv_push(const Params __grid_constant__ p) { - q35draft_gemv::mkd_gemv_kernel(p); + q35draft_gemv::mkd_gemv_kernel(p); +} + +extern "C" __global__ void __launch_bounds__(4 * 32) q35_draft_gemv_gate(const Params __grid_constant__ p) { + q35draft_gemv::mkd_gemv_kernel(p); +} + +extern "C" __global__ void __launch_bounds__(8 * 32) q35_draft_gemv_push_gate(const Params __grid_constant__ p) { + q35draft_gemv::mkd_gemv_kernel(p); } diff --git a/test/kernels/test_qwen3_5_draft_hop.py b/test/kernels/test_qwen3_5_draft_hop.py index 8ef13af8..062b9c64 100644 --- a/test/kernels/test_qwen3_5_draft_hop.py +++ b/test/kernels/test_qwen3_5_draft_hop.py @@ -12,6 +12,8 @@ decode hop's own, so the three mailbox stages cycle; - the draft o_proj GEMV equals an fp32 emulation of its summation order, bit for bit, and its pushed output followed by a consume-only all-reduce equals publish + consume; +- with the attention output gate, the GEMV (plain and pushing) equals SGLang's gate + kernel followed by the plain GEMV, bit for bit; - a draft step's hops replayed from a CUDA graph with fresh inputs equal the eager run. Run with ``python test/kernels/test_qwen3_5_draft_hop.py --num-gpu 4``. @@ -244,6 +246,33 @@ def test_draft_gemv_and_push(rank, draft): torch.cuda.synchronize() +@gpu +def test_draft_gemv_folds_the_output_gate(rank, draft): + from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul + + from sgl_nvidia_mega.qwen3_5.sm103.draft import hop as kernels + + device = torch.device(f"cuda:{torch.cuda.current_device()}") + with torch.inference_mode(): + for seed in range(4): + x = _operands(1, device, rank, seed=500 + seed) + norm = _Norm(x["weight"]) + g = torch.Generator(device=device).manual_seed(600 + seed) + gate = ( + 8.0 * torch.randn(1, O_PROJ_K, device=device, generator=g) + ).bfloat16() + gate[0, :4] = torch.tensor([-0.0, 0.0, 100.0, -100.0]) + want = torch.empty(1, HIDDEN, dtype=torch.bfloat16, device=device) + kernels.draft_gemv(fused_sigmoid_mul(x["attn"], gate), x["o_proj"], want) + got = torch.empty_like(want) + kernels.draft_gemv(x["attn"], x["o_proj"], got, gate=gate) + _same((got,), (want,)) + pushed = draft.o_proj(x["attn"], x["o_proj"], gate=gate) + draft.ffn_input(pushed, x["residual"], norm, moe_single_launch=False) + _same((pushed,), (want,)) + torch.cuda.synchronize() + + @gpu def test_draft_step_replays_from_a_cuda_graph(rank, draft): device = torch.device(f"cuda:{torch.cuda.current_device()}") From a9c8b8d49fef516a4a107e0b97713341e4be7e38 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 10:00:00 -0700 Subject: [PATCH 23/27] [Feat] [GB300] qwen3_5: Serve TP4 on one tray, with slot mkm's engine for intermediate 256 per rank The spec takes TP sizes 4 and 8. Slot mkm picks its engine from the intermediate size a rank holds, adding the TP4 engine of SGLang branch nvidia/qwen35-agentx-gb300 (bitwise against trtllm-gen), and slot draft_hop keeps SGLang's o_proj at TP4. Every other slot builds as at TP8, except gdn_front, which refuses at TP4. --- sgl_nvidia_mega/qwen3_5/draft/hop.py | 20 +- sgl_nvidia_mega/qwen3_5/draft/layer.py | 6 +- sgl_nvidia_mega/qwen3_5/draft/parts.py | 11 +- sgl_nvidia_mega/qwen3_5/moe/mkm.py | 8 +- sgl_nvidia_mega/qwen3_5/moe/route.py | 2 +- sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py | 77 +- .../qwen3_5/sm103/moe/src/mkm_moe_tp4.cu | 1016 +++++++++++++++++ sgl_nvidia_mega/qwen3_5/spec.py | 3 +- test/kernels/test_qwen3_5_draft_front.py | 31 +- test/kernels/test_qwen3_5_moe.py | 27 +- test/unit/test_qwen3_5_spec.py | 8 +- 11 files changed, 1148 insertions(+), 61 deletions(-) create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/moe/src/mkm_moe_tp4.cu diff --git a/sgl_nvidia_mega/qwen3_5/draft/hop.py b/sgl_nvidia_mega/qwen3_5/draft/hop.py index 9db1f519..a0b0dd25 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/hop.py +++ b/sgl_nvidia_mega/qwen3_5/draft/hop.py @@ -36,6 +36,12 @@ # Draft steps carry one row; the verify and extend widths keep their own kernels. MAX_M = 1 +# Whether the attention output's o_proj runs as the one-row GEMV that pushes into the FFN +# input's hop, per TP size, as SGLang branch nvidia/qwen35-agentx-gb300 runs them: TP8 does +# (SGLANG_Q35MF_MKD_GEMV); TP4 runs SGLang's o_proj and the hop publishes its output (that +# branch's TP4 setup also pushes the single-launch MoE into the final norm's hop, +# SGLANG_Q35MF_MKD_HOP_PUSH, which changes no bits and is not ported). +GEMV_O_PROJ = {8: True, 4: False} class DraftHop: @@ -45,9 +51,16 @@ class DraftHop: head (both read only static data before their PDL wait).""" def __init__( - self, hop, *, early: bool = True, early_head: bool = False, delay_ns: int = 0 + self, + hop, + *, + early: bool = True, + early_head: bool = False, + delay_ns: int = 0, + gemv: bool = True, ): self.hop = hop # the target's sm103.hop.Hop + self.gemv = gemv # the o_proj GEMV built: o_proj() pushes into ffn_input's hop self.early = early self.early_head = early and early_head self.delay_ns = delay_ns # diagnostic: hold the consume's output stores @@ -82,7 +95,10 @@ def build( int(deadline_s * 1e9), gemv_ks=() if o_proj_k is None else (o_proj_k,), ) - return cls(hop, early=early, early_head=early_head), "" + return ( + cls(hop, early=early, early_head=early_head, gemv=o_proj_k is not None), + "", + ) def supports(self, m: int) -> bool: return 1 <= m <= MAX_M and self.hop.supports(m) diff --git a/sgl_nvidia_mega/qwen3_5/draft/layer.py b/sgl_nvidia_mega/qwen3_5/draft/layer.py index f66e1db8..b262db49 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/layer.py +++ b/sgl_nvidia_mega/qwen3_5/draft/layer.py @@ -83,7 +83,11 @@ def layer(parts, x, positions, cache_loc, meta, kv, *, rows_per_request: int): meta, kv, rows_per_request=rows_per_request, - o_proj=None if hop is None else partial(hop.o_proj, weight=lay.o_proj.weight), + o_proj=( + partial(hop.o_proj, weight=lay.o_proj.weight) + if hop is not None and hop.gemv + else None + ), ) moe = parts.moe if rows_per_request == 1 else None if hop is not None: diff --git a/sgl_nvidia_mega/qwen3_5/draft/parts.py b/sgl_nvidia_mega/qwen3_5/draft/parts.py index 780c06d5..76aac5af 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/parts.py +++ b/sgl_nvidia_mega/qwen3_5/draft/parts.py @@ -108,12 +108,17 @@ def _build_moe(self): return moe def _build_hop(self): - from sgl_nvidia_mega.qwen3_5.draft.hop import DraftHop + from sgl_nvidia_mega.qwen3_5.draft.hop import GEMV_O_PROJ, DraftHop + tp = self.binding.tp_size hop, why = DraftHop.build( - tp_size=self.binding.tp_size, + tp_size=tp, deadline_s=self.draft.deadline_s, - o_proj_k=int(self.layer.o_proj.weight.shape[1]), + o_proj_k=( + int(self.layer.o_proj.weight.shape[1]) + if GEMV_O_PROJ.get(tp, False) + else None + ), early=True, early_head=True, ) diff --git a/sgl_nvidia_mega/qwen3_5/moe/mkm.py b/sgl_nvidia_mega/qwen3_5/moe/mkm.py index 2d48bcaf..f8962bf6 100644 --- a/sgl_nvidia_mega/qwen3_5/moe/mkm.py +++ b/sgl_nvidia_mega/qwen3_5/moe/mkm.py @@ -73,14 +73,16 @@ def shared_topk_slots(config) -> int: def unsupported(mlp) -> str: """Why the fused experts cannot stand in for ``mlp``'s routed experts ("" when - they can): the TP8 shapes on the trtllm-gen weight layout, no expert parallelism, - softmax top-10 renormalized, plain SwiGLU.""" + they can): the TP8 or TP4 shapes on the trtllm-gen weight layout, no expert + parallelism, softmax top-10 renormalized, plain SwiGLU.""" from sglang.srt.layers.moe.utils import RoutingMethodType from sgl_nvidia_mega.qwen3_5.sm103.moe import mkm experts, cfg = mlp.experts, mlp.topk.topk_config - e, h, i = mkm.EXPERTS, mkm.HIDDEN, mkm.W13_ROWS // 2 + e, h, i = mkm.EXPERTS, mkm.HIDDEN, experts.w2_weight.shape[-1] * 2 + if i not in mkm.INTERMEDIATES: + return f"intermediate {i} per rank (engines: {list(mkm.INTERMEDIATES)})" for name, t, dtype, shape in ( ("w13_weight", experts.w13_weight, torch.uint8, (e, 2 * i, h // 2)), ("w2_weight", experts.w2_weight, torch.uint8, (e, h, i // 2)), diff --git a/sgl_nvidia_mega/qwen3_5/moe/route.py b/sgl_nvidia_mega/qwen3_5/moe/route.py index c8446476..d4beebde 100644 --- a/sgl_nvidia_mega/qwen3_5/moe/route.py +++ b/sgl_nvidia_mega/qwen3_5/moe/route.py @@ -127,7 +127,7 @@ def build(model, max_rows: int, fused: bool, shared: bool = False) -> Bound: reason = mkm.unsupported(mlp) if reason: return Bound(route=True, reason=f"mkm: layer {i}: {reason}") - mkm_kernel.build(device) + mkm_kernel.build(device, layers[0].experts.w2_weight.shape[-1] * 2) return Bound( route=True, mkm={ diff --git a/sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py b/sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py index d21a4890..14e52730 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py @@ -1,10 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -"""The routed experts of up to 7 rows in one launch (``src/mkm_moe.cu``), on the router -front's outputs: the top-10 of the logits, FC1, SwiGLU, the NVFP4 requantization of the -intermediate and FC2, reading the trtllm-gen weight layout in place. It writes the -deferred-finalize rows (row ``t * 10 + k`` is expert ``k`` of token ``t``) and the -expert weights, as the trtllm runner's deferred finalize takes them. +"""The routed experts of up to 7 rows in one launch, on the router front's outputs: the +top-10 of the logits, FC1, SwiGLU, the NVFP4 requantization of the intermediate and FC2, +reading the trtllm-gen weight layout in place. It writes the deferred-finalize rows (row +``t * 10 + k`` is expert ``k`` of token ``t``) and the expert weights, as the trtllm +runner's deferred finalize takes them. One engine per intermediate size a rank holds: +128 at TP8 (``src/mkm_moe.cu``), 256 at TP4 (``src/mkm_moe_tp4.cu``). """ from __future__ import annotations @@ -12,12 +13,12 @@ import ctypes from functools import cache from pathlib import Path +from typing import NamedTuple import torch from sgl_nvidia_mega.common import jit -SOURCE = Path(__file__).parent / "src" / "mkm_moe.cu" HIDDEN, EXPERTS, TOP_K, MAX_TOKENS = 4096, 512, 10, 7 W13_ROWS = 256 # an expert's gate and up rows at TP8 (intermediate 128 per rank) W13_BOX_ROWS = 128 # kW13BoxRows: rows of one W13 TMA box @@ -25,7 +26,24 @@ CTAS = ( 2 * MAX_TOKENS * TOP_K ) # a 2-CTA cluster per selectable expert; the unused ones exit -SMEM_BYTES = 211968 # sizeof(Smem) + 1024, checked when the source compiles + + +class Engine(NamedTuple): + source: str # under src/ + kernel: str + smem_bytes: int # sizeof(Smem) + 1024, checked when the source compiles + w2_swizzle: str # the W2 TMA map's swizzle: its box is a row of I / 2 bytes + + +# per intermediate size a rank holds (TP8, TP4) +ENGINES = { + 128: Engine("mkm_moe.cu", "q35_mkm_moe", 211968, "CU_TENSOR_MAP_SWIZZLE_64B"), + 256: Engine( + "mkm_moe_tp4.cu", "q35_mkm_moe_tp4", 216064, "CU_TENSOR_MAP_SWIZZLE_128B" + ), +} +INTERMEDIATES = tuple(ENGINES) +SMEM_BYTES = ENGINES[128].smem_bytes class Params(ctypes.Structure): @@ -50,20 +68,25 @@ class Params(ctypes.Structure): @cache -def _kernel(device_index: int) -> jit.Kernel: +def _kernel(device_index: int, intermediate: int = 128) -> jit.Kernel: + engine = ENGINES[intermediate] major, minor = torch.cuda.get_device_capability(device_index) cubin = jit.compile_source( - SOURCE.read_text(), - "q35_mkm_moe", + (Path(__file__).parent / "src" / engine.source).read_text(), + engine.kernel, f"sm_{major}{minor}a", - ("-std=c++17", "-default-device", f"-DQ35_MKM_SMEM_BYTES={SMEM_BYTES}"), + ( + "-std=c++17", + "-default-device", + f"-DQ35_MKM_SMEM_BYTES={engine.smem_bytes}", + ), ) - return jit.load(cubin, "q35_mkm_moe", (THREADS, 1, 1), SMEM_BYTES) + return jit.load(cubin, engine.kernel, (THREADS, 1, 1), engine.smem_bytes) -def build(device_index: int) -> None: - """Compile the kernel (no compile inside a step).""" - _kernel(device_index) +def build(device_index: int, intermediate: int = 128) -> None: + """Compile the engine for ``intermediate`` per rank (no compile inside a step).""" + _kernel(device_index, intermediate) def _map( @@ -97,19 +120,28 @@ def _map( def weight_maps(w13: torch.Tensor, w2: torch.Tensor) -> tuple: - """The TMA maps of a layer's W13 [512, 256, 2048] and W2 [512, 4096, 64] (uint8, the - trtllm-gen layout); they stay valid as long as the weights.""" + """The TMA maps of a layer's W13 [512, 2 I, 2048] and W2 [512, 4096, I / 2] (uint8, + the trtllm-gen layout; I = 128 or 256 per rank); they stay valid as long as the + weights.""" + i = w2.shape[2] * 2 swizzle = jit.driver().CUtensorMapSwizzle return ( _map( w13, - EXPERTS * W13_ROWS, + EXPERTS * 2 * i, HIDDEN // 2, 128, W13_BOX_ROWS, swizzle.CU_TENSOR_MAP_SWIZZLE_128B, ), - _map(w2, EXPERTS * HIDDEN, 64, 64, 256, swizzle.CU_TENSOR_MAP_SWIZZLE_64B), + _map( + w2, + EXPERTS * HIDDEN, + i // 2, + i // 2, + 256, + getattr(swizzle, ENGINES[i].w2_swizzle), + ), ) @@ -126,11 +158,12 @@ def mkm_moe( ) -> tuple[torch.Tensor, torch.Tensor]: """(gemm2_out bf16 [M * 10, 4096], expert_weights bf16 [M, 10]) for logits [M, 512] bf16 (M <= 7) and the router front's x_fp4 [M, 2048] / x_sf [M, 256] (uint8, linear - e4m3); w13_sf [512, 256, 256] / w2_sf [512, 4096, 8] e4m3 viewed as uint8; g1_alpha, - g1_scale_c, g2_alpha [512] fp32.""" + e4m3); w13_sf [512, 2 I, 256] / w2_sf [512, 4096, I / 16] e4m3 viewed as uint8 (the + engine follows I); g1_alpha, g1_scale_c, g2_alpha [512] fp32.""" m = logits.shape[0] if not 1 <= m <= MAX_TOKENS: raise ValueError(f"mkm: {m} rows (1-{MAX_TOKENS})") + intermediate = w2_sf.shape[2] * 16 device = logits.device gemm2_out = torch.empty((m * TOP_K, HIDDEN), dtype=torch.bfloat16, device=device) expert_weights = torch.empty((m, TOP_K), dtype=torch.bfloat16, device=device) @@ -151,7 +184,7 @@ def mkm_moe( num_tokens=m, ) jit.launch( - _kernel(device.index), + _kernel(device.index, intermediate), (CTAS, 1, 1), [*maps, x_map, p], torch.cuda.current_stream().cuda_stream, diff --git a/sgl_nvidia_mega/qwen3_5/sm103/moe/src/mkm_moe_tp4.cu b/sgl_nvidia_mega/qwen3_5/sm103/moe/src/mkm_moe_tp4.cu new file mode 100644 index 00000000..50b4b04b --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/moe/src/mkm_moe_tp4.cu @@ -0,0 +1,1016 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Fused small-M NVFP4 MoE expert phase for Qwen3.5 TP4 decode verify (M <= 7 tokens, top-10, 512 experts, +// hidden 4096, intermediate 256 per rank): the top-10 of the router logits in the trtllm-gen routing's order and +// weights, FC1, SwiGLU, the NVFP4 requantization and FC2 in one launch over the trtllm-gen weight layout. Writes the +// deferred-finalize rows (row t * 10 + k, identity permutation) and the expert weights. +// +// Built with NVRTC: the kernel has C linkage (q35_mkm_moe_tp4), and the launcher's dynamic shared memory size +// (Q35_MKM_SMEM_BYTES) is checked against the layout at compile time. + +#include +#include +#include + +// NVRTC has no , or +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; +typedef unsigned long long uintptr_t; +typedef struct __align__(64) { + uint64_t opaque[16]; +} CUtensorMap; +#ifndef INFINITY +#define INFINITY __int_as_float(0x7f800000) +#endif + +// Minimal SM100 (tcgen05 / TMA / mbarrier) PTX helpers for the small-M NVFP4 MoE engine. + +namespace qwen35 { + +__device__ __forceinline__ uint32_t smem_u32(const void* p) { + return static_cast(__cvta_generic_to_shared(p)); +} + +// ---------------------------------------------------------------- mbarrier +__device__ __forceinline__ void mbar_init(uint64_t* bar, uint32_t count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" ::"r"(smem_u32(bar)), "r"(count) : "memory"); +} +__device__ __forceinline__ void fence_mbar_init() { + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); +} +__device__ __forceinline__ void mbar_expect_tx(uint64_t* bar, uint32_t bytes) { + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(smem_u32(bar)), "r"(bytes) : "memory"); +} +__device__ __forceinline__ void mbar_arrive(uint64_t* bar) { + asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];" ::"r"(smem_u32(bar)) : "memory"); +} +__device__ __forceinline__ bool mbar_try_wait(uint64_t* bar, uint32_t phase) { + uint32_t ok; + asm volatile( + "{\n .reg .pred p;\n mbarrier.try_wait.parity.shared::cta.b64 p, [%1], %2;\n selp.u32 %0, 1, 0, p;\n}\n" + : "=r"(ok) + : "r"(smem_u32(bar)), "r"(phase) + : "memory"); + return ok != 0; +} +__device__ __forceinline__ void mbar_wait(uint64_t* bar, uint32_t phase) { + while (!mbar_try_wait(bar, phase)) { + } +} + +// ---------------------------------------------------------------- TMA / bulk copies +__device__ __forceinline__ void tma_load_2d(void* dst, const void* tmap, uint64_t* bar, int c0, int c1) { + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%3, %4}], [%2];" :: + "r"(smem_u32(dst)), + "l"(tmap), "r"(smem_u32(bar)), "r"(c0), "r"(c1) + : "memory"); +} +__device__ __forceinline__ void bulk_load(void* dst, const void* src, uint32_t bytes, uint64_t* bar) { + asm volatile("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"( + smem_u32(dst)), + "l"(src), "r"(bytes), "r"(smem_u32(bar)) + : "memory"); +} +__device__ __forceinline__ void prefetch_tmap(const void* tmap) { + asm volatile("prefetch.tensormap [%0];" ::"l"(tmap) : "memory"); +} +// generic-proxy smem writes -> visible to the async proxy (UMMA / UTCCP operand reads) +__device__ __forceinline__ void fence_proxy_async_smem() { + asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); +} + +// ---------------------------------------------------------------- tcgen05 +__device__ __forceinline__ void tmem_alloc(uint32_t* dst_smem, uint32_t ncols) { + asm volatile("tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" ::"r"(smem_u32(dst_smem)), + "r"(ncols) + : "memory"); +} +__device__ __forceinline__ void tmem_relinquish() { + asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;" ::: "memory"); +} +__device__ __forceinline__ void tmem_dealloc(uint32_t taddr, uint32_t ncols) { + asm volatile("tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" ::"r"(taddr), "r"(ncols) : "memory"); +} +__device__ __forceinline__ void tc_fence_before() { + asm volatile("tcgen05.fence::before_thread_sync;" ::: "memory"); +} +__device__ __forceinline__ void tc_fence_after() { + asm volatile("tcgen05.fence::after_thread_sync;" ::: "memory"); +} +// arrive on `bar` when all prior tcgen05 async ops issued by this thread complete +__device__ __forceinline__ void tc_commit(uint64_t* bar) { + asm volatile("tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%0];" ::"r"(smem_u32(bar)) + : "memory"); +} +// smem -> TMEM copy of one 512-byte scale-factor chunk (32 rows x 16 B), broadcast to the 4 lane quarters +__device__ __forceinline__ void utccp_32x128b_warpx4(uint32_t taddr, uint64_t sdesc) { + asm volatile("tcgen05.cp.cta_group::1.32x128b.warpx4 [%0], %1;" ::"r"(taddr), "l"(sdesc) : "memory"); +} +// D(tmem) (+)= A(smem) * B(smem)^T with NVFP4 block-16 scales from TMEM +__device__ __forceinline__ void umma_nvf4(uint32_t tmem_d, uint64_t adesc, uint64_t bdesc, uint32_t idesc, + uint32_t tmem_sfa, uint32_t tmem_sfb, uint32_t accumulate) { + asm volatile( + "{\n .reg .pred p;\n setp.ne.b32 p, %4, 0;\n" + " tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16 [%0], %1, %2, %3, [%5], [%6], p;\n}\n" ::"r"( + tmem_d), + "l"(adesc), "l"(bdesc), "r"(idesc), "r"(accumulate), "r"(tmem_sfa), "r"(tmem_sfb) + : "memory"); +} +// each warp reads its 32-lane quarter: 32 lanes x 16 consecutive 32-bit columns +__device__ __forceinline__ void tmem_ld_32x32b_x16(uint32_t taddr, uint32_t (&r)[16]) { + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x16.b32 {%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15}, [%16];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]), "=r"(r[4]), "=r"(r[5]), "=r"(r[6]), "=r"(r[7]), + "=r"(r[8]), "=r"(r[9]), "=r"(r[10]), "=r"(r[11]), "=r"(r[12]), "=r"(r[13]), "=r"(r[14]), "=r"(r[15]) + : "r"(taddr) + : "memory"); +} +// L2 prefetch of a contiguous global range (bytes multiple of 16, 16-byte aligned) +__device__ __forceinline__ void bulk_prefetch_l2(const void* gptr, uint32_t bytes) { + asm volatile("cp.async.bulk.prefetch.L2.global [%0], %1;" ::"l"(gptr), "r"(bytes) : "memory"); +} +__device__ __forceinline__ void tmem_ld_wait() { asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); } +// each warp writes its 32-lane quarter: 32 lanes x 16 consecutive 32-bit columns +__device__ __forceinline__ void tmem_st_32x32b_x16(uint32_t taddr, const uint32_t (&r)[16]) { + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x16.b32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16};" ::"r"( + taddr), + "r"(r[0]), "r"(r[1]), "r"(r[2]), "r"(r[3]), "r"(r[4]), "r"(r[5]), "r"(r[6]), "r"(r[7]), "r"(r[8]), "r"(r[9]), + "r"(r[10]), "r"(r[11]), "r"(r[12]), "r"(r[13]), "r"(r[14]), "r"(r[15]) + : "memory"); +} +__device__ __forceinline__ void tmem_st_wait() { asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); } + +// ---------------------------------------------------------------- descriptors +// SM100 shared-memory matrix descriptor (version 1). layout: 0 none, 2 SW128, 4 SW64, 6 SW32. +__device__ __forceinline__ uint64_t make_sdesc(uint32_t saddr, uint32_t lbo, uint32_t sbo, uint32_t layout) { + uint64_t d = 0; + d |= (uint64_t)((saddr >> 4) & 0x3FFF); + d |= (uint64_t)((lbo >> 4) & 0x3FFF) << 16; + d |= (uint64_t)((sbo >> 4) & 0x3FFF) << 32; + d |= (uint64_t)1 << 46; // version + d |= (uint64_t)(layout & 0x7) << 61; + return d; +} +// Block-scaled instruction descriptor: E2M1 x E2M1, UE4M3 scales, both K-major, F32 accumulate. +__host__ __device__ constexpr uint32_t make_idesc_nvf4(uint32_t M, uint32_t N) { + return (1u << 7) // a_format E2M1 + | (1u << 10) // b_format E2M1 + | ((N >> 3) << 17) // n_dim + | (0u << 23) // scale format UE4M3 + | ((M >> 4) << 24); // m_dim +} + +} // namespace qwen35 + +namespace sglang { + +namespace qwen_mkm_moe_tp4 { + +using namespace qwen35; + + +constexpr int kH = 4096, kE = 512, kTopK = 10, kMaxTok = 7; +constexpr int kI = 256; // intermediates per expert per rank (TP4) +constexpr int kN1 = 2 * kI; // W13 rows per expert (gate + up) +constexpr int kStageBytes = 36864; // 32 KB weights + 4 KB scale factors +constexpr int kStages = 5; +constexpr int kN1c = 256; // W13 rows per CTA (row split) +constexpr int kW13BoxRows = 128; // rows of one W13 TMA box; the host tensor map must use the same +constexpr int kFc1Stages = 16; // 4 k-blocks each (full K = 4096): 2 TMA boxes of 128 rows x 128 B (same K range) +constexpr int kFc2Stages = 8; // one tile of 256 W2 rows x 128 B each +constexpr int kTotStages = kFc1Stages + kFc2Stages; +constexpr int kThreads = 256; +// TMEM columns. FC1: accumulator [0,256), x SFA ring [256,320) (16 slots x 4), SFB ring [384,512) (16 slots x 8). +// FC2: accumulator [0,256), h SFA [256,384) (32 atoms x 4), SFB ring [384,512) (16 slots x 8). +constexpr uint32_t kColSfaX = 256, kColSfa = 256, kColSfb = 384; +constexpr int kSfbSlots = 16, kSfaSlots = 16; +constexpr int kARow = 260; // FC1 accumulator row stride in floats: 16 B aligned, rows on distinct banks +#ifndef SF_UTCCP_FC1 +#define SF_UTCCP_FC1 0 +#endif +#ifndef DIAG_NO_SFWAIT // timing diagnostic only: the MMA issuer does not wait for staged scale factors +#define DIAG_NO_SFWAIT 0 +#endif +#ifndef L2_EVICT_FIRST // stream expert weights with an L2 evict-first policy (they are read once per layer) +#define L2_EVICT_FIRST 1 +#endif +#ifndef TOPK_RANK // 1: all-pairs rank top-10; 0: chains of warp max reductions (faster: 13.14 vs 13.32 us) +#define TOPK_RANK 0 +#endif +#ifndef PDL_TRIGGER // 1: let dependent grids launch once the FC2 MMAs are queued (only safe if they all wait) +#define PDL_TRIGGER 0 +#endif +#ifndef HOIST_SCALES // 1: per-expert dequant scales loaded right after routing; 0: at first use +#define HOIST_SCALES 1 +#endif +#ifndef FC2SF_PRE // 1: stage FC2 scale factors of stages 8, 9 during the partial exchange; 0: after h +#define FC2SF_PRE 1 +#endif +#ifndef DIAG_NO_MMA // timing diagnostic only: no MMAs, stages are released as soon as they land +#define DIAG_NO_MMA 0 +#endif // partial row stride in floats: 16 B aligned, rows land on distinct banks + +struct Params { + const uint16_t* logits; // [M, 512] bf16 router logits (router_gemm_quant output) + uint16_t* expert_weights; // [M, 10] bf16 out: DynBlock softmax-over-top-10 weights (CTA 0 writes) + int32_t* topk_ids; // [M, 10] out (CTA 0 writes) + const uint8_t* x_sf; // [M, 256] linear e4m3 + const uint8_t* w13_sf; // [E, 4 row blocks x 64 k blocks x 512 B] (128x4 interleaved) + const uint8_t* w2_sf; // [E, 32 row blocks x 4 k blocks x 512 B] + const float* g1_alpha; // [E] gate dequant + const float* g1_scale_c; // [E] up dequant * FC2 input quant scale + const float* g2_alpha; // [E] + __nv_bfloat16* out; // [M * 10, 4096] + long long* trace; // optional [grid, 16] %globaltimer stamps + int num_tokens; +}; + +struct __align__(1024) Smem { + uint8_t ring[kStages][kStageBytes]; + alignas(1024) uint8_t xa[16 * 1024]; // x operand: 8 rows x full K (2048 B), SW128 (16 atoms of 8 rows x 128 B) + float acc1[8 * kARow]; // FC1 accumulator rows 0..7 (tokens) x 256 accumulator columns + alignas(1024) uint8_t ha[1024]; // h operand: 8 rows x 128 B (K = 256), SW128 (atom must be 1024 B aligned) + uint8_t xsf[8][256]; // x scale bytes, full K (row t = token t), by bulk copy + uint32_t hsf[8][4]; // h scale words: token t, k-block kb (4 e4m3 bytes) + uint64_t full[kStages], empty[kStages], sf_full[kStages]; + uint64_t x_full, fc1_done, h_peer, h_ready, fc2_done, route_done; + uint32_t tmem; + int expert; + uint32_t tokmask; + int kslot[8]; + alignas(16) uint32_t cand[8][32]; // read as uint4 by the rank-based top-k + int32_t ids[8][kTopK]; +}; + +__device__ __forceinline__ uint64_t gtime() { + uint64_t t; + asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t)::"memory"); + return t; +} +__device__ __forceinline__ uint32_t cluster_rank() { + uint32_t r; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(r)); + return r; +} +__device__ __forceinline__ void cluster_arrive() { + asm volatile("barrier.cluster.arrive.release.aligned;" ::: "memory"); +} +// no release: the phase only orders DSMEM traffic that mbarriers already completed, so pending global stores +// (stamps, routing outputs, output rows) need not drain before the arrive +__device__ __forceinline__ void cluster_arrive_relaxed() { + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); +} +__device__ __forceinline__ void cluster_wait() { + asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); +} +__device__ __forceinline__ uint32_t mapa(uint32_t saddr, uint32_t rank) { + uint32_t r; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(r) : "r"(saddr), "r"(rank)); + return r; +} +// smem (this CTA) -> smem (peer CTA in the cluster), completes tx bytes on the peer's mbarrier +__device__ __forceinline__ void bulk_s2s(uint32_t dst_cluster, uint32_t src, uint32_t bytes, uint32_t bar_cluster) { + asm volatile("cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"( + dst_cluster), + "r"(src), "r"(bytes), "r"(bar_cluster) + : "memory"); +} +// registers -> the peer CTA's shared memory; completes the bytes on the peer's mbarrier +__device__ __forceinline__ void st_async_v4(uint32_t dst, const uint4& v, uint32_t bar) { + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 [%0], {%1, %2, %3, %4}, [%5];" ::"r"(dst), + "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w), "r"(bar) + : "memory"); +} +__device__ __forceinline__ void st_async_b32(uint32_t dst, uint32_t v, uint32_t bar) { + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.b32 [%0], %1, [%2];" ::"r"(dst), "r"(v), "r"(bar) + : "memory"); +} +__device__ __forceinline__ void tmem_st_x4(uint32_t taddr, const uint4& v) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x4.b32 [%0], {%1, %2, %3, %4};" ::"r"(taddr), "r"(v.x), "r"(v.y), + "r"(v.z), "r"(v.w) + : "memory"); +} +__device__ __forceinline__ void named_bar(int id, int n) { + asm volatile("bar.sync %0, %1;" ::"r"(id), "r"(n) : "memory"); +} +__device__ __forceinline__ void tmem_ld_x32(uint32_t taddr, uint32_t (&r)[32]) { + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x32.b32 {%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16,%17,%18,%19," + "%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,%31}, [%32];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]), "=r"(r[4]), "=r"(r[5]), "=r"(r[6]), "=r"(r[7]), "=r"(r[8]), + "=r"(r[9]), "=r"(r[10]), "=r"(r[11]), "=r"(r[12]), "=r"(r[13]), "=r"(r[14]), "=r"(r[15]), "=r"(r[16]), + "=r"(r[17]), "=r"(r[18]), "=r"(r[19]), "=r"(r[20]), "=r"(r[21]), "=r"(r[22]), "=r"(r[23]), "=r"(r[24]), + "=r"(r[25]), "=r"(r[26]), "=r"(r[27]), "=r"(r[28]), "=r"(r[29]), "=r"(r[30]), "=r"(r[31]) + : "r"(taddr) + : "memory"); +} +__device__ __forceinline__ void tmem_ld_x64(uint32_t taddr, uint32_t (&r)[64]) { + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x64.b32 {%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16,%17,%18,%19," + "%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,%31,%32,%33,%34,%35,%36,%37,%38,%39,%40,%41,%42,%43,%44,%45,%46,%47," + "%48,%49,%50,%51,%52,%53,%54,%55,%56,%57,%58,%59,%60,%61,%62,%63}, [%64];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]), "=r"(r[4]), "=r"(r[5]), "=r"(r[6]), "=r"(r[7]), "=r"(r[8]), + "=r"(r[9]), "=r"(r[10]), "=r"(r[11]), "=r"(r[12]), "=r"(r[13]), "=r"(r[14]), "=r"(r[15]), "=r"(r[16]), + "=r"(r[17]), "=r"(r[18]), "=r"(r[19]), "=r"(r[20]), "=r"(r[21]), "=r"(r[22]), "=r"(r[23]), "=r"(r[24]), + "=r"(r[25]), "=r"(r[26]), "=r"(r[27]), "=r"(r[28]), "=r"(r[29]), "=r"(r[30]), "=r"(r[31]), "=r"(r[32]), + "=r"(r[33]), "=r"(r[34]), "=r"(r[35]), "=r"(r[36]), "=r"(r[37]), "=r"(r[38]), "=r"(r[39]), "=r"(r[40]), + "=r"(r[41]), "=r"(r[42]), "=r"(r[43]), "=r"(r[44]), "=r"(r[45]), "=r"(r[46]), "=r"(r[47]), "=r"(r[48]), + "=r"(r[49]), "=r"(r[50]), "=r"(r[51]), "=r"(r[52]), "=r"(r[53]), "=r"(r[54]), "=r"(r[55]), "=r"(r[56]), + "=r"(r[57]), "=r"(r[58]), "=r"(r[59]), "=r"(r[60]), "=r"(r[61]), "=r"(r[62]), "=r"(r[63]) + : "r"(taddr) + : "memory"); +} +__device__ __forceinline__ void tmem_st_x8(uint32_t taddr, const uint32_t (&r)[8]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x8.b32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8};" ::"r"(taddr), "r"(r[0]), + "r"(r[1]), "r"(r[2]), "r"(r[3]), "r"(r[4]), "r"(r[5]), "r"(r[6]), "r"(r[7]) + : "memory"); +} +__device__ __forceinline__ void tmem_st_x16(uint32_t taddr, const uint32_t (&r)[16]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x16.b32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16};" :: + "r"(taddr), + "r"(r[0]), "r"(r[1]), "r"(r[2]), "r"(r[3]), "r"(r[4]), "r"(r[5]), "r"(r[6]), "r"(r[7]), "r"(r[8]), "r"(r[9]), + "r"(r[10]), "r"(r[11]), "r"(r[12]), "r"(r[13]), "r"(r[14]), "r"(r[15]) + : "memory"); +} +__device__ __forceinline__ void tmem_st_x32(uint32_t taddr, const uint32_t (&r)[32]) { + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x32.b32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16,%17,%18," + "%19,%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,%31,%32};" ::"r"(taddr), + "r"(r[0]), "r"(r[1]), "r"(r[2]), "r"(r[3]), "r"(r[4]), "r"(r[5]), "r"(r[6]), "r"(r[7]), "r"(r[8]), "r"(r[9]), + "r"(r[10]), "r"(r[11]), "r"(r[12]), "r"(r[13]), "r"(r[14]), "r"(r[15]), "r"(r[16]), "r"(r[17]), "r"(r[18]), + "r"(r[19]), "r"(r[20]), "r"(r[21]), "r"(r[22]), "r"(r[23]), "r"(r[24]), "r"(r[25]), "r"(r[26]), "r"(r[27]), + "r"(r[28]), "r"(r[29]), "r"(r[30]), "r"(r[31]) + : "memory"); +} +__device__ __forceinline__ float ex2_approx(float x) { + float y; + asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} +__device__ __forceinline__ float mul_ftz(float a, float b) { + float y; + asm("mul.ftz.f32 %0, %1, %2;" : "=f"(y) : "f"(a), "f"(b)); + return y; +} +__device__ __forceinline__ float rcp_approx(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} +__device__ __forceinline__ uint32_t cvt_e4m3(float a) { + uint32_t out; + asm("{\n .reg .b16 pair;\n .reg .f32 zero;\n mov.f32 zero, 0f00000000;\n" + " cvt.rn.satfinite.e4m3x2.f32 pair, zero, %1;\n cvt.u32.u16 %0, pair;\n}" + : "=r"(out) + : "f"(a)); + return out & 0xFFu; +} +__device__ __forceinline__ float e4m3_to_f32(uint32_t v) { + float out; + asm("{\n .reg .b16 pair;\n .reg .b32 h2;\n .reg .b16 lo, hi;\n cvt.u16.u32 pair, %1;\n" + " cvt.rn.f16x2.e4m3x2 h2, pair;\n mov.b32 {lo, hi}, h2;\n cvt.f32.f16 %0, lo;\n}" + : "=f"(out) + : "r"(v)); + return out; +} +__device__ __forceinline__ uint32_t cvt_e2m1x8(const float* v) { + uint32_t out; + asm("{\n .reg .b8 b0, b1, b2, b3;\n" + " cvt.rn.satfinite.e2m1x2.f32 b0, %2, %1;\n cvt.rn.satfinite.e2m1x2.f32 b1, %4, %3;\n" + " cvt.rn.satfinite.e2m1x2.f32 b2, %6, %5;\n cvt.rn.satfinite.e2m1x2.f32 b3, %8, %7;\n" + " mov.b32 %0, {b0, b1, b2, b3};\n}" + : "=r"(out) + : "f"(v[0]), "f"(v[1]), "f"(v[2]), "f"(v[3]), "f"(v[4]), "f"(v[5]), "f"(v[6]), "f"(v[7])); + return out; +} +// ---- router top-k, bit-exact with FlashInfer's single-block routing (routingCustom, RenormalizeNaive on bf16): +// key = bf16 order (cub float twiddle) above 65535 - expert, i.e. the higher score first, the smaller expert on ties. +__device__ __forceinline__ uint32_t topk_key(uint32_t bits, uint32_t expert) { + const uint32_t twiddled = bits ^ ((bits & 0x8000u) ? 0xFFFFu : 0x8000u); + return (twiddled << 16) | (65535u - expert); +} +__device__ __forceinline__ void topk_unkey(uint32_t key, uint32_t& bits, uint32_t& expert) { + expert = 65535u - (key & 0xFFFFu); + const uint32_t twiddled = key >> 16; + bits = twiddled ^ ((twiddled & 0x8000u) ? 0x8000u : 0xFFFFu); +} +__device__ __forceinline__ uint32_t max16(const uint32_t (&k)[16]) { + const uint32_t a = max(max(max(k[0], k[1]), max(k[2], k[3])), max(max(k[4], k[5]), max(k[6], k[7]))); + const uint32_t b = max(max(max(k[8], k[9]), max(k[10], k[11])), max(max(k[12], k[13]), max(k[14], k[15]))); + return max(a, b); +} +__device__ __forceinline__ uint32_t warp_max_u32(uint32_t v) { + uint32_t r; + asm volatile("redux.sync.max.u32 %0, %1, 0xffffffff;" : "=r"(r) : "r"(v)); + return r; +} +__device__ __forceinline__ uint32_t warp_sum_u32(uint32_t v) { + uint32_t r; + asm volatile("redux.sync.add.u32 %0, %1, 0xffffffff;" : "=r"(r) : "r"(v)); + return r; +} +__device__ __forceinline__ uint32_t warp_topk_of_lanes(uint32_t v, uint32_t lane) { + uint32_t mine = 0; +#pragma unroll + for (uint32_t k = 0; k < kTopK; ++k) { + const uint32_t top = warp_max_u32(v); + v = v == top ? 0u : v; + mine = lane == k ? top : mine; + } + return mine; +} +__device__ __forceinline__ uint32_t count_greater(const uint32_t* s_cand, uint32_t v) { + uint32_t r = 0; +#pragma unroll + for (int j = 0; j < 32; j += 4) { + const uint4 q = *reinterpret_cast(s_cand + j); + r += (q.x > v) + (q.y > v) + (q.z > v) + (q.w > v); + } + return r; +} +// Top-10 keys of 512 (16 per lane), lane k < 10 holding the k-th largest (keys unique and nonzero). Ranks come from +// all-pairs compares over shared memory rather than chains of warp reductions. +__device__ __forceinline__ uint32_t warp_top10(uint32_t (&keys)[16], uint32_t lane, uint32_t* s_cand) { + // bound = the 10th largest lane maximum; every top-10 key is >= bound + const uint32_t m = max16(keys); + s_cand[lane] = m; + __syncwarp(); + const uint32_t r = count_greater(s_cand, m); + const uint32_t bound = __shfl_sync(0xffffffffu, m, __ffs(__ballot_sync(0xffffffffu, r == kTopK - 1)) - 1); + uint32_t count = 0; +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) count += keys[i] >= bound; + // exclusive prefix of count over the lanes, from the ballots of its bits (count <= 16) + const uint32_t lt = (1u << lane) - 1u; + uint32_t offset = 0, total = 0; +#pragma unroll + for (int b = 0; b < 5; ++b) { + const uint32_t bb = __ballot_sync(0xffffffffu, (count >> b) & 1u); + offset += (uint32_t)__popc(bb & lt) << b; + total += (uint32_t)__popc(bb) << b; + } + if (total <= 32) { + __syncwarp(); // every lane has read the lane maxima +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) { + if (keys[i] >= bound) s_cand[offset++] = keys[i]; + } + if (lane >= total) s_cand[lane] = 0u; + __syncwarp(); + const uint32_t cand = s_cand[lane]; + const uint32_t rk = count_greater(s_cand, cand); + __syncwarp(); + if (lane < total && rk < kTopK) s_cand[rk] = cand; + __syncwarp(); + return lane < kTopK ? s_cand[lane] : 0u; + } + uint32_t mine = 0; +#pragma unroll 1 + for (uint32_t k = 0; k < kTopK; ++k) { + const uint32_t top = warp_max_u32(max16(keys)); +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) keys[i] = keys[i] == top ? 0u : keys[i]; + mine = lane == k ? top : mine; + } + return mine; +} +// Top-10 keys of 512 (16 per lane), lane k holding the k-th largest (keys unique and nonzero). +__device__ __forceinline__ uint32_t warp_top10_redux(uint32_t (&keys)[16], uint32_t lane, uint32_t* s_cand) { + uint32_t v = max16(keys); + uint32_t bound = 0; +#pragma unroll + for (uint32_t k = 0; k < kTopK; ++k) { + bound = warp_max_u32(v); + v = v == bound ? 0u : v; + } + uint32_t count = 0; +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) count += keys[i] >= bound; + const uint32_t total = warp_sum_u32(count); + if (total <= 32) { + uint32_t offset = count; +#pragma unroll + for (uint32_t d = 1; d < 32; d <<= 1) { + const uint32_t up = __shfl_up_sync(0xffffffffu, offset, d); + offset += lane >= d ? up : 0u; + } + offset -= count; +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) { + if (keys[i] >= bound) s_cand[offset++] = keys[i]; + } + __syncwarp(); + const uint32_t cand = lane < total ? s_cand[lane] : 0u; + __syncwarp(); + return warp_topk_of_lanes(cand, lane); + } + uint32_t mine = 0; +#pragma unroll 1 + for (uint32_t k = 0; k < kTopK; ++k) { + const uint32_t top = warp_max_u32(max16(keys)); +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) keys[i] = keys[i] == top ? 0u : keys[i]; + mine = lane == k ? top : mine; + } + return mine; +} +// FlashInfer calcSoftmax (bf16 scores) as built with -use_fast_math: expf -> ex2.approx.ftz of x * log2(e), +// division -> div.full.ftz; reductions are cooperative_groups::reduce like the reference. +__device__ __forceinline__ float fast_expf(float x) { + float y; + asm("{\n .reg .f32 t;\n mul.ftz.f32 t, %1, 0f3FB8AA3B;\n ex2.approx.ftz.f32 %0, t;\n}" : "=f"(y) : "f"(x)); + return y; +} +__device__ __forceinline__ float fast_div(float a, float b) { + float y; + asm("div.full.ftz.f32 %0, %1, %2;" : "=f"(y) : "f"(a), "f"(b)); + return y; +} +__device__ __forceinline__ uint16_t softmax_topk(uint16_t score_bits, uint32_t lane) { + namespace cg = cooperative_groups; + const auto warp = cg::tiled_partition<32>(cg::this_thread_block()); + const float si = __uint_as_float((uint32_t)score_bits << 16); + float max_score = -INFINITY; + if (lane < kTopK) max_score = si >= max_score ? si : max_score; + max_score = cg::reduce(warp, max_score, cg::greater()); + float sum_score = 0.f, new_score = 0.f; + if (lane < kTopK) { + new_score = fast_expf(__fsub_rn(si, max_score)); + sum_score = __fadd_rn(sum_score, new_score); + } + sum_score = cg::reduce(warp, sum_score, cg::plus()); + uint16_t out = score_bits; + if (lane < kTopK) { + const __nv_bfloat16 b = __float2bfloat16_rn(fast_div(new_score, sum_score)); + out = *reinterpret_cast(&b); + } + return out; +} + +__device__ __forceinline__ uint64_t policy_evict_first() { + uint64_t pol; + asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(pol)); + return pol; +} +__device__ __forceinline__ void tma_load_2d_w(void* dst, const void* tmap, uint64_t* bar, int c0, int c1, uint64_t pol) { + if (L2_EVICT_FIRST) + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint" + " [%0], [%1, {%3, %4}], [%2], %5;" ::"r"(smem_u32(dst)), + "l"(tmap), "r"(smem_u32(bar)), "r"(c0), "r"(c1), "l"(pol) + : "memory"); + else + tma_load_2d(dst, tmap, bar, c0, c1); +} +__device__ __forceinline__ void bulk_load_w(void* dst, const void* src, uint32_t bytes, uint64_t* bar, uint64_t pol) { + if (L2_EVICT_FIRST) + asm volatile("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [%0], [%1], %2, [%3], %4;" ::"r"( + smem_u32(dst)), + "l"(src), "r"(bytes), "r"(smem_u32(bar)), "l"(pol) + : "memory"); + else + bulk_load(dst, src, bytes, bar); +} + +__device__ __forceinline__ void stamp(const Params& p, int slot) { + if (p.trace) p.trace[blockIdx.x * 16 + slot] = (long long)gtime(); +} +// stamp after `dep` is available (the branch needs its value; keys are never 0) +__device__ __forceinline__ void stamp_dep(const Params& p, int slot, uint32_t dep) { + if (dep == 0u) __trap(); + stamp(p, slot); +} +// Physical W13 column (accumulator column) holding reordered row q: block-32 shuffle of FlashInfer's +// get_shuffle_matrix_a_row_indices (epilogue_tile_m = 128); reordered row 2j is up_j, 2j + 1 is gate_j. +__device__ __forceinline__ int w13_col(int q) { return (q & ~31) + ((q & 31) % 4) * 8 + (q & 31) / 4; } +__host__ __device__ constexpr int w13_logical(int p) { + return ((p & ~31) + (p % 8) * 4 + (p & 31) / 8) % 2 ? ((p & ~31) + (p % 8) * 4 + (p & 31) / 8) / 2 + : 128 + ((p & ~31) + (p % 8) * 4 + (p & 31) / 8) / 2; +} + +// FC2 drain of TMEM lane quarter q (tiles 4 q .. 4 q + 3), column half `half` (128 accumulator columns) +__device__ __forceinline__ void fc2_drain(const Params& p, Smem& s, uint32_t tm, int q, int lane, int half, float a2, + uint32_t rank, int e) { + mbar_wait(&s.fc2_done, 0); + if (!HOIST_SCALES) a2 = p.g2_alpha[e]; + tc_fence_after(); + if (q == 0 && half == 0 && lane == 0) stamp(p, 7); + const int row = q * 32 + lane, tile = row / 8, t = row % 8; + const bool valid = t < p.num_tokens && ((s.tokmask >> t) & 1u); + __nv_bfloat16* orow = p.out + (size_t)(t * kTopK + s.kslot[t]) * kH + rank * 2048 + tile * 256 + half * 128; +#pragma unroll 1 + for (int c = 0; c < 2; ++c) { + uint32_t r[64]; + tmem_ld_x64(tm + ((uint32_t)(q * 32) << 16) + (uint32_t)(half * 128 + c * 64), r); + tmem_ld_wait(); + if (valid) { +#pragma unroll + for (int b32 = 0; b32 < 2; ++b32) { + uint32_t pk[16]; +#pragma unroll + for (int l = 0; l < 32; l += 2) { + const float v0 = __uint_as_float(r[b32 * 32 + (l % 4) * 8 + l / 4]) * a2; + const float v1 = __uint_as_float(r[b32 * 32 + ((l + 1) % 4) * 8 + (l + 1) / 4]) * a2; + const __nv_bfloat162 b2 = __floats2bfloat162_rn(v0, v1); + pk[l / 2] = *reinterpret_cast(&b2); + } +#pragma unroll + for (int i = 0; i < 4; ++i) + *reinterpret_cast(orow + c * 64 + b32 * 32 + i * 8) = + make_uint4(pk[4 * i], pk[4 * i + 1], pk[4 * i + 2], pk[4 * i + 3]); + } + } + } +} + +extern "C" __global__ void __cluster_dims__(2, 1, 1) __launch_bounds__(kThreads, 1) +q35_mkm_moe_tp4(const __grid_constant__ CUtensorMap tm13, const __grid_constant__ CUtensorMap tm2, + const __grid_constant__ CUtensorMap tmx, const Params p) { + extern __shared__ __align__(1024) uint8_t raw[]; + Smem& s = *reinterpret_cast((reinterpret_cast(raw) + 1023) & ~uintptr_t(1023)); + const int warp = threadIdx.x / 32, lane = threadIdx.x % 32; + const uint32_t rank = cluster_rank(); + const int pair = blockIdx.x / 2; + if (threadIdx.x == 0) { + stamp(p, 0); + for (int i = 0; i < kStages; ++i) { + mbar_init(&s.full[i], 1); + mbar_init(&s.empty[i], 1); + mbar_init(&s.sf_full[i], 4); + } + mbar_init(&s.x_full, 1); + mbar_init(&s.fc1_done, 1); + mbar_init(&s.h_peer, 1); + mbar_init(&s.h_ready, 4); + mbar_init(&s.fc2_done, 1); + mbar_init(&s.route_done, 1); + mbar_expect_tx(&s.h_peer, 32 * 16 + 16 * 4); // the peer's half of h (8 rows x 64 B) and its 16 scale words + fence_mbar_init(); + } + if (warp == 2) { tmem_alloc(&s.tmem, 512); tmem_relinquish(); tc_fence_before(); } + if (warp == 1 && lane == 0) { prefetch_tmap(&tm13); prefetch_tmap(&tm2); prefetch_tmap(&tmx); } + __syncthreads(); + cluster_arrive(); // waited on before the first DSMEM transfer and at exit + tc_fence_after(); + const uint32_t tm = s.tmem; + // cluster `pair` serves (token my_t, top-k slot my_k) unless an earlier pair (in t * 10 + k order) has its expert + const int my_t = pair / kTopK, my_k = pair % kTopK; + if (my_t >= p.num_tokens) { // both CTAs of the pair agree; nothing was issued + cluster_wait(); + if (warp == 2) tmem_dealloc(tm, 512); + return; + } + auto issue_stage = [&](int st, int e, uint64_t pol) { + const int b = st % kStages; + mbar_expect_tx(&s.full[b], kStageBytes); + uint8_t* dst = s.ring[b]; + if (st < kFc1Stages) { + // rows [256 r, 256 r + 256) of this expert as two boxes of 128 rows, K bytes [128 st, 128 st + 128) + tma_load_2d_w(dst, &tm13, &s.full[b], st * 128, e * kN1 + (int)rank * kN1c, pol); + tma_load_2d_w(dst + 16384, &tm13, &s.full[b], st * 128, e * kN1 + (int)rank * kN1c + 128, pol); + // scale atoms of row blocks 2 r and 2 r + 1, k-blocks [4 st, 4 st + 4) + const uint8_t* sf = p.w13_sf + (size_t)e * (kN1 * kH / 16); + bulk_load_w(dst + 32768, sf + (size_t)(2 * (int)rank * 64 + st * 4) * 512, 2048, &s.full[b], pol); + bulk_load_w(dst + 32768 + 2048, sf + (size_t)((2 * (int)rank + 1) * 64 + st * 4) * 512, 2048, &s.full[b], pol); + } else { + const int j = st - kFc1Stages; + const int row0 = (int)rank * 2048 + j * 256; + tma_load_2d_w(dst, &tm2, &s.full[b], 0, e * kH + row0, pol); + bulk_load_w(dst + 32768, p.w2_sf + (size_t)e * (kH * 16) + (size_t)(row0 / 128) * 4 * 512, 4096, &s.full[b], pol); + } + }; + // routing: warps 0, 2..7 take tokens 0..6 (DynBlock-exact top-10 + softmax); warp 1 recomputes token my_t's top-10 + // and starts this pair's weight stream at once (a duplicate pair only wastes its first ring fill, mostly L2 hits) + const int rtok = warp == 1 ? my_t : (warp == 0 ? 0 : warp - 1); + uint32_t key = 0; + asm volatile("griddepcontrol.wait;" ::: "memory"); + if (threadIdx.x == 32) stamp(p, 10); + if (warp == 3 && lane == 0) { + // x (8 atoms of this CTA's K half, rows >= M zero-filled) and its scale bytes; independent of the routing + mbar_expect_tx(&s.x_full, 16 * 1024 + p.num_tokens * 256); +#pragma unroll + for (int a = 0; a < 16; ++a) tma_load_2d(s.xa + a * 1024, &tmx, &s.x_full, a * 128, 0); + for (int t = 0; t < p.num_tokens; ++t) bulk_load(s.xsf[t], p.x_sf + (size_t)t * (kH / 16), 256, &s.x_full); + } + if (rtok < p.num_tokens) { + const uint4* src = reinterpret_cast(p.logits + (size_t)rtok * kE + lane * 16); + const uint4 v0 = src[0], v1 = src[1]; + uint32_t keys[16]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + const uint32_t w = i < 4 ? (&v0.x)[i] : (&v1.x)[i - 4]; + keys[2 * i] = topk_key(w & 0xFFFFu, (uint32_t)(lane * 16 + 2 * i)); + keys[2 * i + 1] = topk_key(w >> 16, (uint32_t)(lane * 16 + 2 * i + 1)); + } + if (threadIdx.x == 32) stamp_dep(p, 11, keys[0]); + key = TOPK_RANK ? warp_top10(keys, (uint32_t)lane, s.cand[warp]) : warp_top10_redux(keys, (uint32_t)lane, s.cand[warp]); + } + int e; + if (warp == 1) { + uint32_t bits, espec; + topk_unkey(__shfl_sync(0xffffffffu, key, my_k), bits, espec); + if (lane == 0) { + stamp(p, 9); + const uint64_t pol = policy_evict_first(); + for (int st = 0; st < kStages; ++st) issue_stage(st, (int)espec, pol); + stamp(p, 2); + } + mbar_wait(&s.route_done, 0); + e = s.expert; + } else { + if (rtok < p.num_tokens) { + uint32_t bits, expert; + topk_unkey(key, bits, expert); + const uint16_t wbits = softmax_topk((uint16_t)bits, (uint32_t)lane); + if (lane < kTopK) { + s.ids[rtok][lane] = (int32_t)expert; + if (blockIdx.x == 0) { + p.expert_weights[rtok * kTopK + lane] = wbits; + p.topk_ids[rtok * kTopK + lane] = (int32_t)expert; + } + } + } + named_bar(4, 224); + if (warp == 0) { + // this pair's expert, whether an earlier pair picked it, and which tokens use it (top-k slot per token) + const int ee = s.ids[my_t][my_k]; + bool dup = false; + for (int q = lane; q < pair; q += 32) dup |= s.ids[q / kTopK][q % kTopK] == ee; + dup = __any_sync(0xffffffffu, dup); + uint32_t mbit = 0; + int ks = 0; + if (lane < p.num_tokens) { +#pragma unroll + for (int k = 0; k < kTopK; ++k) + if (s.ids[lane][k] == ee) { mbit = 1u << lane; ks = k; } + } + const uint32_t mask = __reduce_or_sync(0xffffffffu, mbit); + if (lane < 8) s.kslot[lane] = ks; + if (lane == 0) { + s.expert = dup ? -1 : ee; + s.tokmask = mask; + stamp(p, 1); + mbar_arrive(&s.route_done); + } + } + named_bar(4, 224); + e = s.expert; + } + if (e < 0) { // duplicate pair (both CTAs agree): let the speculative loads land; nothing was sent over DSMEM + if (warp == 1) { + if (lane == 0) + for (int b = 0; b < kStages; ++b) mbar_wait(&s.full[b], 0); + __syncwarp(); + } + mbar_wait(&s.x_full, 0); + cluster_wait(); + if (warp == 2) tmem_dealloc(tm, 512); + return; + } + tc_fence_after(); + // per-expert dequant scales, loaded now so that no global read sits behind the FC1 -> h -> FC2 chain + float ag = 0.f, sc = 0.f, a2 = 0.f; + if (HOIST_SCALES) { ag = p.g1_alpha[e]; sc = p.g1_scale_c[e]; a2 = p.g2_alpha[e]; } + + if (warp == 1) { + if (lane == 0) { + // ---------------- TMA producer: the rest of the weight stream (stages 0 .. kStages - 1 went out above) + const uint64_t pol = policy_evict_first(); + for (int st = kStages; st < kTotStages; ++st) { + mbar_wait(&s.empty[st % kStages], ((st / kStages) - 1) & 1); + issue_stage(st, e, pol); + } + } + __syncwarp(); + } else if (warp == 2) { + if (lane == 0) { + // ---------------- MMA issuer (tensor pipe: MMAs only) + const uint32_t idesc = make_idesc_nvf4(128u, 256u); + mbar_wait(&s.x_full, 0); + tc_fence_after(); + int n = 0; + for (int st = 0; st < kFc1Stages; ++st) { + const int b = st % kStages; + mbar_wait(&s.full[b], (st / kStages) & 1); + if (!SF_UTCCP_FC1 && !DIAG_NO_SFWAIT) mbar_wait(&s.sf_full[b], (st / kStages) & 1); + tc_fence_after(); + if (st == 0) stamp(p, 3); + if (DIAG_NO_MMA) { n += 4; tc_commit(&s.empty[b]); continue; } +#pragma unroll + for (int q = 0; q < 4; ++q, ++n) { + const int kb = st * 4 + q; // k-blocks in K order: one accumulator, the stock summation order + umma_nvf4(tm, make_sdesc(smem_u32(s.xa + (kb / 4) * 1024) + (kb % 4) * 32, 0, 0, 2), + make_sdesc(smem_u32(s.ring[b]) + q * 32, 0, 1024, 2), idesc, + tm + kColSfaX + (uint32_t)(kb % kSfaSlots) * 4, tm + kColSfb + (uint32_t)(n % kSfbSlots) * 8, kb > 0); + } + tc_commit(&s.empty[b]); + } + tc_commit(&s.fc1_done); + mbar_wait(&s.h_ready, 0); + tc_fence_after(); + stamp(p, 6); + for (int st = kFc1Stages; st < kTotStages; ++st) { + const int b = st % kStages; + mbar_wait(&s.full[b], (st / kStages) & 1); + if (!DIAG_NO_SFWAIT) mbar_wait(&s.sf_full[b], (st / kStages) & 1); + tc_fence_after(); + if (DIAG_NO_MMA) { tc_commit(&s.empty[b]); continue; } + const int j = st - kFc1Stages; +#pragma unroll + for (int kb = 0; kb < 4; ++kb) { + const int n2 = 64 + j * 4 + kb; // scale-factor slot, numbered on from FC1's 64 k-blocks + umma_nvf4(tm, make_sdesc(smem_u32(s.ha) + kb * 32, 0, 0, 2), + make_sdesc(smem_u32(s.ring[b]) + kb * 32, 0, 1024, 2), idesc, + tm + kColSfa + (uint32_t)(j * 4 + kb) * 4, tm + kColSfb + (uint32_t)(n2 % kSfbSlots) * 8, (j | kb) != 0); + } + tc_commit(&s.empty[b]); + } + tc_commit(&s.fc2_done); + if (PDL_TRIGGER) asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); + } + __syncwarp(); + } else if (warp >= 4) { + // ---------------- TMEM-side warpgroup: q = lane quarter (TMEM lanes 32 q .. 32 q + 31) + const int q = warp - 4; + const uint32_t lane_addr = (uint32_t)(q * 32) << 16; + mbar_wait(&s.x_full, 0); + // weight scale factors of every stage: k-block slot (8 columns) = [atom rows 0..127 | atom rows 128..255] + auto stage_sf = [&](int st, int n0, int nkb, int atom_stride, int half_off) { + const int b = st % kStages; + mbar_wait(&s.full[b], (st / kStages) & 1); + // the slots about to be overwritten were last read by the MMAs of stage st - 4 + if (st >= 4) mbar_wait(&s.empty[(st - 4) % kStages], ((st - 4) / kStages) & 1); + const uint8_t* sf = s.ring[b] + 32768; + for (int i = 0; i < nkb; ++i) { + const int tl = i / 2, kb = i % 2; + const uint8_t* a0 = atom_stride ? sf + tl * 2048 + kb * 512 : sf + i * 512; + const uint8_t* a1 = a0 + half_off; + const uint4 v0 = *reinterpret_cast(a0 + lane * 16); + const uint4 v1 = *reinterpret_cast(a1 + lane * 16); + const uint32_t r[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w}; + tmem_st_x8(tm + lane_addr + kColSfb + (uint32_t)((n0 + i) % kSfbSlots) * 8, r); + } + tmem_st_wait(); + tc_fence_before(); + __syncwarp(); + if (lane == 0) mbar_arrive(&s.sf_full[b]); + }; + // FC1 stage st: weight scale factors of k-blocks [4 st, 4 st + 4) (8 columns each: row blocks 2 r | 2 r + 1) and + // the x scale factors of the same k-blocks (row m -> lane m % 32, column m / 32; every quarter holds a copy) + for (int st = 0; st < kFc1Stages; ++st) { + const int b = st % kStages; + mbar_wait(&s.full[b], (st / kStages) & 1); + // the slots about to be overwritten were last read by the MMAs of stage st - 4 + if (st >= 4) mbar_wait(&s.empty[(st - 4) % kStages], ((st - 4) / kStages) & 1); + const uint8_t* sf = s.ring[b] + 32768; +#pragma unroll + for (int i = 0; i < 4; ++i) { + const uint4 v0 = *reinterpret_cast(sf + i * 512 + lane * 16); + const uint4 v1 = *reinterpret_cast(sf + 2048 + i * 512 + lane * 16); + const uint32_t r[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w}; + tmem_st_x8(tm + lane_addr + kColSfb + (uint32_t)((st * 4 + i) % kSfbSlots) * 8, r); + } + uint32_t xr[16]; +#pragma unroll + for (int i = 0; i < 16; ++i) { + const int kb = st * 4 + i / 4; + xr[i] = (i % 4 == 0 && lane < p.num_tokens) ? *reinterpret_cast(&s.xsf[lane][kb * 4]) : 0u; + } + tmem_st_x16(tm + lane_addr + kColSfaX + (uint32_t)((st * 4) % kSfaSlots) * 4, xr); + tmem_st_wait(); + tc_fence_before(); + __syncwarp(); + if (lane == 0) mbar_arrive(&s.sf_full[b]); + } + + // FC1 accumulator: TMEM lanes 0..7 (quarter 0) hold the token rows; quarter 0 drains all 256 columns + mbar_wait(&s.fc1_done, 0); + tc_fence_after(); + if (q == 0) { + if (lane == 0) stamp(p, 4); +#pragma unroll 1 + for (int c = 0; c < 4; ++c) { + uint32_t r[64]; + tmem_ld_x64(tm + (uint32_t)(c * 64), r); + tmem_ld_wait(); + if (lane < 8) { +#pragma unroll + for (int i = 0; i < 16; ++i) + *reinterpret_cast(s.acc1 + lane * kARow + c * 64 + 4 * i) = + make_uint4(r[4 * i], r[4 * i + 1], r[4 * i + 2], r[4 * i + 3]); + } + } + } + // FC2 scale factors of the first two FC2 stages while h is computed (all FC1 MMAs are done) + if (FC2SF_PRE) { + stage_sf(kFc1Stages, 64, 4, 0, 2048); + stage_sf(kFc1Stages + 1, 68, 4, 0, 2048); + } + if (threadIdx.x == 128) stamp(p, 15); + tc_fence_before(); + named_bar(2, 128); // quarter 0's accumulator rows are visible to the whole warpgroup + + if (threadIdx.x == 128) stamp(p, 5); + if (!HOIST_SCALES) { ag = p.g1_alpha[e]; sc = p.g1_scale_c[e]; } + { + // this CTA's 128 intermediates: (token row, 16-wide block, half). Accumulator columns of each 32-block: + // [0, 8) up 2m, [8, 16) gate 2m, [16, 24) up 2m + 1, [24, 32) gate 2m + 1 (m = 0..7 within the 16 indices) + const int it = threadIdx.x - 128; // 0..127 + const int row = it / 16, blk = (it / 2) % 8, hf = it % 2; + const float4* ar = reinterpret_cast(s.acc1 + row * kARow + 32 * blk + 4 * hf); + const float4 u_even = ar[0], g_even = ar[2], u_odd = ar[4], g_odd = ar[6]; + const float ga[8] = {g_even.x, g_odd.x, g_even.y, g_odd.y, g_even.z, g_odd.z, g_even.w, g_odd.w}; + const float ua[8] = {u_even.x, u_odd.x, u_even.y, u_odd.y, u_even.z, u_odd.z, u_even.w, u_odd.w}; + // trtllm-gen's SwiGLU + NVFP4 epilogue op for op (bit for bit, test_qwen3_5_moe.py); a = FC1 dequant, + // c = a x FC2 input scale: + // h' = (u a)(g sigmoid(g a)), sigmoid = rcp(ex2(-(g (a log2e))) + 1), block scale e4m3(max|h'| c / 6), FP4 (h' / scale) c + const float a_l2e = mul_ftz(ag, 1.4426950216293334961f); + float h[8]; + float amax = 0.f; +#pragma unroll + for (int i = 0; i < 8; ++i) { + const float e = ex2_approx(-(ga[i] * a_l2e)); + const float sg = rcp_approx(e + 1.0f); + const float v = fmaf(ua[i], ag, 0.0f) * (ga[i] * sg); + h[i] = v; + amax = fmaxf(amax, fabsf(v)); + } + amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, 1)); + const uint32_t sf = cvt_e4m3(fmaxf(0.f, fabsf(amax * sc)) * 0.16666667163372039795f); + const float sff = e4m3_to_f32(sf); + const float osc = sff == 0.f ? 0.f : rcp_approx(sff); + float qv[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) qv[i] = mul_ftz(h[i], osc) * sc; + const uint32_t packed = cvt_e2m1x8(qv); + const int gblk = 8 * (int)rank + blk; // 16-wide block over the 256 intermediates; k-blocks 2 r, 2 r + 1 + const int chunk = gblk / 2; + *reinterpret_cast(s.ha + row * 128 + ((chunk ^ row) * 16) + (gblk % 2) * 8 + hf * 4) = packed; + if (hf == 0) reinterpret_cast(&s.hsf[row][gblk / 4])[gblk % 4] = (uint8_t)sf; + } + named_bar(1, 128); + if (q == 0) { + // send this CTA's k-block of h (8 rows x two 16-B chunks) and its scale words into the peer's buffers + cluster_wait(); // phase 1: the peer's barriers are initialized + const uint32_t pbar = mapa(smem_u32(&s.h_peer), rank ^ 1u); + { + const int row = lane / 4, chunk = 4 * (int)rank + lane % 4; + const uint32_t off = (uint32_t)(row * 128 + ((chunk ^ row) * 16)); + st_async_v4(mapa(smem_u32(s.ha + off), rank ^ 1u), *reinterpret_cast(s.ha + off), pbar); + } + if (lane < 16) { + const int row = lane / 2, w = 2 * (int)rank + lane % 2; + st_async_b32(mapa(smem_u32(&s.hsf[row][w]), rank ^ 1u), s.hsf[row][w], pbar); + } + __syncwarp(); + } + mbar_wait(&s.h_peer, 0); // the peer's half of h and its scale words + if (threadIdx.x == 128) stamp(p, 5); + // h scale factors: 32 atoms (tile jt, k-block kb) of 4 columns; row m = 32 c + lane is nonzero only in its tile + { + uint32_t r[32]; +#pragma unroll 1 + for (int half = 0; half < 4; ++half) { +#pragma unroll + for (int i = 0; i < 32; ++i) { + const int a = half * 8 + i / 4, col = i % 4; // atom a = 4 jt + kb + const int jt = a / 4, kb = a % 4; + const int row = 32 * col + lane; + r[i] = (row / 8 == jt) ? s.hsf[row % 8][kb] : 0u; + } + tmem_st_x32(tm + lane_addr + kColSfa + half * 32, r); + } + tmem_st_wait(); + } + fence_proxy_async_smem(); + tc_fence_before(); + __syncwarp(); + if (lane == 0) mbar_arrive(&s.h_ready); + // FC2 weight scale factors (the MMA issuer waits for each stage's) + for (int st = kFc1Stages + (FC2SF_PRE ? 2 : 0); st < kTotStages; ++st) + stage_sf(st, 64 + (st - kFc1Stages) * 4, 4, 0, 2048); + + if (q < 2) fc2_drain(p, s, tm, q, lane, 0, a2, rank, e); + } + if (warp < 2) fc2_drain(p, s, tm, warp, lane, 1, a2, rank, e); + tc_fence_before(); + __syncthreads(); + if (threadIdx.x == 0) stamp(p, 13); + // every warp but warp 4 still owes its wait on the first cluster phase; phase 2 then ends the DSMEM exchange + if (warp != 4) cluster_wait(); + cluster_arrive_relaxed(); + cluster_wait(); + if (threadIdx.x == 0) stamp(p, 8); + if (warp == 2) tmem_dealloc(tm, 512); +} + +static_assert(sizeof(Smem) + 1024 == Q35_MKM_SMEM_BYTES, "the launcher's dynamic shared memory size"); + +} // namespace qwen_mkm_moe_tp4 + +} // namespace sglang diff --git a/sgl_nvidia_mega/qwen3_5/spec.py b/sgl_nvidia_mega/qwen3_5/spec.py index 78861069..aae586eb 100644 --- a/sgl_nvidia_mega/qwen3_5/spec.py +++ b/sgl_nvidia_mega/qwen3_5/spec.py @@ -62,7 +62,8 @@ REQUIREMENTS = Requirements( isas=frozenset({"sm_103"}), - tp_sizes=frozenset({8}), + # TP8 over the two 4-GPU trays of one NVL72 rack, TP4 on one tray + tp_sizes=frozenset({4, 8}), dtypes=frozenset({"bfloat16"}), # the decode CUDA graphs the covered step is tested with (eager steps too) decode_graph_backends=frozenset({"full", "disabled"}), diff --git a/test/kernels/test_qwen3_5_draft_front.py b/test/kernels/test_qwen3_5_draft_front.py index 6d9c4059..d3d1ca2d 100644 --- a/test/kernels/test_qwen3_5_draft_front.py +++ b/test/kernels/test_qwen3_5_draft_front.py @@ -3,26 +3,27 @@ """Slot draft_front's attention front (sm103/draft/front.py, 2.2's ATTN_FRONT) against SGLang's two launches it replaces, on one GPU: the fused q / k Gemma RMSNorm + RoPE + gate split, then the fused FP8 q / k / v + paged KV write. The FP8 q, the gate and the K / V cache -rows must be the same bytes, at the draft's widths (1 and 7 rows), in eager and in a -CUDA graph replayed with new positions and slots.""" +rows must be the same bytes, at the draft's widths (1 and 7 rows) and query heads a rank +(4 at TP8, 8 at TP4), in eager and in a CUDA graph replayed with new positions and +slots.""" import pytest import torch cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") -HEADS, KV_HEADS, HEAD_DIM, ROTARY = 4, 1, 256, 64 # the TP8 draft layer's shapes +KV_HEADS, HEAD_DIM, ROTARY = 1, 256, 64 # the draft layer's shapes at TP8 and TP4 SLOTS = 4096 -def _inputs(rows, seed): +def _inputs(rows, seed, heads): g = torch.Generator(device="cuda").manual_seed(seed) dev = "cuda" def randn(*shape, scale=1.0): return (torch.randn(*shape, generator=g, device=dev) * scale).to(torch.bfloat16) - q_gate = randn(rows, HEADS * 2 * HEAD_DIM, scale=2.0) + q_gate = randn(rows, heads * 2 * HEAD_DIM, scale=2.0) k = randn(rows, KV_HEADS * HEAD_DIM, scale=2.0) v = randn(rows, KV_HEADS * HEAD_DIM, scale=3.0) q_gate[0, :8] = -0.0 @@ -55,9 +56,9 @@ def _stock(inputs, kv): ) q_gate, k, v, q_w, k_w, cos_sin, positions, cache_loc, k_scale, v_scale = inputs - rows = q_gate.shape[0] + rows, heads = q_gate.shape[0], q_gate.shape[1] // (2 * HEAD_DIM) q, k_out, gate = fused_qk_gemma_rmsnorm_rope_gate( - q_gate, k, q_w, k_w, cos_sin, positions, 1e-6, HEADS, KV_HEADS, HEAD_DIM, ROTARY + q_gate, k, q_w, k_w, cos_sin, positions, 1e-6, heads, KV_HEADS, HEAD_DIM, ROTARY ) q8 = fused_fp8_qkv_kv_cache( q=q.view(rows, -1), @@ -78,7 +79,7 @@ def _plugin(inputs, kv): ) q_gate, k, v, q_w, k_w, cos_sin, positions, cache_loc, k_scale, v_scale = inputs - rows = q_gate.shape[0] + rows, heads = q_gate.shape[0], q_gate.shape[1] // (2 * HEAD_DIM) q8, k_none, gate = fused_qk_gemma_rmsnorm_rope_gate( q_gate, k, @@ -87,7 +88,7 @@ def _plugin(inputs, kv): cos_sin, positions, 1e-6, - HEADS, + heads, KV_HEADS, HEAD_DIM, ROTARY, @@ -108,10 +109,11 @@ def _same(a, b): @cuda +@pytest.mark.parametrize("heads", [4, 8]) @pytest.mark.parametrize("rows", [1, 7]) @pytest.mark.parametrize("seed", [0, 1, 2]) -def test_the_front_writes_what_sglangs_two_launches_write(rows, seed): - inputs = _inputs(rows, seed) +def test_the_front_writes_what_sglangs_two_launches_write(rows, seed, heads): + inputs = _inputs(rows, seed, heads) kv_s, kv_p = _caches(), _caches() q_s, gate_s = _stock(inputs, kv_s) q_p, gate_p = _plugin(inputs, kv_p) @@ -121,9 +123,10 @@ def test_the_front_writes_what_sglangs_two_launches_write(rows, seed): @cuda -def test_the_front_replays_in_a_graph_with_new_positions_and_slots(): +@pytest.mark.parametrize("heads", [4, 8]) +def test_the_front_replays_in_a_graph_with_new_positions_and_slots(heads): rows = 1 - inputs = _inputs(rows, 5) + inputs = _inputs(rows, 5, heads) kv_s, kv_p = _caches(), _caches() static = [t.clone() for t in inputs] _plugin(static, kv_p) # compile outside capture @@ -132,7 +135,7 @@ def test_the_front_replays_in_a_graph_with_new_positions_and_slots(): with torch.cuda.graph(graph): q_p, gate_p = _plugin(static, kv_p) for seed in (11, 12, 13): - fresh = _inputs(rows, seed) + fresh = _inputs(rows, seed, heads) for dst, src in zip(static, fresh): dst.copy_(src) graph.replay() diff --git a/test/kernels/test_qwen3_5_moe.py b/test/kernels/test_qwen3_5_moe.py index feed304a..e0a16e5d 100644 --- a/test/kernels/test_qwen3_5_moe.py +++ b/test/kernels/test_qwen3_5_moe.py @@ -6,8 +6,9 @@ GEMM SGLang runs (ReplicatedLinear, bf16), and its NVFP4 input against the trtllm runner's quantizer (flashinfer.fp4_quantize, linear scale layout), at 1-16 rows; - the fused routed experts (slot mkm), against FlashInfer's trtllm-gen NVFP4 MoE as - SGLang serves Qwen3.5 at TP8 (routing from the logits, RenormalizeNaive, deferred - finalize): each token's 10 expert rows and their weights, at 1, 4 and 7 rows; + SGLang serves Qwen3.5 at TP8 and TP4 (intermediate 128 / 256 a rank; routing from the + logits, RenormalizeNaive, deferred finalize): each token's 10 expert rows and their + weights, at 1, 4 and 7 rows; - the router front with the shared gate/up stage (slot route_shared, numeric): its logits and NVFP4 input bit for bit the plain launch's, and its gate/up rows against fp64 on SGLang's static FP8 quantization of the rows, at most 1.05 x the mean relative @@ -19,7 +20,6 @@ from sgl_nvidia_mega.qwen3_5.sm103.moe import mkm, router E, H, TOP_K = mkm.EXPERTS, mkm.HIDDEN, mkm.TOP_K -I = mkm.W13_ROWS // 2 # intermediate per rank X_SCALE = 448.0 * 6.0 / 12.0 # the static input scale of the routed experts H_SCALE = 30.0 / (448.0 * 6.0) # the static FC2 input scale @@ -59,29 +59,32 @@ def _experts(g, rows, cols, std): return packed, sf, scale -@pytest.fixture(scope="module") -def layer(): - """One TP8 Qwen3.5 MoE layer on the trtllm-gen weight layout, as SGLang loads it.""" +@pytest.fixture(scope="module", params=mkm.INTERMEDIATES, ids=lambda i: f"I{i}") +def layer(request): + """One TP8 (intermediate 128 a rank) or TP4 (256) Qwen3.5 MoE layer on the trtllm-gen + weight layout, as SGLang loads it.""" from sglang.srt.layers.quantization.utils import ( prepare_static_weights_for_trtllm_fp4_moe, ) + inter = request.param # intermediate per rank g = torch.Generator(device="cuda").manual_seed(11) - w13, sf13, s13 = _experts(g, 2 * I, H, 0.02) - w2, sf2, s2 = _experts(g, H, I, 0.02) + w13, sf13, s13 = _experts(g, 2 * inter, H, 0.02) + w2, sf2, s2 = _experts(g, H, inter, 0.02) w13, sf13, w2, sf2 = prepare_static_weights_for_trtllm_fp4_moe( - w13, w2, sf13, sf2, H, I, E, is_gated=True + w13, w2, sf13, sf2, H, inter, E, is_gated=True ) g1 = (1.0 / (X_SCALE * s13)).float() return { "w13": w13, - "w13_sf": sf13.view(torch.float8_e4m3fn).reshape(E, 2 * I, H // 16), + "w13_sf": sf13.view(torch.float8_e4m3fn).reshape(E, 2 * inter, H // 16), "w2": w2, - "w2_sf": sf2.view(torch.float8_e4m3fn).reshape(E, H, I // 16), + "w2_sf": sf2.view(torch.float8_e4m3fn).reshape(E, H, inter // 16), "g1": g1, "g1c": (g1 / H_SCALE).float(), "g2": (H_SCALE / s2).float(), "maps": mkm.weight_maps(w13, w2), + "inter": inter, } @@ -148,7 +151,7 @@ def _stock(layer, logits, x_fp4, x_sf): top_k=TOP_K, n_group=None, topk_group=None, - intermediate_size=I, + intermediate_size=layer["inter"], local_expert_offset=0, local_num_experts=E, routed_scaling_factor=None, diff --git a/test/unit/test_qwen3_5_spec.py b/test/unit/test_qwen3_5_spec.py index b5abc9f5..b87324ab 100644 --- a/test/unit/test_qwen3_5_spec.py +++ b/test/unit/test_qwen3_5_spec.py @@ -81,6 +81,10 @@ def test_the_documented_setup_is_accepted(): assert _refusal() == "" +def test_tp4_on_one_tray_is_accepted(): + assert _refusal(tp_size=4, nnodes=1) == "" + + BREAKABLE = with_phase( default_cuda_graph_config(), Phase.DECODE, backend=Backend.BREAKABLE ) @@ -89,7 +93,7 @@ def test_the_documented_setup_is_accepted(): @pytest.mark.parametrize( "kw, want", [ - (dict(tp_size=4), "tp_size 4"), + (dict(tp_size=2, nnodes=1), "tp_size 2"), (dict(pp_size=2), "pipeline parallelism is in use"), (dict(enable_dp_attention=True, dp_size=8), "DP attention is in use"), (dict(kv_cache_dtype="auto"), "kv_cache_dtype 'auto'"), @@ -149,7 +153,7 @@ def test_a_slot_set_that_cannot_run_is_refused_before_the_load( def test_every_reason_is_listed_at_once(): - message = _refusal(tp_size=4, kv_cache_dtype="auto", pp_size=2) + message = _refusal(tp_size=2, nnodes=1, kv_cache_dtype="auto", pp_size=2) assert message.count("\n - ") == 3, message From 43b40a0447788ff3d093a706531ba5c8b155bca6 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 10:01:00 -0700 Subject: [PATCH 24/27] [Perf] [GB300] qwen3_5: Opt-in multi_request: cover TP8's verify steps of 2 and 4 requests With multi_request in the slot list, TP8 also covers the verify steps of 2 and 4 requests (14 and 28 rows), each slot only up to the rows its kernels serve: route 16, hop 15, gdn_push, route_shared and mka 8, mkm 7, gdn_front and skv2 one request, gdn_verify and gdn_pdl every width. Off by default; TP4 keeps one request. The parity check counts the requests' own rows apart in graph steps. Each slot keeps its claim. --- sgl_nvidia_mega/qwen3_5/boundary/hop.py | 8 +- sgl_nvidia_mega/qwen3_5/moe/__init__.py | 6 +- sgl_nvidia_mega/qwen3_5/plan.py | 119 +++++++++++++++----- sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 33 ++++-- sgl_nvidia_mega/qwen3_5/runtime/parity.py | 40 +++++-- test/kernels/test_qwen3_5_gdn.py | 40 ++++--- test/unit/test_qwen3_5_runtime.py | 46 +++++++- test/unit/test_qwen3_5_spec.py | 45 ++++++++ 8 files changed, 274 insertions(+), 63 deletions(-) diff --git a/sgl_nvidia_mega/qwen3_5/boundary/hop.py b/sgl_nvidia_mega/qwen3_5/boundary/hop.py index de081384..e321cbd4 100644 --- a/sgl_nvidia_mega/qwen3_5/boundary/hop.py +++ b/sgl_nvidia_mega/qwen3_5/boundary/hop.py @@ -145,11 +145,15 @@ def build(model, ctx) -> Bound: from sglang.srt.megakernels.runtime import megakernel_agree from sglang.srt.runtime_context import get_parallel - from sgl_nvidia_mega.qwen3_5 import runtime + from sgl_nvidia_mega.qwen3_5 import plan, runtime from sgl_nvidia_mega.qwen3_5.sm103.hop import Hop if "hop" not in runtime.SLOTS: return Bound(reason="slot off") + # the widest step the hop covers: its mailbox's rows + rows = plan.widest(ctx.widths, plan.HOP_MAX_ROWS) + if not rows: + return Bound(reason=f"no width of {list(ctx.widths)} within its rows") m = model.model group = get_parallel().tp_group reasons = refusals(m, group.world_size) @@ -158,7 +162,7 @@ def build(model, ctx) -> Bound: if reasons: return Bound(reason="; ".join(reasons)) hop = Hop( - max_m=max(ctx.widths), + max_m=rows, eps=m.flashinfer_mnnvl_cutedsl_fusion.rms_epsilon, group=group.device_group, device=torch.device("cuda", torch.cuda.current_device()), diff --git a/sgl_nvidia_mega/qwen3_5/moe/__init__.py b/sgl_nvidia_mega/qwen3_5/moe/__init__.py index abf28068..e4719d0b 100644 --- a/sgl_nvidia_mega/qwen3_5/moe/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/moe/__init__.py @@ -71,12 +71,16 @@ def build(model, ctx) -> route.Bound: """Slots ``route`` and ``mkm`` for every MoE layer; the result is ``st.bound["moe"]``.""" + from sgl_nvidia_mega.qwen3_5 import plan from sgl_nvidia_mega.qwen3_5.runtime import SLOTS if "route" not in SLOTS: return route.Bound(reason="stock (SGLANG_NVIDIA_MEGAKERNEL_SLOTS)") + rows = plan.widest(ctx.widths, plan.ROUTE_MAX_ROWS) + if not rows: + return route.Bound(reason=f"no width of {list(ctx.widths)} within its rows") return route.build( - model, max(ctx.widths), fused="mkm" in SLOTS, shared="route_shared" in SLOTS + model, rows, fused="mkm" in SLOTS, shared="route_shared" in SLOTS ) diff --git a/sgl_nvidia_mega/qwen3_5/plan.py b/sgl_nvidia_mega/qwen3_5/plan.py index c10d2c9f..f306fd77 100644 --- a/sgl_nvidia_mega/qwen3_5/plan.py +++ b/sgl_nvidia_mega/qwen3_5/plan.py @@ -3,29 +3,47 @@ """Which kernel fills each slot of a covered step, per width (rows), fixed at build. A slot runs only where it is requested (``SGLANG_NVIDIA_MEGAKERNEL_SLOTS``), built and -inside its row bounds; every other op is SGLang's own. ``route`` (the router -GEMM and the routed experts' NVFP4 input in one launch) and ``mkm`` (on its outputs, the -top-10 and both expert GEMMs in one launch, up to 7 rows) cover every width, as do -``gdn_verify`` (the GatedDeltaNet verify chain with its launch choices changed), -``hop`` (the boundary's fused all-reduce points on the decode hop) and ``gdn_push`` -(numeric: out_proj as the hop's push GEMM, on gdn_verify's FP8 norm). ``gdn_front`` -(numeric: the GDN projections and conv in one launch, then gdn_pdl's verify) covers one -request of 7 rows, ``route_shared`` (numeric: the shared expert's gate/up in route's -launch) up to 8 rows. ``skv2`` runs the -full attention's core through the split-KV attention of SGLang branch -nvidia/qwen35-agentx-gb300, ``mka`` the fused attention prologue and epilogue around it, -and ``mka_push`` (with ``hop``) has the epilogue push its rows into the hop. +inside its row bounds; every other op is SGLang's own. The covered steps are the +target verify of 1, 2 or 4 requests (``REQUESTS``: SGLang's decode graphs of batch +sizes 1, 2 and 4); a width of 2 or 4 requests is covered only where a slot runs there. +``route`` (the router GEMM and the routed experts' NVFP4 input in one launch) runs up to +16 rows, ``mkm`` (on its outputs, the top-10 and both expert GEMMs in one launch) up to +7. ``gdn_verify`` (the GatedDeltaNet verify chain with its launch choices changed) covers +every width, and ``hop`` (the boundary's fused all-reduce points on the decode hop) the +widths where FlashInfer's low-latency kernels, which it equals bit for bit, run. +``gdn_push`` (numeric: out_proj as the hop's push GEMM, on gdn_verify's FP8 norm) runs up +to 8 rows. ``gdn_front`` (numeric: the GDN projections and conv in one launch, then +gdn_pdl's verify) covers one request of 7 rows, ``route_shared`` (numeric: the shared +expert's gate/up in route's launch) up to 8 rows. ``skv2`` runs the full attention's core +of one request through the split-KV attention of SGLang branch +nvidia/qwen35-agentx-gb300, ``mka`` the fused attention prologue and epilogue around it +(up to 8 rows), and ``mka_push`` (with ``hop``) has the epilogue push its rows into the +hop. """ from __future__ import annotations import msgspec -MAX_ROWS = 7 # the widest covered step: one request of 7 verify rows (TP8 bs1) +MAX_ROWS = 28 # the widest covered step: four requests of 7 verify rows (TP8 bs4) +REQUESTS = (1, 2, 4) # the requests of a covered step (SGLang's decode graph sizes) +# the TP sizes whose steps of more than one request are covered when requested (checked at +# TP8); at the others a covered step is one request +MULTI_REQUEST_TP = (8,) +# the name that requests them in the slot list +MULTI_REQUEST = "multi_request" GDN_FRONT_ROWS = 7 # the rows slot gdn_front's front is built for -ROUTE_SHARED_ROWS = ( - 8 # the rows slot route_shared's shared stage fits shared memory for -) +# Each slot's widest step: its launcher's bound (sm103/moe/router.py MAX_TOKENS, +# sm103/moe/mkm.py MAX_TOKENS, sm103/hop OPROJ_MAX_M, sm103/attention/mka.py), and for +# hop the rows up to which FlashInfer's MNNVL CuTe DSL fusion runs the one-shot +# all-reduce the hop equals bit for bit (its finalize runs one-shot up to 23 rows) +ROUTE_MAX_ROWS = 16 +MKM_MAX_ROWS = 7 +HOP_MAX_ROWS = 15 +GDN_PUSH_MAX_ROWS = 8 +# the rows slot route_shared's shared stage fits shared memory for +ROUTE_SHARED_ROWS = 8 +MKA_MAX_ROWS = 8 class Slots(msgspec.Struct, frozen=True): @@ -57,7 +75,7 @@ class Facts(msgspec.Struct, frozen=True): mkm: bool = False # the fused routed experts built for every MoE layer gdn_verify: bool = False # the GDN verify slot built for every GDN layer gdn_pdl: bool = False # with its dependent verify launch - hop: bool = False # the decode hop, built for the widest covered step + hop: bool = False # the decode hop, built for the widest step it covers gdn_push: bool = False # out_proj packed for the push GEMM on every GDN layer gdn_front: bool = False # in_proj_qkvz packed for the front on every GDN layer route_shared: bool = ( @@ -68,26 +86,71 @@ class Facts(msgspec.Struct, frozen=True): mka_push: bool = False # with the epilogue's push into the hop -def slots(rows: int, facts: Facts) -> Slots: +def slots(rows: int, facts: Facts, tokens_per_request: int = GDN_FRONT_ROWS) -> Slots: if not 1 <= rows <= MAX_ROWS: raise ValueError(f"{rows} rows are outside the covered widths 1-{MAX_ROWS}") + route = facts.route and rows <= ROUTE_MAX_ROWS + hop = facts.hop and rows <= HOP_MAX_ROWS + mka = facts.mka and rows <= MKA_MAX_ROWS return Slots( - route=facts.route, - mkm=facts.route and facts.mkm, + route=route, + mkm=route and facts.mkm and rows <= MKM_MAX_ROWS, gdn_verify=facts.gdn_verify, gdn_pdl=facts.gdn_verify and facts.gdn_pdl, - hop=facts.hop, - gdn_push=facts.gdn_verify and facts.hop and facts.gdn_push, + hop=hop, + gdn_push=facts.gdn_verify + and hop + and facts.gdn_push + and rows <= GDN_PUSH_MAX_ROWS, gdn_front=facts.gdn_verify and facts.gdn_pdl and facts.gdn_front and rows == GDN_FRONT_ROWS, - route_shared=facts.route and facts.route_shared and rows <= ROUTE_SHARED_ROWS, - skv2=facts.skv2, - mka=facts.mka, - mka_push=facts.mka and facts.mka_push and facts.hop, + route_shared=route and facts.route_shared and rows <= ROUTE_SHARED_ROWS, + # SKV2 attends one sequence + skv2=facts.skv2 and rows <= tokens_per_request, + mka=mka, + mka_push=mka and facts.mka_push and hop, ) -def plan(widths: tuple[int, ...], facts: Facts) -> dict[int, Slots]: - return {w: slots(w, facts) for w in widths} +def plan( + widths: tuple[int, ...], facts: Facts, tokens_per_request: int = GDN_FRONT_ROWS +) -> dict[int, Slots]: + return {w: slots(w, facts, tokens_per_request) for w in widths} + + +def multi_request(tp_size: int, requested: frozenset) -> bool: + """Whether steps of more than one request are covered: ``MULTI_REQUEST`` requested + at a TP size in ``MULTI_REQUEST_TP``.""" + return MULTI_REQUEST in requested and tp_size in MULTI_REQUEST_TP + + +def candidates( + offered: tuple[int, ...], tokens_per_request: int, multi: bool = True +) -> tuple[int, ...]: + """The widths SGLang offers (``BuildContext.widths``) of ``REQUESTS`` requests, or of + one request without ``multi``.""" + requests = REQUESTS if multi else (1,) + return tuple( + w + for w in offered + if w % tokens_per_request == 0 and w // tokens_per_request in requests + ) + + +def widest(widths: tuple[int, ...], max_rows: int) -> int: + """The widest of ``widths`` a slot bounded by ``max_rows`` rows serves (0: none).""" + return max((w for w in widths if w <= max_rows), default=0) + + +def covered( + widths: tuple[int, ...], facts: Facts, tokens_per_request: int +) -> tuple[int, ...]: + """The widths a build covers: one request always (with no slot it runs the stock + step), more requests only where a slot runs.""" + return tuple( + w + for w in widths + if w == tokens_per_request or slots(w, facts, tokens_per_request) != Slots() + ) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index f1a271b2..efd07423 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -27,6 +27,10 @@ "mka", "mka_push", ) +# Steps of more than one request (TP8's decode graphs of batch sizes 2 and 4), covered only +# when requested in the same variable: covering them costs the one-request step about +# 0.3% at TP8 +OPT_IN_COVERAGE: tuple[str, ...] = ("multi_request",) # The MTP draft's EAGLE entries and kernel groups (qwen3_5/draft), requested in the same # variable; the draft spec's build reads them, the target's ignores them DRAFT_ENTRIES: tuple[str, ...] = ("draft_loop", "draft_extend") @@ -58,6 +62,7 @@ def slot_reasons() -> list[str]: known = ( *SHIPPED_SLOTS, *OPT_IN_SLOTS, + *OPT_IN_COVERAGE, *DRAFT_ENTRIES, *DRAFT_SLOTS, ) @@ -115,6 +120,7 @@ def slot_reasons() -> list[str]: "deadline_s", "binding", ), + "srt/runtime_context.py": ("get_parallel", '"tp_size"'), } @@ -136,13 +142,19 @@ def build(model, ctx): from sgl_nvidia_mega.qwen3_5.runtime import parity from sgl_nvidia_mega.qwen3_5.runtime.executor import Qwen35Verify - if not ctx.widths: + multi = slot_plan.MULTI_REQUEST in SLOTS + if multi: + from sglang.srt.runtime_context import get_parallel + + multi = slot_plan.multi_request(get_parallel().tp_size, SLOTS) + widths = slot_plan.candidates(ctx.widths, ctx.tokens_per_request, multi) + if not widths: raise RuntimeError( f"NVIDIA megakernel: {ctx.tokens_per_request} rows a request leave no " "covered width" ) pctx = PackageContext( - widths=ctx.widths, + widths=widths, tokens_per_request=ctx.tokens_per_request, deadline_s=ctx.deadline_s, ) @@ -165,10 +177,17 @@ def build(model, ctx): mka=bound["attention"].mka, mka_push=bound["attention"].mka_push, ) - slots = slot_plan.plan(pctx.widths, facts) + covered = slot_plan.covered(pctx.widths, facts, pctx.tokens_per_request) + slots = slot_plan.plan(covered, facts, pctx.tokens_per_request) + # the slots of every width wider than one request, for the startup line + wider = "".join( + f"; {w} rows: {[n for n, ok in msgspec.structs.asdict(s).items() if ok]}" + for w, s in slots.items() + if w != pctx.tokens_per_request + ) built = msgspec.structs.asdict(facts) # every package's build agrees across ranks, so this raises on every rank alike - target_slots = SLOTS - set(DRAFT_ENTRIES) - set(DRAFT_SLOTS) + target_slots = SLOTS - set(DRAFT_ENTRIES) - set(DRAFT_SLOTS) - set(OPT_IN_COVERAGE) missing = sorted(name for name in target_slots if not built[name]) if missing: raise RuntimeError( @@ -178,7 +197,7 @@ def build(model, ctx): f"attention: {bound['attention'].reason or 'built'})" ) checker = ( - parity.Parity(pctx.widths, torch.cuda.current_device(), model) + parity.Parity(covered, torch.cuda.current_device(), model) if parity.ENABLED else None ) @@ -187,8 +206,8 @@ def build(model, ctx): model, slots, bound=bound, - widths=pctx.widths, + widths=covered, binding=ctx.binding, - summary=f"slots {on or 'none (stock)'}", + summary=f"slots {on or 'none (stock)'}{wider}", parity=checker, ) diff --git a/sgl_nvidia_mega/qwen3_5/runtime/parity.py b/sgl_nvidia_mega/qwen3_5/runtime/parity.py index e0e72e80..78958700 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/parity.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/parity.py @@ -11,8 +11,11 @@ floor of the same comparison. ``trace`` also hooks every module both forwards call and logs the first module call whose input or output differs (eager steps only). -Every line also reports how far the two forwards' outputs are apart, which is what a -numeric slot (not bitwise) is read by: per verify row, whether the two top-1 tokens +Graph lines also count the differing elements of the requests' own rows apart +(valid_*): a captured step of more requests replays for fewer, and SGLang's ops leave a +padded request's rows unwritten (its mamba slot is the pad slot), so both forwards read +whatever those rows held. Every line also reports how far the two forwards' outputs are +apart, which is what a numeric slot (not bitwise) is read by: per verify row, whether the two top-1 tokens differ and the |difference| of the two log-softmaxes at SGLang's top-1, and the max abs difference of the hidden rows. @@ -59,14 +62,15 @@ class Parity: """One rank's comparison counters per width: covered steps, the elements of the - logits and of the hidden rows whose bits differ, and the elements compared (zero - when a forward returned no such output, e.g. SGLang's autotune dummy run).""" + logits and of the hidden rows whose bits differ, the elements compared (zero when a + forward returned no such output, e.g. SGLang's autotune dummy run), and the + differing elements of the logits and hidden rows of requests with a mamba slot.""" def __init__(self, widths: tuple[int, ...], device, model=None): self.counters = { - w: torch.zeros(5, dtype=torch.int64, device=device) for w in widths + w: torch.zeros(7, dtype=torch.int64, device=device) for w in widths } - self.logged = {w: [0] * 5 for w in widths} + self.logged = {w: [0] * 7 for w in widths} # rows, top-1 differs, sum and max of |dlp|, hidden max abs; read at each flush self.distance = { w: torch.zeros(5, dtype=torch.float64, device=device) for w in widths @@ -101,9 +105,15 @@ def run(self, step, original, megakernel): if torch.cuda.is_current_stream_capturing(): counts = self.counters[step.rows] counts[0:1].add_(1) + valid = ( + None + if slots is None or step.rows % slots.numel() + else valid_rows(slots, step.rows) + ) for i, (a, b) in enumerate(pairs, start=1): counts[i : i + 1].add_(differing(a, b)) counts[i + 2 : i + 3].add_(0 if a is None else a.numel()) + counts[i + 4 : i + 5].add_(differing(a, b, valid)) if apart is not None: d = self.distance[step.rows] d[0:4:2].add_(apart[0:4:2]) @@ -144,7 +154,7 @@ def flush(self) -> None: logger.info( "NVIDIA megakernel parity%s graph target_verify rows=%d steps=%d " "logits_differ=%d hidden_differ=%d logits_compared=%d " - "hidden_compared=%d%s", + "hidden_compared=%d valid_logits_differ=%d valid_hidden_differ=%d%s", self.label, rows, *added, @@ -255,8 +265,15 @@ def _distance_text(apart) -> str: ) -def differing(a, b): - """The number of elements of a and b whose bits differ (a device scalar).""" +def valid_rows(slots: torch.Tensor, rows: int) -> torch.Tensor: + """[rows] bool: the rows of requests with a mamba slot (not the pad slot), for a step + of ``slots.numel()`` requests of equal rows.""" + return (slots >= 0).repeat_interleave(rows // slots.numel()) + + +def differing(a, b, valid=None): + """The number of elements of a and b whose bits differ (a device scalar); with + ``valid`` ([rows] bool), only in those rows.""" if a is None or b is None: if a is None and b is None: return 0 @@ -266,7 +283,10 @@ def differing(a, b): f"parity: {a.dtype}{tuple(a.shape)} against {b.dtype}{tuple(b.shape)}" ) bits = _BITS[a.element_size()] - return (a.contiguous().view(bits) != b.contiguous().view(bits)).sum() + differ = a.contiguous().view(bits) != b.contiguous().view(bits) + if valid is not None: + differ = differ & valid.view(-1, *([1] * (differ.dim() - 1))) + return differ.sum() def _clone(t): diff --git a/test/kernels/test_qwen3_5_gdn.py b/test/kernels/test_qwen3_5_gdn.py index bea4871d..c76b9af5 100644 --- a/test/kernels/test_qwen3_5_gdn.py +++ b/test/kernels/test_qwen3_5_gdn.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. """The kernels slot gdn_verify launches its own way, on one SM100-family GPU (GB300), -against SGLang's, bit for bit, at Qwen3.5's TP8 GatedDeltaNet shapes: +against SGLang's, bit for bit, at Qwen3.5's TP8 GatedDeltaNet shapes, for steps of 1, 2 +and 4 requests: - the causal conv update with the 128-channel tile and the token-major output, against ``causal_conv1d_update`` (output, conv states and intermediate windows), on the @@ -55,15 +56,22 @@ def _projection(bs, views, g): return qkvz[:, :D] if views else qkvz[:, :D].contiguous() +# a covered step's requests (slot gdn_verify covers B1, B2 and B4) and their slots +REQUESTS = (1, 2, 4) +STATE_SLOTS = (4, 7, 0, 8) +RING_SLOTS = (3, 5, 0, 8) +REPLAY_SLOTS = (1, 2, 4, 6) + + +@pytest.mark.parametrize("bs", REQUESTS) @pytest.mark.parametrize("views", [False, True]) @pytest.mark.parametrize("block_n", [256, 128]) -def test_conv_update_is_sglangs(views, block_n): +def test_conv_update_is_sglangs(views, block_n, bs): from sglang.kernels.ops.mamba.causal_conv1d_triton import causal_conv1d_update from sgl_nvidia_mega.qwen3_5.gdn.conv import conv_update - g = torch.Generator(device="cuda").manual_seed(block_n + views) - bs = 2 + g = torch.Generator(device="cuda").manual_seed(block_n + views + 10 * bs) rows = _projection(bs, views, g) weight = torch.randn(D, WIDTH, device="cuda", generator=g).to(torch.bfloat16) bias = torch.randn(D, device="cuda", generator=g).to(torch.bfloat16) @@ -73,8 +81,8 @@ def test_conv_update_is_sglangs(views, block_n): window = torch.zeros( SLOTS, DRAFT, D, WIDTH - 1, dtype=torch.bfloat16, device="cuda" ) - slots = torch.tensor([4, 7], dtype=torch.int32, device="cuda") - inter = torch.tensor([0, 1], dtype=torch.int32, device="cuda") + slots = torch.tensor(STATE_SLOTS[:bs], dtype=torch.int32, device="cuda") + inter = torch.arange(bs, dtype=torch.int32, device="cuda") x = rows.view(bs, DRAFT, D).transpose(1, 2) want_state, want_window = state.clone(), window.clone() want = causal_conv1d_update( @@ -105,15 +113,16 @@ def test_conv_update_is_sglangs(views, block_n): assert got.transpose(1, 2).is_contiguous() +@pytest.mark.parametrize("bs", REQUESTS) @pytest.mark.parametrize("views", [False, True]) -def test_norm_fp8_is_sglangs_norm_then_static_quant(views): +def test_norm_fp8_is_sglangs_norm_then_static_quant(views, bs): from sglang.kernels.ops.attention.fla.layernorm_gated import RMSNorm from sglang.kernels.ops.quantization.fp8_kernel import static_quant_fp8 from sgl_nvidia_mega.qwen3_5.gdn.norm_fp8 import rms_norm_gated_fp8 - g = torch.Generator(device="cuda").manual_seed(int(views)) - tokens = 2 * DRAFT + g = torch.Generator(device="cuda").manual_seed(int(views) + 10 * bs) + tokens = bs * DRAFT core = torch.randn(tokens * HV, V, device="cuda", generator=g).to(torch.bfloat16) qkvz = torch.randn(tokens, D + HV * V, device="cuda", generator=g).to( torch.bfloat16 @@ -137,8 +146,9 @@ def test_norm_fp8_is_sglangs_norm_then_static_quant(views): ) +@pytest.mark.parametrize("bs", REQUESTS) @pytest.mark.parametrize("pdl, gate", [(False, False), (True, False), (True, True)]) -def test_replayssm_verify_is_sglangs(pdl, gate): +def test_replayssm_verify_is_sglangs(pdl, gate, bs): """The slot's verify launch (32-wide value tiles on 2 warps, and with or without the dependent launch, waiting before the gates with ``gate``) against gdn_replayssm_spec_decode's verify launch: the output and this step's ring entries.""" @@ -153,8 +163,10 @@ def test_replayssm_verify_is_sglangs(pdl, gate): from sgl_nvidia_mega.qwen3_5.gdn import replayssm - g = torch.Generator(device="cuda").manual_seed(3 + int(pdl) + 2 * int(gate)) - bs, ring_len, slots = 2, 16, 9 + g = torch.Generator(device="cuda").manual_seed( + 3 + int(pdl) + 2 * int(gate) + 10 * bs + ) + ring_len, slots = 16, 9 tokens = bs * DRAFT def rand(*shape, dtype=torch.bfloat16): @@ -181,8 +193,8 @@ def rand(*shape, dtype=torch.bfloat16): cache_base = cache_base.to(torch.int32) is_flush = torch.zeros(slots, dtype=torch.int8, device="cuda") qsl = torch.arange(0, tokens + 1, DRAFT, dtype=torch.int32, device="cuda") - states = torch.tensor([3, 5], dtype=torch.int32, device="cuda") - replay = torch.tensor([1, 2], dtype=torch.int32, device="cuda") + states = torch.tensor(RING_SLOTS[:bs], dtype=torch.int32, device="cuda") + replay = torch.tensor(REPLAY_SLOTS[:bs], dtype=torch.int32, device="cuda") want_ring, got_ring = [t.clone() for t in ring], [t.clone() for t in ring] want, got = (q.new_empty(tokens, HV, V) for _ in range(2)) gdn_replayssm_spec_decode( diff --git a/test/unit/test_qwen3_5_runtime.py b/test/unit/test_qwen3_5_runtime.py index 51290e22..3f189869 100644 --- a/test/unit/test_qwen3_5_runtime.py +++ b/test/unit/test_qwen3_5_runtime.py @@ -56,6 +56,37 @@ def test_only_target_verify_steps_of_built_widths_are_covered(): assert built.plan(ids, _batch(), input_embeds=None) is None +def test_the_covered_widths_are_the_ones_the_build_returns(): + built = _verify(widths=(7, 14, 28)) + for rows in (7, 14, 28): + ids = torch.zeros(rows, dtype=torch.int64) + assert built.plan(ids, _batch()) == Step(mode="target_verify", rows=rows) + assert built.plan(torch.zeros(21, dtype=torch.int64), _batch()) is None + + +@pytest.mark.parametrize("tp_size, want", [(8, (7,)), (4, (7,))]) +def test_the_build_covers_more_requests_only_where_a_slot_runs( + monkeypatch, tp_size, want +): + monkeypatch.setattr(runtime, "SLOTS", frozenset()) + monkeypatch.setattr( + "sglang.srt.runtime_context.get_parallel", + lambda: SimpleNamespace(tp_size=tp_size), + ) + model = SimpleNamespace( + model=SimpleNamespace(start_layer=0, end_layer=0, layers=[]) + ) + # what SGLang offers for the spec's MAX_ROWS with 7 rows a request + ctx = SimpleNamespace( + widths=(7, 14, 21, 28), + tokens_per_request=7, + deadline_s=60.0, + binding="EagleBinding", + ) + built = runtime.build(model, ctx) + assert built.widths == want and built.summary() == "slots none (stock)" + + def test_the_build_hands_over_the_binding_and_every_entry_stays_sglangs(): from sglang.srt.megakernels.spec import ENTRIES @@ -363,6 +394,18 @@ def test_parity_compares_bits_not_values(): parity.differing(a, None) +def test_parity_counts_the_requests_own_rows_apart(): + # two requests of 2 rows; the second is a padded one (the pad slot) + valid = parity.valid_rows(torch.tensor([3, -1]), 4) + assert valid.tolist() == [True, True, False, False] + a = torch.zeros(4, 3) + b = a.clone() + b[3] = 1.0 # a padded row + assert int(parity.differing(a, b)) == 3 and int(parity.differing(a, b, valid)) == 0 + b[0, 1] = 1.0 # a request's row + assert int(parity.differing(a, b, valid)) == 1 + + @pytest.mark.parametrize("mode", ["1", "control"]) def test_an_eager_parity_step_runs_sglangs_forward_first( eager, monkeypatch, caplog, mode @@ -437,7 +480,7 @@ def run(): def test_graph_parity_counts_are_logged_once_each(eager, caplog): checker = parity.Parity((7,), "cpu") - checker.counters[7] += torch.tensor([3, 0, 5, 63, 0]) + checker.counters[7] += torch.tensor([3, 0, 5, 63, 0, 0, 2]) checker.distance[7] += torch.tensor( [21.0, 2.0, 0.42, 0.25, 0.5], dtype=torch.float64 ) @@ -448,6 +491,7 @@ def test_graph_parity_counts_are_logged_once_each(eager, caplog): assert lines == [ "NVIDIA megakernel parity graph target_verify rows=7 steps=3 " "logits_differ=0 hidden_differ=5 logits_compared=63 hidden_compared=0 " + "valid_logits_differ=0 valid_hidden_differ=2 " "rows_compared=21 top1_differ=2 dlp_mean=0.02 dlp_max=0.25 hidden_dist_max=0.5" ] # the distance restarts with every flush diff --git a/test/unit/test_qwen3_5_spec.py b/test/unit/test_qwen3_5_spec.py index b87324ab..846fdf42 100644 --- a/test/unit/test_qwen3_5_spec.py +++ b/test/unit/test_qwen3_5_spec.py @@ -183,6 +183,51 @@ def test_the_slot_plan_covers_one_request_of_7_rows(): plan.slots(plan.MAX_ROWS + 1, plan.Facts()) +def _on(slots): + return sorted(name for name in plan.Slots.__struct_fields__ if getattr(slots, name)) + + +def test_the_slot_plan_gates_each_slot_by_the_rows_its_kernels_serve(): + every = plan.Facts(**dict.fromkeys(plan.Facts.__struct_fields__, True)) + assert _on(plan.slots(7, every)) == sorted(plan.Slots.__struct_fields__) + # two requests: the 16-row router front, the GDN chain and the hop's one-shot rows + assert _on(plan.slots(14, every)) == ["gdn_pdl", "gdn_verify", "hop", "route"] + # four requests: past the router front's and the one-shot all-reduce's rows + assert _on(plan.slots(28, every)) == ["gdn_pdl", "gdn_verify"] + # SGLang offers every multiple of 7 up to MAX_ROWS; three requests are not covered + offered = tuple(range(7, plan.MAX_ROWS + 1, 7)) + assert plan.candidates(offered, 7) == (7, 14, 28) + assert plan.candidates(offered, 7, multi=False) == (7,) + # more than one request only when requested, and only at TP8; TP4 keeps the + # one-request step it is checked at + assert plan.multi_request(8, frozenset({"multi_request"})) + assert not plan.multi_request(8, frozenset()) + assert not plan.multi_request(4, frozenset({"multi_request"})) + assert plan.covered((7, 14, 28), every, 7) == (7, 14, 28) + # one request is always covered, more only where a slot runs + assert plan.covered((7, 14, 28), plan.Facts(), 7) == (7,) + assert plan.covered((7, 14, 28), plan.Facts(route=True, mkm=True), 7) == (7, 14) + attention = plan.Facts(skv2=True, mka=True, mka_push=True, hop=True) + assert plan.covered((7, 14, 28), attention, 7) == (7, 14) + assert _on(plan.slots(14, attention)) == ["hop"] + # the router front and the hop's mailbox are built for the widest width they serve + assert plan.widest((7, 14, 28), plan.ROUTE_MAX_ROWS) == 14 + assert plan.widest((7, 14, 28), plan.HOP_MAX_ROWS) == 14 + assert plan.widest((28,), plan.HOP_MAX_ROWS) == 0 + + +def test_the_slot_plans_row_bounds_are_the_kernels(): + from sgl_nvidia_mega.qwen3_5.sm103 import hop + from sgl_nvidia_mega.qwen3_5.sm103.attention import mka + from sgl_nvidia_mega.qwen3_5.sm103.moe import mkm, router + + assert plan.ROUTE_MAX_ROWS == router.MAX_TOKENS + assert plan.ROUTE_SHARED_ROWS == router.SHARED_MAX_M + assert plan.MKM_MAX_ROWS == mkm.MAX_TOKENS + assert plan.GDN_PUSH_MAX_ROWS == hop.OPROJ_MAX_M + assert plan.MKA_MAX_ROWS == mka.MAX_ROWS + + def test_a_numeric_slot_turns_the_claim_into_its_tolerances(): from sglang.srt.megakernels.spec import NumericsKind From 01d1711b58ae3c16376a8ed122c39c0738348c36 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 10:02:00 -0700 Subject: [PATCH 25/27] [Perf] [GB300] qwen3_5: Slot mkm_wide: the fused routed experts of 8-16 rows in one launch The 16-token engine of SGLang branch nvidia/qwen35-agentx-gb300 (a CTA per (token, top-k) slot, no clusters) at the TP8 expert shapes runs slot mkm's steps of 8 to 16 rows. Slot mkm_wide (with mkm and multi_request; vacuous where no covered step is wider than 7 rows); bitwise against FlashInfer's trtllm-gen MoE. --- sgl_nvidia_mega/qwen3_5/moe/__init__.py | 6 +- sgl_nvidia_mega/qwen3_5/moe/route.py | 27 +- sgl_nvidia_mega/qwen3_5/plan.py | 8 +- sgl_nvidia_mega/qwen3_5/runtime/__init__.py | 18 +- sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py | 50 +- .../qwen3_5/sm103/moe/src/mkm_moe_m16.cu | 1176 +++++++++++++++++ test/kernels/test_qwen3_5_moe.py | 23 +- test/unit/test_qwen3_5_spec.py | 16 +- 8 files changed, 1294 insertions(+), 30 deletions(-) create mode 100644 sgl_nvidia_mega/qwen3_5/sm103/moe/src/mkm_moe_m16.cu diff --git a/sgl_nvidia_mega/qwen3_5/moe/__init__.py b/sgl_nvidia_mega/qwen3_5/moe/__init__.py index e4719d0b..1a785ab4 100644 --- a/sgl_nvidia_mega/qwen3_5/moe/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/moe/__init__.py @@ -80,7 +80,11 @@ def build(model, ctx) -> route.Bound: if not rows: return route.Bound(reason=f"no width of {list(ctx.widths)} within its rows") return route.build( - model, rows, fused="mkm" in SLOTS, shared="route_shared" in SLOTS + model, + rows, + fused="mkm" in SLOTS, + shared="route_shared" in SLOTS, + wide="mkm_wide" in SLOTS, ) diff --git a/sgl_nvidia_mega/qwen3_5/moe/route.py b/sgl_nvidia_mega/qwen3_5/moe/route.py index d4beebde..9b1362d6 100644 --- a/sgl_nvidia_mega/qwen3_5/moe/route.py +++ b/sgl_nvidia_mega/qwen3_5/moe/route.py @@ -95,15 +95,20 @@ class Bound(msgspec.Struct, frozen=True): route: bool = False # the router front stands in for every layer's gate mkm: dict = msgspec.field(default_factory=dict) # layer id -> its weight TMA maps - identity: object = None # [70] int32 0, 1, ...: slot mkm's rows are in expert order + mkm_wide: bool = False # slot mkm_wide: mkm's 16-token engine built + # [160] int32 0, 1, ...: slot mkm's rows are in expert order + identity: object = None shared: bool = False # slot route_shared: the shared gate/up in the router launch reason: str = "" # why a requested slot is not built -def build(model, max_rows: int, fused: bool, shared: bool = False) -> Bound: +def build( + model, max_rows: int, fused: bool, shared: bool = False, wide: bool = False +) -> Bound: """The router front for every MoE layer, compiled for 1..max_rows rows, with - ``fused`` slot mkm's kernel and weight maps, and with ``shared`` slot route_shared's - launch; or why a layer cannot take them.""" + ``fused`` slot mkm's kernel and weight maps (with ``wide`` its 16-token engine too, + for the rows past 7 up to max_rows), and with ``shared`` slot route_shared's launch; + or why a layer cannot take them.""" from sgl_nvidia_mega.qwen3_5.moe import mkm from sgl_nvidia_mega.qwen3_5.sm103.moe import mkm as mkm_kernel from sgl_nvidia_mega.qwen3_5.sm103.moe import router @@ -127,7 +132,14 @@ def build(model, max_rows: int, fused: bool, shared: bool = False) -> Bound: reason = mkm.unsupported(mlp) if reason: return Bound(route=True, reason=f"mkm: layer {i}: {reason}") - mkm_kernel.build(device, layers[0].experts.w2_weight.shape[-1] * 2) + intermediate = layers[0].experts.w2_weight.shape[-1] * 2 + # the 16-token engine serves the rows past 7 where this rank's intermediate has one + wide = ( + wide + and max_rows > mkm_kernel.MAX_TOKENS + and intermediate in mkm_kernel.WIDE_ENGINES + ) + mkm_kernel.build(device, intermediate, max_rows if wide else mkm_kernel.MAX_TOKENS) return Bound( route=True, mkm={ @@ -136,8 +148,11 @@ def build(model, max_rows: int, fused: bool, shared: bool = False) -> Bound: ) for mlp in layers }, + mkm_wide=wide, identity=torch.arange( - mkm_kernel.MAX_TOKENS * mkm_kernel.TOP_K, dtype=torch.int32, device="cuda" + mkm_kernel.M16_MAX_TOKENS * mkm_kernel.TOP_K, + dtype=torch.int32, + device="cuda", ), shared=shared, ) diff --git a/sgl_nvidia_mega/qwen3_5/plan.py b/sgl_nvidia_mega/qwen3_5/plan.py index f306fd77..89c52f91 100644 --- a/sgl_nvidia_mega/qwen3_5/plan.py +++ b/sgl_nvidia_mega/qwen3_5/plan.py @@ -8,7 +8,7 @@ sizes 1, 2 and 4); a width of 2 or 4 requests is covered only where a slot runs there. ``route`` (the router GEMM and the routed experts' NVFP4 input in one launch) runs up to 16 rows, ``mkm`` (on its outputs, the top-10 and both expert GEMMs in one launch) up to -7. ``gdn_verify`` (the GatedDeltaNet verify chain with its launch choices changed) covers +7, and with ``mkm_wide`` (its 16-token engine) up to 16. ``gdn_verify`` (the GatedDeltaNet verify chain with its launch choices changed) covers every width, and ``hop`` (the boundary's fused all-reduce points on the decode hop) the widths where FlashInfer's low-latency kernels, which it equals bit for bit, run. ``gdn_push`` (numeric: out_proj as the hop's push GEMM, on gdn_verify's FP8 norm) runs up @@ -39,6 +39,7 @@ # all-reduce the hop equals bit for bit (its finalize runs one-shot up to 23 rows) ROUTE_MAX_ROWS = 16 MKM_MAX_ROWS = 7 +MKM_WIDE_MAX_ROWS = 16 HOP_MAX_ROWS = 15 GDN_PUSH_MAX_ROWS = 8 # the rows slot route_shared's shared stage fits shared memory for @@ -73,6 +74,7 @@ class Facts(msgspec.Struct, frozen=True): route: bool = False # the router front built for every MoE layer mkm: bool = False # the fused routed experts built for every MoE layer + mkm_wide: bool = False # with their 16-token engine gdn_verify: bool = False # the GDN verify slot built for every GDN layer gdn_pdl: bool = False # with its dependent verify launch hop: bool = False # the decode hop, built for the widest step it covers @@ -94,7 +96,9 @@ def slots(rows: int, facts: Facts, tokens_per_request: int = GDN_FRONT_ROWS) -> mka = facts.mka and rows <= MKA_MAX_ROWS return Slots( route=route, - mkm=route and facts.mkm and rows <= MKM_MAX_ROWS, + mkm=route + and facts.mkm + and rows <= (MKM_WIDE_MAX_ROWS if facts.mkm_wide else MKM_MAX_ROWS), gdn_verify=facts.gdn_verify, gdn_pdl=facts.gdn_verify and facts.gdn_pdl, hop=hop, diff --git a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py index efd07423..75ec1c75 100644 --- a/sgl_nvidia_mega/qwen3_5/runtime/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/runtime/__init__.py @@ -17,6 +17,7 @@ OPT_IN_SLOTS: tuple[str, ...] = ( "route", "mkm", + "mkm_wide", "gdn_verify", "gdn_pdl", "hop", @@ -78,6 +79,13 @@ def slot_reasons() -> list[str]: ) if "mkm" in SLOTS and "route" not in SLOTS: reasons.append(f"{SLOTS_ENV}: slot 'mkm' runs on slot 'route''s outputs") + if "mkm_wide" in SLOTS and "mkm" not in SLOTS: + reasons.append(f"{SLOTS_ENV}: slot 'mkm_wide' is slot 'mkm''s 16-token engine") + if "mkm_wide" in SLOTS and "multi_request" not in SLOTS: + reasons.append( + f"{SLOTS_ENV}: slot 'mkm_wide' serves the steps of more than one request " + "('multi_request')" + ) if "route_shared" in SLOTS and "route" not in SLOTS: reasons.append( f"{SLOTS_ENV}: slot 'route_shared' runs in slot 'route''s launch" @@ -167,6 +175,7 @@ def build(model, ctx): facts = slot_plan.Facts( route=bound["moe"].route, mkm=bool(bound["moe"].mkm), + mkm_wide=bound["moe"].mkm_wide, gdn_verify=bound["gdn"].gdn_verify, gdn_pdl=bound["gdn"].gdn_pdl, hop=bound["boundary"].hop is not None, @@ -188,7 +197,14 @@ def build(model, ctx): built = msgspec.structs.asdict(facts) # every package's build agrees across ranks, so this raises on every rank alike target_slots = SLOTS - set(DRAFT_ENTRIES) - set(DRAFT_SLOTS) - set(OPT_IN_COVERAGE) - missing = sorted(name for name in target_slots if not built[name]) + # mkm's 16-token engine has nothing to serve where no covered step is wider than 7 rows + vacuous = ( + {"mkm_wide"} + if slot_plan.widest(pctx.widths, slot_plan.MKM_WIDE_MAX_ROWS) + <= slot_plan.MKM_MAX_ROWS + else set() + ) + missing = sorted(name for name in target_slots - vacuous if not built[name]) if missing: raise RuntimeError( f"NVIDIA megakernel: requested slots {missing} built no width (MoE: " diff --git a/sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py b/sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py index 14e52730..576a4e69 100644 --- a/sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py +++ b/sgl_nvidia_mega/qwen3_5/sm103/moe/mkm.py @@ -5,7 +5,9 @@ reading the trtllm-gen weight layout in place. It writes the deferred-finalize rows (row ``t * 10 + k`` is expert ``k`` of token ``t``) and the expert weights, as the trtllm runner's deferred finalize takes them. One engine per intermediate size a rank holds: -128 at TP8 (``src/mkm_moe.cu``), 256 at TP4 (``src/mkm_moe_tp4.cu``). +128 at TP8 (``src/mkm_moe.cu``), 256 at TP4 (``src/mkm_moe_tp4.cu``), a 2-CTA cluster per +selectable expert. At TP8, 8 to 16 rows run the 16-token engine (``src/mkm_moe_m16.cu``: +a CTA per (token, top-k) slot; a slot whose expert an earlier slot holds exits). """ from __future__ import annotations @@ -44,6 +46,14 @@ class Engine(NamedTuple): } INTERMEDIATES = tuple(ENGINES) SMEM_BYTES = ENGINES[128].smem_bytes +# the 16-token engines, per intermediate size a rank holds (TP8) +M16_MAX_TOKENS = 16 +M16_CTAS = M16_MAX_TOKENS * TOP_K # a CTA per (token, top-k) slot; the unused ones exit +WIDE_ENGINES = { + 128: Engine( + "mkm_moe_m16.cu", "q35_mkm_moe_m16", 221184, "CU_TENSOR_MAP_SWIZZLE_64B" + ), +} class Params(ctypes.Structure): @@ -68,8 +78,10 @@ class Params(ctypes.Structure): @cache -def _kernel(device_index: int, intermediate: int = 128) -> jit.Kernel: - engine = ENGINES[intermediate] +def _kernel( + device_index: int, intermediate: int = 128, wide: bool = False +) -> jit.Kernel: + engine = (WIDE_ENGINES if wide else ENGINES)[intermediate] major, minor = torch.cuda.get_device_capability(device_index) cubin = jit.compile_source( (Path(__file__).parent / "src" / engine.source).read_text(), @@ -84,9 +96,19 @@ def _kernel(device_index: int, intermediate: int = 128) -> jit.Kernel: return jit.load(cubin, engine.kernel, (THREADS, 1, 1), engine.smem_bytes) -def build(device_index: int, intermediate: int = 128) -> None: - """Compile the engine for ``intermediate`` per rank (no compile inside a step).""" +def max_tokens(intermediate: int) -> int: + """The widest rows an engine serves for ``intermediate`` per rank.""" + return M16_MAX_TOKENS if intermediate in WIDE_ENGINES else MAX_TOKENS + + +def build( + device_index: int, intermediate: int = 128, max_rows: int = MAX_TOKENS +) -> None: + """Compile the engines that serve ``intermediate`` per rank up to ``max_rows`` rows + (no compile inside a step).""" _kernel(device_index, intermediate) + if max_rows > MAX_TOKENS: + _kernel(device_index, intermediate, wide=True) def _map( @@ -157,19 +179,21 @@ def mkm_moe( g2_alpha: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: """(gemm2_out bf16 [M * 10, 4096], expert_weights bf16 [M, 10]) for logits [M, 512] - bf16 (M <= 7) and the router front's x_fp4 [M, 2048] / x_sf [M, 256] (uint8, linear - e4m3); w13_sf [512, 2 I, 256] / w2_sf [512, 4096, I / 16] e4m3 viewed as uint8 (the - engine follows I); g1_alpha, g1_scale_c, g2_alpha [512] fp32.""" + bf16 (M <= 7, at TP8 M <= 16) and the router front's x_fp4 [M, 2048] / x_sf [M, 256] + (uint8, linear e4m3); w13_sf [512, 2 I, 256] / w2_sf [512, 4096, I / 16] e4m3 viewed + as uint8 (the engine follows I); g1_alpha, g1_scale_c, g2_alpha [512] fp32.""" m = logits.shape[0] - if not 1 <= m <= MAX_TOKENS: - raise ValueError(f"mkm: {m} rows (1-{MAX_TOKENS})") intermediate = w2_sf.shape[2] * 16 + if not 1 <= m <= max_tokens(intermediate): + raise ValueError(f"mkm: {m} rows (1-{max_tokens(intermediate)})") + wide = m > MAX_TOKENS device = logits.device gemm2_out = torch.empty((m * TOP_K, HIDDEN), dtype=torch.bfloat16, device=device) expert_weights = torch.empty((m, TOP_K), dtype=torch.bfloat16, device=device) topk_ids = torch.empty((m, TOP_K), dtype=torch.int32, device=device) swizzle = jit.driver().CUtensorMapSwizzle.CU_TENSOR_MAP_SWIZZLE_128B - x_map = _map(x_fp4, m, HIDDEN // 2, 128, 8, swizzle) + # the x rows past m read as zeros: 8 rows a box, 16 for the 16-token engine + x_map = _map(x_fp4, m, HIDDEN // 2, 128, 16 if wide else 8, swizzle) p = Params( logits=logits.data_ptr(), expert_weights=expert_weights.data_ptr(), @@ -184,8 +208,8 @@ def mkm_moe( num_tokens=m, ) jit.launch( - _kernel(device.index, intermediate), - (CTAS, 1, 1), + _kernel(device.index, intermediate, wide), + (M16_CTAS if wide else CTAS, 1, 1), [*maps, x_map, p], torch.cuda.current_stream().cuda_stream, ) diff --git a/sgl_nvidia_mega/qwen3_5/sm103/moe/src/mkm_moe_m16.cu b/sgl_nvidia_mega/qwen3_5/sm103/moe/src/mkm_moe_m16.cu new file mode 100644 index 00000000..6a8b938e --- /dev/null +++ b/sgl_nvidia_mega/qwen3_5/sm103/moe/src/mkm_moe_m16.cu @@ -0,0 +1,1176 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Fused small-M NVFP4 MoE expert phase for Qwen3.5 TP8 decode verify (M <= 16 tokens, top-10, 512 experts, +// hidden 4096, intermediate 128 per rank): the top-10 of the router logits in the trtllm-gen routing's order and +// weights, FC1, SwiGLU, the NVFP4 requantization and FC2 in one launch over the trtllm-gen weight layout. Writes the +// deferred-finalize rows (row t * 10 + k, identity permutation) and the expert weights. +// +// The 16-token engine of SGLang branch nvidia/qwen35-agentx-gb300's TP4 work (one CTA per (token, top-k) slot, no +// clusters; a slot whose expert an earlier slot holds exits) at the TP8 expert shapes: one FC1 pass over the expert's +// 256 W13 rows, and FC2 stages of two 256-row W2 tiles of 64 B, staged as the TP8 7-row engine (mkm_moe.cu) stages +// them. +// +// Built with NVRTC: the kernel has C linkage (q35_mkm_moe_m16), and the launcher's dynamic shared memory size +// (Q35_MKM_SMEM_BYTES) is checked against the layout at compile time. +#include +#include +#include + +// NVRTC has no , or +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; +typedef unsigned long long uintptr_t; +typedef struct __align__(64) { + uint64_t opaque[16]; +} CUtensorMap; +#ifndef INFINITY +#define INFINITY __int_as_float(0x7f800000) +#endif +// Minimal SM100 (tcgen05 / TMA / mbarrier) PTX helpers for the small-M NVFP4 MoE engine. + +namespace qwen35 { + +__device__ __forceinline__ uint32_t smem_u32(const void* p) { + return static_cast(__cvta_generic_to_shared(p)); +} + +// ---------------------------------------------------------------- mbarrier +__device__ __forceinline__ void mbar_init(uint64_t* bar, uint32_t count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" ::"r"(smem_u32(bar)), "r"(count) : "memory"); +} +__device__ __forceinline__ void fence_mbar_init() { + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); +} +__device__ __forceinline__ void mbar_expect_tx(uint64_t* bar, uint32_t bytes) { + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(smem_u32(bar)), "r"(bytes) : "memory"); +} +__device__ __forceinline__ void mbar_arrive(uint64_t* bar) { + asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];" ::"r"(smem_u32(bar)) : "memory"); +} +__device__ __forceinline__ bool mbar_try_wait(uint64_t* bar, uint32_t phase) { + uint32_t ok; + asm volatile( + "{\n .reg .pred p;\n mbarrier.try_wait.parity.shared::cta.b64 p, [%1], %2;\n selp.u32 %0, 1, 0, p;\n}\n" + : "=r"(ok) + : "r"(smem_u32(bar)), "r"(phase) + : "memory"); + return ok != 0; +} +__device__ __forceinline__ void mbar_wait(uint64_t* bar, uint32_t phase) { + while (!mbar_try_wait(bar, phase)) { + } +} + +// ---------------------------------------------------------------- TMA / bulk copies +__device__ __forceinline__ void tma_load_2d(void* dst, const void* tmap, uint64_t* bar, int c0, int c1) { + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%3, %4}], [%2];" :: + "r"(smem_u32(dst)), + "l"(tmap), "r"(smem_u32(bar)), "r"(c0), "r"(c1) + : "memory"); +} +__device__ __forceinline__ void bulk_load(void* dst, const void* src, uint32_t bytes, uint64_t* bar) { + asm volatile("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"( + smem_u32(dst)), + "l"(src), "r"(bytes), "r"(smem_u32(bar)) + : "memory"); +} +__device__ __forceinline__ void prefetch_tmap(const void* tmap) { + asm volatile("prefetch.tensormap [%0];" ::"l"(tmap) : "memory"); +} +// generic-proxy smem writes -> visible to the async proxy (UMMA / UTCCP operand reads) +__device__ __forceinline__ void fence_proxy_async_smem() { + asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); +} + +// ---------------------------------------------------------------- tcgen05 +__device__ __forceinline__ void tmem_alloc(uint32_t* dst_smem, uint32_t ncols) { + asm volatile("tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" ::"r"(smem_u32(dst_smem)), + "r"(ncols) + : "memory"); +} +__device__ __forceinline__ void tmem_relinquish() { + asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;" ::: "memory"); +} +__device__ __forceinline__ void tmem_dealloc(uint32_t taddr, uint32_t ncols) { + asm volatile("tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" ::"r"(taddr), "r"(ncols) : "memory"); +} +__device__ __forceinline__ void tc_fence_before() { + asm volatile("tcgen05.fence::before_thread_sync;" ::: "memory"); +} +__device__ __forceinline__ void tc_fence_after() { + asm volatile("tcgen05.fence::after_thread_sync;" ::: "memory"); +} +// arrive on `bar` when all prior tcgen05 async ops issued by this thread complete +__device__ __forceinline__ void tc_commit(uint64_t* bar) { + asm volatile("tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%0];" ::"r"(smem_u32(bar)) + : "memory"); +} +// smem -> TMEM copy of one 512-byte scale-factor chunk (32 rows x 16 B), broadcast to the 4 lane quarters +__device__ __forceinline__ void utccp_32x128b_warpx4(uint32_t taddr, uint64_t sdesc) { + asm volatile("tcgen05.cp.cta_group::1.32x128b.warpx4 [%0], %1;" ::"r"(taddr), "l"(sdesc) : "memory"); +} +// D(tmem) (+)= A(smem) * B(smem)^T with NVFP4 block-16 scales from TMEM +__device__ __forceinline__ void umma_nvf4(uint32_t tmem_d, uint64_t adesc, uint64_t bdesc, uint32_t idesc, + uint32_t tmem_sfa, uint32_t tmem_sfb, uint32_t accumulate) { + asm volatile( + "{\n .reg .pred p;\n setp.ne.b32 p, %4, 0;\n" + " tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16 [%0], %1, %2, %3, [%5], [%6], p;\n}\n" ::"r"( + tmem_d), + "l"(adesc), "l"(bdesc), "r"(idesc), "r"(accumulate), "r"(tmem_sfa), "r"(tmem_sfb) + : "memory"); +} +// each warp reads its 32-lane quarter: 32 lanes x 16 consecutive 32-bit columns +__device__ __forceinline__ void tmem_ld_32x32b_x16(uint32_t taddr, uint32_t (&r)[16]) { + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x16.b32 {%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15}, [%16];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]), "=r"(r[4]), "=r"(r[5]), "=r"(r[6]), "=r"(r[7]), + "=r"(r[8]), "=r"(r[9]), "=r"(r[10]), "=r"(r[11]), "=r"(r[12]), "=r"(r[13]), "=r"(r[14]), "=r"(r[15]) + : "r"(taddr) + : "memory"); +} +// L2 prefetch of a contiguous global range (bytes multiple of 16, 16-byte aligned) +__device__ __forceinline__ void bulk_prefetch_l2(const void* gptr, uint32_t bytes) { + asm volatile("cp.async.bulk.prefetch.L2.global [%0], %1;" ::"l"(gptr), "r"(bytes) : "memory"); +} +__device__ __forceinline__ void tmem_ld_wait() { asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); } +// each warp writes its 32-lane quarter: 32 lanes x 16 consecutive 32-bit columns +__device__ __forceinline__ void tmem_st_32x32b_x16(uint32_t taddr, const uint32_t (&r)[16]) { + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x16.b32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16};" ::"r"( + taddr), + "r"(r[0]), "r"(r[1]), "r"(r[2]), "r"(r[3]), "r"(r[4]), "r"(r[5]), "r"(r[6]), "r"(r[7]), "r"(r[8]), "r"(r[9]), + "r"(r[10]), "r"(r[11]), "r"(r[12]), "r"(r[13]), "r"(r[14]), "r"(r[15]) + : "memory"); +} +__device__ __forceinline__ void tmem_st_wait() { asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); } + +// ---------------------------------------------------------------- descriptors +// SM100 shared-memory matrix descriptor (version 1). layout: 0 none, 2 SW128, 4 SW64, 6 SW32. +__device__ __forceinline__ uint64_t make_sdesc(uint32_t saddr, uint32_t lbo, uint32_t sbo, uint32_t layout) { + uint64_t d = 0; + d |= (uint64_t)((saddr >> 4) & 0x3FFF); + d |= (uint64_t)((lbo >> 4) & 0x3FFF) << 16; + d |= (uint64_t)((sbo >> 4) & 0x3FFF) << 32; + d |= (uint64_t)1 << 46; // version + d |= (uint64_t)(layout & 0x7) << 61; + return d; +} +// Block-scaled instruction descriptor: E2M1 x E2M1, UE4M3 scales, both K-major, F32 accumulate. +__host__ __device__ constexpr uint32_t make_idesc_nvf4(uint32_t M, uint32_t N) { + return (1u << 7) // a_format E2M1 + | (1u << 10) // b_format E2M1 + | ((N >> 3) << 17) // n_dim + | (0u << 23) // scale format UE4M3 + | ((M >> 4) << 24); // m_dim +} + +} // namespace qwen35 + +namespace sglang { + +namespace qwen_mkm_moe_m16 { + +using namespace qwen35; + + +constexpr int kH = 4096, kE = 512, kTopK = 10, kMaxTok = 16; +constexpr int kI = 128; // intermediates per expert per rank (TP8) +constexpr int kN1 = 2 * kI; // W13 rows per expert (gate + up) +constexpr int kStageBytes = 36864; // 32 KB weights + 4 KB scale factors +constexpr int kStages = 4; +constexpr int kW13BoxRows = 128; // rows of one W13 TMA box; the host tensor map must use the same +constexpr int kFc1Stages = 16; // the expert's 256 W13 rows, 4 k-blocks each (full K = 4096) +constexpr int kFc2Stages = 8; // two tiles of 256 W2 rows x 64 B each +constexpr int kTotStages = kFc1Stages + kFc2Stages; +constexpr int kThreads = 256; +// TMEM columns. FC1: accumulator [0,256), x SFA ring [256,320) (16 slots x 4), SFB ring [384,512) (16 slots x 8). +// FC2: accumulator [0,256), h SFA [256,320) (16 atoms x 4), SFB ring [384,512) (16 slots x 8). +constexpr uint32_t kColSfaX = 256, kColSfa = 256, kColSfb = 384; +constexpr int kSfbSlots = 16, kSfaSlots = 16; +constexpr int kARow = 260; // FC1 accumulator row stride in floats: 16 B aligned, rows on distinct banks +#ifndef L2_EVICT_FIRST // stream expert weights with an L2 evict-first policy (they are read once per layer) +#define L2_EVICT_FIRST 1 +#endif +#ifndef TOPK_RANK +#define TOPK_RANK 0 +#endif +#ifndef HOIST_SCALES +#define HOIST_SCALES 1 +#endif +#ifndef SF_UTCCP_FC1 +#define SF_UTCCP_FC1 0 +#endif +#ifndef SPEC_STAGES // ring stages issued for the speculated expert before the routing verdict (rest: after it) +#define SPEC_STAGES 4 +#endif +#ifndef STAGE_STAMPS // DIAG: per-stage issue (slots 64 + st) and landing (16 + st, seen by warp 4) stamps +#define STAGE_STAMPS 0 +#endif +constexpr int kTraceStride = STAGE_STAMPS ? 128 : 16; +#ifndef SMEM_SPACE // 1: shared-memory accesses through Smem compile to shared-space instructions +#define SMEM_SPACE 1 +#endif +#ifndef PF_SET // L2 prefetch of the stages that wait for a ring slot at: 1 = start, 4 = FC2, 8 = FC2 pass 1 +#define PF_SET 5 +#endif +#ifndef PF_N1 // stages prefetched at the start trigger +#define PF_N1 4 +#endif +#ifndef PF_N2 // stages prefetched at the FC2 trigger +#define PF_N2 4 +#endif +#ifndef TAILV // FC2 drains: bit 0 = last pass on all 8 warps; bit 1 = TMEM loads double-buffered; bit 2 = coalesced stores +#define TAILV 3 +#endif +#define TAIL8 (TAILV & 1) +#define DRAIN_PIPE ((TAILV >> 1) & 1) +#define DRAIN_SMEM ((TAILV >> 2) & 1) +#ifndef EARLY_SF2 // FC2 stages whose scale factors are staged before the h epilogue (old order: 2) +#define EARLY_SF2 0 +#endif +#ifndef FC2_PF // FC2 tiles (beyond the first kStages) prefetched to L2 ahead of the ring; 0 = off +#define FC2_PF 0 +#endif +#ifndef FC2_PF_LEAD // issue the FC2 prefetch when the producer reaches FC1 stage kFc1Stages - FC2_PF_LEAD +#define FC2_PF_LEAD 4 +#endif + +struct Params { + const uint16_t* logits; // [M, 512] bf16 router logits (router_gemm_quant output) + uint16_t* expert_weights; // [M, 10] bf16 out: DynBlock softmax-over-top-10 weights (CTA 0 writes) + int32_t* topk_ids; // [M, 10] out (CTA 0 writes) + const uint8_t* x_sf; // [M, 256] linear e4m3 + const uint8_t* w13_sf; // [E, 2 row blocks x 64 k blocks x 512 B] (128x4 interleaved) + const uint8_t* w2_sf; // [E, 32 row blocks x 2 k blocks x 512 B] + const float* g1_alpha; // [E] gate dequant + const float* g1_scale_c; // [E] up dequant * FC2 input quant scale + const float* g2_alpha; // [E] + __nv_bfloat16* out; // [M * 10, 4096] + long long* trace; // optional [grid, 16] %globaltimer stamps + int num_tokens; +}; + +struct __align__(1024) Smem { + uint8_t ring[kStages][kStageBytes]; + alignas(1024) uint8_t xa[32 * 1024]; // x: per 128-B K chunk c, 2 KB = SW128 atoms of rows 0..7 | rows 8..15 + alignas(1024) uint8_t ha[16 * 1024]; // h: 8 copies of (rows 0..7 | rows 8..15) x 128 B (K = 128: 64 B), SW128 + float acc1[16 * kARow]; // FC1 accumulator rows 0..15 (tokens) x 256 columns + uint8_t xsf[16][256]; // x scale bytes, full K (row t = token t), by bulk copy + uint32_t hsf[16][2]; // h scale words: token t, k-block kb (4 e4m3 bytes) + uint64_t full[kStages], empty[kStages], sf_full[kStages]; + uint64_t x_full, fc1_done, acc_ready, h_ready, fc2_pass[2], fc2_free, route_done; + uint32_t tmem; + int expert; + uint32_t tokmask; + int kslot[16]; + alignas(16) uint32_t cand[8][32]; // read as uint4 by the rank-based top-k + int32_t ids[16][kTopK]; +}; + +__device__ __forceinline__ uint64_t gtime() { + uint64_t t; + asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t)::"memory"); + return t; +} +__device__ __forceinline__ uint32_t cluster_rank() { + uint32_t r; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(r)); + return r; +} +__device__ __forceinline__ void cluster_arrive() { + asm volatile("barrier.cluster.arrive.release.aligned;" ::: "memory"); +} +// no release: the phase only orders DSMEM traffic that mbarriers already completed, so pending global stores +// (stamps, routing outputs, output rows) need not drain before the arrive +__device__ __forceinline__ void cluster_arrive_relaxed() { + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); +} +__device__ __forceinline__ void cluster_wait() { + asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); +} +__device__ __forceinline__ uint32_t mapa(uint32_t saddr, uint32_t rank) { + uint32_t r; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(r) : "r"(saddr), "r"(rank)); + return r; +} +// smem (this CTA) -> smem (peer CTA in the cluster), completes tx bytes on the peer's mbarrier +__device__ __forceinline__ void bulk_s2s(uint32_t dst_cluster, uint32_t src, uint32_t bytes, uint32_t bar_cluster) { + asm volatile("cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"( + dst_cluster), + "r"(src), "r"(bytes), "r"(bar_cluster) + : "memory"); +} +// registers -> the peer CTA's shared memory; completes the bytes on the peer's mbarrier +__device__ __forceinline__ void st_async_v4(uint32_t dst, const uint4& v, uint32_t bar) { + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 [%0], {%1, %2, %3, %4}, [%5];" ::"r"(dst), + "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w), "r"(bar) + : "memory"); +} +__device__ __forceinline__ void st_async_b32(uint32_t dst, uint32_t v, uint32_t bar) { + asm volatile("st.async.shared::cluster.mbarrier::complete_tx::bytes.b32 [%0], %1, [%2];" ::"r"(dst), "r"(v), "r"(bar) + : "memory"); +} +__device__ __forceinline__ void tmem_st_x4(uint32_t taddr, const uint4& v) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x4.b32 [%0], {%1, %2, %3, %4};" ::"r"(taddr), "r"(v.x), "r"(v.y), + "r"(v.z), "r"(v.w) + : "memory"); +} +// bar.sync counts arrivals per warp execution: reconverge first, so a warp whose lane 0 just ran a stamp or an +// mbarrier arrive cannot arrive twice and release the barrier early +__device__ __forceinline__ void named_bar(int id, int n) { + __syncwarp(); + asm volatile("bar.sync %0, %1;" ::"r"(id), "r"(n) : "memory"); +} +__device__ __forceinline__ void tmem_ld_x32(uint32_t taddr, uint32_t (&r)[32]) { + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x32.b32 {%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16,%17,%18,%19," + "%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,%31}, [%32];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]), "=r"(r[4]), "=r"(r[5]), "=r"(r[6]), "=r"(r[7]), "=r"(r[8]), + "=r"(r[9]), "=r"(r[10]), "=r"(r[11]), "=r"(r[12]), "=r"(r[13]), "=r"(r[14]), "=r"(r[15]), "=r"(r[16]), + "=r"(r[17]), "=r"(r[18]), "=r"(r[19]), "=r"(r[20]), "=r"(r[21]), "=r"(r[22]), "=r"(r[23]), "=r"(r[24]), + "=r"(r[25]), "=r"(r[26]), "=r"(r[27]), "=r"(r[28]), "=r"(r[29]), "=r"(r[30]), "=r"(r[31]) + : "r"(taddr) + : "memory"); +} +__device__ __forceinline__ void tmem_ld_x64(uint32_t taddr, uint32_t (&r)[64]) { + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x64.b32 {%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16,%17,%18,%19," + "%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,%31,%32,%33,%34,%35,%36,%37,%38,%39,%40,%41,%42,%43,%44,%45,%46,%47," + "%48,%49,%50,%51,%52,%53,%54,%55,%56,%57,%58,%59,%60,%61,%62,%63}, [%64];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]), "=r"(r[4]), "=r"(r[5]), "=r"(r[6]), "=r"(r[7]), "=r"(r[8]), + "=r"(r[9]), "=r"(r[10]), "=r"(r[11]), "=r"(r[12]), "=r"(r[13]), "=r"(r[14]), "=r"(r[15]), "=r"(r[16]), + "=r"(r[17]), "=r"(r[18]), "=r"(r[19]), "=r"(r[20]), "=r"(r[21]), "=r"(r[22]), "=r"(r[23]), "=r"(r[24]), + "=r"(r[25]), "=r"(r[26]), "=r"(r[27]), "=r"(r[28]), "=r"(r[29]), "=r"(r[30]), "=r"(r[31]), "=r"(r[32]), + "=r"(r[33]), "=r"(r[34]), "=r"(r[35]), "=r"(r[36]), "=r"(r[37]), "=r"(r[38]), "=r"(r[39]), "=r"(r[40]), + "=r"(r[41]), "=r"(r[42]), "=r"(r[43]), "=r"(r[44]), "=r"(r[45]), "=r"(r[46]), "=r"(r[47]), "=r"(r[48]), + "=r"(r[49]), "=r"(r[50]), "=r"(r[51]), "=r"(r[52]), "=r"(r[53]), "=r"(r[54]), "=r"(r[55]), "=r"(r[56]), + "=r"(r[57]), "=r"(r[58]), "=r"(r[59]), "=r"(r[60]), "=r"(r[61]), "=r"(r[62]), "=r"(r[63]) + : "r"(taddr) + : "memory"); +} +__device__ __forceinline__ void tmem_st_x8(uint32_t taddr, const uint32_t (&r)[8]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x8.b32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8};" ::"r"(taddr), "r"(r[0]), + "r"(r[1]), "r"(r[2]), "r"(r[3]), "r"(r[4]), "r"(r[5]), "r"(r[6]), "r"(r[7]) + : "memory"); +} +__device__ __forceinline__ void tmem_st_x16(uint32_t taddr, const uint32_t (&r)[16]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x16.b32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16};" :: + "r"(taddr), + "r"(r[0]), "r"(r[1]), "r"(r[2]), "r"(r[3]), "r"(r[4]), "r"(r[5]), "r"(r[6]), "r"(r[7]), "r"(r[8]), "r"(r[9]), + "r"(r[10]), "r"(r[11]), "r"(r[12]), "r"(r[13]), "r"(r[14]), "r"(r[15]) + : "memory"); +} +__device__ __forceinline__ void tmem_st_x32(uint32_t taddr, const uint32_t (&r)[32]) { + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x32.b32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15,%16,%17,%18," + "%19,%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,%31,%32};" ::"r"(taddr), + "r"(r[0]), "r"(r[1]), "r"(r[2]), "r"(r[3]), "r"(r[4]), "r"(r[5]), "r"(r[6]), "r"(r[7]), "r"(r[8]), "r"(r[9]), + "r"(r[10]), "r"(r[11]), "r"(r[12]), "r"(r[13]), "r"(r[14]), "r"(r[15]), "r"(r[16]), "r"(r[17]), "r"(r[18]), + "r"(r[19]), "r"(r[20]), "r"(r[21]), "r"(r[22]), "r"(r[23]), "r"(r[24]), "r"(r[25]), "r"(r[26]), "r"(r[27]), + "r"(r[28]), "r"(r[29]), "r"(r[30]), "r"(r[31]) + : "memory"); +} +__device__ __forceinline__ float ex2_approx(float x) { + float y; + asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} +__device__ __forceinline__ float mul_ftz(float a, float b) { + float y; + asm("mul.ftz.f32 %0, %1, %2;" : "=f"(y) : "f"(a), "f"(b)); + return y; +} +__device__ __forceinline__ float rcp_approx(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} +__device__ __forceinline__ uint32_t cvt_e4m3(float a) { + uint32_t out; + asm("{\n .reg .b16 pair;\n .reg .f32 zero;\n mov.f32 zero, 0f00000000;\n" + " cvt.rn.satfinite.e4m3x2.f32 pair, zero, %1;\n cvt.u32.u16 %0, pair;\n}" + : "=r"(out) + : "f"(a)); + return out & 0xFFu; +} +__device__ __forceinline__ float e4m3_to_f32(uint32_t v) { + float out; + asm("{\n .reg .b16 pair;\n .reg .b32 h2;\n .reg .b16 lo, hi;\n cvt.u16.u32 pair, %1;\n" + " cvt.rn.f16x2.e4m3x2 h2, pair;\n mov.b32 {lo, hi}, h2;\n cvt.f32.f16 %0, lo;\n}" + : "=f"(out) + : "r"(v)); + return out; +} +__device__ __forceinline__ uint32_t cvt_e2m1x8(const float* v) { + uint32_t out; + asm("{\n .reg .b8 b0, b1, b2, b3;\n" + " cvt.rn.satfinite.e2m1x2.f32 b0, %2, %1;\n cvt.rn.satfinite.e2m1x2.f32 b1, %4, %3;\n" + " cvt.rn.satfinite.e2m1x2.f32 b2, %6, %5;\n cvt.rn.satfinite.e2m1x2.f32 b3, %8, %7;\n" + " mov.b32 %0, {b0, b1, b2, b3};\n}" + : "=r"(out) + : "f"(v[0]), "f"(v[1]), "f"(v[2]), "f"(v[3]), "f"(v[4]), "f"(v[5]), "f"(v[6]), "f"(v[7])); + return out; +} +// ---- router top-k, bit-exact with FlashInfer's single-block routing (routingCustom, RenormalizeNaive on bf16): +// key = bf16 order (cub float twiddle) above 65535 - expert, i.e. the higher score first, the smaller expert on ties. +__device__ __forceinline__ uint32_t topk_key(uint32_t bits, uint32_t expert) { + const uint32_t twiddled = bits ^ ((bits & 0x8000u) ? 0xFFFFu : 0x8000u); + return (twiddled << 16) | (65535u - expert); +} +__device__ __forceinline__ void topk_unkey(uint32_t key, uint32_t& bits, uint32_t& expert) { + expert = 65535u - (key & 0xFFFFu); + const uint32_t twiddled = key >> 16; + bits = twiddled ^ ((twiddled & 0x8000u) ? 0x8000u : 0xFFFFu); +} +__device__ __forceinline__ uint32_t max16(const uint32_t (&k)[16]) { + const uint32_t a = max(max(max(k[0], k[1]), max(k[2], k[3])), max(max(k[4], k[5]), max(k[6], k[7]))); + const uint32_t b = max(max(max(k[8], k[9]), max(k[10], k[11])), max(max(k[12], k[13]), max(k[14], k[15]))); + return max(a, b); +} +__device__ __forceinline__ uint32_t warp_max_u32(uint32_t v) { + uint32_t r; + asm volatile("redux.sync.max.u32 %0, %1, 0xffffffff;" : "=r"(r) : "r"(v)); + return r; +} +__device__ __forceinline__ uint32_t warp_sum_u32(uint32_t v) { + uint32_t r; + asm volatile("redux.sync.add.u32 %0, %1, 0xffffffff;" : "=r"(r) : "r"(v)); + return r; +} +__device__ __forceinline__ uint32_t warp_topk_of_lanes(uint32_t v, uint32_t lane) { + uint32_t mine = 0; +#pragma unroll + for (uint32_t k = 0; k < kTopK; ++k) { + const uint32_t top = warp_max_u32(v); + v = v == top ? 0u : v; + mine = lane == k ? top : mine; + } + return mine; +} +__device__ __forceinline__ uint32_t count_greater(const uint32_t* s_cand, uint32_t v) { + uint32_t r = 0; +#pragma unroll + for (int j = 0; j < 32; j += 4) { + const uint4 q = *reinterpret_cast(s_cand + j); + r += (q.x > v) + (q.y > v) + (q.z > v) + (q.w > v); + } + return r; +} +// Top-10 keys of 512 (16 per lane), lane k < 10 holding the k-th largest (keys unique and nonzero). Ranks come from +// all-pairs compares over shared memory rather than chains of warp reductions. +__device__ __forceinline__ uint32_t warp_top10(uint32_t (&keys)[16], uint32_t lane, uint32_t* s_cand) { + // bound = the 10th largest lane maximum; every top-10 key is >= bound + const uint32_t m = max16(keys); + s_cand[lane] = m; + __syncwarp(); + const uint32_t r = count_greater(s_cand, m); + const uint32_t bound = __shfl_sync(0xffffffffu, m, __ffs(__ballot_sync(0xffffffffu, r == kTopK - 1)) - 1); + uint32_t count = 0; +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) count += keys[i] >= bound; + // exclusive prefix of count over the lanes, from the ballots of its bits (count <= 16) + const uint32_t lt = (1u << lane) - 1u; + uint32_t offset = 0, total = 0; +#pragma unroll + for (int b = 0; b < 5; ++b) { + const uint32_t bb = __ballot_sync(0xffffffffu, (count >> b) & 1u); + offset += (uint32_t)__popc(bb & lt) << b; + total += (uint32_t)__popc(bb) << b; + } + if (total <= 32) { + __syncwarp(); // every lane has read the lane maxima +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) { + if (keys[i] >= bound) s_cand[offset++] = keys[i]; + } + if (lane >= total) s_cand[lane] = 0u; + __syncwarp(); + const uint32_t cand = s_cand[lane]; + const uint32_t rk = count_greater(s_cand, cand); + __syncwarp(); + if (lane < total && rk < kTopK) s_cand[rk] = cand; + __syncwarp(); + return lane < kTopK ? s_cand[lane] : 0u; + } + uint32_t mine = 0; +#pragma unroll 1 + for (uint32_t k = 0; k < kTopK; ++k) { + const uint32_t top = warp_max_u32(max16(keys)); +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) keys[i] = keys[i] == top ? 0u : keys[i]; + mine = lane == k ? top : mine; + } + return mine; +} +// Top-10 keys of 512 (16 per lane), lane k holding the k-th largest (keys unique and nonzero). +__device__ __forceinline__ uint32_t warp_top10_redux(uint32_t (&keys)[16], uint32_t lane, uint32_t* s_cand) { + uint32_t v = max16(keys); + uint32_t bound = 0; +#pragma unroll + for (uint32_t k = 0; k < kTopK; ++k) { + bound = warp_max_u32(v); + v = v == bound ? 0u : v; + } + uint32_t count = 0; +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) count += keys[i] >= bound; + const uint32_t total = warp_sum_u32(count); + if (total <= 32) { + uint32_t offset = count; +#pragma unroll + for (uint32_t d = 1; d < 32; d <<= 1) { + const uint32_t up = __shfl_up_sync(0xffffffffu, offset, d); + offset += lane >= d ? up : 0u; + } + offset -= count; +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) { + if (keys[i] >= bound) s_cand[offset++] = keys[i]; + } + __syncwarp(); + const uint32_t cand = lane < total ? s_cand[lane] : 0u; + __syncwarp(); + return warp_topk_of_lanes(cand, lane); + } + uint32_t mine = 0; +#pragma unroll 1 + for (uint32_t k = 0; k < kTopK; ++k) { + const uint32_t top = warp_max_u32(max16(keys)); +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) keys[i] = keys[i] == top ? 0u : keys[i]; + mine = lane == k ? top : mine; + } + return mine; +} +// FlashInfer calcSoftmax (bf16 scores) as built with -use_fast_math: expf -> ex2.approx.ftz of x * log2(e), +// division -> div.full.ftz; reductions are cooperative_groups::reduce like the reference. +__device__ __forceinline__ float fast_expf(float x) { + float y; + asm("{\n .reg .f32 t;\n mul.ftz.f32 t, %1, 0f3FB8AA3B;\n ex2.approx.ftz.f32 %0, t;\n}" : "=f"(y) : "f"(x)); + return y; +} +__device__ __forceinline__ float fast_div(float a, float b) { + float y; + asm("div.full.ftz.f32 %0, %1, %2;" : "=f"(y) : "f"(a), "f"(b)); + return y; +} +__device__ __forceinline__ uint16_t softmax_topk(uint16_t score_bits, uint32_t lane) { + namespace cg = cooperative_groups; + const auto warp = cg::tiled_partition<32>(cg::this_thread_block()); + const float si = __uint_as_float((uint32_t)score_bits << 16); + float max_score = -INFINITY; + if (lane < kTopK) max_score = si >= max_score ? si : max_score; + max_score = cg::reduce(warp, max_score, cg::greater()); + float sum_score = 0.f, new_score = 0.f; + if (lane < kTopK) { + new_score = fast_expf(__fsub_rn(si, max_score)); + sum_score = __fadd_rn(sum_score, new_score); + } + sum_score = cg::reduce(warp, sum_score, cg::plus()); + uint16_t out = score_bits; + if (lane < kTopK) { + const __nv_bfloat16 b = __float2bfloat16_rn(fast_div(new_score, sum_score)); + out = *reinterpret_cast(&b); + } + return out; +} + +__device__ __forceinline__ uint64_t policy_evict_first() { + uint64_t pol; + asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(pol)); + return pol; +} +__device__ __forceinline__ void tma_load_2d_w(void* dst, const void* tmap, uint64_t* bar, int c0, int c1, uint64_t pol) { + if (L2_EVICT_FIRST) + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint" + " [%0], [%1, {%3, %4}], [%2], %5;" ::"r"(smem_u32(dst)), + "l"(tmap), "r"(smem_u32(bar)), "r"(c0), "r"(c1), "l"(pol) + : "memory"); + else + tma_load_2d(dst, tmap, bar, c0, c1); +} +__device__ __forceinline__ void bulk_load_w(void* dst, const void* src, uint32_t bytes, uint64_t* bar, uint64_t pol) { + if (L2_EVICT_FIRST) + asm volatile("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [%0], [%1], %2, [%3], %4;" ::"r"( + smem_u32(dst)), + "l"(src), "r"(bytes), "r"(smem_u32(bar)), "l"(pol) + : "memory"); + else + bulk_load(dst, src, bytes, bar); +} + +__device__ __forceinline__ void stamp(const Params& p, int slot) { + if (p.trace) p.trace[blockIdx.x * kTraceStride + slot] = (long long)gtime(); +} +// stamp after `dep` is available (the branch needs its value; keys are never 0) +__device__ __forceinline__ void stamp_dep(const Params& p, int slot, uint32_t dep) { + if (dep == 0u) __trap(); + stamp(p, slot); +} +__device__ __forceinline__ void tma_prefetch_2d(const void* tmap, int c0, int c1) { + asm volatile("cp.async.bulk.prefetch.tensor.2d.L2.global.tile [%0, {%1, %2}];" ::"l"(tmap), "r"(c0), "r"(c1) : "memory"); +} +// Physical W13 column (accumulator column) holding reordered row q: block-32 shuffle of FlashInfer's +// get_shuffle_matrix_a_row_indices (epilogue_tile_m = 128); reordered row 2j is up_j, 2j + 1 is gate_j. +__device__ __forceinline__ int w13_col(int q) { return (q & ~31) + ((q & 31) % 4) * 8 + (q & 31) / 4; } +__host__ __device__ constexpr int w13_logical(int p) { + return ((p & ~31) + (p % 8) * 4 + (p & 31) / 8) % 2 ? ((p & ~31) + (p % 8) * 4 + (p & 31) / 8) / 2 + : 128 + ((p & ~31) + (p % 8) * 4 + (p & 31) / 8) / 2; +} + + +// FC2 drain of one pass: TMEM lane quarter q (tiles 2 q, 2 q + 1 of the pass), 256 accumulator columns +__device__ __forceinline__ void reg_fence64(uint32_t (&r)[64]) { // keep uses of r after the tcgen05.wait::ld +#pragma unroll + for (int i = 0; i < 64; ++i) asm volatile("" : "+r"(r[i])); +} + +// FC2 accumulator columns [64 C0, 64 C1) of this warp's lane quarter -> bf16 gemm2_out rows (row = tile token) +template +__device__ __forceinline__ void fc2_drain16(const Params& p, Smem& s, uint32_t tm, int q, int lane, int pass, float a2) { + const int row = q * 32 + lane, tile = pass * 8 + row / 16, t = row % 16; + const bool valid = t < p.num_tokens && ((s.tokmask >> t) & 1u); + __nv_bfloat16* orow = valid ? p.out + (size_t)(t * kTopK + s.kslot[t]) * kH + tile * 256 : nullptr; + const uint32_t base = tm + ((uint32_t)(q * 32) << 16); + __nv_bfloat16* orow4[4]; // DRAIN_SMEM readback: token rows (k * 4 + lane / 8) % 16 of tile 2 q (+ 256 for 2 q + 1) +#pragma unroll + for (int k = 0; k < 4; ++k) { + const int tr = (k * 4 + lane / 8) % 16; + orow4[k] = (DRAIN_SMEM && tr < p.num_tokens && ((s.tokmask >> tr) & 1u)) + ? p.out + (size_t)(tr * kTopK + s.kslot[tr]) * kH + (pass * 8 + 2 * q) * 256 + : nullptr; + } + auto put = [&](const uint32_t (&r)[64], int c) { + if (DRAIN_SMEM) { + // rows go through a per-warp 4 KB buffer in x's smem (dead after FC1): 32 rows x 128 B, 16-B chunks XOR-swizzled + // by row, read back so that 8 lanes store one row's 128 B (full lines instead of 16-B pieces of 32 rows). + // Shared-space instructions: generic loads after global stores serialize. + const uint32_t stg = smem_u32(s.xa) + (threadIdx.x / 32) * 4096; +#pragma unroll + for (int b32 = 0; b32 < 2; ++b32) { + uint32_t pk[16]; +#pragma unroll + for (int l = 0; l < 32; l += 2) { + const float v0 = __uint_as_float(r[b32 * 32 + (l % 4) * 8 + l / 4]) * a2; + const float v1 = __uint_as_float(r[b32 * 32 + ((l + 1) % 4) * 8 + (l + 1) / 4]) * a2; + const __nv_bfloat162 b2 = __floats2bfloat162_rn(v0, v1); + pk[l / 2] = *reinterpret_cast(&b2); + } +#pragma unroll + for (int i = 0; i < 4; ++i) + asm volatile("st.shared.v4.b32 [%0], {%1, %2, %3, %4};" ::"r"(stg + lane * 128 + (((b32 * 4 + i) ^ (lane & 7)) * 16)), + "r"(pk[4 * i]), "r"(pk[4 * i + 1]), "r"(pk[4 * i + 2]), "r"(pk[4 * i + 3]) : "memory"); + } + __syncwarp(); +#pragma unroll + for (int k = 0; k < 8; ++k) { + const int rr = k * 4 + lane / 8, ch = lane % 8; + uint4 v; + asm volatile("ld.shared.v4.b32 {%0, %1, %2, %3}, [%4];" : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) + : "r"(stg + rr * 128 + ((ch ^ (rr & 7)) * 16)) : "memory"); + if (orow4[k % 4]) *reinterpret_cast(orow4[k % 4] + (k / 4) * 256 + c * 64 + ch * 8) = v; + } + __syncwarp(); + } else if (valid) { +#pragma unroll + for (int b32 = 0; b32 < 2; ++b32) { + uint32_t pk[16]; +#pragma unroll + for (int l = 0; l < 32; l += 2) { + const float v0 = __uint_as_float(r[b32 * 32 + (l % 4) * 8 + l / 4]) * a2; + const float v1 = __uint_as_float(r[b32 * 32 + ((l + 1) % 4) * 8 + (l + 1) / 4]) * a2; + const __nv_bfloat162 b2 = __floats2bfloat162_rn(v0, v1); + pk[l / 2] = *reinterpret_cast(&b2); + } +#pragma unroll + for (int i = 0; i < 4; ++i) + *reinterpret_cast(orow + c * 64 + b32 * 32 + i * 8) = + make_uint4(pk[4 * i], pk[4 * i + 1], pk[4 * i + 2], pk[4 * i + 3]); + } + } + }; + if (DRAIN_PIPE) { // the next chunk's TMEM load is in flight while this chunk converts and stores + uint32_t ra[64], rb[64]; + tmem_ld_x64(base + C0 * 64, ra); + tmem_ld_wait(); + reg_fence64(ra); +#pragma unroll + for (int c = C0; c < C1; c += 2) { + if (c + 1 < C1) tmem_ld_x64(base + (uint32_t)((c + 1) * 64), rb); + put(ra, c); + if (c + 1 < C1) { + tmem_ld_wait(); + reg_fence64(rb); + if (c + 2 < C1) tmem_ld_x64(base + (uint32_t)((c + 2) * 64), ra); + put(rb, c + 1); + if (c + 2 < C1) { tmem_ld_wait(); reg_fence64(ra); } + } + } + } else { +#pragma unroll 1 + for (int c = C0; c < C1; ++c) { + uint32_t r[64]; + tmem_ld_x64(base + (uint32_t)(c * 64), r); + tmem_ld_wait(); + put(r, c); + } + } +} + +extern "C" __global__ void __launch_bounds__(kThreads, 1) +q35_mkm_moe_m16(const __grid_constant__ CUtensorMap tm13, const __grid_constant__ CUtensorMap tm2, + const __grid_constant__ CUtensorMap tmx, const Params p) { + extern __shared__ __align__(1024) uint8_t raw[]; +#if SMEM_SPACE // pointer arithmetic on the __shared__ array keeps the shared address space (STS/LDS, not generic) + Smem& s = *reinterpret_cast(raw + ((1024u - (smem_u32(raw) & 1023u)) & 1023u)); +#else + Smem& s = *reinterpret_cast((reinterpret_cast(raw) + 1023) & ~uintptr_t(1023)); +#endif + const int warp = threadIdx.x / 32, lane = threadIdx.x % 32; + const int slot = blockIdx.x; // (token, top-k slot) served by this CTA + const int M = p.num_tokens; + if (threadIdx.x == 0) { + stamp(p, 0); + for (int i = 0; i < kStages; ++i) { + mbar_init(&s.full[i], 1); + mbar_init(&s.empty[i], 1); + mbar_init(&s.sf_full[i], 4); + } + mbar_init(&s.x_full, 1); + mbar_init(&s.fc1_done, 1); + mbar_init(&s.acc_ready, 2); // warps 0 and 4 drained their columns into acc1 + mbar_init(&s.h_ready, 4); + mbar_init(&s.fc2_pass[0], 1); // one single-phase barrier per FC2 pass + mbar_init(&s.fc2_pass[1], 1); + mbar_init(&s.fc2_free, 4); + mbar_init(&s.route_done, 1); + fence_mbar_init(); + } + if (warp == 2) { tmem_alloc(&s.tmem, 512); tmem_relinquish(); tc_fence_before(); } + if (warp == 1 && lane == 0) { prefetch_tmap(&tm13); prefetch_tmap(&tm2); prefetch_tmap(&tmx); } + __syncthreads(); + tc_fence_after(); + const uint32_t tm = s.tmem; + const int my_t = slot / kTopK, my_k = slot % kTopK; + if (my_t >= M) { + if (warp == 2) tmem_dealloc(tm, 512); + return; + } + auto issue_stage = [&](int st, int e, uint64_t pol) { + const int b = st % kStages; + mbar_expect_tx(&s.full[b], kStageBytes); + uint8_t* dst = s.ring[b]; + if (st < kFc1Stages) { + // the expert's rows as two boxes of 128 rows, K bytes [128 st, 128 st + 128) + tma_load_2d_w(dst, &tm13, &s.full[b], st * 128, e * kN1, pol); + tma_load_2d_w(dst + 16384, &tm13, &s.full[b], st * 128, e * kN1 + 128, pol); + // scale atoms of row blocks 0 and 1, k-blocks [4 st, 4 st + 4) + const uint8_t* sf = p.w13_sf + (size_t)e * (kN1 * kH / 16); + bulk_load_w(dst + 32768, sf + (size_t)(st * 4) * 512, 2048, &s.full[b], pol); + bulk_load_w(dst + 32768 + 2048, sf + (size_t)(64 + st * 4) * 512, 2048, &s.full[b], pol); + } else { + // W2 tiles 2 f and 2 f + 1 (256 rows x 64 B each) and their scale atoms (2 row blocks x 2 k-blocks) + const int f = st - kFc1Stages; +#pragma unroll + for (int tl = 0; tl < 2; ++tl) { + const int row0 = (2 * f + tl) * 256; + tma_load_2d_w(dst + tl * 16384, &tm2, &s.full[b], 0, e * kH + row0, pol); + bulk_load_w(dst + 32768 + tl * 2048, p.w2_sf + (size_t)e * (kH * 8) + (size_t)(row0 / 128) * 2 * 512, 2048, + &s.full[b], pol); + } + } + }; + auto prefetch_stage = [&](int st, int e) { // the same bytes as issue_stage, to L2 only + if (st < kFc1Stages) { + tma_prefetch_2d(&tm13, st * 128, e * kN1); + tma_prefetch_2d(&tm13, st * 128, e * kN1 + 128); + const uint8_t* sf = p.w13_sf + (size_t)e * (kN1 * kH / 16); + bulk_prefetch_l2(sf + (size_t)(st * 4) * 512, 2048); + bulk_prefetch_l2(sf + (size_t)(64 + st * 4) * 512, 2048); + } else { + const int row0 = (st - kFc1Stages) * 512; + tma_prefetch_2d(&tm2, 0, e * kH + row0); + tma_prefetch_2d(&tm2, 0, e * kH + row0 + 256); + bulk_prefetch_l2(p.w2_sf + (size_t)e * (kH * 8) + (size_t)(row0 / 128) * 2 * 512, 4096); + } + }; + // routing: warps 0, 2..7 take tokens 0..15 (DynBlock-exact top-10 + softmax), up to 3 rounds; warp 1 recomputes token + // my_t's top-10 and starts this CTA's weight stream at once (a duplicate slot only wastes its first ring fill) + asm volatile("griddepcontrol.wait;" ::: "memory"); + if (threadIdx.x == 32) stamp(p, 10); + if (warp == 3 && lane == 0) { + // x (16 chunks of 16 rows x 128 B, rows >= M zero-filled) and its scale bytes; independent of the routing + mbar_expect_tx(&s.x_full, 32 * 1024 + M * 256); +#pragma unroll + for (int a = 0; a < 16; ++a) tma_load_2d(s.xa + a * 2048, &tmx, &s.x_full, a * 128, 0); + for (int t = 0; t < M; ++t) bulk_load(s.xsf[t], p.x_sf + (size_t)t * (kH / 16), 256, &s.x_full); + } + auto top10v = [&](const uint4 v0, const uint4 v1) -> uint32_t { + uint32_t keys[16]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + const uint32_t w = i < 4 ? (&v0.x)[i] : (&v1.x)[i - 4]; + keys[2 * i] = topk_key(w & 0xFFFFu, (uint32_t)(lane * 16 + 2 * i)); + keys[2 * i + 1] = topk_key(w >> 16, (uint32_t)(lane * 16 + 2 * i + 1)); + } + return TOPK_RANK ? warp_top10(keys, (uint32_t)lane, s.cand[warp]) : warp_top10_redux(keys, (uint32_t)lane, s.cand[warp]); + }; + auto top10 = [&](int rtok) -> uint32_t { + const uint4* src = reinterpret_cast(p.logits + (size_t)rtok * kE + lane * 16); + return top10v(src[0], src[1]); + }; + int e; + if (warp == 1) { + const uint32_t key = top10(my_t); + uint32_t bits, espec; + topk_unkey(__shfl_sync(0xffffffffu, key, my_k), bits, espec); + if (lane == 0) { + stamp(p, 9); + const uint64_t pol = policy_evict_first(); + for (int st = 0; st < SPEC_STAGES; ++st) { + issue_stage(st, (int)espec, pol); + if (STAGE_STAMPS) stamp(p, 64 + st); + } + stamp(p, 2); + } + mbar_wait(&s.route_done, 0); + e = s.expert; + } else { + const int base = warp == 0 ? 0 : warp - 1; + // every round's logits are requested before the first ranking, so later rounds do not pay the load latency again + uint4 lv[3][2]; +#pragma unroll + for (int r = 0; r < 3; ++r) { + const int rtok = base + 7 * r; + if (rtok < M) { + const uint4* src = reinterpret_cast(p.logits + (size_t)rtok * kE + lane * 16); + lv[r][0] = src[0]; + lv[r][1] = src[1]; + } + } +#pragma unroll + for (int r = 0; r < 3; ++r) { + const int rtok = base + 7 * r; + if (rtok >= M) break; + const uint32_t key = top10v(lv[r][0], lv[r][1]); + uint32_t bits, expert; + topk_unkey(key, bits, expert); + const uint16_t wbits = softmax_topk((uint16_t)bits, (uint32_t)lane); + if (lane < kTopK) { + s.ids[rtok][lane] = (int32_t)expert; + if (blockIdx.x == 0) { + p.expert_weights[rtok * kTopK + lane] = wbits; + p.topk_ids[rtok * kTopK + lane] = (int32_t)expert; + } + } + __syncwarp(); + } + named_bar(4, 224); + if (warp == 0) { + // this slot's expert, whether an earlier slot picked it, and which tokens use it (top-k slot per token) + const int ee = s.ids[my_t][my_k]; + bool dup = false; + for (int q = lane; q < slot; q += 32) dup |= s.ids[q / kTopK][q % kTopK] == ee; + dup = __any_sync(0xffffffffu, dup); + uint32_t mbit = 0; + int ks = 0; + if (lane < M) { +#pragma unroll + for (int k = 0; k < kTopK; ++k) + if (s.ids[lane][k] == ee) { mbit = 1u << lane; ks = k; } + } + const uint32_t mask = __reduce_or_sync(0xffffffffu, mbit); + if (lane < 16) s.kslot[lane] = ks; + if (lane == 0) { + s.expert = dup ? -1 : ee; + s.tokmask = mask; + stamp(p, 1); + mbar_arrive(&s.route_done); + } + } + named_bar(4, 224); + e = s.expert; + } + if (e < 0) { // duplicate slot: let the speculative loads land, then leave + if (warp == 1) { + if (lane == 0) + for (int b = 0; b < SPEC_STAGES; ++b) mbar_wait(&s.full[b], 0); + __syncwarp(); + } + mbar_wait(&s.x_full, 0); + named_bar(7, kThreads); + if (warp == 2) tmem_dealloc(tm, 512); + return; + } + tc_fence_after(); + float ag = 0.f, sc = 0.f, a2 = 0.f; + if (HOIST_SCALES) { ag = p.g1_alpha[e]; sc = p.g1_scale_c[e]; a2 = p.g2_alpha[e]; } + // FC1 drain of quarter-0 accumulator columns [64 c0, 64 c0 + 128) (TMEM lanes 0..15 hold the token rows) into acc1 + auto drain_fc1 = [&](int c0) { +#pragma unroll 1 + for (int c = c0; c < c0 + 2; ++c) { + uint32_t r[64]; + tmem_ld_x64(tm + (uint32_t)(c * 64), r); + tmem_ld_wait(); + if (lane < 16) { +#pragma unroll + for (int i = 0; i < 16; ++i) + *reinterpret_cast(s.acc1 + lane * kARow + c * 64 + 4 * i) = + make_uint4(r[4 * i], r[4 * i + 1], r[4 * i + 2], r[4 * i + 3]); + } + } + tc_fence_before(); + __syncwarp(); + }; + // SwiGLU + NVFP4 epilogue (h = 0: the one FC1 pass), tasks (token row, 16-wide block, half) = 16 x 8 x 2 = 256, + // task = et + k * nt. + // Accumulator columns of each 32-block: [0, 8) up 2m, [8, 16) gate 2m, [16, 24) up 2m + 1, [24, 32) gate 2m + 1 + auto epi_tasks = [&](int h, int et, int nt) { + if (!HOIST_SCALES) { ag = p.g1_alpha[e]; sc = p.g1_scale_c[e]; } +#pragma unroll 1 + for (int task = et; task < 256; task += nt) { + const int row = task / 16, blk = (task / 2) % 8, hf = task % 2; + const float4* ar = reinterpret_cast(s.acc1 + row * kARow + 32 * blk + 4 * hf); + const float4 u_even = ar[0], g_even = ar[2], u_odd = ar[4], g_odd = ar[6]; + const float ga[8] = {g_even.x, g_odd.x, g_even.y, g_odd.y, g_even.z, g_odd.z, g_even.w, g_odd.w}; + const float ua[8] = {u_even.x, u_odd.x, u_even.y, u_odd.y, u_even.z, u_odd.z, u_even.w, u_odd.w}; + // trtllm-gen's SwiGLU + NVFP4 epilogue op for op (from its SASS); a = FC1 dequant, c = a x FC2 input scale: + // h' = (u a)(g sigmoid(g a)), sigmoid = rcp(ex2(-(g (a log2e))) + 1), block scale e4m3(max|h'| c / 6), FP4 (h' / scale) c + const float a_l2e = mul_ftz(ag, 1.4426950216293334961f); + float hv[8]; + float amax = 0.f; +#pragma unroll + for (int i = 0; i < 8; ++i) { + const float ex = ex2_approx(-(ga[i] * a_l2e)); + const float sg = rcp_approx(ex + 1.0f); + const float v = fmaf(ua[i], ag, 0.0f) * (ga[i] * sg); + hv[i] = v; + amax = fmaxf(amax, fabsf(v)); + } + amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, 1)); + const uint32_t sf = cvt_e4m3(fmaxf(0.f, fabsf(amax * sc)) * 0.16666667163372039795f); + const float sff = e4m3_to_f32(sf); + const float osc = sff == 0.f ? 0.f : rcp_approx(sff); + float qv[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) qv[i] = mul_ftz(hv[i], osc) * sc; + const uint32_t packed = cvt_e2m1x8(qv); + const int gblk = 8 * h + blk; // 16-wide block over the 128 intermediates; k-blocks 0, 1 + const int chunk = gblk / 2, r8 = row % 8; + const uint32_t off = (uint32_t)((row / 8) * 1024 + r8 * 128 + ((chunk ^ r8) * 16) + (gblk % 2) * 8 + hf * 4); +#pragma unroll + for (int cp = 0; cp < 8; ++cp) *reinterpret_cast(s.ha + cp * 2048 + off) = packed; + if (hf == 0) reinterpret_cast(&s.hsf[row][gblk / 4])[gblk % 4] = (uint8_t)sf; + } + }; + + if (warp == 1) { + if (lane == 0) { + // ---------------- TMA producer: the rest of the weight stream (stages 0 .. kStages - 1 went out above) + const uint64_t pol = policy_evict_first(); + for (int st = SPEC_STAGES; st < kTotStages; ++st) { + if (FC2_PF > 0 && st == kFc1Stages - FC2_PF_LEAD) { + // the FC2 tiles beyond the ring go to L2 now, while the last FC1 stages and h are in progress + for (int j = kStages; j < kStages + FC2_PF; ++j) prefetch_stage(kFc1Stages + j, e); + } + if (PF_SET != 0 && ((st == SPEC_STAGES && (PF_SET & 1)) || (st == kFc1Stages + 4 && (PF_SET & 4)) || + (st == kFc1Stages + 6 && (PF_SET & 8)))) { + // these stages wait for MMAs that are slow to start after a phase change; let HBM fetch them meanwhile + const int n = st == SPEC_STAGES ? PF_N1 : st == kFc1Stages + 4 ? PF_N2 : 4; + for (int k = st; k < st + n && k < kTotStages; ++k) prefetch_stage(k, e); + } + if (st >= kStages) mbar_wait(&s.empty[st % kStages], ((st / kStages) - 1) & 1); + issue_stage(st, e, pol); + if (STAGE_STAMPS) stamp(p, 64 + st); + } + } + __syncwarp(); + } else if (warp == 2) { + if (lane == 0) { + // ---------------- MMA issuer (tensor pipe: MMAs only) + const uint32_t idesc = make_idesc_nvf4(128u, 256u); + mbar_wait(&s.x_full, 0); + tc_fence_after(); + int n = 0; + for (int st = 0; st < kFc1Stages; ++st) { + const int b = st % kStages, ks = st; + mbar_wait(&s.full[b], (st / kStages) & 1); + mbar_wait(&s.sf_full[b], (st / kStages) & 1); + tc_fence_after(); + if (st == 0) stamp(p, 3); +#pragma unroll + for (int q = 0; q < 4; ++q, ++n) { + const int kb = ks * 4 + q; // k-blocks in K order: one accumulator, the stock summation order + umma_nvf4(tm, make_sdesc(smem_u32(s.xa + (kb / 4) * 2048) + (kb % 4) * 32, 0, 1024, 2), + make_sdesc(smem_u32(s.ring[b]) + q * 32, 0, 1024, 2), idesc, + tm + kColSfaX + (uint32_t)(kb % kSfaSlots) * 4, tm + kColSfb + (uint32_t)(n % kSfbSlots) * 8, kb > 0); + } + tc_commit(&s.empty[b]); + } + tc_commit(&s.fc1_done); + mbar_wait(&s.h_ready, 0); + tc_fence_after(); + stamp(p, 6); + for (int st = kFc1Stages; st < kTotStages; ++st) { + const int b = st % kStages, f = st - kFc1Stages; + if (f == 4) { mbar_wait(&s.fc2_free, 0); tc_fence_after(); } + mbar_wait(&s.full[b], (st / kStages) & 1); + mbar_wait(&s.sf_full[b], (st / kStages) & 1); + tc_fence_after(); +#pragma unroll + for (int tl = 0; tl < 2; ++tl) { + const int j = 2 * f + tl, jt = j % 8; +#pragma unroll + for (int kb = 0; kb < 2; ++kb) { + const int n2 = 64 + j * 2 + kb; // scale-factor slot, numbered on from FC1's 64 k-blocks + umma_nvf4(tm, make_sdesc(smem_u32(s.ha) + kb * 32, 0, 1024, 2), + make_sdesc(smem_u32(s.ring[b] + tl * 16384) + kb * 32, 0, 512, 4), idesc, + tm + kColSfa + (uint32_t)(jt * 2 + kb) * 4, tm + kColSfb + (uint32_t)(n2 % kSfbSlots) * 8, + (jt | kb) != 0); + } + } + tc_commit(&s.empty[b]); + if (f % 4 == 3) tc_commit(&s.fc2_pass[f / 4]); + } + } + __syncwarp(); + } else if (warp == 0 || warp == 3) { + // ---------------- FC1 drains (warp 0: columns [0, 128); warp 4 drains [128, 256)) and the SwiGLU + NVFP4 epilogue + const int et = (warp == 0 ? 0 : 32) + lane; // 0..63 + if (warp == 0) { + mbar_wait(&s.fc1_done, 0); + tc_fence_after(); + if (lane == 0) stamp(p, 4); + drain_fc1(0); + if (lane == 0) { mbar_arrive(&s.acc_ready); stamp(p, 13); } + } + mbar_wait(&s.acc_ready, 0); + epi_tasks(0, et, 192); + if (warp == 3 && lane == 0) stamp(p, 14); + fence_proxy_async_smem(); + named_bar(6, 192); // h and its scale words are complete for the TMEM warpgroup + } else { + // ---------------- TMEM-side warpgroup (warps 4..7): q = lane quarter (TMEM lanes 32 q .. 32 q + 31) + const int q = warp - 4; + const uint32_t lane_addr = (uint32_t)(q * 32) << 16; + mbar_wait(&s.x_full, 0); + auto stage_sf2 = [&](int st) { // FC2 tiles 2 f, 2 f + 1: k-blocks 0, 1 of their row blocks (rows 0..127 | 128..255) + const int b = st % kStages; + mbar_wait(&s.full[b], (st / kStages) & 1); + if (STAGE_STAMPS && threadIdx.x == 128) stamp(p, 16 + st); + if (st >= 4) mbar_wait(&s.empty[(st - 4) % kStages], ((st - 4) / kStages) & 1); + const uint8_t* sf = s.ring[b] + 32768; + const int n0 = 64 + (st - kFc1Stages) * 4; + for (int i = 0; i < 4; ++i) { + const int tl = i / 2, kb = i % 2; + const uint4 v0 = *reinterpret_cast(sf + tl * 2048 + kb * 512 + lane * 16); + const uint4 v1 = *reinterpret_cast(sf + tl * 2048 + 1024 + kb * 512 + lane * 16); + const uint32_t r[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w}; + tmem_st_x8(tm + lane_addr + kColSfb + (uint32_t)((n0 + i) % kSfbSlots) * 8, r); + } + tmem_st_wait(); + tc_fence_before(); + __syncwarp(); + if (lane == 0) mbar_arrive(&s.sf_full[b]); + }; + // FC1 stage st: weight scale factors of k-blocks [4 ks, 4 ks + 4) (8 columns each: row blocks 2 h | 2 h + 1) and + // the x scale factors of the same k-blocks (row m -> lane m % 32, column m / 32 of each 4-column atom, every + // quarter holds a copy; rows >= M, including the A rows read past the 16 real ones, get zero scales) + for (int st = 0; st < kFc1Stages; ++st) { + const int b = st % kStages, ks = st; + mbar_wait(&s.full[b], (st / kStages) & 1); + if (STAGE_STAMPS && threadIdx.x == 128) stamp(p, 16 + st); + if (st >= 4) mbar_wait(&s.empty[(st - 4) % kStages], ((st - 4) / kStages) & 1); + const uint8_t* sf = s.ring[b] + 32768; +#pragma unroll + for (int i = 0; i < 4; ++i) { + const uint4 v0 = *reinterpret_cast(sf + i * 512 + lane * 16); + const uint4 v1 = *reinterpret_cast(sf + 2048 + i * 512 + lane * 16); + const uint32_t r[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w}; + tmem_st_x8(tm + lane_addr + kColSfb + (uint32_t)((st * 4 + i) % kSfbSlots) * 8, r); + } + uint32_t xr[16]; +#pragma unroll + for (int i = 0; i < 16; ++i) { + const int kb = ks * 4 + i / 4; + xr[i] = (i % 4 == 0 && lane < M) ? *reinterpret_cast(&s.xsf[lane][kb * 4]) : 0u; + } + tmem_st_x16(tm + lane_addr + kColSfaX + (uint32_t)((ks * 4) % kSfaSlots) * 4, xr); + tmem_st_wait(); + tc_fence_before(); + __syncwarp(); + if (lane == 0) mbar_arrive(&s.sf_full[b]); + } + // EARLY_SF2: stage the first FC2 scale factors before h (they wait for FC2 data to land, which delays h) + for (int st = kFc1Stages; st < kFc1Stages + EARLY_SF2; ++st) stage_sf2(st); + if (q == 0) { // drain the upper 128 columns + mbar_wait(&s.fc1_done, 0); + tc_fence_after(); + drain_fc1(2); + if (lane == 0) mbar_arrive(&s.acc_ready); + } + mbar_wait(&s.acc_ready, 0); + epi_tasks(0, 64 + (warp - 4) * 32 + lane, 192); + fence_proxy_async_smem(); + named_bar(6, 192); // h complete + if (threadIdx.x == 128) stamp(p, 5); + // h scale factors: 16 atoms (tile jt, k-block kb) of 4 columns; row m = 32 c + lane is nonzero only in its tile + { + uint32_t r[32]; +#pragma unroll 1 + for (int half = 0; half < 2; ++half) { +#pragma unroll + for (int i = 0; i < 32; ++i) { + const int a = half * 8 + i / 4, col = i % 4; // atom a = 2 jt + kb + const int jt = a / 2, kb = a % 2; + const int row = 32 * col + lane; + r[i] = (row / 16 == jt && row % 16 < M) ? s.hsf[row % 16][kb] : 0u; + } + tmem_st_x32(tm + lane_addr + kColSfa + half * 32, r); + } + tmem_st_wait(); + } + tc_fence_before(); + __syncwarp(); + if (lane == 0) mbar_arrive(&s.h_ready); + if (q == 0 && lane == 0) stamp(p, 15); + for (int st = kFc1Stages + EARLY_SF2; st < kTotStages; ++st) { + if (st == kFc1Stages + 4) { + // pass 0 is fully staged; drain it once its MMAs are done, then free the accumulator for pass 1 + mbar_wait(&s.fc2_pass[0], 0); + tc_fence_after(); + if (q == 0 && lane == 0) stamp(p, 7); + fc2_drain16<0, 4>(p, s, tm, q, lane, 0, a2); + tc_fence_before(); + __syncwarp(); + if (lane == 0) mbar_arrive(&s.fc2_free); + } + stage_sf2(st); + } + mbar_wait(&s.fc2_pass[1], 0); + tc_fence_after(); + if (q == 0 && lane == 0) stamp(p, 12); + fc2_drain16<0, (TAIL8 ? 2 : 4)>(p, s, tm, q, lane, 1, a2); + } + if (TAIL8 && warp < 4) { // the last pass's upper columns: every warp is free now (TMEM lane quarter = warp) + mbar_wait(&s.fc2_pass[1], 0); + tc_fence_after(); + fc2_drain16<2, 4>(p, s, tm, warp, lane, 1, HOIST_SCALES ? a2 : p.g2_alpha[e]); + } + tc_fence_before(); + named_bar(7, kThreads); // every warp, including the drains, before TMEM is released + if (threadIdx.x == 128) stamp(p, 8); + if (warp == 2) tmem_dealloc(tm, 512); +} + +static_assert(sizeof(Smem) + 1024 == Q35_MKM_SMEM_BYTES, "the launcher's dynamic shared memory size"); + +} // namespace qwen_mkm_moe_m16 + + +} // namespace sglang diff --git a/test/kernels/test_qwen3_5_moe.py b/test/kernels/test_qwen3_5_moe.py index e0a16e5d..e4c9cac9 100644 --- a/test/kernels/test_qwen3_5_moe.py +++ b/test/kernels/test_qwen3_5_moe.py @@ -8,7 +8,9 @@ - the fused routed experts (slot mkm), against FlashInfer's trtllm-gen NVFP4 MoE as SGLang serves Qwen3.5 at TP8 and TP4 (intermediate 128 / 256 a rank; routing from the logits, RenormalizeNaive, deferred finalize): each token's 10 expert rows and their - weights, at 1, 4 and 7 rows; + weights, at 1, 4 and 7 rows (the 2-CTA engines) and at TP8 also 8, 14 and 16 rows (the + 16-token engine), also with every token routed to the same experts; graph replay of + both; - the router front with the shared gate/up stage (slot route_shared, numeric): its logits and NVFP4 input bit for bit the plain launch's, and its gate/up rows against fp64 on SGLang's static FP8 quantization of the rows, at most 1.05 x the mean relative @@ -158,16 +160,22 @@ def _stock(layer, logits, x_fp4, x_sf): routing_method_type=RoutingMethodType.RenormalizeNaive, do_finalize=False, activation_type=ActivationType.Swiglu, - tune_max_num_tokens=8, + tune_max_num_tokens=max(8, m), enable_pdl=True, )[:3] return rows[permuted.reshape(-1).long()[: m * TOP_K]], weights -@pytest.mark.parametrize("m", [1, 4, 7]) +@pytest.mark.parametrize("m", [1, 4, 7, 8, 14, 16]) @pytest.mark.parametrize("seed", [0, 1]) -def test_mkm_is_the_trtllm_routed_experts(layer, m, seed): +@pytest.mark.parametrize("shared_route", [False, True]) +def test_mkm_is_the_trtllm_routed_experts(layer, m, seed, shared_route): + if m > mkm.max_tokens(layer["inter"]): + pytest.skip("no 16-token engine at this intermediate") logits, x_fp4, x_sf = _inputs(m, seed) + if shared_route: + # every token routes to the same 10 experts: one slot computes each for all rows + logits = logits[:1].expand(m, -1).contiguous() rows, weights = _fused(layer, logits, x_fp4, x_sf) want_rows, want_weights = _stock(layer, logits, x_fp4, x_sf) assert want_weights.dtype == torch.bfloat16 @@ -176,8 +184,11 @@ def test_mkm_is_the_trtllm_routed_experts(layer, m, seed): assert not differ.any(), f"rows {differ.nonzero().flatten().tolist()} differ" -def test_mkm_replays_from_a_cuda_graph(layer): - logits, x_fp4, x_sf = _inputs(7, 2) +@pytest.mark.parametrize("m", [7, 14]) +def test_mkm_replays_from_a_cuda_graph(layer, m): + if m > mkm.max_tokens(layer["inter"]): + pytest.skip("no 16-token engine at this intermediate") + logits, x_fp4, x_sf = _inputs(m, 2) eager = _fused(layer, logits, x_fp4, x_sf) graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): diff --git a/test/unit/test_qwen3_5_spec.py b/test/unit/test_qwen3_5_spec.py index 846fdf42..073707f3 100644 --- a/test/unit/test_qwen3_5_spec.py +++ b/test/unit/test_qwen3_5_spec.py @@ -143,6 +143,11 @@ def test_an_sglang_without_the_eagle_forward_view_is_refused(monkeypatch): "slot 'gdn_front' runs slot 'gdn_pdl''s dependent verify", ), ({"route_shared"}, "slot 'route_shared' runs in slot 'route''s launch"), + ({"route", "mkm_wide"}, "slot 'mkm_wide' is slot 'mkm''s 16-token engine"), + ( + {"route", "mkm", "mkm_wide"}, + "slot 'mkm_wide' serves the steps of more than one request ('multi_request')", + ), ], ) def test_a_slot_set_that_cannot_run_is_refused_before_the_load( @@ -188,7 +193,10 @@ def _on(slots): def test_the_slot_plan_gates_each_slot_by_the_rows_its_kernels_serve(): - every = plan.Facts(**dict.fromkeys(plan.Facts.__struct_fields__, True)) + # every slot built except mkm's 16-token engine (mkm_wide, below) + every = plan.Facts( + **{**dict.fromkeys(plan.Facts.__struct_fields__, True), "mkm_wide": False} + ) assert _on(plan.slots(7, every)) == sorted(plan.Slots.__struct_fields__) # two requests: the 16-row router front, the GDN chain and the hop's one-shot rows assert _on(plan.slots(14, every)) == ["gdn_pdl", "gdn_verify", "hop", "route"] @@ -210,6 +218,11 @@ def test_the_slot_plan_gates_each_slot_by_the_rows_its_kernels_serve(): attention = plan.Facts(skv2=True, mka=True, mka_push=True, hop=True) assert plan.covered((7, 14, 28), attention, 7) == (7, 14) assert _on(plan.slots(14, attention)) == ["hop"] + # mkm_wide: mkm's 16-token engine at two requests, not past the router front's rows + wide = plan.Facts(route=True, mkm=True, mkm_wide=True) + assert _on(plan.slots(14, wide)) == ["mkm", "route"] + assert _on(plan.slots(28, wide)) == [] + assert _on(plan.slots(14, plan.Facts(route=True, mkm=True))) == ["route"] # the router front and the hop's mailbox are built for the widest width they serve assert plan.widest((7, 14, 28), plan.ROUTE_MAX_ROWS) == 14 assert plan.widest((7, 14, 28), plan.HOP_MAX_ROWS) == 14 @@ -224,6 +237,7 @@ def test_the_slot_plans_row_bounds_are_the_kernels(): assert plan.ROUTE_MAX_ROWS == router.MAX_TOKENS assert plan.ROUTE_SHARED_ROWS == router.SHARED_MAX_M assert plan.MKM_MAX_ROWS == mkm.MAX_TOKENS + assert plan.MKM_WIDE_MAX_ROWS == mkm.M16_MAX_TOKENS assert plan.GDN_PUSH_MAX_ROWS == hop.OPROJ_MAX_M assert plan.MKA_MAX_ROWS == mka.MAX_ROWS From c5c714d386d388cb0edcab3b3a1916dfe138effa Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Wed, 21 Oct 2026 10:03:00 -0700 Subject: [PATCH 26/27] [Perf] [GB300] qwen3_5: Entry draft_loop serves two requests at TP8 with multi_request A draft step of two requests runs the split fc, the hop's all-reduce points and the two-token MoE; q1, the o_proj push and the exchange stay one-request. It serves them where multi_request covers the verify steps of several requests (TP8); TP4 keeps one request. Numeric as the draft slots. --- sgl_nvidia_mega/qwen3_5/draft/__init__.py | 4 ++ sgl_nvidia_mega/qwen3_5/draft/attention.py | 8 ++- sgl_nvidia_mega/qwen3_5/draft/layer.py | 15 +++-- sgl_nvidia_mega/qwen3_5/draft/loop.py | 42 +++++++++--- test/kernels/test_qwen3_5_draft_front.py | 4 +- test/kernels/test_qwen3_5_draft_hop.py | 4 +- test/unit/test_qwen3_5_runtime.py | 74 +++++++++++++++++++++- 7 files changed, 130 insertions(+), 21 deletions(-) diff --git a/sgl_nvidia_mega/qwen3_5/draft/__init__.py b/sgl_nvidia_mega/qwen3_5/draft/__init__.py index 2b7457dd..4a58cc8a 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/draft/__init__.py @@ -85,6 +85,10 @@ def numerics(slots: frozenset) -> Numerics: # the one request a draft entry serves: 2.2's draft kernels are single-row SERVED_BS = 1 +# the requests a draft_loop step serves (two only where plan.multi_request): two run the +# draft steps on 2.2's two-row kernels (the split fc, the hop's all-reduce points, the +# two-token MoE), the rest SGLang's ops +LOOP_BS = (1, 2) class MtpDraft: diff --git a/sgl_nvidia_mega/qwen3_5/draft/attention.py b/sgl_nvidia_mega/qwen3_5/draft/attention.py index 0c5e50fa..9297bde7 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/attention.py +++ b/sgl_nvidia_mega/qwen3_5/draft/attention.py @@ -299,7 +299,13 @@ def decode(self, q8, meta, kv, rows_per_request: int): attn = self.layer.attn bmm1_scale = 1.0 * kv.k_scale[0] * attn.scaling bmm2_scale = kv.v_scale[0] - if self.q1 is not None and rows_per_request == 1 and _q1_label() != Q1_STOCK: + # q1 decodes one request + if ( + self.q1 is not None + and rows_per_request == 1 + and q8.shape[0] == 1 + and _q1_label() != Q1_STOCK + ): return self.q1(q8, kv, meta).view(-1, self.heads * self.head_dim) if self.skv2 is not None and rows_per_request > 1 and self.skv2.long(): return self.skv2.run(attn, q8, kv, meta) diff --git a/sgl_nvidia_mega/qwen3_5/draft/layer.py b/sgl_nvidia_mega/qwen3_5/draft/layer.py index b262db49..547dc46c 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/layer.py +++ b/sgl_nvidia_mega/qwen3_5/draft/layer.py @@ -75,6 +75,9 @@ def layer(parts, x, positions, cache_loc, meta, kv, *, rows_per_request: int): lay = parts.layer residual = x hop = parts.hop if rows_per_request == 1 else None + # a draft step of one request pushes its o_proj into the hop and releases the next + # launch early; of two requests it takes only the hop's all-reduce points + push = hop if x.shape[0] == 1 else None extend_hop = parts.hop if rows_per_request > 1 and EXTEND_MNNVL else None attn_out = parts.attention( lay.input_layernorm(x), @@ -84,8 +87,8 @@ def layer(parts, x, positions, cache_loc, meta, kv, *, rows_per_request: int): kv, rows_per_request=rows_per_request, o_proj=( - partial(hop.o_proj, weight=lay.o_proj.weight) - if hop is not None and hop.gemv + partial(push.o_proj, weight=lay.o_proj.weight) + if push is not None and push.gemv else None ), ) @@ -95,7 +98,9 @@ def layer(parts, x, positions, cache_loc, meta, kv, *, rows_per_request: int): attn_out, residual, lay.post_attention_layernorm, - moe_single_launch=moe is not None and moe.single_launch(1), + moe_single_launch=push is not None + and moe is not None + and moe.single_launch(1), ) elif extend_hop is not None: moe_in, residual = extend_hop.ffn_input( @@ -111,7 +116,9 @@ def layer(parts, x, positions, cache_loc, meta, kv, *, rows_per_request: int): moe_out, residual, parts.model.norm, - head_follows=parts.mkd is not None and parts.mkd.head_follows(), + head_follows=push is not None + and parts.mkd is not None + and parts.mkd.head_follows(), ) return out out, _ = parts.model.norm(tensor_model_parallel_all_reduce(moe_out), residual) diff --git a/sgl_nvidia_mega/qwen3_5/draft/loop.py b/sgl_nvidia_mega/qwen3_5/draft/loop.py index dae86bb2..190fd5a8 100644 --- a/sgl_nvidia_mega/qwen3_5/draft/loop.py +++ b/sgl_nvidia_mega/qwen3_5/draft/loop.py @@ -3,18 +3,20 @@ # Adapted from SGLang (Apache-2.0, SGLang Team): # python/sglang/srt/speculative/eagle_worker_v2.py (EagleDraftWorker.draft_forward, the # topk-1 chain with draft_topk1_postprocess). -"""Entry ``draft_loop``: a served draft graph's whole draft loop for one request. +"""Entry ``draft_loop``: a served draft graph's whole draft loop for one or two requests. SGLang calls the factory once before it captures its draft graphs; the step it returns -for bs 1 runs inside the capture in place of ``draft_forward``. Draft step i (of +for bs 1 and 2 runs inside the capture in place of ``draft_forward``. Draft step i (of ``num_steps - 1``) runs the model step at ``positions + i`` on the KV slot ``out_cache_loc[i]`` and writes its top-1 into ``draft_tokens_out[:, i + 1]`` (column 0 is the round's first token); the step advances the positions in place and restores them before it returns, as the binding asks. -Stock: SGLang's model step and top-1 (``layer``). Slot ``draft_xchg`` ends each step -in the exchange: the next step's fc comes from it, the LM head is the shard GEMV, and -the token is chosen across the shards in the same launch (``draft/xchg``). +Stock: SGLang's model step and top-1 (``layer``). Slot ``draft_xchg`` ends each +one-request step in the exchange: the next step's fc comes from it, the LM head is the +shard GEMV, and the token is chosen across the shards in the same launch +(``draft/xchg``); a two-request step takes its fc from the exchange's row-split fc (2.2's +DRAFT_FC_SPLIT at two rows) and SGLang's top-1. ``begin_draft_round`` then fills the first step's fc when the round's input did not come from a served draft extend (2.2's take_fc_buf / fill_fc decision). """ @@ -23,7 +25,8 @@ import torch -from sgl_nvidia_mega.qwen3_5.draft import SERVED_BS, layer +from sgl_nvidia_mega.qwen3_5 import plan +from sgl_nvidia_mega.qwen3_5.draft import LOOP_BS, SERVED_BS, layer # The SGLang names this module reads (test/unit/test_contract.py pins them) SGLANG_READS = { @@ -41,6 +44,7 @@ "draft_tokens_out", "EagleDraftRoundIO", "from_served_extend", + "tp_size", ), } @@ -57,7 +61,12 @@ def __init__(self, parts, *, num_steps: int): parts.attention.q1 = build_q1(parts) def step_for(self, bs: int): - return self.step if bs == SERVED_BS else None + # two requests where the verify steps of several requests are covered + from sgl_nvidia_mega.qwen3_5.runtime import SLOTS + + if not plan.multi_request(self.parts.binding.tp_size, SLOTS): + return self.step if bs == SERVED_BS else None + return self.step if bs in LOOP_BS else None def begin_round(self, io) -> None: """On the host before a served draft graph replays: the first step's fc when @@ -79,12 +88,12 @@ def step(self, io) -> None: tokens = io.draft_tokens_out tokens[:, :1].copy_(io.topk_index) positions, mrope = io.positions, io.mrope_positions - if parts.mkd is not None: + if parts.mkd is not None and io.num_requests == SERVED_BS: self._mkd_steps(io, steps) else: token, hidden = io.topk_index.view(-1), io.hidden_states for i in range(steps): - hidden = self._model_step(io, i, layer.fc_input(parts, token, hidden)) + hidden = self._model_step(io, i, self._fc(token, hidden)) token = layer.top1(parts, hidden) tokens[:, i + 1].copy_(token) positions.add_(1) @@ -94,13 +103,26 @@ def step(self, io) -> None: if mrope is not None: mrope.sub_(steps) + def _fc(self, token, hidden): + """A step's fc output outside the exchange: slot draft_xchg's row-split fc, else + SGLang's.""" + mkd = self.parts.mkd + if mkd is None: + return layer.fc_input(self.parts, token, hidden) + return mkd.split_fc(self.parts.model.embed_tokens(token), hidden) + def _model_step(self, io, i: int, x): """Draft step i's decoder layer on its fc output ``x``: the final-norm rows.""" + # request-major slots: step i's of several requests are a strided view, which the + # front and the FP8 KV write read as dense rows + cache_loc = io.out_cache_loc[i :: io.num_steps] + if io.num_requests > 1: + cache_loc = cache_loc.contiguous() return layer.layer( self.parts, x, io.positions, - io.out_cache_loc[i :: io.num_steps], + cache_loc, io.steps[i], io.kv, rows_per_request=1, diff --git a/test/kernels/test_qwen3_5_draft_front.py b/test/kernels/test_qwen3_5_draft_front.py index d3d1ca2d..f5268172 100644 --- a/test/kernels/test_qwen3_5_draft_front.py +++ b/test/kernels/test_qwen3_5_draft_front.py @@ -3,7 +3,7 @@ """Slot draft_front's attention front (sm103/draft/front.py, 2.2's ATTN_FRONT) against SGLang's two launches it replaces, on one GPU: the fused q / k Gemma RMSNorm + RoPE + gate split, then the fused FP8 q / k / v + paged KV write. The FP8 q, the gate and the K / V cache -rows must be the same bytes, at the draft's widths (1 and 7 rows) and query heads a rank +rows must be the same bytes, at the draft's widths (1, 2 and 7 rows) and query heads a rank (4 at TP8, 8 at TP4), in eager and in a CUDA graph replayed with new positions and slots.""" @@ -110,7 +110,7 @@ def _same(a, b): @cuda @pytest.mark.parametrize("heads", [4, 8]) -@pytest.mark.parametrize("rows", [1, 7]) +@pytest.mark.parametrize("rows", [1, 2, 7]) @pytest.mark.parametrize("seed", [0, 1, 2]) def test_the_front_writes_what_sglangs_two_launches_write(rows, seed, heads): inputs = _inputs(rows, seed, heads) diff --git a/test/kernels/test_qwen3_5_draft_hop.py b/test/kernels/test_qwen3_5_draft_hop.py index 062b9c64..7ed90ec9 100644 --- a/test/kernels/test_qwen3_5_draft_hop.py +++ b/test/kernels/test_qwen3_5_draft_hop.py @@ -8,7 +8,7 @@ - mnnvl: its residual output equals bf16(bf16(sum) + residual) bit for bit, and its norm stays within one bf16 ulp of an fp32 reference (its bits against FlashInfer's legacy MNNVL kernel are checked with SGLang branch nvidia/qwen35-agentx-gb300, not here); -- rows 1 and 7, both entry releases, a late rank in turn, hops interleaved with the +- rows 1, 2 and 7, both entry releases, a late rank in turn, hops interleaved with the decode hop's own, so the three mailbox stages cycle; - the draft o_proj GEMV equals an fp32 emulation of its summation order, bit for bit, and its pushed output followed by a consume-only all-reduce equals publish + consume; @@ -171,7 +171,7 @@ def test_draft_hop_rounding(rank, draft): hop = draft.hop calls = [ (m, add_norm, early) - for m in (1, 7) + for m in (1, 2, 7) for add_norm in (False, True) for early in (False, True) ] diff --git a/test/unit/test_qwen3_5_runtime.py b/test/unit/test_qwen3_5_runtime.py index 3f189869..92b39297 100644 --- a/test/unit/test_qwen3_5_runtime.py +++ b/test/unit/test_qwen3_5_runtime.py @@ -237,8 +237,9 @@ def test_the_draft_slots_name_what_they_need(monkeypatch, slots, want): class _DraftParts: """Parts with no slot built: the stock model step.""" - def __init__(self, slots=frozenset()): + def __init__(self, slots=frozenset(), tp_size=8): self.slots, self.mkd, self.engaged = slots, None, [] + self.binding = SimpleNamespace(tp_size=tp_size) def note_engaged(self, entry, bs): self.engaged.append((entry, bs)) @@ -275,7 +276,7 @@ def test_the_draft_loop_writes_each_steps_token_and_restores_the_positions( _stub_model_step(monkeypatch, calls) parts = _DraftParts() loop = DraftLoop(parts, num_steps=4) - assert loop.step_for(SERVED_BS) == loop.step and loop.step_for(2) is None + assert loop.step_for(SERVED_BS) == loop.step io = SimpleNamespace( num_requests=1, num_steps=4, @@ -309,6 +310,75 @@ def test_the_draft_loop_writes_each_steps_token_and_restores_the_positions( ] +@pytest.mark.parametrize( + "tp_size, slots, served", + [ + (8, {"multi_request"}, (1, 2)), + (8, set(), (1,)), + (4, {"multi_request"}, (1,)), + ], +) +def test_the_draft_loop_serves_two_requests_at_tp8_on_request( + monkeypatch, tp_size, slots, served +): + from sgl_nvidia_mega.qwen3_5.draft.loop import DraftLoop + + monkeypatch.setattr(runtime, "SLOTS", frozenset(slots)) + loop = DraftLoop(_DraftParts(tp_size=tp_size), num_steps=4) + assert tuple(bs for bs in (1, 2, 4) if loop.step_for(bs) == loop.step) == served + + +@pytest.mark.parametrize("split_fc", [False, True]) +def test_a_two_request_draft_loop_runs_each_step_on_dense_rows(monkeypatch, split_fc): + from sgl_nvidia_mega.qwen3_5.draft import layer + from sgl_nvidia_mega.qwen3_5.draft.loop import DraftLoop + + calls = [] + _stub_model_step(monkeypatch, calls) + stub_layer = layer.layer + + def run_layer(parts, x, positions, cache_loc, meta, kv, *, rows_per_request): + calls.append(("dense", cache_loc.is_contiguous())) + return stub_layer( + parts, x, positions, cache_loc, meta, kv, rows_per_request=rows_per_request + ) + + monkeypatch.setattr(layer, "layer", run_layer) + parts = _DraftParts() + if split_fc: + # slot draft_xchg's row-split fc, never its one-request exchange steps + parts.model = SimpleNamespace(embed_tokens=lambda t: t.to(torch.float32)) + parts.mkd = SimpleNamespace( + split_fc=lambda e, h: (calls.append(("split", e.tolist())), h + 1)[1] + ) + loop = DraftLoop(parts, num_steps=4) + io = SimpleNamespace( + num_requests=2, + num_steps=4, + topk_index=torch.tensor([[7], [8]]), + hidden_states=torch.tensor([[1.0, 2.0], [3.0, 4.0]]), + positions=torch.tensor([100, 200]), + mrope_positions=None, + # request-major: request r's step i writes slot 40 + 10 r + i + out_cache_loc=torch.tensor([40, 41, 42, 43, 50, 51, 52, 53]), + steps=("m0", "m1", "m2"), + kv="kv", + draft_tokens_out=torch.full((2, 4), -1, dtype=torch.int64), + ) + loop.step(io) + assert parts.engaged == [("draft_loop", 2)] + assert [c[1:] for c in calls if c[0] == "layer"] == [ + ([100, 200], [40, 50], "m0", "kv"), + ([101, 201], [41, 51], "m1", "kv"), + ([102, 202], [42, 52], "m2", "kv"), + ] + assert [c[1] for c in calls if c[0] == "dense"] == [True] * 3 + fc = "split" if split_fc else "fc" + assert [c[1] for c in calls if c[0] == fc] == [[7, 8], [20, 40], [210, 410]] + assert io.draft_tokens_out.tolist() == [[7, 20, 210, 110], [8, 40, 410, 110]] + assert io.positions.tolist() == [100, 200] + + def test_the_draft_extend_writes_the_selected_rows_outputs(monkeypatch): from sgl_nvidia_mega.qwen3_5.draft.extend import DraftExtend From 3bd8c0ba45b447d092361577dbae592affcf084d Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Sun, 25 Oct 2026 21:12:14 -0700 Subject: [PATCH 27/27] Adapt megakernel specs to SGLang API v2 --- sgl_nvidia_mega/deepseek_v41/moe/step.py | 6 +++-- sgl_nvidia_mega/deepseek_v41/spec.py | 2 +- sgl_nvidia_mega/qwen3_5/boundary/hop.py | 34 +++++++++++++----------- sgl_nvidia_mega/qwen3_5/moe/__init__.py | 8 +++--- sgl_nvidia_mega/qwen3_5/spec.py | 2 +- 5 files changed, 29 insertions(+), 23 deletions(-) diff --git a/sgl_nvidia_mega/deepseek_v41/moe/step.py b/sgl_nvidia_mega/deepseek_v41/moe/step.py index bff95922..cb29fec1 100644 --- a/sgl_nvidia_mega/deepseek_v41/moe/step.py +++ b/sgl_nvidia_mega/deepseek_v41/moe/step.py @@ -63,7 +63,7 @@ "moe_runner_config", ), "srt/layers/moe/utils.py": ("sum_post_experts_output",), - "srt/layers/layer_boundary/fusions/cutedsl.py": ( + "srt/layers/layer_boundary/fusions/moe_finalize.py": ( "MoeDeferredFinalize", "from_flashinfer", ), @@ -324,7 +324,9 @@ def _dual_stream( current.wait_stream(mlp.alt_stream) if deferred and fwd.defer_moe_finalize: - from sglang.srt.layers.layer_boundary.fusions.cutedsl import MoeDeferredFinalize + from sglang.srt.layers.layer_boundary.fusions.moe_finalize import ( + MoeDeferredFinalize, + ) from sglang.srt.layers.moe.utils import sum_post_experts_output return MoeDeferredFinalize.from_flashinfer( diff --git a/sgl_nvidia_mega/deepseek_v41/spec.py b/sgl_nvidia_mega/deepseek_v41/spec.py index c3aad628..2821be85 100644 --- a/sgl_nvidia_mega/deepseek_v41/spec.py +++ b/sgl_nvidia_mega/deepseek_v41/spec.py @@ -64,7 +64,7 @@ ARCHITECTURE = "DeepseekV4ForCausalLM" # the SGLang megakernel contract this spec is written against -API_VERSION = 1 +API_VERSION = 2 # The FlashInfer low-latency GEMM file the wo_b publish epilogue subclasses, by the # sha256 of its source; it is byte-identical in FlashInfer 0.7.0, 0.7.1 and 0.7.2 FLASHINFER_LL_GEMM = "gemm/kernels/cute_dsl/low_latency_blockscaled_gemm.py" diff --git a/sgl_nvidia_mega/qwen3_5/boundary/hop.py b/sgl_nvidia_mega/qwen3_5/boundary/hop.py index d9249338..18a445d2 100644 --- a/sgl_nvidia_mega/qwen3_5/boundary/hop.py +++ b/sgl_nvidia_mega/qwen3_5/boundary/hop.py @@ -79,8 +79,6 @@ ), "srt/layers/layer_boundary/fusions/cutedsl.py": ( "CuteDSLFusion", - "MoeDeferredFinalize", - "_fused_norm_gamma", "_finalize_add_norm", "_ffn_input_all_reduce_add_norm", "_finalize_eligible", @@ -88,12 +86,16 @@ "hidden_size", "top_k", "rms_epsilon", + ), + "srt/layers/layer_boundary/fusions/moe_finalize.py": ( + "MoeDeferredFinalize", "routed_output", "expert_weights", "permuted_indices", "gated_shared_output", "m", ), + "srt/layers/layer_boundary/fusions/norm.py": ("fused_norm_gamma",), "srt/layers/layer_boundary/residual/batch.py": ("stream_of", "clear"), "srt/layers/layer_boundary/residual/stream.py": ( "DeclaredSum", @@ -123,9 +125,9 @@ "srt/layers/layer_boundary/stage.py:StageBoundary._select": "f317ed475a7672cf", "srt/layers/layer_boundary/prepare.py:_run_entry": "639740496fc09d2d", "srt/layers/layer_boundary/prepare.py:_reduce_update_read": "67badae13f96d80f", - "srt/layers/layer_boundary/fusions/cutedsl.py:CuteDSLFusion._finalize_add_norm": "46767c09458d7e1e", - "srt/layers/layer_boundary/fusions/cutedsl.py:CuteDSLFusion._ffn_input_all_reduce_add_norm": "84163a5a5a695823", - "srt/layers/layer_boundary/residual/batch.py:final_norm": "b4a1b4813ca5ac17", + "srt/layers/layer_boundary/fusions/cutedsl.py:CuteDSLFusion._finalize_add_norm": "51d463a1cecbbb64", + "srt/layers/layer_boundary/fusions/cutedsl.py:CuteDSLFusion._ffn_input_all_reduce_add_norm": "010e06f02e64affa", + "srt/layers/layer_boundary/residual/batch.py:final_norm": "13008609a1a7fd8a", } # the (point, rows) the slot has run at (each logged once, as engagement evidence) @@ -214,10 +216,8 @@ def refusals(m, world: int) -> list[str]: def path_refusals(layer) -> list[str]: """Why the ordinary paths of ``layer``'s two stages are not the ones mirrored.""" from sglang.srt.layers.layer_boundary.construction import BatchVariant - from sglang.srt.layers.layer_boundary.fusions.cutedsl import ( - CuteDSLFusion, - _fused_norm_gamma, - ) + from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion + from sglang.srt.layers.layer_boundary.fusions.norm import fused_norm_gamma from sglang.srt.layers.layer_boundary.prepare import ( _attn_input_default, _reduce_update_read, @@ -229,7 +229,7 @@ def path_refusals(layer) -> list[str]: plan, name = stage.plan, stage.kind.name.lower() if ( not isinstance(plan.fusions, CuteDSLFusion) - or _fused_norm_gamma(plan.norm) is None + or fused_norm_gamma(plan.norm) is None ): out.append(f"its {name} input runs no CuTe DSL fusion") continue @@ -295,10 +295,10 @@ def layer_input(hop, layer, hidden, st): """``attn_boundary.prepare`` of a layer whose input completes the previous MoE's finalize handoff (``hidden``), with the hop as its kernel; None where that prepare would not run the CuTe DSL kernel.""" - from sglang.srt.layers.layer_boundary.fusions.cutedsl import ( + from sglang.srt.layers.layer_boundary.fusions.moe_finalize import ( MoeDeferredFinalize, - _fused_norm_gamma, ) + from sglang.srt.layers.layer_boundary.fusions.norm import fused_norm_gamma from sglang.srt.layers.layer_boundary.residual.batch import stream_of stream = stream_of(st.fb) @@ -313,7 +313,7 @@ def layer_input(hop, layer, hidden, st): return None owed, residual = stream.input(hidden) norm, residual = _finalize( - hop, "layer input", owed, residual, _fused_norm_gamma(plan.norm) + hop, "layer input", owed, residual, fused_norm_gamma(plan.norm) ) stream.write(residual) return norm @@ -340,7 +340,7 @@ def moe_input(hop, layer, hidden, st): ``attn_boundary.finish`` recorded it), with the hop as its kernel; None where that prepare would not run the CuTe DSL kernel. A partial the mixer already pushed is only consumed.""" - from sglang.srt.layers.layer_boundary.fusions.cutedsl import _fused_norm_gamma + from sglang.srt.layers.layer_boundary.fusions.norm import fused_norm_gamma from sglang.srt.layers.layer_boundary.residual.batch import stream_of from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum @@ -362,7 +362,7 @@ def moe_input(hop, layer, hidden, st): return None _engage("MoE input", rows) norm, residual = hop.all_reduce_residual_rms_norm( - local_contribution=part, residual=residual, gamma=_fused_norm_gamma(plan.norm) + local_contribution=part, residual=residual, gamma=fused_norm_gamma(plan.norm) ) stream.write(residual) return norm @@ -371,7 +371,9 @@ def moe_input(hop, layer, hidden, st): def final_norm(hop, hidden, st): """``residual_batch.final_norm`` of the last MoE's finalize handoff (``hidden``) with the hop as its kernel; None when the stream holds no handoff the hop takes.""" - from sglang.srt.layers.layer_boundary.fusions.cutedsl import MoeDeferredFinalize + from sglang.srt.layers.layer_boundary.fusions.moe_finalize import ( + MoeDeferredFinalize, + ) from sglang.srt.layers.layer_boundary.residual.batch import clear, stream_of stream = stream_of(st.fb) diff --git a/sgl_nvidia_mega/qwen3_5/moe/__init__.py b/sgl_nvidia_mega/qwen3_5/moe/__init__.py index 1a785ab4..8e2ec90d 100644 --- a/sgl_nvidia_mega/qwen3_5/moe/__init__.py +++ b/sgl_nvidia_mega/qwen3_5/moe/__init__.py @@ -48,7 +48,7 @@ ), "srt/layers/moe/fused_moe_triton/layer.py": ("forward_deferred_finalize",), "srt/layers/moe/utils.py": ("sum_post_experts_output",), - "srt/layers/layer_boundary/fusions/cutedsl.py": ( + "srt/layers/layer_boundary/fusions/moe_finalize.py": ( "MoeDeferredFinalize", "from_flashinfer", "from_flashinfer(gated_shared_output)", @@ -62,7 +62,7 @@ # The SGLang functions whose body this module mirrors, by the digest of their code # (test/unit/test_contract.py recomputes them on the installed SGLang) SGLANG_MIRRORS = { - "srt/models/qwen2_moe.py:Qwen2MoeSparseMoeBlock.forward": "1994cd43fba4fbde", + "srt/models/qwen2_moe.py:Qwen2MoeSparseMoeBlock.forward": "633ff9a686d3ea28", "srt/models/qwen2_moe.py:Qwen2MoeSparseMoeBlock.forward_normal_dual_stream": "afae959064cc8cd7", "srt/models/qwen2_moe.py:Qwen2MoeSparseMoeBlock._forward_router_experts": "32d09f01b77d76b5", } @@ -92,7 +92,9 @@ def step(layer, x: torch.Tensor, st, defer: bool): """``layer``'s MoE on the normed input ``x`` [rows, hidden]: the deferred-finalize handoff with ``defer``, else the unreduced rows [rows, hidden].""" from sglang.kernels.ops.elementwise.elementwise import fused_gate_sigmoid_mul_add - from sglang.srt.layers.layer_boundary.fusions.cutedsl import MoeDeferredFinalize + from sglang.srt.layers.layer_boundary.fusions.moe_finalize import ( + MoeDeferredFinalize, + ) from sglang.srt.layers.moe.utils import sum_post_experts_output from sglang.srt.model_executor.runner import get_is_capture_mode diff --git a/sgl_nvidia_mega/qwen3_5/spec.py b/sgl_nvidia_mega/qwen3_5/spec.py index c92c5a53..3d2f760b 100644 --- a/sgl_nvidia_mega/qwen3_5/spec.py +++ b/sgl_nvidia_mega/qwen3_5/spec.py @@ -70,7 +70,7 @@ ARCHITECTURE = "Qwen3_5MoeForConditionalGeneration" # The SGLang megakernel contract this spec is written against. -API_VERSION = 1 +API_VERSION = 2 REQUIREMENTS = Requirements( isas=frozenset({"sm_103"}),