From cbdc00a2599506e14b9837b65448d35a6ec9222f Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:43:43 +0000 Subject: [PATCH 01/11] feat(qwen3-next): MoE route/combine contract with a batch-invariant grouped expert GEMM RFC #428 moe_route_combine_contract (CUDA). - Routing: FP32 router through the pinned vLLM batch-invariant GEMM, a fixed-order softmax, stable descending argsort (ties go to the lower expert id) and FP32 renormalisation of the ten selected weights. - Experts: per-route gate_up and down projections through vLLM 0.30.0's fused_moe Triton kernel with the batch-invariant tile passed explicitly (BLOCK 64/64/32, SPLIT_K=1); no routed-weight multiply and no sum inside the kernel, so every (token, route) row has one writer. Bitwise equal to the previous per-expert GEMM loop at TP4 and TP1 widths; fails closed if VLLM_TRITON_USE_TD is enabled. - Combine: ten ordered FP32 multiply-adds and one BF16 cast; no atomics. - Backward: per-expert pinned GEMMs into one dense gradient buffer; dx is summed in ascending expert order. - TP4 block: HF load/export for all 512 experts, replicated router and shared-expert gate, Megatron-style copy/reduce autograd boundaries moved into qwen3_next_tp.py. - Prior-art runner and plot comparing HF transformers, vLLM fused_moe (BI=0/1) and FlashInfer cutlass_fused_moe; TP4 real-weight gate script. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- .github/workflows/qwen3-next-provider-gpu.yml | 11 +- rl_engine/integrations/qwen3_next_forward.py | 338 ++++++++++++++ rl_engine/integrations/qwen3_next_tp.py | 65 +++ .../integrations/qwen3_next_tp_blocks.py | 138 ++++++ rl_engine/testing/qwen3_next_moe_prior_art.py | 440 ++++++++++++++++++ rl_engine/testing/tensor_identity.py | 38 ++ scripts/plot_qwen3_next_moe_prior_art.py | 159 +++++++ scripts/qwen3_next_moe_prior_art.py | 87 ++++ scripts/qwen3_next_tp_moe_check.py | 207 ++++++++ tests/check_qwen3_next_forward.py | 238 ++++++++++ tests/test_qwen3_next_forward_contract.py | 62 +++ tests/test_qwen3_next_tp_blocks.py | 93 ++++ tests/test_tensor_identity.py | 41 ++ 14 files changed, 1915 insertions(+), 4 deletions(-) create mode 100644 rl_engine/integrations/qwen3_next_forward.py create mode 100644 rl_engine/integrations/qwen3_next_tp.py create mode 100644 rl_engine/integrations/qwen3_next_tp_blocks.py create mode 100644 rl_engine/testing/qwen3_next_moe_prior_art.py create mode 100644 rl_engine/testing/tensor_identity.py create mode 100644 scripts/plot_qwen3_next_moe_prior_art.py create mode 100644 scripts/qwen3_next_moe_prior_art.py create mode 100644 scripts/qwen3_next_tp_moe_check.py create mode 100644 tests/check_qwen3_next_forward.py create mode 100644 tests/test_qwen3_next_forward_contract.py create mode 100644 tests/test_qwen3_next_tp_blocks.py create mode 100644 tests/test_tensor_identity.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 814a4ae22..2fb92d641 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,7 @@ jobs: run: | python -m pytest rl_engine/tests/test_dispatch.py -v PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest tests/test_attention_correctness.py -q -rs - python -m pytest tests/test_forward_invariance.py tests/test_tolerance_contract.py tests/test_ws1_workload.py tests/test_gradient_invariance.py tests/test_elementwise_inventory.py tests/test_four_judgment_matrix.py tests/test_op_checks.py tests/test_operator_inputs.py tests/test_profiler.py tests/test_kv_consistency.py tests/test_ws1_qwen3_dense.py tests/test_ws1_chain_integration.py tests/test_qwen3_next_norm.py tests/test_gdn_state_contract.py -q + python -m pytest tests/test_forward_invariance.py tests/test_tolerance_contract.py tests/test_ws1_workload.py tests/test_gradient_invariance.py tests/test_elementwise_inventory.py tests/test_four_judgment_matrix.py tests/test_op_checks.py tests/test_operator_inputs.py tests/test_profiler.py tests/test_kv_consistency.py tests/test_ws1_qwen3_dense.py tests/test_ws1_chain_integration.py tests/test_qwen3_next_norm.py tests/test_gdn_state_contract.py tests/test_tensor_identity.py tests/test_qwen3_next_forward_contract.py tests/test_qwen3_next_tp_blocks.py -q - name: Run Cross-Configuration Contract Tests (CPU-safe) run: | diff --git a/.github/workflows/qwen3-next-provider-gpu.yml b/.github/workflows/qwen3-next-provider-gpu.yml index 146818019..691489402 100644 --- a/.github/workflows/qwen3-next-provider-gpu.yml +++ b/.github/workflows/qwen3-next-provider-gpu.yml @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 -# RFC #428 Qwen3-Next provider comparisons: the C1/C3/C4 norm providers and the C6 -# GDN decode-step goldens, each checked against the real vLLM 0.30.0 provider. +# RFC #428 Qwen3-Next provider comparisons: the C1/C3/C4 norm providers, the C6 +# GDN decode-step goldens and the shared GEMM/MoE route-combine provider, each +# checked against the real vLLM 0.30.0 provider. # # These check_ files import vLLM, so the default pytest collection never runs them # (tests/test_framework_operator_integrations.py asserts vLLM is not imported). This @@ -19,6 +20,7 @@ on: branches: [main] paths: - "rl_engine/kernels/ops/**" + - "rl_engine/integrations/qwen3_next*" - "rl_engine/testing/qwen3_next*" - "tests/check_*qwen3_next*.py" - "tests/check_gdn_recurrent_golden.py" @@ -78,5 +80,8 @@ jobs: print("rl_engine._C:", loaded) PY - name: Check pinned providers in their own process + # The FP32 router needs IEEE Triton dots from process start. + env: + TRITON_F32_DEFAULT: ieee run: | - python3 -m pytest -q tests/check_qwen3_next_norm_providers.py tests/check_gdn_recurrent_golden.py + python3 -m pytest -q tests/check_qwen3_next_norm_providers.py tests/check_gdn_recurrent_golden.py tests/check_qwen3_next_forward.py diff --git a/rl_engine/integrations/qwen3_next_forward.py b/rl_engine/integrations/qwen3_next_forward.py new file mode 100644 index 000000000..b5a1e20e0 --- /dev/null +++ b/rl_engine/integrations/qwen3_next_forward.py @@ -0,0 +1,338 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Explicit CUDA forward primitives for the Qwen3-Next shared TP4 profile. + +These functions do not install framework hooks or advertise engine capabilities. +The caller owns parameter sharding, TP collectives, positions and cache identity. +Weights remain BF16; only the router explicitly computes in FP32, as in VIME's +Qwen3-Next configuration. No CPU or alternate-kernel fallback is permitted. +""" + +from dataclasses import dataclass +from functools import lru_cache +from importlib.metadata import version + +import torch +import torch.nn.functional as F +from torch.autograd.function import once_differentiable + +FORWARD_PROVIDER_ID = "qwen3-next-shared-cuda-forward-v1" + + +@lru_cache(maxsize=1) +def _vllm_linear(): + if version("vllm") != "0.30.0": + raise RuntimeError("Shared Qwen3-Next linear requires pinned vLLM 0.30.0") + from vllm.model_executor.determinism.batch_invariant import linear_batch_invariant + + return linear_batch_invariant + + +# The batch-invariant tile vLLM itself selects under VLLM_BATCH_INVARIANT=1 +# (fused_moe.get_default_config), passed explicitly so no environment variable, +# tuned-config file or token count can change it. +_GROUPED_CONFIG = { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + "SPLIT_K": 1, +} + + +@lru_cache(maxsize=1) +def _vllm_grouped(): + if version("vllm") != "0.30.0": + raise RuntimeError("Shared Qwen3-Next experts require pinned vLLM 0.30.0") + from vllm.model_executor.layers.fused_moe.fused_moe import ( + _prepare_expert_assignment, + dispatch_fused_moe_kernel, + ) + from vllm.model_executor.layers.fused_moe.utils import resolve_moe_use_td + + return _prepare_expert_assignment, dispatch_fused_moe_kernel, resolve_moe_use_td + + +def grouped_route_linear(rows, weight, route_indices): + """Per-route ``rows @ weight[expert].T`` through vLLM's fused MoE Triton kernel. + + ``rows`` is [T, K] (one row per token, shared by its ten routes) or + [T*10, K] (one row per route); the result is [T, 10, N] in route order. + No routed weight is applied and nothing is summed: every output row has + exactly one writer, and its K loop has the fixed order of the fixed tile. + """ + prepare, dispatch, use_td = _vllm_grouped() + if use_td(): + raise RuntimeError("VLLM_TRITON_USE_TD changes the MoE kernel; strict mode requires 0") + import triton.language as tl + + tokens, top_k = route_indices.shape + out = rows.new_empty((tokens, top_k, weight.shape[1])) + if tokens == 0: + return out + indices = route_indices.to(torch.int32).contiguous() + sorted_ids, expert_ids, padded = prepare( + indices, _GROUPED_CONFIG, tokens, top_k, weight.shape[0], None, ignore_invalid_experts=True + ) + dispatch( + rows.contiguous(), + weight, + out, + None, + None, + None, + None, + sorted_ids, + expert_ids, + padded, + False, + top_k if rows.shape[0] == tokens else 1, + _GROUPED_CONFIG, + compute_type=tl.bfloat16, + use_fp8_w8a8=False, + use_int8_w8a8=False, + use_int8_w8a16=False, + use_int4_w4a16=False, + per_channel_quant=False, + ) + return out + + +def _cuda_tensors(*tensors): + if torch.version.hip is not None or any(not value.is_cuda for value in tensors): + raise ValueError("Shared Qwen3-Next forward requires CUDA tensors; no CPU fallback") + if any(value.device != tensors[0].device for value in tensors[1:]): + raise ValueError("Shared Qwen3-Next tensors must be on the same device") + + +class _SharedLinear(torch.autograd.Function): + @staticmethod + def forward(ctx, x, weight, bias): + ctx.save_for_backward(x, weight) + ctx.has_bias = bias is not None + if x.shape[0] == 0: + return x.new_empty((0, weight.shape[0])) + with torch.cuda.device(x.device): + return _vllm_linear()(x, weight, bias) + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + x, weight = ctx.saved_tensors + if x.shape[0] == 0: + return ( + torch.empty_like(x) if ctx.needs_input_grad[0] else None, + torch.zeros_like(weight) if ctx.needs_input_grad[1] else None, + weight.new_zeros(weight.shape[0]) if ctx.has_bias else None, + ) + grad = grad_output.contiguous() + with torch.cuda.device(x.device): + linear = _vllm_linear() + dx = linear(grad, weight.t()) if ctx.needs_input_grad[0] else None + dw = linear(grad.t(), x.t()) if ctx.needs_input_grad[1] else None + db = grad.sum(dim=0) if ctx.has_bias and ctx.needs_input_grad[2] else None + return dx, dw, db + + +def shared_linear(x, weight, bias=None): + """Pinned vLLM deterministic GEMM with an explicit first-order VJP. + + x is [..., K], weight is [N, K], output is [..., N]. Operands have one + common BF16 or FP32 dtype. Empty token dimensions preserve autograd. + Backward executes dX=dY@W and dW=dY.T@X through the same fixed provider. + """ + _cuda_tensors(x, weight, *(() if bias is None else (bias,))) + if x.ndim < 2 or weight.ndim != 2 or x.shape[-1] != weight.shape[1]: + raise ValueError("Shared linear requires x[..., K] and weight[N, K]") + if x.dtype not in (torch.bfloat16, torch.float32) or weight.dtype != x.dtype: + raise ValueError("Shared linear operands must have one common BF16 or FP32 dtype") + if x.dtype == torch.float32: + import triton + + if triton.knobs.language.fp32_default != "ieee": + raise RuntimeError( + "FP32 shared linear requires TRITON_F32_DEFAULT=ieee at process startup" + ) + if min(weight.shape) < 1: + raise ValueError("Shared linear weight dimensions must be positive") + if bias is not None and (bias.shape != (weight.shape[0],) or bias.dtype != x.dtype): + raise ValueError("Shared linear bias must be [N] with the input dtype") + out = _SharedLinear.apply(x.reshape(-1, x.shape[-1]), weight, bias) + return out.reshape(*x.shape[:-1], weight.shape[0]) + + +def shared_router(x, weight): + """Compute the 512-expert FP32 router from BF16 inputs/parameter storage.""" + _cuda_tensors(x, weight) + if x.dtype != torch.bfloat16 or weight.dtype != torch.bfloat16: + raise ValueError("Qwen3-Next router input and stored weight must be BF16") + if x.ndim != 2 or weight.shape != (512, x.shape[-1]): + raise ValueError("Qwen3-Next router requires x[T, H] and weight[512, H]") + return shared_linear(x.float(), weight.float()) + + +@dataclass(frozen=True) +class Routing: + """Descending probability routes; equal scores prefer smaller expert IDs.""" + + weights: torch.Tensor + indices: torch.Tensor + + +def fixed_order_row_sum(values): + """Sum the last dimension by a pairwise tree of elementwise adds. + + ``torch.sum`` picks its reduction order from the tensor shape, so the same + row summed inside an 8-row and a 32-row tensor can differ by one ulp unless + vLLM's batch-invariant aten overrides are installed. Qualification runs + execute with ``VLLM_BATCH_INVARIANT=0``, where those overrides are absent. + Elementwise adds have no such freedom: every row is summed in one fixed + tree whatever the row count. Odd widths are padded with exact zeros. + """ + while values.shape[-1] > 1: + if values.shape[-1] % 2: + values = torch.cat((values, values.new_zeros((*values.shape[:-1], 1))), dim=-1) + values = values[..., 0::2] + values[..., 1::2] + return values + + +def fixed_order_softmax(logits): + """Row softmax from a max subtraction, exp and one fixed-order row sum.""" + shifted = logits - torch.amax(logits, dim=-1, keepdim=True) + exponentials = torch.exp(shifted) + return exponentials / fixed_order_row_sum(exponentials) + + +def stable_top10_routes(router_logits): + """Select ten distinct experts and normalize their FP32 probabilities. + + The full 512-way softmax precedes selection, preserving Qwen's normalized + top-k arithmetic. Stable sort fixes the otherwise ambiguous tie boundary. + Indices are discrete; gradients flow through the selected probabilities. + Both row sums use the fixed-order tree, so a token's route weights do not + depend on how many other tokens share its forward pass. + """ + _cuda_tensors(router_logits) + if router_logits.ndim != 2 or router_logits.shape[1] != 512: + raise ValueError("Qwen3-Next routing requires logits[T, 512]") + if router_logits.dtype != torch.float32: + raise ValueError("Qwen3-Next router logits must be FP32") + if not torch.isfinite(router_logits).all(): + raise ValueError("Qwen3-Next router logits must be finite") + probabilities = fixed_order_softmax(router_logits) + indices = torch.argsort(router_logits, dim=-1, descending=True, stable=True)[:, :10] + selected = probabilities.gather(1, indices) + return Routing(selected / fixed_order_row_sum(selected), indices) + + +def combine_routes(expert_outputs, route_weights): + """Combine [T, 10, H] BF16 outputs by ten ordered FP32 multiply-adds. + + There are no atomic sums. Each route's product and each addition is a + separate eager operation; the single BF16 cast follows the final route. + """ + _cuda_tensors(expert_outputs, route_weights) + if expert_outputs.ndim != 3 or expert_outputs.shape[1] != 10: + raise ValueError("Qwen3-Next expert outputs must have shape [T, 10, H]") + if route_weights.shape != expert_outputs.shape[:2]: + raise ValueError("Route weights must have shape [T, 10]") + if expert_outputs.dtype != torch.bfloat16 or route_weights.dtype != torch.float32: + raise ValueError("Combine requires BF16 expert outputs and FP32 route weights") + output = expert_outputs[:, 0].float() * route_weights[:, 0, None] + for slot in range(1, 10): + output = output + expert_outputs[:, slot].float() * route_weights[:, slot, None] + return output.to(torch.bfloat16) + + +def _expert_activation(gate_up_rows): + """SwiGLU in FP32 from the BF16 [gate, up] projection; returns its parts too.""" + gate, up = gate_up_rows.float().chunk(2, dim=-1) + silu_gate = F.silu(gate) + return silu_gate * up, silu_gate, gate, up + + +class _RoutedExperts(torch.autograd.Function): + """Grouped expert forward; per-expert backward into one dense gradient buffer.""" + + @staticmethod + def forward(ctx, x, gate_up, down, route_indices): + ctx.save_for_backward(x, gate_up, down, route_indices) + with torch.cuda.device(x.device): + projected = grouped_route_linear(x, gate_up, route_indices) + activated = _expert_activation(projected)[0] + activated = activated.to(x.dtype).reshape(-1, activated.shape[-1]) + return grouped_route_linear(activated, down, route_indices) + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + x, gate_up, down, route_indices = ctx.saved_tensors + dx = torch.zeros_like(x) if ctx.needs_input_grad[0] else None + dgate_up = torch.zeros_like(gate_up) if ctx.needs_input_grad[1] else None + ddown = torch.zeros_like(down) if ctx.needs_input_grad[2] else None + top_k = route_indices.shape[1] + slots_by_expert = [[] for _ in range(gate_up.shape[0])] + for slot, expert in enumerate(route_indices.flatten().tolist()): + slots_by_expert[expert].append(slot) + grad_flat = grad_output.reshape(-1, x.shape[1]) + with torch.cuda.device(x.device): + linear = _vllm_linear() + projected = grouped_route_linear(x, gate_up, route_indices) + projected = projected.reshape(-1, projected.shape[-1]) + for expert, slots in enumerate(slots_by_expert): + if not slots: + continue + slots = torch.tensor(slots, dtype=torch.long, device=x.device) + tokens = slots // top_k + activated, silu_gate, gate32, up32 = _expert_activation( + projected.index_select(0, slots) + ) + activated = activated.to(x.dtype) + grad = grad_flat.index_select(0, slots) + if ddown is not None: + ddown[expert].copy_(linear(grad.t(), activated.t())) + if dx is None and dgate_up is None: + continue + rows = x.index_select(0, tokens) + dactivated = linear(grad, down[expert].t()).float() + sigmoid = gate32.sigmoid() + dgate = dactivated * up32 * (sigmoid + gate32 * sigmoid * (1 - sigmoid)) + dup = dactivated * silu_gate + dprojected = torch.cat((dgate, dup), dim=-1).to(x.dtype) + if dgate_up is not None: + dgate_up[expert].copy_(linear(dprojected.t(), rows.t())) + if dx is not None: + # Top-k has distinct experts, so tokens are unique within this + # launch. Ascending expert order fixes each token's sum order. + partial = linear(dprojected, gate_up[expert].t()) + dx.index_copy_(0, tokens, dx.index_select(0, tokens) + partial) + return dx, dgate_up, ddown, None + + +def shared_moe(x, router_weight, gate_up_weights, down_weights): + """Evaluate TP-local routed experts; caller reduces the returned output. + + Weights are [512, 2*I_local, H] (gate then up) and [512, H, I_local]. + All experts belong to EP=1. Each (token, route) output row has exactly one + writer and the ten routes are combined in route order. Shared-expert and + TP-reduction ownership remains with the model adapter. The forward has no + host synchronisation; the backward builds per-expert row lists on the host. + """ + _cuda_tensors(x, router_weight, gate_up_weights, down_weights) + if x.ndim != 2 or x.dtype != torch.bfloat16: + raise ValueError("Qwen3-Next MoE input must be BF16 [T, H]") + if gate_up_weights.ndim != 3 or down_weights.ndim != 3: + raise ValueError("Qwen3-Next expert weights must have three dimensions") + local_intermediate = down_weights.shape[-1] + if local_intermediate < 1: + raise ValueError("Expert intermediate dimension must be positive") + if gate_up_weights.shape != (512, 2 * local_intermediate, x.shape[1]): + raise ValueError("gate_up_weights must have shape [512, 2*I_local, H]") + if down_weights.shape != (512, x.shape[1], local_intermediate): + raise ValueError("down_weights must have shape [512, H, I_local]") + if gate_up_weights.dtype != x.dtype or down_weights.dtype != x.dtype: + raise ValueError("Qwen3-Next expert weights must remain BF16") + routes = stable_top10_routes(shared_router(x, router_weight)) + outputs = _RoutedExperts.apply(x, gate_up_weights, down_weights, routes.indices) + return combine_routes(outputs, routes.weights), routes diff --git a/rl_engine/integrations/qwen3_next_tp.py b/rl_engine/integrations/qwen3_next_tp.py new file mode 100644 index 000000000..884a2d043 --- /dev/null +++ b/rl_engine/integrations/qwen3_next_tp.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Differentiable TP4 boundaries and parameter metadata shared by Qwen3-Next blocks. + +The forward copy and reduction follow Megatron's column/row-parallel autograd +pair. Reductions go through ``collective_for_group`` so a strict deployment +can pin the collective algorithm; nothing here selects one implicitly. +""" + +import torch +import torch.distributed as dist + + +def _reduce(value, group): + from rl_engine.distributed.collectives import collective_for_group + + collective = collective_for_group(group, min_size_bytes=value.numel() * value.element_size()) + return collective.all_reduce(value.contiguous()) + + +class _CopyToTP(torch.autograd.Function): + @staticmethod + def forward(ctx, value, group): + ctx.group = group + return value + + @staticmethod + def backward(ctx, gradient): + return _reduce(gradient, ctx.group), None + + +class _ReduceFromTP(torch.autograd.Function): + @staticmethod + def forward(ctx, value, group): + return _reduce(value, group) + + @staticmethod + def backward(ctx, gradient): + return gradient, None + + +def _rank(rank): + if isinstance(rank, bool) or not isinstance(rank, int) or rank not in range(4): + raise ValueError("TP4 rank must be an integer in [0, 4)") + + +def _weight(value, shape, name): + if value.dtype != torch.bfloat16 or value.shape != shape: + raise ValueError(f"{name} must be BF16 {shape}") + + +def _tp_group(group): + if not dist.is_initialized() or dist.get_world_size(group) != 4: + raise ValueError("Qwen3-Next blocks require an initialized four-rank TP group") + return dist.group.WORLD if group is None else group + + +def _parallel_parameter(parameter, dimension, *, stride=1, duplicate=False): + # MCore uses these attributes when computing the global optimizer gradient + # norm. A replicated copy updates normally but must not count twice. + parameter.tensor_model_parallel = True + parameter.partition_dim = dimension + parameter.partition_stride = stride + parameter.shared = duplicate diff --git a/rl_engine/integrations/qwen3_next_tp_blocks.py b/rl_engine/integrations/qwen3_next_tp_blocks.py new file mode 100644 index 000000000..1c80530d4 --- /dev/null +++ b/rl_engine/integrations/qwen3_next_tp_blocks.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Official Qwen3-Next TP4 MoE block for shared adapters. + +The block owns the local arithmetic and differentiable TP boundaries. It does +not register an engine backend or advertise full-model acceptance. +""" + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch import nn + +from rl_engine.integrations.qwen3_next_forward import shared_linear, shared_moe +from rl_engine.integrations.qwen3_next_tp import ( + _CopyToTP, + _parallel_parameter, + _rank, + _ReduceFromTP, + _tp_group, + _weight, +) +from rl_engine.testing.tensor_identity import tensor_bitwise_equal + + +def moe_hf_shapes(): + result = { + "gate.weight": (512, 2048), + "shared_expert_gate.weight": (1, 2048), + } + for prefix in ("shared_expert", *(f"experts.{expert}" for expert in range(512))): + result[f"{prefix}.gate_proj.weight"] = (512, 2048) + result[f"{prefix}.up_proj.weight"] = (512, 2048) + result[f"{prefix}.down_proj.weight"] = (2048, 512) + return result + + +def shard_moe_weight(name, value, rank): + """Shard one official HF tensor without materializing all expert weights.""" + _rank(rank) + shapes = moe_hf_shapes() + if name not in shapes: + raise ValueError(f"Unknown Qwen3-Next MoE weight: {name}") + _weight(value, shapes[name], name) + if name in ("gate.weight", "shared_expert_gate.weight"): + return value + return value.chunk(4, dim=1 if name.endswith("down_proj.weight") else 0)[rank].contiguous() + + +def assemble_moe_weight(name, values): + if len(values) != 4: + raise ValueError("Exactly four MoE weight shards are required") + shapes = moe_hf_shapes() + if name not in shapes: + raise ValueError(f"Unknown Qwen3-Next MoE weight: {name}") + if name in ("gate.weight", "shared_expert_gate.weight"): + for value in values: + _weight(value, shapes[name], name) + if any(not tensor_bitwise_equal(value, values[0]) for value in values[1:]): + raise ValueError(f"Replicated MoE weight differs across TP: {name}") + return values[0] + axis = 1 if name.endswith("down_proj.weight") else 0 + shape = list(shapes[name]) + shape[axis] //= 4 + for value in values: + _weight(value, tuple(shape), name) + return torch.cat(tuple(values), dim=axis) + + +class TP4MoE(nn.Module): + """All 512 experts at EP1 with intermediate width 128 per TP rank.""" + + def __init__(self, *, group, device): + super().__init__() + self.group = _tp_group(group) + self.rank = dist.get_rank(self.group) + factory = {"device": device, "dtype": torch.bfloat16} + self.gate = nn.Linear(2048, 512, bias=False, **factory) + self.experts = nn.Module() + self.experts.register_parameter( + "gate_up", nn.Parameter(torch.empty(512, 256, 2048, **factory)) + ) + self.experts.register_parameter( + "down", nn.Parameter(torch.empty(512, 2048, 128, **factory)) + ) + self.shared_expert = nn.Module() + self.shared_expert.gate_proj = nn.Linear(2048, 128, bias=False, **factory) + self.shared_expert.up_proj = nn.Linear(2048, 128, bias=False, **factory) + self.shared_expert.down_proj = nn.Linear(128, 2048, bias=False, **factory) + self.shared_expert_gate = nn.Linear(2048, 1, bias=False, **factory) + _parallel_parameter(self.experts.gate_up, 1, stride=2) + _parallel_parameter(self.experts.down, 2) + _parallel_parameter(self.shared_expert.gate_proj.weight, 0) + _parallel_parameter(self.shared_expert.up_proj.weight, 0) + _parallel_parameter(self.shared_expert.down_proj.weight, 1) + + def _hf_tensor(self, name): + if name.startswith("experts."): + _, expert, projection, _ = name.split(".") + expert = int(expert) + if projection == "down_proj": + return self.experts.down[expert] + offset = 0 if projection == "gate_proj" else 128 + return self.experts.gate_up[expert, offset : offset + 128] + obj = self + for part in name.split("."): + obj = getattr(obj, part) + return obj + + def load_hf_weights(self, weights): + if set(weights) != set(moe_hf_shapes()): + raise ValueError("MoE weight names do not match the official HF contract") + with torch.no_grad(): + for name in moe_hf_shapes(): + self._hf_tensor(name).copy_(shard_moe_weight(name, weights[name], self.rank)) + + def export_local_hf_weights(self): + return {name: self._hf_tensor(name).detach() for name in moe_hf_shapes()} + + def forward(self, hidden): + if hidden.ndim != 2 or hidden.shape[1] != 2048 or hidden.dtype != torch.bfloat16: + raise ValueError("MoE hidden must be BF16 [tokens, 2048]") + copied = _CopyToTP.apply(hidden, self.group) + routed, _ = shared_moe( + copied, + _CopyToTP.apply(self.gate.weight, self.group), + self.experts.gate_up, + self.experts.down, + ) + gate = shared_linear(copied, self.shared_expert.gate_proj.weight) + up = shared_linear(copied, self.shared_expert.up_proj.weight) + activated = (F.silu(gate.float()) * up.float()).to(hidden.dtype) + shared = shared_linear(activated, self.shared_expert.down_proj.weight) + shared_gate = shared_linear( + copied, _CopyToTP.apply(self.shared_expert_gate.weight, self.group) + ).sigmoid() + return _ReduceFromTP.apply(routed + shared * shared_gate, self.group) diff --git a/rl_engine/testing/qwen3_next_moe_prior_art.py b/rl_engine/testing/qwen3_next_moe_prior_art.py new file mode 100644 index 000000000..d8ad92234 --- /dev/null +++ b/rl_engine/testing/qwen3_next_moe_prior_art.py @@ -0,0 +1,440 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Prior-art comparison for the Qwen3-Next routed MoE (RFC #428 ``moe_route_combine_contract``). + +Every candidate computes the routed half of ``Qwen3NextSparseMoeBlock`` at the +official TP1 shape (H=2048, 512 experts, top-10, expert width 512): router +logits, top-10 selection with renormalised weights, the selected SwiGLU +experts and the weighted combine. The shared expert is left out because every +candidate evaluates it as the same dense MLP. + +For each candidate the report records + +* **routing batch invariance** - the expert ids and weights of a fixed block + of probe tokens computed alone, and embedded first and last in larger + batches, must be bitwise equal; +* **output batch invariance** - the same for the routed output; +* **dx / dW batch invariance** (candidates with a backward) - the probe rows' + input gradient must not change when unrelated rows join the batch, and the + expert weight gradient must not change when rows whose output gradient is + zero join it (mathematically neither can); +* **repeatability** - two identical calls are bitwise equal; +* **accuracy** - error against an FP64 evaluation of the HF formula with FP64 + routing, plus the fraction of tokens whose selected expert set differs; +* **performance** - median CUDA-event latency, candidates interleaved. + +A candidate that cannot be imported or launched in the pinned environment is +recorded as ``unavailable`` with the reason instead of failing the run. +""" + +from __future__ import annotations + +import os +import statistics +from contextlib import contextmanager +from typing import Any, Callable + +import torch +import torch.nn.functional as F + +HIDDEN = 2048 +EXPERTS = 512 +TOPK = 10 +WIDTH = 512 +PROBE = 8 +BI_SIZES = (16, 64, 256, 1024) +PERF_TOKENS = (1, 8, 64, 256, 1024, 4096) +BACKWARD_TOKENS = (64, 1024) + + +# --------------------------------------------------------------------------- # +# Measurement helpers +# --------------------------------------------------------------------------- # + + +def _sample_us(fn: Callable[[], Any]) -> float: + torch.cuda.synchronize() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + fn() + end.record() + torch.cuda.synchronize() + return start.elapsed_time(end) * 1000.0 + + +def time_interleaved(calls: dict[str, Callable[[], Any]], warmup: int = 5, iters: int = 30): + """Median latency per candidate; the order alternates so none always runs first.""" + keys = list(calls) + orders = (keys, list(reversed(keys))) + for i in range(warmup): + for key in orders[i % 2]: + calls[key]() + samples = {key: [] for key in keys} + for i in range(iters): + for key in orders[i % 2]: + samples[key].append(_sample_us(calls[key])) + return {key: statistics.median(values) for key, values in samples.items()} + + +def _bitwise(a: torch.Tensor, b: torch.Tensor) -> bool: + return a.shape == b.shape and a.dtype == b.dtype and torch.equal(a, b) + + +def _rel(value: torch.Tensor, reference: torch.Tensor) -> float: + reference = reference.double() + return float((value.double() - reference).norm() / reference.norm().clamp_min(1e-300)) + + +def _guard(key: str, factory: Callable[[], dict[str, Any]]) -> dict[str, Any]: + try: + return {"key": key, **factory()} + except Exception as exc: # noqa: BLE001 - record and continue + return {"key": key, "unavailable": f"{type(exc).__name__}: {str(exc)[:300]}"} + + +@contextmanager +def _env(name: str, value: str | None): + old = os.environ.get(name) + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + try: + yield + finally: + if old is None: + os.environ.pop(name, None) + else: + os.environ[name] = old + + +# --------------------------------------------------------------------------- # +# Inputs and the FP64 reference +# --------------------------------------------------------------------------- # + + +def make_weights(seed: int = 0) -> dict[str, torch.Tensor]: + g = torch.Generator(device="cuda").manual_seed(seed) + + def randn(*shape, scale): + return (torch.randn(shape, device="cuda", generator=g) * scale).to(torch.bfloat16) + + return { + "router": randn(EXPERTS, HIDDEN, scale=0.02), + "gate_up": randn(EXPERTS, 2 * WIDTH, HIDDEN, scale=0.02), + "down": randn(EXPERTS, HIDDEN, WIDTH, scale=0.02), + } + + +def make_tokens(count: int, seed: int) -> torch.Tensor: + g = torch.Generator(device="cuda").manual_seed(seed) + return torch.randn(count, HIDDEN, device="cuda", generator=g).to(torch.bfloat16) + + +def fp64_reference(x: torch.Tensor, w: dict[str, torch.Tensor]): + """HF formula in FP64 with FP64 routing; returns (output, expert ids).""" + x64 = x.double() + probs = (x64 @ w["router"].double().T).softmax(-1) + weights, ids = probs.topk(TOPK, dim=-1) + weights = weights / weights.sum(-1, keepdim=True) + out = torch.zeros_like(x64) + gate_up, down = w["gate_up"], w["down"] + for expert in ids.unique().tolist(): + token, slot = torch.where(ids == expert) + gate, up = (x64[token] @ gate_up[expert].double().T).chunk(2, dim=-1) + y = (F.silu(gate) * up) @ down[expert].double().T + out.index_add_(0, token, y * weights[token, slot, None]) + return out, ids + + +# --------------------------------------------------------------------------- # +# Candidates +# +# ``route(x) -> (ids [T,10], weights [T,10])`` exposes the routing decision; +# ``fwd(x) -> [T, H]`` is the routed output; ``graph(x) -> (y, leaves)`` (when +# the candidate is differentiable) returns the output and [x, gate_up, down]. +# --------------------------------------------------------------------------- # + + +def _rl_kernel(w): + from rl_engine.integrations import qwen3_next_forward as provider + + def route(x): + routes = provider.stable_top10_routes(provider.shared_router(x, w["router"])) + return routes.indices, routes.weights + + def fwd(x): + with torch.no_grad(): + return provider.shared_moe(x, w["router"], w["gate_up"], w["down"])[0] + + gate_up = w["gate_up"].detach().clone().requires_grad_(True) + down = w["down"].detach().clone().requires_grad_(True) + + def graph(x): + leaf = x.detach().clone().requires_grad_(True) + y, _ = provider.shared_moe(leaf, w["router"], gate_up, down) + return y, [leaf, gate_up, down] + + return { + "name": "rl-kernel shared_moe", + "source": "rl_engine.integrations.qwen3_next_forward.shared_moe", + "route": route, + "fwd": fwd, + "graph": graph, + } + + +def _hf_modules(w, device="cuda"): + from transformers import Qwen3NextConfig + from transformers.models.qwen3_next import modeling_qwen3_next as hf + + config = Qwen3NextConfig( + hidden_size=HIDDEN, + num_experts=EXPERTS, + num_experts_per_tok=TOPK, + moe_intermediate_size=WIDTH, + norm_topk_prob=True, + ) + router = hf.Qwen3NextTopKRouter(config).to(device=device, dtype=torch.bfloat16) + experts = hf.Qwen3NextExperts(config).to(device=device, dtype=torch.bfloat16) + with torch.no_grad(): + router.weight.copy_(w["router"]) + experts.gate_up_proj.copy_(w["gate_up"]) + experts.down_proj.copy_(w["down"]) + return router, experts + + +def _hf(w): + import transformers + + router, experts = _hf_modules(w) + + def route(x): + _, weights, ids = router(x) + return ids, weights + + def call(x, mod): + _, weights, ids = router(x) + return mod(x, ids, weights) + + def fwd(x): + with torch.no_grad(): + return call(x, experts) + + def graph(x): + leaf = x.detach().clone().requires_grad_(True) + return call(leaf, experts), [leaf, experts.gate_up_proj, experts.down_proj] + + return { + "name": "HF transformers Qwen3NextExperts (eager)", + "source": f"transformers {transformers.__version__} modeling_qwen3_next", + "route": route, + "fwd": fwd, + "graph": graph, + } + + +def _vllm(w, batch_invariant: bool): + import vllm + from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts + from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk + + flag = "1" if batch_invariant else "0" + linear = F.linear + if batch_invariant: + from vllm.model_executor.determinism.batch_invariant import linear_batch_invariant + + linear = linear_batch_invariant + + def route(x): + with _env("VLLM_BATCH_INVARIANT", flag): + # vLLM's Qwen3-Next gate is a BF16 ReplicatedLinear; batch-invariant + # mode replaces its aten GEMM with vLLM's persistent Triton matmul. + logits = linear(x, w["router"]) + weights, ids, _ = fused_topk(x, logits, TOPK, renormalize=True) + return ids.long(), weights + + def fwd(x): + with torch.no_grad(), _env("VLLM_BATCH_INVARIANT", flag): + ids, weights = route(x) + return fused_experts(x, w["gate_up"], w["down"], weights, ids.int()) + + return { + "name": f"vLLM fused_topk + fused_experts (Triton), VLLM_BATCH_INVARIANT={flag}", + "source": f"vllm {vllm.__version__} model_executor/layers/fused_moe", + "route": route, + "fwd": fwd, + } + + +def _flashinfer(w): + import flashinfer + from flashinfer.fused_moe import cutlass_fused_moe + + # CUTLASS SwiGLU takes fc1 as [up; gate] (the opposite half order of HF). + gate, up = w["gate_up"].chunk(2, dim=1) + fc1 = torch.cat((up, gate), dim=1).contiguous() + + def route(x): + probs = F.linear(x, w["router"]).float().softmax(-1) + weights, ids = probs.topk(TOPK, dim=-1) + return ids, weights / weights.sum(-1, keepdim=True) + + def fwd(x): + with torch.no_grad(): + ids, weights = route(x) + out = cutlass_fused_moe( + x, ids.int(), weights, fc1, w["down"], torch.bfloat16, quant_scales=[] + ) + return out[0] if isinstance(out, (list, tuple)) else out + + return { + "name": "FlashInfer cutlass_fused_moe (routes from torch.topk)", + "source": f"flashinfer {flashinfer.__version__} fused_moe.cutlass_fused_moe", + "route": route, + "fwd": fwd, + } + + +def candidates(w) -> list[dict[str, Any]]: + return [ + _guard("rl_kernel_cuda", lambda: _rl_kernel(w)), + _guard("hf_transformers", lambda: _hf(w)), + _guard("vllm_bi0", lambda: _vllm(w, False)), + _guard("vllm_bi1", lambda: _vllm(w, True)), + _guard("flashinfer_cutlass", lambda: _flashinfer(w)), + ] + + +# --------------------------------------------------------------------------- # +# Checks +# --------------------------------------------------------------------------- # + + +def _embedded(x: torch.Tensor, size: int): + """Probe rows first in a batch of ``size``, and the same probe rows last.""" + first = x[:size] + last = x[:size].clone() + last[size - PROBE :] = x[:PROBE] + return first, last + + +def _rows_bi(fn, x) -> dict[str, bool]: + alone = fn(x[:PROBE]) + result = {} + for size in BI_SIZES: + first, last = _embedded(x, size) + a, b = fn(first), fn(last) + if isinstance(alone, tuple): + ok = all( + _bitwise(p[:PROBE], q) and _bitwise(r[size - PROBE :], q) + for p, r, q in zip(a, b, alone) + ) + else: + ok = _bitwise(a[:PROBE], alone) and _bitwise(b[size - PROBE :], alone) + result[str(size)] = bool(ok) + return result + + +def _grad_bi(cand, x) -> dict[str, Any]: + """dx of the probe rows, and dW with zero-gradient rows appended.""" + g = torch.Generator(device="cuda").manual_seed(7) + dy = torch.randn(x.shape, device="cuda", generator=g).to(torch.bfloat16) + + def grads(rows, dy_rows): + y, leaves = cand["graph"](rows) + return torch.autograd.grad(y, leaves, dy_rows) + + alone = grads(x[:PROBE], dy[:PROBE]) + dx, dw = {}, {} + for size in BI_SIZES: + full = grads(x[:size], dy[:size]) + dx[str(size)] = bool(_bitwise(full[0][:PROBE], alone[0])) + padded = dy[:size].clone() + padded[PROBE:] = 0 + masked = grads(x[:size], padded) + dw[str(size)] = bool(all(_bitwise(m, a) for m, a in zip(masked[1:], alone[1:]))) + return {"dx_rows_bitwise": dx, "dweight_zero_rows_bitwise": dw} + + +def _accuracy(cand, x, w) -> dict[str, Any]: + ref, ref_ids = fp64_reference(x, w) + ids, _ = cand["route"](x) + same = ids.long().sort(-1).values == ref_ids.sort(-1).values + out = cand["fwd"](x) + return { + "rel_l2_vs_fp64": _rel(out, ref), + "max_abs_vs_fp64": float((out.double() - ref).abs().max()), + "tokens_with_different_expert_set": int((~same.all(-1)).sum()), + "tokens": int(x.shape[0]), + } + + +def _one(cand, x, w) -> dict[str, Any]: + if "unavailable" in cand: + return cand + out = {"key": cand["key"], "name": cand["name"], "source": cand["source"]} + try: + out["route_rows_bitwise"] = _rows_bi(cand["route"], x) + out["output_rows_bitwise"] = _rows_bi(cand["fwd"], x) + out["repeatable"] = bool(_bitwise(cand["fwd"](x[:256]), cand["fwd"](x[:256]))) + if "graph" in cand: + out.update(_grad_bi(cand, x)) + out["accuracy"] = _accuracy(cand, x[:256], w) + except Exception as exc: # noqa: BLE001 - a crash is a result, not a harness failure + out["failed"] = f"{type(exc).__name__}: {str(exc)[:300]}" + out["batch_invariant"] = bool( + "failed" not in out + and all(out["route_rows_bitwise"].values()) + and all(out["output_rows_bitwise"].values()) + and all(out.get("dx_rows_bitwise", {"": True}).values()) + and all(out.get("dweight_zero_rows_bitwise", {"": True}).values()) + ) + return out + + +def _latency(cands, tokens) -> dict[str, Any]: + ready = [c for c in cands if "unavailable" not in c] + forward = {} + for count in tokens: + x = make_tokens(count, seed=100 + count) + calls = {} + for cand in ready: + try: + cand["fwd"](x) + calls[cand["key"]] = lambda c=cand: c["fwd"](x) + except Exception: # noqa: BLE001 - recorded by the BI section + continue + forward[str(count)] = time_interleaved(calls) + backward = {} + for count in BACKWARD_TOKENS: + x = make_tokens(count, seed=200 + count) + dy = torch.randn_like(x) + calls = {} + for cand in ready: + if "graph" not in cand: + continue + + def step(c=cand): + y, leaves = c["graph"](x) + torch.autograd.grad(y, leaves, dy) + + calls[cand["key"]] = step + backward[str(count)] = time_interleaved(calls, warmup=2, iters=10) + return {"forward_us": forward, "forward_plus_backward_us": backward} + + +def moe_report() -> dict[str, Any]: + torch.manual_seed(0) + w = make_weights() + x = make_tokens(max(BI_SIZES), seed=1) + cands = candidates(w) + return { + "op": "qwen3_next_routed_moe", + "shape": {"hidden": HIDDEN, "experts": EXPERTS, "top_k": TOPK, "expert_width": WIDTH}, + "probe_rows": PROBE, + "candidates": [_one(c, x, w) for c in cands], + "latency": _latency(cands, PERF_TOKENS), + } diff --git a/rl_engine/testing/tensor_identity.py b/rl_engine/testing/tensor_identity.py new file mode 100644 index 000000000..5d05c0fe9 --- /dev/null +++ b/rl_engine/testing/tensor_identity.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Finite tensor identity in the original storage dtype, including signed zero.""" + +import torch + + +def tensor_bitwise_equal(actual: torch.Tensor, expected: torch.Tensor) -> bool: + """Compare logical values by bytes without widening or accepting NaN/Inf. + + Strides need not match. A cross-device comparison copies expected bytes to + the actual tensor's device; it never converts either tensor's dtype. + """ + if actual.shape != expected.shape or actual.dtype != expected.dtype: + return False + for value in (actual, expected): + if (value.is_floating_point() or value.is_complex()) and not bool( + torch.isfinite(value).all() + ): + return False + actual_bytes = actual.detach().contiguous().reshape(-1).view(torch.uint8) + expected_bytes = expected.detach().contiguous().reshape(-1).view(torch.uint8) + if actual_bytes.device != expected_bytes.device: + expected_bytes = expected_bytes.to(actual_bytes.device) + return bool(torch.equal(actual_bytes, expected_bytes)) + + +def assert_tensor_bitwise_equal( + actual: torch.Tensor, expected: torch.Tensor, *, name: str = "tensor" +) -> None: + """Require finite, same-dtype, same-shape raw-bit identity.""" + if not tensor_bitwise_equal(actual, expected): + raise AssertionError( + f"{name}: finite raw-bit identity failed; " + f"actual={tuple(actual.shape)}/{actual.dtype}, " + f"expected={tuple(expected.shape)}/{expected.dtype}" + ) diff --git a/scripts/plot_qwen3_next_moe_prior_art.py b/scripts/plot_qwen3_next_moe_prior_art.py new file mode 100644 index 000000000..17a966b61 --- /dev/null +++ b/scripts/plot_qwen3_next_moe_prior_art.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Render the routed-MoE prior-art report (``scripts/qwen3_next_moe_prior_art.py``) as a PNG. + +Needs only the report JSON and matplotlib, so it runs anywhere: + + python scripts/plot_qwen3_next_moe_prior_art.py \\ + docs/usage/evidence/qwen3-next-moe-route-b200/report.json + +writes ``figure.png`` next to the report. Three panels: which batch-invariance +checks each implementation passes, accuracy against an FP64 evaluation with +FP64 routing, and forward latency. RL-Kernel is blue, existing libraries orange. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.colors import ListedColormap # noqa: E402 + +RL_KERNEL = "#2a78d6" +INK = "#2b2b29" +MUTED = "#6f6e69" +GRID = "#e4e3dd" +SURFACE = "#fcfcfb" +PASS = "#1baf7a" +FAIL = "#d64545" +NA = "#e4e3dd" +EXISTING = ("#eb6834", "#b8501f", "#f29a62", "#8a3a15") + +LABELS = { + "rl_kernel_cuda": "RL-Kernel shared_moe", + "hf_transformers": "HF Qwen3NextExperts", + "vllm_bi0": "vLLM fused_moe (BI=0)", + "vllm_bi1": "vLLM fused_moe (BI=1)", + "flashinfer_cutlass": "FlashInfer cutlass_fused_moe", +} +CHECKS = ( + ("route_rows_bitwise", "routes"), + ("output_rows_bitwise", "output"), + ("dx_rows_bitwise", "dx"), + ("dweight_zero_rows_bitwise", "dW"), +) + + +def _colors(keys): + existing = iter(EXISTING) + return {k: RL_KERNEL if k == "rl_kernel_cuda" else next(existing) for k in keys} + + +def _style(ax): + ax.set_facecolor(SURFACE) + for side in ("top", "right"): + ax.spines[side].set_visible(False) + ax.tick_params(colors=INK, labelsize=8) + ax.grid(color=GRID, linewidth=0.6) + + +def plot(report: dict, out: Path) -> None: + cands = [c for c in report["candidates"] if "unavailable" not in c] + keys = [c["key"] for c in cands] + colors = _colors(keys) + sizes = list(next(iter(cands))["output_rows_bitwise"]) + fig, axes = plt.subplots(1, 3, figsize=(17, 4.6), gridspec_kw={"width_ratios": [1.5, 1, 1.4]}) + fig.patch.set_facecolor("white") + + # Panel 1: batch-invariance matrix; 1 = pass, 0 = fail, 0.5 = no backward. + ax = axes[0] + columns = [f"{label} {size}" for _, label in CHECKS for size in sizes] + grid = [] + for cand in cands: + row = [] + for field, _ in CHECKS: + values = cand.get(field) + row.extend(0.5 if values is None else float(values[s]) for s in sizes) + grid.append(row) + ax.imshow(grid, cmap=ListedColormap([FAIL, NA, PASS]), vmin=0, vmax=1, aspect="auto") + ax.set_xticks(range(len(columns)), columns, fontsize=7, rotation=90) + ax.set_yticks(range(len(cands)), [LABELS.get(k, k) for k in keys], fontsize=8) + for i, cand in enumerate(cands): + mark = "BI" if cand.get("batch_invariant") else "not BI" + ax.text(len(columns) - 0.4, i, f" {mark}", va="center", fontsize=8, color=INK) + ax.set_title( + f"Batch invariance: {report['probe_rows']} probe tokens alone vs. in a batch\n" + "(green bitwise equal, red differs, grey: no backward)", + fontsize=9, + color=INK, + ) + ax.tick_params(length=0) + + # Panel 2: accuracy. + ax = axes[1] + _style(ax) + rel = [c["accuracy"]["rel_l2_vs_fp64"] if "accuracy" in c else float("nan") for c in cands] + ax.barh(range(len(cands)), rel, color=[colors[k] for k in keys]) + for i, cand in enumerate(cands): + acc = cand.get("accuracy") + if acc: + ax.text( + rel[i], + i, + f" {rel[i]:.2e} ({acc['tokens_with_different_expert_set']}/{acc['tokens']}" + " tokens route differently)", + va="center", + fontsize=7, + color=MUTED, + ) + ax.set_yticks(range(len(cands)), [LABELS.get(k, k) for k in keys], fontsize=8) + ax.invert_yaxis() + ax.set_xlim(0, 2.2 * max(r for r in rel if r == r)) + ax.set_xlabel("relative L2 error vs. FP64 HF formula with FP64 routing", fontsize=8) + ax.set_title("Accuracy (256 tokens)", fontsize=9, color=INK) + + # Panel 3: forward latency. + ax = axes[2] + _style(ax) + forward = report["latency"]["forward_us"] + tokens = [int(t) for t in forward] + for key in keys: + ys = [forward[str(t)].get(key, float("nan")) / 1000.0 for t in tokens] + ax.plot(tokens, ys, marker="o", ms=3.5, color=colors[key], label=LABELS.get(key, key)) + ax.set_xscale("log", base=2) + ax.set_yscale("log") + ax.set_xlabel("tokens", fontsize=8) + ax.set_ylabel("forward latency (ms, median)", fontsize=8) + ax.set_title("Forward latency\n(H=2048, 512 experts, top-10, width 512)", fontsize=9) + ax.legend(fontsize=7, frameon=False) + + env = report["environment"] + fig.suptitle( + f"Qwen3-Next routed MoE prior art - {env['gpu']}, torch {env['torch']}, " + f"commit {report['rl_kernel_commit'][:7]}", + fontsize=10, + color=INK, + ) + fig.tight_layout() + fig.savefig(out, dpi=160) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report", type=Path) + args = parser.parse_args() + report = json.loads(args.report.read_text()) + out = args.report.with_name("figure.png") + plot(report, out) + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/qwen3_next_moe_prior_art.py b/scripts/qwen3_next_moe_prior_art.py new file mode 100644 index 000000000..398245eee --- /dev/null +++ b/scripts/qwen3_next_moe_prior_art.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Write the Qwen3-Next routed-MoE prior-art report (BI + accuracy + latency) as JSON. + +Run it on a clean tree with ``TRITON_F32_DEFAULT=ieee`` (the FP32 router needs +IEEE Triton dots from process start). The report records the commit, whether +the tracked tree was dirty and the version of every library compared. +``scripts/plot_qwen3_next_moe_prior_art.py`` turns the report into a figure. + + TRITON_F32_DEFAULT=ieee python scripts/qwen3_next_moe_prior_art.py \\ + --out docs/usage/evidence/qwen3-next-moe-route-b200/report.json +""" + +from __future__ import annotations + +import argparse +import importlib.metadata as md +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +import torch # noqa: E402 + +from rl_engine.testing.qwen3_next_moe_prior_art import moe_report # noqa: E402 + +LIBRARIES = ( + "vllm", + "flashinfer-python", + "transformers", + "triton", + "sglang", + "megatron-core", + "transformer_engine", + "vime", +) + + +def _git(*args: str) -> str: + return subprocess.run( + ["git", *args], cwd=ROOT, check=True, capture_output=True, text=True + ).stdout.strip() + + +def _version(name: str) -> str | None: + try: + return md.version(name) + except md.PackageNotFoundError: + return None + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + if not torch.cuda.is_available(): + raise SystemExit("needs a CUDA device") + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + report = { + "kind": "qwen3_next_prior_art_report", + "rfc": 428, + "rl_kernel_commit": _git("rev-parse", "HEAD"), + "tracked_tree_dirty": bool(_git("status", "--porcelain", "--untracked-files=no")), + "environment": { + "gpu": torch.cuda.get_device_name(), + "torch": torch.__version__, + "cuda": torch.version.cuda, + "libraries": {name: _version(name) for name in LIBRARIES}, + }, + **moe_report(), + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2) + "\n") + print( + f"wrote {args.out} (commit {report['rl_kernel_commit'][:7]}, " + f"dirty={report['tracked_tree_dirty']})" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/qwen3_next_tp_moe_check.py b/scripts/qwen3_next_tp_moe_check.py new file mode 100644 index 000000000..817c07bcc --- /dev/null +++ b/scripts/qwen3_next_tp_moe_check.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""TP4 MoE gate on the official checkpoint's layer-0 weights; four GPUs, torchrun. + + TRITON_F32_DEFAULT=ieee torchrun --nproc-per-node 4 scripts/qwen3_next_tp_moe_check.py \\ + --checkpoint --output + +Each rank writes ``rank-.json``. The gate requires, bitwise: + +* the HF load -> local shard -> all-gather -> HF export round trip of all 512 experts; +* identical top-10 routes on every rank (the router is replicated); +* full batch == concatenated chunks, and reordered tokens == reordered output, + for 8, 64, 256 and 1024 tokens; +* the training forward (autograd on) == the no-grad forward; +* identical replicated router and shared-expert-gate gradients on every rank, + and a finite, nonzero gradient for every parameter. +""" + +import argparse +import json +import os +import subprocess +from collections.abc import Mapping +from contextlib import ExitStack +from datetime import timedelta +from importlib.metadata import version +from pathlib import Path + +import torch +import torch.distributed as dist +from safetensors import safe_open + +from rl_engine.integrations.qwen3_next_forward import ( + FORWARD_PROVIDER_ID, + shared_router, + stable_top10_routes, +) +from rl_engine.integrations.qwen3_next_tp_blocks import TP4MoE, assemble_moe_weight, moe_hf_shapes +from rl_engine.testing.tensor_identity import assert_tensor_bitwise_equal as exact + +ROOT = Path(__file__).resolve().parents[1] +TOKEN_COUNTS = (8, 64, 256, 1024) + + +class CheckpointWeights(Mapping): + """Keep shard headers open while reading only the requested layer's tensors.""" + + def __init__(self, checkpoint, prefix, names, stack): + self.root, self.prefix, self.names, self.stack = checkpoint, prefix, tuple(names), stack + self.index = json.loads((checkpoint / "model.safetensors.index.json").read_text())[ + "weight_map" + ] + self.readers = {} + + def __iter__(self): + return iter(self.names) + + def __len__(self): + return len(self.names) + + def __getitem__(self, name): + if name not in self.names: + raise KeyError(name) + key = self.prefix + name + shard = self.index[key] + if shard not in self.readers: + self.readers[shard] = self.stack.enter_context( + safe_open(self.root / shard, framework="pt", device="cpu") + ) + return self.readers[shard].get_tensor(key) + + +def atomic_json(path, payload): + temporary = path.with_suffix(path.suffix + ".partial") + temporary.write_text(json.dumps(payload, indent=2) + "\n") + temporary.replace(path) + + +def gather(value): + peers = [torch.empty_like(value) for _ in range(4)] + dist.all_gather(peers, value.contiguous()) + return peers + + +def check_gradients(module): + result = {} + for name, parameter in module.named_parameters(): + grad = parameter.grad + if grad is None or not torch.isfinite(grad).all() or not grad.count_nonzero(): + raise AssertionError(f"Missing finite nonzero gradient: {name}") + result[name] = float(grad.float().abs().max()) + return result + + +def timed(fn): + start, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + start.record() + out = fn() + end.record() + torch.cuda.synchronize() + return out, start.elapsed_time(end) * 1000.0 + + +def moe_gate(checkpoint, generator): + module = TP4MoE(group=dist.group.WORLD, device="cuda") + with ExitStack() as stack: + weights = CheckpointWeights(checkpoint, "model.layers.0.mlp.", moe_hf_shapes(), stack) + module.load_hf_weights(weights) + for name, value in module.export_local_hf_weights().items(): + restored = assemble_moe_weight(name, [peer.cpu() for peer in gather(value)]) + exact(restored, weights[name], name=f"MoE HF {name}") + cases = ["hf-load-export-all512experts"] + forward_us = {} + with torch.no_grad(): + for length in TOKEN_COUNTS: + hidden = ( + torch.randn(length, 2048, device="cuda", dtype=torch.bfloat16, generator=generator) + * 0.1 + ) + # Every rank draws the same tokens from the same generator; verify it. + for peer in gather(hidden): + exact(peer, hidden, name="replicated input") + routes = stable_top10_routes(shared_router(hidden, module.gate.weight)) + for peer in gather(routes.indices): + exact(peer, routes.indices, name=f"routes across ranks {length}") + module(hidden) + whole, forward_us[str(length)] = timed(lambda: module(hidden)) + cut = min(65, length - 1) + exact( + torch.cat((module(hidden[:cut]), module(hidden[cut:]))), + whole, + name=f"MoE full/chunk {length}", + ) + order = torch.arange(length - 1, -1, -1, device="cuda") + exact(module(hidden[order]), whole[order], name=f"MoE reorder {length}") + cases.extend((f"routes-replicated-{length}", f"full-chunk-reorder-{length}")) + hidden = ( + torch.randn(16, 2048, device="cuda", dtype=torch.bfloat16, generator=generator) * 0.1 + ).requires_grad_() + with torch.no_grad(): + expected = module(hidden) + output = module(hidden) + exact(output, expected, name="MoE independent training forward") + output.float().square().mean().backward() + if ( + hidden.grad is None + or not torch.isfinite(hidden.grad).all() + or not hidden.grad.count_nonzero() + ): + raise AssertionError("MoE input gradient is missing") + gradients = check_gradients(module) + for parameter in (module.gate.weight, module.shared_expert_gate.weight): + for peer in gather(parameter.grad): + exact(peer, parameter.grad, name="MoE replicated router/gate gradient") + cases.extend(("training-forward", "all-parameter-gradient", "replicated-router-gradient")) + return {"cases": cases, "gradient_max_abs": gradients, "forward_us": forward_us} + + +def _git(*args): + return subprocess.run( + ["git", *args], cwd=ROOT, check=True, capture_output=True, text=True + ).stdout.strip() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + rank = int(os.environ["LOCAL_RANK"]) + if int(os.environ["WORLD_SIZE"]) != 4 or torch.cuda.device_count() < 4: + raise RuntimeError("The TP4 MoE gate requires exactly four ranks on four GPUs") + torch.cuda.set_device(rank) + dist.init_process_group( + "nccl", timeout=timedelta(minutes=15), device_id=torch.device("cuda", rank) + ) + try: + generator = torch.Generator(device="cuda").manual_seed(1234) + moe = moe_gate(args.checkpoint, generator) + args.output.mkdir(parents=True, exist_ok=True) + atomic_json( + args.output / f"rank-{rank}.json", + { + "status": "passed", + "scope": "tp4_moe_block_layer0", + "rank": rank, + "rl_kernel_commit": _git("rev-parse", "HEAD"), + "tracked_tree_dirty": bool(_git("status", "--porcelain", "--untracked-files=no")), + "provider": FORWARD_PROVIDER_ID, + "environment": { + "gpu": torch.cuda.get_device_name(), + "torch": torch.__version__, + "vllm": version("vllm"), + "nccl": ".".join(map(str, torch.cuda.nccl.version())), + }, + "moe": moe, + }, + ) + dist.barrier() + finally: + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/tests/check_qwen3_next_forward.py b/tests/check_qwen3_next_forward.py new file mode 100644 index 000000000..ce4beb1f7 --- /dev/null +++ b/tests/check_qwen3_next_forward.py @@ -0,0 +1,238 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Mandatory shared CUDA GEMM and MoE checks; these do not claim model L2.""" + +import pytest +import torch +import torch.nn.functional as F + +from rl_engine.integrations.qwen3_next_forward import ( + combine_routes, + shared_linear, + shared_moe, + stable_top10_routes, +) +from rl_engine.testing.tensor_identity import assert_tensor_bitwise_equal as exact + + +def random(shape, seed=801, *, grad=False, dtype=torch.bfloat16): + assert torch.cuda.is_available(), "Mandatory shared-forward CUDA gates cannot skip" + gen = torch.Generator(device="cuda").manual_seed(seed) + return ( + torch.randn(shape, generator=gen, device="cuda", dtype=dtype) + .mul_(0.02) + .requires_grad_(grad) + ) + + +@pytest.mark.parametrize( + "k,n", [(2048, 3072), (2048, 16), (1024, 2048), (2048, 256), (128, 2048), (2048, 37984)] +) +@torch.no_grad() +def test_qwen_linear_shapes_batch_chunk_and_reorder(k, n): + x, weight = random((17, k)), random((n, k), seed=802) + output = shared_linear(x, weight) + exact( + torch.cat( + [ + shared_linear(x[:1], weight), + shared_linear(x[1:4], weight), + shared_linear(x[4:], weight), + ] + ), + output, + ) + order = torch.arange(16, -1, -1, device="cuda") + exact(shared_linear(x[order], weight), output[order]) + + +def test_shared_linear_vjp_and_empty_tokens(): + x, weight, bias = ( + random((4, 2048), grad=True), + random((16, 2048), 802, grad=True), + random((16,), 803, grad=True), + ) + output = shared_linear(x, weight, bias) + grad = random(output.shape, 804) + output.backward(grad) + refs = [value.detach().double().requires_grad_() for value in (x, weight, bias)] + F.linear(*refs).backward(grad.double()) + for value, ref in zip((x, weight, bias), refs): + torch.testing.assert_close(value.grad.float(), ref.grad.float(), atol=0.002, rtol=0.016) + assert (value.grad.double() - ref.grad).norm() / ref.grad.norm() < 0.016 + with torch.no_grad(): + exact(output, shared_linear(x, weight, bias)) + empty = random((0, 2048), grad=True) + weight.grad = bias.grad = None + shared_linear(empty, weight, bias).sum().backward() + assert empty.grad.shape == empty.shape + assert torch.count_nonzero(weight.grad) == 0 and torch.count_nonzero(bias.grad) == 0 + + +def test_stable_route_tie_boundary_and_router_gradient(): + logits = random((4, 512), dtype=torch.float32) + logits.zero_().requires_grad_() + routes = stable_top10_routes(logits) + assert torch.equal(routes.indices, torch.arange(10, device="cuda").expand(4, 10)) + exact(routes.weights, torch.full((4, 10), 0.1, device="cuda")) + (routes.weights * torch.arange(10, device="cuda")).sum().backward() + assert torch.isfinite(logits.grad).all() and logits.grad.abs().max() > 0 + bad = logits.detach().clone() + bad[0, 0] = torch.nan + with pytest.raises(ValueError, match="finite"): + stable_top10_routes(bad) + + +def test_combine_has_fixed_route_order_and_differentiable_weights(): + outputs = random((4, 10, 2048), grad=True) + weights = random((4, 10), 802, dtype=torch.float32, grad=True) + actual = combine_routes(outputs, weights) + exact( + torch.cat([combine_routes(outputs[i : i + 1], weights[i : i + 1]) for i in range(4)]), + actual, + ) + actual.float().square().sum().backward() + assert weights.grad.abs().max() > 0 and outputs.grad.abs().max() > 0 + refs = [value.detach().double().requires_grad_() for value in (outputs, weights)] + expected = (refs[0] * refs[1].unsqueeze(-1)).sum(dim=1) + expected.backward((2 * actual.float()).double()) + for value, reference in zip((outputs, weights), refs): + torch.testing.assert_close( + value.grad.float(), reference.grad.float(), atol=0.002, rtol=0.016 + ) + + +def test_qwen_moe_actual_tp4_shapes_and_independent_training_forward(): + x = random((4, 2048), grad=True) + router = random((512, 2048), 802, grad=True) + gate_up = random((512, 256, 2048), 803, grad=True) + down = random((512, 2048, 128), 804, grad=True) + output, routes = shared_moe(x, router, gate_up, down) + with torch.no_grad(): + row_output, row_routes = shared_moe(x[:1], router, gate_up, down) + exact(row_output, output[:1]) + exact(row_routes.weights, routes.weights[:1]) + assert torch.equal(row_routes.indices, routes.indices[:1]) + output.float().square().sum().backward() + for value in (x, router, gate_up, down): + assert value.grad is not None and torch.isfinite(value.grad).all() + assert value.grad.abs().max() > 0 + active = routes.indices.unique() + inactive = torch.ones(512, dtype=torch.bool, device="cuda") + inactive[active] = False + assert torch.count_nonzero(gate_up.grad[inactive]) == 0 + assert torch.count_nonzero(down.grad[inactive]) == 0 + + +def test_moe_vjp_matches_independent_double_formula(): + # A small hidden width isolates the derivative; the previous gate covers + # the unmodified official TP4 widths and expert count. + x = random((2, 32)).mul_(10).requires_grad_() + router = random((512, 32), 802).mul_(5).requires_grad_() + gate_up = random((512, 16, 32), 803).mul_(10).requires_grad_() + down = random((512, 32, 8), 804).mul_(10).requires_grad_() + actual, routes = shared_moe(x, router, gate_up, down) + grad = random(actual.shape, 805) + actual.backward(grad) + xr, rr, gu, dr = [v.detach().double().requires_grad_() for v in (x, router, gate_up, down)] + probability = (xr @ rr.t()).softmax(-1) + selected = probability.gather(1, routes.indices) + selected = selected / selected.sum(-1, keepdim=True) + rows = [] + for token in range(2): + slots = [] + for slot, expert in enumerate(routes.indices[token].tolist()): + gate, up = F.linear(xr[token], gu[expert]).chunk(2) + slots.append(F.linear(F.silu(gate) * up, dr[expert]) * selected[token, slot]) + rows.append(torch.stack(slots).sum(0)) + torch.stack(rows).backward(grad.double()) + for value, reference in zip((x, router, gate_up, down), (xr, rr, gu, dr)): + assert torch.isfinite(value.grad).all() + # BF16 intermediates are rounded by the actual provider; this compares + # derivative correctness, never the raw-bit L1/L2 acceptance criterion. + torch.testing.assert_close( + value.grad.float(), reference.grad.float(), atol=0.00002, rtol=0.016 + ) + assert (value.grad.double() - reference.grad).norm() / reference.grad.norm() < 0.016 + + +@torch.no_grad() +def test_route_weights_are_batch_invariant(): + logits = random((32, 512), 805, dtype=torch.float32).mul_(100) + whole = stable_top10_routes(logits) + for count in (1, 8, 17): + part = stable_top10_routes(logits[:count]) + assert torch.equal(part.indices, whole.indices[:count]) + exact(part.weights, whole.weights[:count]) + moe_x, router = random((32, 2048), 806), random((512, 2048), 807) + gate_up, down = random((512, 256, 2048), 808), random((512, 2048, 128), 809) + whole_out, _ = shared_moe(moe_x, router, gate_up, down) + part_out, _ = shared_moe(moe_x[:8], router, gate_up, down) + exact(part_out, whole_out[:8]) + + +class _PerExpertReference(torch.autograd.Function): + """The previous provider: one pinned vLLM GEMM pair per expert, in expert order.""" + + @staticmethod + def forward(ctx, x, gate_up, down, route_indices): + from rl_engine.integrations.qwen3_next_forward import _vllm_linear + + output = x.new_empty((x.shape[0] * 10, x.shape[1])) + flat = route_indices.flatten() + for expert in flat.unique().tolist(): + slots = (flat == expert).nonzero().squeeze(1) + rows = x.index_select(0, slots // 10) + gate, up = _vllm_linear()(rows, gate_up[expert]).chunk(2, dim=-1) + activated = (F.silu(gate.float()) * up.float()).to(x.dtype) + output.index_copy_(0, slots, _vllm_linear()(activated, down[expert])) + return output.reshape(x.shape[0], 10, x.shape[1]) + + +@torch.no_grad() +@pytest.mark.parametrize("width", [128, 512]) +@pytest.mark.parametrize("tokens", [1, 7, 13, 300]) +def test_grouped_experts_are_bitwise_the_per_expert_gemms(width, tokens): + from rl_engine.integrations.qwen3_next_forward import _RoutedExperts, shared_router + + x, router = random((tokens, 2048), 811), random((512, 2048), 812) + gate_up, down = random((512, 2 * width, 2048), 813), random((512, 2048, width), 814) + indices = stable_top10_routes(shared_router(x, router)).indices + exact( + _RoutedExperts.apply(x, gate_up, down, indices), + _PerExpertReference.apply(x, gate_up, down, indices), + ) + + +@torch.no_grad() +@pytest.mark.parametrize("width", [128, 512]) +def test_grouped_projection_is_bitwise_the_per_expert_gemm(width): + # The backward recomputes [gate, up] with the grouped kernel; equality here + # leaves every gradient bitwise what the per-expert backward produced. + from rl_engine.integrations.qwen3_next_forward import ( + _vllm_linear, + grouped_route_linear, + shared_router, + ) + + x, router = random((300, 2048), 817), random((512, 2048), 818) + gate_up = random((512, 2 * width, 2048), 819) + indices = stable_top10_routes(shared_router(x, router)).indices + grouped = grouped_route_linear(x, gate_up, indices).reshape(-1, 2 * width) + flat = indices.flatten() + for expert in flat.unique().tolist(): + slots = (flat == expert).nonzero().squeeze(1) + exact(grouped[slots], _vllm_linear()(x[slots // 10], gate_up[expert]), name=f"e{expert}") + + +def test_grouped_experts_fail_closed_on_the_tensor_descriptor_path(monkeypatch): + from rl_engine.integrations import qwen3_next_forward as provider + + prepare, dispatch, _ = provider._vllm_grouped() + monkeypatch.setattr(provider, "_vllm_grouped", lambda: (prepare, dispatch, lambda: True)) + x = random((4, 2048), 815) + with pytest.raises(RuntimeError, match="VLLM_TRITON_USE_TD"): + provider.grouped_route_linear( + x, random((512, 256, 2048), 816), torch.zeros(4, 10, device="cuda", dtype=torch.long) + ) diff --git a/tests/test_qwen3_next_forward_contract.py b/tests/test_qwen3_next_forward_contract.py new file mode 100644 index 000000000..76d6df183 --- /dev/null +++ b/tests/test_qwen3_next_forward_contract.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""CPU checks of the strict shared-forward boundary, without a CPU fallback.""" + +import pytest +import torch + +from rl_engine.integrations import qwen3_next_forward as provider + + +@pytest.mark.parametrize( + "call", + [ + lambda: provider.shared_linear(torch.zeros(1, 8), torch.zeros(8, 8)), + lambda: provider.stable_top10_routes(torch.zeros(1, 512)), + lambda: provider.combine_routes(torch.zeros(1, 10, 8), torch.zeros(1, 10)), + ], +) +def test_required_cuda_primitives_reject_cpu(call): + with pytest.raises(ValueError, match="no CPU fallback"): + call() + + +def test_shared_linear_rejects_an_unpinned_vllm(monkeypatch): + provider._vllm_linear.cache_clear() + monkeypatch.setattr(provider, "version", lambda package: "0.30.1") + with pytest.raises(RuntimeError, match="pinned vLLM 0.30.0"): + provider._vllm_linear() + provider._vllm_linear.cache_clear() + + +def _rows(count, width, seed): + generator = torch.Generator().manual_seed(seed) + return torch.randn((count, width), generator=generator, dtype=torch.float32) * 4 + + +@pytest.mark.parametrize("width", [1, 2, 7, 10, 512]) +def test_fixed_order_row_sum_matches_sequential_double_sum(width): + values = _rows(5, width, 901) + actual = provider.fixed_order_row_sum(values) + assert actual.shape == (5, 1) + reference = values.double().sum(-1, keepdim=True).float() + torch.testing.assert_close(actual, reference, rtol=1e-5, atol=1e-4) + + +@pytest.mark.parametrize("width", [10, 512]) +def test_fixed_order_row_sum_is_independent_of_row_count(width): + values = _rows(32, width, 902) + whole = provider.fixed_order_row_sum(values) + for count in (1, 8, 17): + assert torch.equal(provider.fixed_order_row_sum(values[:count]), whole[:count]) + + +def test_fixed_order_softmax_rows_do_not_depend_on_batch(): + logits = _rows(32, 512, 903) + whole = provider.fixed_order_softmax(logits) + torch.testing.assert_close(whole.double().sum(-1), torch.ones(32, dtype=torch.float64)) + for count in (1, 8): + assert torch.equal(provider.fixed_order_softmax(logits[:count]), whole[:count]) + # The row sum is a fixed sequential chain, so each result is a pure function of + # its own row and cannot depend on how many rows share the call. diff --git a/tests/test_qwen3_next_tp_blocks.py b/tests/test_qwen3_next_tp_blocks.py new file mode 100644 index 000000000..f2ac9c822 --- /dev/null +++ b/tests/test_qwen3_next_tp_blocks.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""HF mappings, replica checks and official Qwen3-Next MoE dimensions.""" + +import pytest +import torch + +from rl_engine.integrations import qwen3_next_tp +from rl_engine.integrations import qwen3_next_tp_blocks as blocks +from rl_engine.testing.tensor_identity import assert_tensor_bitwise_equal as exact + + +def pattern(shape): + return ( + torch.arange(torch.Size(shape).numel(), dtype=torch.int32) + .remainder(251) + .to(torch.bfloat16) + .reshape(shape) + ) + + +@pytest.mark.parametrize( + "name", + [ + "gate.weight", + "shared_expert_gate.weight", + "shared_expert.gate_proj.weight", + "shared_expert.up_proj.weight", + "shared_expert.down_proj.weight", + "experts.0.gate_proj.weight", + "experts.0.up_proj.weight", + "experts.511.down_proj.weight", + ], +) +def test_moe_hf_weight_roundtrip(name): + value = pattern(blocks.moe_hf_shapes()[name]) + shards = [blocks.shard_moe_weight(name, value, rank) for rank in range(4)] + exact(blocks.assemble_moe_weight(name, shards), value, name=name) + if name.endswith("down_proj.weight"): + assert shards[0].shape == (2048, 128) + elif name.endswith(("gate_proj.weight", "up_proj.weight")): + assert shards[0].shape == (128, 2048) + + +def test_moe_export_rejects_router_replica_drift(): + values = [pattern((512, 2048)) for _ in range(4)] + values[2][0, 0] = -1 + with pytest.raises(ValueError, match="Replicated MoE"): + blocks.assemble_moe_weight("gate.weight", values) + + +@pytest.mark.parametrize("rank", [-1, 4, True, 0.5]) +def test_invalid_shard_rank_rejected(rank): + with pytest.raises(ValueError, match="TP4 rank"): + blocks.shard_moe_weight("gate.weight", torch.empty(512, 2048, dtype=torch.bfloat16), rank) + + +def test_official_shared_expert_width_is_512_and_each_expert_has_three_weights(): + shapes = blocks.moe_hf_shapes() + assert len(shapes) == 512 * 3 + 5 + assert shapes["shared_expert.gate_proj.weight"] == (512, 2048) + assert shapes["shared_expert.down_proj.weight"] == (2048, 512) + + +def test_tp4_moe_rejects_a_missing_or_wrong_size_tp_group(monkeypatch): + monkeypatch.setattr(qwen3_next_tp.dist, "is_initialized", lambda: False) + with pytest.raises(ValueError, match="four-rank TP group"): + blocks.TP4MoE(group=None, device="meta") + monkeypatch.setattr(qwen3_next_tp.dist, "is_initialized", lambda: True) + monkeypatch.setattr(qwen3_next_tp.dist, "get_world_size", lambda group: 8) + with pytest.raises(ValueError, match="four-rank TP group"): + blocks.TP4MoE(group=None, device="meta") + + +def test_tp4_moe_parameter_ownership(monkeypatch): + group = object() + monkeypatch.setattr(qwen3_next_tp.dist, "is_initialized", lambda: True) + monkeypatch.setattr(qwen3_next_tp.dist, "get_world_size", lambda group: 4) + monkeypatch.setattr(blocks.dist, "get_rank", lambda group: 2) + module = blocks.TP4MoE(group=group, device="meta") + assert module.rank == 2 + assert module.experts.gate_up.shape == (512, 256, 2048) + assert module.experts.down.shape == (512, 2048, 128) + assert ( + module.experts.gate_up.partition_dim == 1 and module.experts.gate_up.partition_stride == 2 + ) + assert module.experts.down.partition_dim == 2 + assert module.shared_expert.down_proj.weight.partition_dim == 1 + # Router and shared-expert gate are replicated: no TP attributes, full shapes. + assert not getattr(module.gate.weight, "tensor_model_parallel", False) + assert not getattr(module.shared_expert_gate.weight, "tensor_model_parallel", False) + assert module.gate.weight.shape == (512, 2048) diff --git a/tests/test_tensor_identity.py b/tests/test_tensor_identity.py new file mode 100644 index 000000000..f85c0c4dd --- /dev/null +++ b/tests/test_tensor_identity.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""CPU regressions for the raw-bit identity used by Qwen3-Next acceptance checks.""" + +import pytest +import torch + +from rl_engine.testing.tensor_identity import assert_tensor_bitwise_equal, tensor_bitwise_equal + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16, torch.float16]) +def test_signed_zero_is_not_identical(dtype): + positive = torch.tensor([0.0], dtype=dtype) + negative = torch.tensor([-0.0], dtype=dtype) + assert not tensor_bitwise_equal(positive, negative) + with pytest.raises(AssertionError, match="raw-bit"): + assert_tensor_bitwise_equal(positive, negative) + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), -float("inf")]) +def test_identical_nonfinite_values_are_rejected(value): + tensor = torch.tensor([value]) + assert not tensor_bitwise_equal(tensor, tensor) + + +def test_same_numbers_in_different_dtypes_are_rejected(): + assert not tensor_bitwise_equal(torch.ones(2), torch.ones(2, dtype=torch.bfloat16)) + + +def test_adjacent_bfloat16_values_are_rejected(): + value = torch.ones(2, dtype=torch.bfloat16) + changed = (value.view(torch.int16) + 1).view(torch.bfloat16) + assert not tensor_bitwise_equal(value, changed) + + +@pytest.mark.parametrize( + "value", [torch.tensor(1.0), torch.empty(0), torch.arange(12).reshape(3, 4).T] +) +def test_logical_identity_does_not_require_matching_strides(value): + assert_tensor_bitwise_equal(value, value.contiguous().clone()) From 362b80b5ae0feefcaafc7698b9894b671b126c24 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:12:10 +0000 Subject: [PATCH 02/11] docs(qwen3-next): MoE route/combine contract, prior-art and TP4 evidence (B200, 2e950f3) Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- docs/operators/README.md | 1 + .../operators/qwen3-next-moe-route-combine.md | 137 ++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 docs/operators/qwen3-next-moe-route-combine.md diff --git a/docs/operators/README.md b/docs/operators/README.md index 0b93b3d90..be74cb6d0 100644 --- a/docs/operators/README.md +++ b/docs/operators/README.md @@ -34,4 +34,5 @@ Every operator page should include: - [Token Embedding](embedding.md) - [Qwen3-Next RMSNorm (zero-centred)](qwen3-next-rms-norm.md) - [Qwen3-Next Gated RMSNorm](qwen3-next-rms-norm-gated.md) +- [Qwen3-Next MoE route / combine](qwen3-next-moe-route-combine.md) - [Operator Doc Template](../contributing/operator-doc-template.md) diff --git a/docs/operators/qwen3-next-moe-route-combine.md b/docs/operators/qwen3-next-moe-route-combine.md new file mode 100644 index 000000000..ed11301a0 --- /dev/null +++ b/docs/operators/qwen3-next-moe-route-combine.md @@ -0,0 +1,137 @@ +# Qwen3-Next MoE route / combine contract + +## Summary + +Qwen3-Next's sparse MoE block routes every token to 10 of 512 experts (expert +width 512, TP4-local width 128) and adds a gated shared expert. RFC #428 row +`moe_route_combine_contract` asks for a routed MoE whose output for a token is a +pure function of that token: the same bits whatever else is in the batch, in +training replay and in rollout. This page states the contract and how +`rl_engine.integrations.qwen3_next_forward.shared_moe` meets it. + +Upstream references: `transformers` `Qwen3NextSparseMoeBlock` (5.17.0) and vLLM +0.30.0 `fused_topk` + `fused_experts`. + +## Entry Point + +```python +from rl_engine.integrations.qwen3_next_forward import shared_moe, stable_top10_routes +from rl_engine.integrations.qwen3_next_tp_blocks import TP4MoE + +routed, routes = shared_moe(x, router_weight, gate_up, down) # TP-local, not reduced +block = TP4MoE(group=tp_group, device="cuda") # routed + shared, reduced +block.load_hf_weights(hf_layer_mlp_weights) +y = block(hidden) +``` + +`TRITON_F32_DEFAULT=ieee` must be set before the process starts: the FP32 router +GEMM refuses to run on TF32 Triton dots. + +## Contract + +| Stage | Order and dtype | Atomics | +| --- | --- | --- | +| Router | BF16 `x`, `W` upcast to FP32; one pinned vLLM batch-invariant GEMM (IEEE FP32) | none | +| Softmax | max subtraction, `exp`, row sum by a fixed pairwise tree (`fixed_order_row_sum`) | none | +| Selection | `argsort(logits, descending, stable)[:, :10]`: score descending, equal scores to the lower expert id | none | +| Weights | the ten selected FP32 probabilities divided by their fixed-tree sum | none | +| Experts | per (token, route) row: `gate_up` GEMM (BF16 in, FP32 accumulate, BF16 out), SwiGLU in FP32, BF16 cast, `down` GEMM | none: one writer per row | +| Combine | `out = e0*w0`, then `out += e_k*w_k` for k = 1..9 in route order, FP32; one BF16 cast | none | +| Shared expert (TP4 block) | `routed + shared * sigmoid(gate)` in BF16, then one TP all-reduce | collective as configured | + +Payload dtype between stages: BF16 expert outputs, FP32 route weights, BF16 +block output. The expert GEMMs are vLLM 0.30.0's `fused_moe` Triton kernel with +the tile vLLM itself uses under `VLLM_BATCH_INVARIANT=1` +(`BLOCK_SIZE_M/N/K = 64/64/32`, `GROUP_SIZE_M = 8`, `SPLIT_K = 1`), passed +explicitly so no environment variable, tuned-config file or token count can +change it. The routed weight is *not* multiplied inside the kernel and the +kernel does not sum over routes; that stays in the combine above. + +Fail-closed checks: CUDA tensors only (no CPU fallback, no ROCm), BF16 weights of +the exact `[512, 2*I, H]` / `[512, H, I]` shapes, finite FP32 router logits, +vLLM exactly 0.30.0, `VLLM_TRITON_USE_TD` off (the tensor-descriptor path is a +different instruction stream), and IEEE Triton FP32. + +### Backward + +`_RoutedExperts.backward` recomputes the `[gate, up]` projection with the same +grouped kernel, then visits experts in ascending id. Each expert's `dgate_up` +and `ddown` slice is one pinned GEMM written once into a dense buffer; `dx` is +accumulated per token in ascending expert order (a token's ten experts are +distinct, so each launch has unique rows). Router-weight gradients flow through +the selected probabilities; the indices are discrete. + +## Reuse decision + +| Implementation | Routes BI | Output BI | Backward | Why not reused as is | +| --- | --- | --- | --- | --- | +| vLLM `fused_topk` + `fused_experts`, `VLLM_BATCH_INVARIANT=0` | no | no | none | tuned configs and the BF16 router GEMM depend on the token count | +| vLLM, `VLLM_BATCH_INVARIANT=1` | yes | yes | none | no VJP for training; BF16 router logits (Qwen3-Next and VIME route in FP32); routed weight applied in the expert GEMM epilogue on the BF16 output, then routes summed by `moe_sum` | +| HF `Qwen3NextExperts` (eager) | yes | yes | autograd | `dx`/`dW` change with batch size (cuBLAS shape heuristics); BF16 router; `index_add_` combine in BF16 | +| FlashInfer `cutlass_fused_moe` | (torch routing) | no | none | output changes with batch size; no VJP | +| SGLang, Megatron-core, Transformer Engine | not measured | not measured | - | not installable next to the pinned vLLM 0.30.0 / torch 2.13 runtime | + +What RL-Kernel reuses: vLLM's batch-invariant GEMM for the router and the +backward, and vLLM's `fused_moe` Triton kernel for the forward expert GEMMs, +which is batch-invariant per route at the fixed tile. What it implements: the +FP32 fixed-order routing, the stable tie rule, the atomic-free FP32 combine, the +deterministic backward and the TP4 boundaries. The grouped kernel's per-route +output is bitwise equal to the per-expert pinned GEMM loop it replaced +(`tests/check_qwen3_next_forward.py::test_grouped_*`), so this is a speed change +with no change in bits. + +## Results + +Measured on B200 with `scripts/qwen3_next_moe_prior_art.py` and the TP4 gate +`scripts/qwen3_next_tp_moe_check.py`; the reports, the figure and the exact +commands are in +[`docs/usage/evidence/qwen3-next-moe-route-b200/`](../usage/evidence/qwen3-next-moe-route-b200/README.md). + +![Qwen3-Next routed MoE prior art](../usage/evidence/qwen3-next-moe-route-b200/figure.png) + +TP1 shape (H=2048, 512 experts, top-10, width 512), random weights, BF16: + +| | RL-Kernel `shared_moe` | vLLM BI=1 | vLLM BI=0 | FlashInfer CUTLASS | HF eager | +| --- | --- | --- | --- | --- | --- | +| routes / output BI (16-1024 tokens) | **yes / yes** | yes / yes | no / no (256+) | no / no (256+) | yes / yes | +| dx / dW BI | **yes / yes** | no backward | no backward | no backward | no / no (1024) | +| rel. L2 error vs FP64 | **3.9e-3** | 7.0e-2 | 7.0e-2 | 7.0e-2 | 7.0e-2 | +| tokens routed unlike FP64 (of 256) | **0** | 11 | 11 | 11 | 11 | +| forward, 1 / 64 / 1024 / 4096 tokens (ms) | 1.38 / 1.71 / 2.05 / 3.05 | 0.37 / 0.66 / 0.88 / 1.06 | 0.36 / 0.65 / 0.87 / 1.06 | 0.34 / 0.67 / 0.85 / 1.08 | 2.0 / 61 / 89 / 89 | +| forward + backward, 64 / 1024 tokens (ms) | 112 / 166 | - | - | - | 878 / 1283 | + +* The accuracy gap is the router: every other candidate computes BF16 router + logits, and 11 of 256 tokens then select a different expert set than FP64 + routing does. RL-Kernel routes in FP32 and selects the same experts. +* RL-Kernel's forward is 2-4x slower than vLLM's. About 0.5 ms of it, at any + token count, is the FP32 IEEE router GEMM, which is part of the contract. + The expert GEMMs themselves run in vLLM's kernel. The per-expert GEMM loop + they replace (same bits) took 47 / 66 / 71 ms at 64 / 1024 / 4096 tokens in + a local B200 run of the same runner; that run is not part of this evidence. +* The backward is still a per-expert loop of pinned GEMMs (512 experts x 4 + GEMMs) and dominates a training step's MoE time. + +TP4 gate on the real layer-0 weights (`tp4-moe/rank-*.json`): all twelve cases +pass on all four ranks, including the round trip of all 512 experts, identical +routes on every rank, full == chunked and reordered batches at 8-1024 tokens, +and identical replicated router gradients. The TP4 block forward (routed + +shared expert + all-reduce) takes 1.9-2.4 ms at 8-1024 tokens. + +## Tests + +| File | Device | Covers | +| --- | --- | --- | +| `tests/test_qwen3_next_forward_contract.py` | CPU | CPU rejection, vLLM pin, fixed-order sum/softmax row independence | +| `tests/test_qwen3_next_tp_blocks.py` | CPU | HF shard/assemble round trips, replica drift rejection, TP4 parameter ownership | +| `tests/test_tensor_identity.py` | CPU | raw-bit identity (signed zero, NaN/Inf, dtype) | +| `tests/check_qwen3_next_forward.py` | CUDA + vLLM | GEMM batch/chunk/reorder, route ties, combine order, MoE VJP vs FP64, route and output batch invariance, grouped == per-expert bitwise, fail-closed TD path | +| `scripts/qwen3_next_tp_moe_check.py` | 4 x CUDA | real layer-0 weights: HF round trip, replicated routes, chunk/reorder, training forward, gradients | + +The `check_` file imports vLLM, so it runs in the `Qwen3-Next-provider-GPU` +workflow rather than the default collection. + +## Scope + +TP4 with EP1 (all 512 experts on every rank) on CUDA B200. ROCm, EP > 1 and +other TP sizes are separate claims. Model-level parity uses this block but is +the `full_model_chain` row. From dc79953b823bcaaf5619e36605ea7650ebfc2814 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:49:38 +0000 Subject: [PATCH 03/11] test(qwen3-next): measure Megatron-core/TE and SGLang in the MoE prior-art runner - Megatron-core 0.16 MoELayer with Transformer Engine grouped GEMM and fused permute, configured as VIME runs Qwen3-Next (FP32 softmax router, top-10, all-to-all dispatcher), including its backward. - SGLang 0.5.21 Triton fused_moe in default and deterministic-inference mode. sgl_kernel cannot load next to torch 2.13, so its two kernels on this path are replaced by SGLang's own Triton moe_sum_reduce and JIT moe_align_block_size; any other sgl_kernel call fails loudly. - --only and --merge combine reports from different Python environments. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- rl_engine/testing/qwen3_next_moe_prior_art.py | 264 +++++++++++++++++- scripts/plot_qwen3_next_moe_prior_art.py | 5 +- scripts/qwen3_next_moe_prior_art.py | 31 +- 3 files changed, 290 insertions(+), 10 deletions(-) diff --git a/rl_engine/testing/qwen3_next_moe_prior_art.py b/rl_engine/testing/qwen3_next_moe_prior_art.py index d8ad92234..90ed6dea3 100644 --- a/rl_engine/testing/qwen3_next_moe_prior_art.py +++ b/rl_engine/testing/qwen3_next_moe_prior_art.py @@ -298,6 +298,224 @@ def fwd(x): } +def _megatron_layer(w): + """Megatron-core MoELayer configured as VIME runs Qwen3-Next. + + VIME's ``scripts/models/qwen3-next-80B-A3B.sh``: + + Softmax router computed in FP32, top-10 of 512, all-to-all dispatcher, TE + grouped GEMM and TE fused permute, no auxiliary loss. Single process, TP1/EP1. + """ + import os + + import torch.distributed as dist + from megatron.core import parallel_state + from megatron.core.models.gpt.moe_module_specs import get_moe_module_spec + from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed + from megatron.core.transformer.spec_utils import build_module + from megatron.core.transformer.transformer_config import TransformerConfig + + if not dist.is_initialized(): + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", "29531") + device = torch.device("cuda", torch.cuda.current_device()) + dist.init_process_group("nccl", rank=0, world_size=1, device_id=device) + if not parallel_state.model_parallel_is_initialized(): + parallel_state.initialize_model_parallel(1, 1) + model_parallel_cuda_manual_seed(0) + config = TransformerConfig( + num_layers=1, + hidden_size=HIDDEN, + num_attention_heads=16, + ffn_hidden_size=WIDTH, + moe_ffn_hidden_size=WIDTH, + num_moe_experts=EXPERTS, + moe_router_topk=TOPK, + moe_router_score_function="softmax", + moe_router_dtype="fp32", + moe_token_dispatcher_type="alltoall", + moe_grouped_gemm=True, + moe_permute_fusion=True, + moe_aux_loss_coeff=0.0, + moe_router_load_balancing_type="none", + add_bias_linear=False, + gated_linear_unit=True, + activation_func=F.silu, + bf16=True, + params_dtype=torch.bfloat16, + ) + spec = get_moe_module_spec(use_te=True, num_experts=EXPERTS, moe_grouped_gemm=True) + layer = build_module(spec, config=config).cuda() + with torch.no_grad(): + layer.router.weight.copy_(w["router"]) + for expert in range(EXPERTS): + getattr(layer.experts.linear_fc1, f"weight{expert}").copy_(w["gate_up"][expert]) + getattr(layer.experts.linear_fc2, f"weight{expert}").copy_(w["down"][expert]) + return layer + + +def _megatron(w): + import megatron.core + import transformer_engine + + layer = _megatron_layer(w) + gate_up = [getattr(layer.experts.linear_fc1, f"weight{e}") for e in range(EXPERTS)] + down = [getattr(layer.experts.linear_fc2, f"weight{e}") for e in range(EXPERTS)] + + def route(x): + with torch.no_grad(): + probs, routing_map = layer.router(x) + weights, ids = probs.topk(TOPK, dim=-1) + if not bool(routing_map.gather(1, ids).all()): + raise RuntimeError("Megatron routing map disagrees with its routed probabilities") + return ids, weights + + def call(x): + return layer(x.unsqueeze(1))[0].squeeze(1) + + def fwd(x): + with torch.no_grad(): + return call(x) + + def grads(x, dy): + leaf = x.detach().clone().requires_grad_(True) + dx, *dw = torch.autograd.grad(call(leaf), [leaf, *gate_up, *down], dy) + return dx, torch.stack(dw[:EXPERTS]), torch.stack(dw[EXPERTS:]) + + return { + "name": "Megatron-core MoELayer + TE grouped GEMM (VIME's Qwen3-Next config)", + "source": f"megatron-core {megatron.core.__version__}, " + f"transformer-engine {transformer_engine.__version__}", + "route": route, + "fwd": fwd, + "grads": grads, + } + + +def _sglang_shims(): + """Make SGLang's Triton MoE path importable next to torch 2.13. + + ``sgl_kernel`` 0.3.21 is built against another libtorch ABI and cannot load + here. On CUDA, SGLang's Triton MoE calls two of its kernels; both are + replaced by SGLang's own implementations of the same operation: the Triton + ``moe_sum_reduce_triton`` and the JIT ``moe_align_block_size`` that SGLang + registers with the same signature. Every other ``sgl_kernel`` symbol only + has to import; calling one raises. + """ + import importlib.abc + import importlib.machinery + import sys + import types + + class Missing: + def __init__(self, name): + self.name = name + + def __getattr__(self, name): + if name.startswith("__"): + raise AttributeError(name) + return Missing(f"{self.name}.{name}") + + def __call__(self, *args, **kwargs): + raise RuntimeError(f"{self.name} called, but sgl_kernel cannot load next to torch 2.13") + + class Stub(types.ModuleType): + def __getattr__(self, name): + if name.startswith("__"): + raise AttributeError(name) + return Missing(f"{self.__name__}.{name}") + + class Finder(importlib.abc.MetaPathFinder, importlib.abc.Loader): + def find_spec(self, name, path=None, target=None): + if name in ("sgl_kernel", "gguf") or name.startswith("sgl_kernel."): + return importlib.machinery.ModuleSpec(name, self, is_package=True) + + def create_module(self, spec): + module = Stub(spec.name) + module.__path__ = [] + return module + + def exec_module(self, module): + pass + + if not any(type(f).__name__ == "Finder" for f in sys.meta_path): + sys.meta_path.insert(0, Finder()) + import sgl_kernel + import sglang.kernels.ops.moe as moe_ops + from sglang.kernels.ops.moe.fused_moe_triton_kernels import moe_sum_reduce_triton + from sglang.kernels.spec import KernelBackend + + sgl_kernel.moe_sum_reduce = moe_sum_reduce_triton + lookup = moe_ops.get_kernel + moe_ops.get_kernel = lambda op, backend: lookup( + op, KernelBackend.JIT if op == "moe.moe_align_block_size" else backend + ) + + +def _sglang(w, deterministic: bool): + import sglang + + _sglang_shims() + from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig + from sglang.srt.layers.moe.moe_runner.triton_utils import fused_moe as fm + from sglang.srt.layers.moe.moe_runner.triton_utils import fused_moe_triton_config as fc + from sglang.srt.layers.moe.topk import StandardTopKOutput, fused_topk_torch_native + + class Bag: + def __init__(self, **values): + self.__dict__.update(values) + + def __getattr__(self, name): + return False + + # SGLang reads these switches from its published server config; publishing + # needs a full server (and sgl_kernel). Only the deterministic switch matters. + settings = Bag(deterministic=Bag(enable_deterministic_inference=deterministic), moe=Bag()) + from sglang.srt.batch_invariant_ops import batch_invariant_ops as bio + + def linear(x, weight): + # Deterministic mode: what enable_batch_invariant_mode() installs for aten::mm, + # called directly so the override does not leak into other candidates. + return bio.mm_batch_invariant(x, weight.t()) if deterministic else F.linear(x, weight) + + config = MoeRunnerConfig( + num_experts=EXPERTS, + num_local_experts=EXPERTS, + hidden_size=HIDDEN, + intermediate_size_per_partition=WIDTH, + top_k=TOPK, + params_dtype=torch.bfloat16, + inplace=False, + ) + + def set_mode(): + fm.get_exec = fc.get_exec = lambda: settings + fm.is_batch_invariant_mode_enabled = lambda: deterministic + # One process, no TP group: symmetric memory is disabled, the group unused. + fm.get_parallel = lambda: Bag(tp_group=None) + fm.is_allocation_symmetric = lambda: False + + def route(x): + set_mode() + logits = linear(x, w["router"]) + weights, ids = fused_topk_torch_native(x, logits, TOPK, renormalize=True)[:2] + return ids.long(), weights + + def fwd(x): + with torch.no_grad(): + ids, weights = route(x) + topk = StandardTopKOutput(weights, ids.int(), None) + return fm.fused_experts(x, w["gate_up"], w["down"], topk, config) + + mode = "deterministic inference" if deterministic else "default" + return { + "name": f"SGLang fused_moe (Triton), {mode}", + "source": f"sglang {sglang.__version__} moe_runner/triton_utils", + "route": route, + "fwd": fwd, + } + + def candidates(w) -> list[dict[str, Any]]: return [ _guard("rl_kernel_cuda", lambda: _rl_kernel(w)), @@ -305,6 +523,9 @@ def candidates(w) -> list[dict[str, Any]]: _guard("vllm_bi0", lambda: _vllm(w, False)), _guard("vllm_bi1", lambda: _vllm(w, True)), _guard("flashinfer_cutlass", lambda: _flashinfer(w)), + _guard("megatron_te", lambda: _megatron(w)), + _guard("sglang_triton", lambda: _sglang(w, False)), + _guard("sglang_deterministic", lambda: _sglang(w, True)), ] @@ -338,14 +559,25 @@ def _rows_bi(fn, x) -> dict[str, bool]: return result +def has_backward(cand) -> bool: + return "graph" in cand or "grads" in cand + + +def cand_grads(cand, rows, dy_rows): + """``(dx, d gate_up, d down)`` of one backward.""" + if "grads" in cand: + return cand["grads"](rows, dy_rows) + y, leaves = cand["graph"](rows) + return torch.autograd.grad(y, leaves, dy_rows) + + def _grad_bi(cand, x) -> dict[str, Any]: """dx of the probe rows, and dW with zero-gradient rows appended.""" g = torch.Generator(device="cuda").manual_seed(7) dy = torch.randn(x.shape, device="cuda", generator=g).to(torch.bfloat16) def grads(rows, dy_rows): - y, leaves = cand["graph"](rows) - return torch.autograd.grad(y, leaves, dy_rows) + return cand_grads(cand, rows, dy_rows) alone = grads(x[:PROBE], dy[:PROBE]) dx, dw = {}, {} @@ -380,7 +612,7 @@ def _one(cand, x, w) -> dict[str, Any]: out["route_rows_bitwise"] = _rows_bi(cand["route"], x) out["output_rows_bitwise"] = _rows_bi(cand["fwd"], x) out["repeatable"] = bool(_bitwise(cand["fwd"](x[:256]), cand["fwd"](x[:256]))) - if "graph" in cand: + if has_backward(cand): out.update(_grad_bi(cand, x)) out["accuracy"] = _accuracy(cand, x[:256], w) except Exception as exc: # noqa: BLE001 - a crash is a result, not a harness failure @@ -414,23 +646,39 @@ def _latency(cands, tokens) -> dict[str, Any]: dy = torch.randn_like(x) calls = {} for cand in ready: - if "graph" not in cand: + if not has_backward(cand): continue def step(c=cand): - y, leaves = c["graph"](x) - torch.autograd.grad(y, leaves, dy) + cand_grads(c, x, dy) calls[cand["key"]] = step backward[str(count)] = time_interleaved(calls, warmup=2, iters=10) return {"forward_us": forward, "forward_plus_backward_us": backward} -def moe_report() -> dict[str, Any]: +def moe_report(only=None) -> dict[str, Any]: + """The report for every candidate, or only for the keys in ``only``. + + Some candidates need another Python environment (SGLang's pinned torch ABI, + Megatron's training stack); run each environment with ``only`` and merge. + """ torch.manual_seed(0) w = make_weights() x = make_tokens(max(BI_SIZES), seed=1) - cands = candidates(w) + keys = [ + "rl_kernel_cuda", + "hf_transformers", + "vllm_bi0", + "vllm_bi1", + "flashinfer_cutlass", + "megatron_te", + "sglang_triton", + "sglang_deterministic", + ] + if only is not None and set(only) - set(keys): + raise ValueError(f"Unknown candidates: {sorted(set(only) - set(keys))}") + cands = [c for c in candidates(w) if only is None or c["key"] in only] return { "op": "qwen3_next_routed_moe", "shape": {"hidden": HIDDEN, "experts": EXPERTS, "top_k": TOPK, "expert_width": WIDTH}, diff --git a/scripts/plot_qwen3_next_moe_prior_art.py b/scripts/plot_qwen3_next_moe_prior_art.py index 17a966b61..f7e358b63 100644 --- a/scripts/plot_qwen3_next_moe_prior_art.py +++ b/scripts/plot_qwen3_next_moe_prior_art.py @@ -34,7 +34,7 @@ PASS = "#1baf7a" FAIL = "#d64545" NA = "#e4e3dd" -EXISTING = ("#eb6834", "#b8501f", "#f29a62", "#8a3a15") +EXISTING = ("#eb6834", "#b8501f", "#f29a62", "#8a3a15", "#f6bd94", "#d9773f", "#6b2a0e") LABELS = { "rl_kernel_cuda": "RL-Kernel shared_moe", @@ -42,6 +42,9 @@ "vllm_bi0": "vLLM fused_moe (BI=0)", "vllm_bi1": "vLLM fused_moe (BI=1)", "flashinfer_cutlass": "FlashInfer cutlass_fused_moe", + "megatron_te": "Megatron-core + TE (VIME config)", + "sglang_triton": "SGLang fused_moe (default)", + "sglang_deterministic": "SGLang fused_moe (deterministic)", } CHECKS = ( ("route_rows_bitwise", "routes"), diff --git a/scripts/qwen3_next_moe_prior_art.py b/scripts/qwen3_next_moe_prior_art.py index 398245eee..ae3a028b4 100644 --- a/scripts/qwen3_next_moe_prior_art.py +++ b/scripts/qwen3_next_moe_prior_art.py @@ -35,6 +35,8 @@ "transformers", "triton", "sglang", + "sgl-kernel", + "megatron_core", "megatron-core", "transformer_engine", "vime", @@ -54,9 +56,34 @@ def _version(name: str) -> str | None: return None +def merge(base: dict, extra: dict) -> dict: + """``base`` with ``extra``'s candidates and latencies added (same commit and GPU).""" + for field in ("rl_kernel_commit", "tracked_tree_dirty", "shape", "op"): + if base[field] != extra[field]: + raise SystemExit(f"Reports differ in {field}; they cannot be merged") + if base["environment"]["gpu"] != extra["environment"]["gpu"]: + raise SystemExit("Reports were measured on different GPUs") + keys = {c["key"] for c in base["candidates"]} + added = [c for c in extra["candidates"] if c["key"] not in keys] + merged = dict(base, candidates=base["candidates"] + added) + merged["latency"] = { + section: { + size: {**values, **extra["latency"][section].get(size, {})} + for size, values in sizes.items() + } + for section, sizes in base["latency"].items() + } + merged["environments"] = {"main": base["environment"], "extra": extra["environment"]} + return merged + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--only", help="comma-separated candidate keys (default: all)") + parser.add_argument( + "--merge", type=Path, help="a report from another environment to merge into this one" + ) args = parser.parse_args() if not torch.cuda.is_available(): raise SystemExit("needs a CUDA device") @@ -73,8 +100,10 @@ def main() -> None: "cuda": torch.version.cuda, "libraries": {name: _version(name) for name in LIBRARIES}, }, - **moe_report(), + **moe_report(args.only.split(",") if args.only else None), } + if args.merge is not None: + report = merge(json.loads(args.merge.read_text()), report) args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(json.dumps(report, indent=2) + "\n") print( From 53dce6e9db4fd8941732579074db2ef74295f004 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:49:57 +0000 Subject: [PATCH 04/11] docs(qwen3-next): MoE prior art with Megatron-core/TE and SGLang measured (B200, c58816b) Every library in the reuse table is now measured: Megatron-core 0.16 + TE 2.16 as VIME configures Qwen3-Next, and SGLang 0.5.21 in default and deterministic-inference mode. The TP4 gate evidence is unchanged (2e950f3, identical MoE code). Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../operators/qwen3-next-moe-route-combine.md | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/docs/operators/qwen3-next-moe-route-combine.md b/docs/operators/qwen3-next-moe-route-combine.md index ed11301a0..20ed1d6ba 100644 --- a/docs/operators/qwen3-next-moe-route-combine.md +++ b/docs/operators/qwen3-next-moe-route-combine.md @@ -69,7 +69,9 @@ the selected probabilities; the indices are discrete. | vLLM, `VLLM_BATCH_INVARIANT=1` | yes | yes | none | no VJP for training; BF16 router logits (Qwen3-Next and VIME route in FP32); routed weight applied in the expert GEMM epilogue on the BF16 output, then routes summed by `moe_sum` | | HF `Qwen3NextExperts` (eager) | yes | yes | autograd | `dx`/`dW` change with batch size (cuBLAS shape heuristics); BF16 router; `index_add_` combine in BF16 | | FlashInfer `cutlass_fused_moe` | (torch routing) | no | none | output changes with batch size; no VJP | -| SGLang, Megatron-core, Transformer Engine | not measured | not measured | - | not installable next to the pinned vLLM 0.30.0 / torch 2.13 runtime | +| Megatron-core 0.16 `MoELayer` + TE 2.16 grouped GEMM, as VIME configures Qwen3-Next | no (256+) | no (1024) | autograd, not BI | FP32 router routes like FP64, but the routing, output, dx and dW change with batch size | +| SGLang 0.5.21 Triton `fused_moe`, default | no (256+) | no (256+) | none | tuned configs and the BF16 router GEMM depend on the token count | +| SGLang, deterministic inference | yes | yes | none | inference only; BF16 router logits; its deterministic tile (64/64/32) is the one vLLM uses in batch-invariant mode | What RL-Kernel reuses: vLLM's batch-invariant GEMM for the router and the backward, and vLLM's `fused_moe` Triton kernel for the forward expert GEMMs, @@ -91,25 +93,32 @@ commands are in TP1 shape (H=2048, 512 experts, top-10, width 512), random weights, BF16: -| | RL-Kernel `shared_moe` | vLLM BI=1 | vLLM BI=0 | FlashInfer CUTLASS | HF eager | -| --- | --- | --- | --- | --- | --- | -| routes / output BI (16-1024 tokens) | **yes / yes** | yes / yes | no / no (256+) | no / no (256+) | yes / yes | -| dx / dW BI | **yes / yes** | no backward | no backward | no backward | no / no (1024) | -| rel. L2 error vs FP64 | **3.9e-3** | 7.0e-2 | 7.0e-2 | 7.0e-2 | 7.0e-2 | -| tokens routed unlike FP64 (of 256) | **0** | 11 | 11 | 11 | 11 | -| forward, 1 / 64 / 1024 / 4096 tokens (ms) | 1.38 / 1.71 / 2.05 / 3.05 | 0.37 / 0.66 / 0.88 / 1.06 | 0.36 / 0.65 / 0.87 / 1.06 | 0.34 / 0.67 / 0.85 / 1.08 | 2.0 / 61 / 89 / 89 | -| forward + backward, 64 / 1024 tokens (ms) | 112 / 166 | - | - | - | 878 / 1283 | - -* The accuracy gap is the router: every other candidate computes BF16 router - logits, and 11 of 256 tokens then select a different expert set than FP64 - routing does. RL-Kernel routes in FP32 and selects the same experts. +| | routes / output BI (16-1024 tokens) | dx / dW BI | rel. L2 vs FP64 | tokens routed unlike FP64 (of 256) | forward, 1 / 64 / 1024 / 4096 tokens (ms) | fwd + bwd, 64 / 1024 (ms) | +| --- | --- | --- | --- | --- | --- | --- | +| **RL-Kernel `shared_moe`** | **yes / yes** | **yes / yes** | **3.9e-3** | **0** | 1.39 / 1.69 / 2.04 / 3.05 | 114 / 169 | +| Megatron-core + TE (VIME config) | no / no | no / no (1024) | 4.6e-3 | 0 | 3.8 / 9.7 / 11.7 / 12.1 | 45 / 51 | +| vLLM BI=1 | yes / yes | no backward | 7.0e-2 | 11 | 0.39 / 0.66 / 0.87 / 1.05 | - | +| SGLang deterministic | yes / yes | no backward | 7.0e-2 | 11 | 0.35 / 0.66 / 0.84 / 1.37 | - | +| vLLM BI=0 | no / no | no backward | 7.0e-2 | 11 | 0.38 / 0.66 / 0.87 / 1.06 | - | +| SGLang default | no / no | no backward | 7.0e-2 | 11 | 0.34 / 0.61 / 0.83 / 1.35 | - | +| FlashInfer CUTLASS | no / no | no backward | 7.0e-2 | 11 | 0.37 / 0.72 / 0.90 / 1.12 | - | +| HF eager | yes / yes | no / no (1024) | 7.0e-2 | 11 | 2.1 / 62 / 89 / 87 | 877 / 1284 | + +* The accuracy gap is the router: every candidate with BF16 router logits sends + 11 of 256 tokens to a different expert set than FP64 routing does. RL-Kernel + and Megatron-core (VIME's training path) route in FP32 and select the same + experts as FP64. +* The only batch-invariant candidates are RL-Kernel and the inference-only + modes of vLLM and SGLang; of these, only RL-Kernel has a backward, so only it + can run the same forward on the training and the rollout side. * RL-Kernel's forward is 2-4x slower than vLLM's. About 0.5 ms of it, at any token count, is the FP32 IEEE router GEMM, which is part of the contract. The expert GEMMs themselves run in vLLM's kernel. The per-expert GEMM loop they replace (same bits) took 47 / 66 / 71 ms at 64 / 1024 / 4096 tokens in a local B200 run of the same runner; that run is not part of this evidence. * The backward is still a per-expert loop of pinned GEMMs (512 experts x 4 - GEMMs) and dominates a training step's MoE time. + GEMMs): 169 ms at 1024 tokens, against 51 ms for Megatron-core + TE, which is + not batch-invariant. It dominates a training step's MoE time. TP4 gate on the real layer-0 weights (`tp4-moe/rank-*.json`): all twelve cases pass on all four ranks, including the round trip of all 512 experts, identical From 7317e5a806deaff2c4210c85205d28d2a1e1e205 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:00:16 +0000 Subject: [PATCH 05/11] style(tools): mark the Qwen3-Next TP gate imports after sys.path setup as E402-exempt The gate scripts insert the repo root into sys.path before importing rl_engine and tools, which flake8 reports as E402. Mark those imports with noqa, as the other tools/ scripts do. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- tools/validation/models/qwen3_next_tp_moe_check.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/validation/models/qwen3_next_tp_moe_check.py b/tools/validation/models/qwen3_next_tp_moe_check.py index c9079b7df..775c9a84b 100644 --- a/tools/validation/models/qwen3_next_tp_moe_check.py +++ b/tools/validation/models/qwen3_next_tp_moe_check.py @@ -32,21 +32,23 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[3])) -import torch -import torch.distributed as dist -from safetensors import safe_open +import torch # noqa: E402 +import torch.distributed as dist # noqa: E402 +from safetensors import safe_open # noqa: E402 -from rl_engine.models.qwen3_next.qwen3_next_forward import ( +from rl_engine.models.qwen3_next.qwen3_next_forward import ( # noqa: E402 FORWARD_PROVIDER_ID, shared_router, stable_top10_routes, ) -from rl_engine.models.qwen3_next.qwen3_next_tp_blocks import ( +from rl_engine.models.qwen3_next.qwen3_next_tp_blocks import ( # noqa: E402 TP4MoE, assemble_moe_weight, moe_hf_shapes, ) -from rl_engine.validation.common.tensor_identity import assert_tensor_bitwise_equal as exact +from rl_engine.validation.common.tensor_identity import ( # noqa: E402 + assert_tensor_bitwise_equal as exact, +) ROOT = Path(__file__).resolve().parents[3] TOKEN_COUNTS = (8, 64, 256, 1024) From 52b97e3c788291f6611228aac3c46e4a232c8cb6 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:34:36 +0000 Subject: [PATCH 06/11] fix(qwen3-next): type annotations for mypy in the MoE prior-art runner Annotate the latency samples and accept the default-argument lambdas used as timed calls. No behaviour change. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- rl_engine/validation/models/qwen3_next_moe_prior_art.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rl_engine/validation/models/qwen3_next_moe_prior_art.py b/rl_engine/validation/models/qwen3_next_moe_prior_art.py index c0582726f..7c0cc33c9 100644 --- a/rl_engine/validation/models/qwen3_next_moe_prior_art.py +++ b/rl_engine/validation/models/qwen3_next_moe_prior_art.py @@ -64,14 +64,14 @@ def _sample_us(fn: Callable[[], Any]) -> float: return start.elapsed_time(end) * 1000.0 -def time_interleaved(calls: dict[str, Callable[[], Any]], warmup: int = 5, iters: int = 30): +def time_interleaved(calls: dict[str, Callable[..., Any]], warmup: int = 5, iters: int = 30): """Median latency per candidate; the order alternates so none always runs first.""" keys = list(calls) orders = (keys, list(reversed(keys))) for i in range(warmup): for key in orders[i % 2]: calls[key]() - samples = {key: [] for key in keys} + samples: dict[str, list[float]] = {key: [] for key in keys} for i in range(iters): for key in orders[i % 2]: samples[key].append(_sample_us(calls[key])) From 57216db60948b2381abea0005045153fa1c97e50 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Wed, 14 Oct 2026 20:24:12 +0000 Subject: [PATCH 07/11] refactor(qwen3-next): MoE route branch to the reviewed layout Per the layout review on #549: - the TP4 MoE gate moves to tools/validation/models/qwen3_next/; - the prior-art runner and its plot move to the Qwen3-Next benchmark PR (benchmarks/models/qwen3_next/); - the committed evidence directory becomes a ZIP release attachment with a short summary in reports/experiments/qwen3_next/ (download link, SHA-256, tested commits, commands); the bytes are unchanged. The GPU workflow now also triggers on files under the qwen3_next/ validation directories. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .github/workflows/qwen3-next-provider-gpu.yml | 2 + .../operators/qwen3-next-moe-route-combine.md | 15 +- .../qwen3_next/qwen3-next-moe-route-b200.md | 57 ++ .../models/qwen3_next_moe_prior_art.py | 688 ------------------ .../models/plot_qwen3_next_moe_prior_art.py | 162 ----- .../validation/models/qwen3_next/__init__.py | 2 + .../qwen3_next_tp_moe_check.py | 6 +- .../models/qwen3_next_moe_prior_art.py | 116 --- 8 files changed, 73 insertions(+), 975 deletions(-) create mode 100644 reports/experiments/qwen3_next/qwen3-next-moe-route-b200.md delete mode 100644 rl_engine/validation/models/qwen3_next_moe_prior_art.py delete mode 100644 tools/validation/models/plot_qwen3_next_moe_prior_art.py create mode 100644 tools/validation/models/qwen3_next/__init__.py rename tools/validation/models/{ => qwen3_next}/qwen3_next_tp_moe_check.py (97%) delete mode 100644 tools/validation/models/qwen3_next_moe_prior_art.py diff --git a/.github/workflows/qwen3-next-provider-gpu.yml b/.github/workflows/qwen3-next-provider-gpu.yml index 84b7a7b0e..46a42a2b3 100644 --- a/.github/workflows/qwen3-next-provider-gpu.yml +++ b/.github/workflows/qwen3-next-provider-gpu.yml @@ -25,6 +25,8 @@ on: - "rl_engine/integrations/engines/train/vllm/qwen3_next*" - "rl_engine/validation/models/qwen3_next*" - "tools/validation/models/*qwen3_next*" + - "rl_engine/validation/models/qwen3_next/**" + - "tools/validation/models/qwen3_next/**" - "tests/models/qwen3_next/**" - "tests/models/qwen3_next/check_gdn_recurrent_golden.py" - ".github/workflows/qwen3-next-provider-gpu.yml" diff --git a/docs/operators/qwen3-next-moe-route-combine.md b/docs/operators/qwen3-next-moe-route-combine.md index 07804b8bc..06d84e879 100644 --- a/docs/operators/qwen3-next-moe-route-combine.md +++ b/docs/operators/qwen3-next-moe-route-combine.md @@ -84,12 +84,15 @@ with no change in bits. ## Results -Measured on B200 with `tools/validation/models/qwen3_next_moe_prior_art.py` and the TP4 gate -`tools/validation/models/qwen3_next_tp_moe_check.py`; the reports, the figure and the exact -commands are in -[`docs/usage/evidence/qwen3-next-moe-route-b200/`](../usage/evidence/qwen3-next-moe-route-b200/README.md). +Measured on B200 with the prior-art runner of the Qwen3-Next benchmark PR +(`benchmarks/models/qwen3_next/qwen3_next_moe_prior_art.py`) and this PR's TP4 gate +`tools/validation/models/qwen3_next/qwen3_next_tp_moe_check.py`. The summary, tested +commits and commands are in +[`reports/experiments/qwen3_next/qwen3-next-moe-route-b200.md`](https://github.com/fusheng-ji/RL-Kernel/blob/feat/ws1-qwen3-next-moe-route-combine/reports/experiments/qwen3_next/qwen3-next-moe-route-b200.md); +the raw reports and the figure are a +[ZIP release attachment](https://github.com/fusheng-ji/RL-Kernel/releases/download/qwen3-next-rfc428-evidence/qwen3-next-moe-route-b200.zip). -![Qwen3-Next routed MoE prior art](../usage/evidence/qwen3-next-moe-route-b200/figure.png) +![Qwen3-Next routed MoE prior art](https://github.com/fusheng-ji/RL-Kernel/releases/download/qwen3-next-rfc428-evidence/qwen3-next-moe-route-b200.png) TP1 shape (H=2048, 512 experts, top-10, width 512), random weights, BF16: @@ -134,7 +137,7 @@ shared expert + all-reduce) takes 1.9-2.4 ms at 8-1024 tokens. | `tests/models/qwen3_next/test_qwen3_next_tp_blocks.py` | CPU | HF shard/assemble round trips, replica drift rejection, TP4 parameter ownership | | `tests/validation/common/test_tensor_identity.py` | CPU | raw-bit identity (signed zero, NaN/Inf, dtype) | | `tests/models/qwen3_next/check_qwen3_next_forward.py` | CUDA + vLLM | GEMM batch/chunk/reorder, route ties, combine order, MoE VJP vs FP64, route and output batch invariance, grouped == per-expert bitwise, fail-closed TD path | -| `tools/validation/models/qwen3_next_tp_moe_check.py` | 4 x CUDA | real layer-0 weights: HF round trip, replicated routes, chunk/reorder, training forward, gradients | +| `tools/validation/models/qwen3_next/qwen3_next_tp_moe_check.py` | 4 x CUDA | real layer-0 weights: HF round trip, replicated routes, chunk/reorder, training forward, gradients | The `check_` file imports vLLM, so it runs in the `Qwen3-Next-provider-GPU` workflow rather than the default collection. diff --git a/reports/experiments/qwen3_next/qwen3-next-moe-route-b200.md b/reports/experiments/qwen3_next/qwen3-next-moe-route-b200.md new file mode 100644 index 000000000..a7d90d059 --- /dev/null +++ b/reports/experiments/qwen3_next/qwen3-next-moe-route-b200.md @@ -0,0 +1,57 @@ +# Qwen3-Next routed MoE: prior art and TP4 gate (B200) + +[Download raw results (ZIP)](https://github.com/fusheng-ji/RL-Kernel/releases/download/qwen3-next-rfc428-evidence/qwen3-next-moe-route-b200.zip). + +ZIP SHA-256: `9fcc704291bf697274edfe16d6958ca799134008acd9d5f452767b9a92758cb0`. + +The archive holds `report.json`, `figure.png`, the TP4 gate's +`tp4-moe/rank-{0..3}.json` and the original README, byte for byte as measured, +plus `SHA256SUMS`. The prior art was measured at commit `c58816b` from a clean +clone on one B200 (torch 2.13.0+cu130, Triton 3.7.1); the TP4 gate at +`2e950f3`, whose MoE code is identical, on four B200. The prior art uses +random weights (scale 0.02) at the TP1 shape (H=2048, 512 experts, top-10, +width 512); the TP4 gate reads layer 0 of the official +`Qwen/Qwen3-Next-80B-A3B-Instruct` revision +`9c7f2fbe84465e40164a94cc16cd30b6999b0cc7`. + +```bash +# prior art (benchmark PR, benchmarks/models/qwen3_next/), training environment +TRITON_F32_DEFAULT=ieee python benchmarks/models/qwen3_next/qwen3_next_moe_prior_art.py \ + --only rl_kernel_cuda,hf_transformers,vllm_bi0,vllm_bi1,flashinfer_cutlass,megatron_te --out main.json +# SGLang environment +TRITON_F32_DEFAULT=ieee python benchmarks/models/qwen3_next/qwen3_next_moe_prior_art.py \ + --only sglang_triton,sglang_deterministic --merge main.json --out report.json +python benchmarks/models/qwen3_next/plot_qwen3_next_moe_prior_art.py report.json +# TP4 gate on real layer-0 weights (this PR) +TRITON_F32_DEFAULT=ieee torchrun --nproc-per-node 4 tools/validation/models/qwen3_next/qwen3_next_tp_moe_check.py \ + --checkpoint --output tp4-moe +``` + +At `c58816b` / `2e950f3` the runners lived in `tools/validation/models/` and +`rl_engine/validation/models/`; the commands above use the current layout. + +| Candidate | Forward batch invariant | Backward batch invariant | Different expert set (of 256 tokens) | Max abs vs FP64 | Forward, 1024 tokens | +| --- | --- | --- | --- | --- | --- | +| **RL-Kernel `shared_moe`** | **yes** | **yes** | **0** | 2.1e-3 | 2.04 ms | +| HF transformers `Qwen3NextExperts` (eager) | yes | no (1024) | 11 | 1.1e-1 | 89.3 ms | +| vLLM `fused_experts`, `VLLM_BATCH_INVARIANT=0` | no (256, 1024) | n/a (inference) | 11 | 1.1e-1 | 0.87 ms | +| vLLM `fused_experts`, `VLLM_BATCH_INVARIANT=1` | yes | n/a (inference) | 11 | 1.1e-1 | 0.87 ms | +| FlashInfer `cutlass_fused_moe` | no (256, 1024) | n/a (inference) | 11 | 1.1e-1 | 0.90 ms | +| Megatron-core `MoELayer` + TE grouped GEMM | no (1024) | no (1024) | 0 | 2.2e-3 | 11.7 ms | +| SGLang `fused_moe` (Triton), default | no (256, 1024) | n/a (inference) | 11 | 1.1e-1 | 0.83 ms | +| SGLang `fused_moe`, deterministic inference | yes | n/a (inference) | 11 | 1.1e-1 | 0.84 ms | + +- Batch invariance: 8 probe tokens computed alone and placed first and last in + batches of 16, 64, 256 and 1024 tokens; the token counts in parentheses are + the batches that differ. +- Accuracy: 256 tokens against the HF formula in FP64 with FP64 routing. The + candidates with a BF16 router pick a different expert set for 11 of 256 + tokens, which dominates their error. +- SGLang's compiled `sgl_kernel` 0.3.21 cannot load next to torch 2.13, so its + two kernels on this path are SGLang's own replacements (Triton + `moe_sum_reduce`, JIT `moe_align_block_size`); `environments` in + `report.json` records both Python environments. +- TP4 gate (`tp4-moe/rank-*.json`): HF round trip, replicated routes, chunk and + reorder invariance, training forward and gradients pass on every rank. + +Site paths, host names and the scheduler do not appear in any file. diff --git a/rl_engine/validation/models/qwen3_next_moe_prior_art.py b/rl_engine/validation/models/qwen3_next_moe_prior_art.py deleted file mode 100644 index 7c0cc33c9..000000000 --- a/rl_engine/validation/models/qwen3_next_moe_prior_art.py +++ /dev/null @@ -1,688 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2026 RL-Kernel Contributors - -"""Prior-art comparison for the Qwen3-Next routed MoE (RFC #428 ``moe_route_combine_contract``). - -Every candidate computes the routed half of ``Qwen3NextSparseMoeBlock`` at the -official TP1 shape (H=2048, 512 experts, top-10, expert width 512): router -logits, top-10 selection with renormalised weights, the selected SwiGLU -experts and the weighted combine. The shared expert is left out because every -candidate evaluates it as the same dense MLP. - -For each candidate the report records - -* **routing batch invariance** - the expert ids and weights of a fixed block - of probe tokens computed alone, and embedded first and last in larger - batches, must be bitwise equal; -* **output batch invariance** - the same for the routed output; -* **dx / dW batch invariance** (candidates with a backward) - the probe rows' - input gradient must not change when unrelated rows join the batch, and the - expert weight gradient must not change when rows whose output gradient is - zero join it (mathematically neither can); -* **repeatability** - two identical calls are bitwise equal; -* **accuracy** - error against an FP64 evaluation of the HF formula with FP64 - routing, plus the fraction of tokens whose selected expert set differs; -* **performance** - median CUDA-event latency, candidates interleaved. - -A candidate that cannot be imported or launched in the pinned environment is -recorded as ``unavailable`` with the reason instead of failing the run. -""" - -from __future__ import annotations - -import os -import statistics -from contextlib import contextmanager -from typing import Any, Callable - -import torch -import torch.nn.functional as F - -HIDDEN = 2048 -EXPERTS = 512 -TOPK = 10 -WIDTH = 512 -PROBE = 8 -BI_SIZES = (16, 64, 256, 1024) -PERF_TOKENS = (1, 8, 64, 256, 1024, 4096) -BACKWARD_TOKENS = (64, 1024) - - -# --------------------------------------------------------------------------- # -# Measurement helpers -# --------------------------------------------------------------------------- # - - -def _sample_us(fn: Callable[[], Any]) -> float: - torch.cuda.synchronize() - start = torch.cuda.Event(enable_timing=True) - end = torch.cuda.Event(enable_timing=True) - start.record() - fn() - end.record() - torch.cuda.synchronize() - return start.elapsed_time(end) * 1000.0 - - -def time_interleaved(calls: dict[str, Callable[..., Any]], warmup: int = 5, iters: int = 30): - """Median latency per candidate; the order alternates so none always runs first.""" - keys = list(calls) - orders = (keys, list(reversed(keys))) - for i in range(warmup): - for key in orders[i % 2]: - calls[key]() - samples: dict[str, list[float]] = {key: [] for key in keys} - for i in range(iters): - for key in orders[i % 2]: - samples[key].append(_sample_us(calls[key])) - return {key: statistics.median(values) for key, values in samples.items()} - - -def _bitwise(a: torch.Tensor, b: torch.Tensor) -> bool: - return a.shape == b.shape and a.dtype == b.dtype and torch.equal(a, b) - - -def _rel(value: torch.Tensor, reference: torch.Tensor) -> float: - reference = reference.double() - return float((value.double() - reference).norm() / reference.norm().clamp_min(1e-300)) - - -def _guard(key: str, factory: Callable[[], dict[str, Any]]) -> dict[str, Any]: - try: - return {"key": key, **factory()} - except Exception as exc: # noqa: BLE001 - record and continue - return {"key": key, "unavailable": f"{type(exc).__name__}: {str(exc)[:300]}"} - - -@contextmanager -def _env(name: str, value: str | None): - old = os.environ.get(name) - if value is None: - os.environ.pop(name, None) - else: - os.environ[name] = value - try: - yield - finally: - if old is None: - os.environ.pop(name, None) - else: - os.environ[name] = old - - -# --------------------------------------------------------------------------- # -# Inputs and the FP64 reference -# --------------------------------------------------------------------------- # - - -def make_weights(seed: int = 0) -> dict[str, torch.Tensor]: - g = torch.Generator(device="cuda").manual_seed(seed) - - def randn(*shape, scale): - return (torch.randn(shape, device="cuda", generator=g) * scale).to(torch.bfloat16) - - return { - "router": randn(EXPERTS, HIDDEN, scale=0.02), - "gate_up": randn(EXPERTS, 2 * WIDTH, HIDDEN, scale=0.02), - "down": randn(EXPERTS, HIDDEN, WIDTH, scale=0.02), - } - - -def make_tokens(count: int, seed: int) -> torch.Tensor: - g = torch.Generator(device="cuda").manual_seed(seed) - return torch.randn(count, HIDDEN, device="cuda", generator=g).to(torch.bfloat16) - - -def fp64_reference(x: torch.Tensor, w: dict[str, torch.Tensor]): - """HF formula in FP64 with FP64 routing; returns (output, expert ids).""" - x64 = x.double() - probs = (x64 @ w["router"].double().T).softmax(-1) - weights, ids = probs.topk(TOPK, dim=-1) - weights = weights / weights.sum(-1, keepdim=True) - out = torch.zeros_like(x64) - gate_up, down = w["gate_up"], w["down"] - for expert in ids.unique().tolist(): - token, slot = torch.where(ids == expert) - gate, up = (x64[token] @ gate_up[expert].double().T).chunk(2, dim=-1) - y = (F.silu(gate) * up) @ down[expert].double().T - out.index_add_(0, token, y * weights[token, slot, None]) - return out, ids - - -# --------------------------------------------------------------------------- # -# Candidates -# -# ``route(x) -> (ids [T,10], weights [T,10])`` exposes the routing decision; -# ``fwd(x) -> [T, H]`` is the routed output; ``graph(x) -> (y, leaves)`` (when -# the candidate is differentiable) returns the output and [x, gate_up, down]. -# --------------------------------------------------------------------------- # - - -def _rl_kernel(w): - from rl_engine.models.qwen3_next import qwen3_next_forward as provider - - def route(x): - routes = provider.stable_top10_routes(provider.shared_router(x, w["router"])) - return routes.indices, routes.weights - - def fwd(x): - with torch.no_grad(): - return provider.shared_moe(x, w["router"], w["gate_up"], w["down"])[0] - - gate_up = w["gate_up"].detach().clone().requires_grad_(True) - down = w["down"].detach().clone().requires_grad_(True) - - def graph(x): - leaf = x.detach().clone().requires_grad_(True) - y, _ = provider.shared_moe(leaf, w["router"], gate_up, down) - return y, [leaf, gate_up, down] - - return { - "name": "rl-kernel shared_moe", - "source": "rl_engine.models.qwen3_next.qwen3_next_forward.shared_moe", - "route": route, - "fwd": fwd, - "graph": graph, - } - - -def _hf_modules(w, device="cuda"): - from transformers import Qwen3NextConfig - from transformers.models.qwen3_next import modeling_qwen3_next as hf - - config = Qwen3NextConfig( - hidden_size=HIDDEN, - num_experts=EXPERTS, - num_experts_per_tok=TOPK, - moe_intermediate_size=WIDTH, - norm_topk_prob=True, - ) - router = hf.Qwen3NextTopKRouter(config).to(device=device, dtype=torch.bfloat16) - experts = hf.Qwen3NextExperts(config).to(device=device, dtype=torch.bfloat16) - with torch.no_grad(): - router.weight.copy_(w["router"]) - experts.gate_up_proj.copy_(w["gate_up"]) - experts.down_proj.copy_(w["down"]) - return router, experts - - -def _hf(w): - import transformers - - router, experts = _hf_modules(w) - - def route(x): - _, weights, ids = router(x) - return ids, weights - - def call(x, mod): - _, weights, ids = router(x) - return mod(x, ids, weights) - - def fwd(x): - with torch.no_grad(): - return call(x, experts) - - def graph(x): - leaf = x.detach().clone().requires_grad_(True) - return call(leaf, experts), [leaf, experts.gate_up_proj, experts.down_proj] - - return { - "name": "HF transformers Qwen3NextExperts (eager)", - "source": f"transformers {transformers.__version__} modeling_qwen3_next", - "route": route, - "fwd": fwd, - "graph": graph, - } - - -def _vllm(w, batch_invariant: bool): - import vllm - from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts - from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk - - flag = "1" if batch_invariant else "0" - linear = F.linear - if batch_invariant: - from vllm.model_executor.determinism.batch_invariant import linear_batch_invariant - - linear = linear_batch_invariant - - def route(x): - with _env("VLLM_BATCH_INVARIANT", flag): - # vLLM's Qwen3-Next gate is a BF16 ReplicatedLinear; batch-invariant - # mode replaces its aten GEMM with vLLM's persistent Triton matmul. - logits = linear(x, w["router"]) - weights, ids, _ = fused_topk(x, logits, TOPK, renormalize=True) - return ids.long(), weights - - def fwd(x): - with torch.no_grad(), _env("VLLM_BATCH_INVARIANT", flag): - ids, weights = route(x) - return fused_experts(x, w["gate_up"], w["down"], weights, ids.int()) - - return { - "name": f"vLLM fused_topk + fused_experts (Triton), VLLM_BATCH_INVARIANT={flag}", - "source": f"vllm {vllm.__version__} model_executor/layers/fused_moe", - "route": route, - "fwd": fwd, - } - - -def _flashinfer(w): - import flashinfer - from flashinfer.fused_moe import cutlass_fused_moe - - # CUTLASS SwiGLU takes fc1 as [up; gate] (the opposite half order of HF). - gate, up = w["gate_up"].chunk(2, dim=1) - fc1 = torch.cat((up, gate), dim=1).contiguous() - - def route(x): - probs = F.linear(x, w["router"]).float().softmax(-1) - weights, ids = probs.topk(TOPK, dim=-1) - return ids, weights / weights.sum(-1, keepdim=True) - - def fwd(x): - with torch.no_grad(): - ids, weights = route(x) - out = cutlass_fused_moe( - x, ids.int(), weights, fc1, w["down"], torch.bfloat16, quant_scales=[] - ) - return out[0] if isinstance(out, (list, tuple)) else out - - return { - "name": "FlashInfer cutlass_fused_moe (routes from torch.topk)", - "source": f"flashinfer {flashinfer.__version__} fused_moe.cutlass_fused_moe", - "route": route, - "fwd": fwd, - } - - -def _megatron_layer(w): - """Megatron-core MoELayer configured as VIME runs Qwen3-Next. - - VIME's ``scripts/models/qwen3-next-80B-A3B.sh``: - - Softmax router computed in FP32, top-10 of 512, all-to-all dispatcher, TE - grouped GEMM and TE fused permute, no auxiliary loss. Single process, TP1/EP1. - """ - import os - - import torch.distributed as dist - from megatron.core import parallel_state - from megatron.core.models.gpt.moe_module_specs import get_moe_module_spec - from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed - from megatron.core.transformer.spec_utils import build_module - from megatron.core.transformer.transformer_config import TransformerConfig - - if not dist.is_initialized(): - os.environ.setdefault("MASTER_ADDR", "127.0.0.1") - os.environ.setdefault("MASTER_PORT", "29531") - device = torch.device("cuda", torch.cuda.current_device()) - dist.init_process_group("nccl", rank=0, world_size=1, device_id=device) - if not parallel_state.model_parallel_is_initialized(): - parallel_state.initialize_model_parallel(1, 1) - model_parallel_cuda_manual_seed(0) - config = TransformerConfig( - num_layers=1, - hidden_size=HIDDEN, - num_attention_heads=16, - ffn_hidden_size=WIDTH, - moe_ffn_hidden_size=WIDTH, - num_moe_experts=EXPERTS, - moe_router_topk=TOPK, - moe_router_score_function="softmax", - moe_router_dtype="fp32", - moe_token_dispatcher_type="alltoall", - moe_grouped_gemm=True, - moe_permute_fusion=True, - moe_aux_loss_coeff=0.0, - moe_router_load_balancing_type="none", - add_bias_linear=False, - gated_linear_unit=True, - activation_func=F.silu, - bf16=True, - params_dtype=torch.bfloat16, - ) - spec = get_moe_module_spec(use_te=True, num_experts=EXPERTS, moe_grouped_gemm=True) - layer = build_module(spec, config=config).cuda() - with torch.no_grad(): - layer.router.weight.copy_(w["router"]) - for expert in range(EXPERTS): - getattr(layer.experts.linear_fc1, f"weight{expert}").copy_(w["gate_up"][expert]) - getattr(layer.experts.linear_fc2, f"weight{expert}").copy_(w["down"][expert]) - return layer - - -def _megatron(w): - import megatron.core - import transformer_engine - - layer = _megatron_layer(w) - gate_up = [getattr(layer.experts.linear_fc1, f"weight{e}") for e in range(EXPERTS)] - down = [getattr(layer.experts.linear_fc2, f"weight{e}") for e in range(EXPERTS)] - - def route(x): - with torch.no_grad(): - probs, routing_map = layer.router(x) - weights, ids = probs.topk(TOPK, dim=-1) - if not bool(routing_map.gather(1, ids).all()): - raise RuntimeError("Megatron routing map disagrees with its routed probabilities") - return ids, weights - - def call(x): - return layer(x.unsqueeze(1))[0].squeeze(1) - - def fwd(x): - with torch.no_grad(): - return call(x) - - def grads(x, dy): - leaf = x.detach().clone().requires_grad_(True) - dx, *dw = torch.autograd.grad(call(leaf), [leaf, *gate_up, *down], dy) - return dx, torch.stack(dw[:EXPERTS]), torch.stack(dw[EXPERTS:]) - - return { - "name": "Megatron-core MoELayer + TE grouped GEMM (VIME's Qwen3-Next config)", - "source": f"megatron-core {megatron.core.__version__}, " - f"transformer-engine {transformer_engine.__version__}", - "route": route, - "fwd": fwd, - "grads": grads, - } - - -def _sglang_shims(): - """Make SGLang's Triton MoE path importable next to torch 2.13. - - ``sgl_kernel`` 0.3.21 is built against another libtorch ABI and cannot load - here. On CUDA, SGLang's Triton MoE calls two of its kernels; both are - replaced by SGLang's own implementations of the same operation: the Triton - ``moe_sum_reduce_triton`` and the JIT ``moe_align_block_size`` that SGLang - registers with the same signature. Every other ``sgl_kernel`` symbol only - has to import; calling one raises. - """ - import importlib.abc - import importlib.machinery - import sys - import types - - class Missing: - def __init__(self, name): - self.name = name - - def __getattr__(self, name): - if name.startswith("__"): - raise AttributeError(name) - return Missing(f"{self.name}.{name}") - - def __call__(self, *args, **kwargs): - raise RuntimeError(f"{self.name} called, but sgl_kernel cannot load next to torch 2.13") - - class Stub(types.ModuleType): - def __getattr__(self, name): - if name.startswith("__"): - raise AttributeError(name) - return Missing(f"{self.__name__}.{name}") - - class Finder(importlib.abc.MetaPathFinder, importlib.abc.Loader): - def find_spec(self, name, path=None, target=None): - if name in ("sgl_kernel", "gguf") or name.startswith("sgl_kernel."): - return importlib.machinery.ModuleSpec(name, self, is_package=True) - - def create_module(self, spec): - module = Stub(spec.name) - module.__path__ = [] - return module - - def exec_module(self, module): - pass - - if not any(type(f).__name__ == "Finder" for f in sys.meta_path): - sys.meta_path.insert(0, Finder()) - import sgl_kernel - import sglang.kernels.ops.moe as moe_ops - from sglang.kernels.ops.moe.fused_moe_triton_kernels import moe_sum_reduce_triton - from sglang.kernels.spec import KernelBackend - - sgl_kernel.moe_sum_reduce = moe_sum_reduce_triton - lookup = moe_ops.get_kernel - moe_ops.get_kernel = lambda op, backend: lookup( - op, KernelBackend.JIT if op == "moe.moe_align_block_size" else backend - ) - - -def _sglang(w, deterministic: bool): - import sglang - - _sglang_shims() - from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig - from sglang.srt.layers.moe.moe_runner.triton_utils import fused_moe as fm - from sglang.srt.layers.moe.moe_runner.triton_utils import fused_moe_triton_config as fc - from sglang.srt.layers.moe.topk import StandardTopKOutput, fused_topk_torch_native - - class Bag: - def __init__(self, **values): - self.__dict__.update(values) - - def __getattr__(self, name): - return False - - # SGLang reads these switches from its published server config; publishing - # needs a full server (and sgl_kernel). Only the deterministic switch matters. - settings = Bag(deterministic=Bag(enable_deterministic_inference=deterministic), moe=Bag()) - from sglang.srt.batch_invariant_ops import batch_invariant_ops as bio - - def linear(x, weight): - # Deterministic mode: what enable_batch_invariant_mode() installs for aten::mm, - # called directly so the override does not leak into other candidates. - return bio.mm_batch_invariant(x, weight.t()) if deterministic else F.linear(x, weight) - - config = MoeRunnerConfig( - num_experts=EXPERTS, - num_local_experts=EXPERTS, - hidden_size=HIDDEN, - intermediate_size_per_partition=WIDTH, - top_k=TOPK, - params_dtype=torch.bfloat16, - inplace=False, - ) - - def set_mode(): - fm.get_exec = fc.get_exec = lambda: settings - fm.is_batch_invariant_mode_enabled = lambda: deterministic - # One process, no TP group: symmetric memory is disabled, the group unused. - fm.get_parallel = lambda: Bag(tp_group=None) - fm.is_allocation_symmetric = lambda: False - - def route(x): - set_mode() - logits = linear(x, w["router"]) - weights, ids = fused_topk_torch_native(x, logits, TOPK, renormalize=True)[:2] - return ids.long(), weights - - def fwd(x): - with torch.no_grad(): - ids, weights = route(x) - topk = StandardTopKOutput(weights, ids.int(), None) - return fm.fused_experts(x, w["gate_up"], w["down"], topk, config) - - mode = "deterministic inference" if deterministic else "default" - return { - "name": f"SGLang fused_moe (Triton), {mode}", - "source": f"sglang {sglang.__version__} moe_runner/triton_utils", - "route": route, - "fwd": fwd, - } - - -def candidates(w) -> list[dict[str, Any]]: - return [ - _guard("rl_kernel_cuda", lambda: _rl_kernel(w)), - _guard("hf_transformers", lambda: _hf(w)), - _guard("vllm_bi0", lambda: _vllm(w, False)), - _guard("vllm_bi1", lambda: _vllm(w, True)), - _guard("flashinfer_cutlass", lambda: _flashinfer(w)), - _guard("megatron_te", lambda: _megatron(w)), - _guard("sglang_triton", lambda: _sglang(w, False)), - _guard("sglang_deterministic", lambda: _sglang(w, True)), - ] - - -# --------------------------------------------------------------------------- # -# Checks -# --------------------------------------------------------------------------- # - - -def _embedded(x: torch.Tensor, size: int): - """Probe rows first in a batch of ``size``, and the same probe rows last.""" - first = x[:size] - last = x[:size].clone() - last[size - PROBE :] = x[:PROBE] - return first, last - - -def _rows_bi(fn, x) -> dict[str, bool]: - alone = fn(x[:PROBE]) - result = {} - for size in BI_SIZES: - first, last = _embedded(x, size) - a, b = fn(first), fn(last) - if isinstance(alone, tuple): - ok = all( - _bitwise(p[:PROBE], q) and _bitwise(r[size - PROBE :], q) - for p, r, q in zip(a, b, alone) - ) - else: - ok = _bitwise(a[:PROBE], alone) and _bitwise(b[size - PROBE :], alone) - result[str(size)] = bool(ok) - return result - - -def has_backward(cand) -> bool: - return "graph" in cand or "grads" in cand - - -def cand_grads(cand, rows, dy_rows): - """``(dx, d gate_up, d down)`` of one backward.""" - if "grads" in cand: - return cand["grads"](rows, dy_rows) - y, leaves = cand["graph"](rows) - return torch.autograd.grad(y, leaves, dy_rows) - - -def _grad_bi(cand, x) -> dict[str, Any]: - """dx of the probe rows, and dW with zero-gradient rows appended.""" - g = torch.Generator(device="cuda").manual_seed(7) - dy = torch.randn(x.shape, device="cuda", generator=g).to(torch.bfloat16) - - def grads(rows, dy_rows): - return cand_grads(cand, rows, dy_rows) - - alone = grads(x[:PROBE], dy[:PROBE]) - dx, dw = {}, {} - for size in BI_SIZES: - full = grads(x[:size], dy[:size]) - dx[str(size)] = bool(_bitwise(full[0][:PROBE], alone[0])) - padded = dy[:size].clone() - padded[PROBE:] = 0 - masked = grads(x[:size], padded) - dw[str(size)] = bool(all(_bitwise(m, a) for m, a in zip(masked[1:], alone[1:]))) - return {"dx_rows_bitwise": dx, "dweight_zero_rows_bitwise": dw} - - -def _accuracy(cand, x, w) -> dict[str, Any]: - ref, ref_ids = fp64_reference(x, w) - ids, _ = cand["route"](x) - same = ids.long().sort(-1).values == ref_ids.sort(-1).values - out = cand["fwd"](x) - return { - "rel_l2_vs_fp64": _rel(out, ref), - "max_abs_vs_fp64": float((out.double() - ref).abs().max()), - "tokens_with_different_expert_set": int((~same.all(-1)).sum()), - "tokens": int(x.shape[0]), - } - - -def _one(cand, x, w) -> dict[str, Any]: - if "unavailable" in cand: - return cand - out = {"key": cand["key"], "name": cand["name"], "source": cand["source"]} - try: - out["route_rows_bitwise"] = _rows_bi(cand["route"], x) - out["output_rows_bitwise"] = _rows_bi(cand["fwd"], x) - out["repeatable"] = bool(_bitwise(cand["fwd"](x[:256]), cand["fwd"](x[:256]))) - if has_backward(cand): - out.update(_grad_bi(cand, x)) - out["accuracy"] = _accuracy(cand, x[:256], w) - except Exception as exc: # noqa: BLE001 - a crash is a result, not a harness failure - out["failed"] = f"{type(exc).__name__}: {str(exc)[:300]}" - out["batch_invariant"] = bool( - "failed" not in out - and all(out["route_rows_bitwise"].values()) - and all(out["output_rows_bitwise"].values()) - and all(out.get("dx_rows_bitwise", {"": True}).values()) - and all(out.get("dweight_zero_rows_bitwise", {"": True}).values()) - ) - return out - - -def _latency(cands, tokens) -> dict[str, Any]: - ready = [c for c in cands if "unavailable" not in c] - forward = {} - for count in tokens: - x = make_tokens(count, seed=100 + count) - calls = {} - for cand in ready: - try: - cand["fwd"](x) - calls[cand["key"]] = lambda c=cand: c["fwd"](x) - except Exception: # noqa: BLE001 - recorded by the BI section - continue - forward[str(count)] = time_interleaved(calls) - backward = {} - for count in BACKWARD_TOKENS: - x = make_tokens(count, seed=200 + count) - dy = torch.randn_like(x) - calls = {} - for cand in ready: - if not has_backward(cand): - continue - - def step(c=cand): - cand_grads(c, x, dy) - - calls[cand["key"]] = step - backward[str(count)] = time_interleaved(calls, warmup=2, iters=10) - return {"forward_us": forward, "forward_plus_backward_us": backward} - - -def moe_report(only=None) -> dict[str, Any]: - """The report for every candidate, or only for the keys in ``only``. - - Some candidates need another Python environment (SGLang's pinned torch ABI, - Megatron's training stack); run each environment with ``only`` and merge. - """ - torch.manual_seed(0) - w = make_weights() - x = make_tokens(max(BI_SIZES), seed=1) - keys = [ - "rl_kernel_cuda", - "hf_transformers", - "vllm_bi0", - "vllm_bi1", - "flashinfer_cutlass", - "megatron_te", - "sglang_triton", - "sglang_deterministic", - ] - if only is not None and set(only) - set(keys): - raise ValueError(f"Unknown candidates: {sorted(set(only) - set(keys))}") - cands = [c for c in candidates(w) if only is None or c["key"] in only] - return { - "op": "qwen3_next_routed_moe", - "shape": {"hidden": HIDDEN, "experts": EXPERTS, "top_k": TOPK, "expert_width": WIDTH}, - "probe_rows": PROBE, - "candidates": [_one(c, x, w) for c in cands], - "latency": _latency(cands, PERF_TOKENS), - } diff --git a/tools/validation/models/plot_qwen3_next_moe_prior_art.py b/tools/validation/models/plot_qwen3_next_moe_prior_art.py deleted file mode 100644 index 21cee4fb9..000000000 --- a/tools/validation/models/plot_qwen3_next_moe_prior_art.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2026 RL-Kernel Contributors - -"""Render the routed-MoE prior-art report as a PNG. - -Needs only the report JSON and matplotlib, so it runs anywhere: - - python tools/validation/models/plot_qwen3_next_moe_prior_art.py \\ - docs/usage/evidence/qwen3-next-moe-route-b200/report.json - -writes ``figure.png`` next to the report. Three panels: which batch-invariance -checks each implementation passes, accuracy against an FP64 evaluation with -FP64 routing, and forward latency. RL-Kernel is blue, existing libraries orange. -""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - -import matplotlib - -matplotlib.use("Agg") -import matplotlib.pyplot as plt # noqa: E402 -from matplotlib.colors import ListedColormap # noqa: E402 - -RL_KERNEL = "#2a78d6" -INK = "#2b2b29" -MUTED = "#6f6e69" -GRID = "#e4e3dd" -SURFACE = "#fcfcfb" -PASS = "#1baf7a" -FAIL = "#d64545" -NA = "#e4e3dd" -EXISTING = ("#eb6834", "#b8501f", "#f29a62", "#8a3a15", "#f6bd94", "#d9773f", "#6b2a0e") - -LABELS = { - "rl_kernel_cuda": "RL-Kernel shared_moe", - "hf_transformers": "HF Qwen3NextExperts", - "vllm_bi0": "vLLM fused_moe (BI=0)", - "vllm_bi1": "vLLM fused_moe (BI=1)", - "flashinfer_cutlass": "FlashInfer cutlass_fused_moe", - "megatron_te": "Megatron-core + TE (VIME config)", - "sglang_triton": "SGLang fused_moe (default)", - "sglang_deterministic": "SGLang fused_moe (deterministic)", -} -CHECKS = ( - ("route_rows_bitwise", "routes"), - ("output_rows_bitwise", "output"), - ("dx_rows_bitwise", "dx"), - ("dweight_zero_rows_bitwise", "dW"), -) - - -def _colors(keys): - existing = iter(EXISTING) - return {k: RL_KERNEL if k == "rl_kernel_cuda" else next(existing) for k in keys} - - -def _style(ax): - ax.set_facecolor(SURFACE) - for side in ("top", "right"): - ax.spines[side].set_visible(False) - ax.tick_params(colors=INK, labelsize=8) - ax.grid(color=GRID, linewidth=0.6) - - -def plot(report: dict, out: Path) -> None: - cands = [c for c in report["candidates"] if "unavailable" not in c] - keys = [c["key"] for c in cands] - colors = _colors(keys) - sizes = list(next(iter(cands))["output_rows_bitwise"]) - fig, axes = plt.subplots(1, 3, figsize=(17, 4.6), gridspec_kw={"width_ratios": [1.5, 1, 1.4]}) - fig.patch.set_facecolor("white") - - # Panel 1: batch-invariance matrix; 1 = pass, 0 = fail, 0.5 = no backward. - ax = axes[0] - columns = [f"{label} {size}" for _, label in CHECKS for size in sizes] - grid = [] - for cand in cands: - row = [] - for field, _ in CHECKS: - values = cand.get(field) - row.extend(0.5 if values is None else float(values[s]) for s in sizes) - grid.append(row) - ax.imshow(grid, cmap=ListedColormap([FAIL, NA, PASS]), vmin=0, vmax=1, aspect="auto") - ax.set_xticks(range(len(columns)), columns, fontsize=7, rotation=90) - ax.set_yticks(range(len(cands)), [LABELS.get(k, k) for k in keys], fontsize=8) - for i, cand in enumerate(cands): - mark = "BI" if cand.get("batch_invariant") else "not BI" - ax.text(len(columns) - 0.4, i, f" {mark}", va="center", fontsize=8, color=INK) - ax.set_title( - f"Batch invariance: {report['probe_rows']} probe tokens alone vs. in a batch\n" - "(green bitwise equal, red differs, grey: no backward)", - fontsize=9, - color=INK, - ) - ax.tick_params(length=0) - - # Panel 2: accuracy. - ax = axes[1] - _style(ax) - rel = [c["accuracy"]["rel_l2_vs_fp64"] if "accuracy" in c else float("nan") for c in cands] - ax.barh(range(len(cands)), rel, color=[colors[k] for k in keys]) - for i, cand in enumerate(cands): - acc = cand.get("accuracy") - if acc: - ax.text( - rel[i], - i, - f" {rel[i]:.2e} ({acc['tokens_with_different_expert_set']}/{acc['tokens']}" - " tokens route differently)", - va="center", - fontsize=7, - color=MUTED, - ) - ax.set_yticks(range(len(cands)), [LABELS.get(k, k) for k in keys], fontsize=8) - ax.invert_yaxis() - ax.set_xlim(0, 2.2 * max(r for r in rel if r == r)) - ax.set_xlabel("relative L2 error vs. FP64 HF formula with FP64 routing", fontsize=8) - ax.set_title("Accuracy (256 tokens)", fontsize=9, color=INK) - - # Panel 3: forward latency. - ax = axes[2] - _style(ax) - forward = report["latency"]["forward_us"] - tokens = [int(t) for t in forward] - for key in keys: - ys = [forward[str(t)].get(key, float("nan")) / 1000.0 for t in tokens] - ax.plot(tokens, ys, marker="o", ms=3.5, color=colors[key], label=LABELS.get(key, key)) - ax.set_xscale("log", base=2) - ax.set_yscale("log") - ax.set_xlabel("tokens", fontsize=8) - ax.set_ylabel("forward latency (ms, median)", fontsize=8) - ax.set_title("Forward latency\n(H=2048, 512 experts, top-10, width 512)", fontsize=9) - ax.legend(fontsize=7, frameon=False) - - env = report["environment"] - fig.suptitle( - f"Qwen3-Next routed MoE prior art - {env['gpu']}, torch {env['torch']}, " - f"commit {report['rl_kernel_commit'][:7]}", - fontsize=10, - color=INK, - ) - fig.tight_layout() - fig.savefig(out, dpi=160) - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("report", type=Path) - args = parser.parse_args() - report = json.loads(args.report.read_text()) - out = args.report.with_name("figure.png") - plot(report, out) - print(f"wrote {out}") - - -if __name__ == "__main__": - main() diff --git a/tools/validation/models/qwen3_next/__init__.py b/tools/validation/models/qwen3_next/__init__.py new file mode 100644 index 000000000..86cf4c9da --- /dev/null +++ b/tools/validation/models/qwen3_next/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors diff --git a/tools/validation/models/qwen3_next_tp_moe_check.py b/tools/validation/models/qwen3_next/qwen3_next_tp_moe_check.py similarity index 97% rename from tools/validation/models/qwen3_next_tp_moe_check.py rename to tools/validation/models/qwen3_next/qwen3_next_tp_moe_check.py index 775c9a84b..f5a65f8ab 100644 --- a/tools/validation/models/qwen3_next_tp_moe_check.py +++ b/tools/validation/models/qwen3_next/qwen3_next_tp_moe_check.py @@ -5,7 +5,7 @@ """TP4 MoE gate on the official checkpoint's layer-0 weights; four GPUs, torchrun. TRITON_F32_DEFAULT=ieee torchrun --nproc-per-node 4 \\ - tools/validation/models/qwen3_next_tp_moe_check.py \\ + tools/validation/models/qwen3_next/qwen3_next_tp_moe_check.py \\ --checkpoint --output Each rank writes ``rank-.json``. The gate requires, bitwise: @@ -30,7 +30,7 @@ from importlib.metadata import version from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parents[3])) +sys.path.insert(0, str(Path(__file__).resolve().parents[4])) import torch # noqa: E402 import torch.distributed as dist # noqa: E402 @@ -50,7 +50,7 @@ assert_tensor_bitwise_equal as exact, ) -ROOT = Path(__file__).resolve().parents[3] +ROOT = Path(__file__).resolve().parents[4] TOKEN_COUNTS = (8, 64, 256, 1024) diff --git a/tools/validation/models/qwen3_next_moe_prior_art.py b/tools/validation/models/qwen3_next_moe_prior_art.py deleted file mode 100644 index c3c8d088a..000000000 --- a/tools/validation/models/qwen3_next_moe_prior_art.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2026 RL-Kernel Contributors - -"""Write the Qwen3-Next routed-MoE prior-art report (BI + accuracy + latency) as JSON. - -Run it on a clean tree with ``TRITON_F32_DEFAULT=ieee`` (the FP32 router needs -IEEE Triton dots from process start). The report records the commit, whether -the tracked tree was dirty and the version of every library compared. -``tools/validation/models/plot_qwen3_next_moe_prior_art.py`` turns the report into a figure. - - TRITON_F32_DEFAULT=ieee python tools/validation/models/qwen3_next_moe_prior_art.py \\ - --out docs/usage/evidence/qwen3-next-moe-route-b200/report.json -""" - -from __future__ import annotations - -import argparse -import importlib.metadata as md -import json -import subprocess -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[3] -sys.path.insert(0, str(ROOT)) - -import torch # noqa: E402 - -from rl_engine.validation.models.qwen3_next_moe_prior_art import moe_report # noqa: E402 - -LIBRARIES = ( - "vllm", - "flashinfer-python", - "transformers", - "triton", - "sglang", - "sgl-kernel", - "megatron_core", - "megatron-core", - "transformer_engine", - "vime", -) - - -def _git(*args: str) -> str: - return subprocess.run( - ["git", *args], cwd=ROOT, check=True, capture_output=True, text=True - ).stdout.strip() - - -def _version(name: str) -> str | None: - try: - return md.version(name) - except md.PackageNotFoundError: - return None - - -def merge(base: dict, extra: dict) -> dict: - """``base`` with ``extra``'s candidates and latencies added (same commit and GPU).""" - for field in ("rl_kernel_commit", "tracked_tree_dirty", "shape", "op"): - if base[field] != extra[field]: - raise SystemExit(f"Reports differ in {field}; they cannot be merged") - if base["environment"]["gpu"] != extra["environment"]["gpu"]: - raise SystemExit("Reports were measured on different GPUs") - keys = {c["key"] for c in base["candidates"]} - added = [c for c in extra["candidates"] if c["key"] not in keys] - merged = dict(base, candidates=base["candidates"] + added) - merged["latency"] = { - section: { - size: {**values, **extra["latency"][section].get(size, {})} - for size, values in sizes.items() - } - for section, sizes in base["latency"].items() - } - merged["environments"] = {"main": base["environment"], "extra": extra["environment"]} - return merged - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--out", type=Path, required=True) - parser.add_argument("--only", help="comma-separated candidate keys (default: all)") - parser.add_argument( - "--merge", type=Path, help="a report from another environment to merge into this one" - ) - args = parser.parse_args() - if not torch.cuda.is_available(): - raise SystemExit("needs a CUDA device") - torch.backends.cuda.matmul.allow_tf32 = False - torch.backends.cudnn.allow_tf32 = False - report = { - "kind": "qwen3_next_prior_art_report", - "rfc": 428, - "rl_kernel_commit": _git("rev-parse", "HEAD"), - "tracked_tree_dirty": bool(_git("status", "--porcelain", "--untracked-files=no")), - "environment": { - "gpu": torch.cuda.get_device_name(), - "torch": torch.__version__, - "cuda": torch.version.cuda, - "libraries": {name: _version(name) for name in LIBRARIES}, - }, - **moe_report(args.only.split(",") if args.only else None), - } - if args.merge is not None: - report = merge(json.loads(args.merge.read_text()), report) - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text(json.dumps(report, indent=2) + "\n") - print( - f"wrote {args.out} (commit {report['rl_kernel_commit'][:7]}, " - f"dirty={report['tracked_tree_dirty']})" - ) - - -if __name__ == "__main__": - main() From 8943c14fdab9d374dbbd5dcdccabf7dcc4743a5f Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:56:55 +0000 Subject: [PATCH 08/11] docs: move the measurement outputs out of the pull request The raw results (JSON, figures, audit files) now live on the fork's pr_assets branch; the documents link to them at a fixed commit. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- docs/operators/qwen3-next-rms-norm-gated.md | 6 +++--- docs/operators/qwen3-next-rms-norm.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/operators/qwen3-next-rms-norm-gated.md b/docs/operators/qwen3-next-rms-norm-gated.md index 217fa45ed..38441ed42 100644 --- a/docs/operators/qwen3-next-rms-norm-gated.md +++ b/docs/operators/qwen3-next-rms-norm-gated.md @@ -121,13 +121,13 @@ python tools/validation/operators/check_operator.py --op rms_norm_gated --candid ## Evidence -![gated RMSNorm vs existing implementations on B200](../usage/evidence/qwen3-next-rms-norm-gated-b200/figure-gated_rmsnorm.png) +![gated RMSNorm vs existing implementations on B200](https://raw.githubusercontent.com/fusheng-ji/RL-Kernel/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr468/docs/usage/evidence/qwen3-next-rms-norm-gated-b200/figure-gated_rmsnorm.png) -[`report.json`](../usage/evidence/qwen3-next-rms-norm-gated-b200/report.json) was written by +[`report.json`](https://github.com/fusheng-ji/RL-Kernel/blob/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr468/docs/usage/evidence/qwen3-next-rms-norm-gated-b200/report.json) was written by `tools/validation/models/qwen3_next_norm_evidence.py` from a clean tree at `822b085`, on an otherwise idle B200 (torch 2.13.0+cu130, transformers 5.17.0, vLLM 0.30.0). Head dim 128, BF16; the FP64 golden uses vLLM's convention. The same report re-measures the zero-centred op -([figure](../usage/evidence/qwen3-next-rms-norm-gated-b200/figure-zero_centred_rmsnorm.png)). +([figure](https://raw.githubusercontent.com/fusheng-ji/RL-Kernel/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr468/docs/usage/evidence/qwen3-next-rms-norm-gated-b200/figure-zero_centred_rmsnorm.png)). | | rl-kernel CUDA | PyTorch reference | transformers (cast-first) | vLLM `RMSNormGated`* | |---|---|---|---|---| diff --git a/docs/operators/qwen3-next-rms-norm.md b/docs/operators/qwen3-next-rms-norm.md index 2fbd8baa5..897b0ffff 100644 --- a/docs/operators/qwen3-next-rms-norm.md +++ b/docs/operators/qwen3-next-rms-norm.md @@ -100,9 +100,9 @@ element and no extra memory traffic. ## Evidence -![zero-centred RMSNorm vs existing implementations on B200](../usage/evidence/qwen3-next-rms-norm-b200/figure.png) +![zero-centred RMSNorm vs existing implementations on B200](https://raw.githubusercontent.com/fusheng-ji/RL-Kernel/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr467/docs/usage/evidence/qwen3-next-rms-norm-b200/figure.png) -[`report.json`](../usage/evidence/qwen3-next-rms-norm-b200/report.json) was written by +[`report.json`](https://github.com/fusheng-ji/RL-Kernel/blob/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr467/docs/usage/evidence/qwen3-next-rms-norm-b200/report.json) was written by `tools/validation/models/qwen3_next_norm_evidence.py` from a clean tree at `3d0bae7`, on an otherwise idle B200 (torch 2.13.0+cu130, transformers 5.17.0, vLLM 0.30.0, FlashInfer 0.6.18). Hidden 2048, BF16. From bc431137651b4dbda982bd36f4db7dc28b88471c Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:06:05 +0000 Subject: [PATCH 09/11] docs: point commit hashes at the rewritten history The history of this branch was rewritten on 2026-10-23 (evidence directories removed from every commit, sign-offs added). Commit hashes quoted in the reports and documents now name the rewritten commits; the evidence archives still record the pre-rewrite hashes. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- docs/operators/qwen3-next-rms-norm-gated.md | 2 +- reports/experiments/qwen3_next/qwen3-next-moe-route-b200.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/operators/qwen3-next-rms-norm-gated.md b/docs/operators/qwen3-next-rms-norm-gated.md index 38441ed42..a3895292a 100644 --- a/docs/operators/qwen3-next-rms-norm-gated.md +++ b/docs/operators/qwen3-next-rms-norm-gated.md @@ -124,7 +124,7 @@ python tools/validation/operators/check_operator.py --op rms_norm_gated --candid ![gated RMSNorm vs existing implementations on B200](https://raw.githubusercontent.com/fusheng-ji/RL-Kernel/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr468/docs/usage/evidence/qwen3-next-rms-norm-gated-b200/figure-gated_rmsnorm.png) [`report.json`](https://github.com/fusheng-ji/RL-Kernel/blob/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr468/docs/usage/evidence/qwen3-next-rms-norm-gated-b200/report.json) was written by -`tools/validation/models/qwen3_next_norm_evidence.py` from a clean tree at `822b085`, on an otherwise idle +`tools/validation/models/qwen3_next_norm_evidence.py` from a clean tree at `ff271d5`, on an otherwise idle B200 (torch 2.13.0+cu130, transformers 5.17.0, vLLM 0.30.0). Head dim 128, BF16; the FP64 golden uses vLLM's convention. The same report re-measures the zero-centred op ([figure](https://raw.githubusercontent.com/fusheng-ji/RL-Kernel/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr468/docs/usage/evidence/qwen3-next-rms-norm-gated-b200/figure-zero_centred_rmsnorm.png)). diff --git a/reports/experiments/qwen3_next/qwen3-next-moe-route-b200.md b/reports/experiments/qwen3_next/qwen3-next-moe-route-b200.md index a7d90d059..61a0aeb12 100644 --- a/reports/experiments/qwen3_next/qwen3-next-moe-route-b200.md +++ b/reports/experiments/qwen3_next/qwen3-next-moe-route-b200.md @@ -6,9 +6,9 @@ ZIP SHA-256: `9fcc704291bf697274edfe16d6958ca799134008acd9d5f452767b9a92758cb0`. The archive holds `report.json`, `figure.png`, the TP4 gate's `tp4-moe/rank-{0..3}.json` and the original README, byte for byte as measured, -plus `SHA256SUMS`. The prior art was measured at commit `c58816b` from a clean +plus `SHA256SUMS`. The prior art was measured at commit `dc79953` from a clean clone on one B200 (torch 2.13.0+cu130, Triton 3.7.1); the TP4 gate at -`2e950f3`, whose MoE code is identical, on four B200. The prior art uses +`cbdc00a`, whose MoE code is identical, on four B200. The prior art uses random weights (scale 0.02) at the TP1 shape (H=2048, 512 experts, top-10, width 512); the TP4 gate reads layer 0 of the official `Qwen/Qwen3-Next-80B-A3B-Instruct` revision @@ -27,7 +27,7 @@ TRITON_F32_DEFAULT=ieee torchrun --nproc-per-node 4 tools/validation/models/qwen --checkpoint --output tp4-moe ``` -At `c58816b` / `2e950f3` the runners lived in `tools/validation/models/` and +At `dc79953` / `cbdc00a` the runners lived in `tools/validation/models/` and `rl_engine/validation/models/`; the commands above use the current layout. | Candidate | Forward batch invariant | Backward batch invariant | Different expert set (of 256 tokens) | Max abs vs FP64 | Forward, 1024 tokens | From d53ac95edb2746e5948194c1df339b4391a29227 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:53:31 +0000 Subject: [PATCH 10/11] ci(qwen3-next): trigger the provider job on validation/common check_qwen3_next_forward.py imports rl_engine.validation.common.tensor_identity for its raw-bit comparisons, so a change there must run the provider job. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .github/workflows/qwen3-next-provider-gpu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/qwen3-next-provider-gpu.yml b/.github/workflows/qwen3-next-provider-gpu.yml index 30189fafb..299f184a8 100644 --- a/.github/workflows/qwen3-next-provider-gpu.yml +++ b/.github/workflows/qwen3-next-provider-gpu.yml @@ -32,6 +32,7 @@ on: - "tools/validation/models/qwen3_next/**" - "tests/models/qwen3_next/**" - "rl_engine/config/**" + - "rl_engine/validation/common/**" - ".github/workflows/qwen3-next-provider-gpu.yml" workflow_dispatch: inputs: From e514d42f3d4f07b5b8d271442f49b31d933093f4 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:07:36 +0000 Subject: [PATCH 11/11] fix(qwen3-next): fixed-order route-weight gradient in combine_routes The autograd backward of combine_routes summed grad * expert_output over H with torch.sum, whose reduction order depends on the batch shape, so the route-weight gradient stopped being batch-invariant from 16 rows up. Add a custom autograd Function that sums H with fixed_order_row_sum, and a test that compares the first 8 rows alone against the same rows inside larger batches. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../operators/qwen3-next-moe-route-combine.md | 8 ++- .../models/qwen3_next/qwen3_next_forward.py | 57 +++++++++++++------ .../qwen3_next/check_qwen3_next_forward.py | 19 +++++++ 3 files changed, 66 insertions(+), 18 deletions(-) diff --git a/docs/operators/qwen3-next-moe-route-combine.md b/docs/operators/qwen3-next-moe-route-combine.md index 06d84e879..38fcf9f9c 100644 --- a/docs/operators/qwen3-next-moe-route-combine.md +++ b/docs/operators/qwen3-next-moe-route-combine.md @@ -58,8 +58,12 @@ different instruction stream), and IEEE Triton FP32. grouped kernel, then visits experts in ascending id. Each expert's `dgate_up` and `ddown` slice is one pinned GEMM written once into a dense buffer; `dx` is accumulated per token in ascending expert order (a token's ten experts are -distinct, so each launch has unique rows). Router-weight gradients flow through -the selected probabilities; the indices are discrete. +distinct, so each launch has unique rows). `combine_routes` has its own backward: +the expert-output gradient is `grad * weight` per route, and the route-weight +gradient sums `grad * expert_output` over H with `fixed_order_row_sum`, because +`torch.sum` picks its reduction order from the batch shape and made that gradient +batch-variant from 16 rows up. Router-weight gradients then flow through the +selected probabilities; the indices are discrete. ## Reuse decision diff --git a/rl_engine/models/qwen3_next/qwen3_next_forward.py b/rl_engine/models/qwen3_next/qwen3_next_forward.py index b5a1e20e0..4109e724d 100644 --- a/rl_engine/models/qwen3_next/qwen3_next_forward.py +++ b/rl_engine/models/qwen3_next/qwen3_next_forward.py @@ -226,23 +226,48 @@ def stable_top10_routes(router_logits): return Routing(selected / fixed_order_row_sum(selected), indices) -def combine_routes(expert_outputs, route_weights): - """Combine [T, 10, H] BF16 outputs by ten ordered FP32 multiply-adds. +class _CombineRoutes(torch.autograd.Function): + """Ten ordered FP32 multiply-adds; route-weight gradients sum H with fixed_order_row_sum.""" - There are no atomic sums. Each route's product and each addition is a - separate eager operation; the single BF16 cast follows the final route. - """ - _cuda_tensors(expert_outputs, route_weights) - if expert_outputs.ndim != 3 or expert_outputs.shape[1] != 10: - raise ValueError("Qwen3-Next expert outputs must have shape [T, 10, H]") - if route_weights.shape != expert_outputs.shape[:2]: - raise ValueError("Route weights must have shape [T, 10]") - if expert_outputs.dtype != torch.bfloat16 or route_weights.dtype != torch.float32: - raise ValueError("Combine requires BF16 expert outputs and FP32 route weights") - output = expert_outputs[:, 0].float() * route_weights[:, 0, None] - for slot in range(1, 10): - output = output + expert_outputs[:, slot].float() * route_weights[:, slot, None] - return output.to(torch.bfloat16) + @staticmethod + def forward(ctx, expert_outputs, route_weights): + _cuda_tensors(expert_outputs, route_weights) + if expert_outputs.ndim != 3 or expert_outputs.shape[1] != 10: + raise ValueError("Qwen3-Next expert outputs must have shape [T, 10, H]") + if route_weights.shape != expert_outputs.shape[:2]: + raise ValueError("Route weights must have shape [T, 10]") + if expert_outputs.dtype != torch.bfloat16 or route_weights.dtype != torch.float32: + raise ValueError("Combine requires BF16 expert outputs and FP32 route weights") + ctx.save_for_backward(expert_outputs, route_weights) + output = expert_outputs[:, 0].float() * route_weights[:, 0, None] + for slot in range(1, 10): + output = output + expert_outputs[:, slot].float() * route_weights[:, slot, None] + return output.to(torch.bfloat16) + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + expert_outputs, route_weights = ctx.saved_tensors + grad_expert = grad_weight = None + grad_f = grad_output.float() + if ctx.needs_input_grad[0]: + grad_expert = torch.zeros_like(expert_outputs) + for slot in range(10): + grad_expert[:, slot] = (grad_f * route_weights[:, slot, None]).to( + expert_outputs.dtype + ) + if ctx.needs_input_grad[1]: + grad_weight = route_weights.new_zeros(route_weights.shape) + for slot in range(10): + grad_weight[:, slot] = fixed_order_row_sum( + grad_f * expert_outputs[:, slot].float() + ).reshape(route_weights.shape[0]) + return grad_expert, grad_weight + + +def combine_routes(expert_outputs, route_weights): + """Combine [T, 10, H] BF16 outputs by ten ordered FP32 multiply-adds.""" + return _CombineRoutes.apply(expert_outputs, route_weights) def _expert_activation(gate_up_rows): diff --git a/tests/models/qwen3_next/check_qwen3_next_forward.py b/tests/models/qwen3_next/check_qwen3_next_forward.py index 29ae44382..7ae46fdb7 100644 --- a/tests/models/qwen3_next/check_qwen3_next_forward.py +++ b/tests/models/qwen3_next/check_qwen3_next_forward.py @@ -103,6 +103,25 @@ def test_combine_has_fixed_route_order_and_differentiable_weights(): ) +def test_combine_gradients_are_batch_invariant(): + gen = torch.Generator(device="cuda").manual_seed(803) + outputs = (torch.randn(256, 10, 2048, generator=gen, device="cuda") * 0.02).bfloat16() + weights = torch.rand(256, 10, generator=gen, device="cuda") + grad_out = (torch.randn(256, 2048, generator=gen, device="cuda") * 0.02).bfloat16() + + def run(rows): + e = outputs[:rows].clone().requires_grad_() + w = weights[:rows].clone().requires_grad_() + out = combine_routes(e, w) + d_e, d_w = torch.autograd.grad(out, [e, w], grad_out[:rows]) + return out.detach(), d_e, d_w + + reference = run(8) + for rows in (9, 16, 32, 64, 256): + for got, want in zip(run(rows), reference): + exact(got[:8], want) + + def test_qwen_moe_actual_tp4_shapes_and_independent_training_forward(): x = random((4, 2048), grad=True) router = random((512, 2048), 802, grad=True)