diff --git a/ci/scripts/run_ws1_gtest.sh b/ci/scripts/run_ws1_gtest.sh index 52caa1c7d..73674a5c6 100755 --- a/ci/scripts/run_ws1_gtest.sh +++ b/ci/scripts/run_ws1_gtest.sh @@ -76,3 +76,18 @@ for cell in required: ) print("[ws1-gtest] C8 gate passed") PY + +echo "[ws1-gtest] Qwen3-Next C3/C4 norms (operator scope)" +QWEN3_NEXT_NORM_MANIFEST=rl_engine/validation/models/qwen3_next_norm_manifest.json +# --hidden is the model hidden size the manifest pins: validate_norm_dimensions requires +# --hidden 2048 --head-dim 128 for both ops. rms_norm_gated does not run at 2048: its inputs and +# weight are built at --head-dim (128, the GDN value-head width), so the same flags give each +# op its checkpoint width. +"$PY" tools/validation/operators/check_forward_invariance.py --manifest "$QWEN3_NEXT_NORM_MANIFEST" \ + --op qwen3_next_rms_norm --candidate cuda --backend-profile cuda_bf16 --hidden 2048 --head-dim 128 +"$PY" tools/validation/operators/check_gradient_invariance.py --manifest "$QWEN3_NEXT_NORM_MANIFEST" \ + --op qwen3_next_rms_norm --candidate cuda --backend-profile cuda_bf16 --hidden 2048 --head-dim 128 +"$PY" tools/validation/operators/check_forward_invariance.py --manifest "$QWEN3_NEXT_NORM_MANIFEST" \ + --op rms_norm_gated --candidate cuda --backend-profile cuda_bf16 --hidden 2048 --head-dim 128 +"$PY" tools/validation/operators/check_gradient_invariance.py --manifest "$QWEN3_NEXT_NORM_MANIFEST" \ + --op rms_norm_gated --candidate cuda --backend-profile cuda_bf16 --hidden 2048 --head-dim 128 diff --git a/csrc/bindings/ops.cpp b/csrc/bindings/ops.cpp index 6fcf90752..d71ea1401 100644 --- a/csrc/bindings/ops.cpp +++ b/csrc/bindings/ops.cpp @@ -3,6 +3,7 @@ #include #include +#include // Fused LogP Declarations torch::Tensor fused_logp_forward(torch::Tensor logits, torch::Tensor token_ids); @@ -274,6 +275,26 @@ void rmsnorm_backward_dx_cuda( torch::Tensor dx, double weight_offset); +void rmsnorm_gated_forward_cuda( + torch::Tensor x, + torch::Tensor weight, + torch::Tensor gate, + torch::Tensor y, + torch::Tensor rstd, + double eps, + double weight_offset, + int64_t activation); + +void rmsnorm_gated_backward_dx_cuda( + torch::Tensor dy, + torch::Tensor x, + torch::Tensor weight, + torch::Tensor gate, + torch::Tensor rstd, + torch::Tensor dx, + double weight_offset, + int64_t activation); + void rmsnorm_backward_partial_dw_cuda( torch::Tensor dy, torch::Tensor x, @@ -298,6 +319,27 @@ static void rmsnorm_check_input(const torch::Tensor& x, const char* name) { TORCH_CHECK(x.is_contiguous(), name, " must be contiguous"); } +static void rmsnorm_check_weight(const torch::Tensor& x, const torch::Tensor& weight) { + TORCH_CHECK(x.dim() == 2 && x.size(1) > 0, "x must be 2D with positive hidden size"); + // The kernels address rows as int row * H. + TORCH_CHECK(x.numel() <= std::numeric_limits::max(), + "x must have at most INT_MAX elements (32-bit row offsets)"); + TORCH_CHECK(x.scalar_type() == torch::kFloat32 || x.scalar_type() == torch::kFloat16 || + x.scalar_type() == torch::kBFloat16, "x must be float32, float16 or bfloat16"); + TORCH_CHECK(weight.dim() == 1 && weight.size(0) == x.size(1), "weight must be [H]"); + TORCH_CHECK(weight.device() == x.device(), "weight must be on the same device as x"); + TORCH_CHECK(weight.scalar_type() == x.scalar_type() || weight.scalar_type() == torch::kFloat32, + "weight must have x dtype or float32"); +} + +static void rmsnorm_check_backward(const torch::Tensor& dy, const torch::Tensor& x, + const torch::Tensor& rstd) { + TORCH_CHECK(dy.device() == x.device() && rstd.device() == x.device(), + "dy and rstd must be on the same device as x"); + TORCH_CHECK(dy.scalar_type() == x.scalar_type(), "dy must have the same dtype as x"); + TORCH_CHECK(rstd.scalar_type() == torch::kFloat32, "rstd must be float32"); +} + std::vector rmsnorm_forward( torch::Tensor x, torch::Tensor weight, @@ -306,10 +348,7 @@ std::vector rmsnorm_forward( { rmsnorm_check_input(x, "x"); rmsnorm_check_input(weight, "weight"); - - TORCH_CHECK(x.dim() == 2, "x must be 2D [T, H]"); - TORCH_CHECK(weight.dim() == 1, "weight must be 1D [H]"); - TORCH_CHECK(x.size(1) == weight.size(0), "x.size(1) must equal weight.size(0)"); + rmsnorm_check_weight(x, weight); auto T = x.size(0); auto y = torch::empty_like(x); @@ -330,11 +369,11 @@ torch::Tensor rmsnorm_backward_dx( rmsnorm_check_input(dy, "dy"); rmsnorm_check_input(x, "x"); rmsnorm_check_input(weight, "weight"); + rmsnorm_check_weight(x, weight); rmsnorm_check_input(rstd, "rstd"); + rmsnorm_check_backward(dy, x, rstd); TORCH_CHECK(dy.sizes() == x.sizes(), "dy and x must have same shape"); - TORCH_CHECK(x.dim() == 2, "x must be 2D [T, H]"); - TORCH_CHECK(weight.dim() == 1, "weight must be 1D [H]"); TORCH_CHECK(rstd.dim() == 1, "rstd must be 1D [T]"); TORCH_CHECK(rstd.size(0) == x.size(0), "rstd.size(0) must equal x.size(0)"); @@ -345,6 +384,72 @@ torch::Tensor rmsnorm_backward_dx( return dx; } +static void rmsnorm_gated_check( + const torch::Tensor& x, + const torch::Tensor& weight, + const torch::Tensor& gate, + int64_t activation) +{ + rmsnorm_check_input(x, "x"); + rmsnorm_check_input(weight, "weight"); + rmsnorm_check_weight(x, weight); + rmsnorm_check_input(gate, "gate"); + TORCH_CHECK(gate.device() == x.device(), "gate must be on the same device as x"); + TORCH_CHECK(gate.sizes() == x.sizes(), "gate must have the same shape as x"); + TORCH_CHECK( + gate.scalar_type() == x.scalar_type(), + "gate must have the same dtype as x"); + // 0 = silu/swish, 1 = sigmoid. Anything else fails closed rather than + // silently computing a different activation (RFC #428 section 6, item 7). + TORCH_CHECK( + activation == 0 || activation == 1, + "activation must be 0 (silu) or 1 (sigmoid), got ", activation); +} + +std::vector rmsnorm_gated_forward( + torch::Tensor x, + torch::Tensor weight, + torch::Tensor gate, + double eps, + double weight_offset, + int64_t activation) +{ + rmsnorm_gated_check(x, weight, gate, activation); + + auto T = x.size(0); + auto y = torch::empty_like(x); + auto rstd = torch::empty({T}, x.options().dtype(torch::kFloat32)); + + rmsnorm_gated_forward_cuda(x, weight, gate, y, rstd, eps, weight_offset, activation); + + return {y, rstd}; +} + +torch::Tensor rmsnorm_gated_backward_dx( + torch::Tensor dy, + torch::Tensor x, + torch::Tensor weight, + torch::Tensor gate, + torch::Tensor rstd, + double weight_offset, + int64_t activation) +{ + rmsnorm_gated_check(x, weight, gate, activation); + rmsnorm_check_input(dy, "dy"); + rmsnorm_check_input(rstd, "rstd"); + rmsnorm_check_backward(dy, x, rstd); + + TORCH_CHECK(dy.sizes() == x.sizes(), "dy must have the same shape as x"); + TORCH_CHECK(rstd.dim() == 1 && rstd.size(0) == x.size(0), "rstd must be [T]"); + + auto dx = torch::empty_like(x); + + rmsnorm_gated_backward_dx_cuda( + dy, x, weight, gate, rstd, dx, weight_offset, activation); + + return dx; +} + torch::Tensor rmsnorm_backward_dw( torch::Tensor dy, torch::Tensor x, @@ -354,6 +459,7 @@ torch::Tensor rmsnorm_backward_dw( rmsnorm_check_input(dy, "dy"); rmsnorm_check_input(x, "x"); rmsnorm_check_input(rstd, "rstd"); + rmsnorm_check_backward(dy, x, rstd); rmsnorm_check_input(mask, "mask"); TORCH_CHECK(dy.sizes() == x.sizes(), "dy and x must have same shape"); @@ -361,8 +467,12 @@ torch::Tensor rmsnorm_backward_dw( TORCH_CHECK(rstd.dim() == 1, "rstd must be 1D [T]"); TORCH_CHECK(mask.dim() == 1, "mask must be 1D [T]"); TORCH_CHECK(mask.scalar_type() == torch::kBool, "mask must be bool"); + TORCH_CHECK(mask.device() == x.device(), "mask must be on the same device as x"); + TORCH_CHECK(x.size(1) > 0, "hidden size must be positive"); TORCH_CHECK(rstd.size(0) == x.size(0), "rstd.size(0) must equal x.size(0)"); TORCH_CHECK(mask.size(0) == x.size(0), "mask.size(0) must equal x.size(0)"); + TORCH_CHECK(x.numel() <= std::numeric_limits::max(), + "x must have at most INT_MAX elements (32-bit row offsets)"); auto T = x.size(0); auto H = x.size(1); @@ -731,6 +841,15 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { py::arg("dy"), py::arg("x"), py::arg("weight"), py::arg("rstd"), py::arg("weight_offset") = 0.0); m.def("rmsnorm_backward_dw", &rmsnorm_backward_dw, "Deterministic RMSNorm backward dweight CUDA"); + m.def("rmsnorm_gated_forward", &rmsnorm_gated_forward, + "Batch-invariant gated RMSNorm forward CUDA", + py::arg("x"), py::arg("weight"), py::arg("gate"), py::arg("eps"), + py::arg("weight_offset") = 0.0, py::arg("activation") = 0); + m.def("rmsnorm_gated_backward_dx", &rmsnorm_gated_backward_dx, + "Batch-invariant gated RMSNorm backward dx CUDA", + py::arg("dy"), py::arg("x"), py::arg("weight"), py::arg("gate"), + py::arg("rstd"), py::arg("weight_offset") = 0.0, + py::arg("activation") = 0); #if !defined(USE_ROCM) m.def( "reduce_rows_fp32_left_fold", diff --git a/csrc/cuda/norm/rmsnorm.cu b/csrc/cuda/norm/rmsnorm.cu index 595131522..e256b7261 100644 --- a/csrc/cuda/norm/rmsnorm.cu +++ b/csrc/cuda/norm/rmsnorm.cu @@ -198,6 +198,126 @@ __global__ void rmsnorm_bwd_dx_kernel( } +// --------------------------------------------------------------------------- +// Gated RMSNorm (Qwen3-Next GDN block). +// +// y = x * rstd * (weight_offset + weight) * act(gate) +// +// The gate activation and the weight multiply are both evaluated in fp32 and +// there is exactly one cast, at the store. This mirrors vLLM's RMSNormGated +// with norm_before_gate=True, which is how the GDN block constructs it. +// +// ACT selects the gate activation: 0 = silu/swish, 1 = sigmoid. expf (not the +// __expf intrinsic) is used deliberately -- the fast intrinsic trades accuracy +// for speed and would put the result further from the fp32 reference. +// --------------------------------------------------------------------------- + +template +__device__ __forceinline__ float gate_activation(float z) { + const float sigma = 1.0f / (1.0f + expf(-z)); + return (ACT == 0) ? z * sigma : sigma; +} + + +template +__global__ void rmsnorm_gated_fwd_kernel( + const scalar_t* __restrict__ x, + const weight_t* __restrict__ weight, + const scalar_t* __restrict__ gate, + scalar_t* __restrict__ y, + float* __restrict__ rstd, + int T, + int H, + float eps, + float weight_offset +) { + int row = blockIdx.x; + int tid = threadIdx.x; + + const scalar_t* x_row = x + row * H; + const scalar_t* gate_row = gate + row * H; + scalar_t* y_row = y + row * H; + + float local_sum = 0.0f; + + // The statistic is over x only; the gate never enters the reduction, so + // rstd here is bit-identical to the ungated kernel's for the same x. + for (int col = tid; col < H; col += blockDim.x) { + float xv = load_as_float(x_row + col); + local_sum += xv * xv; + } + + float sum = block_reduce_sum(local_sum); + + float row_rstd = rsqrtf(sum / static_cast(H) + eps); + + if (tid == 0) { + rstd[row] = row_rstd; + } + + __syncthreads(); + + for (int col = tid; col < H; col += blockDim.x) { + float xv = load_as_float(x_row + col); + float wv = load_as_float(weight + col); + if (weight_offset != 0.0f) wv += weight_offset; + float zv = load_as_float(gate_row + col); + float out = xv * row_rstd * wv * gate_activation(zv); + store_from_float(y_row + col, out); + } +} + + +template +__global__ void rmsnorm_gated_bwd_dx_kernel( + const scalar_t* __restrict__ dy, + const scalar_t* __restrict__ x, + const weight_t* __restrict__ weight, + const scalar_t* __restrict__ gate, + const float* __restrict__ rstd, + scalar_t* __restrict__ dx, + int T, + int H, + float weight_offset +) { + int row = blockIdx.x; + int tid = threadIdx.x; + + const scalar_t* dy_row = dy + row * H; + const scalar_t* x_row = x + row * H; + const scalar_t* gate_row = gate + row * H; + scalar_t* dx_row = dx + row * H; + + float local_dot = 0.0f; + + // Identical to the ungated dx, with the per-column scale (w + offset) + // replaced by (w + offset) * act(gate): the gate is a constant wrt x. + for (int col = tid; col < H; col += blockDim.x) { + float dyv = load_as_float(dy_row + col); + float xv = load_as_float(x_row + col); + float wv = load_as_float(weight + col); + if (weight_offset != 0.0f) wv += weight_offset; + float zv = load_as_float(gate_row + col); + local_dot += dyv * (wv * gate_activation(zv)) * xv; + } + + float dot = block_reduce_sum(local_dot); + + float r = rstd[row]; + float coeff = dot * r * r * r / static_cast(H); + + for (int col = tid; col < H; col += blockDim.x) { + float dyv = load_as_float(dy_row + col); + float xv = load_as_float(x_row + col); + float wv = load_as_float(weight + col); + if (weight_offset != 0.0f) wv += weight_offset; + float zv = load_as_float(gate_row + col); + + float out = r * dyv * (wv * gate_activation(zv)) - xv * coeff; + store_from_float(dx_row + col, out); + } +} + template __global__ void rmsnorm_partial_dw_kernel( const scalar_t* __restrict__ dy, @@ -265,6 +385,7 @@ void rmsnorm_forward_cuda( // device, which need not be x's. const at::cuda::OptionalCUDAGuard device_guard(device_of(x)); int T = x.size(0); + if (T == 0) return; int H = x.size(1); int threads = choose_threads(H); size_t smem = threads * sizeof(float); @@ -287,6 +408,7 @@ void rmsnorm_forward_cuda( ); }); }); + C10_CUDA_KERNEL_LAUNCH_CHECK(); } @@ -300,6 +422,7 @@ void rmsnorm_backward_dx_cuda( ) { const at::cuda::OptionalCUDAGuard device_guard(device_of(x)); int T = x.size(0); + if (T == 0) return; int H = x.size(1); int threads = choose_threads(H); size_t smem = threads * sizeof(float); @@ -322,9 +445,89 @@ void rmsnorm_backward_dx_cuda( ); }); }); + C10_CUDA_KERNEL_LAUNCH_CHECK(); } +void rmsnorm_gated_forward_cuda( + torch::Tensor x, + torch::Tensor weight, + torch::Tensor gate, + torch::Tensor y, + torch::Tensor rstd, + double eps, + double weight_offset, + int64_t activation +) { + const at::cuda::OptionalCUDAGuard device_guard(device_of(x)); + int T = x.size(0); + if (T == 0) return; + int H = x.size(1); + int threads = choose_threads(H); + size_t smem = threads * sizeof(float); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND2(at::kHalf, at::kBFloat16, x.scalar_type(), "rmsnorm_gated_forward_cuda", [&] { + using x_t = scalar_t; + AT_DISPATCH_FLOATING_TYPES_AND2(at::kHalf, at::kBFloat16, weight.scalar_type(), "rmsnorm_gated_forward_weight_cuda", [&] { + using w_t = scalar_t; + if (activation == 0) { + rmsnorm_gated_fwd_kernel<<>>( + x.data_ptr(), weight.data_ptr(), gate.data_ptr(), + y.data_ptr(), rstd.data_ptr(), + T, H, static_cast(eps), static_cast(weight_offset)); + } else { + rmsnorm_gated_fwd_kernel<<>>( + x.data_ptr(), weight.data_ptr(), gate.data_ptr(), + y.data_ptr(), rstd.data_ptr(), + T, H, static_cast(eps), static_cast(weight_offset)); + } + }); + }); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + + +void rmsnorm_gated_backward_dx_cuda( + torch::Tensor dy, + torch::Tensor x, + torch::Tensor weight, + torch::Tensor gate, + torch::Tensor rstd, + torch::Tensor dx, + double weight_offset, + int64_t activation +) { + const at::cuda::OptionalCUDAGuard device_guard(device_of(x)); + int T = x.size(0); + if (T == 0) return; + int H = x.size(1); + int threads = choose_threads(H); + size_t smem = threads * sizeof(float); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND2(at::kHalf, at::kBFloat16, x.scalar_type(), "rmsnorm_gated_backward_dx_cuda", [&] { + using x_t = scalar_t; + AT_DISPATCH_FLOATING_TYPES_AND2(at::kHalf, at::kBFloat16, weight.scalar_type(), "rmsnorm_gated_backward_dx_weight_cuda", [&] { + using w_t = scalar_t; + if (activation == 0) { + rmsnorm_gated_bwd_dx_kernel<<>>( + dy.data_ptr(), x.data_ptr(), weight.data_ptr(), + gate.data_ptr(), rstd.data_ptr(), dx.data_ptr(), + T, H, static_cast(weight_offset)); + } else { + rmsnorm_gated_bwd_dx_kernel<<>>( + dy.data_ptr(), x.data_ptr(), weight.data_ptr(), + gate.data_ptr(), rstd.data_ptr(), dx.data_ptr(), + T, H, static_cast(weight_offset)); + } + }); + }); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + void rmsnorm_backward_partial_dw_cuda( torch::Tensor dy, torch::Tensor x, @@ -334,6 +537,7 @@ void rmsnorm_backward_partial_dw_cuda( ) { const at::cuda::OptionalCUDAGuard device_guard(device_of(x)); int T = x.size(0); + if (T == 0) return; int H = x.size(1); int chunks = (T + RMSNORM_DW_ROWS_PER_CHUNK - 1) / RMSNORM_DW_ROWS_PER_CHUNK; @@ -353,6 +557,7 @@ void rmsnorm_backward_partial_dw_cuda( H ); }); + C10_CUDA_KERNEL_LAUNCH_CHECK(); } @@ -375,6 +580,7 @@ void rmsnorm_backward_reduce_dw_cuda( chunks, H ); + C10_CUDA_KERNEL_LAUNCH_CHECK(); } #if !defined(USE_ROCM) diff --git a/docs/.nav.yml b/docs/.nav.yml index ebb5476de..4e04fd061 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -30,6 +30,7 @@ nav: - operators/det-gemm.md - operators/embedding.md - operators/qwen3-next-rms-norm.md + - operators/qwen3-next-rms-norm-gated.md - Developer Guide: - contributing/README.md - Contributor Guide: contributing/contributor-guide.md diff --git a/docs/operators/README.md b/docs/operators/README.md index 0d955ed18..0b93b3d90 100644 --- a/docs/operators/README.md +++ b/docs/operators/README.md @@ -33,4 +33,5 @@ Every operator page should include: - [Sampling](sampling.md) - [Token Embedding](embedding.md) - [Qwen3-Next RMSNorm (zero-centred)](qwen3-next-rms-norm.md) +- [Qwen3-Next Gated RMSNorm](qwen3-next-rms-norm-gated.md) - [Operator Doc Template](../contributing/operator-doc-template.md) diff --git a/docs/operators/qwen3-next-rms-norm-gated.md b/docs/operators/qwen3-next-rms-norm-gated.md new file mode 100644 index 000000000..deb3903c7 --- /dev/null +++ b/docs/operators/qwen3-next-rms-norm-gated.md @@ -0,0 +1,208 @@ +# Qwen3-Next Gated RMSNorm + +## Summary + +The norm inside Qwen3-Next's Gated DeltaNet block: + +``` +out = x * rstd * weight * silu(gate) +``` + +It is a different operator from the decoder norm, not a variant of it: the weight is +plain rather than zero-centred (upstream initializes it to ones), it takes a second +input tensor, and — critically — `transformers` and vLLM disagree about where the +weight multiply happens. See [Qwen3-Next RMSNorm](qwen3-next-rms-norm.md) for the +decoder-norm convention. + +Added for RFC #428 C1 on the Qwen3-Next rollout-vs-replay path. + +## Entry Point + +```python +from rl_engine.reference.norm.qwen3_next_rms_norm import ( + Qwen3NextRMSNormGatedOp, # strict / vLLM convention + Qwen3NextRMSNormGatedHFOp, # transformers convention, kept as a witness +) +from rl_engine.backends.cuda.norm.rmsnorm import ( + Qwen3NextRMSNormGatedCudaOp, + rmsnorm_gated_cuda, +) + +y = Qwen3NextRMSNormGatedCudaOp().forward(x, weight, gate, eps=1e-6) +y = Qwen3NextRMSNormGatedCudaOp(activation="sigmoid").forward(x, weight, gate, eps=1e-6) +y = rmsnorm_gated_cuda(x, weight, gate, eps=1e-6, activation="sigmoid") +``` + +## Backends + +| Backend | Wrapper | Native symbol | Status | +| --- | --- | --- | --- | +| CUDA | `Qwen3NextRMSNormGatedCudaOp` | `rl_engine._C.rmsnorm_gated_forward` | Supported | +| ROCm | — | — | Not implemented | +| PyTorch fallback | `Qwen3NextRMSNormGatedOp` | — | Supported (WS1 gold) | + +The CUDA op is deliberately **not** a subclass of `RMSNormCudaOp`: it takes an extra +required tensor, so it cannot stand in for one. + +## Tensor Contract + +| Argument | Shape | Dtype | Requirements | +| --- | --- | --- | --- | +| `x` | `[..., H]` | fp32 / bf16 / fp16 | H > 0; wrapper makes contiguous copies | +| `weight` | `[H]` | matches `x` or fp32 | plain, NOT zero-centred | +| `gate` | same as `x` | same as `x` | shape, dtype and device must match before flattening | +| `eps` | scalar | float | `1e-6` for Qwen3-Next | +| `activation` | — | `"silu"`/`"swish"`/`"sigmoid"` | constructor argument; `"swish"` is an alias for `"silu"`, as in vLLM; anything else is rejected | + +All tensors must share a device. Low-level CUDA bindings require contiguous +2-D inputs, and validate backward gradient dtype and FP32 statistics. Empty +batches are supported by the bindings. + +Only `norm_before_gate=True` and `group_size=None` are implemented — the +configuration vLLM's GDN block constructs. The op has no parameter for either, so +other configurations are not implemented rather than rejected at runtime. + +## Dispatch Behavior + +Registered as `rms_norm_gated`. CUDA prefers the kernel; every other platform +resolves to the PyTorch reference. `__init__` validates the compiled symbols, so on a +build without the extension the registry falls through instead of returning an op +that raises at call time. + +## Accuracy + +Claim levels: + +- **L1** for `Qwen3NextRMSNormGatedCudaOp`: prefix slices in + `tests/models/qwen3_next/test_qwen3_next_norm.py`, and batch size, chunking, padding and permutation + in the C3/C4 gates (`ci/scripts/run_ws1_gtest.sh`). +- **L0** is not separately tested for the CUDA op; no test repeats it. +- L2 is not claimed. + +Accuracy tests resolve their tolerances from `tolerance_contract.json` +(`forward_accuracy`, `reduction` × dtype). The bounds in +`tests/models/qwen3_next/check_qwen3_next_norm_providers.py` are provider-gap bounds, not contract +thresholds. + +`transformers` casts the normalized value back to the input dtype *before* the weight +multiply; vLLM keeps it in fp32. `Qwen3NextRMSNormGatedOp` follows vLLM and +`Qwen3NextRMSNormGatedHFOp` keeps the transformers convention as a witness. +`test_gated_conventions_diverge_in_low_precision` asserts that the two differ in bf16 +(by more than `1e-3`) and agree bitwise in fp32. One earlier measurement, with no +committed script, found them differing on 35% of elements with `max|diff| = 6.25e-2` +(one seed, B200, bf16, `head_v_dim=128`, 512 rows, direct eager calls). That is a +one-off observation, not an assertion. Which convention the strict profile should use +is an open question for RFC #428. + +"Bitwise equal to vLLM" is undefined until a provider is named. Over 40 seeds (bf16, +`head_v_dim=128`, 512 rows, B200), vLLM's eager `forward_native` and `forward_cuda` +disagreed on 21, worst `1.56e-2`; in fp32 they stay within the asserted `1e-5`. These +figures come from `tests/models/qwen3_next/check_qwen3_next_norm_providers.py`, which imports real vLLM +and must be run explicitly. The PyTorch reference reproduces the convention, not +vLLM's reduction tree. + +`rstd` is bitwise identical to the ungated kernel's for the same `x`, asserted over +fp32/fp16/bf16 × H ∈ {128, 2048, 5120} × {silu, sigmoid} × offset ∈ {0, 1}, so a gate +leaking into the statistic would fail. + +Backward is assembled from deterministic pieces: `dx` from a row-local kernel, +`dweight` from fp32 row contributions reduced by the ascending-row left fold, and +`dgate` elementwise in fp32 with no reduction. + +## Performance Notes + +Reuses the existing `block_reduce_sum` / `choose_threads(H)` reduction, so the gate +costs one extra load and one fp32 multiply per element. + +```bash +python tools/validation/operators/check_operator.py --op rms_norm_gated --candidate cuda \ + --device cuda --dtype bf16 --check-grad +``` + +## Evidence + +![gated RMSNorm vs existing implementations on B200](https://raw.githubusercontent.com/fusheng-ji/RL-Kernel/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr468/docs/usage/evidence/qwen3-next-rms-norm-gated-b200/figure-gated_rmsnorm.png) + +[`report.json`](https://github.com/fusheng-ji/RL-Kernel/blob/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr468/docs/usage/evidence/qwen3-next-rms-norm-gated-b200/report.json) was written by +`tools/validation/models/qwen3_next_norm_evidence.py` from a clean tree at `ff271d5`, on an otherwise idle +B200 (torch 2.13.0+cu130, transformers 5.17.0, vLLM 0.30.0). Head dim 128, BF16; the FP64 +golden uses vLLM's convention. The same report re-measures the zero-centred op +([figure](https://raw.githubusercontent.com/fusheng-ji/RL-Kernel/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr468/docs/usage/evidence/qwen3-next-rms-norm-gated-b200/figure-zero_centred_rmsnorm.png)). + +| | rl-kernel CUDA | PyTorch reference | transformers (cast-first) | vLLM `RMSNormGated`* | +|---|---|---|---|---| +| rows differing alone vs in a batch, forward / `dx`,`dgate` (of 768) | **0 / 0** | 0 / 0 | 0 / 0 | 0 / n/a | +| forward elements correctly rounded vs the golden | **99.999%** | 99.999% | 65.6% | 99.999% | +| forward, 262144 rows | 235 µs | 774 µs | 697 µs | **81 µs** | +| backward, 262144 rows | 114 ms | **1.4 ms** | 1.5 ms | n/a | + +\* forward only, no backward. + +- **Every implementation is row-invariant** for this op. +- **transformers computes a different function:** it casts to BF16 before the weight + multiply, so 34% of its forward elements differ from vLLM's convention, and its gradients + are about twice as far from the golden. +- **The backward is about 75× slower than transformers at 262144 rows**, for the same reason + as the zero-centred op: `dweight` (128 columns here) is folded over the rows in ascending + order, one thread per column, so that it meets the gradient-invariance contract's + singleton-aggregate check bitwise. + +## Existing implementations: measured batch invariance, accuracy, gates + +![rms_norm_gated vs existing implementations](https://raw.githubusercontent.com/fusheng-ji/RL-Kernel/692c4082f9fe6ca0f0dba37d277a4422dd689ba2/pr468/docs/usage/evidence/qwen3-next-norm-reuse-b200/rms_norm_gated.png) + +| Implementation | Batch-invariance checks | Forward correctly rounded | Worst grad err | Forward / fwd+bwd | C3/C4 gates | +|---|---|---|---|---|---| +| rl-kernel Qwen3NextRMSNormGatedCudaOp | yes (sampled) | 99.9989% | 2.4e-03 | 235 / 115293 µs | pass | +| rl-kernel PyTorch reference | yes (sampled) | 99.9990% | 2.4e-03 | 772 / 1991 µs | — | +| transformers 5.17.0 Qwen3NextRMSNormGated | yes (sampled) | 65.4748% | 5.6e-03 | 695 / 2093 µs | C4 **fail** | +| FLA 0.5.2 layernorm_gated.rmsnorm_fn | yes (sampled) | 99.9987% | 2.4e-03 | 177 / 882 µs | C4 **fail** | +| FLA 0.5.2 fused_norm_gate.rms_norm_gated | **no** (247 sub-batches) | 99.9987% | 2.4e-03 | 93 / 865 µs | C4 **fail** | +| vLLM 0.30.0 RMSNormGated.forward_cuda | **no** (3 rows, 3 sub-batches) | 99.9988% | — | 87 / — µs | — | + +Batch invariance is bitwise and covers three checks: every row computed alone vs inside full +batches of three sizes; sampled small sub-batches and exhaustive larger sub-batches +of the full workload-size batch; and a batch-size sweep over probe rows. In these +historical reports, sub-batch sizes 1 and 7 visit only 512 starts per seed, so a +"yes (sampled)" does not establish every-row coverage at those sizes. The JSON +records the actual stride and rows checked per size. The current script instead +partitions the full batch at every advertised sub-batch size, covering every row +including a final partial batch. These historical measurements have not been rerun. +A "no" counts the rows, sub-batches or sweep cases that differed. Accuracy is against FP64 at the workload size; latency is the median on an otherwise +idle B200. The C3/C4 column runs this repository's own gate scripts unchanged, with the CUDA +candidate replaced by a subclass of this op whose forward and backward call the other library. +The subclass drops this op's FP32 `dweight` row contributions +(`parameter_vjp_contributions_fp32`), so every C4 `dweight` check judges the `dweight` that +library's own backward returns. C3 passes for every implementation. Each existing +implementation fails C4 on `dweight` in the chunked-prefill and singleton-aggregate +configurations (max |diff| up to 5.7e-6): both split the batch into several calls and the harness +adds the per-call `dweight`, which each library returns in BF16; the split sum is not bitwise +equal to the one-call `dweight`. This op passes because it declares FP32 per-row `dweight` +contributions that the harness folds in a fixed order. An earlier run reported C4 passes for +these libraries: the swapped op still inherited this op's row contributions, so those checks +measured this op's `dweight`, not the library's. The gates were rerun on a B200 from a clean +tree at `339bc3ff` ([`rms_norm_gated_gates.json`](https://github.com/fusheng-ji/RL-Kernel/blob/692c4082f9fe6ca0f0dba37d277a4422dd689ba2/pr468/docs/usage/evidence/qwen3-next-norm-reuse-b200/rms_norm_gated_gates.json), +`--checks gates`); the other columns were not rerun. +[`rms_norm_gated.json`](https://github.com/fusheng-ji/RL-Kernel/blob/692c4082f9fe6ca0f0dba37d277a4422dd689ba2/pr468/docs/usage/evidence/qwen3-next-norm-reuse-b200/rms_norm_gated.json), with its `contract_gates` taken from that +rerun, was originally written from a clean tree at `abb56c2` by + +```bash +python tools/validation/models/qwen3_next_norm_reuse_check.py --op rms_norm_gated \ + --out docs/usage/evidence/qwen3-next-norm-reuse-b200/rms_norm_gated.json [--megatron-src ] +``` + +## Tests + +```bash +python -m pytest tests/models/qwen3_next/test_qwen3_next_norm.py -v +# imports real vLLM, so it is not collected by default: +python -m pytest tests/models/qwen3_next/check_qwen3_next_norm_providers.py -v +``` + +## Known Limitations + +- CUDA only; no ROCm, Ascend or Triton backend. +- `norm_before_gate=False` and grouped norms are not implemented. +- Not bitwise against either vLLM path (see Accuracy). +- Measured on sm_100 (B200); RFC #428 §2.2 forbids carrying the claim across + H100/H200/B100/B200. diff --git a/docs/operators/qwen3-next-rms-norm.md b/docs/operators/qwen3-next-rms-norm.md index a4bd85df7..18e9c04ef 100644 --- a/docs/operators/qwen3-next-rms-norm.md +++ b/docs/operators/qwen3-next-rms-norm.md @@ -9,8 +9,8 @@ offset away and silently breaks any bitwise claim. This operator exists for RFC #428 C1 (Embedding / RMSNorm / residual / final norm exactness) on the Qwen3-Next rollout-vs-replay path. The Gated DeltaNet block uses -a *different* weight convention and a different cast order; it is a separate -operator. +a *different* weight convention and a different cast order; see +[Gated RMSNorm](qwen3-next-rms-norm-gated.md). Upstream references: `transformers` `Qwen3NextRMSNorm`, and vLLM's `GemmaRMSNorm`, which `vllm/model_executor/models/qwen3_next.py` aliases as `Qwen3NextRMSNorm`. @@ -49,11 +49,11 @@ y = rmsnorm_cuda(x, weight, eps=1e-6, weight_offset=1.0) ## Dispatch Behavior -Not registered on this branch: there is no `qwen3_next_rms_norm` gtest spec and no -registry entry yet. Both arrive with the gated-norm PR. Until then, construct the -ops directly as in "Entry Point". The CUDA op validates the compiled symbols in -`__init__`, so on a build without the extension construction raises instead of -handing out an op that fails at call time. +Registered as the `qwen3_next_rms_norm` gtest operator. On CUDA the registry +prefers `Qwen3NextRMSNormCudaOp`; every other platform resolves to the PyTorch +reference. The CUDA op validates the compiled symbols in `__init__`, so on a build +without the extension construction raises and the registry falls through to the +reference rather than handing out an op that fails at call time. ## Accuracy @@ -126,31 +126,37 @@ BF16. gradient-invariance contract's singleton-aggregate check). That serial fold over rows is the cost; a faster tree fold would not pass that check. -## Existing implementations: batch invariance of every row, accuracy, gates +## Existing implementations: measured batch invariance, accuracy, gates -![qwen3_next_rms_norm vs existing implementations](https://raw.githubusercontent.com/fusheng-ji/RL-Kernel/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr467/docs/usage/evidence/qwen3-next-norm-reuse-b200/qwen3_next_rms_norm.png) +![qwen3_next_rms_norm vs existing implementations](https://raw.githubusercontent.com/fusheng-ji/RL-Kernel/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr468/docs/usage/evidence/qwen3-next-norm-reuse-b200/qwen3_next_rms_norm.png) -| Implementation | Batch-invariant | Forward correctly rounded | Worst grad err | Forward / fwd+bwd | C3/C4 gates | +| Implementation | Batch-invariance checks | Forward correctly rounded | Worst grad err | Forward / fwd+bwd | C3/C4 gates | |---|---|---|---|---|---| -| rl-kernel Qwen3NextRMSNormCudaOp | yes | 99.9993% | 2.4e-03 | 426 / 31485 µs | not on this branch | +| rl-kernel Qwen3NextRMSNormCudaOp | yes (sampled) | 99.9993% | 2.4e-03 | 426 / 31485 µs | see gate table below | | rl-kernel PyTorch reference | **no** (49 rows, 94 sub-batches) | 99.9993% | 2.4e-03 | 2146 / 5024 µs | — | -| transformers 5.17.0 Qwen3NextRMSNorm | **no** (90 rows, 142 sub-batches) | 99.9992% | 2.4e-03 | 1449 / 4468 µs | not on this branch | -| Liger 0.8.4 RMSNorm, offset 1, gemma | yes | 99.9993% | 2.4e-03 | 133 / 974 µs | not on this branch | -| FLA 0.5.2 rms_norm, weight passed as 1 + w | yes | 73.0161% | 5.5e-03 | 147 / 1051 µs | not on this branch | +| transformers 5.17.0 Qwen3NextRMSNorm | **no** (90 rows, 142 sub-batches) | 99.9992% | 2.4e-03 | 1449 / 4468 µs | see gate table below | +| Liger 0.8.4 RMSNorm, offset 1, gemma | yes (sampled) | 99.9993% | 2.4e-03 | 133 / 974 µs | see gate table below | +| FLA 0.5.2 rms_norm, weight passed as 1 + w | yes (sampled) | 73.0161% | 5.5e-03 | 147 / 1051 µs | see gate table below | | TE 2.20.2 RMSNorm(zero_centered_gamma) | **no** (175 rows, 245 sub-batches) | 99.9992% | 2.4e-03 | 156 / 779 µs | — | -| FlashInfer 0.6.18.post1 gemma_rmsnorm | yes | 99.9992% | — | 91 / — µs | — | +| FlashInfer 0.6.18.post1 gemma_rmsnorm | yes (sampled) | 99.9992% | — | 91 / — µs | — | | vLLM 0.30.0 GemmaRMSNorm.forward_cuda | **no** (16 rows, 26 sub-batches) | 99.9992% | — | 1454 / — µs | — | Batch invariance is bitwise and covers three checks: every row computed alone vs inside full -batches of three sizes; the full workload-size batch vs sub-batches that together cover every -row; and a dense batch-size sweep. A "no" counts the rows, sub-batches or sweep cases that -differed. Accuracy is against FP64 at the workload size; latency is the median on an otherwise +batches of three sizes; sampled small sub-batches and exhaustive larger sub-batches +of the full workload-size batch; and a batch-size sweep over probe rows. In these +historical reports, sub-batch sizes 1 and 7 visit only 512 starts per seed, so a +"yes (sampled)" does not establish every-row coverage at those sizes. The JSON +records the actual stride and rows checked per size. The current script instead +partitions the full batch at every advertised sub-batch size, covering every row +including a final partial batch. These historical measurements have not been rerun. +A "no" counts the rows, sub-batches or sweep cases that differed. Accuracy is against FP64 at the workload size; latency is the median on an otherwise idle B200. The C3/C4 column runs this repository's own gate scripts unchanged, with the CUDA candidate replaced by a subclass of this op whose forward and backward call the other library. -The subclass keeps this op's FP32 `dweight` row contributions, so singleton-aggregate compares -like with like. The gate scripts for this op arrive with #468 (`qwen3_next_norm_manifest.json`); its gate results are in #468's copy of this page. [`qwen3_next_rms_norm.json`](https://github.com/fusheng-ji/RL-Kernel/blob/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr467/docs/usage/evidence/qwen3-next-norm-reuse-b200/qwen3_next_rms_norm.json) +The subclass drops this op's FP32 `dweight` row contributions +(`parameter_vjp_contributions_fp32`), so every C4 `dweight` check judges the `dweight` that +library's own backward returns. The C3/C4 results for this op are in the gate table below. [`qwen3_next_rms_norm.json`](https://github.com/fusheng-ji/RL-Kernel/blob/6d6193d94a83266d5f5f62bb9d90fb3215716ee3/pr468/docs/usage/evidence/qwen3-next-norm-reuse-b200/qwen3_next_rms_norm.json) was originally written from a clean tree at `a66493c` by the command below. The -Megatron `52fbcbc` result has been excluded from the report, table and figure: +Megatron `52fbcbc` result has been excluded from the reports, tables and figure: that revision computes `weight_eff` but uses the original `weight` in its forward output, so it does not implement the zero-centred operation despite accepting the flag. The remaining measurements are unchanged; the figure was regenerated from @@ -164,20 +170,39 @@ python tools/validation/models/qwen3_next_norm_reuse_check.py --op qwen3_next_rm --out docs/usage/evidence/qwen3-next-norm-reuse-b200/qwen3_next_rms_norm.json [--megatron-src ] ``` + +The gate scripts support this op on this branch. Run unchanged on a B200 from a clean tree at `339bc3ff`, with the CUDA candidate swapped for each implementation ([`qwen3_next_rms_norm_gates.json`](https://github.com/fusheng-ji/RL-Kernel/blob/692c4082f9fe6ca0f0dba37d277a4422dd689ba2/pr468/docs/usage/evidence/qwen3-next-norm-reuse-b200/qwen3_next_rms_norm_gates.json), `tools/validation/models/qwen3_next_norm_reuse_check.py --op qwen3_next_rms_norm --checks gates`): + +| Implementation swapped into the CUDA candidate | C3 forward | C4 gradient (incl. singleton-aggregate) | +|---|---|---| +| rl-kernel Qwen3NextRMSNormCudaOp | pass | pass | +| transformers 5.17.0 Qwen3NextRMSNorm | pass | **fail** | +| Liger 0.8.4 RMSNorm, offset 1, gemma | pass | **fail** | +| FLA 0.5.2 rms_norm, weight passed as 1 + w | pass | **fail** | + +Each existing implementation fails C4 on `dweight` in the chunked-prefill and +singleton-aggregate configurations (max |diff| 1.8e-4); transformers also fails `dx` under +chunking and for one singleton sample. Both `dweight` configurations split the batch into +several calls and the harness adds the per-call `dweight`, which each library returns in BF16; +the split sum is not bitwise equal to the one-call `dweight`. This op passes because it declares +FP32 per-row `dweight` contributions that the harness folds in a fixed order. An +earlier run reported C4 passes for Liger and FLA: the swapped op still inherited this op's +row contributions, so those checks measured this op's `dweight`, not the library's. + ## Tests ```bash python -m pytest tests/models/qwen3_next/test_qwen3_next_norm.py -v +python tools/validation/operators/check_operator.py --op qwen3_next_rms_norm --candidate cuda \ + --device cuda --dtype bf16 --check-grad ``` ## Known Limitations - CUDA only; no ROCm, Ascend or Triton backend. -- Not registered as a gtest operator or in the registry on this branch (see - "Dispatch Behavior"). - Not bitwise against vLLM (see Accuracy). An L2 claim needs a single source of truth for the forward on both sides, per RFC #428 §0 item 1. - The gated pair (`Qwen3NextRMSNormGatedOp`, `Qwen3NextRMSNormGatedHFOp`) is - documented with its CUDA kernel in the gated-norm PR, not on this page. + documented with its CUDA kernel on [Gated RMSNorm](qwen3-next-rms-norm-gated.md). - Measured on sm_100 (B200). Per RFC #428 §2.2 no claim carries across H100/H200/B100/B200. diff --git a/rl_engine/_C.pyi b/rl_engine/_C.pyi index 225f997b6..c0f9cb02c 100644 --- a/rl_engine/_C.pyi +++ b/rl_engine/_C.pyi @@ -272,6 +272,23 @@ def rmsnorm_backward_dx( rstd: torch.Tensor, weight_offset: float = ..., ) -> torch.Tensor: ... +def rmsnorm_gated_forward( + x: torch.Tensor, + weight: torch.Tensor, + gate: torch.Tensor, + eps: float, + weight_offset: float = ..., + activation: int = ..., +) -> list[torch.Tensor]: ... +def rmsnorm_gated_backward_dx( + dy: torch.Tensor, + x: torch.Tensor, + weight: torch.Tensor, + gate: torch.Tensor, + rstd: torch.Tensor, + weight_offset: float = ..., + activation: int = ..., +) -> torch.Tensor: ... def rmsnorm_backward_dw( dy: torch.Tensor, x: torch.Tensor, diff --git a/rl_engine/backends/cuda/norm/rmsnorm.py b/rl_engine/backends/cuda/norm/rmsnorm.py index a64b59fde..33c5d0c27 100644 --- a/rl_engine/backends/cuda/norm/rmsnorm.py +++ b/rl_engine/backends/cuda/norm/rmsnorm.py @@ -7,6 +7,16 @@ _RMSNORM_API_VERSION = 2 +def _fold_dweight_rows(rows: torch.Tensor, dtype: torch.dtype) -> torch.Tensor: + """The single left-fold entrypoint for this backend's dweight reductions. + + Both the plain and the gated backward route through here so the file keeps + one auditable reduction path; the ascending-row fp32 fold is what makes + dweight independent of the batch layout. + """ + return reduce_rows_fp32(rows).to(dtype) + + def _require_cuda_rmsnorm() -> None: """Raise when the compiled RMSNorm bindings are missing or incompatible. @@ -99,7 +109,7 @@ def backward(ctx, grad_out): # Multiplication is part of the pre-existing mask contract, including # IEEE propagation for non-finite inactive contributions. rows = rows * mask.to(dtype=rows.dtype).unsqueeze(-1) - dw = reduce_rows_fp32(rows).to(weight.dtype) + dw = _fold_dweight_rows(rows, weight.dtype) record_backward( "rms_norm", kernel_id=( @@ -150,6 +160,11 @@ def parameter_vjp_contributions_fp32(self, *, x, weight, grad_output, eps=1e-6): hidden = x.shape[-1] # Only `rstd` is used, and it does not depend on the offset; the offset is # passed so this is the same call the forward makes, not because it matters. + # The kernel's own `rstd` is deliberate: the contributions must fold to exactly the + # `dweight` the CUDA backward produces, and that backward uses the saved kernel `rstd`. + # A torch-computed `rstd` differs in the last bits and breaks + # test_cuda_parameter_contributions_reproduce_backward. The op's constructor has + # already checked the extension (`_require_cuda_rmsnorm`). _, rstd = _C.rmsnorm_forward( x.contiguous().reshape(-1, hidden), weight.contiguous(), @@ -169,3 +184,226 @@ class Qwen3NextRMSNormCudaOp(RMSNormCudaOp): """ weight_offset = 1.0 + + +# --------------------------------------------------------------------------- # +# Gated RMSNorm (Qwen3-Next GDN block) +# --------------------------------------------------------------------------- # + + +def _require_cuda_symbols(what: str, *names: str) -> None: + """Raise when the compiled kernels backing ``what`` are missing. + + The registry treats a backend whose construction raises as unavailable and + falls through, so calling this from ``__init__`` is what lets a CUDA-first + priority list degrade to the PyTorch reference on a build without the + extension. Mirrors ``_require_cuda_activation`` in the activation ops. + """ + if not _EXT_AVAILABLE or _C is None: + raise RuntimeError(f"{what} requires the compiled rl_engine._C extension.") + missing = [name for name in names if not hasattr(_C, name)] + if missing: + raise RuntimeError( + f"{what} symbols ({', '.join(missing)}) are not compiled into _C. " + "Rebuild the extension with csrc/cuda/norm/rmsnorm.cu." + ) + + +#: Gate activations understood by the CUDA kernel, in binding order. ``swish`` is +#: an alias for ``silu``, as in vLLM's GDN block, which maps ``output_gate_type`` +#: "swish" to "silu" before constructing ``RMSNormGated``. +_GATE_ACTIVATIONS = {"silu": 0, "swish": 0, "sigmoid": 1} + + +def _check_gate_activation(activation: str) -> int: + if activation not in _GATE_ACTIVATIONS: + raise ValueError( + f"activation must be one of {sorted(_GATE_ACTIVATIONS)}, got {activation!r}" + ) + return _GATE_ACTIVATIONS[activation] + + +def _require_cuda_gated() -> None: + """Guard shared by the gated op's constructor and its autograd forward. + + Checks the two gated symbols and the same RMSNorm API version the plain op + requires, so a stale extension that exports the symbols with an older + signature is rejected instead of being called. + """ + _require_cuda_symbols( + "Gated CUDA RMSNorm", "rmsnorm_gated_forward", "rmsnorm_gated_backward_dx" + ) + api_version = getattr(_C, "rmsnorm_api_version", None) + if api_version != _RMSNORM_API_VERSION: + raise RuntimeError( + f"Gated CUDA RMSNorm requires rl_engine._C RMSNorm API version {_RMSNORM_API_VERSION} " + f"(loaded {api_version!r}). Rebuild the extension with csrc/cuda/norm/rmsnorm.cu." + ) + + +def _gate_activation_fp32(gate: torch.Tensor, activation: int) -> torch.Tensor: + """act(gate) in fp32, matching the kernel's ``gate_activation``.""" + gate32 = gate.float() + return torch.nn.functional.silu(gate32) if activation == 0 else torch.sigmoid(gate32) + + +def _gate_activation_grad_fp32(gate: torch.Tensor, activation: int) -> torch.Tensor: + """d act(gate) / d gate in fp32: sigma * (1 + z * (1 - sigma)) for silu, sigma * (1 - sigma) + for sigmoid, with sigma = sigmoid(z).""" + gate32 = gate.float() + sigma = torch.sigmoid(gate32) + if activation == 0: + return sigma * (1.0 + gate32 * (1.0 - sigma)) + return sigma * (1.0 - sigma) + + +class RMSNormGatedCuda(torch.autograd.Function): + """Autograd wrapper for the gated CUDA RMSNorm. + + Forward is the fused kernel. Backward is assembled from deterministic + pieces: ``dx`` from a row-local CUDA kernel, ``dweight`` from fp32 row + contributions reduced by the ascending-row left fold, and ``dgate`` purely + elementwise in fp32 (no reduction, so batch invariance is trivial). + """ + + @staticmethod + def forward(ctx, x, weight, gate, eps=1e-6, weight_offset=0.0, activation=0): + """ + Forward: + y = x * rsqrt(mean(x^2) + eps) * (weight_offset + weight) * act(gate) + + Input: + x, gate: [T, H], fp16/bf16/fp32 CUDA tensors of matching dtype + weight: [H] + activation: 0 = silu/swish, 1 = sigmoid + """ + _require_cuda_gated() + assert x.is_cuda and weight.is_cuda and gate.is_cuda, "inputs must be CUDA tensors" + assert x.is_contiguous() and weight.is_contiguous() and gate.is_contiguous() + assert x.dim() == 2, "x must be [T, H]" + assert weight.dim() == 1, "weight must be [H]" + assert gate.shape == x.shape, "gate must match x" + y, rstd = _C.rmsnorm_gated_forward( + x, weight, gate, float(eps), float(weight_offset), int(activation) + ) + + ctx.save_for_backward(x, weight, gate, rstd) + ctx.eps = eps + ctx.weight_offset = float(weight_offset) + ctx.activation = int(activation) + + return y + + @staticmethod + def backward(ctx, grad_out): + x, weight, gate, rstd = ctx.saved_tensors + dy = grad_out.contiguous() + act = ctx.activation + + dx = _C.rmsnorm_gated_backward_dx(dy, x, weight, gate, rstd, ctx.weight_offset, act) + + # dweight: the gate is a per-element constant here, so the ungated row + # contributions apply once dy carries act(gate). + gate_act = _gate_activation_fp32(gate, act) + rows = rmsnorm_dweight_rows_fp32(x, dy.float() * gate_act, rstd=rstd.reshape(x.shape[:-1])) + dw = _fold_dweight_rows(rows, weight.dtype) + + # dgate: row-local and reduction-free. + normed = x.float() * rstd.reshape(x.shape[:-1]).unsqueeze(-1) + # Same guard as the kernels: an unconditional `+ 0.0` turns -0.0 weights into +0.0. + scale = weight.float() + if ctx.weight_offset != 0.0: + scale = scale + ctx.weight_offset + dgate = (dy.float() * normed * scale * _gate_activation_grad_fp32(gate, act)).to(gate.dtype) + + record_backward( + "rms_norm_gated", + kernel_id=( + "rl_engine._C.rmsnorm_gated_backward_dx" + "+rl_engine.ops.autograd.vjp_fp32.rmsnorm_dweight_rows_fp32" + "+rl_engine.ops.autograd.vjp_fp32.reduce_rows_fp32" + ), + impl="cuda_rmsnorm_gated_dx_declared_fp32_rowfold_dw", + family="cuda", + ) + + return dx, dw, dgate, None, None, None + + +def rmsnorm_gated_cuda(x, weight, gate, eps=1e-6, weight_offset=0.0, activation="silu"): + """ + use: + y = rmsnorm_gated_cuda(x, weight, gate) + y = rmsnorm_gated_cuda(x, weight, gate, activation="sigmoid") + """ + act = _check_gate_activation(activation) + return RMSNormGatedCuda.apply(x, weight, gate, eps, weight_offset, act) + + +class Qwen3NextRMSNormGatedCudaOp: + """CUDA gated RMSNorm for the Qwen3-Next GDN block. + + Deliberately not a subclass of :class:`RMSNormCudaOp`: it takes an extra + required tensor, so it cannot stand in for one. + + ``out = x * rstd * weight * silu(gate)``, every multiply in fp32 with a + single cast at the store. The weight is plain, not zero-centred, matching + vLLM's ``RMSNormGated`` with ``norm_before_gate=True`` and ``group_size=None`` + -- which is how the GDN block constructs it. The op has no ``norm_before_gate`` + or ``group_size`` parameter, so other configurations are not implemented. + + ``activation`` is fixed at construction; the registry constructs the + released config's ``"silu"``. + """ + + backward_impl = "cuda_rmsnorm_gated_dx_declared_fp32_rowfold_dw" + + #: The gated weight is plain; kept as an attribute so the surface matches + #: the ungated op and a zero-centred variant stays one subclass away. + weight_offset = 0.0 + + def __init__(self, activation: str = "silu") -> None: + _check_gate_activation(activation) + self.activation = activation + _require_cuda_gated() + + def __call__(self, x, weight, gate, *, eps=1e-6): + return self.forward(x, weight, gate, eps=eps) + + def forward(self, x, weight, gate, *, eps=1e-6): + if gate.shape != x.shape: + raise ValueError(f"gate must match x, got {tuple(gate.shape)} vs {tuple(x.shape)}") + hidden = x.shape[-1] + x_2d = x.contiguous().view(-1, hidden) + gate_2d = gate.contiguous().view(-1, hidden) + y_2d = rmsnorm_gated_cuda( + x_2d, + weight.contiguous(), + gate_2d, + eps=eps, + weight_offset=self.weight_offset, + activation=self.activation, + ) + return y_2d.view_as(x) + + def parameter_vjp_contributions_fp32(self, *, x, weight, gate, grad_output, eps=1e-6): + if gate.shape != x.shape: + raise ValueError(f"gate must match x, got {tuple(gate.shape)} vs {tuple(x.shape)}") + hidden = x.shape[-1] + act = _GATE_ACTIVATIONS[self.activation] + # Kernel `rstd` on purpose, as in RMSNormCudaOp.parameter_vjp_contributions_fp32; + # the constructor has already run `_require_cuda_gated`. + _, rstd = _C.rmsnorm_gated_forward( + x.contiguous().reshape(-1, hidden), + weight.contiguous(), + gate.contiguous().reshape(-1, hidden), + float(eps), + float(self.weight_offset), + act, + ) + rows = rmsnorm_dweight_rows_fp32( + x, + grad_output.float() * _gate_activation_fp32(gate, act), + rstd=rstd.reshape(x.shape[:-1]), + ) + return {"weight": rows} diff --git a/rl_engine/config/workload.py b/rl_engine/config/workload.py index 69b21c20f..f59ff7fb3 100644 --- a/rl_engine/config/workload.py +++ b/rl_engine/config/workload.py @@ -264,10 +264,34 @@ def load_manifest(path: str | Path | None = None) -> WS1Manifest: raw = json.load(fh) if not isinstance(raw, dict): raise WorkloadError("manifest root must be a JSON object") - validate_manifest(raw) + # A manifest without a scope is the Dense workload, as in workload_report. + scope = raw.get("scope", "qwen3_8b_dense") + if scope == "qwen3_next_norm_operators": + from rl_engine.validation.models.qwen3_next_workload import validate_norm_manifest + + validate_norm_manifest(raw) + elif scope == "qwen3_8b_dense": + validate_manifest(raw) + else: + raise WorkloadError(f"unknown manifest scope {scope!r}") return WS1Manifest(raw=raw, path=manifest_path) +def workload_report(manifest: WS1Manifest) -> dict[str, Any]: + """The ``workload`` block the C3/C4 gate scripts attach to their JSON report. + + ``full_model_evidence`` is whatever the manifest declares, and ``None`` when + it declares nothing. + """ + return { + "workload_id": manifest.workload_id, + "scope": manifest.raw.get("scope", "qwen3_8b_dense"), + "fixture_identity_sha256": manifest.raw["fixture_identity_sha256"], + "model_id": manifest.model_identity["model_id"], + "full_model_evidence": manifest.raw.get("full_model_evidence"), + } + + def validate_manifest(raw: Mapping[str, Any]) -> None: """Hard-fail if any required C2 pin is missing or inconsistent.""" missing = [k for k in _REQUIRED_TOP_LEVEL if k not in raw] @@ -292,20 +316,25 @@ def validate_manifest(raw: Mapping[str, Any]) -> None: ) -def _validate_model_identity(identity: Mapping[str, Any]) -> None: +def _validate_model_identity( + identity: Mapping[str, Any], + *, + fingerprint: Mapping[str, Any] = _OFFICIAL_FINGERPRINT, + model_label: str = "Qwen3-8B Dense", +) -> None: for key in ("model_id", "revision", "config_fingerprint", "weight_snapshot"): if key not in identity: raise WorkloadError(f"model_identity missing {key!r}") fp = identity["config_fingerprint"] if not isinstance(fp, Mapping): raise WorkloadError("config_fingerprint must be an object") - for key, expected in _OFFICIAL_FINGERPRINT.items(): + for key, expected in fingerprint.items(): if key not in fp: raise WorkloadError(f"config_fingerprint missing {key!r}") if fp[key] != expected: raise WorkloadError( f"config_fingerprint {key}={fp[key]!r} does not match official " - f"Qwen3-8B Dense pin {expected!r}; architecture shrink is forbidden" + f"{model_label} pin {expected!r}; architecture shrink is forbidden" ) if not identity.get("exit_forbids_architecture_shrink", False): raise WorkloadError("exit_forbids_architecture_shrink must be true") diff --git a/rl_engine/reference/norm/qwen3_next_rms_norm.py b/rl_engine/reference/norm/qwen3_next_rms_norm.py index adb024bcc..41c3b92fc 100644 --- a/rl_engine/reference/norm/qwen3_next_rms_norm.py +++ b/rl_engine/reference/norm/qwen3_next_rms_norm.py @@ -22,7 +22,8 @@ All three reuse :func:`shape_invariant_rstd`, a fixed-order reduction. They reproduce the weight convention and cast order, not vLLM's reduction tree, and are not bitwise equal to any vLLM path probed so far. Claim levels, measurements and -limitations are in ``docs/operators/qwen3-next-rms-norm.md``. +limitations are in ``docs/operators/qwen3-next-rms-norm.md`` and, for the gated +pair, ``docs/operators/qwen3-next-rms-norm-gated.md``. """ from __future__ import annotations @@ -61,7 +62,7 @@ class Qwen3NextRMSNormGatedOp: ``out = (x * rstd * weight) * silu(gate)``, with every multiply in fp32 and a single cast on the way out. This is the convention vLLM's ``RMSNormGated`` uses with ``norm_before_gate=True``. Which gated convention is the strict - default is still open; see the operator page. + default is still open; see ``docs/operators/qwen3-next-rms-norm-gated.md``. Not a subclass of the plain op: it takes an extra tensor and its epilogue differs, so it is not a drop-in substitute for one. @@ -113,6 +114,12 @@ def _normalized( f"gate must match x, got tuple(gate.shape)={tuple(gate.shape)} " f"vs tuple(x.shape)={tuple(x.shape)}" ) + if gate.dtype != x.dtype or gate.device != x.device: + raise ValueError( + "gate must have the same dtype and device as x, got " + f"gate=(dtype={gate.dtype}, device={gate.device}) vs " + f"x=(dtype={x.dtype}, device={x.device})" + ) x_f = x.float() rstd = shape_invariant_rstd(x_f, float(eps)).unsqueeze(-1) return x_f * rstd diff --git a/rl_engine/runtime/registry.py b/rl_engine/runtime/registry.py index a190bbca1..9a6494d5d 100644 --- a/rl_engine/runtime/registry.py +++ b/rl_engine/runtime/registry.py @@ -159,6 +159,18 @@ class OpBackend(Enum, metaclass=_KernelEnumMeta): TRITON_RMS_NORM = "rl_engine.kernels.ops.triton.rmsnorm_triton.RMSNormTritonOp" PYTORCH_NATIVE_RMS_NORM = "rl_engine.kernels.ops.pytorch.norm.rms_norm.NativeRMSNormOp" + # Zero-centred RMSNorm (Qwen3-Next decoder / final norm) + CUDA_QWEN3_NEXT_RMS_NORM = "rl_engine.kernels.ops.cuda.norm.rmsnorm.Qwen3NextRMSNormCudaOp" + PYTORCH_NATIVE_QWEN3_NEXT_RMS_NORM = ( + "rl_engine.reference.norm.qwen3_next_rms_norm.Qwen3NextRMSNormOp" + ) + + # Gated RMSNorm (Qwen3-Next Gated DeltaNet block) + CUDA_RMS_NORM_GATED = "rl_engine.kernels.ops.cuda.norm.rmsnorm.Qwen3NextRMSNormGatedCudaOp" + PYTORCH_NATIVE_RMS_NORM_GATED = ( + "rl_engine.reference.norm.qwen3_next_rms_norm.Qwen3NextRMSNormGatedOp" + ) + # Generic fallback TRITON_GENERIC = "rl_engine.kernels.ops.triton.generic.TritonOp" PYTORCH_ATTN = "rl_engine.kernels.ops.pytorch.attention.NativeAttentionOp" @@ -594,6 +606,14 @@ def __init__(self): OpBackend.CUDA_RMS_NORM, OpBackend.PYTORCH_NATIVE_RMS_NORM, ], + "rms_norm_gated": [ + OpBackend.CUDA_RMS_NORM_GATED, + OpBackend.PYTORCH_NATIVE_RMS_NORM_GATED, + ], + "qwen3_next_rms_norm": [ + OpBackend.CUDA_QWEN3_NEXT_RMS_NORM, + OpBackend.PYTORCH_NATIVE_QWEN3_NEXT_RMS_NORM, + ], "lm_head": [OpBackend.PYTORCH_NATIVE_LM_HEAD], "embedding": [OpBackend.PYTORCH_NATIVE_EMBEDDING], "silu": [ @@ -649,6 +669,8 @@ def __init__(self): ], "matmul": [OpBackend.PYTORCH_NATIVE_MATMUL], "rms_norm": [OpBackend.PYTORCH_NATIVE_RMS_NORM], + "rms_norm_gated": [OpBackend.PYTORCH_NATIVE_RMS_NORM_GATED], + "qwen3_next_rms_norm": [OpBackend.PYTORCH_NATIVE_QWEN3_NEXT_RMS_NORM], "lm_head": [OpBackend.PYTORCH_NATIVE_LM_HEAD], "embedding": [OpBackend.PYTORCH_NATIVE_EMBEDDING], "silu": [OpBackend.TRITON_SILU, OpBackend.PYTORCH_NATIVE_SILU], @@ -694,6 +716,8 @@ def __init__(self): OpBackend.TRITON_RMS_NORM, OpBackend.PYTORCH_NATIVE_RMS_NORM, ], + "rms_norm_gated": [OpBackend.PYTORCH_NATIVE_RMS_NORM_GATED], + "qwen3_next_rms_norm": [OpBackend.PYTORCH_NATIVE_QWEN3_NEXT_RMS_NORM], "lm_head": [OpBackend.PYTORCH_NATIVE_LM_HEAD], "embedding": [ OpBackend.TRITON_EMBEDDING, @@ -722,6 +746,8 @@ def __init__(self): "batch_invariant_logp": [OpBackend.PYTORCH_BATCH_INVARIANT_LOGP], "matmul": [OpBackend.PYTORCH_NATIVE_MATMUL], "rms_norm": [OpBackend.PYTORCH_NATIVE_RMS_NORM], + "rms_norm_gated": [OpBackend.PYTORCH_NATIVE_RMS_NORM_GATED], + "qwen3_next_rms_norm": [OpBackend.PYTORCH_NATIVE_QWEN3_NEXT_RMS_NORM], "lm_head": [OpBackend.PYTORCH_NATIVE_LM_HEAD], "embedding": [OpBackend.PYTORCH_NATIVE_EMBEDDING], "silu": [OpBackend.PYTORCH_NATIVE_SILU], @@ -762,6 +788,12 @@ def __init__(self): OpBackend.ASCEND_RMS_NORM, OpBackend.PYTORCH_NATIVE_RMS_NORM, ] + self._priority_map["npu"]["rms_norm_gated"] = [ + OpBackend.PYTORCH_NATIVE_RMS_NORM_GATED, + ] + self._priority_map["npu"]["qwen3_next_rms_norm"] = [ + OpBackend.PYTORCH_NATIVE_QWEN3_NEXT_RMS_NORM, + ] self._priority_map["npu"]["embedding"] = [ OpBackend.ASCEND_EMBEDDING, OpBackend.PYTORCH_NATIVE_EMBEDDING, diff --git a/rl_engine/validation/models/qwen3_next_norm_manifest.json b/rl_engine/validation/models/qwen3_next_norm_manifest.json new file mode 100644 index 000000000..9cf5977ef --- /dev/null +++ b/rl_engine/validation/models/qwen3_next_norm_manifest.json @@ -0,0 +1,759 @@ +{ + "version": "1.0", + "workload_id": "qwen3-next-80b-a3b-norm-c3-c4-v1", + "seed": 20260812, + "model_identity": { + "model_id": "Qwen/Qwen3-Next-80B-A3B-Instruct", + "revision": "9c7f2fbe84465e40164a94cc16cd30b6999b0cc7", + "config_fingerprint": { + "num_hidden_layers": 48, + "hidden_size": 2048, + "intermediate_size": 5120, + "num_attention_heads": 16, + "num_key_value_heads": 2, + "head_dim": 256, + "vocab_size": 151936, + "linear_key_head_dim": 128, + "linear_value_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "num_experts": 512, + "num_experts_per_tok": 10 + }, + "exit_forbids_architecture_shrink": true, + "weight_snapshot": { + "pin_method": "revision-and-sha256", + "total_size_bytes": 162649725440, + "index_file": "model.safetensors.index.json", + "index_sha256": "0e5274f538d750abdd7fd55a36bad64100f9e881d59c65f6a59f364d2f8b6dd7", + "content_hash_algorithm": "sha256-of-sorted-shard-records-v1", + "content_hash": "00a6a3b5573ccbac2e01fadb673c2cd7bca172d40ddd73f95ed7afa5cbdb9dcd", + "shards": [ + { + "filename": "model-00001-of-00041.safetensors", + "size_bytes": 3999619256, + "sha256": "d8908fd48650169854b5ed815cb01bfc1741152cc58795ec99188862c6a29e11" + }, + { + "filename": "model-00002-of-00041.safetensors", + "size_bytes": 3999841784, + "sha256": "c753c9bfaca220781d4030c3a99e69b4a256434c9d6ec223f5147edc265289df" + }, + { + "filename": "model-00003-of-00041.safetensors", + "size_bytes": 3999515584, + "sha256": "51aaa14dd50c5ab90c363227bfb1ac51182118f588e0141dcaadda012548407c" + }, + { + "filename": "model-00004-of-00041.safetensors", + "size_bytes": 3999842000, + "sha256": "82a33096134fb6e7751a423f142e79b1bbe89f45242b75397c2d7170fffa75bd" + }, + { + "filename": "model-00005-of-00041.safetensors", + "size_bytes": 3999842208, + "sha256": "41abda7bf93f27c36cb28382114f5b33defa6dbfd154abb6886a6fec20f9e479" + }, + { + "filename": "model-00006-of-00041.safetensors", + "size_bytes": 3999853216, + "sha256": "a7794475040ebd62c9a1f9c94c17e7c600873e14fcabc656b32faea09bc7fd2d" + }, + { + "filename": "model-00007-of-00041.safetensors", + "size_bytes": 3999841912, + "sha256": "64d7e90d00ce15cc8bbf7677ad07bf3cffc9e90a1b777e3334fa15ffea219d6b" + }, + { + "filename": "model-00008-of-00041.safetensors", + "size_bytes": 3999842000, + "sha256": "9ace9b99e71490d619656956d457744f727647b3f36fa3d801798a5156599d35" + }, + { + "filename": "model-00009-of-00041.safetensors", + "size_bytes": 3999843192, + "sha256": "5b1b374c9e6100077c446a293566c1f644475c94739ab08b7fa26ba847110216" + }, + { + "filename": "model-00010-of-00041.safetensors", + "size_bytes": 3999517808, + "sha256": "38e51dc850a39c325324e6ddd23347e6e3e76cba1341aeef2e1f260ca2cb9f49" + }, + { + "filename": "model-00011-of-00041.safetensors", + "size_bytes": 4000181296, + "sha256": "c06719dc79fbbc796b8751ccbb29c8ecbdb686de9f89e3c741b5bbfec203cf0c" + }, + { + "filename": "model-00012-of-00041.safetensors", + "size_bytes": 3999843880, + "sha256": "349036c3e3f6a8645af47906dae7272bd5352f822a3c354a16ef4084b3555679" + }, + { + "filename": "model-00013-of-00041.safetensors", + "size_bytes": 3999517472, + "sha256": "bfab17d1535ef23e99e54ff3156ce1456aa5f0a4a93b281a7adb9d8b921dc829" + }, + { + "filename": "model-00014-of-00041.safetensors", + "size_bytes": 3999843984, + "sha256": "318d10df78f1647189941884acc01616a6a78d5d45d74ea30955be7f509c580a" + }, + { + "filename": "model-00015-of-00041.safetensors", + "size_bytes": 4000181736, + "sha256": "aa3400abde789ecca625b8cea37aa31232bf6696923f10981d028518a2393173" + }, + { + "filename": "model-00016-of-00041.safetensors", + "size_bytes": 3999517256, + "sha256": "3776700f68222f0174bb14fea8480da41d928a1bfd6b5317c4fc2b14006d506e" + }, + { + "filename": "model-00017-of-00041.safetensors", + "size_bytes": 3999843880, + "sha256": "052726cd55224d3d217dac93eaed060215b8029afb44861c0d927d87c6a046ab" + }, + { + "filename": "model-00018-of-00041.safetensors", + "size_bytes": 3999843880, + "sha256": "dd1b8d843e0013ed3beb6ba7fc36dc4b1f2df5d71b40520f82f369cd8fce5cba" + }, + { + "filename": "model-00019-of-00041.safetensors", + "size_bytes": 3999844096, + "sha256": "bfe5489c61334f9ce9bc6b369c3edf44420d7f585c6438f3afb98a120b26e6f3" + }, + { + "filename": "model-00020-of-00041.safetensors", + "size_bytes": 3999855040, + "sha256": "eba3113cbd34304751e148502f314be5ceae6e7d830bd33f2c0cf3bb8dfb28c2" + }, + { + "filename": "model-00021-of-00041.safetensors", + "size_bytes": 3999843792, + "sha256": "9ddd72e8ada4480e86e4230639210df1039802db9f78537673c19117d2e69b95" + }, + { + "filename": "model-00022-of-00041.safetensors", + "size_bytes": 3999843880, + "sha256": "c6fc2bb1762d34b8db83fb11566e462c9166e132bb2bfcef45a33a4bd41c02db" + }, + { + "filename": "model-00023-of-00041.safetensors", + "size_bytes": 3999517464, + "sha256": "484bf8c327f7f7aaa20fbc4f6800d4aab7870749335b6f6bbfdb83570a1f66e2" + }, + { + "filename": "model-00024-of-00041.safetensors", + "size_bytes": 3999844264, + "sha256": "cfbb94709f5dac71ffe144ee3b9a976a49a2a9b9863caa29620fbc65820c2342" + }, + { + "filename": "model-00025-of-00041.safetensors", + "size_bytes": 4000181296, + "sha256": "b5799d19dcccfb17b108b5349900a4c42eb7bc241149cb7d1b38b48c707070cf" + }, + { + "filename": "model-00026-of-00041.safetensors", + "size_bytes": 3999517472, + "sha256": "98003819929196d2f6f05c42603e9124a7a7853f372089eed3ffbd0c244fbdac" + }, + { + "filename": "model-00027-of-00041.safetensors", + "size_bytes": 3999843880, + "sha256": "3d6b8b9d1c0c262cc8a63c41c3f75053ab8b31b6107130800718048da64b86db" + }, + { + "filename": "model-00028-of-00041.safetensors", + "size_bytes": 3999843984, + "sha256": "836d4a85cc72243c6caee2b6cf470cd598d6ab817e0685daf1128fd97bb133e3" + }, + { + "filename": "model-00029-of-00041.safetensors", + "size_bytes": 3999855320, + "sha256": "7b083d02c732458cfaaee8b0d1407626ad5ee10fc7eeed6b294b7d6853ac30e8" + }, + { + "filename": "model-00030-of-00041.safetensors", + "size_bytes": 3999843672, + "sha256": "e81006eeaa1152c67798640be46b710e87d44a307effca60cbdb7e9d1a3b268b" + }, + { + "filename": "model-00031-of-00041.safetensors", + "size_bytes": 3999843880, + "sha256": "4fd46dbc55ceefa6146a7687cd02d040087c58b0f7ea42c4cd7dc66b838f531c" + }, + { + "filename": "model-00032-of-00041.safetensors", + "size_bytes": 3999843880, + "sha256": "7edf6738b9bc204f0b39e9c7899515d1a1ed9d7d5f7c1cce32c981120c4f034d" + }, + { + "filename": "model-00033-of-00041.safetensors", + "size_bytes": 3999517688, + "sha256": "9f0f735e75e8d648b4b8d601a598d489fc4e4c7876da4f3406bf3bcc8990a181" + }, + { + "filename": "model-00034-of-00041.safetensors", + "size_bytes": 4000181496, + "sha256": "3499ba70dbb6a9acf87509c3b82c5790461a73f8d0d12f0e3699354f34dc02b7" + }, + { + "filename": "model-00035-of-00041.safetensors", + "size_bytes": 3999843792, + "sha256": "de04b5e39b9494b44becacf22fbe9c02890ef323a3dfe37640a2bfa80335504c" + }, + { + "filename": "model-00036-of-00041.safetensors", + "size_bytes": 3999517472, + "sha256": "3eddb88c62194513f94ef3b1c11577ad324fdcc21265ff7b11a4b50d50b03c5f" + }, + { + "filename": "model-00037-of-00041.safetensors", + "size_bytes": 3999843872, + "sha256": "01992493e00d45d5fec4b5989392f401c9ebe3e13973edaaf1973577e132ca11" + }, + { + "filename": "model-00038-of-00041.safetensors", + "size_bytes": 3999844264, + "sha256": "e328c6d2e5f2a42eb99a0335bde7aa4c42cbd72b88fb988d5a2c96ba6231e95e" + }, + { + "filename": "model-00039-of-00041.safetensors", + "size_bytes": 3999854888, + "sha256": "15063cd0a46af3fa37e74590534f62a81f1ede5340e7640713bf98f39f20961d" + }, + { + "filename": "model-00040-of-00041.safetensors", + "size_bytes": 3365572496, + "sha256": "e458c4f4345d7082e7063f381ee1962725f84c44d53a5b70afb77e601ff22017" + }, + { + "filename": "model-00041-of-00041.safetensors", + "size_bytes": 3301131296, + "sha256": "1f3ac4d828f7e08dd14eb4dc6f282139ae1fa0d894e43f247da2539d2ef43826" + } + ], + "weight_files_total_size_bytes": 162659161528 + } + }, + "chain_semantics": { + "execution_dtype": "bfloat16", + "reference_dtype": "float32", + "accumulation_dtype": "float32", + "temperature": 1.0, + "loss_reduction": "sum_over_active_tokens_then_optional_mean_by_active_count", + "logprob_selection": "selected_token_logprob_on_active_mask", + "active_token_policy": "active selected tokens only", + "aggregates": [ + "max_abs_dlogp", + "approx_kl0", + "clipfrac0" + ], + "clip_interval": [ + 0.8, + 1.2 + ], + "clip_interval_note": "Pinned for clipfrac0; must match C1 chain_logprob_aggregates.default_clip_interval unless an explicit contract revision changes both.", + "comparison_roles_source": "rl_engine/kernels/gtest/tolerance_contract.json", + "forbidden_comparison_roles": [ + "baseline", + "singleton_aggregate" + ], + "singleton_aggregate_note": "singleton_aggregate is a C2 execution/aggregation mode only. It must never populate comparison_lhs_role or comparison_rhs_role.", + "tf32_policy_ref": "rl_engine/kernels/gtest/tolerance_contract.json#/policy/tf32", + "tf32_note": "WS1 TF32 enable/disable is owned by the C1 contract; C2 gates must not introduce a private TF32 policy.", + "report_naming": { + "comparison_lhs_role": "from_c1_by_report_kind", + "comparison_rhs_role": "from_c1_by_report_kind", + "forbidden_in_reports": [ + "baseline", + "singleton_aggregate" + ], + "singleton_aggregate_is": "c2_execution_aggregation_mode_only", + "note": "C2 freezes naming rules; C3+ emit reports that must obey these roles." + }, + "backend_actual_semantics": { + "c2_representative_actual_source": "scripts/ws1_candidate_evidence.py runtime execution", + "full_model_runtime_observed_actual_owner": [ + "C3", + "C8", + "C10", + "C11" + ], + "note": "C2 executes every representative case and records runtime-observed actual backend/kernel provenance. Later children own full-model dispatch provenance." + } + }, + "stochastic_policy": { + "dropout": 0.0, + "attention_dropout": 0.0, + "sampling_in_logprob_parity": false, + "canonical_gate_uses_dropout_zero": true, + "rng_source": "manifest_seed_plus_logical_sample_token_identity", + "undeclared_randomness": "hard_fail", + "retained_stochastic_ops": [] + }, + "primary_matrix": { + "description": "Fixed #150 Batch \u00d7 Chunked-Prefill matrix prerequisite workload cells.", + "N": 4, + "batch_size_bn": 4, + "sample_ids": [ + "s0", + "s1", + "s2", + "s3" + ], + "sample_order_fixed": true, + "batch_permutation": { + "enabled": true, + "permutation": [ + 2, + 0, + 3, + 1 + ], + "target_sample_position_in_bn": 0, + "note": "Permutation exercises layout invariance; logical compare restores sample_id order." + }, + "chunk": { + "chunk_size_tokens": 7, + "require_ge_2_chunks": true, + "non_divisible_case": true, + "note": "Longest primary seq_len=19 with chunk_size=7 yields chunks [7,7,5]." + }, + "cells": [ + { + "cell_id": "B1-singleton_aggregate/full", + "batch_mode": "singleton_aggregate", + "batch_size_per_run": 1, + "num_runs": 4, + "prefill_mode": "full", + "aggregation": { + "order": "sample_ids", + "denominator": "active_token_count_across_all_samples" + } + }, + { + "cell_id": "BN/full", + "batch_mode": "batched", + "batch_size_per_run": 4, + "num_runs": 1, + "prefill_mode": "full", + "aggregation": { + "order": "sample_ids", + "denominator": "active_token_count_across_all_samples" + } + }, + { + "cell_id": "B1-singleton_aggregate/chunked", + "batch_mode": "singleton_aggregate", + "batch_size_per_run": 1, + "num_runs": 4, + "prefill_mode": "chunked", + "aggregation": { + "order": "sample_ids", + "denominator": "active_token_count_across_all_samples" + } + }, + { + "cell_id": "BN/chunked", + "batch_mode": "batched", + "batch_size_per_run": 4, + "num_runs": 1, + "prefill_mode": "chunked", + "aggregation": { + "order": "sample_ids", + "denominator": "active_token_count_across_all_samples" + } + } + ] + }, + "fixtures": { + "prompt_template": "ws1_fixed_token_fixture", + "dtype_for_token_tensors": "int64", + "position_ids": { + "basis": "logical_zero_based_per_sample", + "reset_after_pack_boundary": true + }, + "attention_mask": { + "active_value": 1, + "padding_value": 0, + "causal": true + }, + "primary_seq_len": 19, + "primary_prompt_len": 8, + "short_seq_len": 8, + "long_seq_len": 32, + "varlen_seq_lens": [ + 11, + 16, + 13, + 19 + ], + "padding": { + "modes": [ + "right", + "left" + ], + "pad_token_id": 151643, + "primary_padded_len": 20 + }, + "packing": { + "status": "supported", + "implementation": "rl_engine.kernels.ops.pytorch.packing.pack.NativePackOp", + "packed_fixture": { + "sample_order": [ + "s0", + "s1", + "s2", + "s3" + ], + "segment_lengths": [ + 11, + 16, + 13, + 19 + ], + "total_tokens": 59, + "restore_key": [ + "sample_id", + "token_position" + ] + } + }, + "loss_mask": { + "prompt_tokens_active": false, + "completion_tokens_active": true + }, + "samples": [ + { + "sample_id": "s0", + "seq_len": 11, + "prompt_len": 8, + "token_ids": [ + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 200, + 201, + 202 + ] + }, + { + "sample_id": "s1", + "seq_len": 16, + "prompt_len": 8, + "token_ids": [ + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217 + ] + }, + { + "sample_id": "s2", + "seq_len": 13, + "prompt_len": 8, + "token_ids": [ + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 220, + 221, + 222, + 223, + 224 + ] + }, + { + "sample_id": "s3", + "seq_len": 19, + "prompt_len": 8, + "token_ids": [ + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240 + ] + } + ], + "short_full_model_fixture": { + "fixture_id": "short_full_model_seq8", + "seq_len": 8, + "prompt_len": 4, + "token_ids": [ + 310, + 311, + 312, + 313, + 410, + 411, + 412, + 413 + ], + "note": "Shorter sequence on full architecture+weights only; never shrinks layers/hidden/heads/vocab.", + "candidate_case_ids": [ + "short_full_model_seq8_qwen3_next_rms_norm", + "short_full_model_seq8_rms_norm_gated" + ] + }, + "long_full_model_fixture": { + "fixture_id": "long_full_model_seq32", + "seq_len": 32, + "prompt_len": 16, + "token_ids": [ + 500, + 501, + 502, + 503, + 504, + 505, + 506, + 507, + 508, + 509, + 510, + 511, + 512, + 513, + 514, + 515, + 600, + 601, + 602, + 603, + 604, + 605, + 606, + 607, + 608, + 609, + 610, + 611, + 612, + 613, + 614, + 615 + ], + "note": "Long fixed sequence on the same full architecture and pinned weight snapshot.", + "candidate_case_ids": [ + "long_full_model_seq32_qwen3_next_rms_norm", + "long_full_model_seq32_rms_norm_gated" + ] + }, + "representative_full_model_fixture": { + "fixture_id": "rep_full_model_seq16", + "seq_len": 16, + "prompt_len": 8, + "sample_ids": [ + "s0", + "s1", + "s2", + "s3" + ], + "note": "Primary variable-length matrix fixture; full architecture+weights.", + "candidate_case_ids": [ + "rep_full_model_seq16_qwen3_next_rms_norm", + "rep_full_model_seq16_rms_norm_gated" + ] + }, + "prompt_lens": [ + 8, + 8, + 8, + 8 + ], + "completion_lens": [ + 3, + 8, + 5, + 11 + ], + "max_completion_len": 11 + }, + "logical_identity": { + "key": [ + "sample_id", + "token_position" + ], + "token_position_basis": "logical_unpadded_index_in_sample", + "restore_before_compare_after": [ + "pad", + "pack", + "chunk", + "batch_permute" + ], + "gradient_singleton_aggregate": { + "definition": "N independent B=1 runs of the same N logical samples, aggregated with fixed sample order and active-token denominator", + "compare_to": "single B=N run of the same logical sample/token multiset", + "forbid_different_sample_sets": true + } + }, + "capabilities": { + "required_chain_ops": [ + { + "op": "qwen3_next_rms_norm", + "status": "required" + }, + { + "op": "rms_norm_gated", + "status": "required" + } + ], + "operator_spec_map": { + "qwen3_next_rms_norm": "qwen3_next_rms_norm", + "rms_norm_gated": "rms_norm_gated" + } + }, + "backend_profiles": { + "cuda_bf16": { + "backend_family": "cuda", + "execution_dtype": "bfloat16", + "required_nodes": [ + { + "node": "qwen3_next_rms_norm", + "status": "declared", + "expected_backend_id": "cuda", + "expected_kernel_config_id": "rl_engine.kernels.ops.cuda.norm.rmsnorm.Qwen3NextRMSNormCudaOp", + "algorithm_property": "fixed row reduction and ordered FP32 parameter fold" + }, + { + "node": "rms_norm_gated", + "status": "declared", + "expected_backend_id": "cuda", + "expected_kernel_config_id": "rl_engine.kernels.ops.cuda.norm.rmsnorm.Qwen3NextRMSNormGatedCudaOp", + "algorithm_property": "fixed row reduction and ordered FP32 parameter fold" + } + ] + }, + "triton_cuda_bf16": { + "backend_family": "triton", + "execution_dtype": "bfloat16", + "required_nodes": [ + { + "node": "qwen3_next_rms_norm", + "status": "missing_required", + "expected_backend_id": null, + "expected_kernel_config_id": null, + "algorithm_property": "fixed row reduction and ordered FP32 parameter fold" + }, + { + "node": "rms_norm_gated", + "status": "missing_required", + "expected_backend_id": null, + "expected_kernel_config_id": null, + "algorithm_property": "fixed row reduction and ordered FP32 parameter fold" + } + ] + }, + "ascend_bf16": { + "backend_family": "ascend", + "execution_dtype": "bfloat16", + "required_nodes": [ + { + "node": "qwen3_next_rms_norm", + "status": "missing_required", + "expected_backend_id": null, + "expected_kernel_config_id": null, + "algorithm_property": "fixed row reduction and ordered FP32 parameter fold" + }, + { + "node": "rms_norm_gated", + "status": "missing_required", + "expected_backend_id": null, + "expected_kernel_config_id": null, + "algorithm_property": "fixed row reduction and ordered FP32 parameter fold" + } + ] + } + }, + "representative_cases": [ + { + "case_id": "short_full_model_seq8_qwen3_next_rms_norm", + "fixture_id": "short_full_model_seq8", + "operator_spec": "qwen3_next_rms_norm", + "hidden": 2048, + "architecture_identity": "qwen3_next_80b_a3b_norm_operators" + }, + { + "case_id": "short_full_model_seq8_rms_norm_gated", + "fixture_id": "short_full_model_seq8", + "operator_spec": "rms_norm_gated", + "hidden": 128, + "architecture_identity": "qwen3_next_80b_a3b_norm_operators" + }, + { + "case_id": "long_full_model_seq32_qwen3_next_rms_norm", + "fixture_id": "long_full_model_seq32", + "operator_spec": "qwen3_next_rms_norm", + "hidden": 2048, + "architecture_identity": "qwen3_next_80b_a3b_norm_operators" + }, + { + "case_id": "long_full_model_seq32_rms_norm_gated", + "fixture_id": "long_full_model_seq32", + "operator_spec": "rms_norm_gated", + "hidden": 128, + "architecture_identity": "qwen3_next_80b_a3b_norm_operators" + }, + { + "case_id": "rep_full_model_seq16_qwen3_next_rms_norm", + "fixture_id": "rep_full_model_seq16", + "operator_spec": "qwen3_next_rms_norm", + "hidden": 2048, + "architecture_identity": "qwen3_next_80b_a3b_norm_operators" + }, + { + "case_id": "rep_full_model_seq16_rms_norm_gated", + "fixture_id": "rep_full_model_seq16", + "operator_spec": "rms_norm_gated", + "hidden": 128, + "architecture_identity": "qwen3_next_80b_a3b_norm_operators" + } + ], + "fixture_identity_sha256": "3ac9490ffbb1486edac6ea509811fd4d5affbe880b208bd7d9b052b0ce418442", + "provenance_boundary": { + "scope": "Synthetic norm inputs at checkpoint dimensions; not checkpoint execution or model-level L2.", + "runtime_verified": false + }, + "scope": "qwen3_next_norm_operators", + "full_model_evidence": false +} diff --git a/rl_engine/validation/models/qwen3_next_workload.py b/rl_engine/validation/models/qwen3_next_workload.py new file mode 100644 index 000000000..beaf372c2 --- /dev/null +++ b/rl_engine/validation/models/qwen3_next_workload.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Qwen3-Next norm-only C3/C4 workload, explicitly separate from the Dense chain.""" + +from collections.abc import Mapping +from typing import Any + +from rl_engine.config.workload import ( + _REQUIRED_TOP_LEVEL, + WorkloadError, + _validate_backend_profiles, + _validate_chain_semantics, + _validate_fixtures, + _validate_logical_identity, + _validate_model_identity, + _validate_primary_matrix, + _validate_stochastic_policy, + manifest_identity_hash, +) + +MODEL_ID = "Qwen/Qwen3-Next-80B-A3B-Instruct" +REVISION = "9c7f2fbe84465e40164a94cc16cd30b6999b0cc7" +FINGERPRINT = { + "num_hidden_layers": 48, + "hidden_size": 2048, + "intermediate_size": 5120, + "num_attention_heads": 16, + "num_key_value_heads": 2, + "head_dim": 256, + "vocab_size": 151936, + "linear_key_head_dim": 128, + "linear_value_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "num_experts": 512, + "num_experts_per_tok": 10, +} +NORM_OPS = {"qwen3_next_rms_norm": 2048, "rms_norm_gated": 128} + + +def validate_norm_manifest(raw: Mapping[str, Any]) -> None: + """Validate a norm-operator manifest; every malformed manifest raises ``WorkloadError``.""" + try: + _validate_norm_manifest(raw) + except (KeyError, TypeError) as exc: + raise WorkloadError(f"norm workload manifest is malformed: {exc!r}") from exc + + +def _validate_norm_manifest(raw: Mapping[str, Any]) -> None: + missing = [k for k in _REQUIRED_TOP_LEVEL if k not in raw] + if missing: + raise WorkloadError( + f"norm workload manifest is malformed: missing top-level keys {missing}" + ) + identity = raw["model_identity"] + if identity["model_id"] != MODEL_ID or identity["revision"] != REVISION: + raise WorkloadError("Qwen3-Next norm workload requires the pinned official checkpoint") + _validate_model_identity(identity, fingerprint=FINGERPRINT, model_label="Qwen3-Next") + if ( + raw.get("scope") != "qwen3_next_norm_operators" + or raw.get("full_model_evidence") is not False + ): + raise WorkloadError("norm workload cannot claim full-model evidence") + _validate_chain_semantics(raw["chain_semantics"]) + _validate_stochastic_policy(raw["stochastic_policy"]) + _validate_primary_matrix(raw["primary_matrix"], raw["fixtures"]) + _validate_fixtures(raw["fixtures"], raw["primary_matrix"]) + _validate_logical_identity(raw["logical_identity"]) + caps = raw["capabilities"] + if sorted(e["op"] for e in caps["required_chain_ops"]) != sorted(NORM_OPS): + raise WorkloadError("norm workload must contain exactly the two Qwen3-Next norms") + if any(e["status"] != "required" for e in caps["required_chain_ops"]): + raise WorkloadError("both norm operators are required") + _validate_backend_profiles(raw["backend_profiles"], caps) + cases = {case["case_id"]: case for case in raw["representative_cases"]} + if len(cases) != len(raw["representative_cases"]): + raise WorkloadError("duplicate norm case ID") + referenced = set() + for key in ( + "short_full_model_fixture", + "long_full_model_fixture", + "representative_full_model_fixture", + ): + fixture = raw["fixtures"][key] + for case_id in fixture["candidate_case_ids"]: + if case_id not in cases or cases[case_id]["fixture_id"] != fixture["fixture_id"]: + raise WorkloadError("norm case fixture binding mismatch") + referenced.add(case_id) + if referenced != set(cases): + raise WorkloadError("unreferenced norm case") + if {case["operator_spec"] for case in cases.values()} != set(NORM_OPS): + raise WorkloadError("representative cases must cover both norms") + for case in cases.values(): + if case["hidden"] != NORM_OPS[case["operator_spec"]]: + raise WorkloadError("norm case hidden dimension does not match checkpoint") + if case["architecture_identity"] != "qwen3_next_80b_a3b_norm_operators": + raise WorkloadError("norm cases cannot claim Dense or full-model architecture evidence") + if raw["fixture_identity_sha256"] != manifest_identity_hash(raw): + raise WorkloadError("Qwen3-Next fixture identity hash mismatch") + + +def validate_norm_dimensions(raw: Mapping[str, Any], op: str, hidden: int, head_dim: int) -> None: + if raw.get("scope") != "qwen3_next_norm_operators": + return + # Both gates take the same pair: --hidden is the model hidden size, the zero-centred + # norm's width, and the gated norm runs at --head-dim (linear_value_head_dim), not at + # --hidden. Pinning the pair to the two checkpoint widths pins each op's width. + expected = (NORM_OPS["qwen3_next_rms_norm"], NORM_OPS["rms_norm_gated"]) + if op not in NORM_OPS or (hidden, head_dim) != expected: + raise WorkloadError( + "Qwen3-Next norm gate requires its two norm ops, " + f"--hidden {expected[0]} --head-dim {expected[1]}" + ) diff --git a/rl_engine/validation/operators/gradient_adapters.py b/rl_engine/validation/operators/gradient_adapters.py index f22d80285..9831fb670 100644 --- a/rl_engine/validation/operators/gradient_adapters.py +++ b/rl_engine/validation/operators/gradient_adapters.py @@ -53,6 +53,7 @@ class GradientAdapterSpec: source_files: tuple[str, ...] shape_dependent_bwd_accum: str = "forbidden" atomic_add: str = "forbidden" + model_id: str | None = None @dataclass(frozen=True) @@ -115,6 +116,26 @@ def to_dict(self) -> dict[str, Any]: "csrc/cuda/norm/rmsnorm.cu", ), ), + "qwen3_next_rms_norm": GradientAdapterSpec( + op_name="qwen3_next_rms_norm", + chain_node="qwen3_next_rms_norm", + op_class="reduction", + spec_name="qwen3_next_rms_norm", + tensors=(_DX, _DWEIGHT), + requirement="required", + source_files=("rl_engine/backends/cuda/norm/rmsnorm.py", "csrc/cuda/norm/rmsnorm.cu"), + model_id="Qwen/Qwen3-Next-80B-A3B-Instruct", + ), + "rms_norm_gated": GradientAdapterSpec( + op_name="rms_norm_gated", + chain_node="rms_norm_gated", + op_class="reduction", + spec_name="rms_norm_gated", + tensors=(_DX, _DWEIGHT, _DGATE), + requirement="required", + source_files=("rl_engine/backends/cuda/norm/rmsnorm.py", "csrc/cuda/norm/rmsnorm.cu"), + model_id="Qwen/Qwen3-Next-80B-A3B-Instruct", + ), "qk_norm": GradientAdapterSpec( op_name="qk_norm", chain_node="qk_norm", @@ -291,18 +312,27 @@ def get_adapter(op_name: str) -> GradientAdapterSpec: raise KeyError(f"unknown gradient adapter {op_name!r}") from exc -def required_gradient_adapters() -> tuple[GradientAdapterSpec, ...]: +def required_gradient_adapters( + manifest: WS1Manifest | None = None, +) -> tuple[GradientAdapterSpec, ...]: + selected = manifest or load_manifest() + model_id = selected.model_identity["model_id"] + subset = selected.raw.get("scope") == "qwen3_next_norm_operators" return tuple( spec for spec in GRADIENT_ADAPTERS.values() if spec.requirement in ("required", "layout_supported") + and spec.model_id in (None, model_id) + and (not subset or spec.model_id == model_id) ) -def required_forward_adapters() -> tuple[GradientAdapterSpec, ...]: +def required_forward_adapters( + manifest: WS1Manifest | None = None, +) -> tuple[GradientAdapterSpec, ...]: """Same enumerable WS1 ops as C4; C3 reuses the registry, not a second list.""" - return required_gradient_adapters() + return required_gradient_adapters(manifest) @dataclass(frozen=True) @@ -620,9 +650,9 @@ def _row_parameters( head_dim: int = 16, ) -> dict[str, torch.Tensor]: """Config-independent trainable parameters, built in the execution dtype.""" - if op_name == "rms_norm": + if op_name in {"rms_norm", "qwen3_next_rms_norm"}: return {"weight": _shared_parameter((hidden,), device=device, dtype=dtype, offset=1)} - if op_name == "qk_norm": + if op_name in {"qk_norm", "rms_norm_gated"}: return {"weight": _shared_parameter((head_dim,), device=device, dtype=dtype, offset=1)} if op_name == "det_gemm": return {"b": _shared_parameter((hidden, hidden), device=device, dtype=dtype, offset=2)} @@ -663,12 +693,19 @@ def _row_inputs( """ n = len(keys) leading = (n,) - if op_name == "rms_norm": + if op_name in {"rms_norm", "qwen3_next_rms_norm"}: return { "x": _stack_rows(keys, leading, (hidden,), device=device, dtype=dtype), "weight": params["weight"], "eps": 1.0e-6, } + if op_name == "rms_norm_gated": + return { + "x": _stack_rows(keys, leading, (head_dim,), device=device, dtype=dtype), + "gate": _stack_rows(keys, leading, (head_dim,), device=device, dtype=dtype, offset=7), + "weight": params["weight"], + "eps": 1.0e-6, + } if op_name == "qk_norm": return { "x": _stack_rows(keys, leading, (head_dim,), device=device, dtype=dtype), @@ -1228,6 +1265,15 @@ def resolve_profile_candidate( manifest: WS1Manifest | None = None, ) -> dict[str, Any]: m = manifest if manifest is not None else load_manifest() + subset = m.raw.get("scope") == "qwen3_next_norm_operators" + if adapter.model_id not in (None, m.model_identity["model_id"]) or ( + subset and adapter.model_id != m.model_identity["model_id"] + ): + return { + "status": "absent_not_required", + "expected_backend_id": None, + "candidate_path": None, + } if adapter.requirement == "absent_not_required": return { "status": "absent_not_required", diff --git a/rl_engine/validation/operators/operator_inputs.py b/rl_engine/validation/operators/operator_inputs.py index ca3b7c120..1cd03d31f 100644 --- a/rl_engine/validation/operators/operator_inputs.py +++ b/rl_engine/validation/operators/operator_inputs.py @@ -26,6 +26,8 @@ def make_operator_inputs( ) -> dict[str, Any]: builders = { "rms_norm": _make_rms_norm_inputs, + "rms_norm_gated": _make_rms_norm_gated_inputs, + "qwen3_next_rms_norm": _make_rms_norm_inputs, "qk_norm": _make_qk_norm_inputs, "pack": _make_pack_inputs, "matmul": _make_matmul_inputs, @@ -54,6 +56,8 @@ def operator_shape_name(op_name: str, args: argparse.Namespace) -> str: vocab = _arg_int(args, "vocab", DEFAULT_VOCAB) names = { "rms_norm": f"{batch}x{seq}x{_normalized_dim(args)}", + "rms_norm_gated": f"{batch}x{seq}x{_arg_int(args, 'head_dim', DEFAULT_HEAD_DIM)}", + "qwen3_next_rms_norm": f"{batch}x{seq}x{_normalized_dim(args)}", "qk_norm": f"{batch}x{seq}x{_arg_int(args, 'n_heads', DEFAULT_N_HEADS)}x" f"{_arg_int(args, 'head_dim', DEFAULT_HEAD_DIM)}", "pack": f"{batch}x{seq}x{_normalized_dim(args)}", @@ -91,6 +95,25 @@ def _make_rms_norm_inputs( } +def _make_rms_norm_gated_inputs( + args: argparse.Namespace, dtype: torch.dtype, device: torch.device +) -> dict[str, Any]: + """Gated RMSNorm inputs at the GDN block's width. + + The gated norm normalizes over ``linear_value_head_dim`` (128 for + Qwen3-Next), not ``hidden_size``, so it follows ``head_dim`` rather than + ``normalized_dim``. + """ + batch, seq = _batch_seq(args) + head_dim = _arg_int(args, "head_dim", DEFAULT_HEAD_DIM) + return { + "x": _floating_tensor((batch, seq, head_dim), args, dtype, device, offset=0), + "weight": _floating_tensor((head_dim,), args, dtype, device, offset=1), + "gate": _floating_tensor((batch, seq, head_dim), args, dtype, device, offset=2), + "eps": _arg_float(args, "eps", DEFAULT_RMS_EPS), + } + + def _make_qk_norm_inputs( args: argparse.Namespace, dtype: torch.dtype, device: torch.device ) -> dict[str, Any]: diff --git a/rl_engine/validation/operators/operator_specs.py b/rl_engine/validation/operators/operator_specs.py index 2323f4787..7f17c369d 100644 --- a/rl_engine/validation/operators/operator_specs.py +++ b/rl_engine/validation/operators/operator_specs.py @@ -46,6 +46,30 @@ def _load_object(path: str) -> Any: }, grad_input_names=("x", "weight"), ), + "rms_norm_gated": OperatorSpec( + name="rms_norm_gated", + op_class="reduction", + gold_path=("rl_engine.reference.norm.qwen3_next_rms_norm.Qwen3NextRMSNormGatedOp"), + gold_method="forward_fp32", + candidate_paths={ + "pytorch": ("rl_engine.reference.norm.qwen3_next_rms_norm.Qwen3NextRMSNormGatedOp"), + "cuda": ("rl_engine.kernels.ops.cuda.norm.rmsnorm.Qwen3NextRMSNormGatedCudaOp"), + "cuda-sm90": ("rl_engine.kernels.ops.cuda.norm.rmsnorm.Qwen3NextRMSNormGatedCudaOp"), + }, + grad_input_names=("x", "weight", "gate"), + ), + "qwen3_next_rms_norm": OperatorSpec( + name="qwen3_next_rms_norm", + op_class="reduction", + gold_path=("rl_engine.reference.norm.qwen3_next_rms_norm.Qwen3NextRMSNormOp"), + gold_method="forward_fp32", + candidate_paths={ + "pytorch": ("rl_engine.reference.norm.qwen3_next_rms_norm.Qwen3NextRMSNormOp"), + "cuda": "rl_engine.kernels.ops.cuda.norm.rmsnorm.Qwen3NextRMSNormCudaOp", + "cuda-sm90": ("rl_engine.kernels.ops.cuda.norm.rmsnorm.Qwen3NextRMSNormCudaOp"), + }, + grad_input_names=("x", "weight"), + ), "qk_norm": OperatorSpec( name="qk_norm", op_class="reduction", diff --git a/tests/models/qwen3_next/test_qwen3_next_norm.py b/tests/models/qwen3_next/test_qwen3_next_norm.py index 5c9d5feff..77cf3b5c5 100644 --- a/tests/models/qwen3_next/test_qwen3_next_norm.py +++ b/tests/models/qwen3_next/test_qwen3_next_norm.py @@ -347,6 +347,13 @@ def test_gated_shape_mismatch_raises(): op.forward_fp32(x, w, _rand((3, _HEAD_V_DIM), seed=32)) +def test_gated_dtype_mismatch_names_both_dtypes(): + op = Qwen3NextRMSNormGatedOp() + x, w = _rand((2, _HEAD_V_DIM), seed=33), _rand((_HEAD_V_DIM,), seed=34) + with pytest.raises(ValueError, match=r"gate=\(dtype=torch.float64"): + op.forward_fp32(x, w, _rand((2, _HEAD_V_DIM), seed=35).double()) + + # --------------------------------------------------------------------------- # # 6. CUDA kernel: the offset is applied in fp32 inside the kernel # --------------------------------------------------------------------------- # @@ -462,6 +469,279 @@ def test_cuda_zero_centred_backward_is_offset_aware(): assert torch.equal(grads["plain"][1], grads["zero_centred"][1]) # dw does not +# --------------------------------------------------------------------------- # +# 7. Gated RMSNorm CUDA kernel (the GDN block's norm) +# --------------------------------------------------------------------------- # +_HAS_CUDA_GATED = False +if torch.cuda.is_available(): # pragma: no branch - probe only + try: + from rl_engine.backends.extension import _C as _C_probe + from rl_engine.backends.extension import _EXT_AVAILABLE as _EXT_probe + + _HAS_CUDA_GATED = _EXT_probe and hasattr(_C_probe, "rmsnorm_gated_forward") + except ImportError: # pragma: no cover + _HAS_CUDA_GATED = False + +requires_cuda_gated = pytest.mark.skipif( + not _HAS_CUDA_GATED, reason="gated RMSNorm CUDA extension is not available" +) + + +def _gated_cuda_inputs(seed=0, rows=512, hidden=_HEAD_V_DIM, dtype=torch.bfloat16): + g = torch.Generator(device="cuda").manual_seed(seed) + x = torch.randn(rows, hidden, device="cuda", dtype=dtype, generator=g) + gate = torch.randn(rows, hidden, device="cuda", dtype=dtype, generator=g) + weight = torch.randn(hidden, device="cuda", dtype=dtype, generator=g) + return x, weight, gate + + +@requires_cuda_gated +def test_cuda_gated_matches_golden_within_contract(): + from rl_engine.backends.cuda.norm.rmsnorm import Qwen3NextRMSNormGatedCudaOp + + x, w, gate = _gated_cuda_inputs() + got = Qwen3NextRMSNormGatedCudaOp().forward(x, w, gate, eps=_EPS) + ref = Qwen3NextRMSNormGatedOp().forward_fp32(x, w, gate, eps=_EPS) + torch.testing.assert_close(got.float(), ref, **_forward_tol(torch.bfloat16)) + + +@requires_cuda_gated +@pytest.mark.parametrize("offset", [0.0, 1.0]) +@pytest.mark.parametrize("activation", [0, 1], ids=["silu", "sigmoid"]) +@pytest.mark.parametrize("hidden", [_HEAD_V_DIM, 2048, 5120]) +@pytest.mark.parametrize( + "dtype", [torch.float32, torch.float16, torch.bfloat16], ids=["fp32", "fp16", "bf16"] +) +def test_cuda_gated_rstd_is_bitwise_identical_to_plain_kernel(dtype, hidden, activation, offset): + """The gate must not perturb the normalization statistic. + + Same x, same rstd, bitwise -- otherwise the gate has leaked into the + reduction and the two kernels no longer share a contract. Both kernels + launch with ``choose_threads(H)``: H=128 runs 128 threads with one column + each, 2048 and 5120 run 512 threads with 4 and 10 serial columns each. + Each activation is its own template instantiation, and the offset is + applied only after the statistic, so neither it nor the gate may move it. + """ + from rl_engine.backends.extension import _C + + x, w, gate = _gated_cuda_inputs(hidden=hidden, dtype=dtype) + _, rstd_gated = _C.rmsnorm_gated_forward(x, w, gate, _EPS, offset, activation) + _, rstd_plain = _C.rmsnorm_forward(x, w, _EPS, offset) + assert torch.equal(rstd_gated, rstd_plain) + + +@requires_cuda_gated +@pytest.mark.parametrize("batch", [1, 2, 8, 16, 32, 48, 64, 512]) +def test_cuda_gated_batch_invariance(batch): + """L1 across the RFC #428 concurrency axis, bitwise.""" + from rl_engine.backends.cuda.norm.rmsnorm import Qwen3NextRMSNormGatedCudaOp + + op = Qwen3NextRMSNormGatedCudaOp() + x, w, gate = _gated_cuda_inputs() + full = op.forward(x, w, gate, eps=_EPS) + assert torch.equal(op.forward(x[:batch], w, gate[:batch], eps=_EPS), full[:batch]) + + +@requires_cuda_gated +def test_cuda_gated_zero_gate_zeroes_output(): + from rl_engine.backends.cuda.norm.rmsnorm import Qwen3NextRMSNormGatedCudaOp + + x, w, _ = _gated_cuda_inputs() + out = Qwen3NextRMSNormGatedCudaOp().forward(x, w, torch.zeros_like(x), eps=_EPS) + assert torch.equal(out, torch.zeros_like(out)) + + +@requires_cuda_gated +def test_cuda_gated_unit_weight_is_plain_norm_times_silu(): + from rl_engine.backends.cuda.norm.rmsnorm import Qwen3NextRMSNormGatedCudaOp, rmsnorm_cuda + + x, _, gate = _gated_cuda_inputs() + ones = torch.ones(_HEAD_V_DIM, device="cuda", dtype=x.dtype) + got = Qwen3NextRMSNormGatedCudaOp().forward(x, ones, gate, eps=_EPS).float() + ref = rmsnorm_cuda(x, ones, eps=_EPS).float() * F.silu(gate.float()) + torch.testing.assert_close(got, ref, **_forward_tol(torch.bfloat16)) + + +@requires_cuda_gated +def test_cuda_gated_sigmoid_activation(): + from rl_engine.backends.cuda.norm.rmsnorm import rmsnorm_cuda, rmsnorm_gated_cuda + + x, w, gate = _gated_cuda_inputs() + got = rmsnorm_gated_cuda(x, w, gate, eps=_EPS, activation="sigmoid").float() + ref = rmsnorm_cuda(x, w, eps=_EPS).float() * torch.sigmoid(gate.float()) + torch.testing.assert_close(got, ref, **_forward_tol(torch.bfloat16)) + + +@requires_cuda_gated +def test_cuda_gated_backward_matches_autograd_golden(): + """dx, dweight and dgate against the fp32 reference's autograd.""" + from rl_engine.backends.cuda.norm.rmsnorm import rmsnorm_gated_cuda + + x, w, gate = _gated_cuda_inputs() + x, w, gate = x.float(), w.float(), gate.float() + dy = torch.randn_like(x) + + got = [t.clone().requires_grad_(True) for t in (x, w, gate)] + rmsnorm_gated_cuda(*got, eps=_EPS).backward(dy) + ref = [t.clone().requires_grad_(True) for t in (x, w, gate)] + Qwen3NextRMSNormGatedOp().forward_fp32(*ref, eps=_EPS).backward(dy) + + for name, a, b in zip(("dx", "dweight", "dgate"), got, ref): + torch.testing.assert_close(a.grad, b.grad, atol=1e-4, rtol=1e-4, msg=name) + + +@requires_cuda_gated +@pytest.mark.parametrize("batch", [1, 8, 64, 512]) +def test_cuda_gated_dweight_is_batch_invariant(batch): + """A row's contribution to dweight must not depend on the batch it sits in. + + The ascending-row fp32 left fold makes dweight over the first n rows exactly + the fold of those n row-contributions -- so the n-row run must reproduce the + 512-row run's prefix, bitwise. + """ + from rl_engine.backends.cuda.norm.rmsnorm import rmsnorm_gated_cuda + from rl_engine.backends.extension import _C + from rl_engine.ops.autograd.vjp_fp32 import reduce_rows_fp32 + + x, w, gate = _gated_cuda_inputs() + x, w, gate = x.float(), w.float(), gate.float() + dy = torch.randn_like(x) + + wg = w.clone().requires_grad_(True) + rmsnorm_gated_cuda(x[:batch], wg, gate[:batch], eps=_EPS).backward(dy[:batch]) + + # rstd from the full-batch forward: the row statistic is batch-invariant, so + # its prefix is what the n-row run must have seen. Recomputing it with + # mean(-1) instead would compare against a different reduction. + _, rstd_full = _C.rmsnorm_gated_forward(x, w, gate, _EPS, 0.0, 0) + rows = dy * F.silu(gate) * x * rstd_full.unsqueeze(-1) + assert torch.equal(wg.grad, reduce_rows_fp32(rows[:batch])) + + +@requires_cuda_gated +def test_cuda_gated_weight_offset_is_applied_in_fp32(): + """The gated kernel carries the same offset contract as the plain one.""" + from rl_engine.backends.cuda.norm.rmsnorm import rmsnorm_gated_cuda + + x, w, gate = _gated_cuda_inputs() + offset = rmsnorm_gated_cuda(x, w, gate, eps=_EPS, weight_offset=1.0) + explicit = rmsnorm_gated_cuda(x, (1.0 + w.float()), gate, eps=_EPS) + assert torch.equal(offset, explicit) + folded = rmsnorm_gated_cuda(x, (1.0 + w.float()).bfloat16(), gate, eps=_EPS) + assert not torch.equal(offset, folded) + + +@requires_cuda_gated +def test_cuda_gated_backward_is_offset_aware(): + """dx and dgate must see (1 + w); dw is offset-independent since d/dw (1+w) == d/dw w.""" + from rl_engine.backends.cuda.norm.rmsnorm import rmsnorm_gated_cuda + + x, w, gate = (t.float() for t in _gated_cuda_inputs(rows=128)) + dy = torch.randn_like(x) + grads = {} + for name, weight, off in (("offset", w, 1.0), ("explicit", 1.0 + w, 0.0), ("plain", w, 0.0)): + xg, gg = x.clone().requires_grad_(True), gate.clone().requires_grad_(True) + wg = weight.clone().requires_grad_(True) + rmsnorm_gated_cuda(xg, wg, gg, eps=_EPS, weight_offset=off).backward(dy.clone()) + grads[name] = (xg.grad, gg.grad, wg.grad) + for got, want in zip(grads["offset"], grads["explicit"], strict=True): + assert torch.equal(got, want) + assert not torch.equal(grads["offset"][0], grads["plain"][0]) # dx differs + assert not torch.equal(grads["offset"][1], grads["plain"][1]) # dgate differs + assert torch.equal(grads["offset"][2], grads["plain"][2]) # dw does not + + +@requires_cuda_gated +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("activation", ["silu", "sigmoid"]) +@pytest.mark.parametrize("shape", [(512,), (2, 256)]) +def test_cuda_gated_parameter_vjp_contributions_match_the_fold(dtype, activation, shape): + """Compare the harness hook with actual autograd, including its CUDA rstd and N-D x.""" + from rl_engine.backends.cuda.norm.rmsnorm import Qwen3NextRMSNormGatedCudaOp, _fold_dweight_rows + + x, w, gate = (t.to(dtype) for t in _gated_cuda_inputs(rows=512)) + x, gate = x.reshape(*shape, -1), gate.reshape(*shape, -1) + w = w.float().requires_grad_() + dy = torch.randn_like(x) + op = Qwen3NextRMSNormGatedCudaOp(activation=activation) + op.forward(x, w, gate, eps=_EPS).backward(dy) + rows = op.parameter_vjp_contributions_fp32(x=x, weight=w, gate=gate, grad_output=dy, eps=_EPS)[ + "weight" + ] + assert torch.equal(_fold_dweight_rows(rows.reshape(-1, x.shape[-1]), torch.float32), w.grad) + + +@requires_cuda_gated +def test_cuda_gated_wrapper_rejects_equal_numel_wrong_shape(): + from rl_engine.backends.cuda.norm.rmsnorm import Qwen3NextRMSNormGatedCudaOp + + x, w, gate = _gated_cuda_inputs(rows=6) + x = x.reshape(2, 3, -1) + with pytest.raises(ValueError, match="gate must match x"): + Qwen3NextRMSNormGatedCudaOp().forward(x, w, gate) + + +@requires_cuda_gated +def test_cuda_gated_sigmoid_backward_uses_the_sigmoid_derivative(): + """The activation gradient has two branches; only silu was covered.""" + from rl_engine.backends.cuda.norm.rmsnorm import rmsnorm_gated_cuda + + x, w, gate = _gated_cuda_inputs(rows=64) + x, w, gate = x.float(), w.float(), gate.float() + dy = torch.randn_like(x) + + got = [t.clone().requires_grad_(True) for t in (x, w, gate)] + rmsnorm_gated_cuda(*got, eps=_EPS, activation="sigmoid").backward(dy) + + ref = [t.clone().requires_grad_(True) for t in (x, w, gate)] + rstd = torch.rsqrt(ref[0].square().mean(dim=-1) + _EPS) + out = (ref[0] * rstd.unsqueeze(-1) * ref[1]) * torch.sigmoid(ref[2]) + out.backward(dy) + for name, a, b in zip(("dx", "dweight", "dgate"), got, ref): + torch.testing.assert_close(a.grad, b.grad, atol=1e-4, rtol=1e-4, msg=name) + + +@requires_cuda_gated +def test_cuda_gated_rejects_bad_activation(): + from rl_engine.backends.cuda.norm.rmsnorm import rmsnorm_gated_cuda + + x, w, gate = _gated_cuda_inputs(rows=8) + with pytest.raises(ValueError, match="activation must be one of"): + rmsnorm_gated_cuda(x, w, gate, eps=_EPS, activation="gelu") + + +def test_gated_cuda_op_rejects_bad_activation_at_construction(): + """The activation is validated before the extension check, so this runs on CPU.""" + from rl_engine.backends.cuda.norm.rmsnorm import Qwen3NextRMSNormGatedCudaOp + + with pytest.raises(ValueError, match="activation must be one of"): + Qwen3NextRMSNormGatedCudaOp(activation="gelu") + + +@requires_cuda_gated +def test_cuda_gated_swish_is_an_alias_for_silu(): + """vLLM maps output_gate_type "swish" to "silu"; the alias must not select sigmoid.""" + from rl_engine.backends.cuda.norm.rmsnorm import Qwen3NextRMSNormGatedCudaOp + + x, w, gate = _gated_cuda_inputs(rows=64) + swish = Qwen3NextRMSNormGatedCudaOp(activation="swish").forward(x, w, gate, eps=_EPS) + silu = Qwen3NextRMSNormGatedCudaOp(activation="silu").forward(x, w, gate, eps=_EPS) + sigmoid = Qwen3NextRMSNormGatedCudaOp(activation="sigmoid").forward(x, w, gate, eps=_EPS) + assert torch.equal(swish, silu) + assert not torch.equal(swish, sigmoid) + + +@requires_cuda_gated +def test_cuda_gated_rejects_mismatched_gate(): + from rl_engine.backends.cuda.norm.rmsnorm import rmsnorm_gated_cuda + + x, w, gate = _gated_cuda_inputs(rows=8) + with pytest.raises((AssertionError, RuntimeError)): + rmsnorm_gated_cuda(x, w, gate[:4], eps=_EPS) + with pytest.raises((AssertionError, RuntimeError)): + rmsnorm_gated_cuda(x, w, gate.float(), eps=_EPS) + + # --------------------------------------------------------------------------- # # 8. `__call__` is the documented entry point and must agree with `forward` # --------------------------------------------------------------------------- # @@ -553,3 +833,110 @@ def test_zero_centred_cuda_constructor_rejects_missing_extension(monkeypatch): monkeypatch.setattr(rmsnorm, "_C", None) with pytest.raises(RuntimeError, match="requires the compiled"): rmsnorm.Qwen3NextRMSNormCudaOp() + + +def test_gated_cuda_constructor_rejects_stale_api_version(monkeypatch): + """The gated op needs the same RMSNorm API version as the plain op.""" + from types import SimpleNamespace + + from rl_engine.backends.cuda.norm import rmsnorm + + stale = SimpleNamespace( + rmsnorm_gated_forward=object(), rmsnorm_gated_backward_dx=object(), rmsnorm_api_version=1 + ) + monkeypatch.setattr(rmsnorm, "_EXT_AVAILABLE", True) + monkeypatch.setattr(rmsnorm, "_C", stale) + with pytest.raises(RuntimeError, match="API version 2"): + rmsnorm.Qwen3NextRMSNormGatedCudaOp() + del stale.rmsnorm_api_version + with pytest.raises(RuntimeError, match="API version 2"): + rmsnorm.Qwen3NextRMSNormGatedCudaOp() + + +def test_gated_cuda_forward_raises_runtime_error_without_extension(monkeypatch): + """Not an ``assert``: it must survive ``python -O`` and stay a RuntimeError.""" + from rl_engine.backends.cuda.norm import rmsnorm + + monkeypatch.setattr(rmsnorm, "_EXT_AVAILABLE", False) + monkeypatch.setattr(rmsnorm, "_C", None) + x = torch.zeros(2, 4) + with pytest.raises(RuntimeError, match="requires the compiled"): + rmsnorm.rmsnorm_gated_cuda(x, torch.ones(4), torch.zeros(2, 4)) + + +@requires_cuda_gated +@pytest.mark.parametrize( + "fault", ["weight_shape", "weight_dtype", "gate_dtype", "dy_dtype", "rstd_dtype"] +) +def test_gated_extension_rejects_invalid_tensor_contract(fault): + from rl_engine import _C + + x, w, gate = _gated_cuda_inputs(rows=8) + dy = torch.ones_like(x) + rstd = torch.ones(8, device=x.device, dtype=torch.float32) + if fault == "weight_shape": + w = w[:-1] + elif fault == "weight_dtype": + w = w.half() + elif fault == "gate_dtype": + gate = gate.float() + elif fault == "dy_dtype": + dy = dy.float() + else: + rstd = rstd.bfloat16() + with pytest.raises(RuntimeError): + _C.rmsnorm_gated_backward_dx(dy, x, w, gate, rstd) + + +@requires_cuda_gated +def test_gated_extension_handles_empty_batch_and_nondefault_stream(): + from rl_engine import _C + + stream = torch.cuda.Stream() + with torch.cuda.stream(stream): + x, w, gate = _gated_cuda_inputs(rows=8) + actual, rstd = _C.rmsnorm_gated_forward(x, w, gate, _EPS) + dx = _C.rmsnorm_gated_backward_dx(torch.ones_like(x), x, w, gate, rstd) + empty, empty_rstd = _C.rmsnorm_gated_forward(x[:0], w, gate[:0], _EPS) + empty_dx = _C.rmsnorm_gated_backward_dx(x[:0], x[:0], w, gate[:0], empty_rstd) + stream.synchronize() + expected, expected_rstd = _C.rmsnorm_gated_forward(x, w, gate, _EPS) + expected_dx = _C.rmsnorm_gated_backward_dx(torch.ones_like(x), x, w, gate, expected_rstd) + assert torch.equal(actual, expected) + assert torch.equal(dx, expected_dx) + assert empty.shape == empty_dx.shape == (0, x.shape[-1]) + assert empty_rstd.numel() == 0 + + +@requires_cuda_gated +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +def test_cuda_gated_signed_zero_weight_is_preserved(dtype): + from rl_engine.backends.cuda.norm.rmsnorm import rmsnorm_gated_cuda + + x = torch.ones(2, 128, device="cuda", dtype=dtype) + x[1].neg_() + weight = torch.full((128,), -0.0, device="cuda", dtype=dtype) + actual = rmsnorm_gated_cuda(x, weight, torch.ones_like(x)) + expected = x * weight + bits = torch.int32 if dtype == torch.float32 else torch.int16 + assert torch.equal(actual.view(bits), expected.view(bits)) + + +@requires_cuda_gated +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +def test_cuda_gated_dgate_keeps_signed_zero_weight(dtype): + """dgate scales by the weight too, so it must not turn a -0.0 weight into +0.0. + + dgate = dy * x * rstd * w * act'(gate); with dy, rstd, act'(1) > 0 its sign is + the sign of ``x * w``, zeros included. + """ + from rl_engine.backends.cuda.norm.rmsnorm import rmsnorm_gated_cuda + + x = torch.ones(2, 128, device="cuda", dtype=dtype) + x[1].neg_() + weight = torch.full((128,), -0.0, device="cuda", dtype=dtype) + gate = torch.ones_like(x).requires_grad_() + rmsnorm_gated_cuda(x, weight, gate).backward(torch.ones_like(x)) + expected = x * weight + bits = torch.int32 if dtype == torch.float32 else torch.int16 + assert torch.equal(gate.grad.view(bits), expected.view(bits)) diff --git a/tests/models/qwen3_next/test_qwen3_next_norm_reuse_check.py b/tests/models/qwen3_next/test_qwen3_next_norm_reuse_check.py index f77b86bf7..3c4eaa84d 100644 --- a/tests/models/qwen3_next/test_qwen3_next_norm_reuse_check.py +++ b/tests/models/qwen3_next/test_qwen3_next_norm_reuse_check.py @@ -47,3 +47,128 @@ def apply(x, weight, eps, zero_centered_gamma): output.sum().backward() assert x.grad is not None assert weight.grad is not None + + +@pytest.mark.parametrize("op", ["qwen3_next_rms_norm", "rms_norm_gated"]) +def test_required_candidate_failure_writes_no_report(monkeypatch, tmp_path, op): + """A missing primary extension must fail before creating report artifacts.""" + + def unavailable(): + raise RuntimeError("CUDA extension needs rebuilding") + + factory_name = "_c1_candidates" if op == "qwen3_next_rms_norm" else "_gated_candidates" + monkeypatch.setattr(reuse, factory_name, lambda hidden: [("rl_kernel", unavailable)]) + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(reuse, "plot", lambda *args: pytest.fail("must not plot a failed run")) + report = tmp_path / "evidence" / "report.json" + monkeypatch.setattr(sys, "argv", ["reuse", "--op", op, "--out", str(report)]) + + with pytest.raises(RuntimeError, match="Required rl_kernel candidate is unavailable"): + reuse.main() + + assert not report.parent.exists() + + +@pytest.mark.parametrize("op", ["qwen3_next_rms_norm", "rms_norm_gated"]) +def test_optional_candidate_failure_is_reported(monkeypatch, op): + """Optional dependencies may be absent when the primary candidate is usable.""" + + def primary(): + return "rl-kernel", lambda x, w: x, "full" + + def optional(): + raise ImportError("optional provider not installed") + + factory_name = "_c1_candidates" if op == "qwen3_next_rms_norm" else "_gated_candidates" + monkeypatch.setattr( + reuse, factory_name, lambda hidden: [("rl_kernel", primary), ("optional", optional)] + ) + found = reuse.candidates(op) + assert callable(found["rl_kernel"]["fn"]) + assert found["optional"] == {"unavailable": "ImportError: optional provider not installed"} + + +@pytest.mark.parametrize("op", ["qwen3_next_rms_norm", "rms_norm_gated"]) +@pytest.mark.parametrize("defect", [None, "forward", "gradient"]) +def test_full_batch_sub_batches_visit_previously_skipped_rows(monkeypatch, op, defect): + """Catch a singleton-only defect at an odd row missed by the old stride.""" + monkeypatch.setattr(reuse, "DEV", "cpu") + monkeypatch.setattr(reuse, "DT", torch.float32) + monkeypatch.setitem(reuse.SHAPES, op, (1, (8,), 1025, (1, 7, 64), 1025)) + + def inputs(op, seed, n): + tensors = {"x": torch.arange(n, dtype=torch.float32).reshape(n, 1), "w": torch.ones(1)} + if op == "rms_norm_gated": + tensors["z"] = torch.ones(n, 1) + return tensors + + monkeypatch.setattr(reuse, "_inputs", inputs) + + def provider(x, w, z=None): + output = x * w + if z is not None: + output = output * z + if len(x) == 1: + affected = (x == 1001).float() + if defect == "forward": + output = output + affected + elif defect == "gradient": + output = output + (x - x.detach()) * affected + return output + + result = reuse.batch_invariance(op, {"fn": provider, "backward": "x-only"}, quick=False) + check = result["full_vs_sub_batches"] + assert check["coverage"] == "every_row_per_sub_batch_size" + assert check["sub_batches"] == 2 * (1025 + 147 + 17) + assert check["fwd_differ"] == (2 if defect == "forward" else 0) + assert check["rowgrad_differ"] == (2 if defect == "gradient" else 0) + assert result["batch_invariant"] is (defect is None) + + +@pytest.mark.parametrize("op", ["qwen3_next_rms_norm", "rms_norm_gated"]) +def test_swapped_operator_c4_uses_the_library_dweight(monkeypatch, op): + """C4 must judge the swapped library's dweight, not rl-kernel's row contributions.""" + from rl_engine.backends.cuda.norm import rmsnorm as R + from rl_engine.config.workload import load_manifest, singleton_aggregate_plan + from rl_engine.validation.operators.forward_invariance import build_config_matrix + from rl_engine.validation.operators.gradient_adapters import make_gradient_runner + + # Constructing the subclass needs the compiled extension only for the hook being removed. + monkeypatch.setattr(R, "_require_cuda_rmsnorm", lambda: None) + monkeypatch.setattr(R, "_require_cuda_gated", lambda: None) + library_dweights = [] + + def library(x, *rest): + weight = rest[-1] + weight.register_hook(lambda grad: library_dweights.append(grad.float())) + out = x * (1.0 + weight) + return out * torch.sigmoid(rest[0]) if len(rest) == 2 else out + + swapped = reuse._swapped_operator(op, library) + assert not callable(getattr(swapped, "parameter_vjp_contributions_fp32", None)) + + configs = {config.config_id: config for config in build_config_matrix(load_manifest())} + canonical = configs["BN/full"] + run = make_gradient_runner( + op, + swapped, + device=torch.device("cpu"), + dtype=torch.float32, + reference=True, + hidden=8, + head_dim=8, + ) + for config_id in ("BN/full", "BN/chunked", "B1-singleton_aggregate/full/s0"): + library_dweights.clear() + grads = run( + configs[config_id], + active_token_denominator=canonical.logical_batch.active_token_count(), + loss_reduction="sum_over_active_tokens_then_optional_mean_by_active_count", + aggregation_order=singleton_aggregate_plan(canonical.logical_batch).aggregation_order, + ) + assert "__parameter_contributions__" not in grads + assert library_dweights + expected = library_dweights[0] + for grad in library_dweights[1:]: + expected = expected + grad + assert torch.equal(grads["dweight"], expected) diff --git a/tests/models/qwen3_next/test_qwen3_next_workload.py b/tests/models/qwen3_next/test_qwen3_next_workload.py new file mode 100644 index 000000000..6ae23dc4b --- /dev/null +++ b/tests/models/qwen3_next/test_qwen3_next_workload.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""The norm workload cannot stand in for Dense or full-checkpoint evidence.""" + +import copy +import json +from pathlib import Path + +import pytest + +from rl_engine.config.workload import WorkloadError, load_manifest, manifest_identity_hash +from rl_engine.validation.models.qwen3_next_workload import validate_norm_manifest +from rl_engine.validation.operators.gradient_adapters import get_adapter, resolve_profile_candidate + +MANIFEST = ( + Path(__file__).resolve().parents[3] + / "rl_engine/validation/models/qwen3_next_norm_manifest.json" +) + + +def test_norm_manifest_pins_real_architecture_and_separate_scope(): + manifest = load_manifest(MANIFEST) + assert manifest.model_identity["config_fingerprint"]["hidden_size"] == 2048 + assert manifest.raw["full_model_evidence"] is False + for name, gradients in ( + ("qwen3_next_rms_norm", ("dx", "dweight")), + ("rms_norm_gated", ("dx", "dweight", "dgate")), + ): + adapter = get_adapter(name) + assert tuple(t.name for t in adapter.tensors) == gradients + assert resolve_profile_candidate(adapter, "cuda_bf16", manifest)["status"] == "declared" + assert ( + resolve_profile_candidate(adapter, "triton_cuda_bf16", manifest)["status"] + == "missing_required" + ) + assert ( + resolve_profile_candidate(adapter, "cuda_bf16", load_manifest())["status"] + == "absent_not_required" + ) + + +@pytest.mark.parametrize( + "fault", + ["architecture", "full_model", "revision", "shape", "binding", "chain_dtype", "duplicate_op"], +) +def test_norm_manifest_rejects_false_evidence_even_with_regenerated_hash(fault): + raw = copy.deepcopy(load_manifest(MANIFEST).raw) + if fault == "architecture": + raw["model_identity"]["config_fingerprint"]["hidden_size"] = 4096 + elif fault == "full_model": + raw["full_model_evidence"] = True + elif fault == "revision": + raw["model_identity"]["revision"] = "main" + elif fault == "chain_dtype": + raw["chain_semantics"]["execution_dtype"] = "float16" + elif fault == "duplicate_op": + ops = raw["capabilities"]["required_chain_ops"] + ops.append(dict(ops[0])) + elif fault == "shape": + raw["representative_cases"][0]["hidden"] = 64 + else: + raw["representative_cases"][0]["fixture_id"] = "wrong_fixture" + raw["fixture_identity_sha256"] = manifest_identity_hash(raw) + with pytest.raises(WorkloadError): + validate_norm_manifest(raw) + + +@pytest.mark.parametrize( + "missing", + [ + "model_identity", + "stochastic_policy", + "primary_matrix", + "fixtures", + "capabilities", + "chain_semantics", + "seed", + "workload_id", + "version", + "provenance_boundary", + ], +) +def test_norm_manifest_missing_block_raises_workload_error(missing): + """A required block removed from a norm manifest is a WorkloadError, not a KeyError.""" + raw = copy.deepcopy(load_manifest(MANIFEST).raw) + del raw[missing] + with pytest.raises(WorkloadError, match="malformed"): + validate_norm_manifest(raw) + + +@pytest.mark.parametrize( + "op,hidden,head_dim", + [ + ("qwen3_next_rms_norm", 64, 128), + ("qwen3_next_rms_norm", 2048, 256), + ("rms_norm_gated", 2048, 256), + ("rms_norm_gated", 128, 128), + ("rms_norm", 2048, 128), + ], +) +def test_norm_dimension_gate_rejects_other_widths(op, hidden, head_dim): + """Both gates run only at the checkpoint widths: hidden 2048, gated head_dim 128.""" + from rl_engine.validation.models.qwen3_next_workload import validate_norm_dimensions + + raw = load_manifest(MANIFEST).raw + with pytest.raises(WorkloadError, match="hidden 2048"): + validate_norm_dimensions(raw, op, hidden, head_dim) + + +@pytest.mark.parametrize("op", ["qwen3_next_rms_norm", "rms_norm_gated"]) +def test_norm_dimension_gate_accepts_checkpoint_widths(op): + from rl_engine.validation.models.qwen3_next_workload import validate_norm_dimensions + + validate_norm_dimensions(load_manifest(MANIFEST).raw, op, 2048, 128) + + +def test_norm_adapters_are_required_by_the_norm_manifest_only(): + """The Qwen3-Next norms are required where the manifest declares them, not in Dense.""" + from rl_engine.validation.operators.gradient_adapters import required_gradient_adapters + + norm = {spec.op_name for spec in required_gradient_adapters(load_manifest(MANIFEST))} + assert norm == {"qwen3_next_rms_norm", "rms_norm_gated"} + dense = {spec.op_name for spec in required_gradient_adapters(load_manifest())} + assert dense.isdisjoint(norm) + + +@pytest.mark.parametrize( + "source,scope", + [ + (MANIFEST, "qwen3-next-norm-operators"), + (None, "qwen3_next_norm_operator"), + (MANIFEST, None), + ], +) +def test_unknown_manifest_scope_is_rejected(tmp_path, source, scope): + """A typo'd scope is a scope error, not a Dense fingerprint mismatch or a silent Dense.""" + raw = copy.deepcopy(load_manifest(source).raw) + raw["scope"] = scope + path = tmp_path / "manifest.json" + path.write_text(json.dumps(raw)) + with pytest.raises(WorkloadError, match="unknown manifest scope"): + load_manifest(path) diff --git a/tests/runtime/test_dispatch.py b/tests/runtime/test_dispatch.py index 3c47bbbcf..6517d3309 100644 --- a/tests/runtime/test_dispatch.py +++ b/tests/runtime/test_dispatch.py @@ -4,6 +4,7 @@ import sys from types import ModuleType +import pytest import torch import rl_engine.platforms.device as device_module @@ -258,3 +259,67 @@ def test_executor_flow(): print("\n All infrastructure tests passed!") except Exception as e: print(f"\n Test failed with error: {e}") + + +_QWEN3_NEXT_NORMS = [ + pytest.param( + "rms_norm_gated", + OpBackend.CUDA_RMS_NORM_GATED, + OpBackend.PYTORCH_NATIVE_RMS_NORM_GATED, + "Qwen3NextRMSNormGatedCudaOp", + "Qwen3NextRMSNormGatedOp", + id="rms_norm_gated", + ), + pytest.param( + "qwen3_next_rms_norm", + OpBackend.CUDA_QWEN3_NEXT_RMS_NORM, + OpBackend.PYTORCH_NATIVE_QWEN3_NEXT_RMS_NORM, + "Qwen3NextRMSNormCudaOp", + "Qwen3NextRMSNormOp", + id="qwen3_next_rms_norm", + ), +] + + +@pytest.mark.parametrize("op_type, cuda_backend, ref_backend, cuda_cls, ref_cls", _QWEN3_NEXT_NORMS) +def test_qwen3_next_norm_priority_is_cuda_first_with_pytorch_fallback( + op_type, cuda_backend, ref_backend, cuda_cls, ref_cls +): + """Both Qwen3-Next norms have a CUDA kernel; every other platform falls back. + + No Triton, ROCm or Ascend kernel exists yet, so those platforms must resolve + to the PyTorch reference rather than to nothing -- an operator missing from a + priority map falls through to ``OpBackend.PYTORCH_NATIVE``, which is the + logprob op, not a norm. + """ + registry = KernelRegistry() + + assert registry._priority_map["cuda"][op_type] == [cuda_backend, ref_backend] + for platform in ("rocm", "musa", "cpu", "npu"): + assert registry._priority_map[platform][op_type] == [ref_backend], platform + + +@pytest.mark.skipif(torch.version.hip is not None, reason="a cuda device maps to rocm on HIP") +@pytest.mark.parametrize("op_type, cuda_backend, ref_backend, cuda_cls, ref_cls", _QWEN3_NEXT_NORMS) +def test_qwen3_next_norm_cuda_backend_absence_falls_through_to_reference( + monkeypatch, op_type, cuda_backend, ref_backend, cuda_cls, ref_cls +): + """Without the compiled symbols the CUDA backend must be skipped, not returned. + + The zero-centred op inherits its check from ``RMSNormCudaOp.__init__``. The + lookup names a CUDA device so the CUDA-first list is walked even on a + CPU-only host; resolving for the host's own platform would reach the + reference through the CPU list without ever trying the CUDA backend. + """ + from rl_engine.backends.cuda.norm import rmsnorm as cuda_rmsnorm + from rl_engine.reference.norm import qwen3_next_rms_norm as reference + + monkeypatch.setattr(cuda_rmsnorm, "_EXT_AVAILABLE", False) + monkeypatch.setattr(cuda_rmsnorm, "_C", None) + with pytest.raises(RuntimeError, match="requires the compiled rl_engine._C extension"): + getattr(cuda_rmsnorm, cuda_cls)() + + registry = KernelRegistry() + resolved = registry.get_op(op_type, device="cuda") + assert type(resolved) is getattr(reference, ref_cls) + assert cuda_backend.name in registry._failed_backends diff --git a/tests/validation/operators/test_ws1_ascend_closeout.py b/tests/validation/operators/test_ws1_ascend_closeout.py index 66c8ae4a6..b9b0fe331 100644 --- a/tests/validation/operators/test_ws1_ascend_closeout.py +++ b/tests/validation/operators/test_ws1_ascend_closeout.py @@ -187,9 +187,15 @@ def test_c2_ascend_cases_pin_real_ascend_kernels_and_sources(): def test_c3_c4_every_required_adapter_resolves_an_ascend_candidate(): manifest = load_manifest() + model_id = manifest.model_identity["model_id"] for name, adapter in GRADIENT_ADAPTERS.items(): if adapter.requirement not in ("required",): continue + # An adapter scoped to another model (the CUDA-only Qwen3-Next norms) + # resolves to `absent_not_required` for this manifest by design; this + # test covers the chain of the model whose manifest it loads. + if adapter.model_id not in (None, model_id): + continue resolved = resolve_profile_candidate(adapter, PROFILE, manifest) assert resolved["status"] == "declared", name assert resolved["expected_backend_id"] == "ascend", name @@ -197,6 +203,22 @@ def test_c3_c4_every_required_adapter_resolves_an_ascend_candidate(): assert candidate_family(str(resolved["expected_backend_id"])) == "ascend" +def test_model_scoped_adapters_are_absent_not_required_for_other_models(): + """The scoping the test above relies on must actually hold. + + Without this, skipping scoped adapters could hide one that silently resolves + to a real candidate for the wrong model. + """ + manifest = load_manifest() + model_id = manifest.model_identity["model_id"] + scoped = [a for a in GRADIENT_ADAPTERS.values() if a.model_id not in (None, model_id)] + assert scoped, "expected at least the Qwen3-Next norm adapters to be model-scoped" + for adapter in scoped: + resolved = resolve_profile_candidate(adapter, PROFILE, manifest) + assert resolved["status"] == "absent_not_required", adapter.op_name + assert resolved["candidate_path"] is None, adapter.op_name + + def test_c4_adapter_status_matrix_has_no_red_ascend_rows(): rows = [r for r in gradient_adapter_status_matrix() if r.backend_profile == PROFILE] assert rows diff --git a/tests/validation/operators/test_ws1_gtest_gpu.py b/tests/validation/operators/test_ws1_gtest_gpu.py index bcaebc2f5..60ac5102c 100644 --- a/tests/validation/operators/test_ws1_gtest_gpu.py +++ b/tests/validation/operators/test_ws1_gtest_gpu.py @@ -36,6 +36,8 @@ def test_all_ws1_single_ops_are_registered(): names = set(operator_names()) assert { "rms_norm", + "rms_norm_gated", + "qwen3_next_rms_norm", "qk_norm", "det_gemm", "attention", diff --git a/tools/validation/models/plot_qwen3_next_norm_evidence.py b/tools/validation/models/plot_qwen3_next_norm_evidence.py index 5067d4d7c..ede305a6f 100644 --- a/tools/validation/models/plot_qwen3_next_norm_evidence.py +++ b/tools/validation/models/plot_qwen3_next_norm_evidence.py @@ -4,7 +4,8 @@ """Plot a report from tools/validation/models/qwen3_next_norm_evidence.py: one 2x2 figure per op. - python tools/validation/models/plot_qwen3_next_norm_evidence.py report.json +python -m pip install matplotlib # optional plotting dependency +python tools/validation/models/plot_qwen3_next_norm_evidence.py report.json Writes figure[-].png beside the report. """ diff --git a/tools/validation/models/qwen3_next_norm_evidence.py b/tools/validation/models/qwen3_next_norm_evidence.py index 34018c013..aaeba82c3 100644 --- a/tools/validation/models/qwen3_next_norm_evidence.py +++ b/tools/validation/models/qwen3_next_norm_evidence.py @@ -25,24 +25,30 @@ from typing import Any, Callable import torch +import torch.nn.functional as F REPO_ROOT = Path(__file__).resolve().parents[3] sys.path.insert(0, str(REPO_ROOT)) -from rl_engine.backends.cuda.norm.rmsnorm import Qwen3NextRMSNormCudaOp # noqa: E402 -from rl_engine.reference.norm.qwen3_next_rms_norm import Qwen3NextRMSNormOp # noqa: E402 +from rl_engine.backends.cuda.norm.rmsnorm import ( # noqa: E402 + Qwen3NextRMSNormCudaOp, + Qwen3NextRMSNormGatedCudaOp, +) +from rl_engine.reference.norm.qwen3_next_rms_norm import ( # noqa: E402 + Qwen3NextRMSNormGatedOp, + Qwen3NextRMSNormOp, +) EPS = 1e-6 -HIDDEN = 2048 # Qwen3-Next hidden size (decoder and final norms) -TIMED_ROWS = (1024, 4096, 16384, 65536) # --------------------------------------------------------------------------- # -# Candidates: name -> (forward fn(x, w) -> y, has_backward) +# Ops. Each candidate is fn(*row_inputs, weight) -> y, plus whether it has a +# backward. Row inputs are sliced together for the row-invariance check. # --------------------------------------------------------------------------- # -def zero_centred_candidates() -> dict[str, dict[str, Any]]: +def zero_centred_candidates(hidden: int) -> dict[str, dict[str, Any]]: cands: dict[str, dict[str, Any]] = { "rl-kernel CUDA": {"fn": Qwen3NextRMSNormCudaOp(), "backward": True}, "rl-kernel PyTorch reference": {"fn": Qwen3NextRMSNormOp(), "backward": True}, @@ -50,7 +56,7 @@ def zero_centred_candidates() -> dict[str, dict[str, Any]]: try: from transformers.models.qwen3_next.modeling_qwen3_next import Qwen3NextRMSNorm - module = Qwen3NextRMSNorm(HIDDEN, eps=EPS).cuda() + module = Qwen3NextRMSNorm(hidden, eps=EPS).cuda() def hf(x, w, module=module): return torch.func.functional_call(module, {"weight": w}, (x,)) @@ -64,7 +70,7 @@ def hf(x, w, module=module): with set_current_vllm_config(VllmConfig()): from vllm.model_executor.layers.layernorm import GemmaRMSNorm - gemma = GemmaRMSNorm(HIDDEN, eps=EPS).cuda() + gemma = GemmaRMSNorm(hidden, eps=EPS).cuda() def vllm_fwd(x, w, gemma=gemma): gemma.weight.data = w.detach() @@ -85,46 +91,121 @@ def vllm_fwd(x, w, gemma=gemma): return cands -def zero_centred_golden(x: torch.Tensor, w: torch.Tensor) -> torch.Tensor: +def zero_centred_golden(x, w): x64 = x.double() return x64 * torch.rsqrt(x64.square().mean(-1, keepdim=True) + EPS) * (1.0 + w.double()) +def gated_candidates(hidden: int) -> dict[str, dict[str, Any]]: + cuda_op, ref_op = Qwen3NextRMSNormGatedCudaOp(), Qwen3NextRMSNormGatedOp() + cands: dict[str, dict[str, Any]] = { + "rl-kernel CUDA": {"fn": lambda x, g, w: cuda_op(x, w, g, eps=EPS), "backward": True}, + "rl-kernel PyTorch reference": { + "fn": lambda x, g, w: ref_op(x, w, g, eps=EPS), + "backward": True, + }, + } + try: + from transformers.models.qwen3_next.modeling_qwen3_next import Qwen3NextRMSNormGated + + module = Qwen3NextRMSNormGated(hidden, eps=EPS).cuda() + + def hf(x, g, w, module=module): + return torch.func.functional_call(module, {"weight": w}, (x, g)) + + cands["transformers Qwen3NextRMSNormGated (cast-first)"] = {"fn": hf, "backward": True} + except ImportError: + pass + try: + from vllm.config import VllmConfig, set_current_vllm_config + + with set_current_vllm_config(VllmConfig()): + from vllm.model_executor.layers.layernorm import RMSNormGated + + gated = RMSNormGated(hidden, eps=EPS, norm_before_gate=True).cuda() + + def vllm_fwd(x, g, w, gated=gated): + gated.weight.data = w.detach() + return gated.forward_cuda(x, g) + + cands["vLLM RMSNormGated (forward only)"] = {"fn": vllm_fwd, "backward": False} + except ImportError: + pass + return cands + + +def gated_golden(x, g, w): + """vLLM's convention (the one #468 implements) in FP64: x * rstd * w * silu(gate).""" + + x64 = x.double() + return (x64 * torch.rsqrt(x64.square().mean(-1, keepdim=True) + EPS) * w.double()) * F.silu( + g.double() + ) + + +OPS: dict[str, dict[str, Any]] = { + "zero_centred_rmsnorm": { + "hidden": 2048, # decoder and final norms + "row_inputs": 1, + "weight": lambda h, gen: torch.randn(h, device="cuda", generator=gen) * 0.1, + "candidates": zero_centred_candidates, + "golden": zero_centred_golden, + "timed_rows": (1024, 4096, 16384, 65536), + }, + "gated_rmsnorm": { + "hidden": 128, # GDN value head dim; rows are tokens x heads + "row_inputs": 2, + "weight": lambda h, gen: 1.0 + torch.randn(h, device="cuda", generator=gen) * 0.1, + "candidates": gated_candidates, + "golden": gated_golden, + "timed_rows": (4096, 16384, 65536, 262144), + }, +} + + # --------------------------------------------------------------------------- # # Measurements # --------------------------------------------------------------------------- # -def _inputs(rows: int, seed: int, dtype=torch.bfloat16): +def _inputs(spec, rows: int, seed: int, dtype=torch.bfloat16): g = torch.Generator(device="cuda").manual_seed(seed) - x = (torch.randn(rows, HIDDEN, device="cuda", generator=g) * 2).to(dtype) - w = (torch.randn(HIDDEN, device="cuda", generator=g) * 0.1).to(dtype) - up = torch.randn(rows, HIDDEN, device="cuda", generator=g).to(dtype) - return x, w, up + h = spec["hidden"] + row_inputs = [(torch.randn(rows, h, device="cuda", generator=g) * 2).to(dtype)] + for _ in range(spec["row_inputs"] - 1): + row_inputs.append(torch.randn(rows, h, device="cuda", generator=g).to(dtype)) + w = spec["weight"](h, g).to(dtype) + up = torch.randn(rows, h, device="cuda", generator=g).to(dtype) + return row_inputs, w, up -def _grads(fn, x, w, up, dtype=None): - xl = (x if dtype is None else x.to(dtype)).detach().clone().requires_grad_(True) - wl = (w if dtype is None else w.to(dtype)).detach().clone().requires_grad_(True) - out = fn(xl, wl) +def _grads(fn, row_inputs, w, up, dtype=None): + def leaf(t): + return (t if dtype is None else t.to(dtype)).detach().clone().requires_grad_(True) + + rl, wl = [leaf(t) for t in row_inputs], leaf(w) + out = fn(*rl, wl) out.backward(up if dtype is None else up.to(dtype)) - return out.detach(), xl.grad, wl.grad + return out.detach(), [t.grad for t in rl], wl.grad -def accuracy(cands, golden, rows: int, seed: int) -> dict[str, Any]: - x, w, up = _inputs(rows, seed) - ref_out, ref_dx, ref_dw = _grads(golden, x, w, up, torch.float64) +def accuracy(spec, cands, rows: int, seed: int) -> dict[str, Any]: + row_inputs, w, up = _inputs(spec, rows, seed) + ref_out, ref_drows, ref_dw = _grads(spec["golden"], row_inputs, w, up, torch.float64) result = {} for name, c in cands.items(): entry: dict[str, Any] = {} if c["backward"]: - out, dx, dw = _grads(c["fn"], x, w, up) - for key, got, ref in (("dx", dx, ref_dx), ("dweight", dw, ref_dw)): + out, drows, dw = _grads(c["fn"], row_inputs, w, up) + pairs = [("dx", drows[0], ref_drows[0]), ("dweight", dw, ref_dw)] + if len(drows) > 1: + pairs.append(("dgate", drows[1], ref_drows[1])) + for key, got, ref in pairs: err = (got.double() - ref).abs().max().item() entry[f"{key}_max_abs_over_absmax"] = err / ref.abs().max().item() else: with torch.no_grad(): - out = c["fn"](x, w) + out = c["fn"](*row_inputs, w) err = (out.double() - ref_out).abs() entry["forward_max_abs"] = err.max().item() entry["forward_correctly_rounded_fraction"] = ( @@ -134,27 +215,27 @@ def accuracy(cands, golden, rows: int, seed: int) -> dict[str, Any]: return result -def row_invariance(cands, seeds=(3, 4, 5), rows: int = 4096, step: int = 16) -> dict[str, Any]: +def row_invariance(spec, cands, seeds=(3, 4, 5), rows: int = 4096, step: int = 16): """256 rows computed alone vs the same rows inside a batch, bitwise.""" result = {} for name, c in cands.items(): fwd_bad = dx_bad = checked = 0 for seed in seeds: - x, w, up = _inputs(rows, seed) + row_inputs, w, up = _inputs(spec, rows, seed) if c["backward"]: - full_out, full_dx, _ = _grads(c["fn"], x, w, up) + full_out, full_drows, _ = _grads(c["fn"], row_inputs, w, up) else: with torch.no_grad(): - full_out = c["fn"](x, w) + full_out = c["fn"](*row_inputs, w) for i in range(0, rows, step): - sl = slice(i, i + 1) + part = [t[i : i + 1] for t in row_inputs] if c["backward"]: - out, dx, _ = _grads(c["fn"], x[sl], w, up[sl]) - dx_bad += not torch.equal(dx[0], full_dx[i]) + out, drows, _ = _grads(c["fn"], part, w, up[i : i + 1]) + dx_bad += not all(torch.equal(d[0], fd[i]) for d, fd in zip(drows, full_drows)) else: with torch.no_grad(): - out = c["fn"](x[sl], w) + out = c["fn"](*part, w) fwd_bad += not torch.equal(out[0], full_out[i]) checked += 1 result[name] = { @@ -182,20 +263,19 @@ def _time_us(fn: Callable[[], Any], warmup: int = 10, iters: int = 50) -> float: return statistics.median(samples) -def latency(cands, rows_list=TIMED_ROWS) -> dict[str, Any]: +def latency(spec, cands) -> dict[str, Any]: result: dict[str, Any] = {name: {} for name in cands} - for rows in rows_list: - x, w, up = _inputs(rows, seed=11) + for rows in spec["timed_rows"]: + row_inputs, w, up = _inputs(spec, rows, seed=11) for name, c in cands.items(): row: dict[str, float] = {} with torch.no_grad(): - row["forward_us"] = _time_us(lambda f=c["fn"]: f(x, w)) + row["forward_us"] = _time_us(lambda f=c["fn"]: f(*row_inputs, w)) if c["backward"]: - xl = x.detach().clone().requires_grad_(True) - wl = w.detach().clone().requires_grad_(True) - out = c["fn"](xl, wl) + leaves = [t.detach().clone().requires_grad_(True) for t in [*row_inputs, w]] + out = c["fn"](*leaves) row["backward_us"] = _time_us( - lambda o=out: torch.autograd.grad(o, (xl, wl), up, retain_graph=True) + lambda o=out, lv=leaves: torch.autograd.grad(o, lv, up, retain_graph=True) ) result[name][str(rows)] = row return result @@ -227,19 +307,27 @@ def environment() -> dict[str, Any]: return env -def run_op(name: str, cands, golden) -> dict[str, Any]: +def run_op(name: str, spec) -> dict[str, Any]: + cands = spec["candidates"](spec["hidden"]) print(f"[{name}] candidates: {', '.join(cands)}", flush=True) report = { - "accuracy": {str(r): accuracy(cands, golden, r, seed=r) for r in (257, 4096)}, - "row_invariance": row_invariance(cands), - "latency": latency(cands), + "hidden": spec["hidden"], + "accuracy": {str(r): accuracy(spec, cands, r, seed=r) for r in (257, 4096)}, + "row_invariance": row_invariance(spec, cands), + "latency": latency(spec, cands), } for cand, entry in report["row_invariance"].items(): print(f" BI {cand}: {entry}", flush=True) return report -def build_report(ops: dict[str, tuple[Callable, Callable]]) -> dict[str, Any]: +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--ops", default=",".join(OPS), help="comma list of ops") + args = parser.parse_args() + if not torch.cuda.is_available(): + raise SystemExit("needs a CUDA device") torch.backends.cuda.matmul.allow_tf32 = False report = { "kind": "qwen3_next_norm_evidence", @@ -247,26 +335,10 @@ def build_report(ops: dict[str, tuple[Callable, Callable]]) -> dict[str, Any]: "git_commit": _git("rev-parse", "HEAD") or "unknown", "git_dirty": bool(_git("status", "--porcelain", "--untracked-files=no")), "environment": environment(), - "hidden": HIDDEN, "eps": EPS, "dtype": "bfloat16", - "ops": {}, + "ops": {name: run_op(name, OPS[name]) for name in args.ops.split(",")}, } - for name, (make_cands, golden) in ops.items(): - report["ops"][name] = run_op(name, make_cands(), golden) - return report - - -OPS = {"zero_centred_rmsnorm": (zero_centred_candidates, zero_centred_golden)} - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--out", type=Path, required=True) - args = parser.parse_args() - if not torch.cuda.is_available(): - raise SystemExit("needs a CUDA device") - report = build_report(OPS) args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(json.dumps(report, indent=2) + "\n") print(f"wrote {args.out} (commit {report['git_commit'][:7]}, dirty={report['git_dirty']})") diff --git a/tools/validation/models/qwen3_next_norm_reuse_check.py b/tools/validation/models/qwen3_next_norm_reuse_check.py index 5fd03b53c..266a06d5c 100644 --- a/tools/validation/models/qwen3_next_norm_reuse_check.py +++ b/tools/validation/models/qwen3_next_norm_reuse_check.py @@ -20,8 +20,8 @@ ``tools/validation/operators/check_gradient_invariance.py``) with the norm manifest and the arguments ``ci/scripts/run_ws1_gtest.sh`` uses), unchanged, with the CUDA candidate replaced by a subclass of the rl-kernel op whose forward and backward call the other library. The subclass - keeps the op's ``parameter_vjp_contributions_fp32``, so the singleton-aggregate check - compares that library's ``dweight`` with the same FP32 row contributions. + drops the op's ``parameter_vjp_contributions_fp32``, so every C4 ``dweight`` comparison, + singleton-aggregate included, uses the ``dweight`` that library's own backward returns. python tools/validation/models/qwen3_next_norm_reuse_check.py --op qwen3_next_rms_norm \\ --out docs/usage/evidence/qwen3-next-norm-reuse-b200/qwen3_next_rms_norm.json \\ @@ -259,6 +259,8 @@ def candidates(op: str) -> dict[str, dict[str, Any]]: source, fn, backward = factory() out[name] = {"source": source, "fn": fn, "backward": backward} except Exception as exc: # noqa: BLE001 - optional library missing or unusable + if name == "rl_kernel": + raise RuntimeError(f"Required rl_kernel candidate is unavailable: {exc}") from exc out[name] = {"unavailable": f"{type(exc).__name__}: {exc}"[:300]} return out @@ -345,8 +347,7 @@ def ups_for(seed, n): inp, ups = _inputs(op, seed, big), ups_for(seed, big) fo, fg = _run(fn, inp, rows, grads, ups) for sb in subs: - step = sb if sb >= 1024 else max(sb, big // 512) - for start in range(0, big, step): + for start in range(0, big, sb): idx = torch.arange(start, min(start + sb, big), device=DEV) o, g = _run(fn, inp, rows, grads, ups, idx) checked += 1 @@ -357,6 +358,7 @@ def ups_for(seed, n): res["full_vs_sub_batches"] = { "full_batch": big, "sub_batch_sizes": list(subs), + "coverage": "every_row_per_sub_batch_size", "sub_batches": checked, "fwd_differ": fwd, "rowgrad_differ": grd, @@ -452,13 +454,44 @@ def step(): # --------------------------------------------------------------------------- # +def _swapped_operator(op: str, fn): + """The rl-kernel CUDA op with its forward and backward replaced by ``fn``. + + The C4 harness folds ``parameter_vjp_contributions_fp32`` in place of the autograd + ``dweight`` whenever an op declares it. Those contributions come from the rl-kernel + kernel, so an inherited hook would let C4 judge rl-kernel's ``dweight`` instead of the + other library's. Clearing it makes C4 use ``dweight`` from the library's own backward. + """ + + from rl_engine.backends.cuda.norm import rmsnorm as R + + if op == "qwen3_next_rms_norm": + + class Swapped(R.Qwen3NextRMSNormCudaOp): + parameter_vjp_contributions_fp32 = None + + def forward(self, x, weight, *, eps=EPS): + return fn(x.reshape(-1, x.shape[-1]), weight).view_as(x) + + else: + + class Swapped(R.Qwen3NextRMSNormGatedCudaOp): + parameter_vjp_contributions_fp32 = None + + def __call__(self, x, weight, gate, *, eps=EPS): + return self.forward(x, weight, gate, eps=eps) + + def forward(self, x, weight, gate, *, eps=EPS): + return fn(x, gate, weight) + + return Swapped() + + def _gate_child(which: str, op: str, impl: str) -> None: """Run the canonical C3/C4 CLI with the CUDA candidate replaced by ``impl``.""" import importlib.util - from rl_engine.backends.cuda.norm import rmsnorm as R - path = REPO_ROOT / "tools/validation/operators" / f"check_{which}_invariance.py" spec = importlib.util.spec_from_file_location("gate", path) gate = importlib.util.module_from_spec(spec) @@ -480,25 +513,12 @@ def _gate_child(which: str, op: str, impl: str) -> None: spec.loader.exec_module(gate) if impl != "rl_kernel": fn = candidates(op)[impl]["fn"] - if op == "qwen3_next_rms_norm": - - class Swapped(R.Qwen3NextRMSNormCudaOp): - def forward(self, x, weight, *, eps=EPS): - return fn(x.reshape(-1, x.shape[-1]), weight).view_as(x) - - else: - - class Swapped(R.Qwen3NextRMSNormGatedCudaOp): - def __call__(self, x, weight, gate, *, eps=EPS): - return self.forward(x, weight, gate, eps=eps) - - def forward(self, x, weight, gate, *, eps=EPS): - return fn(x, gate, weight) - original = gate.load_adapter_operator def load_adapter_operator(op_name, candidate): - return Swapped() if candidate == "cuda" else original(op_name, candidate) + return ( + _swapped_operator(op, fn) if candidate == "cuda" else original(op_name, candidate) + ) gate.load_adapter_operator = load_adapter_operator gate.main() @@ -607,9 +627,11 @@ def plot(report: dict[str, Any], path: Path) -> None: for y, (_, e) in zip(ys, entries): bi = e.get("batch_invariance") gates = e.get("contract_gates") - text = "batch-invariant: " + ( + text = "checks passed: " + ( "n/a" if bi is None else ("yes" if bi["batch_invariant"] else "NO") ) + if bi and bi["full_vs_sub_batches"].get("coverage") == "sampled_small_sub_batches": + text += " (sampled)" if gates and "unavailable" not in gates: text += " | C3/C4 gates: " + ( "pass" if all(g["passed"] for g in gates.values()) else "FAIL" @@ -618,7 +640,7 @@ def plot(report: dict[str, Any], path: Path) -> None: ax.text(0.02, y, text, va="center", fontsize=8, color="#3aa676" if good else "#d14b4b") ax.set_ylim(len(entries) - 0.5, -0.5) ax.axis("off") - ax.set_title("every-row batch invariance and this repository's gates") + ax.set_title("measured batch-invariance checks and this repository's gates") fig.savefig(path.with_suffix(".png"), dpi=120) diff --git a/tools/validation/operators/check_forward_invariance.py b/tools/validation/operators/check_forward_invariance.py index c7a836f2c..6830d8315 100644 --- a/tools/validation/operators/check_forward_invariance.py +++ b/tools/validation/operators/check_forward_invariance.py @@ -18,8 +18,9 @@ if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -from rl_engine.config.workload import load_manifest # noqa: E402 +from rl_engine.config.workload import load_manifest, workload_report # noqa: E402 from rl_engine.contracts.numerical import resolve_dtype_policy # noqa: E402 +from rl_engine.validation.models.qwen3_next_workload import validate_norm_dimensions # noqa: E402 from rl_engine.validation.operators import ( # noqa: E402 BackendProvenance, assert_forward_batch_invariant, @@ -113,6 +114,12 @@ def parse_args() -> argparse.Namespace: if adapter.requirement != "absent_not_required" ] parser = argparse.ArgumentParser(description="WS1 C3 forward invariance GPU gate") + parser.add_argument( + "--manifest", + type=pathlib.Path, + default=None, + help="Workload manifest JSON (default: the Qwen3-8B Dense C2 manifest)", + ) parser.add_argument("--op", choices=sorted(runnable), default="rms_norm") parser.add_argument( "--candidate", required=True, help="Manifest-declared CUDA/Triton/Ascend candidate" @@ -146,7 +153,8 @@ def main() -> None: raise SystemExit("ERROR: --vocab must cover every fixed C2 workload token id") contract = load_contract() - manifest = load_manifest() + manifest = load_manifest(args.manifest) + validate_norm_dimensions(manifest.raw, args.op, args.hidden, args.head_dim) adapter = get_adapter(args.op) if adapter.requirement == "layout_supported": raise SystemExit( @@ -233,7 +241,9 @@ def main() -> None: ) if args.json: - print(json.dumps(report.to_dict(), indent=2, default=str)) + payload = report.to_dict() + payload["workload"] = workload_report(manifest) + print(json.dumps(payload, indent=2, default=str)) else: _summarize(report) if not report.passed: diff --git a/tools/validation/operators/check_gradient_invariance.py b/tools/validation/operators/check_gradient_invariance.py index db81d8c58..178a5a4a6 100644 --- a/tools/validation/operators/check_gradient_invariance.py +++ b/tools/validation/operators/check_gradient_invariance.py @@ -18,8 +18,9 @@ if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -from rl_engine.config.workload import load_manifest # noqa: E402 +from rl_engine.config.workload import load_manifest, workload_report # noqa: E402 from rl_engine.contracts.numerical import resolve_dtype_policy # noqa: E402 +from rl_engine.validation.models.qwen3_next_workload import validate_norm_dimensions # noqa: E402 from rl_engine.validation.operators import ( # noqa: E402 BackendProvenance, assert_gradient_batch_invariant, @@ -125,6 +126,12 @@ def parse_args() -> argparse.Namespace: if adapter.requirement != "absent_not_required" ] parser = argparse.ArgumentParser(description="WS1 C4 gradient invariance GPU gate") + parser.add_argument( + "--manifest", + type=pathlib.Path, + default=None, + help="Workload manifest JSON (default: the Qwen3-8B Dense C2 manifest)", + ) parser.add_argument("--op", choices=sorted(runnable), default="rms_norm") parser.add_argument( "--candidate", required=True, help="Manifest-declared CUDA/Triton/Ascend candidate" @@ -159,7 +166,8 @@ def main() -> None: raise SystemExit(f"ERROR: C4 required-profile evidence needs a real device: {exc}") from exc contract = load_contract() - manifest = load_manifest() + manifest = load_manifest(args.manifest) + validate_norm_dimensions(manifest.raw, args.op, args.hidden, args.head_dim) adapter = get_adapter(args.op) if adapter.requirement == "layout_supported": # Pack is the same PyTorch layout op under both profiles and is not a C2 @@ -255,7 +263,9 @@ def main() -> None: ) from exc if args.json: - print(json.dumps(report.to_dict(), indent=2, default=str)) + payload = report.to_dict() + payload["workload"] = workload_report(manifest) + print(json.dumps(payload, indent=2, default=str)) else: _summarize(report) if not report.passed: