Skip to content
Merged
Show file tree
Hide file tree
Changes from 34 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
b11b957
feat(cuda): gated RMSNorm kernel for the Qwen3-Next GDN block
fusheng-ji Sep 30, 2026
7dd291a
fix(norm): enforce gated tensor boundaries and align parameter VJP st…
fusheng-ji Sep 30, 2026
8df1ffb
test(norm): add Qwen3-Next workload and C3 C4 adapters
fusheng-ji Sep 30, 2026
aaf0986
fix(norm): preserve gated signed-zero weights when offset is disabled
fusheng-ji Sep 30, 2026
4ea1611
test(dispatch): cover qwen3_next_rms_norm priority and missing-extens…
fusheng-ji Oct 1, 2026
5eaeca8
test(norm): sweep the gated-vs-plain rstd identity across dtype, H, a…
fusheng-ji Oct 1, 2026
b0f4c5c
fix(norm): address the gated-norm review (R2-2, 7, 8, 9, 12, 13, 14, …
fusheng-ji Oct 1, 2026
0cceb84
fix(rmsnorm): use OptionalCUDAGuard in the launchers outside the ROCm…
fusheng-ji Oct 2, 2026
cf68fd9
docs(norm): reconcile the C1 norm page, docstrings and gated tests wi…
fusheng-ji Oct 4, 2026
8ce67e0
Merge updated C1 norm base into gated RMSNorm branch
fusheng-ji Oct 6, 2026
1edc9d4
Merge Qwen3-Next C1 norm fixes and evidence runner into gated RMSNorm…
fusheng-ji Oct 8, 2026
a441ff9
feat(scripts): extend the Qwen3-Next norm evidence runner to the gate…
fusheng-ji Oct 8, 2026
ff271d5
Merge Qwen3-Next C1 norm plot fix into gated RMSNorm branch
fusheng-ji Oct 8, 2026
6f07962
docs(norm): gated RMSNorm evidence vs existing implementations on B200
fusheng-ji Oct 8, 2026
be485ee
Merge Qwen3-Next C1 norm evidence into gated RMSNorm branch
fusheng-ji Oct 8, 2026
fece41b
docs(norm): correct the gated backward slowdown ratio
fusheng-ji Oct 8, 2026
6ac74cd
Merge the Qwen3-Next norm reuse check into PR #468
fusheng-ji Oct 8, 2026
713c5dc
Merge the #467 norm reuse evidence into PR #468
fusheng-ji Oct 8, 2026
da6aa8d
Merge the norm reuse-check output fix into PR #468
fusheng-ji Oct 8, 2026
d33c3ac
docs(norm): C3/C4 gates for the zero-centred norm with each existing …
fusheng-ji Oct 8, 2026
8ac348b
docs(norm): rms_norm_gated vs existing implementations, every-row bat…
fusheng-ji Oct 8, 2026
d333a43
fix: exclude invalid Megatron zero-centered norm evidence
fusheng-ji Oct 9, 2026
0de9f00
fix: make norm reuse checks exhaustive and require primary candidate
fusheng-ji Oct 9, 2026
102c2c6
Merge test-qwennext refactor and migrate Qwen3-Next code and tests
fusheng-ji Oct 10, 2026
9ffdc1e
docs: move the measurement outputs out of the pull request
fusheng-ji Oct 19, 2026
1e2eaec
docs: point commit hashes at the rewritten history
fusheng-ji Oct 23, 2026
4a3d19f
fix(norm): require the RMSNorm API version in the gated CUDA op and d…
fusheng-ji Oct 25, 2026
f9fa9b8
refactor(norm): reshape the saved rstd to x's leading dims in the gat…
fusheng-ji Oct 25, 2026
d634d0b
fix(workload): raise WorkloadError for a norm manifest with a missing…
fusheng-ji Oct 25, 2026
08eee50
docs(norm): say why the declared contributions use the kernel's rstd
fusheng-ji Oct 25, 2026
5aa9aa1
ci(ws1): say why both Qwen3-Next norm gates take --hidden 2048 --head…
fusheng-ji Oct 25, 2026
6c63f0b
Merge test-qwennext after #467 landed
fusheng-ji Oct 26, 2026
339bc3f
fix(norm): judge the swapped library's own dweight in the reuse-check…
fusheng-ji Oct 26, 2026
789a285
docs(norm): report the reuse-check C4 gates with each library's own d…
fusheng-ji Oct 26, 2026
025476b
test(workload): pin the norm gate widths and the norm adapters' manif…
fusheng-ji Oct 26, 2026
9c84b32
fix(workload): reject an unknown manifest scope instead of validating…
fusheng-ji Oct 26, 2026
acf4aa5
refactor(workload): take the norm gate widths from NORM_OPS
fusheng-ji Oct 26, 2026
caedea5
fix(norm): reject inputs past 32-bit row offsets; drop the unused gat…
fusheng-ji Oct 26, 2026
a203719
fix(workload): check the norm manifest's top-level keys and chain sem…
fusheng-ji Oct 26, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions ci/scripts/run_ws1_gtest.sh
Original file line number Diff line number Diff line change
Expand Up @@ -76,3 +76,18 @@ for cell in required:
)
print("[ws1-gtest] C8 gate passed")
PY

echo "[ws1-gtest] Qwen3-Next C3/C4 norms (operator scope)"
QWEN3_NEXT_NORM_MANIFEST=rl_engine/validation/models/qwen3_next_norm_manifest.json
# --hidden is the model hidden size the manifest pins: validate_norm_dimensions requires
# --hidden 2048 --head-dim 128 for both ops. rms_norm_gated does not run at 2048: its inputs and
# weight are built at --head-dim (128, the GDN value-head width), so the same flags give each
# op its checkpoint width.
"$PY" tools/validation/operators/check_forward_invariance.py --manifest "$QWEN3_NEXT_NORM_MANIFEST" \
--op qwen3_next_rms_norm --candidate cuda --backend-profile cuda_bf16 --hidden 2048 --head-dim 128
Comment thread
fusheng-ji marked this conversation as resolved.
Resolved
"$PY" tools/validation/operators/check_gradient_invariance.py --manifest "$QWEN3_NEXT_NORM_MANIFEST" \
--op qwen3_next_rms_norm --candidate cuda --backend-profile cuda_bf16 --hidden 2048 --head-dim 128
"$PY" tools/validation/operators/check_forward_invariance.py --manifest "$QWEN3_NEXT_NORM_MANIFEST" \
--op rms_norm_gated --candidate cuda --backend-profile cuda_bf16 --hidden 2048 --head-dim 128
"$PY" tools/validation/operators/check_gradient_invariance.py --manifest "$QWEN3_NEXT_NORM_MANIFEST" \
--op rms_norm_gated --candidate cuda --backend-profile cuda_bf16 --hidden 2048 --head-dim 128
Comment thread
fusheng-ji marked this conversation as resolved.
Resolved
125 changes: 119 additions & 6 deletions csrc/bindings/ops.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,26 @@ void rmsnorm_backward_dx_cuda(
torch::Tensor dx,
double weight_offset);

void rmsnorm_gated_forward_cuda(
torch::Tensor x,
torch::Tensor weight,
torch::Tensor gate,
torch::Tensor y,
torch::Tensor rstd,
double eps,
double weight_offset,
int64_t activation);

void rmsnorm_gated_backward_dx_cuda(
torch::Tensor dy,
torch::Tensor x,
torch::Tensor weight,
torch::Tensor gate,
torch::Tensor rstd,
torch::Tensor dx,
double weight_offset,
int64_t activation);

void rmsnorm_backward_partial_dw_cuda(
torch::Tensor dy,
torch::Tensor x,
Expand All @@ -298,6 +318,24 @@ static void rmsnorm_check_input(const torch::Tensor& x, const char* name) {
TORCH_CHECK(x.is_contiguous(), name, " must be contiguous");
}

static void rmsnorm_check_weight(const torch::Tensor& x, const torch::Tensor& weight) {
TORCH_CHECK(x.dim() == 2 && x.size(1) > 0, "x must be 2D with positive hidden size");
TORCH_CHECK(x.scalar_type() == torch::kFloat32 || x.scalar_type() == torch::kFloat16 ||
x.scalar_type() == torch::kBFloat16, "x must be float32, float16 or bfloat16");
TORCH_CHECK(weight.dim() == 1 && weight.size(0) == x.size(1), "weight must be [H]");
Comment thread
fusheng-ji marked this conversation as resolved.
Resolved
TORCH_CHECK(weight.device() == x.device(), "weight must be on the same device as x");
TORCH_CHECK(weight.scalar_type() == x.scalar_type() || weight.scalar_type() == torch::kFloat32,
"weight must have x dtype or float32");
}

static void rmsnorm_check_backward(const torch::Tensor& dy, const torch::Tensor& x,
const torch::Tensor& rstd) {
TORCH_CHECK(dy.device() == x.device() && rstd.device() == x.device(),
"dy and rstd must be on the same device as x");
TORCH_CHECK(dy.scalar_type() == x.scalar_type(), "dy must have the same dtype as x");
TORCH_CHECK(rstd.scalar_type() == torch::kFloat32, "rstd must be float32");
}

std::vector<torch::Tensor> rmsnorm_forward(
torch::Tensor x,
torch::Tensor weight,
Expand All @@ -306,10 +344,7 @@ std::vector<torch::Tensor> rmsnorm_forward(
{
rmsnorm_check_input(x, "x");
rmsnorm_check_input(weight, "weight");

TORCH_CHECK(x.dim() == 2, "x must be 2D [T, H]");
TORCH_CHECK(weight.dim() == 1, "weight must be 1D [H]");
TORCH_CHECK(x.size(1) == weight.size(0), "x.size(1) must equal weight.size(0)");
rmsnorm_check_weight(x, weight);

auto T = x.size(0);
auto y = torch::empty_like(x);
Expand All @@ -330,11 +365,11 @@ torch::Tensor rmsnorm_backward_dx(
rmsnorm_check_input(dy, "dy");
rmsnorm_check_input(x, "x");
rmsnorm_check_input(weight, "weight");
rmsnorm_check_weight(x, weight);
rmsnorm_check_input(rstd, "rstd");
rmsnorm_check_backward(dy, x, rstd);

TORCH_CHECK(dy.sizes() == x.sizes(), "dy and x must have same shape");
TORCH_CHECK(x.dim() == 2, "x must be 2D [T, H]");
TORCH_CHECK(weight.dim() == 1, "weight must be 1D [H]");
TORCH_CHECK(rstd.dim() == 1, "rstd must be 1D [T]");
TORCH_CHECK(rstd.size(0) == x.size(0), "rstd.size(0) must equal x.size(0)");

Expand All @@ -345,6 +380,72 @@ torch::Tensor rmsnorm_backward_dx(
return dx;
}

static void rmsnorm_gated_check(
const torch::Tensor& x,
const torch::Tensor& weight,
const torch::Tensor& gate,
int64_t activation)
{
rmsnorm_check_input(x, "x");
rmsnorm_check_input(weight, "weight");
rmsnorm_check_weight(x, weight);
rmsnorm_check_input(gate, "gate");
TORCH_CHECK(gate.device() == x.device(), "gate must be on the same device as x");
TORCH_CHECK(gate.sizes() == x.sizes(), "gate must have the same shape as x");
TORCH_CHECK(
gate.scalar_type() == x.scalar_type(),
"gate must have the same dtype as x");
// 0 = silu/swish, 1 = sigmoid. Anything else fails closed rather than
// silently computing a different activation (RFC #428 section 6, item 7).
TORCH_CHECK(
activation == 0 || activation == 1,
"activation must be 0 (silu) or 1 (sigmoid), got ", activation);
}

std::vector<torch::Tensor> rmsnorm_gated_forward(
torch::Tensor x,
torch::Tensor weight,
torch::Tensor gate,
double eps,
double weight_offset,
int64_t activation)
{
rmsnorm_gated_check(x, weight, gate, activation);

auto T = x.size(0);
auto y = torch::empty_like(x);
auto rstd = torch::empty({T}, x.options().dtype(torch::kFloat32));

rmsnorm_gated_forward_cuda(x, weight, gate, y, rstd, eps, weight_offset, activation);

return {y, rstd};
}

torch::Tensor rmsnorm_gated_backward_dx(
torch::Tensor dy,
torch::Tensor x,
torch::Tensor weight,
torch::Tensor gate,
torch::Tensor rstd,
double weight_offset,
int64_t activation)
{
rmsnorm_gated_check(x, weight, gate, activation);
rmsnorm_check_input(dy, "dy");
rmsnorm_check_input(rstd, "rstd");
rmsnorm_check_backward(dy, x, rstd);

TORCH_CHECK(dy.sizes() == x.sizes(), "dy must have the same shape as x");
TORCH_CHECK(rstd.dim() == 1 && rstd.size(0) == x.size(0), "rstd must be [T]");

auto dx = torch::empty_like(x);

rmsnorm_gated_backward_dx_cuda(
dy, x, weight, gate, rstd, dx, weight_offset, activation);

return dx;
}

torch::Tensor rmsnorm_backward_dw(
torch::Tensor dy,
torch::Tensor x,
Expand All @@ -354,13 +455,16 @@ torch::Tensor rmsnorm_backward_dw(
rmsnorm_check_input(dy, "dy");
rmsnorm_check_input(x, "x");
rmsnorm_check_input(rstd, "rstd");
rmsnorm_check_backward(dy, x, rstd);
rmsnorm_check_input(mask, "mask");

TORCH_CHECK(dy.sizes() == x.sizes(), "dy and x must have same shape");
TORCH_CHECK(x.dim() == 2, "x must be 2D [T, H]");
TORCH_CHECK(rstd.dim() == 1, "rstd must be 1D [T]");
TORCH_CHECK(mask.dim() == 1, "mask must be 1D [T]");
TORCH_CHECK(mask.scalar_type() == torch::kBool, "mask must be bool");
TORCH_CHECK(mask.device() == x.device(), "mask must be on the same device as x");
TORCH_CHECK(x.size(1) > 0, "hidden size must be positive");
TORCH_CHECK(rstd.size(0) == x.size(0), "rstd.size(0) must equal x.size(0)");
TORCH_CHECK(mask.size(0) == x.size(0), "mask.size(0) must equal x.size(0)");

Expand Down Expand Up @@ -731,6 +835,15 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
py::arg("dy"), py::arg("x"), py::arg("weight"), py::arg("rstd"),
py::arg("weight_offset") = 0.0);
m.def("rmsnorm_backward_dw", &rmsnorm_backward_dw, "Deterministic RMSNorm backward dweight CUDA");
m.def("rmsnorm_gated_forward", &rmsnorm_gated_forward,
"Batch-invariant gated RMSNorm forward CUDA",
py::arg("x"), py::arg("weight"), py::arg("gate"), py::arg("eps"),
py::arg("weight_offset") = 0.0, py::arg("activation") = 0);
m.def("rmsnorm_gated_backward_dx", &rmsnorm_gated_backward_dx,
"Batch-invariant gated RMSNorm backward dx CUDA",
py::arg("dy"), py::arg("x"), py::arg("weight"), py::arg("gate"),
py::arg("rstd"), py::arg("weight_offset") = 0.0,
py::arg("activation") = 0);
#if !defined(USE_ROCM)
m.def(
"reduce_rows_fp32_left_fold",
Expand Down
Loading
Loading