Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 70 additions & 20 deletions atom/model_ops/attention_mha.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import aiter
import torch
from aiter import fused_qk_norm_rope_cache_quant_shuffle
from aiter.jit.utils.build_targets import is_gfx1250_family
from aiter.jit.utils.chip_info import get_gfx
from aiter.ops.mha import _flash_attn_varlen_forward
from aiter.ops.triton.fused_kv_cache import fused_qk_rope_reshape_and_cache
Expand Down Expand Up @@ -302,6 +303,7 @@ def rope_cache(self, q, k, v, qkv, position, fwd_ctx: ForwardContext):
k_scale,
v_scale,
attn_metadata,
qkv,
)
elif isinstance(self.q_norm, GemmaRMSNorm):
# Reshape V cache to SHUFFLE layout for the Triton kernel
Expand Down Expand Up @@ -446,8 +448,49 @@ def rope_cache(self, q, k, v, qkv, position, fwd_ctx: ForwardContext):
# sees q/k with matching token counts (sq == sk).
return q, k, v, k_cache, v_cache, k_scale, v_scale

def _aiter_gemma_norm_rope(self, qkv, position):
"""The Gemma q/k norm + NeoX RoPE of a long prefill's ``qkv`` in one aiter
fused_qknorm_idxrqknorm launch: q into a new tensor, k in place in
``qkv``. None where it is not taken: off the gfx1250 family (where it was
measured), for a non-NeoX rotation, and for a call the op does not take
(aiter's fused_qknorm_idxrqknorm_supports, e.g. head_dim 256)."""
if not (is_gfx1250_family(get_gfx()) and self.rotary_emb.is_neox_style):
return None
from atom.models.minimax_m3.model import _minimax_m3_cos_sin_cache

q = torch.empty(
(qkv.shape[0], self.num_heads * self.head_dim),
dtype=qkv.dtype,
device=qkv.device,
)
args = (
qkv,
self.q_norm.weight,
self.k_norm.weight,
_minimax_m3_cos_sin_cache(self.rotary_emb, qkv),
position,
self.num_heads,
self.num_kv_heads,
self.rotary_emb.rotary_dim,
self.q_norm.variance_epsilon,
)
if not aiter.fused_qknorm_idxrqknorm_supports(*args, q_out=q):
return None
aiter.fused_qknorm_idxrqknorm(*args, q_out=q)
return q

def _gemma_norm_rope_then_cache(
self, q, k, v, position, k_cache, v_cache, k_scale, v_scale, attn_metadata
self,
q,
k,
v,
position,
k_cache,
v_cache,
k_scale,
v_scale,
attn_metadata,
qkv,
):
"""Unfused twin of triton_fused_norm_rope_cache for long batches.

Expand All @@ -458,30 +501,37 @@ def _gemma_norm_rope_then_cache(
module's own rotary when that declines), then the cache write: per-token
FP8 for an FP8 cache, unquantized otherwise. Same SHUFFLE layout and
scales the fused kernel writes.
"""
from atom.model_ops.layernorm import DualRMSNorm
from atom.model_ops.triton_mrope import try_mrope_qk_fused

if getattr(self, "_dual_qk_norm", None) is None:
self._dual_qk_norm = DualRMSNorm(
self.q_norm,
self.k_norm,
Where aiter takes it (``_aiter_gemma_norm_rope``), the norm and RoPE
are one aiter launch instead, k in place in ``qkv``.
"""
q_aiter = self._aiter_gemma_norm_rope(qkv, position)
if q_aiter is not None:
q = q_aiter
else:
from atom.model_ops.layernorm import DualRMSNorm
from atom.model_ops.triton_mrope import try_mrope_qk_fused

if getattr(self, "_dual_qk_norm", None) is None:
self._dual_qk_norm = DualRMSNorm(
self.q_norm,
self.k_norm,
self.num_heads,
self.num_kv_heads,
self.head_dim,
prefix=f"layer_{self.layer_num}.qk_norm",
)
q, k = self._dual_qk_norm(q, k)
roped = try_mrope_qk_fused(
self.rotary_emb,
position,
q,
k,
self.num_heads,
self.num_kv_heads,
self.head_dim,
prefix=f"layer_{self.layer_num}.qk_norm",
)
q, k = self._dual_qk_norm(q, k)
roped = try_mrope_qk_fused(
self.rotary_emb,
position,
q,
k,
self.num_heads,
self.num_kv_heads,
self.head_dim,
)
q, k = roped if roped is not None else self.rotary_emb(position, q, k)
q, k = roped if roped is not None else self.rotary_emb(position, q, k)
q = q.view(-1, self.num_heads, self.head_dim)
k = k.view(-1, self.num_kv_heads, self.head_dim)
v = v.view(-1, self.num_kv_heads, self.head_dim)
Expand Down
2 changes: 1 addition & 1 deletion docs/environment_variables.md
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,7 @@ materializes two `[B, V]` fp32 tensors that only an `argmax` reads. See
| Variable | Type | Default | Description |
|----------|------|---------|-------------|
| **ATOM_ENABLE_QK_NORM_ROPE_CACHE_QUANT_FUSION** | bool | 0 (false) | If set to `1`, fuse QK norm, RoPE, and cache quantization into one kernel for Qwen3 dense and MoE models. |
| **ATOM_QK_NORM_ROPE_CACHE_FUSION_MAX_TOKENS** | int | 8192 | With the fusion above on, Gemma-norm models (Qwen3-Next, Qwen3.5) run the q/k norm, RoPE and cache write as separate kernels when a batch has more tokens than this; the fused kernel is slower at prefill sizes. |
| **ATOM_QK_NORM_ROPE_CACHE_FUSION_MAX_TOKENS** | int | 8192 | With the fusion above on, Gemma-norm models (Qwen3-Next, Qwen3.5) run the q/k norm, RoPE and cache write as separate kernels when a batch has more tokens than this; the fused kernel is slower at prefill sizes. Above this count on the gfx1250 family, the q/k norm + RoPE of those separate kernels is one aiter `fused_qknorm_idxrqknorm` launch where the op takes the layer (`fused_qknorm_idxrqknorm_supports`; the MiniMax-M3 dense layers, not head_dim 256). **Needs aiter with #8011**; upgrade ATOM and aiter together. |

### Llama-style

Expand Down
182 changes: 182 additions & 0 deletions tests/model_ops/test_gemma_qknorm_rope_aiter_gpu.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
# SPDX-License-Identifier: MIT
"""The aiter route of the Gemma q/k norm + RoPE of a long prefill on gfx1250.

Above ATOM_QK_NORM_ROPE_CACHE_FUSION_MAX_TOKENS, `PagedAttentionImpl.rope_cache`
hands the Gemma q/k norm and NeoX RoPE to aiter `fused_qknorm_idxrqknorm` (k in
place in qkv) before the cache write, where the op takes the layer (the
MiniMax-M3 dense layers; not head_dim 256). It must leave the attention the q,
k and K/V cache of the Triton kernel.
Not bitwise: the sum of squares is reduced in another order and aiter rounds k
to bf16 before the per-token FP8 quantization, so a cache is compared
dequantized, within one step at the row maximum (32 of the 448 an FP8 row
maximum is scaled to, below 1/8; 1/128 for bf16).
"""

from types import SimpleNamespace

import pytest
import torch

pytest.importorskip("aiter")

import aiter
from aiter.jit.utils.build_targets import is_gfx1250_family
from aiter.jit.utils.chip_info import get_gfx

from atom.model_ops import attention_mha
from atom.utils import envs

pytestmark = pytest.mark.skipif(
not torch.cuda.is_available() or not is_gfx1250_family(get_gfx()),
reason="the aiter route runs on the gfx1250 family only",
)

NQ, NKV, HD, ROT, MAX_POS = 16, 1, 128, 64, 1 << 16
FUSED_MAX = envs.ATOM_QK_NORM_ROPE_CACHE_FUSION_MAX_TOKENS


def make_impl(kv_cache_dtype, hd=HD):
from aiter.rotary_embedding import get_rope

from atom.model_ops.layernorm import GemmaRMSNorm

norms = []
for _ in range(2):
norm = GemmaRMSNorm(hd, eps=1e-6)
norm.weight.data = (torch.randn(hd) * 0.1).to(torch.bfloat16).cuda()
norms.append(norm)
rope = get_rope(hd, ROT, MAX_POS, 10000, is_neox_style=True, dtype=torch.bfloat16)
return attention_mha.PagedAttentionImpl(
NQ,
hd,
hd**-0.5,
NKV,
None,
kv_cache_dtype=kv_cache_dtype,
rotary_emb=rope.cuda(),
q_norm=norms[0],
k_norm=norms[1],
)


def make_cache(kv_cache_dtype, num_blocks, block_size):
"""SHUFFLE: K [blocks, heads, D / x, block, x], V [blocks, heads, block / x,
D, x], x elements per 16 bytes; per-token scales for FP8."""
dtype = torch.float8_e4m3fn if kv_cache_dtype == "fp8" else torch.bfloat16
x = 16 // dtype.itemsize
k = torch.zeros(num_blocks, NKV, HD // x, block_size, x, dtype=dtype, device="cuda")
v = torch.zeros(num_blocks, NKV, block_size // x, HD, x, dtype=dtype, device="cuda")
if kv_cache_dtype != "fp8":
return k, v, None, None
k_scale, v_scale = torch.zeros(2, num_blocks, NKV, block_size, device="cuda")
return k, v, k_scale, v_scale


def read_cache(cache):
"""[slots, heads, D] K and V, dequantized."""
k, v, k_scale, v_scale = cache
nb, bs = k.shape[0], k.shape[3]
k = k.permute(0, 3, 1, 2, 4).reshape(nb * bs, NKV, HD).float()
v = v.permute(0, 2, 4, 1, 3).reshape(nb * bs, NKV, HD).float()
if k_scale is not None:
k = k * k_scale.permute(0, 2, 1).reshape(nb * bs, NKV, 1)
v = v * v_scale.permute(0, 2, 1).reshape(nb * bs, NKV, 1)
return k, v


def assert_close_to_row_max(got, ref, frac):
err = (got.float() - ref.float()).abs()
bound = frac * ref.float().abs().amax(-1, keepdim=True)
assert (err <= bound).all(), f"max excess {(err - bound).max().item():.3e}"


def run_rope_cache(impl, qkv, position, slots, cache, is_prefill):
fwd_ctx = SimpleNamespace(
attn_metadata=SimpleNamespace(slot_mapping=slots),
kv_cache_data={
f"layer_{impl.layer_num}": SimpleNamespace(
k_cache=cache[0], v_cache=cache[1], k_scale=cache[2], v_scale=cache[3]
)
},
context=SimpleNamespace(is_prefill=is_prefill),
)
q_raw, k_raw, v_raw = qkv.split([NQ * HD, NKV * HD, NKV * HD], dim=-1)
return impl.rope_cache(q_raw, k_raw, v_raw, qkv, position, fwd_ctx)[:3]


@pytest.mark.parametrize("block_size", [16, 128])
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "bf16"])
@pytest.mark.parametrize(
("num_tokens", "aiter_calls"), [(FUSED_MAX, 0), (FUSED_MAX + 1, 1)]
)
def test_rope_cache_matches_the_triton_kernel(
monkeypatch, kv_cache_dtype, block_size, num_tokens, aiter_calls
):
from atom.model_ops.triton_fused_qkv_norm_rope_cache import (
triton_fused_norm_rope_cache,
)

torch.manual_seed(0)
impl = make_impl(kv_cache_dtype)
qkv = torch.randn(num_tokens, (NQ + 2 * NKV) * HD, device="cuda").bfloat16()
position = torch.randint(0, MAX_POS, (num_tokens,), device="cuda")
num_blocks = num_tokens // block_size + 8
slots = torch.randperm(num_blocks * block_size, device="cuda")[:num_tokens]

ref_cache = make_cache(kv_cache_dtype, num_blocks, block_size)
q_raw, k_raw, v_raw = qkv.split([NQ * HD, NKV * HD, NKV * HD], dim=-1)
ref_q, ref_k = triton_fused_norm_rope_cache(
q_raw,
k_raw,
v_raw,
position,
q_norm=impl.q_norm,
k_norm=impl.k_norm,
rotary_emb=impl.rotary_emb,
num_heads=NQ,
num_kv_heads=NKV,
head_dim=HD,
k_cache=ref_cache[0],
v_cache=ref_cache[1],
k_scale=ref_cache[2],
v_scale=ref_cache[3],
slot_mapping=slots,
kv_cache_dtype=kv_cache_dtype,
)

calls = []
op = aiter.fused_qknorm_idxrqknorm
monkeypatch.setattr(
aiter,
"fused_qknorm_idxrqknorm",
lambda *a, **kw: calls.append(1) or op(*a, **kw),
)
cache = make_cache(kv_cache_dtype, num_blocks, block_size)
qkv_in = qkv.clone()
q, k, v = run_rope_cache(impl, qkv, position, slots, cache, True)

assert len(calls) == aiter_calls
torch.testing.assert_close(q.reshape(num_tokens, -1), ref_q.reshape(num_tokens, -1))
frac = 1 / 8 if kv_cache_dtype == "fp8" else 1 / 128
for got, ref in zip(read_cache(cache), read_cache(ref_cache)):
assert_close_to_row_max(got, ref, frac)
# prefill attention reads k
torch.testing.assert_close(k.reshape(num_tokens, -1), ref_k.reshape(num_tokens, -1))
torch.testing.assert_close(v.reshape(num_tokens, -1), qkv_in[:, -NKV * HD :])


def test_aiter_route_declines_what_the_op_does_not_take():
position = torch.zeros(8, dtype=torch.long, device="cuda")
for hd, taken in ((HD, True), (256, False)): # 256: Qwen3-Next, Qwen3.5
impl = make_impl("bf16", hd)
qkv = torch.zeros(8, (NQ + 2 * NKV) * hd, dtype=torch.bfloat16, device="cuda")
assert (impl._aiter_gemma_norm_rope(qkv, position) is not None) == taken
# GPT-J: not the rotation the op computes
from aiter.rotary_embedding import get_rope

impl = make_impl("bf16")
impl.rotary_emb = get_rope(
HD, ROT, MAX_POS, 10000, is_neox_style=False, dtype=torch.bfloat16
).cuda()
qkv = torch.zeros(8, (NQ + 2 * NKV) * HD, dtype=torch.bfloat16, device="cuda")
assert impl._aiter_gemma_norm_rope(qkv, position) is None
Loading