From 8aa792a24a26f9c8dac2ba4bcd65b1137f773cc3 Mon Sep 17 00:00:00 2001 From: ShaoChunLee Date: Thu, 17 Sep 2026 07:52:44 +0000 Subject: [PATCH] feat(v4): reuse sparse prefill ASM for decode Co-authored-by: Cursor --- atom/model_ops/attentions/deepseek_v4_attn.py | 13 ++ atom/model_ops/v4_kernels/paged_decode.py | 72 ++++++++++- atom/models/deepseek_v4.py | 6 +- atom/utils/envs.py | 5 + tests/test_envs.py | 8 ++ tests/test_v4_prefill_asm_decode_dispatch.py | 113 ++++++++++++++++++ 6 files changed, 212 insertions(+), 5 deletions(-) create mode 100644 tests/test_v4_prefill_asm_decode_dispatch.py diff --git a/atom/model_ops/attentions/deepseek_v4_attn.py b/atom/model_ops/attentions/deepseek_v4_attn.py index 55f283ea41..d1e3f7b44a 100644 --- a/atom/model_ops/attentions/deepseek_v4_attn.py +++ b/atom/model_ops/attentions/deepseek_v4_attn.py @@ -311,6 +311,10 @@ class AttentionMetaData_DSV4(AttentionMetaData): kv_last_page_lens: torch.Tensor | None = None """[padded_T] int32 GPU — per-token last-page length `ones(N)` (page_size=1 → every page is full).""" + empty_kv_indptr: torch.Tensor | None = None + """[padded_T+1] all-zero int32 GPU — empty extend-stream CSR used when + `ATOM_USE_V4_PREFILL_ASM_FOR_DECODE=1` reuses the H=128 prefill ASM kernel + for decode. Backed by one immutable buffer shared by every layer.""" # ----- Indexer / sparse-layout side metadata ----- indexer_meta: dict[str, Any] | None = None @@ -2415,6 +2419,9 @@ def prepare_mtp_decode( attn_metadata.qo_indptr = self._stage( "v4_qo_indptr", self._v4_qo_indptr_np[: running_bs + 1] ) + attn_metadata.empty_kv_indptr = self.model_runner.forward_vars[ + "v4_empty_kv_indptr" + ][: running_bs + 1] # NOT rebuilt (unused by SWA-only MTP layer; would block a future # CSA/HCA MTP layer — assert at top guards): @@ -3663,6 +3670,9 @@ def _attach_v4_paged_decode_meta( qo_buf.np[: T + 1] = self._v4_qo_indptr_np[: T + 1] qo_buf.np[T + 1 : T_pad + 1] = T attn_metadata.qo_indptr = qo_buf.copy_to_gpu(T_pad + 1) + attn_metadata.empty_kv_indptr = self.model_runner.forward_vars[ + "v4_empty_kv_indptr" + ][: T_pad + 1] def _build_paged_prefill_meta( self, @@ -4319,6 +4329,9 @@ def _alloc_v4_metadata_buffers(self) -> None: # the per-fwd cost is a slice + H2D. bufs["v4_qo_indptr"] = CpuGpuBuffer(T_dec + 1, **i32) self._v4_qo_indptr_np = np.arange(T_dec + 1, dtype=np.int32) + # Immutable, device-only empty CSR for reusing the H=128 sparse-prefill + # ASM kernel in decode. Shared read-only across layers and TBO ubatches. + bufs["v4_empty_kv_indptr"] = torch.zeros(T_dec + 1, **i32) # Per-seq `ctx_len // 4` (raw, no clamp). Consumed by the indexer's # `cu_committed` cumsum — per-SEQUENCE, host-side, prefill only. # Single per-token mapping shared across ALL V4 consumers: diff --git a/atom/model_ops/v4_kernels/paged_decode.py b/atom/model_ops/v4_kernels/paged_decode.py index 6a5396d3ca..1484dfa355 100644 --- a/atom/model_ops/v4_kernels/paged_decode.py +++ b/atom/model_ops/v4_kernels/paged_decode.py @@ -60,6 +60,7 @@ from atom.model_ops.sparse_attn_v4 import _sparse_attn_ragged_torch from atom.model_ops.v4_kernels.pool_index import row_offset +from atom.utils import envs from atom.utils.decorators import mark_trace LOG2E = 1.4426950408889634 # log2(e); folded into qk_scale so softmax can use exp2. @@ -911,6 +912,50 @@ def sparse_attn_v4_paged_decode_reference( return _sparse_attn_ragged_torch(q, unified_kv, attn_sink, topk_idxs, softmax_scale) +def _sparse_attn_v4_paged_decode_prefill_asm( + unified_kv: torch.Tensor, + kv_indices: torch.Tensor, + kv_indptr: torch.Tensor, + empty_kv_indptr: torch.Tensor, + attn_sink: torch.Tensor, + softmax_scale: float, + unified_kv_rope: torch.Tensor, + q_packed_in: torch.Tensor, + q_rope_in: torch.Tensor, +) -> torch.Tensor: + """Run H=128 fp8 decode through the sparse-prefill ASM kernel. + + Decode has no flat extend source: its current token has already been + written to ``unified_kv`` and included in ``kv_indices``. Feed the decode + CSR as the prefill kernel's prefix stream and an empty CSR as its extend + stream. The KV tensors for that unused stream alias the unified pools so + the launch needs no per-layer dummy allocation. + """ + from aiter.ops.mla_sparse_prefill import mla_sparse_prefill_fp8_asm + + n = q_packed_in.shape[0] + if empty_kv_indptr.numel() < n + 1: + raise ValueError( + "V4 prefill-ASM decode needs an all-zero empty_kv_indptr with at " + f"least N+1={n + 1} entries, got {empty_kv_indptr.numel()}" + ) + + return mla_sparse_prefill_fp8_asm( + q_nope=q_packed_in, + q_rope=q_rope_in, + unified_kv_nope=unified_kv, + unified_kv_rope=unified_kv_rope, + kv_indices_prefix=kv_indices, + kv_indptr_prefix=kv_indptr[: n + 1], + kv_nope=unified_kv, + kv_rope=unified_kv_rope, + kv_indices_extend=kv_indices[:0], + kv_indptr_extend=empty_kv_indptr[: n + 1], + attn_sink=attn_sink, + softmax_scale=softmax_scale, + ) + + def _sparse_attn_v4_paged_decode_asm( unified_kv: torch.Tensor, kv_indices: torch.Tensor, @@ -1065,13 +1110,16 @@ def sparse_attn_v4_paged_decode( q_rope_in: torch.Tensor | None = None, qo_indptr: torch.Tensor | None = None, kv_last_page_lens: torch.Tensor | None = None, + empty_kv_indptr: torch.Tensor | None = None, prefix: str = "", ) -> torch.Tensor: """V4 decode sparse attention over a unified KV pool with paged indices. - Native 2buff fp8 (``unified_kv_rope`` provided): routes to the aiter asm - kernel (op5) with pre-packed fp8 Q (``q_packed_in``/``q_rope_in``); the - fp8 NoPE pool + bf16 RoPE pool are read with no requant. + Native 2buff fp8 (``unified_kv_rope`` provided): normally routes to the + aiter decode ASM kernel (op5). With + ``ATOM_USE_V4_PREFILL_ASM_FOR_DECODE=1``, gfx1250 H=128 instead reuses the + sparse-prefill ASM kernel with an empty extend stream. Both paths consume + pre-packed fp8 Q and the fp8 NoPE + bf16 RoPE pools with no requant. Otherwise (bf16): the existing Triton / reference path. When ``kv_scales`` is provided, ``unified_kv`` must be fp8 (e4m3fnuz) and is dequantized @@ -1079,6 +1127,24 @@ def sparse_attn_v4_paged_decode( unreachable from the model). """ if unified_kv_rope is not None: + if ( + envs.ATOM_USE_V4_PREFILL_ASM_FOR_DECODE + and q_packed_in is not None + and q_packed_in.shape[1] == 128 + and empty_kv_indptr is not None + and get_gfx() == "gfx1250" + ): + return _sparse_attn_v4_paged_decode_prefill_asm( + unified_kv, + kv_indices, + kv_indptr, + empty_kv_indptr, + attn_sink, + softmax_scale, + unified_kv_rope, + q_packed_in, + q_rope_in, + ) return _sparse_attn_v4_paged_decode_asm( unified_kv, kv_indices, diff --git a/atom/models/deepseek_v4.py b/atom/models/deepseek_v4.py index 309d50c91c..63d5e8d666 100644 --- a/atom/models/deepseek_v4.py +++ b/atom/models/deepseek_v4.py @@ -3143,8 +3143,9 @@ def _sparse_attention( kv_indices = attn_md.kv_indices_hca kv_indptr = attn_md.kv_indptr_hca # Dispatch on kv-cache layout inside the wrapper: fp8 2buff - # (unified_kv_rope set) → aiter asm op5 with pre-packed fp8 Q + the - # 2buff fp8/bf16 pools read with no requant; bf16 (unified_kv_rope + # (unified_kv_rope set) → aiter ASM with pre-packed fp8 Q + the + # 2buff fp8/bf16 pools read with no requant. The optional H=128 + # prefill-ASM decode route also consumes the shared empty CSR. o = sparse_attn_v4_paged_decode( qkn.q_sa, self.unified_kv, @@ -3156,6 +3157,7 @@ def _sparse_attention( q_packed_in=qkn.q_packed, q_rope_in=qkn.q_rope, qo_indptr=attn_md.qo_indptr, + empty_kv_indptr=attn_md.empty_kv_indptr, prefix=f"{self.layer_name}.sparse_attn_decode", ) # [S, H, head_dim] else: diff --git a/atom/utils/envs.py b/atom/utils/envs.py index 1b75ecdce9..8b21a0a132 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -497,6 +497,11 @@ def _positive_float_env(name: str, default: str) -> float: "ATOM_FORCE_V4_PREFILL_OPUS": lambda: ( os.getenv("ATOM_FORCE_V4_PREFILL_OPUS", "0") == "1" ), + # Reuse the gfx1250 H=128 sparse-prefill ASM kernel for DeepSeek-V4 fp8 + # decode. Ineligible shapes keep the dedicated decode ASM path. + "ATOM_USE_V4_PREFILL_ASM_FOR_DECODE": lambda: ( + os.getenv("ATOM_USE_V4_PREFILL_ASM_FOR_DECODE", "0") == "1" + ), # Use gluon pa decode for some models "ATOM_USE_GLUON_PA_DECODE": lambda: ( os.getenv("ATOM_USE_GLUON_PA_DECODE", "0") == "1" diff --git a/tests/test_envs.py b/tests/test_envs.py index 7c2f944e5c..66333afeed 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -15,6 +15,7 @@ "ATOM_DP_BASE_PORT", "ATOM_USE_TRITON_GEMM", "ATOM_USE_TRITON_MXFP4_BMM", + "ATOM_USE_V4_PREFILL_ASM_FOR_DECODE", "ATOM_MHC_USE_BF16", "ATOM_ENABLE_QK_NORM_ROPE_CACHE_QUANT_FUSION", "ATOM_ENABLE_DS_INPUT_RMSNORM_QUANT_FUSION", @@ -80,6 +81,9 @@ def test_mhc_use_bf16_default(self): def test_use_triton_gemm_default(self): assert _get_envs().ATOM_USE_TRITON_GEMM is False + def test_use_v4_prefill_asm_for_decode_default_disabled(self): + assert _get_envs().ATOM_USE_V4_PREFILL_ASM_FOR_DECODE is False + def test_ds_input_rmsnorm_quant_fusion_default_enabled(self): assert _get_envs().ATOM_ENABLE_DS_INPUT_RMSNORM_QUANT_FUSION is True @@ -150,6 +154,10 @@ def test_dp_port_overrides(self, monkeypatch): assert _get_envs().ATOM_DP_MASTER_PORT == 29700 assert _get_envs().ATOM_DP_BASE_PORT == 29800 + def test_use_v4_prefill_asm_for_decode_enabled(self, monkeypatch): + monkeypatch.setenv("ATOM_USE_V4_PREFILL_ASM_FOR_DECODE", "1") + assert _get_envs().ATOM_USE_V4_PREFILL_ASM_FOR_DECODE is True + def test_torch_profiler_dir_override(self, monkeypatch): monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", "/tmp/prof") assert _get_envs().ATOM_TORCH_PROFILER_DIR == "/tmp/prof" diff --git a/tests/test_v4_prefill_asm_decode_dispatch.py b/tests/test_v4_prefill_asm_decode_dispatch.py new file mode 100644 index 0000000000..85ad316742 --- /dev/null +++ b/tests/test_v4_prefill_asm_decode_dispatch.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: MIT +"""Dispatch coverage for reusing the V4 H=128 prefill ASM in decode.""" + +from __future__ import annotations + +import pytest +import torch + +pytest.importorskip("triton", reason="paged_decode defines Triton kernels") +pytest.importorskip("aiter", reason="paged_decode imports the AITER runtime") + +from aiter.ops import mla_sparse_prefill + +from atom.model_ops.v4_kernels import paged_decode + + +def _dispatch( + monkeypatch, + *, + enabled: bool, + heads: int = 128, + gfx: str = "gfx1250", + with_empty_indptr: bool = True, +): + monkeypatch.setattr( + paged_decode.envs, "ATOM_USE_V4_PREFILL_ASM_FOR_DECODE", enabled + ) + monkeypatch.setattr(paged_decode, "get_gfx", lambda: gfx) + monkeypatch.setattr( + paged_decode, + "_sparse_attn_v4_paged_decode_prefill_asm", + lambda *args, **kwargs: "prefill", + ) + monkeypatch.setattr( + paged_decode, + "_sparse_attn_v4_paged_decode_asm", + lambda *args, **kwargs: "decode", + ) + + n = 2 + return paged_decode.sparse_attn_v4_paged_decode( + q=torch.empty((n, heads, 512)), + unified_kv=torch.empty((4, 512)), + kv_indices=torch.empty(0, dtype=torch.int32), + kv_indptr=torch.zeros(n + 1, dtype=torch.int32), + attn_sink=torch.empty(heads), + softmax_scale=512**-0.5, + unified_kv_rope=torch.empty((4, 64)), + q_packed_in=torch.empty((n, heads, 512)), + q_rope_in=torch.empty((n, heads, 64)), + qo_indptr=torch.arange(n + 1, dtype=torch.int32), + empty_kv_indptr=( + torch.zeros(n + 1, dtype=torch.int32) if with_empty_indptr else None + ), + ) + + +def test_enabled_ep4_head128_uses_prefill_asm(monkeypatch): + assert _dispatch(monkeypatch, enabled=True) == "prefill" + + +def test_decode_csr_becomes_prefix_and_extend_is_empty(monkeypatch): + captured = {} + sentinel = object() + + def fake_prefill_asm(**kwargs): + captured.update(kwargs) + return sentinel + + monkeypatch.setattr( + mla_sparse_prefill, "mla_sparse_prefill_fp8_asm", fake_prefill_asm + ) + n, h = 2, 128 + q_nope = torch.empty((n, h, 512)) + q_rope = torch.empty((n, h, 64)) + unified_kv = torch.empty((4, 512)) + unified_kv_rope = torch.empty((4, 64)) + kv_indices = torch.tensor([0, 1, 2], dtype=torch.int32) + kv_indptr = torch.tensor([0, 1, 3], dtype=torch.int32) + empty_kv_indptr = torch.zeros(n + 1, dtype=torch.int32) + + result = paged_decode._sparse_attn_v4_paged_decode_prefill_asm( + unified_kv, + kv_indices, + kv_indptr, + empty_kv_indptr, + torch.empty(h), + 512**-0.5, + unified_kv_rope, + q_nope, + q_rope, + ) + + assert result is sentinel + assert torch.equal(captured["kv_indptr_prefix"], kv_indptr) + assert torch.equal(captured["kv_indices_prefix"], kv_indices) + assert captured["kv_nope"] is unified_kv + assert captured["kv_rope"] is unified_kv_rope + assert captured["kv_indices_extend"].numel() == 0 + assert torch.count_nonzero(captured["kv_indptr_extend"]) == 0 + + +@pytest.mark.parametrize( + "kwargs", + [ + {"enabled": False}, + {"enabled": True, "heads": 64}, + {"enabled": True, "gfx": "gfx950"}, + {"enabled": True, "with_empty_indptr": False}, + ], +) +def test_ineligible_decode_keeps_dedicated_asm(monkeypatch, kwargs): + assert _dispatch(monkeypatch, **kwargs) == "decode"