Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions atom/model_ops/attentions/deepseek_v4_attn.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,10 @@ class AttentionMetaData_DSV4(AttentionMetaData):
kv_last_page_lens: torch.Tensor | None = None
"""[padded_T] int32 GPU — per-token last-page length `ones(N)` (page_size=1
→ every page is full)."""
empty_kv_indptr: torch.Tensor | None = None
"""[padded_T+1] all-zero int32 GPU — empty extend-stream CSR used when
`ATOM_USE_V4_PREFILL_ASM_FOR_DECODE=1` reuses the H=128 prefill ASM kernel
for decode. Backed by one immutable buffer shared by every layer."""

# ----- Indexer / sparse-layout side metadata -----
indexer_meta: dict[str, Any] | None = None
Expand Down Expand Up @@ -2415,6 +2419,9 @@ def prepare_mtp_decode(
attn_metadata.qo_indptr = self._stage(
"v4_qo_indptr", self._v4_qo_indptr_np[: running_bs + 1]
)
attn_metadata.empty_kv_indptr = self.model_runner.forward_vars[
"v4_empty_kv_indptr"
][: running_bs + 1]

# NOT rebuilt (unused by SWA-only MTP layer; would block a future
# CSA/HCA MTP layer — assert at top guards):
Expand Down Expand Up @@ -3663,6 +3670,9 @@ def _attach_v4_paged_decode_meta(
qo_buf.np[: T + 1] = self._v4_qo_indptr_np[: T + 1]
qo_buf.np[T + 1 : T_pad + 1] = T
attn_metadata.qo_indptr = qo_buf.copy_to_gpu(T_pad + 1)
attn_metadata.empty_kv_indptr = self.model_runner.forward_vars[
"v4_empty_kv_indptr"
][: T_pad + 1]

def _build_paged_prefill_meta(
self,
Expand Down Expand Up @@ -4319,6 +4329,9 @@ def _alloc_v4_metadata_buffers(self) -> None:
# the per-fwd cost is a slice + H2D.
bufs["v4_qo_indptr"] = CpuGpuBuffer(T_dec + 1, **i32)
self._v4_qo_indptr_np = np.arange(T_dec + 1, dtype=np.int32)
# Immutable, device-only empty CSR for reusing the H=128 sparse-prefill
# ASM kernel in decode. Shared read-only across layers and TBO ubatches.
bufs["v4_empty_kv_indptr"] = torch.zeros(T_dec + 1, **i32)
# Per-seq `ctx_len // 4` (raw, no clamp). Consumed by the indexer's
# `cu_committed` cumsum — per-SEQUENCE, host-side, prefill only.
# Single per-token mapping shared across ALL V4 consumers:
Expand Down
72 changes: 69 additions & 3 deletions atom/model_ops/v4_kernels/paged_decode.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@

from atom.model_ops.sparse_attn_v4 import _sparse_attn_ragged_torch
from atom.model_ops.v4_kernels.pool_index import row_offset
from atom.utils import envs
from atom.utils.decorators import mark_trace

LOG2E = 1.4426950408889634 # log2(e); folded into qk_scale so softmax can use exp2.
Expand Down Expand Up @@ -911,6 +912,50 @@ def sparse_attn_v4_paged_decode_reference(
return _sparse_attn_ragged_torch(q, unified_kv, attn_sink, topk_idxs, softmax_scale)


def _sparse_attn_v4_paged_decode_prefill_asm(
unified_kv: torch.Tensor,
kv_indices: torch.Tensor,
kv_indptr: torch.Tensor,
empty_kv_indptr: torch.Tensor,
attn_sink: torch.Tensor,
softmax_scale: float,
unified_kv_rope: torch.Tensor,
q_packed_in: torch.Tensor,
q_rope_in: torch.Tensor,
) -> torch.Tensor:
"""Run H=128 fp8 decode through the sparse-prefill ASM kernel.

Decode has no flat extend source: its current token has already been
written to ``unified_kv`` and included in ``kv_indices``. Feed the decode
CSR as the prefill kernel's prefix stream and an empty CSR as its extend
stream. The KV tensors for that unused stream alias the unified pools so
the launch needs no per-layer dummy allocation.
"""
from aiter.ops.mla_sparse_prefill import mla_sparse_prefill_fp8_asm

n = q_packed_in.shape[0]
if empty_kv_indptr.numel() < n + 1:
raise ValueError(
"V4 prefill-ASM decode needs an all-zero empty_kv_indptr with at "
f"least N+1={n + 1} entries, got {empty_kv_indptr.numel()}"
)

return mla_sparse_prefill_fp8_asm(
q_nope=q_packed_in,
q_rope=q_rope_in,
unified_kv_nope=unified_kv,
unified_kv_rope=unified_kv_rope,
kv_indices_prefix=kv_indices,
kv_indptr_prefix=kv_indptr[: n + 1],
kv_nope=unified_kv,
kv_rope=unified_kv_rope,
kv_indices_extend=kv_indices[:0],
kv_indptr_extend=empty_kv_indptr[: n + 1],
attn_sink=attn_sink,
softmax_scale=softmax_scale,
)


def _sparse_attn_v4_paged_decode_asm(
unified_kv: torch.Tensor,
kv_indices: torch.Tensor,
Expand Down Expand Up @@ -1065,20 +1110,41 @@ def sparse_attn_v4_paged_decode(
q_rope_in: torch.Tensor | None = None,
qo_indptr: torch.Tensor | None = None,
kv_last_page_lens: torch.Tensor | None = None,
empty_kv_indptr: torch.Tensor | None = None,
prefix: str = "",
) -> torch.Tensor:
"""V4 decode sparse attention over a unified KV pool with paged indices.

Native 2buff fp8 (``unified_kv_rope`` provided): routes to the aiter asm
kernel (op5) with pre-packed fp8 Q (``q_packed_in``/``q_rope_in``); the
fp8 NoPE pool + bf16 RoPE pool are read with no requant.
Native 2buff fp8 (``unified_kv_rope`` provided): normally routes to the
aiter decode ASM kernel (op5). With
``ATOM_USE_V4_PREFILL_ASM_FOR_DECODE=1``, gfx1250 H=128 instead reuses the
sparse-prefill ASM kernel with an empty extend stream. Both paths consume
pre-packed fp8 Q and the fp8 NoPE + bf16 RoPE pools with no requant.

Otherwise (bf16): the existing Triton / reference path. When ``kv_scales``
is provided, ``unified_kv`` must be fp8 (e4m3fnuz) and is dequantized
in-kernel using 1xGROUP_SIZE (default 64) block scales (legacy 1buff,
unreachable from the model).
"""
if unified_kv_rope is not None:
if (
envs.ATOM_USE_V4_PREFILL_ASM_FOR_DECODE
and q_packed_in is not None
and q_packed_in.shape[1] == 128
and empty_kv_indptr is not None
and get_gfx() == "gfx1250"
):
return _sparse_attn_v4_paged_decode_prefill_asm(
unified_kv,
kv_indices,
kv_indptr,
empty_kv_indptr,
attn_sink,
softmax_scale,
unified_kv_rope,
q_packed_in,
q_rope_in,
)
return _sparse_attn_v4_paged_decode_asm(
unified_kv,
kv_indices,
Expand Down
6 changes: 4 additions & 2 deletions atom/models/deepseek_v4.py
Original file line number Diff line number Diff line change
Expand Up @@ -3143,8 +3143,9 @@ def _sparse_attention(
kv_indices = attn_md.kv_indices_hca
kv_indptr = attn_md.kv_indptr_hca
# Dispatch on kv-cache layout inside the wrapper: fp8 2buff
# (unified_kv_rope set) → aiter asm op5 with pre-packed fp8 Q + the
# 2buff fp8/bf16 pools read with no requant; bf16 (unified_kv_rope
# (unified_kv_rope set) → aiter ASM with pre-packed fp8 Q + the
# 2buff fp8/bf16 pools read with no requant. The optional H=128
# prefill-ASM decode route also consumes the shared empty CSR.
o = sparse_attn_v4_paged_decode(
qkn.q_sa,
self.unified_kv,
Expand All @@ -3156,6 +3157,7 @@ def _sparse_attention(
q_packed_in=qkn.q_packed,
q_rope_in=qkn.q_rope,
qo_indptr=attn_md.qo_indptr,
empty_kv_indptr=attn_md.empty_kv_indptr,
prefix=f"{self.layer_name}.sparse_attn_decode",
) # [S, H, head_dim]
else:
Expand Down
5 changes: 5 additions & 0 deletions atom/utils/envs.py
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,11 @@ def _positive_float_env(name: str, default: str) -> float:
"ATOM_FORCE_V4_PREFILL_OPUS": lambda: (
os.getenv("ATOM_FORCE_V4_PREFILL_OPUS", "0") == "1"
),
# Reuse the gfx1250 H=128 sparse-prefill ASM kernel for DeepSeek-V4 fp8
# decode. Ineligible shapes keep the dedicated decode ASM path.
"ATOM_USE_V4_PREFILL_ASM_FOR_DECODE": lambda: (
os.getenv("ATOM_USE_V4_PREFILL_ASM_FOR_DECODE", "0") == "1"
),
# Use gluon pa decode for some models
"ATOM_USE_GLUON_PA_DECODE": lambda: (
os.getenv("ATOM_USE_GLUON_PA_DECODE", "0") == "1"
Expand Down
8 changes: 8 additions & 0 deletions tests/test_envs.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
"ATOM_DP_BASE_PORT",
"ATOM_USE_TRITON_GEMM",
"ATOM_USE_TRITON_MXFP4_BMM",
"ATOM_USE_V4_PREFILL_ASM_FOR_DECODE",
"ATOM_MHC_USE_BF16",
"ATOM_ENABLE_QK_NORM_ROPE_CACHE_QUANT_FUSION",
"ATOM_ENABLE_DS_INPUT_RMSNORM_QUANT_FUSION",
Expand Down Expand Up @@ -80,6 +81,9 @@ def test_mhc_use_bf16_default(self):
def test_use_triton_gemm_default(self):
assert _get_envs().ATOM_USE_TRITON_GEMM is False

def test_use_v4_prefill_asm_for_decode_default_disabled(self):
assert _get_envs().ATOM_USE_V4_PREFILL_ASM_FOR_DECODE is False

def test_ds_input_rmsnorm_quant_fusion_default_enabled(self):
assert _get_envs().ATOM_ENABLE_DS_INPUT_RMSNORM_QUANT_FUSION is True

Expand Down Expand Up @@ -150,6 +154,10 @@ def test_dp_port_overrides(self, monkeypatch):
assert _get_envs().ATOM_DP_MASTER_PORT == 29700
assert _get_envs().ATOM_DP_BASE_PORT == 29800

def test_use_v4_prefill_asm_for_decode_enabled(self, monkeypatch):
monkeypatch.setenv("ATOM_USE_V4_PREFILL_ASM_FOR_DECODE", "1")
assert _get_envs().ATOM_USE_V4_PREFILL_ASM_FOR_DECODE is True

def test_torch_profiler_dir_override(self, monkeypatch):
monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", "/tmp/prof")
assert _get_envs().ATOM_TORCH_PROFILER_DIR == "/tmp/prof"
Expand Down
113 changes: 113 additions & 0 deletions tests/test_v4_prefill_asm_decode_dispatch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
# SPDX-License-Identifier: MIT
"""Dispatch coverage for reusing the V4 H=128 prefill ASM in decode."""

from __future__ import annotations

import pytest
import torch

pytest.importorskip("triton", reason="paged_decode defines Triton kernels")
pytest.importorskip("aiter", reason="paged_decode imports the AITER runtime")

from aiter.ops import mla_sparse_prefill

from atom.model_ops.v4_kernels import paged_decode


def _dispatch(
monkeypatch,
*,
enabled: bool,
heads: int = 128,
gfx: str = "gfx1250",
with_empty_indptr: bool = True,
):
monkeypatch.setattr(
paged_decode.envs, "ATOM_USE_V4_PREFILL_ASM_FOR_DECODE", enabled
)
monkeypatch.setattr(paged_decode, "get_gfx", lambda: gfx)
monkeypatch.setattr(
paged_decode,
"_sparse_attn_v4_paged_decode_prefill_asm",
lambda *args, **kwargs: "prefill",
)
monkeypatch.setattr(
paged_decode,
"_sparse_attn_v4_paged_decode_asm",
lambda *args, **kwargs: "decode",
)

n = 2
return paged_decode.sparse_attn_v4_paged_decode(
q=torch.empty((n, heads, 512)),
unified_kv=torch.empty((4, 512)),
kv_indices=torch.empty(0, dtype=torch.int32),
kv_indptr=torch.zeros(n + 1, dtype=torch.int32),
attn_sink=torch.empty(heads),
softmax_scale=512**-0.5,
unified_kv_rope=torch.empty((4, 64)),
q_packed_in=torch.empty((n, heads, 512)),
q_rope_in=torch.empty((n, heads, 64)),
qo_indptr=torch.arange(n + 1, dtype=torch.int32),
empty_kv_indptr=(
torch.zeros(n + 1, dtype=torch.int32) if with_empty_indptr else None
),
)


def test_enabled_ep4_head128_uses_prefill_asm(monkeypatch):
assert _dispatch(monkeypatch, enabled=True) == "prefill"


def test_decode_csr_becomes_prefix_and_extend_is_empty(monkeypatch):
captured = {}
sentinel = object()

def fake_prefill_asm(**kwargs):
captured.update(kwargs)
return sentinel

monkeypatch.setattr(
mla_sparse_prefill, "mla_sparse_prefill_fp8_asm", fake_prefill_asm
)
n, h = 2, 128
q_nope = torch.empty((n, h, 512))
q_rope = torch.empty((n, h, 64))
unified_kv = torch.empty((4, 512))
unified_kv_rope = torch.empty((4, 64))
kv_indices = torch.tensor([0, 1, 2], dtype=torch.int32)
kv_indptr = torch.tensor([0, 1, 3], dtype=torch.int32)
empty_kv_indptr = torch.zeros(n + 1, dtype=torch.int32)

result = paged_decode._sparse_attn_v4_paged_decode_prefill_asm(
unified_kv,
kv_indices,
kv_indptr,
empty_kv_indptr,
torch.empty(h),
512**-0.5,
unified_kv_rope,
q_nope,
q_rope,
)

assert result is sentinel
assert torch.equal(captured["kv_indptr_prefix"], kv_indptr)
assert torch.equal(captured["kv_indices_prefix"], kv_indices)
assert captured["kv_nope"] is unified_kv
assert captured["kv_rope"] is unified_kv_rope
assert captured["kv_indices_extend"].numel() == 0
assert torch.count_nonzero(captured["kv_indptr_extend"]) == 0


@pytest.mark.parametrize(
"kwargs",
[
{"enabled": False},
{"enabled": True, "heads": 64},
{"enabled": True, "gfx": "gfx950"},
{"enabled": True, "with_empty_indptr": False},
],
)
def test_ineligible_decode_keeps_dedicated_asm(monkeypatch, kwargs):
assert _dispatch(monkeypatch, **kwargs) == "decode"
Loading