Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions atom/model_ops/attention_mha.py
Original file line number Diff line number Diff line change
Expand Up @@ -1734,6 +1734,7 @@ def _sparse_prefill(
emit_sparse_block_table=True,
n_valid_column_per_row=sparse_metadata.n_valid_column_per_row,
index_scorer=self._index_scorer(sparse_metadata),
score_workspace=prefill_md.score_workspace,
**self._select_page_kwargs,
)
self._store_cached_topk(
Expand Down
17 changes: 16 additions & 1 deletion atom/model_ops/attentions/aiter_attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@
from atom.distributed.ulysses_sp import local_num_kv_heads
from atom.model_engine.scheduler import ScheduledBatch
from atom.model_ops.attention_mha import PagedAttentionImpl, use_pa_decode_bf16_asm
from atom.model_ops.minimax_m3.index_buffers import allocate_index_scorers
from atom.model_ops.minimax_m3.index_buffers import (
allocate_index_scorers,
allocate_prefill_scores,
)
from atom.model_ops.minimax_m3.sparse_attn import (
ASM_PAGE_SIZE,
make_sparse_decode_metadata,
Expand Down Expand Up @@ -302,6 +305,17 @@ def __init__(
if self._has_sparse_attention
else {}
)
self._prefill_score_workspace = (
allocate_prefill_scores(
self.model_runner.config,
self.device,
max_bs=self.max_bs,
max_qlen=max_qlen,
num_idx_heads=self._num_idx_heads,
)
if self._has_sparse_attention
else None
)
self._pa_decode_bf16_asm_enabled = (
use_pa_decode_bf16_asm() and model_runner.block_size == 256
)
Expand Down Expand Up @@ -960,6 +974,7 @@ def _finish_prefill_batch(self, batch, attn_metadata, prefix=""):
num_prefill_tokens=batch.scheduled_tokens,
num_idx_heads=self._num_idx_heads,
index_scorer=self._index_scorers.get(""),
score_workspace=self._prefill_score_workspace,
)

def _refuse_ragged_on_rectangular_layers(
Expand Down
35 changes: 29 additions & 6 deletions atom/model_ops/minimax_m3/index_buffers.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# SPDX-License-Identifier: MIT
# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.
"""The MXFP4 index scorer's persistent state as one record, and its allocation, in a
module that imports no aiter or triton at the top: the kernels' modules (`sparse_attn`,
`index_topk`) import aiter, and the non-GPU test suite has to be able to build the
record."""
"""The index scorers' persistent state (the MXFP4 record, the bf16 / FP8 prefill
scores) and its allocation, in a module that imports no aiter or triton at the top:
the kernels' modules (`sparse_attn`, `index_topk`) import aiter, and the non-GPU test
suite has to be able to build the record."""

from dataclasses import dataclass, replace

Expand Down Expand Up @@ -48,6 +48,28 @@ def _shard(cp_world: int, cp_rank: int) -> dict:
return {"cp_world": cp_world, "cp_rank": cp_rank}


def prefill_score_capacity(
config, *, max_bs: int, max_qlen: int, num_idx_heads: int
) -> int:
"""fp32 block scores of the widest prefill step: one row per (index head, query
token) of a chunk or a verify step, over the model length's blocks."""
rows = max(max_bs * max_qlen, config.max_num_batched_tokens)
return num_idx_heads * rows * -(-config.max_model_len // SPARSE_BLOCK_SIZE)


def allocate_prefill_scores(
config, device, *, max_bs: int, max_qlen: int, num_idx_heads: int
) -> LogitsWorkspace | None:
"""The bf16 / FP8 index caches' prefill score workspace, sized from the
configuration; None on MXFP4, whose record holds its own."""
if config.index_cache_dtype == "fp4":
return None
capacity = prefill_score_capacity(
config, max_bs=max_bs, max_qlen=max_qlen, num_idx_heads=num_idx_heads
)
return LogitsWorkspace(capacity, device)


def allocate_index_scorers(
config,
device,
Expand Down Expand Up @@ -101,9 +123,10 @@ def work_map(batch: int, query_len: int) -> torch.Tensor:
)
return torch.empty((rows, 2), dtype=torch.int32, device=device)

rows = max(max_bs * max_qlen, config.max_num_batched_tokens)
capacity = max(
num_idx_heads * rows * max_block,
prefill_score_capacity(
config, max_bs=max_bs, max_qlen=max_qlen, num_idx_heads=num_idx_heads
),
decode_heads * max_bs * max_qlen * decode_block,
)
scorers = {
Expand Down
15 changes: 10 additions & 5 deletions atom/model_ops/minimax_m3/index_topk.py
Original file line number Diff line number Diff line change
Expand Up @@ -1156,14 +1156,16 @@ def minimax_m3_index_topk(
n_valid_column_per_row: torch.Tensor | None = None,
pages_per_block: int = PAGES_PER_SPARSE_BLOCK,
index_scorer=None,
score_workspace=None,
):
"""Index block-score + top-k selection. block_size_q == 1 (per-token).

With an ``index_scorer`` (an `IndexScoreBuffers`, MXFP4 index cache only)
``index_kv_cache`` is the uint8 ``[num_blocks, 8704]`` page cache and aiter's
FlyDSL prefill scorer writes the scores into ``index_scorer.scores`` (shared by
every layer, never filled: the selection reads a row only up to its valid
blocks). Without one the Triton scorer runs. ``pages_per_block`` /
blocks). Without one the Triton scorer runs, into ``score_workspace`` (a
`LogitsWorkspace`, the same way) or else a fresh buffer. ``pages_per_block`` /
``block_page_stride`` describe the emitted block table: 8 / 8 page-16 SHUFFLE,
1 / 1 MXFP4 KV.

Expand Down Expand Up @@ -1225,6 +1227,7 @@ def minimax_m3_index_topk(
max_query_len,
max_block,
sm_scale,
out=score_workspace,
)

topk_idx = torch.empty(
Expand Down Expand Up @@ -1273,13 +1276,15 @@ def _triton_prefill_score(
max_query_len,
max_block,
sm_scale,
out=None,
):
total_q, num_idx_heads, head_dim = idx_q.shape
batch = cu_seqlens_q.shape[0] - 1
score = torch.empty(
(num_idx_heads, total_q, max_block),
dtype=torch.float32,
device=idx_q.device,
shape = (num_idx_heads, total_q, max_block)
score = (
torch.empty(shape, dtype=torch.float32, device=idx_q.device)
if out is None
else out.logits(num_idx_heads * total_q, max_block).view(shape)
)
q_tiles = triton.cdiv(max_query_len, SCORE_BLOCK_SIZE_Q)
chunk_blocks = _score_chunk_blocks(
Expand Down
7 changes: 7 additions & 0 deletions atom/model_ops/minimax_m3/sparse_attn.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
build_n_valid_column_per_row,
flydsl_ops,
)
from atom.model_ops.sparse_indexer_chunk import LogitsWorkspace

# Page-16 SHUFFLE layout for the AITER paged-attention path. The KV
# cache is allocated with physical page size 16 (the ASM kernel page), and each
Expand Down Expand Up @@ -71,6 +72,9 @@ class MiniMaxM3SparsePrefillMetadata:
block_table: torch.Tensor
max_query_len: int
max_seq_len: int
score_workspace: LogitsWorkspace | None = None

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

bridge paths still allocate per chunk and stay exposed to the OOM abort, please track the follow-up or pass the workspace there too

"""Where the bf16 / FP8 index scorer writes; None on MXFP4 (its record's) and in
the bridges, which allocate per step."""


@dataclass
Expand Down Expand Up @@ -117,9 +121,11 @@ def make_sparse_prefill_metadata(
num_idx_heads: int,
n_valid_column_per_row_out: torch.Tensor | None = None,
index_scorer: IndexScoreBuffers | None = None,
score_workspace: LogitsWorkspace | None = None,
) -> MiniMaxM3SparseMetadata:
"""``index_scorer`` is the builder's MXFP4 index scorer record (None for any other
index cache); a prefill step uses only its score workspace, so the map is dropped.
``score_workspace`` is the builder's bf16 / FP8 one (None: the scorer allocates).
"""
query_lens = cu_seqlens_q[1 : num_prefills + 1] - cu_seqlens_q[:num_prefills]
prefix_lens = seq_lens - query_lens
Expand All @@ -132,6 +138,7 @@ def make_sparse_prefill_metadata(
block_table=block_table,
max_query_len=max_query_len,
max_seq_len=max_seq_len,
score_workspace=score_workspace,
)
return MiniMaxM3SparseMetadata(
seq_lens=seq_lens,
Expand Down
43 changes: 43 additions & 0 deletions tests/test_minimax_m3_index_buffers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# SPDX-License-Identifier: MIT
"""MiniMax-M3 index score workspaces, sized from the configuration (no aiter)."""

from types import SimpleNamespace

import pytest

from atom.model_ops.minimax_m3.index_buffers import (
allocate_prefill_scores,
prefill_score_capacity,
)


def _config(index_dtype, mnbt=256, max_model_len=4096):
return SimpleNamespace(
index_cache_dtype=index_dtype,
max_num_batched_tokens=mnbt,
max_model_len=max_model_len,
)


@pytest.mark.parametrize("index_dtype", ["bf16", "fp8"])
def test_the_prefill_workspace_holds_a_full_chunk_at_the_model_length(index_dtype):
ws = allocate_prefill_scores(
_config(index_dtype), "cpu", max_bs=4, max_qlen=1, num_idx_heads=4
)
assert ws.logits(4 * 256, 4096 // 128).shape == (1024, 32)
with pytest.raises(ValueError):
ws.logits(4 * 256 + 1, 4096 // 128)


def test_verify_rows_past_the_chunk_widen_it():
config = _config("fp8", mnbt=16)
capacity = prefill_score_capacity(config, max_bs=8, max_qlen=4, num_idx_heads=2)
assert capacity == 2 * (8 * 4) * (4096 // 128)


def test_mxfp4_scores_into_its_own_record():
config = _config("fp4")
assert (
allocate_prefill_scores(config, "cpu", max_bs=4, max_qlen=1, num_idx_heads=4)
is None
)
32 changes: 32 additions & 0 deletions tests/test_minimax_m3_index_topk.py
Original file line number Diff line number Diff line change
Expand Up @@ -763,3 +763,35 @@ def score(config):
torch.cuda.synchronize()
for old_output, new_output in zip(old_outputs, new_outputs):
assert torch.equal(old_output, new_output)


@gpu
@pytest.mark.parametrize("cache", ["bf16", "fp8"])
@pytest.mark.parametrize(
"prefix", [0, m._AITER_MIN_WIDTH_WITH_EMIT * SPARSE_BLOCK_SIZE]
)
def test_chunks_scored_into_one_stale_workspace_select_as_fresh_buffers(cache, prefix):
"""A long prefill's chunks share one workspace, poisoned between them; both
selectors (the prefix decides which) must return what fresh buffers give."""
from atom.model_ops.sparse_indexer_chunk import LogitsWorkspace

heads, chunk, chunks = 2, 256, 3
width = -(-(prefix + chunks * chunk) // SPARSE_BLOCK_SIZE)
ws = LogitsWorkspace(heads * chunk * width, "cuda")
fp8 = torch.float8_e4m3fnuz if m.get_gfx() == "gfx942" else torch.float8_e4m3fn
for i in range(chunks):
kw = _inputs([chunk], [prefix + i * chunk], heads, "cuda")
if cache == "fp8":
kw["index_kv_cache"] = kw["index_kv_cache"].to(fp8)
kw["n_valid_column_per_row"] = m.build_n_valid_column_per_row(
kw["cu_seqlens_q"], kw["prefix_lens"], batch=1, total_q=chunk,
num_idx_heads=heads, decode_max_q=0,
) # fmt: skip
want = m.minimax_m3_index_topk(**kw, emit_sparse_block_table=True)
ws.logits(1, ws.capacity).fill_(float("inf"))
got = m.minimax_m3_index_topk(
**kw, emit_sparse_block_table=True, score_workspace=ws
)
torch.cuda.synchronize()
for fresh, reused in zip(want, got):
assert torch.equal(fresh, reused)
2 changes: 1 addition & 1 deletion tests/test_minimax_m3_mxfp4_decode_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ def attend_fp8(q, k_cache, v_cache, topk_idx, bt, *a, **kw):
prefill=types.SimpleNamespace(
cu_seqlens_q=torch.arange(batch + 1, dtype=torch.int32) * mq,
seq_lens=seq, context_lens=seq, block_table=table, max_query_len=mq,
max_seq_len=100, qo_indptr=cover,
max_seq_len=100, qo_indptr=cover, score_workspace=None,
),
) # fmt: skip
ctx = types.SimpleNamespace(
Expand Down
Loading