Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 15 additions & 10 deletions atom/model_ops/minimax_m3/index_topk.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
# which couples to vLLM internals. ATOM imports triton directly everywhere else.
import triton
import triton.language as tl
from aiter.jit.utils.chip_info import get_gfx
from aiter.jit.utils.chip_info import get_gfx, get_gfx_runtime

try:
# aiter's chunk-narrowing per-row selector: same forced-block pins and tie
Expand Down Expand Up @@ -588,14 +588,9 @@ def _score_tile_keys(
# because the next shape down (68 blocks) prefers 128 over 64 by 10%, so
# lowering it is not a one-line change; it is worth ~0.1ms of a prefill step.
#
# Decode cannot derive it, because it is captured: the wrapper's Python runs
# once, at capture, and capture is handed max_seqlen_k = max_model_len so the
# grid covers every replay. A derived width would therefore be the
# longest-context one on every replay -- (1024, 8) here, which is 1.4-1.7x off
# the per-shape best at 64 blocks. So decode takes a constant, chosen by total
# regret over the deployed shapes. That regret is real and mostly irreducible:
# the per-shape optima span (64, 1) to (1024, 8), and the best single pair,
# (256, 4), is only 0.6% better than this one over 16 shapes.
# Decode can only use the capture-time max-block bound, never the live replay
# length. Its width is selected from that bound and remains fixed for every
# replay of the graph.
#
# Neither phase may autotune, and not for the usual reason: triton's key here
# would be BLOCK_SIZE_T == next_pow2(topk) == 16, a constant, so the first shape
Expand All @@ -614,6 +609,16 @@ def _score_tile_keys(
DECODE_TOPK_NUM_WARPS = 8


def _decode_topk_block_size_k(arch: str, max_block: int) -> int:

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nothing pins the new policy.. add a parametrized test for the 512/513 and 2047/2048 boundaries and the non-gfx1250 default?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added parameterized coverage for gfx1250 512/513 and 2047/2048, plus the non-gfx1250 default, in 40af5ad. Focused tests, Black, and Ruff pass.

if arch != "gfx1250" or max_block <= 512:
return DECODE_TOPK_BLOCK_SIZE_K
# At 2048+ blocks the production emit path normally selects aiter first.
# This tile serves calls without published bounds or shapes aiter declines.
if max_block >= 2048:
return 2048

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

at 2048 with emit the shipped path already takes aiter, so this tile is only the no-aiter fallback? leave a comment so we know.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added the fallback-path comment in 40af5ad. The fixed-bound retest also confirms that at 2048+ the production emit path normally selects aiter, so this tile is not a production optimization there.

return 1024


@functools.cache
def _compute_units(device_index: int) -> int:
return torch.cuda.get_device_properties(device_index).multi_processor_count
Expand Down Expand Up @@ -1341,7 +1346,7 @@ def minimax_m3_index_topk_decode(
num_idx_heads,
rows_per_req=max_query_len,
decode_max_q=max_query_len,
block_size_k=DECODE_TOPK_BLOCK_SIZE_K,
block_size_k=_decode_topk_block_size_k(get_gfx_runtime(), max_block),
num_warps=DECODE_TOPK_NUM_WARPS,
emit=emit_sparse_block_table,
block_page_stride=block_page_stride,
Expand Down
14 changes: 14 additions & 0 deletions tests/test_minimax_m3_index_topk.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,20 @@ def test_prefill_score_launch_config(monkeypatch, arch, dtype, want):
m._prefill_score_launch_config.cache_clear()


@pytest.mark.parametrize(
"arch,max_block,want",
[
("gfx950", 2048, 512),
("gfx1250", 512, 512),
("gfx1250", 513, 1024),
("gfx1250", 2047, 1024),
("gfx1250", 2048, 2048),
],
)
def test_decode_topk_block_size_k(arch, max_block, want):
assert m._decode_topk_block_size_k(arch, max_block) == want


class TestPrefillTileWidth:
"""clamp(next_pow2(max_block), MIN, MAX), and the two ends of the clamp."""

Expand Down
Loading