Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions atom/model_ops/minimax_m3/decode_score_policy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
"""Pure-Python launch policy for the MiniMax-M3 decode scorer."""


def decode_score_num_warps(
arch: str, batch: int, max_block: int, max_query_len: int
) -> int:
if arch == "gfx1250" and max_query_len in (1, 4) and batch >= 4 and max_block > 512:
return 2
return 4


def use_transposed_decode_score(
arch: str, batch: int, max_block: int, max_query_len: int
) -> bool:
if arch != "gfx1250" or max_query_len not in (1, 4) or batch <= 0 or max_block <= 0:
return False
work = batch * max_block
if max_query_len == 1:
return work >= 12_000
return (max_block >= 1_000 and batch >= 4) or (max_block >= 500 and work >= 8_000)
138 changes: 130 additions & 8 deletions atom/model_ops/minimax_m3/index_topk.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,12 @@
# which couples to vLLM internals. ATOM imports triton directly everywhere else.
import triton
import triton.language as tl
from aiter.jit.utils.chip_info import get_gfx
from aiter.jit.utils.chip_info import get_gfx, get_gfx_runtime

from .decode_score_policy import (
decode_score_num_warps,
use_transposed_decode_score,
)

try:
# aiter's chunk-narrowing per-row selector: same forced-block pins and tie
Expand Down Expand Up @@ -644,8 +649,10 @@ def _decode_score_chunks(batch: int, max_block: int) -> int:
A count, not a size: the grid is (request, chunk), so this IS the second
grid dim and it must stay shape-constant for a cuda graph to replay it.
Enough chunks that a long context is not serialized inside a handful of
CTAs, floored so no chunk shrinks to a single block -- DECODE_SCORE_MIN_BLOCKS
is what bounds the split, and TARGET_GRID still caps the largest batches.
CTAs. This shared legacy policy uses DECODE_SCORE_MIN_BLOCKS to amortize
query loads, while TARGET_GRID caps large batches. Native gfx1250 decode
has a separately measured finer-split exception in _native_decode_score_chunks;
CP continues to use this shared policy.

The round trip through the size is not redundant. The caller turns this
count back into a size with the same cdiv, and a count that does not divide
Expand All @@ -663,13 +670,30 @@ def _decode_score_chunks(batch: int, max_block: int) -> int:
# wait in the collective that follows.
return 1
chunks = min(1 << (target.bit_length() - 1), max_block)
# Floor the blocks one chunk walks. The query tile is loaded once outside
# the block loop, so at one block per chunk that fixed cost is the whole
# cost -- every measured regression was a chunk down to a single block.
# Shared legacy floor: amortize the query-tile load across several blocks.
# This is not a universal minimum; _native_decode_score_chunks below uses
# a measured one/two-block exception for small native gfx1250 grids.
chunks = min(chunks, max(1, triton.cdiv(max_block, DECODE_SCORE_MIN_BLOCKS)))
return triton.cdiv(max_block, triton.cdiv(max_block, chunks))


def _native_decode_score_chunks(batch: int, max_block: int) -> int:
"""Measured native decode policy; CP keeps the shared legacy policy.

The native tiled scorer was measured on gfx1250 batch1/2. The CP scorer
has a different kernel and local-page traversal, so it must not inherit
this launch tuning without its own validation.
"""
if (
0 < batch <= 2
and 0 < max_block
and batch * max_block <= 1024
and get_gfx() == "gfx1250"
):
return max_block if batch * max_block <= 256 else triton.cdiv(max_block, 2)
return _decode_score_chunks(batch, max_block)


def _require_packable(max_block: int) -> None:
"""The packed key spends its low 16 bits on the 1-based block id.

Expand Down Expand Up @@ -1102,6 +1126,97 @@ def _decode_index_score_tiled_kernel(
)


# Wide-grid gfx1250 variant. It is separate so launch-limited fallback shapes
# retain the exact original compiled kernel.
@triton.heuristics(
{
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
"BLOCK_SIZE_N": lambda args: max(
16, triton.next_power_of_2(args["NUM_IDX_HEADS"] * args["MAX_Q"])
),
}
)
@triton.jit
def _decode_index_score_transposed_kernel(
q_ptr,
ik_cache_ptr,
score_ptr,
block_table_ptr,
seq_lens,
head_dim,
sm_scale,
chunk_blocks,
stride_q_n,
stride_q_h,
stride_q_d,
stride_ik_blk,
stride_ik_pos,
stride_ik_d,
stride_s_h,
stride_s_b,
stride_s_k,
stride_bt_b,
NUM_IDX_HEADS: tl.constexpr,
MAX_Q: tl.constexpr,
BLOCK_SIZE_K: tl.constexpr,
BLOCK_SIZE_D: tl.constexpr,
BLOCK_SIZE_N: tl.constexpr,
):
pid_b = tl.program_id(0)
pid_chunk = tl.program_id(1)

seq_len = tl.load(seq_lens + pid_b)
num_blocks = (seq_len + BLOCK_SIZE_K - 1) // BLOCK_SIZE_K
chunk_start = pid_chunk * chunk_blocks
chunk_end = tl.minimum(chunk_start + chunk_blocks, num_blocks)
if chunk_start >= chunk_end:
return

off_n = tl.arange(0, BLOCK_SIZE_N)
tok = off_n // NUM_IDX_HEADS
head = off_n - tok * NUM_IDX_HEADS
n_valid = off_n < NUM_IDX_HEADS * MAX_Q
off_d = tl.arange(0, BLOCK_SIZE_D)
d_mask = off_d < head_dim
off_k = tl.arange(0, BLOCK_SIZE_K)
row = pid_b * MAX_Q + tok

# Exchange the two non-reduction axes so converted BF16 K is WMMA's N
# operand. The D reduction order and FP32 accumulation stay unchanged.
q = tl.load(
q_ptr
+ row[:, None] * stride_q_n
+ head[:, None] * stride_q_h
+ off_d[None, :] * stride_q_d,
mask=n_valid[:, None] & d_mask[None, :],
other=0.0,
)
causal_len = seq_len - MAX_Q + tok + 1
bt_row = block_table_ptr + pid_b * stride_bt_b
sm_scale_log2e = sm_scale * 1.4426950409

for blk in tl.range(chunk_start, chunk_end):
page = tl.load(bt_row + blk).to(tl.int64)
k = tl.load(
ik_cache_ptr
+ page * stride_ik_blk
+ off_k[None, :] * stride_ik_pos
+ off_d[:, None] * stride_ik_d,
mask=d_mask[:, None],
other=0.0,
cache_modifier=".cv",
)
qk = tl.dot(q, k.to(q.dtype), out_dtype=tl.float32) * sm_scale_log2e
pos = blk * BLOCK_SIZE_K + off_k
qk = tl.where(pos[None, :] < causal_len[:, None], qk, float("-inf"))
score = tl.max(qk, axis=1)
tl.store(
score_ptr + head * stride_s_h + row * stride_s_b + blk * stride_s_k,
score,
mask=n_valid,
)


# ---------------------------------------------------------------------------
# Python wrappers
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -1301,8 +1416,14 @@ def minimax_m3_index_topk_decode(
dtype=torch.float32,
device=idx_q.device,
)
num_score_chunks = _decode_score_chunks(batch, max_block)
_decode_index_score_tiled_kernel[(batch, num_score_chunks)](
num_score_chunks = _native_decode_score_chunks(batch, max_block)
arch = get_gfx_runtime()
score_kernel = (
_decode_index_score_transposed_kernel
if use_transposed_decode_score(arch, batch, max_block, max_query_len)
else _decode_index_score_tiled_kernel
)
score_kernel[(batch, num_score_chunks)](
idx_q,
index_kv_cache,
score,
Expand All @@ -1324,6 +1445,7 @@ def minimax_m3_index_topk_decode(
NUM_IDX_HEADS=num_idx_heads,
MAX_Q=max_query_len,
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
num_warps=decode_score_num_warps(arch, batch, max_block, max_query_len),
num_stages=DECODE_SCORE_NUM_STAGES,
)

Expand Down
42 changes: 42 additions & 0 deletions tests/test_minimax_m3_decode_score_policy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# SPDX-License-Identifier: MIT

import pytest

from atom.model_ops.minimax_m3.decode_score_policy import (
decode_score_num_warps,
use_transposed_decode_score,
)


@pytest.mark.parametrize(
"arch,batch,max_block,max_q,want",
[
("gfx950", 128, 2000, 1, 4),
("gfx1250", 1, 2000, 1, 4),
("gfx1250", 4, 512, 4, 4),
("gfx1250", 4, 513, 1, 2),
("gfx1250", 128, 2000, 4, 2),
("gfx1250", 128, 2000, 2, 4),
],
)
def test_decode_score_num_warps(arch, batch, max_block, max_q, want):
assert decode_score_num_warps(arch, batch, max_block, max_q) == want


@pytest.mark.parametrize(
"arch,batch,max_block,max_q,want",
[
("gfx950", 128, 2000, 4, False),
("gfx1250", 0, 2000, 4, False),
("gfx1250", 7, 1500, 1, False),
("gfx1250", 8, 1500, 1, True),
("gfx1250", 4, 999, 4, False),
("gfx1250", 4, 1000, 4, True),
("gfx1250", 15, 500, 4, False),
("gfx1250", 16, 500, 4, True),
("gfx1250", 6, 2000, 2, False),
("gfx1250", 6, 2000, 3, False),
],
)
def test_transposed_decode_score_policy(arch, batch, max_block, max_q, want):
assert use_transposed_decode_score(arch, batch, max_block, max_q) is want
Loading
Loading