Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions atom/model_ops/attention_mha.py
Original file line number Diff line number Diff line change
Expand Up @@ -1551,8 +1551,9 @@ def _sparse_prefill(
return output

def _cp_index_topk_decode(
self, index_q, block_table, seq_lens, max_seq_len, max_query_len
):
self, index_q, block_table, seq_lens, max_seq_len, max_query_len,
index_score_work_map=None, index_score_max_block=0,
): # fmt: skip
"""Decode top-k with the context sharded across the indexer-CP group.

Score all index heads over this rank's 1/P of the blocks, reduce to this
Expand Down Expand Up @@ -1583,6 +1584,11 @@ def _cp_index_topk_decode(
self.indexer_cp_world,
max_query_len,
self.scale,
# Built once per decode step in the metadata against this rank's
# local block bound, never here -- see `indexer_context_scores`.
# None means the scorer builds its own, at ~120us per sparse layer.
work_map=index_score_work_map,
max_block=index_score_max_block,
)
# Forced blocks are pinned in BOTH passes on purpose: one that lost its
# owner shard's top-k would never arrive at the merge to be pinned.
Expand Down Expand Up @@ -1641,6 +1647,8 @@ def _sparse_decode(
decode_md.seq_lens,
sparse_metadata.max_seq_len,
max_query_len,
index_score_work_map=sparse_metadata.index_score_work_map,
index_score_max_block=sparse_metadata.index_score_max_block,
)
else:
topk_idx, sparse_bt, sparse_ctx = minimax_m3_index_topk_decode(
Expand All @@ -1657,6 +1665,13 @@ def _sparse_decode(
emit_sparse_block_table=True,
max_query_len=max_query_len,
n_valid_column_per_row=sparse_metadata.n_valid_column_per_row,
# Built once per decode step in the metadata, never here:
# `make_work_map` is ~15 tiny launches costing ~120 us, so
# per sparse layer it would swamp the kernel it feeds. None
# means the scorer builds its own -- correct, and exactly
# the price this hoist exists to avoid paying 57x a step.
index_score_work_map=sparse_metadata.index_score_work_map,
index_score_max_block=sparse_metadata.index_score_max_block,
)
self._store_cached_topk(
sparse_metadata, topk_key, (topk_idx, sparse_bt, sparse_ctx)
Expand Down
91 changes: 91 additions & 0 deletions atom/model_ops/attentions/aiter_attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,48 @@ def __init__(
num_head_k * self.max_bs * max_qlen,
**i32_kwargs,
)
from atom.distributed.indexer_cp import (
get_indexer_cp_rank,
get_indexer_cp_world_size,
indexer_cp_enabled,
)

# The flydsl index-score kernel's dispatch map, same persistence
# argument. Its row count is that kernel's grid, so the bound has to
# be one that does not move between capture and replay -- the model
# length, never the batch's longest request.
from atom.model_ops.minimax_m3.index_topk import (
SPARSE_BLOCK_SIZE as _M3_SPARSE_BLOCK_SIZE,
)
from atom.model_ops.minimax_m3.index_topk import index_score_work_map_size

# Indexer CP shards the context round-robin, so a rank scores
# ceil(blocks / world) of it and every bound below is that local one.
# Resolved here rather than per step for the same reason the sparse
# layer resolves it in __init__: no per-forward branch in the model.
self._index_score_cp_world = 1
self._index_score_cp_rank = 0
if indexer_cp_enabled():
self._index_score_cp_world = get_indexer_cp_world_size()
self._index_score_cp_rank = get_indexer_cp_rank()
self._index_score_max_block = triton.cdiv(
triton.cdiv(config.max_model_len, _M3_SPARSE_BLOCK_SIZE),
self._index_score_cp_world,
)
rows = index_score_work_map_size(
self.max_bs,
self._index_score_max_block,
max_qlen,
num_head_k,
Comment on lines +330 to +334
self._index_score_cp_world,
self._index_score_cp_rank,
)
Comment on lines +330 to +337
self.model_runner.forward_vars["sparse_attention_index_score_work_map"] = (
torch.empty((rows, 2), **i32_kwargs) if rows else None
)
else:
self._index_score_max_block = 0
self._index_score_cp_world, self._index_score_cp_rank = 1, 0
self._pa_decode_bf16_asm_enabled = (
use_pa_decode_bf16_asm() and model_runner.block_size == 256
)
Expand Down Expand Up @@ -422,6 +464,27 @@ def _allocate_ubatch_buffers(
var[f"{p}sparse_attention_n_valid_column_per_row"] = torch.empty(
self._num_idx_heads * ub_max_bs * max_seqlen_qo, **i32_kwargs
)
from atom.model_ops.minimax_m3.index_topk import (
index_score_work_map_size,
)

# The CP pair has to match the one `_build_ubatch_metadata`
# passes to the builder: the shard's dispatch is denser than the
# whole context's, so a map sized at cp_world=1 is SHORTER than
# the one a CP rank writes -- 64 rows against 128 at bs=8,
# max_block=64 -- and the build rejects the buffer outright
# ("work_map: expected packed [at least N, 2] int32").
rows = index_score_work_map_size(
ub_max_bs,
self._index_score_max_block,
max_seqlen_qo,
self._num_idx_heads,
Comment on lines +477 to +481
self._index_score_cp_world,
self._index_score_cp_rank,
)
var[f"{p}sparse_attention_index_score_work_map"] = (
torch.empty((rows, 2), **i32_kwargs) if rows else None
)

# PA work buffers per ubatch (GPU only)
var[f"{p}work_meta_data"] = torch.empty(
Expand Down Expand Up @@ -747,6 +810,18 @@ def _n_valid_column_per_row_buffer(self, prefix: str = "") -> torch.Tensor:
f"{prefix}sparse_attention_n_valid_column_per_row"
]

def _index_score_work_map_buffer(self, prefix: str = "") -> torch.Tensor | None:
"""The decode work-map buffer, on the same terms as its sibling above.

None when this model never allocated one -- a non-sparse model, or a
prefix whose buffers were not registered. The metadata builder reads
the same absence and leaves the map empty, which costs correctness
nothing: the scorer then builds its own, at ~120 us per sparse layer.
"""
return self.model_runner.forward_vars.get(
f"{prefix}sparse_attention_index_score_work_map"
)

def _get_sparse_attention_block_tables(
self,
block_tables: torch.Tensor,
Expand Down Expand Up @@ -966,6 +1041,10 @@ def prepare_mtp_decode(
max_query_len=1,
num_idx_heads=self._num_idx_heads,
n_valid_column_per_row_out=self._n_valid_column_per_row_buffer(),
index_score_work_map_out=self._index_score_work_map_buffer(),
index_score_max_block=self._index_score_max_block,
index_score_cp_world=self._index_score_cp_world,
index_score_cp_rank=self._index_score_cp_rank,
)
# Same short-circuit as the metadata launch above. A dummy step would
# refresh the live work_info from synthetic lengths, but every real
Expand Down Expand Up @@ -1296,6 +1375,10 @@ def prepare_decode(
max_query_len=int(max_seqlen_q),
num_idx_heads=self._num_idx_heads,
n_valid_column_per_row_out=self._n_valid_column_per_row_buffer(),
index_score_work_map_out=self._index_score_work_map_buffer(),
index_score_max_block=self._index_score_max_block,
index_score_cp_world=self._index_score_cp_world,
index_score_cp_rank=self._index_score_cp_rank,
)
mrope_positions = self._build_mrope_decode_positions(
batch, context_lens, max_seqlen_q, running_tokens=running_tokens
Expand Down Expand Up @@ -1491,6 +1574,10 @@ def build_ubatch_metadata(
max_seq_len=attn.max_seqlen_k,
num_idx_heads=self._num_idx_heads,
n_valid_column_per_row_out=self._n_valid_column_per_row_buffer(p),
index_score_work_map_out=self._index_score_work_map_buffer(p),
index_score_max_block=self._index_score_max_block,
index_score_cp_world=self._index_score_cp_world,
index_score_cp_rank=self._index_score_cp_rank,
)
return attn

Expand Down Expand Up @@ -1537,6 +1624,10 @@ def build_for_cudagraph_capture(self, bs: int) -> AttentionMetaData:
max_query_len=max_q_len,
num_idx_heads=self._num_idx_heads,
n_valid_column_per_row_out=self._n_valid_column_per_row_buffer(),
index_score_work_map_out=self._index_score_work_map_buffer(),
index_score_max_block=self._index_score_max_block,
index_score_cp_world=self._index_score_cp_world,
index_score_cp_rank=self._index_score_cp_rank,
)

positions = var["positions"].copy_to_gpu(scheduled_tokens)
Expand Down
Loading
Loading