Skip to content
Merged
Changes from 17 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
5d620de
Update rocm_aiter_mla.py
larryli2-amd Apr 12, 2026
519e078
Merge pull request #1 from larryli2-amd/larryli2-amd-patch-1
larryli2-amd Apr 12, 2026
1b2a40b
Merge branch 'main' into main
larryli2-amd Apr 12, 2026
0bc57b6
Merge branch 'main' into main
larryli2-amd Apr 13, 2026
9ece574
Merge branch 'main' into main
larryli2-amd Apr 13, 2026
5554777
Update rocm_aiter_mla.py
larryli2-amd Apr 13, 2026
0672b48
Merge branch 'main' into main
larryli2-amd Apr 13, 2026
6358ca9
Merge branch 'main' into main
larryli2-amd Apr 14, 2026
83c30e4
Merge branch 'main' into main
larryli2-amd Apr 14, 2026
a3d5179
Merge branch 'main' into main
larryli2-amd Apr 15, 2026
0eef924
Merge branch 'main' into main
larryli2-amd Apr 15, 2026
2b641f4
Merge branch 'main' into main
larryli2-amd Apr 16, 2026
9fb3832
Merge branch 'main' into main
larryli2-amd Apr 16, 2026
d6617cf
Merge branch 'main' into main
larryli2-amd Apr 17, 2026
78f222a
Merge branch 'main' into main
larryli2-amd Apr 17, 2026
a85a119
Merge branch 'main' into main
larryli2-amd Apr 17, 2026
77b868c
Merge branch 'main' into main
larryli2-amd Apr 18, 2026
d1a885c
Update rocm_aiter_mla.py
larryli2-amd Apr 18, 2026
65c7333
Update rocm_aiter_mla.py
larryli2-amd Apr 18, 2026
556ebaa
Merge branch 'main' into main
larryli2-amd Apr 18, 2026
a4eeb7f
Update attention_backends.md
larryli2-amd Apr 18, 2026
cf7157c
Merge branch 'main' into main
larryli2-amd Apr 18, 2026
61ff878
Update attention_backends.md
larryli2-amd Apr 18, 2026
be725b8
Merge branch 'main' into main
larryli2-amd Apr 19, 2026
21aaf87
Merge branch 'main' into main
larryli2-amd Apr 19, 2026
b8d147b
Merge branch 'main' into main
tjtanaa Apr 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
184 changes: 127 additions & 57 deletions vllm/v1/attention/backends/mla/rocm_aiter_mla.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@ def get_supported_head_sizes(cls) -> list[int]:

@staticmethod
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
return [1]
# The aiter MLA decode kernel always operates with page_size=1
# internally (the wrapper flattens kv_buffer via .view(-1, 1, 1, H)).
# We support any kernel_block_size by expanding block-level indices
# into per-token flat indices in the metadata builder.
return [MultipleOf(1)]

@staticmethod
def get_name() -> str:
Expand Down Expand Up @@ -74,6 +78,8 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
attn_out_dtype: torch.dtype = torch.bfloat16
# The max query output length: int
max_qo_len: int | None = None
# Whether persistent MLA metadata was computed (only for qseqlen=1)
has_persistent_metadata: bool = False


@dataclass
Expand Down Expand Up @@ -105,7 +111,16 @@ def __init__(

self.compilation_config = vllm_config.compilation_config
self.decode_attn_out_dtype = vllm_config.model_config.dtype
# kernel block size is always 1.

# Store the kernel block size from the spec. When kernel_block_size=1
# (no spec-dec), behavior is identical to the original. When > 1
# (e.g. 16 with Eagle3), we expand block-level indices into per-token
# flat indices since the aiter kernel always uses page_size=1 internally.
self.kernel_block_size = kv_cache_spec.block_size

# In the flat view (.view(-1,1,1,H)), each token is its own page,
# so max_num_pages_per_req = max_model_len regardless of
# kernel_block_size.
max_num_pages_per_req = vllm_config.model_config.max_model_len
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
max_num_pages = max_num_reqs * max_num_pages_per_req
Expand All @@ -115,8 +130,9 @@ def __init__(
# so we can only use the persistent buffer if a cudagraph is actually
# being used.

# paged_kv_last_page_len is always 1s (kernel block size is always 1),
# so we create it once and reuse slices in both eager and cudagraph modes.
# paged_kv_last_page_len is always 1s (the aiter kernel always sees
# page_size=1 after .view(-1,1,1,H) flattening), so we create it
# once and reuse slices in both eager and cudagraph modes.
self.paged_kv_last_page_len = torch.ones(
max_num_reqs, dtype=torch.int32, device=device
)
Expand Down Expand Up @@ -196,14 +212,14 @@ def _build_decode(
num_decode_tokens: int,
dcp_tot_seq_lens_device: torch.Tensor | None,
) -> AiterMLADecodeMetadata:
# kernel block size is always 1, although the kv block size is not 1.
device = self.device
num_reqs = seq_lens_device.size(0)

# kernel block size is always 1, so each page has exactly 1 token.
# last_page_len is always 1 - just slice the pre-initialized buffer.
# The aiter kernel always operates with page_size=1 (the wrapper
# flattens kv_buffer). last_page_len is always 1.
paged_kv_last_page_len = self.paged_kv_last_page_len[:num_reqs]

# indptr: cumsum of seq_lens (one page per token in the flat view)
paged_kv_indptr = torch.cat(
[
torch.zeros(1, dtype=seq_lens_device.dtype, device=device),
Expand All @@ -215,11 +231,19 @@ def _build_decode(

if self.compilation_config.cudagraph_mode.has_full_cudagraphs():
self.paged_kv_indices.fill_(-1)
_copy_page_indices_kernel[(num_reqs,)](

# Expand block_table entries into per-token flat indices.

@tjtanaa tjtanaa Apr 19, 2026 •

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NITS: This block of comment is the same as the doc string of _expand_page_indices_kernel , let's remove this to make is less verbose.

# When kernel_block_size=1, this degrades to a direct copy (identical
# to the original _copy_page_indices_kernel).
# When kernel_block_size=K>1, block_table entry b covering K tokens
# gets expanded to flat indices b*K, b*K+1, ..., b*K+(K-1).
_expand_page_indices_kernel[(num_reqs,)](
self.paged_kv_indices,
block_table_tensor,
block_table_tensor.stride(0),
paged_kv_indptr,
seq_lens_device,
KERNEL_BLOCK_SIZE=self.kernel_block_size,
BLOCK_SIZE=1024,
)
paged_kv_indices = self.paged_kv_indices
Expand All @@ -245,27 +269,37 @@ def _build_decode(
0, num_reqs + 1, step=1, dtype=torch.int32, device=device
)

from aiter import get_mla_metadata_v1

get_mla_metadata_v1(
qo_indptr,
paged_kv_indptr,
paged_kv_last_page_len,
self._num_attention_heads,
1,
True,
self._mla_work_meta_data,
self._mla_work_info_set,
self._mla_work_indptr,
self._mla_reduce_indptr,
self._mla_reduce_final_map,
self._mla_reduce_partial_map,
page_size=1,
kv_granularity=16,
max_seqlen_qo=max_qo_len,
uni_seqlen_qo=max_qo_len,
fast_mode=True,
)
# The aiter MLA ASM kernel only supports qseqlen=1 (single-token
# decode). With speculative decoding, the verification step has
# qseqlen > 1 (e.g. 8 for spec7). get_mla_metadata_v1 calls
# get_heuristic_kernel_mla which fails for qseqlen > 1.
# We track whether persistent metadata was successfully computed
# so forward_mqa can skip passing it (falling back to the kernel
# computing its own metadata internally, like v0.18.0).
has_persistent_metadata = False
if max_qo_len == 1:
from aiter import get_mla_metadata_v1

get_mla_metadata_v1(
qo_indptr,
paged_kv_indptr,
paged_kv_last_page_len,
self._num_attention_heads,
1,
True,
self._mla_work_meta_data,
self._mla_work_info_set,
self._mla_work_indptr,
self._mla_reduce_indptr,
self._mla_reduce_final_map,
self._mla_reduce_partial_map,
page_size=1,
kv_granularity=16,
max_seqlen_qo=max_qo_len,
uni_seqlen_qo=max_qo_len,
fast_mode=True,
)
has_persistent_metadata = True

attn_metadata = AiterMLADecodeMetadata(
block_table=block_table_tensor,
Expand All @@ -277,6 +311,7 @@ def _build_decode(
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
max_qo_len=max_qo_len,
attn_out_dtype=self.decode_attn_out_dtype,
has_persistent_metadata=has_persistent_metadata,
)

return attn_metadata
Expand All @@ -290,41 +325,65 @@ def build(
attn_metadata = super().build(
common_prefix_len, common_attn_metadata, fast_build
)
attn_metadata.work_meta_data = self._mla_work_meta_data
attn_metadata.work_indptr = self._mla_work_indptr
attn_metadata.work_info_set = self._mla_work_info_set
attn_metadata.reduce_indptr = self._mla_reduce_indptr
attn_metadata.reduce_final_map = self._mla_reduce_final_map
attn_metadata.reduce_partial_map = self._mla_reduce_partial_map
if (attn_metadata.decode is not None
and attn_metadata.decode.has_persistent_metadata):
attn_metadata.work_meta_data = self._mla_work_meta_data
attn_metadata.work_indptr = self._mla_work_indptr
attn_metadata.work_info_set = self._mla_work_info_set
attn_metadata.reduce_indptr = self._mla_reduce_indptr
attn_metadata.reduce_final_map = self._mla_reduce_final_map
attn_metadata.reduce_partial_map = self._mla_reduce_partial_map
return attn_metadata


@triton.jit
def _copy_page_indices_kernel(
def _expand_page_indices_kernel(
page_indices,
block_table,
block_table_stride,
cu_num_blocks,
cu_num_tokens,
seq_lens,
KERNEL_BLOCK_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
"""Copy block table rows into a flat page_indices buffer using indptr.
Avoids blocking boolean mask indexing (tensor[mask]) which has
data-dependent output size and forces sync.
This is the same kernel as introduced in backends/flashinfer.py.
"""Expand block table entries into per-token flat page indices.

The aiter MLA kernel always operates with page_size=1 internally
(kv_buffer is flattened via .view(-1, 1, 1, H)). This kernel converts
block-level indices from the block table into individual token positions
in the flattened KV buffer.

When KERNEL_BLOCK_SIZE=1: block_idx=t, offset=0, flat=block_id
(equivalent to a direct copy -- no regression from the original kernel).

When KERNEL_BLOCK_SIZE=K: block table entry b (covering K tokens)
is expanded to flat indices b*K, b*K+1, ..., b*K+(K-1).
"""
req_idx = tl.program_id(0)
row_ptr = block_table + req_idx * block_table_stride
start_idx = tl.load(cu_num_blocks + req_idx)
end_idx = tl.load(cu_num_blocks + req_idx + 1)
num_blocks = end_idx - start_idx
start_idx = tl.load(cu_num_tokens + req_idx)
num_tokens = tl.load(seq_lens + req_idx)

offset = tl.arange(0, BLOCK_SIZE)
for i in tl.range(0, num_blocks, BLOCK_SIZE):
block_ids = tl.load(row_ptr + i + offset, mask=i + offset < num_blocks)
for i in tl.range(0, num_tokens, BLOCK_SIZE):
token_offsets = i + offset
mask = token_offsets < num_tokens

# Which block in the block table does this token belong to?
block_idx = token_offsets // KERNEL_BLOCK_SIZE
# Offset within that block
offset_in_block = token_offsets % KERNEL_BLOCK_SIZE

# Load the block ID from the block table
block_ids = tl.load(row_ptr + block_idx, mask=mask)

# Compute flat index in the flattened kv_buffer
flat_indices = block_ids * KERNEL_BLOCK_SIZE + offset_in_block

tl.store(
page_indices + start_idx + i + offset,
block_ids,
mask=i + offset < num_blocks,
page_indices + start_idx + token_offsets,
flat_indices,
mask=mask,
)


Expand Down Expand Up @@ -426,6 +485,24 @@ def forward_mqa(

kv_buffer = kv_c_and_k_pe_cache.unsqueeze(2)

# Build kwargs for mla_decode_fwd. Pass persistent metadata only
# when it was successfully computed (qseqlen=1 decode steps).
# For multi-token verification steps (spec-dec), the kernel falls
# back to computing metadata internally.
mla_kwargs = dict(
q_scale=layer._q_scale,
kv_scale=layer._k_scale,
)
if attn_metadata.work_meta_data is not None:
mla_kwargs.update(
work_meta_data=attn_metadata.work_meta_data,
work_indptr=attn_metadata.work_indptr,
work_info_set=attn_metadata.work_info_set,
reduce_indptr=attn_metadata.reduce_indptr,
reduce_final_map=attn_metadata.reduce_final_map,
reduce_partial_map=attn_metadata.reduce_partial_map,
)

rocm_aiter_ops.mla_decode_fwd(
q,
kv_buffer,
Expand All @@ -436,14 +513,7 @@ def forward_mqa(
attn_metadata.decode.paged_kv_indptr,
attn_metadata.decode.paged_kv_indices,
attn_metadata.decode.paged_kv_last_page_len,
q_scale=layer._q_scale,
kv_scale=layer._k_scale,
work_meta_data=attn_metadata.work_meta_data,
work_indptr=attn_metadata.work_indptr,
work_info_set=attn_metadata.work_info_set,
reduce_indptr=attn_metadata.reduce_indptr,
reduce_final_map=attn_metadata.reduce_final_map,
reduce_partial_map=attn_metadata.reduce_partial_map,
**mla_kwargs,
)

if self._needs_head_repeat:
Expand Down
Loading