Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/atom-vllm-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -149,12 +149,12 @@ jobs:
lm_eval_num_fewshot: 5
accuracy_test_threshold: 0.93
runner: atom-mi355-8gpu-vllm-sgl-ci
- display_name: "Kimi-K3 DSpark TP8"
- display_name: "Kimi-K3 DSpark DCP8 TP8"
model_name: "Kimi-K3-DSpark"
model_path: "moonshotai/Kimi-K3"
draft_model_path: "Inferact/Kimi-K3-DSpark"
client_command: "lm_eval --model local-completions --model_args model=${MODEL_PATH},base_url=http://127.0.0.1:${VLLM_PORT}/v1/completions,num_concurrent=64,max_retries=3,tokenized_requests=False,trust_remote_code=True --tasks gsm8k --num_fewshot ${LM_EVAL_NUM_FEWSHOT} --output_path ${OUTPUT_PATH}"
extra_args: "--trust-remote-code --language-model-only --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 16384 --max-num-seqs 64 --max-num-batched-tokens 16384 --gpu-memory-utilization 0.93 --block-size 128 --enable-prefix-caching --mamba-cache-mode align --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' --additional-config '{\"online_quant_config\":{\"global_quant_config\":\"ptpc_fp8\",\"exclude_layer\":[\"lm_head\",\"model.embed_tokens\",\"*self_attn.[qkv]_conv1d*\",\"*block_sparse_moe.experts*\",\"*block_sparse_moe.routed_expert_*\",\"*vision_tower*\",\"*mm_projector*\"]}}' --speculative-config '{\"method\":\"dspark\",\"model\":\"/models/Inferact/Kimi-K3-DSpark\",\"num_speculative_tokens\":7}'"
extra_args: "--trust-remote-code --language-model-only --tensor-parallel-size 8 --decode-context-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 16384 --max-num-seqs 64 --max-num-batched-tokens 16384 --gpu-memory-utilization 0.7 --block-size 128 --enable-prefix-caching --mamba-cache-mode align --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' --additional-config '{\"online_quant_config\":{\"global_quant_config\":\"ptpc_fp8\",\"exclude_layer\":[\"lm_head\",\"model.embed_tokens\",\"*self_attn.[qkv]_conv1d*\",\"*block_sparse_moe.experts*\",\"*block_sparse_moe.routed_expert_*\",\"*vision_tower*\",\"*mm_projector*\"]}}' --speculative-config '{\"method\":\"dspark\",\"model\":\"/models/Inferact/Kimi-K3-DSpark\",\"num_speculative_tokens\":7}'"
lm_eval_num_fewshot: 5
accuracy_test_threshold: 0.95
runner: atom-mi355-8gpu-vllm-sgl-ci
Expand Down
25 changes: 25 additions & 0 deletions atom/plugin/vllm/attention/layer_mla.py
Original file line number Diff line number Diff line change
Expand Up @@ -862,6 +862,27 @@ def _forward_decode(

return_lse = self.dcp_world_size > 1

# DCP + multi-token decode (DSpark verify, MTP): KV is round-robin
# sharded, so a causal intra-block mask has to be placed on GLOBAL
# positions g(j) = j * world + rank rather than on the rank-local row
# order the kernel otherwise sees. Handing over g_kv_indptr plus the
# cp world/rank selects aiter's cprr variant, which does exactly that.
# A single-token decode needs no mask (one query, all local KV), and
# neither does a bidirectional block -- DSpark drafts non-causally, so
# every query legitimately sees every KV row.
decode_md = attn_metadata.decode
cp_world_size = 1
cp_rank = 0
g_kv_indptr = None
if self.dcp_world_size > 1 and decode_md.max_qo_len > 1 and decode_md.causal:
cp_world_size = self.dcp_world_size
cp_rank = self.dcp_rank
g_kv_indptr = decode_md.g_kv_indptr
assert g_kv_indptr is not None, (
"causal multi-token decode under DCP requires "
"attn_metadata.decode.g_kv_indptr from the metadata builder"
)

_, lse = mla_decode_fwd(
q,
kv_buffer.view(-1, 1, 1, q.shape[-1]),
Expand All @@ -881,6 +902,10 @@ def _forward_decode(
q_scale=self._q_scale,
kv_scale=self._k_scale,
return_lse=return_lse,
g_kv_indptr=g_kv_indptr,
cp_world_size=cp_world_size,
cp_rank=cp_rank,
causal=decode_md.causal,
)
if do_fold:
o = o.view(ori_total_s, ori_nhead, -1)
Expand Down
98 changes: 96 additions & 2 deletions atom/plugin/vllm/attention/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,11 @@ class AiterMlaDecodeMetadataForVllm:
# to the whole block. The asm kernel picks a different .co by this and the
# persistent work descriptors are planned for it, so the two have to agree.
causal: bool = True
# Global (pre-DCP-shard) cumulative KV lengths, shape [num_decode + 1].
# Only built for a causal multi-token decode under DCP, where the kernel
# needs them to place the intra-block mask on global positions; None
# otherwise, which is what selects the plain (non-cprr) kernel.
g_kv_indptr: torch.Tensor | None = None
# The fold factor for handling mqa_ratio=64 in non-persistent mode
fold_factor: int | None = None
# Fold buffers for the MLA nhead-fold workaround. These are populated by
Expand Down Expand Up @@ -1036,7 +1041,22 @@ def __init__(
device=None,
model_runner=None,
):
super().__init__(kv_cache_spec, layer_names, config, device)
# supports_dcp_with_varlen: without it vLLM clamps
# reorder_batch_threshold to 1 under DCP, which sends every
# speculative q > 1 batch -- the target's verify pass and DSpark's
# draft block alike -- down the prefill path. That path is causal-only,
# so a bidirectional draft block would come out wrong, and no decode
# batch would ever reach a FULL cudagraph. This backend handles varlen
# DCP decode instead: causal batches through aiter's round-robin (cprr)
# kernel via g_kv_indptr, non-causal ones through the plain kernel,
# which needs no mask at all.
super().__init__(
kv_cache_spec,
layer_names,
config,
device,
supports_dcp_with_varlen=True,
)
logger.info("init AiterMlaMetadataBuilderForVllm")
from vllm.config import VllmConfig

Expand Down Expand Up @@ -1091,6 +1111,24 @@ def __init__(
)
else:
self.persistent_num_heads = self.padded_num_attention_heads
# DCP rank + a persistent buffer for locally derived decode seq lens.
# Only the DSpark draft step needs the buffer (see build()); allocating
# it unconditionally under DCP keeps that branch allocation-free.
self.dcp_rank = 0
self._dcp_local_seq_lens_buf = None
self._g_kv_indptr_buf = None
if self.dcp_world_size > 1:
from vllm.distributed.parallel_state import get_dcp_group

self.dcp_rank = get_dcp_group().rank_in_group
self._dcp_local_seq_lens_buf = torch.zeros(
max_num_reqs, dtype=torch.int32, device=device
)
# Global cumulative KV lengths for the cprr decode kernel. Persistent
# for the same reason: a captured decode graph re-reads it on replay.
self._g_kv_indptr_buf = torch.zeros(
max_num_reqs + 1, dtype=torch.int32, device=device
)
self.block_size = kv_cache_spec.block_size
self.max_bs = max_num_reqs
self.dtype_kv = get_aiter_kv_cache_dtype(config)
Expand Down Expand Up @@ -1223,6 +1261,7 @@ def _set_mla_persistent_worker_buffers(
cu_seqlens_q: torch.Tensor,
max_q_len: int = 1,
causal: bool = True,
is_cp_round_robin: bool = False,
):
split_params = {
"kv_granularity": max(self.block_size, 16),
Expand All @@ -1238,6 +1277,15 @@ def _set_mla_persistent_worker_buffers(
reduce_indptr = var["reduce_indptr"]
reduce_final_map = var["reduce_final_map"]
reduce_partial_map = var["reduce_partial_map"]
# paged_kv_indices are generated at token granularity (block size 1),
# and mla_decode_fwd is handed the cache as page_size=1 to match. The
# page_size below only feeds the work partitioning, which is size-based,
# so the two have disagreed harmlessly. cprr is the exception: it
# reconstructs each work row's GLOBAL token position from this, and a
# page_size of block_size scales that mapping by block_size -- a causal
# mask placed at the wrong positions, silently. Tell it what the kernel
# actually gets.
page_size = 1 if is_cp_round_robin else self.block_size
get_mla_metadata_v1(
cu_seqlens_q,
self.paged_kv_indptr[: bs + 1], # TODO: support sparse
Expand All @@ -1251,9 +1299,10 @@ def _set_mla_persistent_worker_buffers(
reduce_indptr,
reduce_final_map,
reduce_partial_map,
page_size=self.block_size,
page_size=page_size,
dtype_q=self.dtype_q,
dtype_kv=self.dtype_kv,
is_cp_round_robin=is_cp_round_robin,
**split_params,
)
return {
Expand Down Expand Up @@ -1380,12 +1429,23 @@ def _build_decode(
f"decode query length {max_qo_len} exceeds the persistent MLA "
f"work-descriptor capacity {self.reorder_batch_threshold}"
)
# DCP shards KV round-robin, so a causal multi-token decode cannot read
# its mask off the rank-local row order: local position j is global
# position j * world + rank. aiter's cprr variant reconstructs that, but
# only if BOTH halves agree -- the work descriptors are planned for it
# here and the global cumulative lengths are handed over below. Planning
# one way and dispatching the other leaves the persistent kernel
# spinning on a work queue whose indptr disagrees with its work set.
# A single-token decode sees all of its local KV and a bidirectional
# block masks nothing, so both keep the plain kernel.
cp_round_robin = self.dcp_world_size > 1 and max_qo_len > 1 and causal
if use_persistent_metadata:
ctx_mla_ps = self._set_mla_persistent_worker_buffers(
num_reqs,
qo_indptr,
max_qo_len,
causal=causal,
is_cp_round_robin=cp_round_robin,
)
self.mla_persistent_metadata.update(ctx_mla_ps)

Expand Down Expand Up @@ -1427,6 +1487,21 @@ def _build_decode(
num_reqs=num_reqs,
)

g_kv_indptr = None
if cp_round_robin:
# cprr ships only as a persistent kernel; the non-persistent decode
# would ignore g_kv_indptr and mask on local row order instead --
# wrong output, no error. Refuse the batch rather than serve it.
assert use_persistent_metadata and dcp_tot_seq_lens_device is not None, (
f"causal decode of query length {max_qo_len} under DCP"
f"{self.dcp_world_size} needs the persistent round-robin kernel, "
"which is unavailable here (non-gfx950, or DP > 1)"
)
g_kv_indptr = self._g_kv_indptr_buf[: num_reqs + 1]
# [0] stays 0 (zero-init, never written). cumsum promotes to int64,
# so land it through copy_ rather than out=.
g_kv_indptr[1:].copy_(torch.cumsum(dcp_tot_seq_lens_device, dim=0))

attn_metadata = AiterMlaDecodeMetadataForVllm(
block_table=block_table_tensor,
seq_lens=seq_lens_device,
Expand All @@ -1439,6 +1514,7 @@ def _build_decode(
attn_out_dtype=self.decode_attn_out_dtype,
use_persistent_metadata=use_persistent_metadata,
causal=causal,
g_kv_indptr=g_kv_indptr,
fold_factor=fold_factor,
fold_kv_indptr=fold_kv_indptr,
fold_kv_indices=fold_kv_indices,
Expand Down Expand Up @@ -1489,6 +1565,24 @@ def build(
query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu
seq_lens = common_attn_metadata.seq_lens
dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens
if self.dcp_world_size > 1 and dcp_local_seq_lens is None:
# DSpark's draft step builds its own attention metadata inside
# vLLM's DFlash speculator, which never fills this in -- the target
# model runner is its only producer. The local lengths are a pure
# function of the global ones, so derive them here rather than
# teaching that path about DCP. It lands in a persistent buffer
# because the draft's FULL graph captures whatever tensor the decode
# metadata carries and re-reads it on every replay.
assert self._dcp_local_seq_lens_buf is not None
dcp_local_seq_lens = self._dcp_local_seq_lens_buf[:num_reqs]
dcp_local_seq_lens.copy_(
get_dcp_local_seq_lens(
seq_lens,
self.dcp_world_size,
self.dcp_rank,
self.cp_kv_cache_interleave_size,
)
)

num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
split_decodes_and_prefills(
Expand Down
Loading
Loading