Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions atom/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -699,6 +699,7 @@ def _glm5_next_unsupported_features(config: "Config") -> list[str]:
# multimodal models fully supported by plugin mode
_PLUGIN_SUPPORTED_MULTIMODAL_MODELS: set[str] = {
"kimi_k25",
"kimi_k3",
"qwen3_5",
"qwen3_5_moe",
}
Expand Down
18 changes: 13 additions & 5 deletions atom/plugin/vllm/attention/layer_mla.py
Original file line number Diff line number Diff line change
Expand Up @@ -1058,6 +1058,12 @@ def forward_impl(
q = q[:num_actual_toks, ...]
k_c_normed = k_c_normed[:num_actual_toks, ...]
k_pe = k_pe[:num_actual_toks, ...]
# Model runner V2 sizes slot_mapping to the CUDA-graph token width and
# marks the tail with PAD_SLOT_ID, but leaves num_actual_tokens unpadded
# on the piecewise path; V1 never pads it. The cache kernels launch one
# block per slot and index q/kv by that block, so slot_mapping must not
# outrun the tensors sliced above. Same trim layer_mha already applies.
slot_mapping = attn_metadata.slot_mapping[:num_actual_toks]

decode_q = q[:num_decode_tokens]
prefill_q = q[num_decode_tokens:]
Expand Down Expand Up @@ -1085,7 +1091,7 @@ def forward_impl(
k_c_normed,
self.rotary_emb_cos_sin_cache,
self.rotary_emb.is_neox_style,
attn_metadata.slot_mapping,
slot_mapping,
kv_cache,
self.kv_cache_dtype,
self._k_scale,
Expand All @@ -1097,7 +1103,7 @@ def forward_impl(
k_c_normed,
k_pe.squeeze(1),
kv_cache,
attn_metadata.slot_mapping.flatten(),
slot_mapping.flatten(),
kv_cache_dtype=self.kv_cache_dtype,
scale=self._k_scale,
)
Expand Down Expand Up @@ -1195,7 +1201,7 @@ def forward_impl(
self.kv_lora_rank + self.qk_rope_head_dim,
),
decode_q[:, : self.num_heads] if fused_q_head_pad else decode_q,
attn_metadata.slot_mapping,
slot_mapping,
self._k_scale,
self._q_scale,
positions,
Expand Down Expand Up @@ -1367,6 +1373,7 @@ def forward_impl_sparse(
q = q[:num_actual_toks, ...]
k_c_normed = k_c_normed[:num_actual_toks, ...]
k_pe = k_pe[:num_actual_toks, ...].unsqueeze(1)
slot_mapping = sparse_meta.slot_mapping[:num_actual_toks]

positions = None
if self._is_vllm_forward_context_available():
Expand Down Expand Up @@ -1445,7 +1452,7 @@ def forward_impl_sparse(
kv_cache.shape[0], -1, self.kv_lora_rank + self.qk_rope_head_dim
),
q_out,
sparse_meta.slot_mapping,
slot_mapping,
self._k_scale,
self._q_scale,
positions,
Expand Down Expand Up @@ -1506,8 +1513,9 @@ def forward(
def get_kv_cache_spec(self, vllm_config):
from vllm.v1.kv_cache_interface import MLAAttentionSpec

block_size = vllm_config.cache_config.block_size
return MLAAttentionSpec(
block_size=vllm_config.cache_config.block_size,
block_size=block_size,
num_kv_heads=1,
head_size=self.head_size,
dtype=self.kv_cache_torch_dtype,
Expand Down
5 changes: 4 additions & 1 deletion atom/plugin/vllm/attention/layer_sparse_mla.py
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,10 @@ def sparse_attn_indexer_plugin_mode(
"Sparse MLA metadata not found for indexer cache "
f"{k_cache_prefix!r}. The indexer cannot populate paged_kv_indices."
)
slot_mapping = indexer_meta.slot_mapping
# V2 may pad metadata to the graph width while the tensors above contain
# only live tokens. Cache kernels launch once per slot and index those
# tensors directly, so the mapping must have the same live-token extent.
slot_mapping = indexer_meta.slot_mapping[: q_input.shape[0]]
has_decode = indexer_meta.num_decodes > 0
has_prefill = indexer_meta.num_prefills > 0
num_decode_tokens = indexer_meta.num_decode_tokens
Expand Down
48 changes: 23 additions & 25 deletions atom/plugin/vllm/attention/minimax_m3_attnetion.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@
MiniMax-M3's q/k norm + RoPE transform.
"""

from typing import Optional

import aiter
import torch
from aiter import dtypes
Expand Down Expand Up @@ -148,19 +146,19 @@ def __init__(
head_dim: int,
scale: float,
num_kv_heads: int,
alibi_slopes: Optional[list[float]] = None,
alibi_slopes: list[float] | None = None,
kv_cache_dtype: str = "bf16",
layer_num: int = 0,
use_mla: bool = False,
rotary_emb: Optional[nn.Module] = None,
prefix: Optional[str] = None,
q_norm: Optional[nn.Module] = None,
k_norm: Optional[nn.Module] = None,
rotary_emb: nn.Module | None = None,
prefix: str | None = None,
q_norm: nn.Module | None = None,
k_norm: nn.Module | None = None,
cache_config=None,
quant_config=None,
index_q_norm: Optional[nn.Module] = None,
index_k_norm: Optional[nn.Module] = None,
index_rotary_emb: Optional[nn.Module] = None,
index_q_norm: nn.Module | None = None,
index_k_norm: nn.Module | None = None,
index_rotary_emb: nn.Module | None = None,
index_q_size: int = 0,
index_head_dim: int = 0,
topk: int = 0,
Expand Down Expand Up @@ -461,14 +459,14 @@ def _insert_qkv_and_index(
self.index_q_norm.weight,
self.index_k_norm.weight,
self.num_idx_heads,
slot_mapping=main_metadata.slot_mapping,
slot_mapping=main_metadata.slot_mapping[:num_tokens],
kv_cache_k=k_cache,
kv_cache_v=v_cache,
index_cache=self.index_cache_layer.kv_cache,
block_size=k_cache.shape[3],
q_out=q_out,
index_q_out=index_q,
index_slot_mapping=index_metadata.slot_mapping,
index_slot_mapping=index_metadata.slot_mapping[:num_tokens],
kv_cache_dtype=kv_cache_dtype,
k_scale=k_scale if self.kv_cache_dtype == "fp8" else None,
v_scale=v_scale if self.kv_cache_dtype == "fp8" else None,
Expand Down Expand Up @@ -746,8 +744,8 @@ def _run_sparse_attention(
def _forward_with_output(
self,
qkv: torch.Tensor,
positions: Optional[torch.Tensor] = None,
output: Optional[torch.Tensor] = None,
positions: torch.Tensor | None = None,
output: torch.Tensor | None = None,
) -> torch.Tensor:
main_metadata, index_metadata = self._metadata_for_layer()
num_tokens = qkv.shape[0]
Expand Down Expand Up @@ -823,9 +821,9 @@ def forward(
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
positions: Optional[torch.Tensor] = None,
q_scale: Optional[torch.Tensor] = None,
qkv: Optional[torch.Tensor] = None,
positions: torch.Tensor | None = None,
q_scale: torch.Tensor | None = None,
qkv: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor:
del query, key, value, q_scale, kwargs
Expand All @@ -852,14 +850,14 @@ def __init__(
head_dim: int,
scale: float,
num_kv_heads: int,
alibi_slopes: Optional[list[float]] = None,
alibi_slopes: list[float] | None = None,
kv_cache_dtype: str = "bf16",
layer_num: int = 0,
use_mla: bool = False,
rotary_emb: Optional[nn.Module] = None,
prefix: Optional[str] = None,
q_norm: Optional[nn.Module] = None,
k_norm: Optional[nn.Module] = None,
rotary_emb: nn.Module | None = None,
prefix: str | None = None,
q_norm: nn.Module | None = None,
k_norm: nn.Module | None = None,
cache_config=None,
quant_config=None,
**kwargs,
Expand Down Expand Up @@ -989,9 +987,9 @@ def forward(
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
positions: Optional[torch.Tensor] = None,
q_scale: Optional[torch.Tensor] = None,
qkv: Optional[torch.Tensor] = None,
positions: torch.Tensor | None = None,
q_scale: torch.Tensor | None = None,
qkv: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor:
del query, key, value, q_scale, kwargs
Expand Down
13 changes: 11 additions & 2 deletions atom/plugin/vllm/model_wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ def _maybe_set_v4_expert_dtype(atom_config, vllm_config) -> None:
"Qwen3_5ForConditionalGeneration": "atom.plugin.vllm.models.qwen3_5:Qwen3_5ForConditionalGeneration_",
"KimiK25ForConditionalGeneration": "atom.plugin.vllm.models.kimi_k25:KimiK25ForConditionalGeneration_",
"KimiK3ForConditionalGeneration": (
"atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLM"
"atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGeneration_"
),
"MiniMaxM2ForCausalLM": "atom.models.minimax_m2:MiniMaxM2ForCausalLM",
"DeepseekV4ForCausalLM": "atom.plugin.vllm.models.deepseek_v4:DeepseekV4ForCausalLM",
Expand Down Expand Up @@ -507,7 +507,16 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
else:
self.model = model_cls(self.atom_config)
else:
self.model = model_cls(self.atom_config)
if model_arch == "KimiK3ForConditionalGeneration":
# Kimi-K3's inner class inherits vLLM SupportsQuant. Passing
# VllmConfig lets that protocol apply its quant mapper and
# packed-module mappings before the vision modules are built.
self.model = model_cls(
self.atom_config,
vllm_config=vllm_config,
)
else:
self.model = model_cls(self.atom_config)

num_patched_post_load_hooks = _patch_required_act_dtype_post_load_hooks(
self.model,
Expand Down
Loading
Loading