diff --git a/atom/config.py b/atom/config.py index 8e13b3c8b1..5d675b24cf 100644 --- a/atom/config.py +++ b/atom/config.py @@ -699,6 +699,7 @@ def _glm5_next_unsupported_features(config: "Config") -> list[str]: # multimodal models fully supported by plugin mode _PLUGIN_SUPPORTED_MULTIMODAL_MODELS: set[str] = { "kimi_k25", + "kimi_k3", "qwen3_5", "qwen3_5_moe", } diff --git a/atom/plugin/vllm/attention/layer_mla.py b/atom/plugin/vllm/attention/layer_mla.py index aef5ee7b16..998a84a504 100644 --- a/atom/plugin/vllm/attention/layer_mla.py +++ b/atom/plugin/vllm/attention/layer_mla.py @@ -1058,6 +1058,12 @@ def forward_impl( q = q[:num_actual_toks, ...] k_c_normed = k_c_normed[:num_actual_toks, ...] k_pe = k_pe[:num_actual_toks, ...] + # Model runner V2 sizes slot_mapping to the CUDA-graph token width and + # marks the tail with PAD_SLOT_ID, but leaves num_actual_tokens unpadded + # on the piecewise path; V1 never pads it. The cache kernels launch one + # block per slot and index q/kv by that block, so slot_mapping must not + # outrun the tensors sliced above. Same trim layer_mha already applies. + slot_mapping = attn_metadata.slot_mapping[:num_actual_toks] decode_q = q[:num_decode_tokens] prefill_q = q[num_decode_tokens:] @@ -1085,7 +1091,7 @@ def forward_impl( k_c_normed, self.rotary_emb_cos_sin_cache, self.rotary_emb.is_neox_style, - attn_metadata.slot_mapping, + slot_mapping, kv_cache, self.kv_cache_dtype, self._k_scale, @@ -1097,7 +1103,7 @@ def forward_impl( k_c_normed, k_pe.squeeze(1), kv_cache, - attn_metadata.slot_mapping.flatten(), + slot_mapping.flatten(), kv_cache_dtype=self.kv_cache_dtype, scale=self._k_scale, ) @@ -1195,7 +1201,7 @@ def forward_impl( self.kv_lora_rank + self.qk_rope_head_dim, ), decode_q[:, : self.num_heads] if fused_q_head_pad else decode_q, - attn_metadata.slot_mapping, + slot_mapping, self._k_scale, self._q_scale, positions, @@ -1367,6 +1373,7 @@ def forward_impl_sparse( q = q[:num_actual_toks, ...] k_c_normed = k_c_normed[:num_actual_toks, ...] k_pe = k_pe[:num_actual_toks, ...].unsqueeze(1) + slot_mapping = sparse_meta.slot_mapping[:num_actual_toks] positions = None if self._is_vllm_forward_context_available(): @@ -1445,7 +1452,7 @@ def forward_impl_sparse( kv_cache.shape[0], -1, self.kv_lora_rank + self.qk_rope_head_dim ), q_out, - sparse_meta.slot_mapping, + slot_mapping, self._k_scale, self._q_scale, positions, @@ -1506,8 +1513,9 @@ def forward( def get_kv_cache_spec(self, vllm_config): from vllm.v1.kv_cache_interface import MLAAttentionSpec + block_size = vllm_config.cache_config.block_size return MLAAttentionSpec( - block_size=vllm_config.cache_config.block_size, + block_size=block_size, num_kv_heads=1, head_size=self.head_size, dtype=self.kv_cache_torch_dtype, diff --git a/atom/plugin/vllm/attention/layer_sparse_mla.py b/atom/plugin/vllm/attention/layer_sparse_mla.py index f3b72689ce..f5891cbf60 100644 --- a/atom/plugin/vllm/attention/layer_sparse_mla.py +++ b/atom/plugin/vllm/attention/layer_sparse_mla.py @@ -304,7 +304,10 @@ def sparse_attn_indexer_plugin_mode( "Sparse MLA metadata not found for indexer cache " f"{k_cache_prefix!r}. The indexer cannot populate paged_kv_indices." ) - slot_mapping = indexer_meta.slot_mapping + # V2 may pad metadata to the graph width while the tensors above contain + # only live tokens. Cache kernels launch once per slot and index those + # tensors directly, so the mapping must have the same live-token extent. + slot_mapping = indexer_meta.slot_mapping[: q_input.shape[0]] has_decode = indexer_meta.num_decodes > 0 has_prefill = indexer_meta.num_prefills > 0 num_decode_tokens = indexer_meta.num_decode_tokens diff --git a/atom/plugin/vllm/attention/minimax_m3_attnetion.py b/atom/plugin/vllm/attention/minimax_m3_attnetion.py index bc4faccb2a..31850780a2 100644 --- a/atom/plugin/vllm/attention/minimax_m3_attnetion.py +++ b/atom/plugin/vllm/attention/minimax_m3_attnetion.py @@ -10,8 +10,6 @@ MiniMax-M3's q/k norm + RoPE transform. """ -from typing import Optional - import aiter import torch from aiter import dtypes @@ -148,19 +146,19 @@ def __init__( head_dim: int, scale: float, num_kv_heads: int, - alibi_slopes: Optional[list[float]] = None, + alibi_slopes: list[float] | None = None, kv_cache_dtype: str = "bf16", layer_num: int = 0, use_mla: bool = False, - rotary_emb: Optional[nn.Module] = None, - prefix: Optional[str] = None, - q_norm: Optional[nn.Module] = None, - k_norm: Optional[nn.Module] = None, + rotary_emb: nn.Module | None = None, + prefix: str | None = None, + q_norm: nn.Module | None = None, + k_norm: nn.Module | None = None, cache_config=None, quant_config=None, - index_q_norm: Optional[nn.Module] = None, - index_k_norm: Optional[nn.Module] = None, - index_rotary_emb: Optional[nn.Module] = None, + index_q_norm: nn.Module | None = None, + index_k_norm: nn.Module | None = None, + index_rotary_emb: nn.Module | None = None, index_q_size: int = 0, index_head_dim: int = 0, topk: int = 0, @@ -461,14 +459,14 @@ def _insert_qkv_and_index( self.index_q_norm.weight, self.index_k_norm.weight, self.num_idx_heads, - slot_mapping=main_metadata.slot_mapping, + slot_mapping=main_metadata.slot_mapping[:num_tokens], kv_cache_k=k_cache, kv_cache_v=v_cache, index_cache=self.index_cache_layer.kv_cache, block_size=k_cache.shape[3], q_out=q_out, index_q_out=index_q, - index_slot_mapping=index_metadata.slot_mapping, + index_slot_mapping=index_metadata.slot_mapping[:num_tokens], kv_cache_dtype=kv_cache_dtype, k_scale=k_scale if self.kv_cache_dtype == "fp8" else None, v_scale=v_scale if self.kv_cache_dtype == "fp8" else None, @@ -746,8 +744,8 @@ def _run_sparse_attention( def _forward_with_output( self, qkv: torch.Tensor, - positions: Optional[torch.Tensor] = None, - output: Optional[torch.Tensor] = None, + positions: torch.Tensor | None = None, + output: torch.Tensor | None = None, ) -> torch.Tensor: main_metadata, index_metadata = self._metadata_for_layer() num_tokens = qkv.shape[0] @@ -823,9 +821,9 @@ def forward( query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, - positions: Optional[torch.Tensor] = None, - q_scale: Optional[torch.Tensor] = None, - qkv: Optional[torch.Tensor] = None, + positions: torch.Tensor | None = None, + q_scale: torch.Tensor | None = None, + qkv: torch.Tensor | None = None, **kwargs, ) -> torch.Tensor: del query, key, value, q_scale, kwargs @@ -852,14 +850,14 @@ def __init__( head_dim: int, scale: float, num_kv_heads: int, - alibi_slopes: Optional[list[float]] = None, + alibi_slopes: list[float] | None = None, kv_cache_dtype: str = "bf16", layer_num: int = 0, use_mla: bool = False, - rotary_emb: Optional[nn.Module] = None, - prefix: Optional[str] = None, - q_norm: Optional[nn.Module] = None, - k_norm: Optional[nn.Module] = None, + rotary_emb: nn.Module | None = None, + prefix: str | None = None, + q_norm: nn.Module | None = None, + k_norm: nn.Module | None = None, cache_config=None, quant_config=None, **kwargs, @@ -989,9 +987,9 @@ def forward( query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, - positions: Optional[torch.Tensor] = None, - q_scale: Optional[torch.Tensor] = None, - qkv: Optional[torch.Tensor] = None, + positions: torch.Tensor | None = None, + q_scale: torch.Tensor | None = None, + qkv: torch.Tensor | None = None, **kwargs, ) -> torch.Tensor: del query, key, value, q_scale, kwargs diff --git a/atom/plugin/vllm/model_wrapper.py b/atom/plugin/vllm/model_wrapper.py index 9a621aa0bb..52d8b85fbf 100644 --- a/atom/plugin/vllm/model_wrapper.py +++ b/atom/plugin/vllm/model_wrapper.py @@ -156,7 +156,7 @@ def _maybe_set_v4_expert_dtype(atom_config, vllm_config) -> None: "Qwen3_5ForConditionalGeneration": "atom.plugin.vllm.models.qwen3_5:Qwen3_5ForConditionalGeneration_", "KimiK25ForConditionalGeneration": "atom.plugin.vllm.models.kimi_k25:KimiK25ForConditionalGeneration_", "KimiK3ForConditionalGeneration": ( - "atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLM" + "atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGeneration_" ), "MiniMaxM2ForCausalLM": "atom.models.minimax_m2:MiniMaxM2ForCausalLM", "DeepseekV4ForCausalLM": "atom.plugin.vllm.models.deepseek_v4:DeepseekV4ForCausalLM", @@ -507,7 +507,16 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): else: self.model = model_cls(self.atom_config) else: - self.model = model_cls(self.atom_config) + if model_arch == "KimiK3ForConditionalGeneration": + # Kimi-K3's inner class inherits vLLM SupportsQuant. Passing + # VllmConfig lets that protocol apply its quant mapper and + # packed-module mappings before the vision modules are built. + self.model = model_cls( + self.atom_config, + vllm_config=vllm_config, + ) + else: + self.model = model_cls(self.atom_config) num_patched_post_load_hooks = _patch_required_act_dtype_post_load_hooks( self.model, diff --git a/atom/plugin/vllm/models/kimi_k3.py b/atom/plugin/vllm/models/kimi_k3.py index 35ca84bd62..c34fe279b9 100644 --- a/atom/plugin/vllm/models/kimi_k3.py +++ b/atom/plugin/vllm/models/kimi_k3.py @@ -1,8 +1,10 @@ from dataclasses import replace from types import SimpleNamespace +from typing import Any, ClassVar import torch from aiter.dist.parallel_state import get_pp_group +from torch import nn from vllm.config import VllmConfig, get_current_vllm_config from vllm.forward_context import get_forward_context as get_vllm_forward_context from vllm.model_executor.layers.mamba.abstract import MambaBase @@ -12,11 +14,31 @@ MambaStateDtypeCalculator, MambaStateShapeCalculator, ) +from vllm.model_executor.layers.quantization.compressed_tensors import ( + compressed_tensors, +) from vllm.model_executor.models.interfaces import IsHybrid +from vllm.model_executor.models.kimi_k25_vit import ( + KimiK25MultiModalProjector, + MoonViT3dPretrainedModel, +) +from vllm.model_executor.models.vision import is_vit_use_data_parallel +from vllm.models.kimi_k3 import ( + KimiK3ForConditionalGeneration as vLLMKimiK3, +) +from vllm.models.kimi_k3.common.mm_preprocess import ( + KimiK3DummyInputsBuilder, + KimiK3MultiModalProcessor, + KimiK3ProcessingInfo, +) from vllm.models.kimi_k3.nvidia.kda_metadata import KimiK3KDAMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.platforms import current_platform from vllm.v1.attention.backend import AttentionBackend from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum +from atom.config import Config +from atom.model_loader.loader import WeightsMapper, load_model_in_plugin_mode from atom.models import kimi_k3 as kimi_k3_base from atom.models.kimi_k3 import ( KimiDecoderLayer, @@ -26,11 +48,20 @@ from atom.models.kimi_k3 import ( KimiK3ForCausalLM as KimiK3ForCausalLMBase, ) -from atom.models.utils import IntermediateTensors +from atom.models.utils import IntermediateTensors, PPMissingLayer, maybe_prefix from atom.plugin.vllm.kda_backend import AtomKimiK3KDAAttentionBackend -from atom.plugin.vllm.model_wrapper import ATOMMoEForCausalLM +from atom.plugin.vllm.model_wrapper import ATOMForConditionalGeneration from atom.utils.forward_context import get_forward_context as get_atom_forward_context +_KIMI_K3_QUANT_EXCLUDE_NAME_MAPPING: dict[str, str] = { + "language_model.language_model.model.": ( + "model.language_model.language_model.model." + ), + "language_model.language_model.lm_head": ( + "model.language_model.language_model.lm_head" + ), +} + def _adapt_kda_metadata_for_atom(kda_metadata: KimiK3KDAMetadata) -> None: """Provide the state-read index field expected by native ATOM KDA.""" @@ -460,6 +491,15 @@ def forward( class KimiK3ForCausalLM(KimiK3ForCausalLMBase): + def __setattr__(self, name: str, value: Any) -> None: + if name in {"model", "lm_head"}: + language_model = getattr(self, "_modules", {}).get("language_model") + if language_model is not None: + setattr(language_model, name, value) + object.__setattr__(self, name, value) + return + super().__setattr__(name, value) + def __init__(self, *args, **kwargs): original_kda_cls = kimi_k3_base.KimiKDAAttention original_model_cls = kimi_k3_base.KimiLinearModel @@ -470,6 +510,17 @@ def __init__(self, *args, **kwargs): finally: kimi_k3_base.KimiKDAAttention = original_kda_cls kimi_k3_base.KimiLinearModel = original_model_cls + # vLLM's DSpark loader reads these one level above their real owner. + # Bypass Module registration for the initial aliases so checkpoint + # names stay unchanged. A later normal setattr (weight sharing) then + # replaces the alias with a real registered module instead of being + # hidden behind a read-only property. + object.__setattr__(self, "model", self.language_model.model) + object.__setattr__(self, "lm_head", self.language_model.lm_head) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + # Required by vLLM SupportsMultiModal.get_language_model discovery. + return self.get_input_embeddings(input_ids) def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None: self.language_model.model.set_aux_hidden_state_layers(layers) @@ -480,16 +531,159 @@ def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]: return (2, num_layers // 2, num_layers - 3) -class KimiK3ForCausalLMVllm(ATOMMoEForCausalLM, IsHybrid): - def get_language_model(self): - """The causal LM that owns ``model.embed_tokens`` and ``lm_head``. +class KimiK3ForConditionalGeneration_(vLLMKimiK3): + hf_to_atom_mapper = WeightsMapper( + orig_to_new_prefix={ + # ATOM nests language_model.language_model (ATOM KimiK3ForCausalLM + # wraps KimiLinearForCausalLM); checkpoint stores language_model.model. + "language_model.": "language_model.language_model.", + "mm_projector.proj.0": "mm_projector.linear_1", + "mm_projector.proj.2": "mm_projector.linear_2", + } + ) + packed_modules_mapping = KimiK3ForCausalLMBase.packed_modules_mapping + quant_exclude_name_mapping: ClassVar[dict[str, str]] = ( + _KIMI_K3_QUANT_EXCLUDE_NAME_MAPPING + ) - DSpark's loader binds the target's embedding and head into the draft - through this hook; Kimi-K3 keeps both one level in, under - ``language_model``, so without it the loader would look for them on this - wrapper and silently leave the draft with neither. - """ - return self.model.language_model + def __init__( + self, + atom_config: Config, + prefix: str = "model", + *, + vllm_config: VllmConfig | None = None, + ): + nn.Module.__init__(self) + hf_config = getattr(atom_config, "hf_config", None) + assert hf_config is not None, "hf_config is not found in atom_config" + vision_config = hf_config.vision_config + self.config = hf_config + self.atom_config = atom_config + + if vllm_config is None: + vllm_config = atom_config.plugin_config.vllm_config + model_config = vllm_config.model_config + quant_config = vllm_config.quant_config + atom_quant_config = atom_config.quant_config + multimodal_config = model_config.multimodal_config + self.multimodal_config = multimodal_config + self.use_data_parallel = is_vit_use_data_parallel( + vision_config.num_attention_heads + ) + text_config = getattr(hf_config, "text_config", hf_config) + self.hidden_size = text_config.hidden_size + self.device = current_platform.current_device() + self.has_vision_tower = get_pp_group().is_first_rank + + if self.has_vision_tower: + vision_quant_config = self._maybe_ignore_quant_config( + quant_config, + atom_quant_config.exclude_layers or [], + "vision_tower", + ) + projector_quant_config = self._maybe_ignore_quant_config( + quant_config, + atom_quant_config.exclude_layers or [], + "mm_projector", + ) + with self._mark_tower_model(vllm_config, "image"): + self.vision_tower = MoonViT3dPretrainedModel( + vision_config, + quant_config=vision_quant_config, + prefix=maybe_prefix(prefix, "vision_tower"), + ) + if vision_quant_config is not None: + self.vision_tower = self.vision_tower.to(device=self.device) + else: + self.vision_tower = self.vision_tower.to( + device=self.device, dtype=model_config.dtype + ) + self.mm_projector = KimiK25MultiModalProjector( + config=vision_config, + use_data_parallel=self.use_data_parallel, + quant_config=projector_quant_config, + prefix=maybe_prefix(prefix, "mm_projector"), + ).to(device=self.device, dtype=model_config.dtype) + else: + self.vision_tower = PPMissingLayer() + self.mm_projector = PPMissingLayer() + self.skip_weight_prefixes = ["vision_tower.", "mm_projector."] + + self.quant_config = quant_config + with self._mark_language_model(vllm_config): + self.language_model = KimiK3ForCausalLM( + atom_config=atom_config, + prefix=maybe_prefix(prefix, "language_model"), + ) + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors + ) + self.media_placeholder = ( + self.config.media_placeholder_token_id if self.has_vision_tower else None + ) + self.packed_modules_mapping = self.language_model.packed_modules_mapping + self.weights_mapping = self.language_model.weights_mapping + + def _maybe_ignore_quant_config( + self, + quant_config: Any, + exclude_layers: list[str] | None = None, + layer_name: str | None = None, + ): + if isinstance(quant_config, compressed_tensors.CompressedTensorsConfig): + return None + if layer_name is not None: + # Check the container and a representative child, both with and + # without ATOMModelBase's outer ``model.`` root. The child form is + # required for patterns such as ``*.vision_tower.*``. + candidates = ( + layer_name, + f"{layer_name}.__atom_child__", + f"model.{layer_name}", + f"model.{layer_name}.__atom_child__", + ) + if any( + self.atom_config.quant_config._is_excluded( + candidate, exclude_layers or [] + ) + for candidate in candidates + ): + return None + return quant_config + + def load_weights(self, weights): + return load_model_in_plugin_mode( + model=self, + config=self.atom_config, + prefix="model.", + weights_mapper=self.hf_to_atom_mapper, + ) + + def get_expert_mapping(self): + return self.language_model.get_expert_mapping() + + def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None: + self.language_model.set_aux_hidden_state_layers(layers) + + def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]: + return self.language_model.get_eagle3_aux_hidden_state_layers() + + +@MULTIMODAL_REGISTRY.register_processor( + KimiK3MultiModalProcessor, + info=KimiK3ProcessingInfo, + dummy_inputs=KimiK3DummyInputsBuilder, +) +class KimiK3ForConditionalGenerationVllm(ATOMForConditionalGeneration, IsHybrid): + @classmethod + def get_placeholder_str(cls, modality: str, i: int) -> str | None: + if modality == "image": + return "<|kimi_image_placeholder|>" + raise ValueError(f"Unsupported modality: {modality}") + + def load_weights(self, weights): + return self.model.load_weights(weights) @classmethod def get_mamba_state_dtype_from_config( diff --git a/atom/plugin/vllm/register.py b/atom/plugin/vllm/register.py index bc8b58e444..18fe0cb4b3 100644 --- a/atom/plugin/vllm/register.py +++ b/atom/plugin/vllm/register.py @@ -36,7 +36,7 @@ "Qwen3_5MoeForConditionalGeneration": "atom.plugin.vllm.models.qwen3_5:Qwen3_5MoeForConditionalGeneration", "KimiK25ForConditionalGeneration": "atom.plugin.vllm.models.kimi_k25:KimiK25ForConditionalGeneration", "KimiK3ForConditionalGeneration": ( - "atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLMVllm" + "atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGenerationVllm" ), # vLLM registers this arch too, but only to its NVIDIA implementation. "K3DSparkModel": "atom.plugin.vllm.models.kimi_k3_dspark:KimiK3DSparkVllm", diff --git a/docs/vllm_plugin_backend_guide.md b/docs/vllm_plugin_backend_guide.md index 9f6e4eb478..bd022d7dfc 100644 --- a/docs/vllm_plugin_backend_guide.md +++ b/docs/vllm_plugin_backend_guide.md @@ -185,7 +185,7 @@ Currently, the plugin backend supports the following model architectures: | `GptOssForCausalLM` | `atom.models.gpt_oss.GptOssForCausalLM` | GPT-OSS | | `DeepseekV3ForCausalLM` | `atom.models.deepseek_v2.DeepseekV3ForCausalLM` | DeepSeek-R1 / DeepSeek V3 / Kimi-K2 style models | | `Glm4MoeForCausalLM` | `atom.models.glm4_moe.Glm4MoeForCausalLM` | GLM-4-MoE | -| `KimiK3ForConditionalGeneration` | `atom.plugin.vllm.models.kimi_k3.KimiK3ForCausalLM` | Kimi-K3 text-only KDA + MLA hybrid MoE | +| `KimiK3ForConditionalGeneration` | `atom.plugin.vllm.models.kimi_k3.KimiK3ForConditionalGenerationVllm` | Kimi-K3 multimodal KDA + MLA hybrid MoE | `Kimi-K2` is also supported. Although it is usually loaded with `--trust-remote-code`, it shares the same DeepSeek-style MLA+MoE architecture path and reuses `atom.models.deepseek_v2.DeepseekV3ForCausalLM` in the ATOM vLLM OOT backend. diff --git a/recipes/atom_vllm/Kimi-K3.md b/recipes/atom_vllm/Kimi-K3.md index ac2727db32..169a131292 100644 --- a/recipes/atom_vllm/Kimi-K3.md +++ b/recipes/atom_vllm/Kimi-K3.md @@ -1,9 +1,9 @@ # Kimi-K3 with ATOM vLLM Plugin Backend -This recipe serves the text-only Kimi-K3 backbone -(`KimiK3ForConditionalGeneration`) through the ATOM vLLM out-of-tree plugin. -Kimi-K3 combines KDA recurrent-attention layers, MLA full-attention layers, and -an MXFP4 latent MoE. +This recipe serves multimodal Kimi-K3 (`KimiK3ForConditionalGeneration`) +through the ATOM vLLM out-of-tree plugin. Kimi-K3 combines a MoonViT3d vision +tower with KDA recurrent-attention layers, MLA full-attention layers, and an +MXFP4 latent MoE. The validated configuration requires eight MI355 (gfx950) GPUs with TP8. @@ -32,7 +32,6 @@ vllm serve "${MODEL}" \ --port 8000 \ --tensor-parallel-size 8 \ --trust-remote-code \ - --language-model-only \ --kv-cache-dtype fp8 \ --max-model-len 16384 \ --max-num-seqs 64 \ @@ -142,7 +141,8 @@ Avg draft acceptance rate: 86.1%, 86.1% (whole run, each of the two) ## Current scope -- Text generation only; the vision tower and multimodal projector are skipped. +- Text and image inputs are supported through the Kimi-K3 multimodal processor, + vision tower, and projector. - TP8 on MI355/gfx950 is the validated deployment. - Asynchronous scheduling is supported. Prefix caching is off by default and needs `--mamba-cache-mode align` to be turned on, as the DSpark launch does. diff --git a/tests/plugin/test_vllm_kimi_k3.py b/tests/plugin/test_vllm_kimi_k3.py index b151f98119..91ec71c564 100644 --- a/tests/plugin/test_vllm_kimi_k3.py +++ b/tests/plugin/test_vllm_kimi_k3.py @@ -24,10 +24,11 @@ def test_kimi_k3_plugin_registries_are_synchronized(): arch = "KimiK3ForConditionalGeneration" assert ( _VLLM_MODEL_REGISTRY_OVERRIDES[arch] - == "atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLMVllm" + == "atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGenerationVllm" ) assert ( - _ATOM_MODEL_CLASSES[arch] == "atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLM" + _ATOM_MODEL_CLASSES[arch] + == "atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGeneration_" ) @@ -108,6 +109,146 @@ def test_kimi_k3_temporal_state_uses_fp32(): """) +def test_kimi_k3_outer_is_multimodal_and_hybrid(): + _run_without_test_stubs(""" + from vllm.model_executor.models.interfaces import IsHybrid + + from atom.plugin.vllm.model_wrapper import ATOMForConditionalGeneration + from atom.plugin.vllm.models.kimi_k3 import ( + KimiK3ForConditionalGenerationVllm, + ) + + mro = KimiK3ForConditionalGenerationVllm.__mro__ + # Multimodal via ATOMForConditionalGeneration; hybrid state retained. + assert ATOMForConditionalGeneration in mro + assert IsHybrid in mro + + # Image placeholder matches the checkpoint token. + assert ( + KimiK3ForConditionalGenerationVllm.get_placeholder_str("image", 0) + == "<|kimi_image_placeholder|>" + ) + + # vLLM looks up the processor on the architecture class from + # get_model_architecture (this wrapper), not on the inner module. + assert "_processor_factory" in KimiK3ForConditionalGenerationVllm.__dict__ + """) + + +def test_kimi_k3_inner_conditional_generation_class(): + _run_without_test_stubs(""" + from inspect import Parameter, signature + + from vllm.models.kimi_k3 import ( + KimiK3ForConditionalGeneration as vLLMKimiK3, + ) + from atom.plugin.vllm.models.kimi_k3 import ( + KimiK3ForCausalLM, + KimiK3ForConditionalGeneration_, + ) + + # Inner class subclasses the upstream multimodal model so it inherits + # embed_multimodal / media parsing / vision-tower forward. Do not + # re-register the processor here: vLLM keys it per class, and the + # architecture class that must carry it is the outer wrapper. + assert issubclass(KimiK3ForConditionalGeneration_, vLLMKimiK3) + assert "_processor_factory" not in KimiK3ForConditionalGeneration_.__dict__ + assert hasattr(KimiK3ForConditionalGeneration_, "_processor_factory") + parameters = signature(KimiK3ForConditionalGeneration_.__init__).parameters + assert parameters["vllm_config"].kind is Parameter.KEYWORD_ONLY + + # Weight mapper collapses the double language_model nesting and renames + # the projector layers. + mapper = KimiK3ForConditionalGeneration_.hf_to_atom_mapper + assert mapper.orig_to_new_prefix["language_model."] == ( + "language_model.language_model." + ) + assert mapper.orig_to_new_prefix["mm_projector.proj.0"] == ( + "mm_projector.linear_1" + ) + assert mapper.orig_to_new_prefix["mm_projector.proj.2"] == ( + "mm_projector.linear_2" + ) + + # Language-model wrapper exposes embed_input_ids for vLLM multimodal + # discovery. + assert hasattr(KimiK3ForCausalLM, "embed_input_ids") + + # Quant exclusions are remapped after hf_to_atom_mapper. They must add + # the outer model root without applying the language_model nesting a + # second time. + from atom.config import QuantizationConfig + + quant_config = QuantizationConfig() + quant_config.exclude_layers = [ + "language_model.language_model.model.layers.0.mlp", + "language_model.language_model.lm_head", + ] + quant_config.apply_exclude_name_mapping( + KimiK3ForConditionalGeneration_.quant_exclude_name_mapping + ) + assert quant_config.exclude_layers == [ + "model.language_model.language_model.model.layers.0.mlp", + "model.language_model.language_model.lm_head", + ] + """) + + +def test_kimi_k3_spec_decode_alias_writes_reach_language_model(): + _run_without_test_stubs(""" + import torch.nn as nn + + from atom.plugin.vllm.models.kimi_k3 import KimiK3ForCausalLM + + target = KimiK3ForCausalLM.__new__(KimiK3ForCausalLM) + nn.Module.__init__(target) + target.language_model = nn.Module() + target.language_model.model = nn.Linear(2, 2) + target.language_model.lm_head = nn.Linear(2, 3) + + replacement_model = nn.Linear(2, 2) + replacement_head = nn.Linear(2, 3) + target.model = replacement_model + target.lm_head = replacement_head + + assert target.model is replacement_model + assert target.language_model.model is replacement_model + assert target.lm_head is replacement_head + assert target.language_model.lm_head is replacement_head + """) + + +def test_kimi_k3_vision_quant_excludes_cover_regex_and_globs(): + _run_without_test_stubs(""" + from types import SimpleNamespace + + from atom.config import QuantizationConfig + from atom.plugin.vllm.models.kimi_k3 import ( + KimiK3ForConditionalGeneration_, + ) + + model = SimpleNamespace( + atom_config=SimpleNamespace(quant_config=QuantizationConfig()) + ) + quant_config = object() + for exclude in ( + "re:.*vision_tower.*", + "vision_tower*", + "*.vision_tower.*", + "model.vision_tower", + ): + assert ( + KimiK3ForConditionalGeneration_._maybe_ignore_quant_config( + model, + quant_config, + [exclude], + "vision_tower", + ) + is None + ), exclude + """) + + def test_kimi_k3_post_load_accepts_vllm_dtype(): _run_without_test_stubs(""" from inspect import Parameter, signature @@ -221,6 +362,49 @@ def test_kda_metadata_adapter_compacts_full_graph_padding(): """) +def test_kda_metadata_adapter_leaves_unpadded_decode_unchanged(): + _run_without_test_stubs(""" + from types import SimpleNamespace + + import torch + + import atom.plugin.vllm.kda_backend as kda_backend + + assert ( + "_stage_packed_decode_query_start_loc" + not in kda_backend.AtomKimiK3KDAMetadataBuilder.__dict__ + ) + + builder = SimpleNamespace( + use_full_cuda_graph=True, + decode_cudagraph_max_bs=2, + ) + common = SimpleNamespace( + query_start_loc_cpu=torch.tensor([0, 1, 2], dtype=torch.int32), + num_reqs=2, + ) + metadata = SimpleNamespace( + num_prefills=0, + num_spec_decodes=0, + num_decodes=2, + num_decode_tokens=2, + non_spec_state_indices_tensor=torch.tensor([3, 7], dtype=torch.int32), + non_spec_query_start_loc=torch.tensor([0, 1, 2], dtype=torch.int32), + ) + state_indices = metadata.non_spec_state_indices_tensor + query_start_loc = metadata.non_spec_query_start_loc + + kda_backend.AtomKimiK3KDAMetadataBuilder._adapt_full_graph_decode_metadata( + builder, common, metadata + ) + + assert metadata.non_spec_state_indices_tensor is state_indices + assert metadata.non_spec_query_start_loc is query_start_loc + assert metadata.num_decodes == 2 + assert metadata.num_decode_tokens == 2 + """) + + def test_gdn_metadata_builder_compacts_full_graph_padding(): _run_without_test_stubs(""" from types import SimpleNamespace diff --git a/tests/test_kimi_k3_plugin_config.py b/tests/test_kimi_k3_plugin_config.py new file mode 100644 index 0000000000..cfc5aa13a6 --- /dev/null +++ b/tests/test_kimi_k3_plugin_config.py @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + +from atom.config import ( + _MULTIMODAL_MODEL_TYPES, + _PLUGIN_SUPPORTED_MULTIMODAL_MODELS, +) + + +def test_kimi_k3_is_plugin_supported_multimodal(): + # Keep the full HF vision config when ATOM prepares Kimi-K3 for a plugin. + assert "kimi_k3" in _MULTIMODAL_MODEL_TYPES + assert "kimi_k3" in _PLUGIN_SUPPORTED_MULTIMODAL_MODELS