From 552adb46b8a1597674caf3a8c4600ea6edf24e3b Mon Sep 17 00:00:00 2001 From: whx-sjtu Date: Sat, 15 Aug 2026 22:52:06 -0700 Subject: [PATCH 1/6] feat(k3): reapply remaining K3 DSpark/vision delta on main Keep main's later K3/DSpark/core fixes; overlay only the still-unique plugin vision, DSpark draft, MLA/metadata, and recipe changes. Co-authored-by: Cursor --- atom/config.py | 1 + atom/plugin/vllm/attention/layer_mla.py | 18 +- .../vllm/attention/minimax_m3_attnetion.py | 44 +- atom/plugin/vllm/model_wrapper.py | 2 +- atom/plugin/vllm/models/kimi_k3.py | 146 ++++- .../plans/2026-08-04-k3-vision-vllm-plugin.md | 571 ++++++++++++++++++ ...2026-08-04-k3-vision-vllm-plugin-design.md | 179 ++++++ recipes/atom_vllm/KVV.md | 105 ++++ tests/plugin/test_vllm_kimi_k3.py | 93 ++- 9 files changed, 1119 insertions(+), 40 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-04-k3-vision-vllm-plugin.md create mode 100644 docs/superpowers/specs/2026-08-04-k3-vision-vllm-plugin-design.md create mode 100644 recipes/atom_vllm/KVV.md diff --git a/atom/config.py b/atom/config.py index e4ea51e90a..5eff69023d 100644 --- a/atom/config.py +++ b/atom/config.py @@ -676,6 +676,7 @@ def glm5_kpool_block_size(index_kpool: int) -> int: # multimodal models fully supported by plugin mode _PLUGIN_SUPPORTED_MULTIMODAL_MODELS: set[str] = { "kimi_k25", + "kimi_k3", "qwen3_5", "qwen3_5_moe", } diff --git a/atom/plugin/vllm/attention/layer_mla.py b/atom/plugin/vllm/attention/layer_mla.py index aef5ee7b16..998a84a504 100644 --- a/atom/plugin/vllm/attention/layer_mla.py +++ b/atom/plugin/vllm/attention/layer_mla.py @@ -1058,6 +1058,12 @@ def forward_impl( q = q[:num_actual_toks, ...] k_c_normed = k_c_normed[:num_actual_toks, ...] k_pe = k_pe[:num_actual_toks, ...] + # Model runner V2 sizes slot_mapping to the CUDA-graph token width and + # marks the tail with PAD_SLOT_ID, but leaves num_actual_tokens unpadded + # on the piecewise path; V1 never pads it. The cache kernels launch one + # block per slot and index q/kv by that block, so slot_mapping must not + # outrun the tensors sliced above. Same trim layer_mha already applies. + slot_mapping = attn_metadata.slot_mapping[:num_actual_toks] decode_q = q[:num_decode_tokens] prefill_q = q[num_decode_tokens:] @@ -1085,7 +1091,7 @@ def forward_impl( k_c_normed, self.rotary_emb_cos_sin_cache, self.rotary_emb.is_neox_style, - attn_metadata.slot_mapping, + slot_mapping, kv_cache, self.kv_cache_dtype, self._k_scale, @@ -1097,7 +1103,7 @@ def forward_impl( k_c_normed, k_pe.squeeze(1), kv_cache, - attn_metadata.slot_mapping.flatten(), + slot_mapping.flatten(), kv_cache_dtype=self.kv_cache_dtype, scale=self._k_scale, ) @@ -1195,7 +1201,7 @@ def forward_impl( self.kv_lora_rank + self.qk_rope_head_dim, ), decode_q[:, : self.num_heads] if fused_q_head_pad else decode_q, - attn_metadata.slot_mapping, + slot_mapping, self._k_scale, self._q_scale, positions, @@ -1367,6 +1373,7 @@ def forward_impl_sparse( q = q[:num_actual_toks, ...] k_c_normed = k_c_normed[:num_actual_toks, ...] k_pe = k_pe[:num_actual_toks, ...].unsqueeze(1) + slot_mapping = sparse_meta.slot_mapping[:num_actual_toks] positions = None if self._is_vllm_forward_context_available(): @@ -1445,7 +1452,7 @@ def forward_impl_sparse( kv_cache.shape[0], -1, self.kv_lora_rank + self.qk_rope_head_dim ), q_out, - sparse_meta.slot_mapping, + slot_mapping, self._k_scale, self._q_scale, positions, @@ -1506,8 +1513,9 @@ def forward( def get_kv_cache_spec(self, vllm_config): from vllm.v1.kv_cache_interface import MLAAttentionSpec + block_size = vllm_config.cache_config.block_size return MLAAttentionSpec( - block_size=vllm_config.cache_config.block_size, + block_size=block_size, num_kv_heads=1, head_size=self.head_size, dtype=self.kv_cache_torch_dtype, diff --git a/atom/plugin/vllm/attention/minimax_m3_attnetion.py b/atom/plugin/vllm/attention/minimax_m3_attnetion.py index bc4faccb2a..51041d81f5 100644 --- a/atom/plugin/vllm/attention/minimax_m3_attnetion.py +++ b/atom/plugin/vllm/attention/minimax_m3_attnetion.py @@ -10,8 +10,6 @@ MiniMax-M3's q/k norm + RoPE transform. """ -from typing import Optional - import aiter import torch from aiter import dtypes @@ -148,19 +146,19 @@ def __init__( head_dim: int, scale: float, num_kv_heads: int, - alibi_slopes: Optional[list[float]] = None, + alibi_slopes: list[float] | None = None, kv_cache_dtype: str = "bf16", layer_num: int = 0, use_mla: bool = False, - rotary_emb: Optional[nn.Module] = None, - prefix: Optional[str] = None, - q_norm: Optional[nn.Module] = None, - k_norm: Optional[nn.Module] = None, + rotary_emb: nn.Module | None = None, + prefix: str | None = None, + q_norm: nn.Module | None = None, + k_norm: nn.Module | None = None, cache_config=None, quant_config=None, - index_q_norm: Optional[nn.Module] = None, - index_k_norm: Optional[nn.Module] = None, - index_rotary_emb: Optional[nn.Module] = None, + index_q_norm: nn.Module | None = None, + index_k_norm: nn.Module | None = None, + index_rotary_emb: nn.Module | None = None, index_q_size: int = 0, index_head_dim: int = 0, topk: int = 0, @@ -746,8 +744,8 @@ def _run_sparse_attention( def _forward_with_output( self, qkv: torch.Tensor, - positions: Optional[torch.Tensor] = None, - output: Optional[torch.Tensor] = None, + positions: torch.Tensor | None = None, + output: torch.Tensor | None = None, ) -> torch.Tensor: main_metadata, index_metadata = self._metadata_for_layer() num_tokens = qkv.shape[0] @@ -823,9 +821,9 @@ def forward( query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, - positions: Optional[torch.Tensor] = None, - q_scale: Optional[torch.Tensor] = None, - qkv: Optional[torch.Tensor] = None, + positions: torch.Tensor | None = None, + q_scale: torch.Tensor | None = None, + qkv: torch.Tensor | None = None, **kwargs, ) -> torch.Tensor: del query, key, value, q_scale, kwargs @@ -852,14 +850,14 @@ def __init__( head_dim: int, scale: float, num_kv_heads: int, - alibi_slopes: Optional[list[float]] = None, + alibi_slopes: list[float] | None = None, kv_cache_dtype: str = "bf16", layer_num: int = 0, use_mla: bool = False, - rotary_emb: Optional[nn.Module] = None, - prefix: Optional[str] = None, - q_norm: Optional[nn.Module] = None, - k_norm: Optional[nn.Module] = None, + rotary_emb: nn.Module | None = None, + prefix: str | None = None, + q_norm: nn.Module | None = None, + k_norm: nn.Module | None = None, cache_config=None, quant_config=None, **kwargs, @@ -989,9 +987,9 @@ def forward( query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, - positions: Optional[torch.Tensor] = None, - q_scale: Optional[torch.Tensor] = None, - qkv: Optional[torch.Tensor] = None, + positions: torch.Tensor | None = None, + q_scale: torch.Tensor | None = None, + qkv: torch.Tensor | None = None, **kwargs, ) -> torch.Tensor: del query, key, value, q_scale, kwargs diff --git a/atom/plugin/vllm/model_wrapper.py b/atom/plugin/vllm/model_wrapper.py index 9a621aa0bb..ac0a63f263 100644 --- a/atom/plugin/vllm/model_wrapper.py +++ b/atom/plugin/vllm/model_wrapper.py @@ -156,7 +156,7 @@ def _maybe_set_v4_expert_dtype(atom_config, vllm_config) -> None: "Qwen3_5ForConditionalGeneration": "atom.plugin.vllm.models.qwen3_5:Qwen3_5ForConditionalGeneration_", "KimiK25ForConditionalGeneration": "atom.plugin.vllm.models.kimi_k25:KimiK25ForConditionalGeneration_", "KimiK3ForConditionalGeneration": ( - "atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLM" + "atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGeneration_" ), "MiniMaxM2ForCausalLM": "atom.models.minimax_m2:MiniMaxM2ForCausalLM", "DeepseekV4ForCausalLM": "atom.plugin.vllm.models.deepseek_v4:DeepseekV4ForCausalLM", diff --git a/atom/plugin/vllm/models/kimi_k3.py b/atom/plugin/vllm/models/kimi_k3.py index 35ca84bd62..58926cf5ca 100644 --- a/atom/plugin/vllm/models/kimi_k3.py +++ b/atom/plugin/vllm/models/kimi_k3.py @@ -28,7 +28,6 @@ ) from atom.models.utils import IntermediateTensors from atom.plugin.vllm.kda_backend import AtomKimiK3KDAAttentionBackend -from atom.plugin.vllm.model_wrapper import ATOMMoEForCausalLM from atom.utils.forward_context import get_forward_context as get_atom_forward_context @@ -471,6 +470,28 @@ def __init__(self, *args, **kwargs): kimi_k3_base.KimiKDAAttention = original_kda_cls kimi_k3_base.KimiLinearModel = original_model_cls + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + # Required by vLLM SupportsMultiModal.get_language_model discovery. + return self.get_input_embeddings(input_ids) + + @property + def model(self) -> nn.Module: + """The body the DSpark loader reaches for ``embed_tokens`` on. + + ``load_dspark_model`` reads ``get_language_model().model.embed_tokens`` + and ``get_language_model().lm_head``. Multimodal discovery stops at this + class, one level above the causal LM that actually owns both, so expose + them here rather than redirecting ``get_language_model`` -- which the + vision path also uses. A property, so it stays out of ``named_modules`` + and the checkpoint prefixes are unchanged. + """ + return self.language_model.model + + @property + def lm_head(self) -> nn.Module: + """The head the DSpark draft borrows to score its block.""" + return self.language_model.lm_head + def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None: self.language_model.model.set_aux_hidden_state_layers(layers) @@ -480,16 +501,121 @@ def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]: return (2, num_layers // 2, num_layers - 3) -class KimiK3ForCausalLMVllm(ATOMMoEForCausalLM, IsHybrid): - def get_language_model(self): - """The causal LM that owns ``model.embed_tokens`` and ``lm_head``. +@MULTIMODAL_REGISTRY.register_processor( + KimiK3MultiModalProcessor, + info=KimiK3ProcessingInfo, + dummy_inputs=KimiK3DummyInputsBuilder, +) +class KimiK3ForConditionalGeneration_(vLLMKimiK3): + hf_to_atom_mapper = WeightsMapper( + orig_to_new_prefix={ + # ATOM nests language_model.language_model (ATOM KimiK3ForCausalLM + # wraps KimiLinearForCausalLM); checkpoint stores language_model.model. + "language_model.": "language_model.language_model.", + "mm_projector.proj.0": "mm_projector.linear_1", + "mm_projector.proj.2": "mm_projector.linear_2", + } + ) + packed_modules_mapping = KimiK3ForCausalLMBase.packed_modules_mapping + quant_exclude_name_mapping: ClassVar[dict[str, str]] = { + "language_model.model.": "language_model.language_model.model.", + "language_model.lm_head": "language_model.language_model.lm_head", + } + + def __init__(self, atom_config: Config, prefix: str = "model"): + nn.Module.__init__(self) + hf_config = getattr(atom_config, "hf_config", None) + assert hf_config is not None, "hf_config is not found in atom_config" + vision_config = hf_config.vision_config + self.config = hf_config + self.atom_config = atom_config - DSpark's loader binds the target's embedding and head into the draft - through this hook; Kimi-K3 keeps both one level in, under - ``language_model``, so without it the loader would look for them on this - wrapper and silently leave the draft with neither. - """ - return self.model.language_model + vllm_config = atom_config.plugin_config.vllm_config + quant_config = vllm_config.quant_config + atom_quant_config = atom_config.quant_config + multimodal_config = vllm_config.model_config.multimodal_config + self.multimodal_config = multimodal_config + self.use_data_parallel = is_vit_use_data_parallel( + vision_config.num_attention_heads + ) + + with self._mark_tower_model(vllm_config, "image"): + self.vision_tower = MoonViT3dPretrainedModel( + vision_config, + quant_config=self._maybe_ignore_quant_config( + quant_config, + atom_quant_config.exclude_layers or [], + "vision_tower", + ), + prefix=maybe_prefix(prefix, "vision_tower"), + ) + self.mm_projector = KimiK25MultiModalProjector( + config=vision_config, + use_data_parallel=self.use_data_parallel, + quant_config=self._maybe_ignore_quant_config( + quant_config, + atom_quant_config.exclude_layers or [], + "mm_projector", + ), + prefix=maybe_prefix(prefix, "mm_projector"), + ) + + self.quant_config = quant_config + with self._mark_language_model(vllm_config): + self.language_model = KimiK3ForCausalLM( + atom_config=atom_config, + prefix=maybe_prefix(prefix, "language_model"), + ) + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors + ) + self.media_placeholder = self.config.media_placeholder_token_id + self.packed_modules_mapping = self.language_model.packed_modules_mapping + self.weights_mapping = self.language_model.weights_mapping + + def _maybe_ignore_quant_config( + self, quant_config: Any, exclude_layers: list[str], layer_name: str + ): + for exclude_layer in exclude_layers: + if QuantizationConfig._matches_exclude( + layer_name, exclude_layer, check_contains=True + ): + return None + return quant_config + + def load_weights(self, weights): + return load_model_in_plugin_mode( + model=self, + config=self.atom_config, + prefix="model.", + weights_mapper=self.hf_to_atom_mapper, + ) + + def get_expert_mapping(self): + return self.language_model.get_expert_mapping() + + def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None: + self.language_model.set_aux_hidden_state_layers(layers) + + def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]: + return self.language_model.get_eagle3_aux_hidden_state_layers() + + +@MULTIMODAL_REGISTRY.register_processor( + KimiK3MultiModalProcessor, + info=KimiK3ProcessingInfo, + dummy_inputs=KimiK3DummyInputsBuilder, +) +class KimiK3ForCausalLMVllm(ATOMForConditionalGeneration, IsHybrid): + @classmethod + def get_placeholder_str(cls, modality: str, i: int) -> str | None: + if modality == "image": + return "<|kimi_image_placeholder|>" + raise ValueError(f"Unsupported modality: {modality}") + + def load_weights(self, weights): + return self.model.load_weights(weights) @classmethod def get_mamba_state_dtype_from_config( diff --git a/docs/superpowers/plans/2026-08-04-k3-vision-vllm-plugin.md b/docs/superpowers/plans/2026-08-04-k3-vision-vllm-plugin.md new file mode 100644 index 0000000000..c15d7e6671 --- /dev/null +++ b/docs/superpowers/plans/2026-08-04-k3-vision-vllm-plugin.md @@ -0,0 +1,571 @@ +# Kimi-K3 image support (vLLM plugin) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Enable image (vision) input for Kimi-K3 in ATOM vLLM plugin mode by mirroring K2.5's wiring, reusing vLLM's upstream K3 vision stack while keeping ATOM's KDA-optimized hybrid language model. + +**Architecture:** Three-layer plugin structure. The **outer** class `KimiK3ForCausalLMVllm` (what vLLM's registry instantiates) becomes multimodal by switching its base to `ATOMForConditionalGeneration` while retaining `IsHybrid` + the `get_mamba_state_*` classmethods. A **new inner** class `KimiK3ForConditionalGeneration_` (built by ATOM as `.model`) subclasses vLLM's upstream `KimiK3ForConditionalGeneration` to inherit the vision path (`embed_multimodal`, media parsing, `MoonViT3dPretrainedModel`, `KimiK25MultiModalProjector`), but instantiates ATOM's plugin `KimiK3ForCausalLM` as its `language_model` and loads weights via ATOM's plugin loader. A config gate lets the plugin keep `vision_config` instead of stripping it. + +**Tech Stack:** Python, PyTorch, vLLM (editable checkout at `/shared/amdgpu/home/gyu_qle/vllm`), AITER, ROCm. Tests: pytest (no GPU; real-import checks run via subprocess). + +## Global Constraints + +- **NEVER modify native K3 files** `atom/models/kimi_k3.py` or `atom/models/kimi_k3_dspark.py`. Import them read-only only. All new `embed_input_ids` etc. go on the **plugin** subclass in `atom/plugin/vllm/models/kimi_k3.py`. +- **Plugin mode only.** Do not touch native-mode registration `atom/model_engine/model_runner.py` (K3 stays text-only there). +- **Image only.** No video code paths. +- **Reuse the checkpoint's HF `KimiK3Config` directly.** Do NOT create `atom/model_config/kimi_k3.py`. +- **Do not modify `@support_torch_compile`-decorated model files.** +- Format/lint before every commit: `black . && ruff check .` (CI enforced; ruff only fails on PR-diff lines). +- Reference checkpoint (multimodal): `/workspace/shared/data/amd_int/models/Kimi-K3` — arch `KimiK3ForConditionalGeneration`, `model_type: kimi_k3`, `media_placeholder_token_id: 163605`, `image_placeholder: <|kimi_image_placeholder|>`. Weight prefixes: `language_model.*`, `vision_tower.*`, `mm_projector.{proj.0,proj.2,post_norm}`. +- Before restarting any server: `rm -rf /root/.cache/atom/*` and `export AITER_LOG_LEVEL=WARNING`. + +--- + +### Task 1: Config gate — let plugin mode keep K3's vision_config + +**Files:** +- Modify: `atom/config.py:610-614` (`_PLUGIN_SUPPORTED_MULTIMODAL_MODELS`) +- Test: `tests/plugin/test_vllm_kimi_k3.py` + +**Interfaces:** +- Consumes: nothing. +- Produces: `"kimi_k3"` present in `atom.config._PLUGIN_SUPPORTED_MULTIMODAL_MODELS`. This makes `get_hf_config()` skip the text-config stripping for `model_type == "kimi_k3"` under `is_vllm()`, so `hf_config` retains `.vision_config` + `.text_config`. + +**Context:** In `atom/config.py::get_hf_config`, when `is_vllm()` is true, any `model_type` listed in `_PLUGIN_SUPPORTED_MULTIMODAL_MODELS` is filtered OUT of the `multimodal_model_types` stripping map (lines 630-636), so the full multimodal config is preserved. `kimi_k3` is already in `_MULTIMODAL_MODEL_TYPES` (line 602) but missing from the plugin set (lines 610-614). + +- [ ] **Step 1: Write the failing test** + +Add to `tests/plugin/test_vllm_kimi_k3.py`: + +```python +def test_kimi_k3_is_plugin_supported_multimodal(): + from atom.config import ( + _MULTIMODAL_MODEL_TYPES, + _PLUGIN_SUPPORTED_MULTIMODAL_MODELS, + ) + + # kimi_k3 must be a known multimodal type AND opted into plugin-mode + # pass-through, so get_hf_config keeps vision_config instead of stripping + # to text_config. + assert "kimi_k3" in _MULTIMODAL_MODEL_TYPES + assert "kimi_k3" in _PLUGIN_SUPPORTED_MULTIMODAL_MODELS +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py::test_kimi_k3_is_plugin_supported_multimodal -v` +Expected: FAIL — assertion error (`"kimi_k3"` not in `_PLUGIN_SUPPORTED_MULTIMODAL_MODELS`). + +- [ ] **Step 3: Add kimi_k3 to the plugin-supported set** + +In `atom/config.py`, change: + +```python +_PLUGIN_SUPPORTED_MULTIMODAL_MODELS: set[str] = { + "kimi_k25", + "qwen3_5", + "qwen3_5_moe", +} +``` + +to: + +```python +_PLUGIN_SUPPORTED_MULTIMODAL_MODELS: set[str] = { + "kimi_k25", + "kimi_k3", + "qwen3_5", + "qwen3_5_moe", +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py::test_kimi_k3_is_plugin_supported_multimodal -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +black atom/config.py tests/plugin/test_vllm_kimi_k3.py && ruff check atom/config.py tests/plugin/test_vllm_kimi_k3.py +git add atom/config.py tests/plugin/test_vllm_kimi_k3.py +git commit -m "feat(k3): keep vision_config in plugin mode for kimi_k3" +``` + +--- + +### Task 2: Add the multimodal inner class + language-model embed hook + +**Files:** +- Modify: `atom/plugin/vllm/models/kimi_k3.py` (add imports; add `embed_input_ids` to plugin `KimiK3ForCausalLM`; add new `KimiK3ForConditionalGeneration_`) +- Test: `tests/plugin/test_vllm_kimi_k3.py` + +**Interfaces:** +- Consumes: `atom.config._PLUGIN_SUPPORTED_MULTIMODAL_MODELS` containing `"kimi_k3"` (Task 1); vLLM upstream `vllm.models.kimi_k3.KimiK3ForConditionalGeneration`, `vllm.model_executor.models.kimi_k25_vit.{MoonViT3dPretrainedModel,KimiK25MultiModalProjector}`, `vllm.models.kimi_k3.common.mm_preprocess.{KimiK3MultiModalProcessor,KimiK3ProcessingInfo,KimiK3DummyInputsBuilder}`. +- Produces: + - `atom.plugin.vllm.models.kimi_k3.KimiK3ForConditionalGeneration_` — subclass of vLLM's `KimiK3ForConditionalGeneration`; `__init__(self, atom_config: Config, prefix: str = "model")`; attrs `vision_tower`, `mm_projector`, `language_model` (an ATOM plugin `KimiK3ForCausalLM`); classvar `hf_to_atom_mapper: WeightsMapper`; methods `load_weights(weights) -> set[str]`, `get_expert_mapping()`. + - `KimiK3ForCausalLM.embed_input_ids(self, input_ids) -> torch.Tensor` (plugin subclass only). + +**Context:** Mirror `atom/plugin/vllm/models/kimi_k25.py::KimiK25ForConditionalGeneration_` (lines 178-288). The vision-tower/projector construction must mirror `/shared/amdgpu/home/gyu_qle/vllm/vllm/models/kimi_k3/amd/model.py` lines 96-123 (only the `language_model` line differs — we use ATOM's, not `init_vllm_registered_model`). The weight mapper handles the double `language_model` nesting: the checkpoint stores text under `language_model.model.*`, but ATOM nests `inner.language_model` (an ATOM `KimiK3ForCausalLM`) `.language_model` (a `KimiLinearForCausalLM`) `.model.*`, so `language_model.` → `language_model.language_model.`. + +- [ ] **Step 1: Write the failing test** + +Add to `tests/plugin/test_vllm_kimi_k3.py`: + +```python +def test_kimi_k3_inner_conditional_generation_class(): + _run_without_test_stubs(""" + from vllm.models.kimi_k3 import ( + KimiK3ForConditionalGeneration as vLLMKimiK3, + ) + from atom.plugin.vllm.models.kimi_k3 import ( + KimiK3ForCausalLM, + KimiK3ForConditionalGeneration_, + ) + + # Inner class subclasses the upstream multimodal model so it inherits + # embed_multimodal / media parsing / vision-tower forward. + assert issubclass(KimiK3ForConditionalGeneration_, vLLMKimiK3) + + # ATOM plugin loading + expert routing entry points exist. + assert hasattr(KimiK3ForConditionalGeneration_, "load_weights") + assert hasattr(KimiK3ForConditionalGeneration_, "get_expert_mapping") + + # Weight mapper collapses the double language_model nesting and renames + # the projector layers. + mapper = KimiK3ForConditionalGeneration_.hf_to_atom_mapper + assert mapper.orig_to_new_prefix["language_model."] == ( + "language_model.language_model." + ) + assert mapper.orig_to_new_prefix["mm_projector.proj.0"] == ( + "mm_projector.linear_1" + ) + assert mapper.orig_to_new_prefix["mm_projector.proj.2"] == ( + "mm_projector.linear_2" + ) + + # Language-model wrapper exposes embed_input_ids for vLLM multimodal + # discovery. + assert hasattr(KimiK3ForCausalLM, "embed_input_ids") + """) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py::test_kimi_k3_inner_conditional_generation_class -v` +Expected: FAIL — `ImportError: cannot import name 'KimiK3ForConditionalGeneration_'`. + +- [ ] **Step 3: Add imports at the top of `atom/plugin/vllm/models/kimi_k3.py`** + +Add after the existing imports (keep the existing ones): + +```python +from torch import nn +from vllm.models.kimi_k3 import ( + KimiK3ForConditionalGeneration as vLLMKimiK3, +) +from vllm.models.kimi_k3.common.mm_preprocess import ( + KimiK3DummyInputsBuilder, + KimiK3MultiModalProcessor, + KimiK3ProcessingInfo, +) +from vllm.model_executor.models.kimi_k25_vit import ( + KimiK25MultiModalProjector, + MoonViT3dPretrainedModel, +) +from vllm.model_executor.models.vision import is_vit_use_data_parallel +from vllm.multimodal import MULTIMODAL_REGISTRY + +from atom.config import Config, QuantizationConfig +from atom.model_loader.loader import WeightsMapper, load_model_in_plugin_mode +from atom.models.utils import maybe_prefix +``` + +(Note: `ATOMForConditionalGeneration` is imported in Task 3, where the outer class first uses it — importing it here would trip ruff's unused-import check at the Task 2 lint gate.) + +- [ ] **Step 4: Add `embed_input_ids` to the plugin `KimiK3ForCausalLM`** + +The class currently is (lines ~161-168): + +```python +class KimiK3ForCausalLM(KimiK3ForCausalLMBase): + def __init__(self, *args, **kwargs): + original_kda_cls = kimi_k3_base.KimiKDAAttention + kimi_k3_base.KimiKDAAttention = KimiKDAAttentionVllm + try: + super().__init__(*args, **kwargs) + finally: + kimi_k3_base.KimiKDAAttention = original_kda_cls +``` + +Add the method (delegates to the native `get_input_embeddings`; required by vLLM `SupportsMultiModal.get_language_model` discovery, matching K2.5's `KimiK25ForCausalLM.embed_input_ids`): + +```python +class KimiK3ForCausalLM(KimiK3ForCausalLMBase): + def __init__(self, *args, **kwargs): + original_kda_cls = kimi_k3_base.KimiKDAAttention + kimi_k3_base.KimiKDAAttention = KimiKDAAttentionVllm + try: + super().__init__(*args, **kwargs) + finally: + kimi_k3_base.KimiKDAAttention = original_kda_cls + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + # Required by vLLM SupportsMultiModal.get_language_model discovery. + return self.get_input_embeddings(input_ids) +``` + +- [ ] **Step 5: Add the inner `KimiK3ForConditionalGeneration_` class** + +Add ABOVE the existing `KimiK3ForCausalLMVllm` class (so the outer can reference it is not required — registry wiring happens in Task 3). Mirror `vllm/models/kimi_k3/amd/model.py` lines 96-123 for the vision setup: + +```python +@MULTIMODAL_REGISTRY.register_processor( + KimiK3MultiModalProcessor, + info=KimiK3ProcessingInfo, + dummy_inputs=KimiK3DummyInputsBuilder, +) +class KimiK3ForConditionalGeneration_(vLLMKimiK3): + hf_to_atom_mapper = WeightsMapper( + orig_to_new_prefix={ + # ATOM nests language_model.language_model (ATOM KimiK3ForCausalLM + # wraps KimiLinearForCausalLM); checkpoint stores language_model.model. + "language_model.": "language_model.language_model.", + "mm_projector.proj.0": "mm_projector.linear_1", + "mm_projector.proj.2": "mm_projector.linear_2", + } + ) + + def __init__(self, atom_config: Config, prefix: str = "model"): + # Protocols have no __init__; skip vLLMKimiK3.__init__ (it would build + # vLLM's KimiLinear language model) and set up nn.Module directly. + nn.Module.__init__(self) + hf_config = getattr(atom_config, "hf_config", None) + assert hf_config is not None, "hf_config is not found in atom_config" + vision_config = hf_config.vision_config + self.config = hf_config + self.atom_config = atom_config + + vllm_config = atom_config.plugin_config.vllm_config + quant_config = vllm_config.quant_config + atom_quant_config = atom_config.quant_config + multimodal_config = vllm_config.model_config.multimodal_config + self.multimodal_config = multimodal_config + self.use_data_parallel = is_vit_use_data_parallel( + vision_config.num_attention_heads + ) + + with self._mark_tower_model(vllm_config, "image"): + self.vision_tower = MoonViT3dPretrainedModel( + vision_config, + quant_config=self._maybe_ignore_quant_config( + quant_config, + atom_quant_config.exclude_layers or [], + "vision_tower", + ), + prefix=maybe_prefix(prefix, "vision_tower"), + ) + self.mm_projector = KimiK25MultiModalProjector( + config=vision_config, + use_data_parallel=self.use_data_parallel, + quant_config=self._maybe_ignore_quant_config( + quant_config, + atom_quant_config.exclude_layers or [], + "mm_projector", + ), + prefix=maybe_prefix(prefix, "mm_projector"), + ) + + self.quant_config = quant_config + with self._mark_language_model(vllm_config): + self.language_model = KimiK3ForCausalLM( + atom_config=atom_config, + prefix=maybe_prefix(prefix, "language_model"), + ) + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors + ) + self.media_placeholder = self.config.media_placeholder_token_id + + def _maybe_ignore_quant_config( + self, quant_config, exclude_layers, layer_name: str + ): + for exclude_layer in exclude_layers: + if QuantizationConfig._matches_exclude( + layer_name, exclude_layer, check_contains=True + ): + return None + return quant_config + + def load_weights(self, weights): + # weights generator is discarded; ATOM loads from disk in plugin mode. + # prefix="model." because ATOMModelBase constructs this as `.model`. + return load_model_in_plugin_mode( + model=self, + config=self.atom_config, + prefix="model.", + weights_mapper=self.hf_to_atom_mapper, + ) + + def get_expert_mapping(self): + return self.language_model.get_expert_mapping() +``` + +- [ ] **Step 6: Run the test to verify it passes** + +Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py::test_kimi_k3_inner_conditional_generation_class -v` +Expected: PASS + +- [ ] **Step 7: Run the full K3 plugin test module to confirm nothing else broke** + +Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py -v` +Expected: All existing tests still pass (the registry-sync test is updated in Task 3, not yet). + +- [ ] **Step 8: Commit** + +```bash +black atom/plugin/vllm/models/kimi_k3.py tests/plugin/test_vllm_kimi_k3.py && ruff check atom/plugin/vllm/models/kimi_k3.py tests/plugin/test_vllm_kimi_k3.py +git add atom/plugin/vllm/models/kimi_k3.py tests/plugin/test_vllm_kimi_k3.py +git commit -m "feat(k3): add multimodal inner class reusing upstream vision stack" +``` + +--- + +### Task 3: Make the outer class multimodal + rewire the registry + +**Files:** +- Modify: `atom/plugin/vllm/models/kimi_k3.py` (`KimiK3ForCausalLMVllm` base + methods) +- Modify: `atom/plugin/vllm/model_wrapper.py:158-160` (`_ATOM_MODEL_CLASSES["KimiK3ForConditionalGeneration"]`) +- Test: `tests/plugin/test_vllm_kimi_k3.py` + +**Interfaces:** +- Consumes: `KimiK3ForConditionalGeneration_` (Task 2); `ATOMForConditionalGeneration` (from `atom.plugin.vllm.model_wrapper`); `IsHybrid` (already imported). +- Produces: `KimiK3ForCausalLMVllm` with base `(ATOMForConditionalGeneration, IsHybrid)`, `get_placeholder_str`, `load_weights`, and the retained `get_mamba_state_*` classmethods; `_ATOM_MODEL_CLASSES["KimiK3ForConditionalGeneration"]` pointing at `...kimi_k3:KimiK3ForConditionalGeneration_`. + +**Context:** vLLM inspects the OUTER class (from `register.py`'s `_VLLM_MODEL_REGISTRY_OVERRIDES`, which already maps the arch to `KimiK3ForCausalLMVllm`) for interfaces. So `SupportsMultiModal` (via `ATOMForConditionalGeneration`) and `IsHybrid` must both live there. `ATOMForConditionalGeneration` also brings `SupportsMRoPE`, which stays dormant because K3's config has no `rope_scaling`/mrope. The existing registry-sync test must be updated to the new inner qualname. + +- [ ] **Step 1: Write/adjust the failing tests** + +In `tests/plugin/test_vllm_kimi_k3.py`, UPDATE the existing `test_kimi_k3_plugin_registries_are_synchronized` so the inner map points to the new class: + +```python +def test_kimi_k3_plugin_registries_are_synchronized(): + from atom.plugin.vllm.model_wrapper import _ATOM_MODEL_CLASSES + from atom.plugin.vllm.register import _VLLM_MODEL_REGISTRY_OVERRIDES + + arch = "KimiK3ForConditionalGeneration" + assert ( + _VLLM_MODEL_REGISTRY_OVERRIDES[arch] + == "atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLMVllm" + ) + assert ( + _ATOM_MODEL_CLASSES[arch] + == "atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGeneration_" + ) +``` + +ADD a new structural test for the outer class: + +```python +def test_kimi_k3_outer_is_multimodal_and_hybrid(): + _run_without_test_stubs(""" + from vllm.model_executor.models.interfaces import IsHybrid + + from atom.plugin.vllm.model_wrapper import ATOMForConditionalGeneration + from atom.plugin.vllm.models.kimi_k3 import KimiK3ForCausalLMVllm + + mro = KimiK3ForCausalLMVllm.__mro__ + # Multimodal via ATOMForConditionalGeneration; hybrid state retained. + assert ATOMForConditionalGeneration in mro + assert IsHybrid in mro + + # Image placeholder matches the checkpoint token. + assert ( + KimiK3ForCausalLMVllm.get_placeholder_str("image", 0) + == "<|kimi_image_placeholder|>" + ) + + # Hybrid mamba-state entry points survive the base-class change. + for name in ( + "get_mamba_state_dtype_from_config", + "get_mamba_state_shape_from_config", + "get_mamba_state_copy_func", + ): + assert hasattr(KimiK3ForCausalLMVllm, name) + """) +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py::test_kimi_k3_plugin_registries_are_synchronized tests/plugin/test_vllm_kimi_k3.py::test_kimi_k3_outer_is_multimodal_and_hybrid -v` +Expected: FAIL — registry test asserts old qualname; outer test fails (`ATOMForConditionalGeneration` not in MRO / no `get_placeholder_str`). + +- [ ] **Step 3: Add the `ATOMForConditionalGeneration` import** + +In `atom/plugin/vllm/models/kimi_k3.py`, add the import used by the new outer base: + +```python +from atom.plugin.vllm.model_wrapper import ATOMForConditionalGeneration +``` + +- [ ] **Step 4: Rewire the inner-class registry map** + +In `atom/plugin/vllm/model_wrapper.py`, change the `_ATOM_MODEL_CLASSES` entry: + +```python + "KimiK3ForConditionalGeneration": ( + "atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLM" + ), +``` + +to: + +```python + "KimiK3ForConditionalGeneration": ( + "atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGeneration_" + ), +``` + +- [ ] **Step 5: Make the outer class multimodal** + +In `atom/plugin/vllm/models/kimi_k3.py`, change the `KimiK3ForCausalLMVllm` class definition. Current: + +```python +class KimiK3ForCausalLMVllm(ATOMMoEForCausalLM, IsHybrid): + @classmethod + def get_mamba_state_dtype_from_config( + cls, + vllm_config: VllmConfig, + ) -> tuple[torch.dtype, torch.dtype]: + return _get_k3_state_dtype(vllm_config) + ... +``` + +Change ONLY the base classes and ADD `get_placeholder_str` + `load_weights`; keep all three `get_mamba_state_*` classmethods exactly as-is. Also add the processor decorator (matches K2.5, which decorates the outer too): + +```python +@MULTIMODAL_REGISTRY.register_processor( + KimiK3MultiModalProcessor, + info=KimiK3ProcessingInfo, + dummy_inputs=KimiK3DummyInputsBuilder, +) +class KimiK3ForCausalLMVllm(ATOMForConditionalGeneration, IsHybrid): + @classmethod + def get_placeholder_str(cls, modality: str, i: int) -> str | None: + if modality == "image": + return "<|kimi_image_placeholder|>" + raise ValueError(f"Unsupported modality: {modality}") + + def load_weights(self, weights): + return self.model.load_weights(weights) + + @classmethod + def get_mamba_state_dtype_from_config( + cls, + vllm_config: VllmConfig, + ) -> tuple[torch.dtype, torch.dtype]: + return _get_k3_state_dtype(vllm_config) + + @classmethod + def get_mamba_state_shape_from_config( + cls, + vllm_config: VllmConfig, + ) -> tuple[tuple[int, ...], tuple[int, ...]]: + return _get_k3_state_shape(vllm_config) + + @classmethod + def get_mamba_state_copy_func( + cls, + ) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: + return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func() +``` + +- [ ] **Step 6: Remove the now-unused `ATOMMoEForCausalLM` import if nothing else uses it** + +Run: `grep -n "ATOMMoEForCausalLM" atom/plugin/vllm/models/kimi_k3.py` +If the only remaining reference is the `from atom.plugin.vllm.model_wrapper import ATOMMoEForCausalLM` line, delete that import (ruff will flag it as unused otherwise). Keep it if still referenced. + +- [ ] **Step 7: Run the tests to verify they pass** + +Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py -v` +Expected: All pass (registry-sync updated, outer structural test passes, Task 2 test still passes). + +- [ ] **Step 8: Commit** + +```bash +black atom/plugin/vllm/models/kimi_k3.py atom/plugin/vllm/model_wrapper.py tests/plugin/test_vllm_kimi_k3.py && ruff check atom/plugin/vllm/models/kimi_k3.py atom/plugin/vllm/model_wrapper.py tests/plugin/test_vllm_kimi_k3.py +git add atom/plugin/vllm/models/kimi_k3.py atom/plugin/vllm/model_wrapper.py tests/plugin/test_vllm_kimi_k3.py +git commit -m "feat(k3): wire multimodal outer class + registry for image support" +``` + +--- + +### Task 4: GPU validation — weight-load completeness, image smoke, text-accuracy parity + +**Files:** +- No source changes expected. If validation surfaces a weight-mapping mismatch, fix `hf_to_atom_mapper` in `atom/plugin/vllm/models/kimi_k3.py` (adjust prefixes to match the "unexpected/missing weights" error), re-run, and commit as a fix. + +**Interfaces:** +- Consumes: everything from Tasks 1-3. +- Produces: a validated K3 image path (recorded in the PR description / a note under `k3_traces/` or the memory file). + +**Context:** This task cannot be a pytest unit test — it needs the GPU + the multimodal checkpoint. Use the `/run-atom-workload` skill for the canonical stop→start→workload→drain→stop flow. The K3 checkpoint is `/workspace/shared/data/amd_int/models/Kimi-K3`. Weight-load completeness is the real gate on the mapper: vLLM raises on unexpected/missing weights at load time. + +- [ ] **Step 1: Clean caches and start the server in plugin mode** + +```bash +rm -rf /root/.cache/atom/* +export AITER_LOG_LEVEL=WARNING +``` + +Start the server via the `/run-atom-workload` skill (plugin/vLLM mode) against `/workspace/shared/data/amd_int/models/Kimi-K3`. After startup, confirm the model is actually resident on GPU (not just HTTP 200): + +Run: `rocm-smi --showmemuse` +Expected: VRAM% > 0 on the TP GPUs; server log shows the vision tower + language model loaded with **no "unexpected weights" / "missing weights" errors**. If such errors appear, capture the offending names and adjust `hf_to_atom_mapper` prefixes, then restart from Step 1. + +- [ ] **Step 2: Image smoke test (describe an image)** + +Send one OpenAI-compatible chat request with an image URL/base64 + "Describe this image." (use the `/run-atom-workload` client or a `curl` to `/v1/chat/completions`). +Expected: a coherent, image-relevant description (not gibberish, not a text-only hallucination). This confirms `embed_multimodal` → vision tower → projector → merge → KDA/MLA/MoE forward all compose. + +- [ ] **Step 3: Text-only request still works** + +Send a plain text chat request (no image) to the same server. +Expected: a normal coherent completion — confirms `embed_multimodal` returns `None` and the text path is unchanged. + +- [ ] **Step 4: Text-accuracy parity (GSM8K)** + +Run GSM8K via `/run-atom-workload` (lm_eval) against this server and compare to the current text-only K3 baseline on this branch. +Expected: accuracy within normal run-to-run variance of the text-only baseline — proves the KDA/dual-stream/hybrid-state path did not regress from the rewiring. + +- [ ] **Step 5: Record results** + +Note the smoke output, GSM8K score vs baseline, and any mapper fix in the PR description and update the memory file `k3-vision-vllm-plugin.md` with the GPU-validated status. + +- [ ] **Step 6: Commit any fixes** + +```bash +# only if Step 1 required a mapper fix +black atom/plugin/vllm/models/kimi_k3.py && ruff check atom/plugin/vllm/models/kimi_k3.py +git add atom/plugin/vllm/models/kimi_k3.py +git commit -m "fix(k3): correct vision/language weight prefix mapping" +``` + +--- + +## Self-Review + +**Spec coverage:** +- Config gate (`_PLUGIN_SUPPORTED_MULTIMODAL_MODELS`) → Task 1. ✓ +- New inner `KimiK3ForConditionalGeneration_` (subclass upstream, reuse vision, swap ATOM LM, ATOM weight loading) → Task 2. ✓ +- `embed_input_ids` on plugin LM (not native) → Task 2. ✓ +- Outer class → `ATOMForConditionalGeneration, IsHybrid` + `get_placeholder_str` + `load_weights` + retained `get_mamba_state_*` → Task 3. ✓ +- Registry rewire (`_ATOM_MODEL_CLASSES`; `register.py` unchanged) → Task 3. ✓ +- No new config wrapper; native files untouched; plugin/image only → enforced in Global Constraints and each task's file list. ✓ +- Validation: image smoke + text-accuracy-unchanged → Task 4. ✓ +- Risk: multimodal+hybrid composition, dormant MRoPE, weight mapping, VRAM → addressed in Tasks 3/4 and constraints. ✓ + +**Placeholder scan:** No TBD/TODO; every code step has concrete code; the one runtime-verified detail (weight mapper) has a concrete starting value derived from the checkpoint layout plus an explicit adjust-on-error procedure in Task 4. ✓ + +**Type/name consistency:** `KimiK3ForConditionalGeneration_`, `KimiK3ForCausalLMVllm`, `KimiK3ForCausalLM`, `_ATOM_MODEL_CLASSES`, `hf_to_atom_mapper`, `get_placeholder_str`, `get_mamba_state_*` used consistently across Tasks 2-3 and match the existing code in `atom/plugin/vllm/models/kimi_k3.py` and `model_wrapper.py`. ✓ diff --git a/docs/superpowers/specs/2026-08-04-k3-vision-vllm-plugin-design.md b/docs/superpowers/specs/2026-08-04-k3-vision-vllm-plugin-design.md new file mode 100644 index 0000000000..4fe450050c --- /dev/null +++ b/docs/superpowers/specs/2026-08-04-k3-vision-vllm-plugin-design.md @@ -0,0 +1,179 @@ +# Design: Kimi-K3 image support in the ATOM vLLM plugin + +**Date:** 2026-08-04 +**Branch:** guanbao/k3_rmsnorm_quant_fusion +**Status:** Approved design — pending implementation plan + +## Goal + +Add vision (image) support to Kimi-K3 when running in **ATOM vLLM plugin mode**, by +mirroring how Kimi-K2.5 achieves it. K3 currently runs text-only in the plugin even +though the checkpoint is multimodal. + +## Scope + +- **In scope:** Image input, plugin mode only, always-multimodal (vision tower always + loads, K2.5 pattern). +- **Out of scope:** Video (no upstream K3 reference; the K3 checkpoint has no video + tokens). Native ATOM (non-plugin) mode stays text-only. +- **Hard constraint:** Never modify native K3 files `atom/models/kimi_k3.py` or + `atom/models/kimi_k3_dspark.py`. They may only be imported read-only. + +## Background / current state + +- The K3 checkpoint (`/workspace/shared/data/amd_int/models/Kimi-K3`) is already + multimodal: architecture `KimiK3ForConditionalGeneration`, `model_type: kimi_k3`, + with `vision_config` + `text_config` + `image_placeholder` + + `media_placeholder_token_id`. The **same** architecture name is used for text and + vision checkpoints (as with K2.5). +- vLLM upstream already ships the full K3 **image** stack: + - `vllm/models/kimi_k3/amd/model.py::KimiK3ForConditionalGeneration` — reuses K2.5's + `MoonViT3dPretrainedModel` + `KimiK25MultiModalProjector`; combines + `SupportsMultiModal` + `IsHybrid` + `HasInnerState` + `SupportsPP/Quant/Eagle3`. + - `vllm/models/kimi_k3/common/mm_preprocess.py` — `KimiK3ProcessingInfo`, + `KimiK3MultiModalProcessor`, `KimiK3DummyInputsBuilder` (image-only). Placeholder + token `<|kimi_image_placeholder|>`. +- ATOM forces K3 down a text-only path today: + - `kimi_k3` is absent from `_PLUGIN_SUPPORTED_MULTIMODAL_MODELS` in `atom/config.py`, + so the plugin strips `vision_config`. + - The plugin outer class `KimiK3ForCausalLMVllm` inherits `ATOMMoEForCausalLM, + IsHybrid` (text-only) rather than the multimodal `ATOMForConditionalGeneration`. +- The reference implementation for the plugin wiring is + `atom/plugin/vllm/models/kimi_k25.py` (inner `KimiK25ForConditionalGeneration_` + + outer `KimiK25ForConditionalGeneration`). + +**Novel wrinkle vs K2.5:** K3 is a *hybrid* model (KDA recurrent + MLA). Its outer +class must be multimodal **and** retain the hybrid-state plumbing (`IsHybrid` + +`get_mamba_state_*`). K2.5 is pure-attention and did not need this. vLLM's own upstream +K3 class proves the two interface sets compose. + +## Architecture + +Three layers, mirroring K2.5, with hybrid-state additions on the outer class: + +``` +register.py: "KimiK3ForConditionalGeneration" ─► KimiK3ForCausalLMVllm (OUTER, modified) + = ATOMForConditionalGeneration + IsHybrid + · get_mamba_state_* (kept) + · get_placeholder_str (new) + · load_weights → self.model (new) + │ .model = +model_wrapper._ATOM_MODEL_ARCH_TO_QUALNAME ───────► KimiK3ForConditionalGeneration_ (INNER, new) + = subclass of vLLM KimiK3ForConditionalGeneration + · vision_tower = MoonViT3dPretrainedModel (upstream) + · mm_projector = KimiK25MultiModalProjector (upstream) + · embed_multimodal / media parse (inherited) + · @register_processor(KimiK3MultiModalProcessor…) + │ .language_model = + KimiK3ForCausalLM (ATOM PLUGIN subclass, KDA-monkeypatched) + = existing optimized text path (dual-stream / KDA / fusion) +``` + +Key insight: vLLM inspects the **outer** class for interfaces, so `IsHybrid` and +`SupportsMultiModal` (via `ATOMForConditionalGeneration`) both live there. The +**inner** class owns the vision tower and delegates text to ATOM's KDA language model. +This keeps ATOM's entire K3 text-path optimization intact while the image stack is +reused verbatim from upstream. + +## Detailed changes + +All edits are confined to three files. **No native K3 file is touched.** + +### 1. `atom/config.py` +Add `"kimi_k3"` to `_PLUGIN_SUPPORTED_MULTIMODAL_MODELS` so the plugin passes the full +`hf_config` (including `vision_config`) through instead of stripping to `text_config`. + +### 2. `atom/plugin/vllm/models/kimi_k3.py` + +**New inner class** `KimiK3ForConditionalGeneration_`: +- Subclasses vLLM's `KimiK3ForConditionalGeneration` (from `vllm.models.kimi_k3`, + resolves to the AMD branch on ROCm) to inherit `embed_multimodal`, + `_parse_and_validate_media_input`, `_process_media_input`, `_mark_tower_model`, + `_mark_language_model`. +- Decorated with + `@MULTIMODAL_REGISTRY.register_processor(KimiK3MultiModalProcessor, + info=KimiK3ProcessingInfo, dummy_inputs=KimiK3DummyInputsBuilder)`. +- `__init__(self, atom_config, prefix="model")`: + - `nn.Module.__init__(self)`. + - Read `hf_config.vision_config` / `hf_config.text_config` directly — reuse the + checkpoint's HF `KimiK3Config`; do **not** create a new config wrapper (decided). + - Build `vision_tower` (`MoonViT3dPretrainedModel`) + `mm_projector` + (`KimiK25MultiModalProjector`) under `self._mark_tower_model(vllm_config, "image")`, + handling the same quant-exclude logic as K2.5's `_maybe_ignore_quant_config`. + - Build the ATOM **plugin** `KimiK3ForCausalLM` as `self.language_model` under + `self._mark_language_model(vllm_config)` (this triggers the existing KDA → + `KimiKDAAttentionVllm` self-registration in the static forward context). + - `self.make_empty_intermediate_tensors = + self.language_model.make_empty_intermediate_tensors`. +- `load_weights`: `load_model_in_plugin_mode(model=self, config=self.atom_config, + prefix="model.", weights_mapper=self.hf_to_atom_mapper)` with a `WeightsMapper` + covering vision prefixes, `mm_projector.proj.{0,2}` → `linear_{1,2}`, and + `language_model.` remaps (patterned on K2.5's mapper, adjusted to K3's checkpoint key + layout — verified during implementation against the on-disk weight names). +- `get_expert_mapping` delegating to `self.language_model`. + +**Add `embed_input_ids`** to the existing plugin `KimiK3ForCausalLM` subclass (NOT the +native class) — returns `self.get_input_embeddings(input_ids)`. Required by vLLM's +`SupportsMultiModal.get_language_model` discovery (K2.5 needed the same). + +**Modify outer** `KimiK3ForCausalLMVllm`: +- Change base from `(ATOMMoEForCausalLM, IsHybrid)` to + `(ATOMForConditionalGeneration, IsHybrid)`. +- Keep the three existing `get_mamba_state_*` classmethods unchanged. +- Add `get_placeholder_str(cls, modality, i)` → `"<|kimi_image_placeholder|>"` for + `image`, raise for others. +- Add `load_weights` delegating to `self.model.load_weights`. + +### 3. `atom/plugin/vllm/model_wrapper.py` +Repoint `_ATOM_MODEL_ARCH_TO_QUALNAME["KimiK3ForConditionalGeneration"]` from +`atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLM` to +`atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGeneration_`. + +### Unchanged +- `atom/plugin/vllm/register.py` — already maps the arch to the outer + `KimiK3ForCausalLMVllm`; keep as-is. +- `atom/model_engine/model_runner.py` — native mode stays text-only (out of scope). +- `atom/models/kimi_k3.py`, `atom/models/kimi_k3_dspark.py` — never touched. + +## Data flow (image request) + +1. `KimiK3MultiModalProcessor` (upstream) converts image + `<|kimi_image_placeholder|>` + prompt into `pixel_values` + `grid_thws`, expanding the placeholder into + `media_begin … media_end` pad tokens sized to the image resolution. +2. vLLM calls `embed_multimodal` (inherited) → `vision_tower` + `mm_projector` → image + feature tensors. +3. `embed_input_ids` (vLLM `SupportsMultiModal` default merge) embeds text tokens and + scatters image features at `media_placeholder_token_id` positions. +4. Merged embeddings enter ATOM's `KimiK3ForCausalLM.forward` (KDA + MLA + MoE, + dual-stream) exactly as today; hybrid recurrent state via the outer class's + `IsHybrid` / `get_mamba_state_*`. +5. Text-only request → `embed_multimodal` returns `None`; path is identical to the + current text-only K3. + +## Risks & mitigations + +- **Multimodal + hybrid interface composition** (main novelty). Mitigated: vLLM's own + upstream K3 class combines the same interface sets, and we keep the hybrid interfaces + on the outer class (which vLLM inspects) exactly as the working text path does. +- **Inherited `HasInnerState` / `mamba_cache` methods** on the inner class reference a + `mamba_cache` ATOM does not use. Mitigated: vLLM inspects the *outer* class (which + does not declare `HasInnerState`), so these stay dormant. If they are ever invoked, + fall back to a standalone inner class (no upstream subclassing) for the vision path. +- **Weight-name mapping** — vision/projector/language_model prefixes must all resolve. + Verified during implementation against on-disk weight names and by the smoke test's + load-completeness check (no unexpected/missing weights). +- **VRAM / startup** — the ViT now always loads (accepted). May shift memory headroom + for text-only benchmarks on this branch; note in the PR. + +## Validation + +- **Smoke test:** start the server with the K3 checkpoint in plugin mode, send an + image + text prompt, confirm a coherent description and clean weight loading + (`rocm-smi` VRAM > 0, no unmapped weights). Use `/run-atom-workload`. +- **Text accuracy unchanged:** run GSM8K (or the usual text eval) and confirm parity + with the current text-only K3 — proves the KDA / dual-stream path did not regress. + +## Open questions + +None. Config wrapper skipped (reuse HF `KimiK3Config`); native mode out of scope; video +out of scope. diff --git a/recipes/atom_vllm/KVV.md b/recipes/atom_vllm/KVV.md new file mode 100644 index 0000000000..027adca17a --- /dev/null +++ b/recipes/atom_vllm/KVV.md @@ -0,0 +1,105 @@ +# KVV Run — Commands + +Env: local vLLM for Kimi-K3 at `http://localhost:8000/v1`. +Two Python envs: `/opt/venv` (pytest + BEAM), project `.venv` (inspect-ai benchmarks). + +## Clone KVV + +```bash +git clone git@github.com:MoonshotAI/Kimi-Vendor-Verifier.git +``` + +## Server +Check server launch commands in recipe. + +## 1. tests/params/ + +```bash +cd ~/Kimi-Vendor-Verifier +/opt/venv/bin/pytest tests/params/ \ + --base-url http://localhost:8000/v1 --api-key EMPTY \ + --smoke-model "/workspace/shared/data/amd_int/models/Kimi-K3" \ + --think-mode opensource +``` + +## 2. tests/k3_features/ + +```bash +cd ~/Kimi-Vendor-Verifier +/opt/venv/bin/pytest tests/k3_features/ \ + --base-url http://localhost:8000/v1 --api-key EMPTY \ + --smoke-model "/workspace/shared/data/amd_int/models/Kimi-K3" \ + --think-mode opensource +``` + +## 3. tests/prompt_tokens/ + +```bash +cd ~/Kimi-Vendor-Verifier +/opt/venv/bin/pytest tests/prompt_tokens/ \ + --base-url http://localhost:8000/v1 --api-key EMPTY \ + --smoke-model "/workspace/shared/data/amd_int/models/Kimi-K3" \ + --think-mode opensource +``` + +## 4. tests/tool_call_json_schema/ + +```bash +cd ~/Kimi-Vendor-Verifier +/opt/venv/bin/pytest tests/tool_call_json_schema/ \ + --base-url http://localhost:8000/v1 --api-key EMPTY \ + --smoke-model "/workspace/shared/data/amd_int/models/Kimi-K3" \ + --think-mode opensource +``` + +## 5. OCRBench + +```bash +cd ~/Kimi-Vendor-Verifier +KIMI_BASE_URL=http://localhost:8000/v1 KIMI_API_KEY=EMPTY \ +.venv/bin/python eval.py ocrbench \ + --model "opensource//workspace/shared/data/amd_int/models/Kimi-K3" \ + --max-tokens 16384 --thinking --think-mode opensource --stream \ + --max-connections 50 --temperature 1.0 --top-p 0.95 --thinking-effort high +``` + +## 6. MMMU Pro Vision + +```bash +cd ~/Kimi-Vendor-Verifier +KIMI_BASE_URL=http://localhost:8000/v1 KIMI_API_KEY=EMPTY \ +.venv/bin/python eval.py mmmu \ + --model "opensource//workspace/shared/data/amd_int/models/Kimi-K3" \ + --max-tokens 98304 --thinking --think-mode opensource --stream \ + --max-connections 50 --temperature 1.0 --top-p 0.95 --thinking-effort high +``` + +BEAM and Judge are low priority tests. Let's ignore them for now. + +## 7. BEAM 1M (smoke) + +Generate: + +```bash +cd ~/Kimi-Vendor-Verifier/beam +/opt/venv/bin/python beam_generate.py \ + --model /workspace/shared/data/amd_int/models/Kimi-K3 \ + --base-url http://localhost:8000/v1 --api-key EMPTY \ + --temperature 1.0 --top-p 0.95 --max-tokens 32768 \ + --thinking-json '{"chat_template_kwargs": {"thinking": true, "preserve_thinking": true, "thinking_effort": "high"}}' \ + --concurrency 4 --max-retries 3 --limit 8 \ + --output answers_smoke.jsonl +``` + +Judge (self-judge via same local server): + +```bash +cd ~/Kimi-Vendor-Verifier/beam +/opt/venv/bin/python beam_judge.py \ + --answers answers_smoke.jsonl \ + --judge-model /workspace/shared/data/amd_int/models/Kimi-K3 \ + --judge-base-url http://localhost:8000/v1 --judge-api-key EMPTY \ + --judge-max-tokens 16384 --judge-temperature 0.3 \ + --concurrency 16 \ + --output scores_smoke.jsonl +``` diff --git a/tests/plugin/test_vllm_kimi_k3.py b/tests/plugin/test_vllm_kimi_k3.py index b151f98119..395d67827c 100644 --- a/tests/plugin/test_vllm_kimi_k3.py +++ b/tests/plugin/test_vllm_kimi_k3.py @@ -27,7 +27,8 @@ def test_kimi_k3_plugin_registries_are_synchronized(): == "atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLMVllm" ) assert ( - _ATOM_MODEL_CLASSES[arch] == "atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLM" + _ATOM_MODEL_CLASSES[arch] + == "atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGeneration_" ) @@ -108,6 +109,96 @@ def test_kimi_k3_temporal_state_uses_fp32(): """) +def test_kimi_k3_outer_is_multimodal_and_hybrid(): + _run_without_test_stubs(""" + from vllm.model_executor.models.interfaces import IsHybrid + + from atom.plugin.vllm.model_wrapper import ATOMForConditionalGeneration + from atom.plugin.vllm.models.kimi_k3 import KimiK3ForCausalLMVllm + + mro = KimiK3ForCausalLMVllm.__mro__ + # Multimodal via ATOMForConditionalGeneration; hybrid state retained. + assert ATOMForConditionalGeneration in mro + assert IsHybrid in mro + + # Image placeholder matches the checkpoint token. + assert ( + KimiK3ForCausalLMVllm.get_placeholder_str("image", 0) + == "<|kimi_image_placeholder|>" + ) + + # Hybrid mamba-state entry points survive the base-class change. + for name in ( + "get_mamba_state_dtype_from_config", + "get_mamba_state_shape_from_config", + "get_mamba_state_copy_func", + ): + assert hasattr(KimiK3ForCausalLMVllm, name) + """) + + +def test_kimi_k3_is_plugin_supported_multimodal(): + from atom.config import ( + _MULTIMODAL_MODEL_TYPES, + _PLUGIN_SUPPORTED_MULTIMODAL_MODELS, + ) + + # kimi_k3 must be a known multimodal type AND opted into plugin-mode + # pass-through, so get_hf_config keeps vision_config instead of stripping + # to text_config. + assert "kimi_k3" in _MULTIMODAL_MODEL_TYPES + assert "kimi_k3" in _PLUGIN_SUPPORTED_MULTIMODAL_MODELS + + +def test_kimi_k3_inner_conditional_generation_class(): + _run_without_test_stubs(""" + from vllm.models.kimi_k3 import ( + KimiK3ForConditionalGeneration as vLLMKimiK3, + ) + from atom.plugin.vllm.models.kimi_k3 import ( + KimiK3ForCausalLM, + KimiK3ForConditionalGeneration_, + ) + + # Inner class subclasses the upstream multimodal model so it inherits + # embed_multimodal / media parsing / vision-tower forward. + assert issubclass(KimiK3ForConditionalGeneration_, vLLMKimiK3) + + # ATOM plugin loading + expert routing entry points exist. + assert hasattr(KimiK3ForConditionalGeneration_, "load_weights") + assert hasattr(KimiK3ForConditionalGeneration_, "get_expert_mapping") + + # Weight mapper collapses the double language_model nesting and renames + # the projector layers. + mapper = KimiK3ForConditionalGeneration_.hf_to_atom_mapper + assert mapper.orig_to_new_prefix["language_model."] == ( + "language_model.language_model." + ) + assert mapper.orig_to_new_prefix["mm_projector.proj.0"] == ( + "mm_projector.linear_1" + ) + assert mapper.orig_to_new_prefix["mm_projector.proj.2"] == ( + "mm_projector.linear_2" + ) + + # Language-model wrapper exposes embed_input_ids for vLLM multimodal + # discovery. + assert hasattr(KimiK3ForCausalLM, "embed_input_ids") + + # Quant-remap attributes must be present (ATOMModelBase reads them off + # the inner class); missing them silently corrupts a quantized ckpt. + packed = KimiK3ForConditionalGeneration_.packed_modules_mapping + assert packed, "packed_modules_mapping must be non-empty" + + excl = KimiK3ForConditionalGeneration_.quant_exclude_name_mapping + assert excl, "quant_exclude_name_mapping must be non-empty" + # Values carry the doubled language_model. prefix for the extra nesting + # (inner.language_model = ATOM KimiK3ForCausalLM -> KimiLinearForCausalLM). + for value in excl.values(): + assert value.startswith("language_model.language_model."), value + """) + + def test_kimi_k3_post_load_accepts_vllm_dtype(): _run_without_test_stubs(""" from inspect import Parameter, signature From 6694f1467139a683fbd0f9eacbfc882e0d727831 Mon Sep 17 00:00:00 2001 From: Satya K Jandhyala Date: Mon, 17 Aug 2026 10:32:28 -0500 Subject: [PATCH 2/6] Added changes required due to other changes on main. --- atom/models/kimi_k3_dspark.py | 15 ++++++++++----- atom/plugin/vllm/kda_backend.py | 27 +++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/atom/models/kimi_k3_dspark.py b/atom/models/kimi_k3_dspark.py index f29c3a57be..227d163435 100644 --- a/atom/models/kimi_k3_dspark.py +++ b/atom/models/kimi_k3_dspark.py @@ -545,19 +545,24 @@ def __init__( prefix=f"{prefix}.mlp.gate_up_proj", ) - def write_context_kv(self, ctx_hidden, positions) -> None: + def write_context_kv(self, ctx_hidden, positions, slot_mapping=None) -> None: """Populate this layer's context rows. The context source does NOT go through ``input_layernorm``: the reference feeds every layer the same ``context_norm(context_proj(aux))`` tensor straight into the KV projection, while ``input_layernorm`` applies only to the residual stream carrying the draft block. + + ``slot_mapping`` is optional: the native path reads it from + ``forward_context``, while the vLLM plugin passes per-layer slots + explicitly (groups may differ across layers). """ - from atom.utils.forward_context import get_forward_context + if slot_mapping is None: + from atom.utils.forward_context import get_forward_context - slot_mapping = get_forward_context().attn_metadata.slot_mapping[ - : ctx_hidden.shape[0] - ] + slot_mapping = get_forward_context().attn_metadata.slot_mapping[ + : ctx_hidden.shape[0] + ] self.self_attn.write_context_kv(ctx_hidden, positions, slot_mapping) def forward( diff --git a/atom/plugin/vllm/kda_backend.py b/atom/plugin/vllm/kda_backend.py index 4f02e4fcf5..7de35f66b1 100644 --- a/atom/plugin/vllm/kda_backend.py +++ b/atom/plugin/vllm/kda_backend.py @@ -57,9 +57,34 @@ def build( # type: ignore[override] num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu, fast_build=fast_build, ) + self._stage_packed_decode_query_start_loc(common_attn_metadata, metadata) self._adapt_full_graph_decode_metadata(common_attn_metadata, metadata) return metadata + def _stage_packed_decode_query_start_loc( + self, + common_attn_metadata: CommonAttentionMetadata, + metadata: KimiK3KDAMetadata, + ) -> None: + if ( + metadata.num_decodes <= 0 + or metadata.non_spec_query_start_loc is not None + ): + return + + batch_size = int(common_attn_metadata.num_reqs) + query_start_loc_buf = self.non_spec_query_start_loc[: batch_size + 1] + torch.arange( + metadata.num_decodes + 1, + dtype=torch.int32, + device=query_start_loc_buf.device, + out=query_start_loc_buf[: metadata.num_decodes + 1], + ) + query_start_loc_buf[metadata.num_decodes + 1 :].fill_( + metadata.num_decodes + ) + metadata.non_spec_query_start_loc = query_start_loc_buf + def _adapt_full_graph_decode_metadata( self, common_attn_metadata: CommonAttentionMetadata, @@ -81,8 +106,6 @@ def _adapt_full_graph_decode_metadata( query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] real_decode_mask_cpu = query_lens_cpu > 0 real_num_decodes = int(real_decode_mask_cpu.sum().item()) - if real_num_decodes == metadata.num_decodes: - return batch_size = int(common_attn_metadata.num_reqs) if batch_size > self.decode_cudagraph_max_bs: From 03eb78cb1a415482fa19813bcd02ca4d37e2c978 Mon Sep 17 00:00:00 2001 From: Satya K Jandhyala Date: Mon, 17 Aug 2026 20:21:46 -0700 Subject: [PATCH 3/6] Fix Black and Ruff warnings. --- atom/model_ops/fla_ops/chunk.py | 2 +- atom/model_ops/fla_ops/chunk_vk.py | 2 +- atom/model_ops/fla_ops/fused_sigmoid_gating.py | 2 +- atom/model_ops/fla_ops/layernorm_guard.py | 2 +- atom/plugin/vllm/kda_backend.py | 9 ++------- atom/rollout/async_engine.py | 2 +- 6 files changed, 7 insertions(+), 12 deletions(-) diff --git a/atom/model_ops/fla_ops/chunk.py b/atom/model_ops/fla_ops/chunk.py index 6edbb0dab5..d0585a552a 100644 --- a/atom/model_ops/fla_ops/chunk.py +++ b/atom/model_ops/fla_ops/chunk.py @@ -210,7 +210,7 @@ def chunk_gated_delta_rule( v: torch.Tensor, g: torch.Tensor, beta: torch.Tensor, - scale: float = None, + scale: float | None = None, initial_state: torch.Tensor = None, output_final_state: bool = False, cu_seqlens: torch.LongTensor | None = None, diff --git a/atom/model_ops/fla_ops/chunk_vk.py b/atom/model_ops/fla_ops/chunk_vk.py index 2bb9239b1b..bf2a2767b9 100644 --- a/atom/model_ops/fla_ops/chunk_vk.py +++ b/atom/model_ops/fla_ops/chunk_vk.py @@ -137,7 +137,7 @@ def chunk_gated_delta_rule_vk( v: torch.Tensor, g: torch.Tensor, beta: torch.Tensor, - scale: float = None, + scale: float | None = None, initial_state: torch.Tensor = None, output_final_state: bool = False, cu_seqlens: torch.Tensor | None = None, diff --git a/atom/model_ops/fla_ops/fused_sigmoid_gating.py b/atom/model_ops/fla_ops/fused_sigmoid_gating.py index 526c9dd0dd..11cc27e7a9 100644 --- a/atom/model_ops/fla_ops/fused_sigmoid_gating.py +++ b/atom/model_ops/fla_ops/fused_sigmoid_gating.py @@ -204,7 +204,7 @@ def fused_sigmoid_gating_delta_rule_update( o: torch.Tensor | None = None, beta: float = 1.0, threshold: float = 20.0, - scale: float = None, + scale: float | None = None, initial_state: torch.Tensor = None, inplace_final_state: bool = True, cu_seqlens: torch.LongTensor | None = None, diff --git a/atom/model_ops/fla_ops/layernorm_guard.py b/atom/model_ops/fla_ops/layernorm_guard.py index fcb991c595..e87327a784 100644 --- a/atom/model_ops/fla_ops/layernorm_guard.py +++ b/atom/model_ops/fla_ops/layernorm_guard.py @@ -198,7 +198,7 @@ def layer_norm_fwd( eps: float, z: torch.Tensor = None, out: torch.Tensor = None, - group_size: int = None, + group_size: int | None = None, norm_before_gate: bool = True, is_rms_norm: bool = False, ): diff --git a/atom/plugin/vllm/kda_backend.py b/atom/plugin/vllm/kda_backend.py index 7de35f66b1..00024efec5 100644 --- a/atom/plugin/vllm/kda_backend.py +++ b/atom/plugin/vllm/kda_backend.py @@ -66,10 +66,7 @@ def _stage_packed_decode_query_start_loc( common_attn_metadata: CommonAttentionMetadata, metadata: KimiK3KDAMetadata, ) -> None: - if ( - metadata.num_decodes <= 0 - or metadata.non_spec_query_start_loc is not None - ): + if metadata.num_decodes <= 0 or metadata.non_spec_query_start_loc is not None: return batch_size = int(common_attn_metadata.num_reqs) @@ -80,9 +77,7 @@ def _stage_packed_decode_query_start_loc( device=query_start_loc_buf.device, out=query_start_loc_buf[: metadata.num_decodes + 1], ) - query_start_loc_buf[metadata.num_decodes + 1 :].fill_( - metadata.num_decodes - ) + query_start_loc_buf[metadata.num_decodes + 1 :].fill_(metadata.num_decodes) metadata.non_spec_query_start_loc = query_start_loc_buf def _adapt_full_graph_decode_metadata( diff --git a/atom/rollout/async_engine.py b/atom/rollout/async_engine.py index 3a95487250..2d4a46dfa3 100644 --- a/atom/rollout/async_engine.py +++ b/atom/rollout/async_engine.py @@ -54,7 +54,7 @@ def sleep(self, level: int = 1): logger.info("AsyncLLMEngine sleep: completed") - def wake_up(self, tags: list[str] = None): + def wake_up(self, tags: list[str] | None = None): """Restore GPU resources. Args: From 51a42a7a2980c11761f29fb635d4dfcece52924e Mon Sep 17 00:00:00 2001 From: Satya K Jandhyala Date: Wed, 19 Aug 2026 08:45:55 -0700 Subject: [PATCH 4/6] fix(k3): restore vision imports dropped during rebase The multimodal plugin classes in kimi_k3.py referenced symbols without importing them, which made Ruff fail in pre-checks CI. Co-authored-by: Cursor --- atom/plugin/vllm/models/kimi_k3.py | 33 +++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/atom/plugin/vllm/models/kimi_k3.py b/atom/plugin/vllm/models/kimi_k3.py index 58926cf5ca..81be322a69 100644 --- a/atom/plugin/vllm/models/kimi_k3.py +++ b/atom/plugin/vllm/models/kimi_k3.py @@ -1,8 +1,10 @@ from dataclasses import replace from types import SimpleNamespace +from typing import Any, ClassVar import torch from aiter.dist.parallel_state import get_pp_group +from torch import nn from vllm.config import VllmConfig, get_current_vllm_config from vllm.forward_context import get_forward_context as get_vllm_forward_context from vllm.model_executor.layers.mamba.abstract import MambaBase @@ -13,10 +15,26 @@ MambaStateShapeCalculator, ) from vllm.model_executor.models.interfaces import IsHybrid +from vllm.model_executor.models.kimi_k25_vit import ( + KimiK25MultiModalProjector, + MoonViT3dPretrainedModel, +) +from vllm.model_executor.models.vision import is_vit_use_data_parallel +from vllm.models.kimi_k3 import ( + KimiK3ForConditionalGeneration as vLLMKimiK3, +) +from vllm.models.kimi_k3.common.mm_preprocess import ( + KimiK3DummyInputsBuilder, + KimiK3MultiModalProcessor, + KimiK3ProcessingInfo, +) from vllm.models.kimi_k3.nvidia.kda_metadata import KimiK3KDAMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.v1.attention.backend import AttentionBackend from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum +from atom.config import Config, QuantizationConfig +from atom.model_loader.loader import WeightsMapper, load_model_in_plugin_mode from atom.models import kimi_k3 as kimi_k3_base from atom.models.kimi_k3 import ( KimiDecoderLayer, @@ -26,10 +44,16 @@ from atom.models.kimi_k3 import ( KimiK3ForCausalLM as KimiK3ForCausalLMBase, ) -from atom.models.utils import IntermediateTensors +from atom.models.utils import IntermediateTensors, maybe_prefix from atom.plugin.vllm.kda_backend import AtomKimiK3KDAAttentionBackend +from atom.plugin.vllm.model_wrapper import ATOMForConditionalGeneration from atom.utils.forward_context import get_forward_context as get_atom_forward_context +_KIMI_K3_QUANT_EXCLUDE_NAME_MAPPING: dict[str, str] = { + "language_model.model.": "language_model.language_model.model.", + "language_model.lm_head": "language_model.language_model.lm_head", +} + def _adapt_kda_metadata_for_atom(kda_metadata: KimiK3KDAMetadata) -> None: """Provide the state-read index field expected by native ATOM KDA.""" @@ -517,10 +541,9 @@ class KimiK3ForConditionalGeneration_(vLLMKimiK3): } ) packed_modules_mapping = KimiK3ForCausalLMBase.packed_modules_mapping - quant_exclude_name_mapping: ClassVar[dict[str, str]] = { - "language_model.model.": "language_model.language_model.model.", - "language_model.lm_head": "language_model.language_model.lm_head", - } + quant_exclude_name_mapping: ClassVar[dict[str, str]] = ( + _KIMI_K3_QUANT_EXCLUDE_NAME_MAPPING + ) def __init__(self, atom_config: Config, prefix: str = "model"): nn.Module.__init__(self) From 845dcf236a35c210d9046f2d81df6618beee3b47 Mon Sep 17 00:00:00 2001 From: Satya K Jandhyala Date: Sun, 6 Sep 2026 09:32:24 -0700 Subject: [PATCH 5/6] Removed unnecessary files. --- .../plans/2026-08-04-k3-vision-vllm-plugin.md | 571 ------------------ ...2026-08-04-k3-vision-vllm-plugin-design.md | 179 ------ 2 files changed, 750 deletions(-) delete mode 100644 docs/superpowers/plans/2026-08-04-k3-vision-vllm-plugin.md delete mode 100644 docs/superpowers/specs/2026-08-04-k3-vision-vllm-plugin-design.md diff --git a/docs/superpowers/plans/2026-08-04-k3-vision-vllm-plugin.md b/docs/superpowers/plans/2026-08-04-k3-vision-vllm-plugin.md deleted file mode 100644 index c15d7e6671..0000000000 --- a/docs/superpowers/plans/2026-08-04-k3-vision-vllm-plugin.md +++ /dev/null @@ -1,571 +0,0 @@ -# Kimi-K3 image support (vLLM plugin) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Enable image (vision) input for Kimi-K3 in ATOM vLLM plugin mode by mirroring K2.5's wiring, reusing vLLM's upstream K3 vision stack while keeping ATOM's KDA-optimized hybrid language model. - -**Architecture:** Three-layer plugin structure. The **outer** class `KimiK3ForCausalLMVllm` (what vLLM's registry instantiates) becomes multimodal by switching its base to `ATOMForConditionalGeneration` while retaining `IsHybrid` + the `get_mamba_state_*` classmethods. A **new inner** class `KimiK3ForConditionalGeneration_` (built by ATOM as `.model`) subclasses vLLM's upstream `KimiK3ForConditionalGeneration` to inherit the vision path (`embed_multimodal`, media parsing, `MoonViT3dPretrainedModel`, `KimiK25MultiModalProjector`), but instantiates ATOM's plugin `KimiK3ForCausalLM` as its `language_model` and loads weights via ATOM's plugin loader. A config gate lets the plugin keep `vision_config` instead of stripping it. - -**Tech Stack:** Python, PyTorch, vLLM (editable checkout at `/shared/amdgpu/home/gyu_qle/vllm`), AITER, ROCm. Tests: pytest (no GPU; real-import checks run via subprocess). - -## Global Constraints - -- **NEVER modify native K3 files** `atom/models/kimi_k3.py` or `atom/models/kimi_k3_dspark.py`. Import them read-only only. All new `embed_input_ids` etc. go on the **plugin** subclass in `atom/plugin/vllm/models/kimi_k3.py`. -- **Plugin mode only.** Do not touch native-mode registration `atom/model_engine/model_runner.py` (K3 stays text-only there). -- **Image only.** No video code paths. -- **Reuse the checkpoint's HF `KimiK3Config` directly.** Do NOT create `atom/model_config/kimi_k3.py`. -- **Do not modify `@support_torch_compile`-decorated model files.** -- Format/lint before every commit: `black . && ruff check .` (CI enforced; ruff only fails on PR-diff lines). -- Reference checkpoint (multimodal): `/workspace/shared/data/amd_int/models/Kimi-K3` — arch `KimiK3ForConditionalGeneration`, `model_type: kimi_k3`, `media_placeholder_token_id: 163605`, `image_placeholder: <|kimi_image_placeholder|>`. Weight prefixes: `language_model.*`, `vision_tower.*`, `mm_projector.{proj.0,proj.2,post_norm}`. -- Before restarting any server: `rm -rf /root/.cache/atom/*` and `export AITER_LOG_LEVEL=WARNING`. - ---- - -### Task 1: Config gate — let plugin mode keep K3's vision_config - -**Files:** -- Modify: `atom/config.py:610-614` (`_PLUGIN_SUPPORTED_MULTIMODAL_MODELS`) -- Test: `tests/plugin/test_vllm_kimi_k3.py` - -**Interfaces:** -- Consumes: nothing. -- Produces: `"kimi_k3"` present in `atom.config._PLUGIN_SUPPORTED_MULTIMODAL_MODELS`. This makes `get_hf_config()` skip the text-config stripping for `model_type == "kimi_k3"` under `is_vllm()`, so `hf_config` retains `.vision_config` + `.text_config`. - -**Context:** In `atom/config.py::get_hf_config`, when `is_vllm()` is true, any `model_type` listed in `_PLUGIN_SUPPORTED_MULTIMODAL_MODELS` is filtered OUT of the `multimodal_model_types` stripping map (lines 630-636), so the full multimodal config is preserved. `kimi_k3` is already in `_MULTIMODAL_MODEL_TYPES` (line 602) but missing from the plugin set (lines 610-614). - -- [ ] **Step 1: Write the failing test** - -Add to `tests/plugin/test_vllm_kimi_k3.py`: - -```python -def test_kimi_k3_is_plugin_supported_multimodal(): - from atom.config import ( - _MULTIMODAL_MODEL_TYPES, - _PLUGIN_SUPPORTED_MULTIMODAL_MODELS, - ) - - # kimi_k3 must be a known multimodal type AND opted into plugin-mode - # pass-through, so get_hf_config keeps vision_config instead of stripping - # to text_config. - assert "kimi_k3" in _MULTIMODAL_MODEL_TYPES - assert "kimi_k3" in _PLUGIN_SUPPORTED_MULTIMODAL_MODELS -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py::test_kimi_k3_is_plugin_supported_multimodal -v` -Expected: FAIL — assertion error (`"kimi_k3"` not in `_PLUGIN_SUPPORTED_MULTIMODAL_MODELS`). - -- [ ] **Step 3: Add kimi_k3 to the plugin-supported set** - -In `atom/config.py`, change: - -```python -_PLUGIN_SUPPORTED_MULTIMODAL_MODELS: set[str] = { - "kimi_k25", - "qwen3_5", - "qwen3_5_moe", -} -``` - -to: - -```python -_PLUGIN_SUPPORTED_MULTIMODAL_MODELS: set[str] = { - "kimi_k25", - "kimi_k3", - "qwen3_5", - "qwen3_5_moe", -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py::test_kimi_k3_is_plugin_supported_multimodal -v` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -black atom/config.py tests/plugin/test_vllm_kimi_k3.py && ruff check atom/config.py tests/plugin/test_vllm_kimi_k3.py -git add atom/config.py tests/plugin/test_vllm_kimi_k3.py -git commit -m "feat(k3): keep vision_config in plugin mode for kimi_k3" -``` - ---- - -### Task 2: Add the multimodal inner class + language-model embed hook - -**Files:** -- Modify: `atom/plugin/vllm/models/kimi_k3.py` (add imports; add `embed_input_ids` to plugin `KimiK3ForCausalLM`; add new `KimiK3ForConditionalGeneration_`) -- Test: `tests/plugin/test_vllm_kimi_k3.py` - -**Interfaces:** -- Consumes: `atom.config._PLUGIN_SUPPORTED_MULTIMODAL_MODELS` containing `"kimi_k3"` (Task 1); vLLM upstream `vllm.models.kimi_k3.KimiK3ForConditionalGeneration`, `vllm.model_executor.models.kimi_k25_vit.{MoonViT3dPretrainedModel,KimiK25MultiModalProjector}`, `vllm.models.kimi_k3.common.mm_preprocess.{KimiK3MultiModalProcessor,KimiK3ProcessingInfo,KimiK3DummyInputsBuilder}`. -- Produces: - - `atom.plugin.vllm.models.kimi_k3.KimiK3ForConditionalGeneration_` — subclass of vLLM's `KimiK3ForConditionalGeneration`; `__init__(self, atom_config: Config, prefix: str = "model")`; attrs `vision_tower`, `mm_projector`, `language_model` (an ATOM plugin `KimiK3ForCausalLM`); classvar `hf_to_atom_mapper: WeightsMapper`; methods `load_weights(weights) -> set[str]`, `get_expert_mapping()`. - - `KimiK3ForCausalLM.embed_input_ids(self, input_ids) -> torch.Tensor` (plugin subclass only). - -**Context:** Mirror `atom/plugin/vllm/models/kimi_k25.py::KimiK25ForConditionalGeneration_` (lines 178-288). The vision-tower/projector construction must mirror `/shared/amdgpu/home/gyu_qle/vllm/vllm/models/kimi_k3/amd/model.py` lines 96-123 (only the `language_model` line differs — we use ATOM's, not `init_vllm_registered_model`). The weight mapper handles the double `language_model` nesting: the checkpoint stores text under `language_model.model.*`, but ATOM nests `inner.language_model` (an ATOM `KimiK3ForCausalLM`) `.language_model` (a `KimiLinearForCausalLM`) `.model.*`, so `language_model.` → `language_model.language_model.`. - -- [ ] **Step 1: Write the failing test** - -Add to `tests/plugin/test_vllm_kimi_k3.py`: - -```python -def test_kimi_k3_inner_conditional_generation_class(): - _run_without_test_stubs(""" - from vllm.models.kimi_k3 import ( - KimiK3ForConditionalGeneration as vLLMKimiK3, - ) - from atom.plugin.vllm.models.kimi_k3 import ( - KimiK3ForCausalLM, - KimiK3ForConditionalGeneration_, - ) - - # Inner class subclasses the upstream multimodal model so it inherits - # embed_multimodal / media parsing / vision-tower forward. - assert issubclass(KimiK3ForConditionalGeneration_, vLLMKimiK3) - - # ATOM plugin loading + expert routing entry points exist. - assert hasattr(KimiK3ForConditionalGeneration_, "load_weights") - assert hasattr(KimiK3ForConditionalGeneration_, "get_expert_mapping") - - # Weight mapper collapses the double language_model nesting and renames - # the projector layers. - mapper = KimiK3ForConditionalGeneration_.hf_to_atom_mapper - assert mapper.orig_to_new_prefix["language_model."] == ( - "language_model.language_model." - ) - assert mapper.orig_to_new_prefix["mm_projector.proj.0"] == ( - "mm_projector.linear_1" - ) - assert mapper.orig_to_new_prefix["mm_projector.proj.2"] == ( - "mm_projector.linear_2" - ) - - # Language-model wrapper exposes embed_input_ids for vLLM multimodal - # discovery. - assert hasattr(KimiK3ForCausalLM, "embed_input_ids") - """) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py::test_kimi_k3_inner_conditional_generation_class -v` -Expected: FAIL — `ImportError: cannot import name 'KimiK3ForConditionalGeneration_'`. - -- [ ] **Step 3: Add imports at the top of `atom/plugin/vllm/models/kimi_k3.py`** - -Add after the existing imports (keep the existing ones): - -```python -from torch import nn -from vllm.models.kimi_k3 import ( - KimiK3ForConditionalGeneration as vLLMKimiK3, -) -from vllm.models.kimi_k3.common.mm_preprocess import ( - KimiK3DummyInputsBuilder, - KimiK3MultiModalProcessor, - KimiK3ProcessingInfo, -) -from vllm.model_executor.models.kimi_k25_vit import ( - KimiK25MultiModalProjector, - MoonViT3dPretrainedModel, -) -from vllm.model_executor.models.vision import is_vit_use_data_parallel -from vllm.multimodal import MULTIMODAL_REGISTRY - -from atom.config import Config, QuantizationConfig -from atom.model_loader.loader import WeightsMapper, load_model_in_plugin_mode -from atom.models.utils import maybe_prefix -``` - -(Note: `ATOMForConditionalGeneration` is imported in Task 3, where the outer class first uses it — importing it here would trip ruff's unused-import check at the Task 2 lint gate.) - -- [ ] **Step 4: Add `embed_input_ids` to the plugin `KimiK3ForCausalLM`** - -The class currently is (lines ~161-168): - -```python -class KimiK3ForCausalLM(KimiK3ForCausalLMBase): - def __init__(self, *args, **kwargs): - original_kda_cls = kimi_k3_base.KimiKDAAttention - kimi_k3_base.KimiKDAAttention = KimiKDAAttentionVllm - try: - super().__init__(*args, **kwargs) - finally: - kimi_k3_base.KimiKDAAttention = original_kda_cls -``` - -Add the method (delegates to the native `get_input_embeddings`; required by vLLM `SupportsMultiModal.get_language_model` discovery, matching K2.5's `KimiK25ForCausalLM.embed_input_ids`): - -```python -class KimiK3ForCausalLM(KimiK3ForCausalLMBase): - def __init__(self, *args, **kwargs): - original_kda_cls = kimi_k3_base.KimiKDAAttention - kimi_k3_base.KimiKDAAttention = KimiKDAAttentionVllm - try: - super().__init__(*args, **kwargs) - finally: - kimi_k3_base.KimiKDAAttention = original_kda_cls - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - # Required by vLLM SupportsMultiModal.get_language_model discovery. - return self.get_input_embeddings(input_ids) -``` - -- [ ] **Step 5: Add the inner `KimiK3ForConditionalGeneration_` class** - -Add ABOVE the existing `KimiK3ForCausalLMVllm` class (so the outer can reference it is not required — registry wiring happens in Task 3). Mirror `vllm/models/kimi_k3/amd/model.py` lines 96-123 for the vision setup: - -```python -@MULTIMODAL_REGISTRY.register_processor( - KimiK3MultiModalProcessor, - info=KimiK3ProcessingInfo, - dummy_inputs=KimiK3DummyInputsBuilder, -) -class KimiK3ForConditionalGeneration_(vLLMKimiK3): - hf_to_atom_mapper = WeightsMapper( - orig_to_new_prefix={ - # ATOM nests language_model.language_model (ATOM KimiK3ForCausalLM - # wraps KimiLinearForCausalLM); checkpoint stores language_model.model. - "language_model.": "language_model.language_model.", - "mm_projector.proj.0": "mm_projector.linear_1", - "mm_projector.proj.2": "mm_projector.linear_2", - } - ) - - def __init__(self, atom_config: Config, prefix: str = "model"): - # Protocols have no __init__; skip vLLMKimiK3.__init__ (it would build - # vLLM's KimiLinear language model) and set up nn.Module directly. - nn.Module.__init__(self) - hf_config = getattr(atom_config, "hf_config", None) - assert hf_config is not None, "hf_config is not found in atom_config" - vision_config = hf_config.vision_config - self.config = hf_config - self.atom_config = atom_config - - vllm_config = atom_config.plugin_config.vllm_config - quant_config = vllm_config.quant_config - atom_quant_config = atom_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config - self.multimodal_config = multimodal_config - self.use_data_parallel = is_vit_use_data_parallel( - vision_config.num_attention_heads - ) - - with self._mark_tower_model(vllm_config, "image"): - self.vision_tower = MoonViT3dPretrainedModel( - vision_config, - quant_config=self._maybe_ignore_quant_config( - quant_config, - atom_quant_config.exclude_layers or [], - "vision_tower", - ), - prefix=maybe_prefix(prefix, "vision_tower"), - ) - self.mm_projector = KimiK25MultiModalProjector( - config=vision_config, - use_data_parallel=self.use_data_parallel, - quant_config=self._maybe_ignore_quant_config( - quant_config, - atom_quant_config.exclude_layers or [], - "mm_projector", - ), - prefix=maybe_prefix(prefix, "mm_projector"), - ) - - self.quant_config = quant_config - with self._mark_language_model(vllm_config): - self.language_model = KimiK3ForCausalLM( - atom_config=atom_config, - prefix=maybe_prefix(prefix, "language_model"), - ) - - self.make_empty_intermediate_tensors = ( - self.language_model.make_empty_intermediate_tensors - ) - self.media_placeholder = self.config.media_placeholder_token_id - - def _maybe_ignore_quant_config( - self, quant_config, exclude_layers, layer_name: str - ): - for exclude_layer in exclude_layers: - if QuantizationConfig._matches_exclude( - layer_name, exclude_layer, check_contains=True - ): - return None - return quant_config - - def load_weights(self, weights): - # weights generator is discarded; ATOM loads from disk in plugin mode. - # prefix="model." because ATOMModelBase constructs this as `.model`. - return load_model_in_plugin_mode( - model=self, - config=self.atom_config, - prefix="model.", - weights_mapper=self.hf_to_atom_mapper, - ) - - def get_expert_mapping(self): - return self.language_model.get_expert_mapping() -``` - -- [ ] **Step 6: Run the test to verify it passes** - -Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py::test_kimi_k3_inner_conditional_generation_class -v` -Expected: PASS - -- [ ] **Step 7: Run the full K3 plugin test module to confirm nothing else broke** - -Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py -v` -Expected: All existing tests still pass (the registry-sync test is updated in Task 3, not yet). - -- [ ] **Step 8: Commit** - -```bash -black atom/plugin/vllm/models/kimi_k3.py tests/plugin/test_vllm_kimi_k3.py && ruff check atom/plugin/vllm/models/kimi_k3.py tests/plugin/test_vllm_kimi_k3.py -git add atom/plugin/vllm/models/kimi_k3.py tests/plugin/test_vllm_kimi_k3.py -git commit -m "feat(k3): add multimodal inner class reusing upstream vision stack" -``` - ---- - -### Task 3: Make the outer class multimodal + rewire the registry - -**Files:** -- Modify: `atom/plugin/vllm/models/kimi_k3.py` (`KimiK3ForCausalLMVllm` base + methods) -- Modify: `atom/plugin/vllm/model_wrapper.py:158-160` (`_ATOM_MODEL_CLASSES["KimiK3ForConditionalGeneration"]`) -- Test: `tests/plugin/test_vllm_kimi_k3.py` - -**Interfaces:** -- Consumes: `KimiK3ForConditionalGeneration_` (Task 2); `ATOMForConditionalGeneration` (from `atom.plugin.vllm.model_wrapper`); `IsHybrid` (already imported). -- Produces: `KimiK3ForCausalLMVllm` with base `(ATOMForConditionalGeneration, IsHybrid)`, `get_placeholder_str`, `load_weights`, and the retained `get_mamba_state_*` classmethods; `_ATOM_MODEL_CLASSES["KimiK3ForConditionalGeneration"]` pointing at `...kimi_k3:KimiK3ForConditionalGeneration_`. - -**Context:** vLLM inspects the OUTER class (from `register.py`'s `_VLLM_MODEL_REGISTRY_OVERRIDES`, which already maps the arch to `KimiK3ForCausalLMVllm`) for interfaces. So `SupportsMultiModal` (via `ATOMForConditionalGeneration`) and `IsHybrid` must both live there. `ATOMForConditionalGeneration` also brings `SupportsMRoPE`, which stays dormant because K3's config has no `rope_scaling`/mrope. The existing registry-sync test must be updated to the new inner qualname. - -- [ ] **Step 1: Write/adjust the failing tests** - -In `tests/plugin/test_vllm_kimi_k3.py`, UPDATE the existing `test_kimi_k3_plugin_registries_are_synchronized` so the inner map points to the new class: - -```python -def test_kimi_k3_plugin_registries_are_synchronized(): - from atom.plugin.vllm.model_wrapper import _ATOM_MODEL_CLASSES - from atom.plugin.vllm.register import _VLLM_MODEL_REGISTRY_OVERRIDES - - arch = "KimiK3ForConditionalGeneration" - assert ( - _VLLM_MODEL_REGISTRY_OVERRIDES[arch] - == "atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLMVllm" - ) - assert ( - _ATOM_MODEL_CLASSES[arch] - == "atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGeneration_" - ) -``` - -ADD a new structural test for the outer class: - -```python -def test_kimi_k3_outer_is_multimodal_and_hybrid(): - _run_without_test_stubs(""" - from vllm.model_executor.models.interfaces import IsHybrid - - from atom.plugin.vllm.model_wrapper import ATOMForConditionalGeneration - from atom.plugin.vllm.models.kimi_k3 import KimiK3ForCausalLMVllm - - mro = KimiK3ForCausalLMVllm.__mro__ - # Multimodal via ATOMForConditionalGeneration; hybrid state retained. - assert ATOMForConditionalGeneration in mro - assert IsHybrid in mro - - # Image placeholder matches the checkpoint token. - assert ( - KimiK3ForCausalLMVllm.get_placeholder_str("image", 0) - == "<|kimi_image_placeholder|>" - ) - - # Hybrid mamba-state entry points survive the base-class change. - for name in ( - "get_mamba_state_dtype_from_config", - "get_mamba_state_shape_from_config", - "get_mamba_state_copy_func", - ): - assert hasattr(KimiK3ForCausalLMVllm, name) - """) -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py::test_kimi_k3_plugin_registries_are_synchronized tests/plugin/test_vllm_kimi_k3.py::test_kimi_k3_outer_is_multimodal_and_hybrid -v` -Expected: FAIL — registry test asserts old qualname; outer test fails (`ATOMForConditionalGeneration` not in MRO / no `get_placeholder_str`). - -- [ ] **Step 3: Add the `ATOMForConditionalGeneration` import** - -In `atom/plugin/vllm/models/kimi_k3.py`, add the import used by the new outer base: - -```python -from atom.plugin.vllm.model_wrapper import ATOMForConditionalGeneration -``` - -- [ ] **Step 4: Rewire the inner-class registry map** - -In `atom/plugin/vllm/model_wrapper.py`, change the `_ATOM_MODEL_CLASSES` entry: - -```python - "KimiK3ForConditionalGeneration": ( - "atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLM" - ), -``` - -to: - -```python - "KimiK3ForConditionalGeneration": ( - "atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGeneration_" - ), -``` - -- [ ] **Step 5: Make the outer class multimodal** - -In `atom/plugin/vllm/models/kimi_k3.py`, change the `KimiK3ForCausalLMVllm` class definition. Current: - -```python -class KimiK3ForCausalLMVllm(ATOMMoEForCausalLM, IsHybrid): - @classmethod - def get_mamba_state_dtype_from_config( - cls, - vllm_config: VllmConfig, - ) -> tuple[torch.dtype, torch.dtype]: - return _get_k3_state_dtype(vllm_config) - ... -``` - -Change ONLY the base classes and ADD `get_placeholder_str` + `load_weights`; keep all three `get_mamba_state_*` classmethods exactly as-is. Also add the processor decorator (matches K2.5, which decorates the outer too): - -```python -@MULTIMODAL_REGISTRY.register_processor( - KimiK3MultiModalProcessor, - info=KimiK3ProcessingInfo, - dummy_inputs=KimiK3DummyInputsBuilder, -) -class KimiK3ForCausalLMVllm(ATOMForConditionalGeneration, IsHybrid): - @classmethod - def get_placeholder_str(cls, modality: str, i: int) -> str | None: - if modality == "image": - return "<|kimi_image_placeholder|>" - raise ValueError(f"Unsupported modality: {modality}") - - def load_weights(self, weights): - return self.model.load_weights(weights) - - @classmethod - def get_mamba_state_dtype_from_config( - cls, - vllm_config: VllmConfig, - ) -> tuple[torch.dtype, torch.dtype]: - return _get_k3_state_dtype(vllm_config) - - @classmethod - def get_mamba_state_shape_from_config( - cls, - vllm_config: VllmConfig, - ) -> tuple[tuple[int, ...], tuple[int, ...]]: - return _get_k3_state_shape(vllm_config) - - @classmethod - def get_mamba_state_copy_func( - cls, - ) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: - return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func() -``` - -- [ ] **Step 6: Remove the now-unused `ATOMMoEForCausalLM` import if nothing else uses it** - -Run: `grep -n "ATOMMoEForCausalLM" atom/plugin/vllm/models/kimi_k3.py` -If the only remaining reference is the `from atom.plugin.vllm.model_wrapper import ATOMMoEForCausalLM` line, delete that import (ruff will flag it as unused otherwise). Keep it if still referenced. - -- [ ] **Step 7: Run the tests to verify they pass** - -Run: `python -m pytest tests/plugin/test_vllm_kimi_k3.py -v` -Expected: All pass (registry-sync updated, outer structural test passes, Task 2 test still passes). - -- [ ] **Step 8: Commit** - -```bash -black atom/plugin/vllm/models/kimi_k3.py atom/plugin/vllm/model_wrapper.py tests/plugin/test_vllm_kimi_k3.py && ruff check atom/plugin/vllm/models/kimi_k3.py atom/plugin/vllm/model_wrapper.py tests/plugin/test_vllm_kimi_k3.py -git add atom/plugin/vllm/models/kimi_k3.py atom/plugin/vllm/model_wrapper.py tests/plugin/test_vllm_kimi_k3.py -git commit -m "feat(k3): wire multimodal outer class + registry for image support" -``` - ---- - -### Task 4: GPU validation — weight-load completeness, image smoke, text-accuracy parity - -**Files:** -- No source changes expected. If validation surfaces a weight-mapping mismatch, fix `hf_to_atom_mapper` in `atom/plugin/vllm/models/kimi_k3.py` (adjust prefixes to match the "unexpected/missing weights" error), re-run, and commit as a fix. - -**Interfaces:** -- Consumes: everything from Tasks 1-3. -- Produces: a validated K3 image path (recorded in the PR description / a note under `k3_traces/` or the memory file). - -**Context:** This task cannot be a pytest unit test — it needs the GPU + the multimodal checkpoint. Use the `/run-atom-workload` skill for the canonical stop→start→workload→drain→stop flow. The K3 checkpoint is `/workspace/shared/data/amd_int/models/Kimi-K3`. Weight-load completeness is the real gate on the mapper: vLLM raises on unexpected/missing weights at load time. - -- [ ] **Step 1: Clean caches and start the server in plugin mode** - -```bash -rm -rf /root/.cache/atom/* -export AITER_LOG_LEVEL=WARNING -``` - -Start the server via the `/run-atom-workload` skill (plugin/vLLM mode) against `/workspace/shared/data/amd_int/models/Kimi-K3`. After startup, confirm the model is actually resident on GPU (not just HTTP 200): - -Run: `rocm-smi --showmemuse` -Expected: VRAM% > 0 on the TP GPUs; server log shows the vision tower + language model loaded with **no "unexpected weights" / "missing weights" errors**. If such errors appear, capture the offending names and adjust `hf_to_atom_mapper` prefixes, then restart from Step 1. - -- [ ] **Step 2: Image smoke test (describe an image)** - -Send one OpenAI-compatible chat request with an image URL/base64 + "Describe this image." (use the `/run-atom-workload` client or a `curl` to `/v1/chat/completions`). -Expected: a coherent, image-relevant description (not gibberish, not a text-only hallucination). This confirms `embed_multimodal` → vision tower → projector → merge → KDA/MLA/MoE forward all compose. - -- [ ] **Step 3: Text-only request still works** - -Send a plain text chat request (no image) to the same server. -Expected: a normal coherent completion — confirms `embed_multimodal` returns `None` and the text path is unchanged. - -- [ ] **Step 4: Text-accuracy parity (GSM8K)** - -Run GSM8K via `/run-atom-workload` (lm_eval) against this server and compare to the current text-only K3 baseline on this branch. -Expected: accuracy within normal run-to-run variance of the text-only baseline — proves the KDA/dual-stream/hybrid-state path did not regress from the rewiring. - -- [ ] **Step 5: Record results** - -Note the smoke output, GSM8K score vs baseline, and any mapper fix in the PR description and update the memory file `k3-vision-vllm-plugin.md` with the GPU-validated status. - -- [ ] **Step 6: Commit any fixes** - -```bash -# only if Step 1 required a mapper fix -black atom/plugin/vllm/models/kimi_k3.py && ruff check atom/plugin/vllm/models/kimi_k3.py -git add atom/plugin/vllm/models/kimi_k3.py -git commit -m "fix(k3): correct vision/language weight prefix mapping" -``` - ---- - -## Self-Review - -**Spec coverage:** -- Config gate (`_PLUGIN_SUPPORTED_MULTIMODAL_MODELS`) → Task 1. ✓ -- New inner `KimiK3ForConditionalGeneration_` (subclass upstream, reuse vision, swap ATOM LM, ATOM weight loading) → Task 2. ✓ -- `embed_input_ids` on plugin LM (not native) → Task 2. ✓ -- Outer class → `ATOMForConditionalGeneration, IsHybrid` + `get_placeholder_str` + `load_weights` + retained `get_mamba_state_*` → Task 3. ✓ -- Registry rewire (`_ATOM_MODEL_CLASSES`; `register.py` unchanged) → Task 3. ✓ -- No new config wrapper; native files untouched; plugin/image only → enforced in Global Constraints and each task's file list. ✓ -- Validation: image smoke + text-accuracy-unchanged → Task 4. ✓ -- Risk: multimodal+hybrid composition, dormant MRoPE, weight mapping, VRAM → addressed in Tasks 3/4 and constraints. ✓ - -**Placeholder scan:** No TBD/TODO; every code step has concrete code; the one runtime-verified detail (weight mapper) has a concrete starting value derived from the checkpoint layout plus an explicit adjust-on-error procedure in Task 4. ✓ - -**Type/name consistency:** `KimiK3ForConditionalGeneration_`, `KimiK3ForCausalLMVllm`, `KimiK3ForCausalLM`, `_ATOM_MODEL_CLASSES`, `hf_to_atom_mapper`, `get_placeholder_str`, `get_mamba_state_*` used consistently across Tasks 2-3 and match the existing code in `atom/plugin/vllm/models/kimi_k3.py` and `model_wrapper.py`. ✓ diff --git a/docs/superpowers/specs/2026-08-04-k3-vision-vllm-plugin-design.md b/docs/superpowers/specs/2026-08-04-k3-vision-vllm-plugin-design.md deleted file mode 100644 index 4fe450050c..0000000000 --- a/docs/superpowers/specs/2026-08-04-k3-vision-vllm-plugin-design.md +++ /dev/null @@ -1,179 +0,0 @@ -# Design: Kimi-K3 image support in the ATOM vLLM plugin - -**Date:** 2026-08-04 -**Branch:** guanbao/k3_rmsnorm_quant_fusion -**Status:** Approved design — pending implementation plan - -## Goal - -Add vision (image) support to Kimi-K3 when running in **ATOM vLLM plugin mode**, by -mirroring how Kimi-K2.5 achieves it. K3 currently runs text-only in the plugin even -though the checkpoint is multimodal. - -## Scope - -- **In scope:** Image input, plugin mode only, always-multimodal (vision tower always - loads, K2.5 pattern). -- **Out of scope:** Video (no upstream K3 reference; the K3 checkpoint has no video - tokens). Native ATOM (non-plugin) mode stays text-only. -- **Hard constraint:** Never modify native K3 files `atom/models/kimi_k3.py` or - `atom/models/kimi_k3_dspark.py`. They may only be imported read-only. - -## Background / current state - -- The K3 checkpoint (`/workspace/shared/data/amd_int/models/Kimi-K3`) is already - multimodal: architecture `KimiK3ForConditionalGeneration`, `model_type: kimi_k3`, - with `vision_config` + `text_config` + `image_placeholder` + - `media_placeholder_token_id`. The **same** architecture name is used for text and - vision checkpoints (as with K2.5). -- vLLM upstream already ships the full K3 **image** stack: - - `vllm/models/kimi_k3/amd/model.py::KimiK3ForConditionalGeneration` — reuses K2.5's - `MoonViT3dPretrainedModel` + `KimiK25MultiModalProjector`; combines - `SupportsMultiModal` + `IsHybrid` + `HasInnerState` + `SupportsPP/Quant/Eagle3`. - - `vllm/models/kimi_k3/common/mm_preprocess.py` — `KimiK3ProcessingInfo`, - `KimiK3MultiModalProcessor`, `KimiK3DummyInputsBuilder` (image-only). Placeholder - token `<|kimi_image_placeholder|>`. -- ATOM forces K3 down a text-only path today: - - `kimi_k3` is absent from `_PLUGIN_SUPPORTED_MULTIMODAL_MODELS` in `atom/config.py`, - so the plugin strips `vision_config`. - - The plugin outer class `KimiK3ForCausalLMVllm` inherits `ATOMMoEForCausalLM, - IsHybrid` (text-only) rather than the multimodal `ATOMForConditionalGeneration`. -- The reference implementation for the plugin wiring is - `atom/plugin/vllm/models/kimi_k25.py` (inner `KimiK25ForConditionalGeneration_` + - outer `KimiK25ForConditionalGeneration`). - -**Novel wrinkle vs K2.5:** K3 is a *hybrid* model (KDA recurrent + MLA). Its outer -class must be multimodal **and** retain the hybrid-state plumbing (`IsHybrid` + -`get_mamba_state_*`). K2.5 is pure-attention and did not need this. vLLM's own upstream -K3 class proves the two interface sets compose. - -## Architecture - -Three layers, mirroring K2.5, with hybrid-state additions on the outer class: - -``` -register.py: "KimiK3ForConditionalGeneration" ─► KimiK3ForCausalLMVllm (OUTER, modified) - = ATOMForConditionalGeneration + IsHybrid - · get_mamba_state_* (kept) - · get_placeholder_str (new) - · load_weights → self.model (new) - │ .model = -model_wrapper._ATOM_MODEL_ARCH_TO_QUALNAME ───────► KimiK3ForConditionalGeneration_ (INNER, new) - = subclass of vLLM KimiK3ForConditionalGeneration - · vision_tower = MoonViT3dPretrainedModel (upstream) - · mm_projector = KimiK25MultiModalProjector (upstream) - · embed_multimodal / media parse (inherited) - · @register_processor(KimiK3MultiModalProcessor…) - │ .language_model = - KimiK3ForCausalLM (ATOM PLUGIN subclass, KDA-monkeypatched) - = existing optimized text path (dual-stream / KDA / fusion) -``` - -Key insight: vLLM inspects the **outer** class for interfaces, so `IsHybrid` and -`SupportsMultiModal` (via `ATOMForConditionalGeneration`) both live there. The -**inner** class owns the vision tower and delegates text to ATOM's KDA language model. -This keeps ATOM's entire K3 text-path optimization intact while the image stack is -reused verbatim from upstream. - -## Detailed changes - -All edits are confined to three files. **No native K3 file is touched.** - -### 1. `atom/config.py` -Add `"kimi_k3"` to `_PLUGIN_SUPPORTED_MULTIMODAL_MODELS` so the plugin passes the full -`hf_config` (including `vision_config`) through instead of stripping to `text_config`. - -### 2. `atom/plugin/vllm/models/kimi_k3.py` - -**New inner class** `KimiK3ForConditionalGeneration_`: -- Subclasses vLLM's `KimiK3ForConditionalGeneration` (from `vllm.models.kimi_k3`, - resolves to the AMD branch on ROCm) to inherit `embed_multimodal`, - `_parse_and_validate_media_input`, `_process_media_input`, `_mark_tower_model`, - `_mark_language_model`. -- Decorated with - `@MULTIMODAL_REGISTRY.register_processor(KimiK3MultiModalProcessor, - info=KimiK3ProcessingInfo, dummy_inputs=KimiK3DummyInputsBuilder)`. -- `__init__(self, atom_config, prefix="model")`: - - `nn.Module.__init__(self)`. - - Read `hf_config.vision_config` / `hf_config.text_config` directly — reuse the - checkpoint's HF `KimiK3Config`; do **not** create a new config wrapper (decided). - - Build `vision_tower` (`MoonViT3dPretrainedModel`) + `mm_projector` - (`KimiK25MultiModalProjector`) under `self._mark_tower_model(vllm_config, "image")`, - handling the same quant-exclude logic as K2.5's `_maybe_ignore_quant_config`. - - Build the ATOM **plugin** `KimiK3ForCausalLM` as `self.language_model` under - `self._mark_language_model(vllm_config)` (this triggers the existing KDA → - `KimiKDAAttentionVllm` self-registration in the static forward context). - - `self.make_empty_intermediate_tensors = - self.language_model.make_empty_intermediate_tensors`. -- `load_weights`: `load_model_in_plugin_mode(model=self, config=self.atom_config, - prefix="model.", weights_mapper=self.hf_to_atom_mapper)` with a `WeightsMapper` - covering vision prefixes, `mm_projector.proj.{0,2}` → `linear_{1,2}`, and - `language_model.` remaps (patterned on K2.5's mapper, adjusted to K3's checkpoint key - layout — verified during implementation against the on-disk weight names). -- `get_expert_mapping` delegating to `self.language_model`. - -**Add `embed_input_ids`** to the existing plugin `KimiK3ForCausalLM` subclass (NOT the -native class) — returns `self.get_input_embeddings(input_ids)`. Required by vLLM's -`SupportsMultiModal.get_language_model` discovery (K2.5 needed the same). - -**Modify outer** `KimiK3ForCausalLMVllm`: -- Change base from `(ATOMMoEForCausalLM, IsHybrid)` to - `(ATOMForConditionalGeneration, IsHybrid)`. -- Keep the three existing `get_mamba_state_*` classmethods unchanged. -- Add `get_placeholder_str(cls, modality, i)` → `"<|kimi_image_placeholder|>"` for - `image`, raise for others. -- Add `load_weights` delegating to `self.model.load_weights`. - -### 3. `atom/plugin/vllm/model_wrapper.py` -Repoint `_ATOM_MODEL_ARCH_TO_QUALNAME["KimiK3ForConditionalGeneration"]` from -`atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLM` to -`atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGeneration_`. - -### Unchanged -- `atom/plugin/vllm/register.py` — already maps the arch to the outer - `KimiK3ForCausalLMVllm`; keep as-is. -- `atom/model_engine/model_runner.py` — native mode stays text-only (out of scope). -- `atom/models/kimi_k3.py`, `atom/models/kimi_k3_dspark.py` — never touched. - -## Data flow (image request) - -1. `KimiK3MultiModalProcessor` (upstream) converts image + `<|kimi_image_placeholder|>` - prompt into `pixel_values` + `grid_thws`, expanding the placeholder into - `media_begin … media_end` pad tokens sized to the image resolution. -2. vLLM calls `embed_multimodal` (inherited) → `vision_tower` + `mm_projector` → image - feature tensors. -3. `embed_input_ids` (vLLM `SupportsMultiModal` default merge) embeds text tokens and - scatters image features at `media_placeholder_token_id` positions. -4. Merged embeddings enter ATOM's `KimiK3ForCausalLM.forward` (KDA + MLA + MoE, - dual-stream) exactly as today; hybrid recurrent state via the outer class's - `IsHybrid` / `get_mamba_state_*`. -5. Text-only request → `embed_multimodal` returns `None`; path is identical to the - current text-only K3. - -## Risks & mitigations - -- **Multimodal + hybrid interface composition** (main novelty). Mitigated: vLLM's own - upstream K3 class combines the same interface sets, and we keep the hybrid interfaces - on the outer class (which vLLM inspects) exactly as the working text path does. -- **Inherited `HasInnerState` / `mamba_cache` methods** on the inner class reference a - `mamba_cache` ATOM does not use. Mitigated: vLLM inspects the *outer* class (which - does not declare `HasInnerState`), so these stay dormant. If they are ever invoked, - fall back to a standalone inner class (no upstream subclassing) for the vision path. -- **Weight-name mapping** — vision/projector/language_model prefixes must all resolve. - Verified during implementation against on-disk weight names and by the smoke test's - load-completeness check (no unexpected/missing weights). -- **VRAM / startup** — the ViT now always loads (accepted). May shift memory headroom - for text-only benchmarks on this branch; note in the PR. - -## Validation - -- **Smoke test:** start the server with the K3 checkpoint in plugin mode, send an - image + text prompt, confirm a coherent description and clean weight loading - (`rocm-smi` VRAM > 0, no unmapped weights). Use `/run-atom-workload`. -- **Text accuracy unchanged:** run GSM8K (or the usual text eval) and confirm parity - with the current text-only K3 — proves the KDA / dual-stream path did not regress. - -## Open questions - -None. Config wrapper skipped (reuse HF `KimiK3Config`); native mode out of scope; video -out of scope. From ff37588947399c800360fcbccbdd2f0f78e2eb13 Mon Sep 17 00:00:00 2001 From: Satya K Jandhyala Date: Wed, 16 Sep 2026 15:37:02 -0500 Subject: [PATCH 6/6] Addressed PR reviews. --- atom/model_ops/fla_ops/chunk.py | 2 +- atom/model_ops/fla_ops/chunk_vk.py | 2 +- .../model_ops/fla_ops/fused_sigmoid_gating.py | 2 +- atom/model_ops/fla_ops/layernorm_guard.py | 2 +- atom/models/kimi_k3_dspark.py | 15 +- .../plugin/vllm/attention/layer_sparse_mla.py | 5 +- .../vllm/attention/minimax_m3_attnetion.py | 4 +- atom/plugin/vllm/kda_backend.py | 22 +-- atom/plugin/vllm/model_wrapper.py | 11 +- atom/plugin/vllm/models/kimi_k3.py | 155 ++++++++++------ atom/plugin/vllm/register.py | 2 +- atom/rollout/async_engine.py | 2 +- docs/vllm_plugin_backend_guide.md | 2 +- recipes/atom_vllm/KVV.md | 105 ----------- recipes/atom_vllm/Kimi-K3.md | 12 +- tests/plugin/test_vllm_kimi_k3.py | 173 ++++++++++++++---- tests/test_kimi_k3_plugin_config.py | 13 ++ 17 files changed, 282 insertions(+), 247 deletions(-) delete mode 100644 recipes/atom_vllm/KVV.md create mode 100644 tests/test_kimi_k3_plugin_config.py diff --git a/atom/model_ops/fla_ops/chunk.py b/atom/model_ops/fla_ops/chunk.py index d0585a552a..6edbb0dab5 100644 --- a/atom/model_ops/fla_ops/chunk.py +++ b/atom/model_ops/fla_ops/chunk.py @@ -210,7 +210,7 @@ def chunk_gated_delta_rule( v: torch.Tensor, g: torch.Tensor, beta: torch.Tensor, - scale: float | None = None, + scale: float = None, initial_state: torch.Tensor = None, output_final_state: bool = False, cu_seqlens: torch.LongTensor | None = None, diff --git a/atom/model_ops/fla_ops/chunk_vk.py b/atom/model_ops/fla_ops/chunk_vk.py index bf2a2767b9..2bb9239b1b 100644 --- a/atom/model_ops/fla_ops/chunk_vk.py +++ b/atom/model_ops/fla_ops/chunk_vk.py @@ -137,7 +137,7 @@ def chunk_gated_delta_rule_vk( v: torch.Tensor, g: torch.Tensor, beta: torch.Tensor, - scale: float | None = None, + scale: float = None, initial_state: torch.Tensor = None, output_final_state: bool = False, cu_seqlens: torch.Tensor | None = None, diff --git a/atom/model_ops/fla_ops/fused_sigmoid_gating.py b/atom/model_ops/fla_ops/fused_sigmoid_gating.py index 11cc27e7a9..526c9dd0dd 100644 --- a/atom/model_ops/fla_ops/fused_sigmoid_gating.py +++ b/atom/model_ops/fla_ops/fused_sigmoid_gating.py @@ -204,7 +204,7 @@ def fused_sigmoid_gating_delta_rule_update( o: torch.Tensor | None = None, beta: float = 1.0, threshold: float = 20.0, - scale: float | None = None, + scale: float = None, initial_state: torch.Tensor = None, inplace_final_state: bool = True, cu_seqlens: torch.LongTensor | None = None, diff --git a/atom/model_ops/fla_ops/layernorm_guard.py b/atom/model_ops/fla_ops/layernorm_guard.py index e87327a784..fcb991c595 100644 --- a/atom/model_ops/fla_ops/layernorm_guard.py +++ b/atom/model_ops/fla_ops/layernorm_guard.py @@ -198,7 +198,7 @@ def layer_norm_fwd( eps: float, z: torch.Tensor = None, out: torch.Tensor = None, - group_size: int | None = None, + group_size: int = None, norm_before_gate: bool = True, is_rms_norm: bool = False, ): diff --git a/atom/models/kimi_k3_dspark.py b/atom/models/kimi_k3_dspark.py index 227d163435..f29c3a57be 100644 --- a/atom/models/kimi_k3_dspark.py +++ b/atom/models/kimi_k3_dspark.py @@ -545,24 +545,19 @@ def __init__( prefix=f"{prefix}.mlp.gate_up_proj", ) - def write_context_kv(self, ctx_hidden, positions, slot_mapping=None) -> None: + def write_context_kv(self, ctx_hidden, positions) -> None: """Populate this layer's context rows. The context source does NOT go through ``input_layernorm``: the reference feeds every layer the same ``context_norm(context_proj(aux))`` tensor straight into the KV projection, while ``input_layernorm`` applies only to the residual stream carrying the draft block. - - ``slot_mapping`` is optional: the native path reads it from - ``forward_context``, while the vLLM plugin passes per-layer slots - explicitly (groups may differ across layers). """ - if slot_mapping is None: - from atom.utils.forward_context import get_forward_context + from atom.utils.forward_context import get_forward_context - slot_mapping = get_forward_context().attn_metadata.slot_mapping[ - : ctx_hidden.shape[0] - ] + slot_mapping = get_forward_context().attn_metadata.slot_mapping[ + : ctx_hidden.shape[0] + ] self.self_attn.write_context_kv(ctx_hidden, positions, slot_mapping) def forward( diff --git a/atom/plugin/vllm/attention/layer_sparse_mla.py b/atom/plugin/vllm/attention/layer_sparse_mla.py index ddbbf75846..973995eb0e 100644 --- a/atom/plugin/vllm/attention/layer_sparse_mla.py +++ b/atom/plugin/vllm/attention/layer_sparse_mla.py @@ -302,7 +302,10 @@ def sparse_attn_indexer_plugin_mode( "Sparse MLA metadata not found for indexer cache " f"{k_cache_prefix!r}. The indexer cannot populate paged_kv_indices." ) - slot_mapping = indexer_meta.slot_mapping + # V2 may pad metadata to the graph width while the tensors above contain + # only live tokens. Cache kernels launch once per slot and index those + # tensors directly, so the mapping must have the same live-token extent. + slot_mapping = indexer_meta.slot_mapping[: q_input.shape[0]] has_decode = indexer_meta.num_decodes > 0 has_prefill = indexer_meta.num_prefills > 0 num_decode_tokens = indexer_meta.num_decode_tokens diff --git a/atom/plugin/vllm/attention/minimax_m3_attnetion.py b/atom/plugin/vllm/attention/minimax_m3_attnetion.py index 51041d81f5..31850780a2 100644 --- a/atom/plugin/vllm/attention/minimax_m3_attnetion.py +++ b/atom/plugin/vllm/attention/minimax_m3_attnetion.py @@ -459,14 +459,14 @@ def _insert_qkv_and_index( self.index_q_norm.weight, self.index_k_norm.weight, self.num_idx_heads, - slot_mapping=main_metadata.slot_mapping, + slot_mapping=main_metadata.slot_mapping[:num_tokens], kv_cache_k=k_cache, kv_cache_v=v_cache, index_cache=self.index_cache_layer.kv_cache, block_size=k_cache.shape[3], q_out=q_out, index_q_out=index_q, - index_slot_mapping=index_metadata.slot_mapping, + index_slot_mapping=index_metadata.slot_mapping[:num_tokens], kv_cache_dtype=kv_cache_dtype, k_scale=k_scale if self.kv_cache_dtype == "fp8" else None, v_scale=v_scale if self.kv_cache_dtype == "fp8" else None, diff --git a/atom/plugin/vllm/kda_backend.py b/atom/plugin/vllm/kda_backend.py index 00024efec5..4f02e4fcf5 100644 --- a/atom/plugin/vllm/kda_backend.py +++ b/atom/plugin/vllm/kda_backend.py @@ -57,29 +57,9 @@ def build( # type: ignore[override] num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu, fast_build=fast_build, ) - self._stage_packed_decode_query_start_loc(common_attn_metadata, metadata) self._adapt_full_graph_decode_metadata(common_attn_metadata, metadata) return metadata - def _stage_packed_decode_query_start_loc( - self, - common_attn_metadata: CommonAttentionMetadata, - metadata: KimiK3KDAMetadata, - ) -> None: - if metadata.num_decodes <= 0 or metadata.non_spec_query_start_loc is not None: - return - - batch_size = int(common_attn_metadata.num_reqs) - query_start_loc_buf = self.non_spec_query_start_loc[: batch_size + 1] - torch.arange( - metadata.num_decodes + 1, - dtype=torch.int32, - device=query_start_loc_buf.device, - out=query_start_loc_buf[: metadata.num_decodes + 1], - ) - query_start_loc_buf[metadata.num_decodes + 1 :].fill_(metadata.num_decodes) - metadata.non_spec_query_start_loc = query_start_loc_buf - def _adapt_full_graph_decode_metadata( self, common_attn_metadata: CommonAttentionMetadata, @@ -101,6 +81,8 @@ def _adapt_full_graph_decode_metadata( query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] real_decode_mask_cpu = query_lens_cpu > 0 real_num_decodes = int(real_decode_mask_cpu.sum().item()) + if real_num_decodes == metadata.num_decodes: + return batch_size = int(common_attn_metadata.num_reqs) if batch_size > self.decode_cudagraph_max_bs: diff --git a/atom/plugin/vllm/model_wrapper.py b/atom/plugin/vllm/model_wrapper.py index ac0a63f263..52d8b85fbf 100644 --- a/atom/plugin/vllm/model_wrapper.py +++ b/atom/plugin/vllm/model_wrapper.py @@ -507,7 +507,16 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): else: self.model = model_cls(self.atom_config) else: - self.model = model_cls(self.atom_config) + if model_arch == "KimiK3ForConditionalGeneration": + # Kimi-K3's inner class inherits vLLM SupportsQuant. Passing + # VllmConfig lets that protocol apply its quant mapper and + # packed-module mappings before the vision modules are built. + self.model = model_cls( + self.atom_config, + vllm_config=vllm_config, + ) + else: + self.model = model_cls(self.atom_config) num_patched_post_load_hooks = _patch_required_act_dtype_post_load_hooks( self.model, diff --git a/atom/plugin/vllm/models/kimi_k3.py b/atom/plugin/vllm/models/kimi_k3.py index 81be322a69..c34fe279b9 100644 --- a/atom/plugin/vllm/models/kimi_k3.py +++ b/atom/plugin/vllm/models/kimi_k3.py @@ -14,6 +14,9 @@ MambaStateDtypeCalculator, MambaStateShapeCalculator, ) +from vllm.model_executor.layers.quantization.compressed_tensors import ( + compressed_tensors, +) from vllm.model_executor.models.interfaces import IsHybrid from vllm.model_executor.models.kimi_k25_vit import ( KimiK25MultiModalProjector, @@ -30,10 +33,11 @@ ) from vllm.models.kimi_k3.nvidia.kda_metadata import KimiK3KDAMetadata from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.platforms import current_platform from vllm.v1.attention.backend import AttentionBackend from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum -from atom.config import Config, QuantizationConfig +from atom.config import Config from atom.model_loader.loader import WeightsMapper, load_model_in_plugin_mode from atom.models import kimi_k3 as kimi_k3_base from atom.models.kimi_k3 import ( @@ -44,14 +48,18 @@ from atom.models.kimi_k3 import ( KimiK3ForCausalLM as KimiK3ForCausalLMBase, ) -from atom.models.utils import IntermediateTensors, maybe_prefix +from atom.models.utils import IntermediateTensors, PPMissingLayer, maybe_prefix from atom.plugin.vllm.kda_backend import AtomKimiK3KDAAttentionBackend from atom.plugin.vllm.model_wrapper import ATOMForConditionalGeneration from atom.utils.forward_context import get_forward_context as get_atom_forward_context _KIMI_K3_QUANT_EXCLUDE_NAME_MAPPING: dict[str, str] = { - "language_model.model.": "language_model.language_model.model.", - "language_model.lm_head": "language_model.language_model.lm_head", + "language_model.language_model.model.": ( + "model.language_model.language_model.model." + ), + "language_model.language_model.lm_head": ( + "model.language_model.language_model.lm_head" + ), } @@ -483,6 +491,15 @@ def forward( class KimiK3ForCausalLM(KimiK3ForCausalLMBase): + def __setattr__(self, name: str, value: Any) -> None: + if name in {"model", "lm_head"}: + language_model = getattr(self, "_modules", {}).get("language_model") + if language_model is not None: + setattr(language_model, name, value) + object.__setattr__(self, name, value) + return + super().__setattr__(name, value) + def __init__(self, *args, **kwargs): original_kda_cls = kimi_k3_base.KimiKDAAttention original_model_cls = kimi_k3_base.KimiLinearModel @@ -493,29 +510,18 @@ def __init__(self, *args, **kwargs): finally: kimi_k3_base.KimiKDAAttention = original_kda_cls kimi_k3_base.KimiLinearModel = original_model_cls + # vLLM's DSpark loader reads these one level above their real owner. + # Bypass Module registration for the initial aliases so checkpoint + # names stay unchanged. A later normal setattr (weight sharing) then + # replaces the alias with a real registered module instead of being + # hidden behind a read-only property. + object.__setattr__(self, "model", self.language_model.model) + object.__setattr__(self, "lm_head", self.language_model.lm_head) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: # Required by vLLM SupportsMultiModal.get_language_model discovery. return self.get_input_embeddings(input_ids) - @property - def model(self) -> nn.Module: - """The body the DSpark loader reaches for ``embed_tokens`` on. - - ``load_dspark_model`` reads ``get_language_model().model.embed_tokens`` - and ``get_language_model().lm_head``. Multimodal discovery stops at this - class, one level above the causal LM that actually owns both, so expose - them here rather than redirecting ``get_language_model`` -- which the - vision path also uses. A property, so it stays out of ``named_modules`` - and the checkpoint prefixes are unchanged. - """ - return self.language_model.model - - @property - def lm_head(self) -> nn.Module: - """The head the DSpark draft borrows to score its block.""" - return self.language_model.lm_head - def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None: self.language_model.model.set_aux_hidden_state_layers(layers) @@ -525,11 +531,6 @@ def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]: return (2, num_layers // 2, num_layers - 3) -@MULTIMODAL_REGISTRY.register_processor( - KimiK3MultiModalProcessor, - info=KimiK3ProcessingInfo, - dummy_inputs=KimiK3DummyInputsBuilder, -) class KimiK3ForConditionalGeneration_(vLLMKimiK3): hf_to_atom_mapper = WeightsMapper( orig_to_new_prefix={ @@ -545,7 +546,13 @@ class KimiK3ForConditionalGeneration_(vLLMKimiK3): _KIMI_K3_QUANT_EXCLUDE_NAME_MAPPING ) - def __init__(self, atom_config: Config, prefix: str = "model"): + def __init__( + self, + atom_config: Config, + prefix: str = "model", + *, + vllm_config: VllmConfig | None = None, + ): nn.Module.__init__(self) hf_config = getattr(atom_config, "hf_config", None) assert hf_config is not None, "hf_config is not found in atom_config" @@ -553,35 +560,54 @@ def __init__(self, atom_config: Config, prefix: str = "model"): self.config = hf_config self.atom_config = atom_config - vllm_config = atom_config.plugin_config.vllm_config + if vllm_config is None: + vllm_config = atom_config.plugin_config.vllm_config + model_config = vllm_config.model_config quant_config = vllm_config.quant_config atom_quant_config = atom_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = model_config.multimodal_config self.multimodal_config = multimodal_config self.use_data_parallel = is_vit_use_data_parallel( vision_config.num_attention_heads ) - - with self._mark_tower_model(vllm_config, "image"): - self.vision_tower = MoonViT3dPretrainedModel( - vision_config, - quant_config=self._maybe_ignore_quant_config( - quant_config, - atom_quant_config.exclude_layers or [], - "vision_tower", - ), - prefix=maybe_prefix(prefix, "vision_tower"), + text_config = getattr(hf_config, "text_config", hf_config) + self.hidden_size = text_config.hidden_size + self.device = current_platform.current_device() + self.has_vision_tower = get_pp_group().is_first_rank + + if self.has_vision_tower: + vision_quant_config = self._maybe_ignore_quant_config( + quant_config, + atom_quant_config.exclude_layers or [], + "vision_tower", ) - self.mm_projector = KimiK25MultiModalProjector( - config=vision_config, - use_data_parallel=self.use_data_parallel, - quant_config=self._maybe_ignore_quant_config( - quant_config, - atom_quant_config.exclude_layers or [], - "mm_projector", - ), - prefix=maybe_prefix(prefix, "mm_projector"), + projector_quant_config = self._maybe_ignore_quant_config( + quant_config, + atom_quant_config.exclude_layers or [], + "mm_projector", ) + with self._mark_tower_model(vllm_config, "image"): + self.vision_tower = MoonViT3dPretrainedModel( + vision_config, + quant_config=vision_quant_config, + prefix=maybe_prefix(prefix, "vision_tower"), + ) + if vision_quant_config is not None: + self.vision_tower = self.vision_tower.to(device=self.device) + else: + self.vision_tower = self.vision_tower.to( + device=self.device, dtype=model_config.dtype + ) + self.mm_projector = KimiK25MultiModalProjector( + config=vision_config, + use_data_parallel=self.use_data_parallel, + quant_config=projector_quant_config, + prefix=maybe_prefix(prefix, "mm_projector"), + ).to(device=self.device, dtype=model_config.dtype) + else: + self.vision_tower = PPMissingLayer() + self.mm_projector = PPMissingLayer() + self.skip_weight_prefixes = ["vision_tower.", "mm_projector."] self.quant_config = quant_config with self._mark_language_model(vllm_config): @@ -593,16 +619,35 @@ def __init__(self, atom_config: Config, prefix: str = "model"): self.make_empty_intermediate_tensors = ( self.language_model.make_empty_intermediate_tensors ) - self.media_placeholder = self.config.media_placeholder_token_id + self.media_placeholder = ( + self.config.media_placeholder_token_id if self.has_vision_tower else None + ) self.packed_modules_mapping = self.language_model.packed_modules_mapping self.weights_mapping = self.language_model.weights_mapping def _maybe_ignore_quant_config( - self, quant_config: Any, exclude_layers: list[str], layer_name: str + self, + quant_config: Any, + exclude_layers: list[str] | None = None, + layer_name: str | None = None, ): - for exclude_layer in exclude_layers: - if QuantizationConfig._matches_exclude( - layer_name, exclude_layer, check_contains=True + if isinstance(quant_config, compressed_tensors.CompressedTensorsConfig): + return None + if layer_name is not None: + # Check the container and a representative child, both with and + # without ATOMModelBase's outer ``model.`` root. The child form is + # required for patterns such as ``*.vision_tower.*``. + candidates = ( + layer_name, + f"{layer_name}.__atom_child__", + f"model.{layer_name}", + f"model.{layer_name}.__atom_child__", + ) + if any( + self.atom_config.quant_config._is_excluded( + candidate, exclude_layers or [] + ) + for candidate in candidates ): return None return quant_config @@ -630,7 +675,7 @@ def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]: info=KimiK3ProcessingInfo, dummy_inputs=KimiK3DummyInputsBuilder, ) -class KimiK3ForCausalLMVllm(ATOMForConditionalGeneration, IsHybrid): +class KimiK3ForConditionalGenerationVllm(ATOMForConditionalGeneration, IsHybrid): @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality == "image": diff --git a/atom/plugin/vllm/register.py b/atom/plugin/vllm/register.py index bc8b58e444..18fe0cb4b3 100644 --- a/atom/plugin/vllm/register.py +++ b/atom/plugin/vllm/register.py @@ -36,7 +36,7 @@ "Qwen3_5MoeForConditionalGeneration": "atom.plugin.vllm.models.qwen3_5:Qwen3_5MoeForConditionalGeneration", "KimiK25ForConditionalGeneration": "atom.plugin.vllm.models.kimi_k25:KimiK25ForConditionalGeneration", "KimiK3ForConditionalGeneration": ( - "atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLMVllm" + "atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGenerationVllm" ), # vLLM registers this arch too, but only to its NVIDIA implementation. "K3DSparkModel": "atom.plugin.vllm.models.kimi_k3_dspark:KimiK3DSparkVllm", diff --git a/atom/rollout/async_engine.py b/atom/rollout/async_engine.py index 2d4a46dfa3..3a95487250 100644 --- a/atom/rollout/async_engine.py +++ b/atom/rollout/async_engine.py @@ -54,7 +54,7 @@ def sleep(self, level: int = 1): logger.info("AsyncLLMEngine sleep: completed") - def wake_up(self, tags: list[str] | None = None): + def wake_up(self, tags: list[str] = None): """Restore GPU resources. Args: diff --git a/docs/vllm_plugin_backend_guide.md b/docs/vllm_plugin_backend_guide.md index 9f6e4eb478..bd022d7dfc 100644 --- a/docs/vllm_plugin_backend_guide.md +++ b/docs/vllm_plugin_backend_guide.md @@ -185,7 +185,7 @@ Currently, the plugin backend supports the following model architectures: | `GptOssForCausalLM` | `atom.models.gpt_oss.GptOssForCausalLM` | GPT-OSS | | `DeepseekV3ForCausalLM` | `atom.models.deepseek_v2.DeepseekV3ForCausalLM` | DeepSeek-R1 / DeepSeek V3 / Kimi-K2 style models | | `Glm4MoeForCausalLM` | `atom.models.glm4_moe.Glm4MoeForCausalLM` | GLM-4-MoE | -| `KimiK3ForConditionalGeneration` | `atom.plugin.vllm.models.kimi_k3.KimiK3ForCausalLM` | Kimi-K3 text-only KDA + MLA hybrid MoE | +| `KimiK3ForConditionalGeneration` | `atom.plugin.vllm.models.kimi_k3.KimiK3ForConditionalGenerationVllm` | Kimi-K3 multimodal KDA + MLA hybrid MoE | `Kimi-K2` is also supported. Although it is usually loaded with `--trust-remote-code`, it shares the same DeepSeek-style MLA+MoE architecture path and reuses `atom.models.deepseek_v2.DeepseekV3ForCausalLM` in the ATOM vLLM OOT backend. diff --git a/recipes/atom_vllm/KVV.md b/recipes/atom_vllm/KVV.md deleted file mode 100644 index 027adca17a..0000000000 --- a/recipes/atom_vllm/KVV.md +++ /dev/null @@ -1,105 +0,0 @@ -# KVV Run — Commands - -Env: local vLLM for Kimi-K3 at `http://localhost:8000/v1`. -Two Python envs: `/opt/venv` (pytest + BEAM), project `.venv` (inspect-ai benchmarks). - -## Clone KVV - -```bash -git clone git@github.com:MoonshotAI/Kimi-Vendor-Verifier.git -``` - -## Server -Check server launch commands in recipe. - -## 1. tests/params/ - -```bash -cd ~/Kimi-Vendor-Verifier -/opt/venv/bin/pytest tests/params/ \ - --base-url http://localhost:8000/v1 --api-key EMPTY \ - --smoke-model "/workspace/shared/data/amd_int/models/Kimi-K3" \ - --think-mode opensource -``` - -## 2. tests/k3_features/ - -```bash -cd ~/Kimi-Vendor-Verifier -/opt/venv/bin/pytest tests/k3_features/ \ - --base-url http://localhost:8000/v1 --api-key EMPTY \ - --smoke-model "/workspace/shared/data/amd_int/models/Kimi-K3" \ - --think-mode opensource -``` - -## 3. tests/prompt_tokens/ - -```bash -cd ~/Kimi-Vendor-Verifier -/opt/venv/bin/pytest tests/prompt_tokens/ \ - --base-url http://localhost:8000/v1 --api-key EMPTY \ - --smoke-model "/workspace/shared/data/amd_int/models/Kimi-K3" \ - --think-mode opensource -``` - -## 4. tests/tool_call_json_schema/ - -```bash -cd ~/Kimi-Vendor-Verifier -/opt/venv/bin/pytest tests/tool_call_json_schema/ \ - --base-url http://localhost:8000/v1 --api-key EMPTY \ - --smoke-model "/workspace/shared/data/amd_int/models/Kimi-K3" \ - --think-mode opensource -``` - -## 5. OCRBench - -```bash -cd ~/Kimi-Vendor-Verifier -KIMI_BASE_URL=http://localhost:8000/v1 KIMI_API_KEY=EMPTY \ -.venv/bin/python eval.py ocrbench \ - --model "opensource//workspace/shared/data/amd_int/models/Kimi-K3" \ - --max-tokens 16384 --thinking --think-mode opensource --stream \ - --max-connections 50 --temperature 1.0 --top-p 0.95 --thinking-effort high -``` - -## 6. MMMU Pro Vision - -```bash -cd ~/Kimi-Vendor-Verifier -KIMI_BASE_URL=http://localhost:8000/v1 KIMI_API_KEY=EMPTY \ -.venv/bin/python eval.py mmmu \ - --model "opensource//workspace/shared/data/amd_int/models/Kimi-K3" \ - --max-tokens 98304 --thinking --think-mode opensource --stream \ - --max-connections 50 --temperature 1.0 --top-p 0.95 --thinking-effort high -``` - -BEAM and Judge are low priority tests. Let's ignore them for now. - -## 7. BEAM 1M (smoke) - -Generate: - -```bash -cd ~/Kimi-Vendor-Verifier/beam -/opt/venv/bin/python beam_generate.py \ - --model /workspace/shared/data/amd_int/models/Kimi-K3 \ - --base-url http://localhost:8000/v1 --api-key EMPTY \ - --temperature 1.0 --top-p 0.95 --max-tokens 32768 \ - --thinking-json '{"chat_template_kwargs": {"thinking": true, "preserve_thinking": true, "thinking_effort": "high"}}' \ - --concurrency 4 --max-retries 3 --limit 8 \ - --output answers_smoke.jsonl -``` - -Judge (self-judge via same local server): - -```bash -cd ~/Kimi-Vendor-Verifier/beam -/opt/venv/bin/python beam_judge.py \ - --answers answers_smoke.jsonl \ - --judge-model /workspace/shared/data/amd_int/models/Kimi-K3 \ - --judge-base-url http://localhost:8000/v1 --judge-api-key EMPTY \ - --judge-max-tokens 16384 --judge-temperature 0.3 \ - --concurrency 16 \ - --output scores_smoke.jsonl -``` diff --git a/recipes/atom_vllm/Kimi-K3.md b/recipes/atom_vllm/Kimi-K3.md index ac2727db32..169a131292 100644 --- a/recipes/atom_vllm/Kimi-K3.md +++ b/recipes/atom_vllm/Kimi-K3.md @@ -1,9 +1,9 @@ # Kimi-K3 with ATOM vLLM Plugin Backend -This recipe serves the text-only Kimi-K3 backbone -(`KimiK3ForConditionalGeneration`) through the ATOM vLLM out-of-tree plugin. -Kimi-K3 combines KDA recurrent-attention layers, MLA full-attention layers, and -an MXFP4 latent MoE. +This recipe serves multimodal Kimi-K3 (`KimiK3ForConditionalGeneration`) +through the ATOM vLLM out-of-tree plugin. Kimi-K3 combines a MoonViT3d vision +tower with KDA recurrent-attention layers, MLA full-attention layers, and an +MXFP4 latent MoE. The validated configuration requires eight MI355 (gfx950) GPUs with TP8. @@ -32,7 +32,6 @@ vllm serve "${MODEL}" \ --port 8000 \ --tensor-parallel-size 8 \ --trust-remote-code \ - --language-model-only \ --kv-cache-dtype fp8 \ --max-model-len 16384 \ --max-num-seqs 64 \ @@ -142,7 +141,8 @@ Avg draft acceptance rate: 86.1%, 86.1% (whole run, each of the two) ## Current scope -- Text generation only; the vision tower and multimodal projector are skipped. +- Text and image inputs are supported through the Kimi-K3 multimodal processor, + vision tower, and projector. - TP8 on MI355/gfx950 is the validated deployment. - Asynchronous scheduling is supported. Prefix caching is off by default and needs `--mamba-cache-mode align` to be turned on, as the DSpark launch does. diff --git a/tests/plugin/test_vllm_kimi_k3.py b/tests/plugin/test_vllm_kimi_k3.py index 395d67827c..91ec71c564 100644 --- a/tests/plugin/test_vllm_kimi_k3.py +++ b/tests/plugin/test_vllm_kimi_k3.py @@ -24,7 +24,7 @@ def test_kimi_k3_plugin_registries_are_synchronized(): arch = "KimiK3ForConditionalGeneration" assert ( _VLLM_MODEL_REGISTRY_OVERRIDES[arch] - == "atom.plugin.vllm.models.kimi_k3:KimiK3ForCausalLMVllm" + == "atom.plugin.vllm.models.kimi_k3:KimiK3ForConditionalGenerationVllm" ) assert ( _ATOM_MODEL_CLASSES[arch] @@ -114,44 +114,31 @@ def test_kimi_k3_outer_is_multimodal_and_hybrid(): from vllm.model_executor.models.interfaces import IsHybrid from atom.plugin.vllm.model_wrapper import ATOMForConditionalGeneration - from atom.plugin.vllm.models.kimi_k3 import KimiK3ForCausalLMVllm + from atom.plugin.vllm.models.kimi_k3 import ( + KimiK3ForConditionalGenerationVllm, + ) - mro = KimiK3ForCausalLMVllm.__mro__ + mro = KimiK3ForConditionalGenerationVllm.__mro__ # Multimodal via ATOMForConditionalGeneration; hybrid state retained. assert ATOMForConditionalGeneration in mro assert IsHybrid in mro # Image placeholder matches the checkpoint token. assert ( - KimiK3ForCausalLMVllm.get_placeholder_str("image", 0) + KimiK3ForConditionalGenerationVllm.get_placeholder_str("image", 0) == "<|kimi_image_placeholder|>" ) - # Hybrid mamba-state entry points survive the base-class change. - for name in ( - "get_mamba_state_dtype_from_config", - "get_mamba_state_shape_from_config", - "get_mamba_state_copy_func", - ): - assert hasattr(KimiK3ForCausalLMVllm, name) + # vLLM looks up the processor on the architecture class from + # get_model_architecture (this wrapper), not on the inner module. + assert "_processor_factory" in KimiK3ForConditionalGenerationVllm.__dict__ """) -def test_kimi_k3_is_plugin_supported_multimodal(): - from atom.config import ( - _MULTIMODAL_MODEL_TYPES, - _PLUGIN_SUPPORTED_MULTIMODAL_MODELS, - ) - - # kimi_k3 must be a known multimodal type AND opted into plugin-mode - # pass-through, so get_hf_config keeps vision_config instead of stripping - # to text_config. - assert "kimi_k3" in _MULTIMODAL_MODEL_TYPES - assert "kimi_k3" in _PLUGIN_SUPPORTED_MULTIMODAL_MODELS - - def test_kimi_k3_inner_conditional_generation_class(): _run_without_test_stubs(""" + from inspect import Parameter, signature + from vllm.models.kimi_k3 import ( KimiK3ForConditionalGeneration as vLLMKimiK3, ) @@ -161,12 +148,14 @@ def test_kimi_k3_inner_conditional_generation_class(): ) # Inner class subclasses the upstream multimodal model so it inherits - # embed_multimodal / media parsing / vision-tower forward. + # embed_multimodal / media parsing / vision-tower forward. Do not + # re-register the processor here: vLLM keys it per class, and the + # architecture class that must carry it is the outer wrapper. assert issubclass(KimiK3ForConditionalGeneration_, vLLMKimiK3) - - # ATOM plugin loading + expert routing entry points exist. - assert hasattr(KimiK3ForConditionalGeneration_, "load_weights") - assert hasattr(KimiK3ForConditionalGeneration_, "get_expert_mapping") + assert "_processor_factory" not in KimiK3ForConditionalGeneration_.__dict__ + assert hasattr(KimiK3ForConditionalGeneration_, "_processor_factory") + parameters = signature(KimiK3ForConditionalGeneration_.__init__).parameters + assert parameters["vllm_config"].kind is Parameter.KEYWORD_ONLY # Weight mapper collapses the double language_model nesting and renames # the projector layers. @@ -185,17 +174,78 @@ def test_kimi_k3_inner_conditional_generation_class(): # discovery. assert hasattr(KimiK3ForCausalLM, "embed_input_ids") - # Quant-remap attributes must be present (ATOMModelBase reads them off - # the inner class); missing them silently corrupts a quantized ckpt. - packed = KimiK3ForConditionalGeneration_.packed_modules_mapping - assert packed, "packed_modules_mapping must be non-empty" - - excl = KimiK3ForConditionalGeneration_.quant_exclude_name_mapping - assert excl, "quant_exclude_name_mapping must be non-empty" - # Values carry the doubled language_model. prefix for the extra nesting - # (inner.language_model = ATOM KimiK3ForCausalLM -> KimiLinearForCausalLM). - for value in excl.values(): - assert value.startswith("language_model.language_model."), value + # Quant exclusions are remapped after hf_to_atom_mapper. They must add + # the outer model root without applying the language_model nesting a + # second time. + from atom.config import QuantizationConfig + + quant_config = QuantizationConfig() + quant_config.exclude_layers = [ + "language_model.language_model.model.layers.0.mlp", + "language_model.language_model.lm_head", + ] + quant_config.apply_exclude_name_mapping( + KimiK3ForConditionalGeneration_.quant_exclude_name_mapping + ) + assert quant_config.exclude_layers == [ + "model.language_model.language_model.model.layers.0.mlp", + "model.language_model.language_model.lm_head", + ] + """) + + +def test_kimi_k3_spec_decode_alias_writes_reach_language_model(): + _run_without_test_stubs(""" + import torch.nn as nn + + from atom.plugin.vllm.models.kimi_k3 import KimiK3ForCausalLM + + target = KimiK3ForCausalLM.__new__(KimiK3ForCausalLM) + nn.Module.__init__(target) + target.language_model = nn.Module() + target.language_model.model = nn.Linear(2, 2) + target.language_model.lm_head = nn.Linear(2, 3) + + replacement_model = nn.Linear(2, 2) + replacement_head = nn.Linear(2, 3) + target.model = replacement_model + target.lm_head = replacement_head + + assert target.model is replacement_model + assert target.language_model.model is replacement_model + assert target.lm_head is replacement_head + assert target.language_model.lm_head is replacement_head + """) + + +def test_kimi_k3_vision_quant_excludes_cover_regex_and_globs(): + _run_without_test_stubs(""" + from types import SimpleNamespace + + from atom.config import QuantizationConfig + from atom.plugin.vllm.models.kimi_k3 import ( + KimiK3ForConditionalGeneration_, + ) + + model = SimpleNamespace( + atom_config=SimpleNamespace(quant_config=QuantizationConfig()) + ) + quant_config = object() + for exclude in ( + "re:.*vision_tower.*", + "vision_tower*", + "*.vision_tower.*", + "model.vision_tower", + ): + assert ( + KimiK3ForConditionalGeneration_._maybe_ignore_quant_config( + model, + quant_config, + [exclude], + "vision_tower", + ) + is None + ), exclude """) @@ -312,6 +362,49 @@ def test_kda_metadata_adapter_compacts_full_graph_padding(): """) +def test_kda_metadata_adapter_leaves_unpadded_decode_unchanged(): + _run_without_test_stubs(""" + from types import SimpleNamespace + + import torch + + import atom.plugin.vllm.kda_backend as kda_backend + + assert ( + "_stage_packed_decode_query_start_loc" + not in kda_backend.AtomKimiK3KDAMetadataBuilder.__dict__ + ) + + builder = SimpleNamespace( + use_full_cuda_graph=True, + decode_cudagraph_max_bs=2, + ) + common = SimpleNamespace( + query_start_loc_cpu=torch.tensor([0, 1, 2], dtype=torch.int32), + num_reqs=2, + ) + metadata = SimpleNamespace( + num_prefills=0, + num_spec_decodes=0, + num_decodes=2, + num_decode_tokens=2, + non_spec_state_indices_tensor=torch.tensor([3, 7], dtype=torch.int32), + non_spec_query_start_loc=torch.tensor([0, 1, 2], dtype=torch.int32), + ) + state_indices = metadata.non_spec_state_indices_tensor + query_start_loc = metadata.non_spec_query_start_loc + + kda_backend.AtomKimiK3KDAMetadataBuilder._adapt_full_graph_decode_metadata( + builder, common, metadata + ) + + assert metadata.non_spec_state_indices_tensor is state_indices + assert metadata.non_spec_query_start_loc is query_start_loc + assert metadata.num_decodes == 2 + assert metadata.num_decode_tokens == 2 + """) + + def test_gdn_metadata_builder_compacts_full_graph_padding(): _run_without_test_stubs(""" from types import SimpleNamespace diff --git a/tests/test_kimi_k3_plugin_config.py b/tests/test_kimi_k3_plugin_config.py new file mode 100644 index 0000000000..cfc5aa13a6 --- /dev/null +++ b/tests/test_kimi_k3_plugin_config.py @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + +from atom.config import ( + _MULTIMODAL_MODEL_TYPES, + _PLUGIN_SUPPORTED_MULTIMODAL_MODELS, +) + + +def test_kimi_k3_is_plugin_supported_multimodal(): + # Keep the full HF vision config when ATOM prepares Kimi-K3 for a plugin. + assert "kimi_k3" in _MULTIMODAL_MODEL_TYPES + assert "kimi_k3" in _PLUGIN_SUPPORTED_MULTIMODAL_MODELS