From 1e8ec82fa5141f94dc417307a2f8167838615e13 Mon Sep 17 00:00:00 2001 From: Raphael Rialland Date: Thu, 30 Jul 2026 11:15:25 +0200 Subject: [PATCH 1/8] Fix MRv2 multimodal draft capability detection Assisted-by: Codex Signed-off-by: Raphael Rialland --- .../test_gpu_autoregressive_speculator.py | 80 +++++++++++++++++++ .../spec_decode/autoregressive/speculator.py | 24 +++++- 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_autoregressive_speculator.py index 940fb375a928..1900668876de 100644 --- a/tests/v1/worker/test_gpu_autoregressive_speculator.py +++ b/tests/v1/worker/test_gpu_autoregressive_speculator.py @@ -11,6 +11,7 @@ from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( AutoRegressiveSpeculator, ) +from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator class _TestSpeculator(AutoRegressiveSpeculator): @@ -27,6 +28,21 @@ def forward(self, **kwargs): return self.output +class _MultimodalDraftModel(torch.nn.Module): + def embed_input_ids( + self, + input_ids, + multimodal_embeddings=None, + is_multimodal=None, + ): + return input_ids + + +class _TextOnlyDraftModel(torch.nn.Module): + def embed_input_ids(self, input_ids): + return input_ids + + def _make_speculator( monkeypatch, output: torch.Tensor | tuple[torch.Tensor, torch.Tensor], @@ -49,6 +65,70 @@ def _make_speculator( return speculator +def test_mm_support_uses_target_config(monkeypatch): + target_model_config = object() + draft_model_config = object() + vllm_config = SimpleNamespace(model_config=target_model_config) + + def init_base(speculator, vllm_config, device): + speculator.max_num_tokens = 4 + speculator.max_num_reqs = 2 + speculator.hidden_size = 3 + speculator.dtype = torch.float32 + speculator.draft_model_config = draft_model_config + + checked_configs = [] + + def supports_multimodal_inputs(model_config): + checked_configs.append(model_config) + return True + + monkeypatch.setattr(DraftModelSpeculator, "__init__", init_base) + monkeypatch.setattr( + spec_module.MULTIMODAL_REGISTRY, + "supports_multimodal_inputs", + supports_multimodal_inputs, + ) + + speculator = _TestSpeculator(vllm_config, torch.device("cpu")) + + assert checked_configs == [target_model_config] + assert speculator.supports_mm_inputs + assert speculator.inputs_embeds.shape == (4, 3) + + +def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch): + speculator = object.__new__(_TestSpeculator) + speculator.supports_mm_inputs = True + speculator.device = torch.device("cpu") + draft_model = _MultimodalDraftModel() + monkeypatch.setattr( + DraftModelSpeculator, + "load_model", + lambda self, target_model: setattr(self, "model", draft_model), + ) + + speculator.load_model(torch.nn.Module()) + + assert speculator.supports_mm_inputs + + +def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch): + speculator = object.__new__(_TestSpeculator) + speculator.supports_mm_inputs = True + speculator.device = torch.device("cpu") + draft_model = _TextOnlyDraftModel() + monkeypatch.setattr( + DraftModelSpeculator, + "load_model", + lambda self, target_model: setattr(self, "model", draft_model), + ) + + speculator.load_model(torch.nn.Module()) + + assert not speculator.supports_mm_inputs + + def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch): logits_hidden = torch.full((4, 3), 1.0) feedback_hidden = torch.full((4, 3), 2.0) diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index 9035ea38b233..0ffac688196e 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -3,6 +3,7 @@ from typing import Any import torch +import torch.nn as nn from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode @@ -37,8 +38,10 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): self.max_num_reqs, dtype=torch.int64, device=device ) + # Draft heads reuse embeddings produced by the target's multimodal encoder + # and commonly have text-only checkpoint configs. self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs( - self.draft_model_config + vllm_config.model_config ) if self.supports_mm_inputs: self.inputs_embeds = torch.zeros( @@ -48,6 +51,25 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None + def load_model(self, target_model: nn.Module) -> None: + super().load_model(target_model) + if not self.supports_mm_inputs: + return + + try: + dummy_input_ids = torch.tensor([[1]], device=self.device) + self.model.embed_input_ids( + dummy_input_ids, + multimodal_embeddings=None, + is_multimodal=None, + ) + except (NotImplementedError, AttributeError, TypeError): + logger.warning( + "Draft model does not support multimodal inputs, " + "falling back to text-only mode" + ) + self.supports_mm_inputs = False + @property def advance_draft_positions(self) -> bool: """ From 323b82cd666d65d5225773df6e2b3130a37bb900 Mon Sep 17 00:00:00 2001 From: Raphael Rialland Date: Mon, 3 Aug 2026 10:45:18 +0200 Subject: [PATCH 2/8] Use explicit multimodal embedding capability Assisted-by: Codex Signed-off-by: Raphael Rialland --- .../test_gpu_autoregressive_speculator.py | 46 +++++++++++++++++-- vllm/model_executor/models/__init__.py | 2 + vllm/model_executor/models/interfaces.py | 7 +++ vllm/model_executor/models/llama_eagle3.py | 17 ++++++- vllm/v1/spec_decode/llm_base_proposer.py | 25 +++++----- .../spec_decode/autoregressive/speculator.py | 10 +--- 6 files changed, 80 insertions(+), 27 deletions(-) diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_autoregressive_speculator.py index 1900668876de..1dc0887f7a8d 100644 --- a/tests/v1/worker/test_gpu_autoregressive_speculator.py +++ b/tests/v1/worker/test_gpu_autoregressive_speculator.py @@ -7,6 +7,7 @@ import torch from vllm.config.compilation import CUDAGraphMode +from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( AutoRegressiveSpeculator, @@ -29,18 +30,27 @@ def forward(self, **kwargs): class _MultimodalDraftModel(torch.nn.Module): + supports_multimodal_embeddings = True + def embed_input_ids( self, input_ids, multimodal_embeddings=None, + *, is_multimodal=None, ): - return input_ids + raise AssertionError("embed_input_ids should not be called during loading") class _TextOnlyDraftModel(torch.nn.Module): - def embed_input_ids(self, input_ids): - return input_ids + def embed_input_ids( + self, + input_ids, + multimodal_embeddings=None, + *, + is_multimodal=None, + ): + raise AssertionError("embed_input_ids should not be called during loading") def _make_speculator( @@ -100,7 +110,6 @@ def supports_multimodal_inputs(model_config): def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch): speculator = object.__new__(_TestSpeculator) speculator.supports_mm_inputs = True - speculator.device = torch.device("cpu") draft_model = _MultimodalDraftModel() monkeypatch.setattr( DraftModelSpeculator, @@ -116,7 +125,6 @@ def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch): def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch): speculator = object.__new__(_TestSpeculator) speculator.supports_mm_inputs = True - speculator.device = torch.device("cpu") draft_model = _TextOnlyDraftModel() monkeypatch.setattr( DraftModelSpeculator, @@ -129,6 +137,34 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch): assert not speculator.supports_mm_inputs +def test_eagle3_llama_merges_multimodal_embeddings(): + embedded_input_ids = None + + class _EmbeddingModel: + def embed_input_ids(self, input_ids): + nonlocal embedded_input_ids + embedded_input_ids = input_ids + return input_ids.unsqueeze(-1).expand(-1, 2).float() + + model = SimpleNamespace(model=_EmbeddingModel()) + input_ids = torch.tensor([1, 999, 2]) + is_multimodal = torch.tensor([False, True, False]) + multimodal_embeddings = [torch.tensor([[10.0, 11.0]])] + + actual = Eagle3LlamaForCausalLM.embed_input_ids( + model, + input_ids, + multimodal_embeddings, + is_multimodal, + ) + + torch.testing.assert_close(embedded_input_ids, torch.tensor([1, 0, 2])) + torch.testing.assert_close( + actual, + torch.tensor([[1.0, 1.0], [10.0, 11.0], [2.0, 2.0]]), + ) + + def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch): logits_hidden = torch.full((4, 3), 1.0) feedback_hidden = torch.full((4, 3), 2.0) diff --git a/vllm/model_executor/models/__init__.py b/vllm/model_executor/models/__init__.py index 9f8dd042bf83..aa72b8994eaa 100644 --- a/vllm/model_executor/models/__init__.py +++ b/vllm/model_executor/models/__init__.py @@ -12,6 +12,7 @@ supports_lora, supports_mrope, supports_multimodal, + supports_multimodal_embeddings, supports_pp, supports_transcription, ) @@ -35,6 +36,7 @@ "supports_lora", "SupportsMultiModal", "supports_multimodal", + "supports_multimodal_embeddings", "SupportsMRoPE", "supports_mrope", "SupportsPP", diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index efdab8c4d105..e606d353bc08 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -122,6 +122,9 @@ class SupportsMultiModal(Protocol): MRO of your model class. """ + supports_multimodal_embeddings: ClassVar[Literal[True]] = True + """A flag indicating this model can merge external multimodal embeddings.""" + supports_multimodal_raw_input_only: ClassVar[bool] = False """ A flag that indicates this model supports multi-modal inputs and processes @@ -488,6 +491,10 @@ def supports_multimodal( return getattr(model, "supports_multimodal", False) +def supports_multimodal_embeddings(model: type[object] | object) -> bool: + return getattr(model, "supports_multimodal_embeddings", False) + + def supports_multimodal_raw_input_only(model: type[object] | object) -> bool: return getattr(model, "supports_multimodal_raw_input_only", False) diff --git a/vllm/model_executor/models/llama_eagle3.py b/vllm/model_executor/models/llama_eagle3.py index 549e8b7bf63a..2dcf07a2d718 100644 --- a/vllm/model_executor/models/llama_eagle3.py +++ b/vllm/model_executor/models/llama_eagle3.py @@ -18,12 +18,14 @@ ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.models.interfaces import _require_is_multimodal from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM from vllm.multimodal.inputs import NestedTensors from .utils import ( AutoWeightsLoader, WeightsMapper, + _merge_multimodal_embeddings, get_draft_quant_config, maybe_prefix, process_eagle_weight, @@ -270,6 +272,8 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: class Eagle3LlamaForCausalLM(LlamaForCausalLM): + supports_multimodal_embeddings = True + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) self.config = vllm_config.speculative_config.draft_model_config.hf_config @@ -321,7 +325,18 @@ def embed_input_ids( multimodal_embeddings: NestedTensors | None = None, is_multimodal: torch.Tensor | None = None, ) -> torch.Tensor: - return self.model.embed_input_ids(input_ids) + if is_multimodal is not None: + input_ids = input_ids.masked_fill( + is_multimodal.to(device=input_ids.device, non_blocking=True), 0 + ) + inputs_embeds = self.model.embed_input_ids(input_ids) + if multimodal_embeddings is None or len(multimodal_embeddings) == 0: + return inputs_embeds + return _merge_multimodal_embeddings( + inputs_embeds, + multimodal_embeddings, + _require_is_multimodal(is_multimodal), + ) def forward( self, diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index c02c3350d868..8db0b48b0912 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -25,7 +25,10 @@ from vllm.logger import init_logger from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.model_loader import get_model -from vllm.model_executor.models import supports_multimodal +from vllm.model_executor.models import ( + supports_multimodal, + supports_multimodal_embeddings, +) from vllm.model_executor.models.deepseek_eagle3 import Eagle3DeepseekV2ForCausalLM from vllm.model_executor.models.interfaces import SupportsMultiModal from vllm.model_executor.models.laguna_dflash import DFlashLagunaForCausalLM @@ -1349,18 +1352,14 @@ def load_model(self, target_model: nn.Module) -> None: if all_attn_layers[name].get_kv_cache_spec(self.vllm_config) is not None } - if self.supports_mm_inputs: - # Even if the target model is multimodal, we can also use - # text-only draft models - try: - dummy_input_ids = torch.tensor([[1]], device=self.input_ids.device) - self.model.embed_input_ids(dummy_input_ids, multimodal_embeddings=None) - except (NotImplementedError, AttributeError, TypeError): - logger.warning( - "Draft model does not support multimodal inputs, " - "falling back to text-only mode" - ) - self.supports_mm_inputs = False + # Even if the target model is multimodal, we can also use + # text-only draft models + if self.supports_mm_inputs and not supports_multimodal_embeddings(self.model): + logger.warning( + "Draft model does not support multimodal inputs, " + "falling back to text-only mode" + ) + self.supports_mm_inputs = False if supports_multimodal(target_model): # handle multimodality diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index 0ffac688196e..e3952f5e9021 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -9,6 +9,7 @@ from vllm.config.compilation import CUDAGraphMode from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.logger import init_logger +from vllm.model_executor.models import supports_multimodal_embeddings from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.triton_utils import tl, triton from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer @@ -56,14 +57,7 @@ def load_model(self, target_model: nn.Module) -> None: if not self.supports_mm_inputs: return - try: - dummy_input_ids = torch.tensor([[1]], device=self.device) - self.model.embed_input_ids( - dummy_input_ids, - multimodal_embeddings=None, - is_multimodal=None, - ) - except (NotImplementedError, AttributeError, TypeError): + if not supports_multimodal_embeddings(self.model): logger.warning( "Draft model does not support multimodal inputs, " "falling back to text-only mode" From 671b57e4974926bc1add44fb71d8f85f9fd253bf Mon Sep 17 00:00:00 2001 From: Raphael Rialland Date: Mon, 3 Aug 2026 11:13:42 +0200 Subject: [PATCH 3/8] Avoid changing Eagle3Llama embedding behavior Assisted-by: Codex Signed-off-by: Raphael Rialland --- .../test_gpu_autoregressive_speculator.py | 29 ------------------- vllm/model_executor/models/llama_eagle3.py | 17 +---------- 2 files changed, 1 insertion(+), 45 deletions(-) diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_autoregressive_speculator.py index 1dc0887f7a8d..b80a2eab6540 100644 --- a/tests/v1/worker/test_gpu_autoregressive_speculator.py +++ b/tests/v1/worker/test_gpu_autoregressive_speculator.py @@ -7,7 +7,6 @@ import torch from vllm.config.compilation import CUDAGraphMode -from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( AutoRegressiveSpeculator, @@ -137,34 +136,6 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch): assert not speculator.supports_mm_inputs -def test_eagle3_llama_merges_multimodal_embeddings(): - embedded_input_ids = None - - class _EmbeddingModel: - def embed_input_ids(self, input_ids): - nonlocal embedded_input_ids - embedded_input_ids = input_ids - return input_ids.unsqueeze(-1).expand(-1, 2).float() - - model = SimpleNamespace(model=_EmbeddingModel()) - input_ids = torch.tensor([1, 999, 2]) - is_multimodal = torch.tensor([False, True, False]) - multimodal_embeddings = [torch.tensor([[10.0, 11.0]])] - - actual = Eagle3LlamaForCausalLM.embed_input_ids( - model, - input_ids, - multimodal_embeddings, - is_multimodal, - ) - - torch.testing.assert_close(embedded_input_ids, torch.tensor([1, 0, 2])) - torch.testing.assert_close( - actual, - torch.tensor([[1.0, 1.0], [10.0, 11.0], [2.0, 2.0]]), - ) - - def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch): logits_hidden = torch.full((4, 3), 1.0) feedback_hidden = torch.full((4, 3), 2.0) diff --git a/vllm/model_executor/models/llama_eagle3.py b/vllm/model_executor/models/llama_eagle3.py index 2dcf07a2d718..549e8b7bf63a 100644 --- a/vllm/model_executor/models/llama_eagle3.py +++ b/vllm/model_executor/models/llama_eagle3.py @@ -18,14 +18,12 @@ ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.models.interfaces import _require_is_multimodal from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM from vllm.multimodal.inputs import NestedTensors from .utils import ( AutoWeightsLoader, WeightsMapper, - _merge_multimodal_embeddings, get_draft_quant_config, maybe_prefix, process_eagle_weight, @@ -272,8 +270,6 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: class Eagle3LlamaForCausalLM(LlamaForCausalLM): - supports_multimodal_embeddings = True - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) self.config = vllm_config.speculative_config.draft_model_config.hf_config @@ -325,18 +321,7 @@ def embed_input_ids( multimodal_embeddings: NestedTensors | None = None, is_multimodal: torch.Tensor | None = None, ) -> torch.Tensor: - if is_multimodal is not None: - input_ids = input_ids.masked_fill( - is_multimodal.to(device=input_ids.device, non_blocking=True), 0 - ) - inputs_embeds = self.model.embed_input_ids(input_ids) - if multimodal_embeddings is None or len(multimodal_embeddings) == 0: - return inputs_embeds - return _merge_multimodal_embeddings( - inputs_embeds, - multimodal_embeddings, - _require_is_multimodal(is_multimodal), - ) + return self.model.embed_input_ids(input_ids) def forward( self, From e4402735f803bd30cb4f720ebca968a42e2c2286 Mon Sep 17 00:00:00 2001 From: Raphael Rialland Date: Mon, 3 Aug 2026 11:26:12 +0200 Subject: [PATCH 4/8] Advertise multimodal embedding capable drafters Assisted-by: Codex Signed-off-by: Raphael Rialland --- .../test_gpu_autoregressive_speculator.py | 34 +++++++++++++++++++ vllm/model_executor/models/llama4_eagle.py | 2 ++ vllm/model_executor/models/mistral_eagle.py | 2 ++ .../models/mistral_large_3_eagle.py | 2 ++ vllm/v1/spec_decode/llm_base_proposer.py | 8 +++-- .../spec_decode/autoregressive/speculator.py | 8 +++-- 6 files changed, 50 insertions(+), 6 deletions(-) diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_autoregressive_speculator.py index b80a2eab6540..66e156b45733 100644 --- a/tests/v1/worker/test_gpu_autoregressive_speculator.py +++ b/tests/v1/worker/test_gpu_autoregressive_speculator.py @@ -4,9 +4,18 @@ from contextlib import nullcontext from types import SimpleNamespace +import pytest import torch from vllm.config.compilation import CUDAGraphMode +from vllm.model_executor.models import supports_multimodal_embeddings +from vllm.model_executor.models.exaone4_5_mtp import Exaone4_5_MTP +from vllm.model_executor.models.llama4_eagle import EagleLlama4ForCausalLM +from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM +from vllm.model_executor.models.mistral_eagle import EagleMistralForCausalLM +from vllm.model_executor.models.mistral_large_3_eagle import ( + EagleMistralLarge3ForCausalLM, +) from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( AutoRegressiveSpeculator, @@ -125,15 +134,40 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch): speculator = object.__new__(_TestSpeculator) speculator.supports_mm_inputs = True draft_model = _TextOnlyDraftModel() + warning_messages = [] monkeypatch.setattr( DraftModelSpeculator, "load_model", lambda self, target_model: setattr(self, "model", draft_model), ) + monkeypatch.setattr( + spec_module.logger, + "warning_once", + lambda message, *args: warning_messages.append(message % args), + ) speculator.load_model(torch.nn.Module()) assert not speculator.supports_mm_inputs + assert warning_messages == [ + "Draft model _TextOnlyDraftModel does not support external multimodal " + "embeddings. Embeddings from the target model will not be passed to the " + "drafter; using text-only draft inputs instead." + ] + + +@pytest.mark.parametrize( + ("model_cls", "expected"), + [ + (EagleLlama4ForCausalLM, True), + (EagleMistralForCausalLM, True), + (EagleMistralLarge3ForCausalLM, True), + (Exaone4_5_MTP, True), + (Eagle3LlamaForCausalLM, False), + ], +) +def test_draft_model_multimodal_embedding_capability(model_cls, expected): + assert supports_multimodal_embeddings(model_cls) is expected def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch): diff --git a/vllm/model_executor/models/llama4_eagle.py b/vllm/model_executor/models/llama4_eagle.py index d94dbb049668..336122cd9468 100644 --- a/vllm/model_executor/models/llama4_eagle.py +++ b/vllm/model_executor/models/llama4_eagle.py @@ -153,6 +153,8 @@ def pad_layer_name(layer: str) -> str: class EagleLlama4ForCausalLM(Llama4ForCausalLM): + supports_multimodal_embeddings = True + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) self.config = vllm_config.speculative_config.draft_model_config.hf_config diff --git a/vllm/model_executor/models/mistral_eagle.py b/vllm/model_executor/models/mistral_eagle.py index 75d1ebb91a80..c579ceecbe5d 100644 --- a/vllm/model_executor/models/mistral_eagle.py +++ b/vllm/model_executor/models/mistral_eagle.py @@ -110,6 +110,8 @@ def forward( class EagleMistralForCausalLM(MistralForCausalLM): + supports_multimodal_embeddings = True + mistral_mapping = MistralForCausalLM.mistral_mapping | { "eagle_linear": "model.fc", } diff --git a/vllm/model_executor/models/mistral_large_3_eagle.py b/vllm/model_executor/models/mistral_large_3_eagle.py index 8ace01205d03..605095ced8ca 100644 --- a/vllm/model_executor/models/mistral_large_3_eagle.py +++ b/vllm/model_executor/models/mistral_large_3_eagle.py @@ -108,6 +108,8 @@ def forward( class EagleMistralLarge3ForCausalLM(MistralLarge3ForCausalLM): + supports_multimodal_embeddings = True + hf_to_vllm_mapper = MistralLarge3ForCausalLM.hf_to_vllm_mapper | WeightsMapper( orig_to_new_regex={ regex.compile(r"\Aeagle_linear\.weight\Z"): r"model.fc.weight", diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index 8db0b48b0912..675bc4bbe561 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -1355,9 +1355,11 @@ def load_model(self, target_model: nn.Module) -> None: # Even if the target model is multimodal, we can also use # text-only draft models if self.supports_mm_inputs and not supports_multimodal_embeddings(self.model): - logger.warning( - "Draft model does not support multimodal inputs, " - "falling back to text-only mode" + logger.warning_once( + "Draft model %s does not support external multimodal embeddings. " + "Embeddings from the target model will not be passed to the " + "drafter; using text-only draft inputs instead.", + type(self.model).__name__, ) self.supports_mm_inputs = False diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index e3952f5e9021..b275985acd8c 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -58,9 +58,11 @@ def load_model(self, target_model: nn.Module) -> None: return if not supports_multimodal_embeddings(self.model): - logger.warning( - "Draft model does not support multimodal inputs, " - "falling back to text-only mode" + logger.warning_once( + "Draft model %s does not support external multimodal embeddings. " + "Embeddings from the target model will not be passed to the " + "drafter; using text-only draft inputs instead.", + type(self.model).__name__, ) self.supports_mm_inputs = False From aad112c31ac877fdf2b35dc842ffb0b03336dd06 Mon Sep 17 00:00:00 2001 From: Raphael Rialland Date: Mon, 3 Aug 2026 12:08:27 +0200 Subject: [PATCH 5/8] Add multimodal embedding support protocol Assisted-by: Codex Signed-off-by: Raphael Rialland --- vllm/model_executor/models/__init__.py | 2 ++ vllm/model_executor/models/interfaces.py | 36 ++++++++++++++++--- vllm/model_executor/models/llama4_eagle.py | 6 ++-- vllm/model_executor/models/mistral_eagle.py | 9 ++--- .../models/mistral_large_3_eagle.py | 8 ++--- 5 files changed, 44 insertions(+), 17 deletions(-) diff --git a/vllm/model_executor/models/__init__.py b/vllm/model_executor/models/__init__.py index aa72b8994eaa..478b00d0af33 100644 --- a/vllm/model_executor/models/__init__.py +++ b/vllm/model_executor/models/__init__.py @@ -6,6 +6,7 @@ SupportsLoRA, SupportsMRoPE, SupportsMultiModal, + SupportsMultiModalEmbeddings, SupportsPP, SupportsTranscription, has_inner_state, @@ -36,6 +37,7 @@ "supports_lora", "SupportsMultiModal", "supports_multimodal", + "SupportsMultiModalEmbeddings", "supports_multimodal_embeddings", "SupportsMRoPE", "supports_mrope", diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index e606d353bc08..de2dbf2a47f6 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -110,7 +110,22 @@ def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor: @runtime_checkable -class SupportsMultiModal(Protocol): +class SupportsMultiModalEmbeddings(Protocol): + """The interface for models that can merge external multimodal embeddings.""" + + supports_multimodal_embeddings: ClassVar[Literal[True]] = True + + def embed_input_ids( + self, + input_ids: Tensor, + multimodal_embeddings: MultiModalEmbeddings | None = None, + *, + is_multimodal: Tensor | None = None, + ) -> Tensor: ... + + +@runtime_checkable +class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol): """The interface required for all multi-modal models.""" supports_multimodal: ClassVar[Literal[True]] = True @@ -122,9 +137,6 @@ class SupportsMultiModal(Protocol): MRO of your model class. """ - supports_multimodal_embeddings: ClassVar[Literal[True]] = True - """A flag indicating this model can merge external multimodal embeddings.""" - supports_multimodal_raw_input_only: ClassVar[bool] = False """ A flag that indicates this model supports multi-modal inputs and processes @@ -491,7 +503,21 @@ def supports_multimodal( return getattr(model, "supports_multimodal", False) -def supports_multimodal_embeddings(model: type[object] | object) -> bool: +@overload +def supports_multimodal_embeddings( + model: type[object], +) -> TypeIs[type[SupportsMultiModalEmbeddings]]: ... + + +@overload +def supports_multimodal_embeddings( + model: object, +) -> TypeIs[SupportsMultiModalEmbeddings]: ... + + +def supports_multimodal_embeddings( + model: type[object] | object, +) -> TypeIs[type[SupportsMultiModalEmbeddings]] | TypeIs[SupportsMultiModalEmbeddings]: return getattr(model, "supports_multimodal_embeddings", False) diff --git a/vllm/model_executor/models/llama4_eagle.py b/vllm/model_executor/models/llama4_eagle.py index 336122cd9468..e41b0c5dd283 100644 --- a/vllm/model_executor/models/llama4_eagle.py +++ b/vllm/model_executor/models/llama4_eagle.py @@ -35,7 +35,7 @@ from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM from vllm.model_executor.models.utils import extract_layer_index -from .interfaces import SupportsMultiModal +from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings from .utils import ( AutoWeightsLoader, WeightsMapper, @@ -152,9 +152,7 @@ def pad_layer_name(layer: str) -> str: } -class EagleLlama4ForCausalLM(Llama4ForCausalLM): - supports_multimodal_embeddings = True - +class EagleLlama4ForCausalLM(Llama4ForCausalLM, SupportsMultiModalEmbeddings): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) self.config = vllm_config.speculative_config.draft_model_config.hf_config diff --git a/vllm/model_executor/models/mistral_eagle.py b/vllm/model_executor/models/mistral_eagle.py index c579ceecbe5d..6a85f970aee3 100644 --- a/vllm/model_executor/models/mistral_eagle.py +++ b/vllm/model_executor/models/mistral_eagle.py @@ -14,7 +14,10 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding -from vllm.model_executor.models.interfaces import MultiModalEmbeddings +from vllm.model_executor.models.interfaces import ( + MultiModalEmbeddings, + SupportsMultiModalEmbeddings, +) from vllm.model_executor.models.llama import LlamaConfig from vllm.model_executor.models.mistral import ( MistralDecoderLayer, @@ -109,9 +112,7 @@ def forward( return hidden_states, hidden_states -class EagleMistralForCausalLM(MistralForCausalLM): - supports_multimodal_embeddings = True - +class EagleMistralForCausalLM(MistralForCausalLM, SupportsMultiModalEmbeddings): mistral_mapping = MistralForCausalLM.mistral_mapping | { "eagle_linear": "model.fc", } diff --git a/vllm/model_executor/models/mistral_large_3_eagle.py b/vllm/model_executor/models/mistral_large_3_eagle.py index 605095ced8ca..d194536bebab 100644 --- a/vllm/model_executor/models/mistral_large_3_eagle.py +++ b/vllm/model_executor/models/mistral_large_3_eagle.py @@ -22,7 +22,7 @@ ) from vllm.model_executor.models.mistral_large_3 import MistralLarge3ForCausalLM -from .interfaces import SupportsMultiModal +from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings from .utils import WeightsMapper, make_empty_intermediate_tensors_factory, maybe_prefix logger = init_logger(__name__) @@ -107,9 +107,9 @@ def forward( return output -class EagleMistralLarge3ForCausalLM(MistralLarge3ForCausalLM): - supports_multimodal_embeddings = True - +class EagleMistralLarge3ForCausalLM( + MistralLarge3ForCausalLM, SupportsMultiModalEmbeddings +): hf_to_vllm_mapper = MistralLarge3ForCausalLM.hf_to_vllm_mapper | WeightsMapper( orig_to_new_regex={ regex.compile(r"\Aeagle_linear\.weight\Z"): r"model.fc.weight", From 6ad6f392e7a32714636708c6825096ea2ae17b16 Mon Sep 17 00:00:00 2001 From: Raphael Rialland Date: Mon, 3 Aug 2026 12:16:21 +0200 Subject: [PATCH 6/8] Make multimodal embedding contract abstract Assisted-by: Codex Signed-off-by: Raphael Rialland --- vllm/model_executor/models/interfaces.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index de2dbf2a47f6..71028d9caab7 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio +from abc import abstractmethod from collections.abc import ( AsyncGenerator, Callable, @@ -115,6 +116,7 @@ class SupportsMultiModalEmbeddings(Protocol): supports_multimodal_embeddings: ClassVar[Literal[True]] = True + @abstractmethod def embed_input_ids( self, input_ids: Tensor, From 8e2b5f5a80ef752d8a9fedcc2d8252a7f21d198c Mon Sep 17 00:00:00 2001 From: Raphael Rialland Date: Mon, 3 Aug 2026 12:19:54 +0200 Subject: [PATCH 7/8] Match model protocol conventions Assisted-by: Codex Signed-off-by: Raphael Rialland --- vllm/model_executor/models/interfaces.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 71028d9caab7..de2dbf2a47f6 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio -from abc import abstractmethod from collections.abc import ( AsyncGenerator, Callable, @@ -116,7 +115,6 @@ class SupportsMultiModalEmbeddings(Protocol): supports_multimodal_embeddings: ClassVar[Literal[True]] = True - @abstractmethod def embed_input_ids( self, input_ids: Tensor, From 3d13da128811cbe4e3fce95c0f0fb4948ff44095 Mon Sep 17 00:00:00 2001 From: Raphael Rialland Date: Mon, 3 Aug 2026 18:30:32 +0200 Subject: [PATCH 8/8] Centralize draft multimodal capability detection Assisted-by: Codex Signed-off-by: Raphael Rialland --- .../test_gpu_autoregressive_speculator.py | 101 +++++++++++++++--- vllm/models/inkling/amd/mtp.py | 3 +- vllm/models/inkling/nvidia/mtp.py | 3 +- .../spec_decode/autoregressive/speculator.py | 27 ++--- .../multi_module_mtp/speculator.py | 46 ++++---- vllm/v1/worker/gpu/spec_decode/speculator.py | 17 +++ 6 files changed, 137 insertions(+), 60 deletions(-) diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_autoregressive_speculator.py index 66e156b45733..9f5f90a9775e 100644 --- a/tests/v1/worker/test_gpu_autoregressive_speculator.py +++ b/tests/v1/worker/test_gpu_autoregressive_speculator.py @@ -16,16 +16,20 @@ from vllm.model_executor.models.mistral_large_3_eagle import ( EagleMistralLarge3ForCausalLM, ) +from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( AutoRegressiveSpeculator, ) +from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import ( + MultiModuleMTPSpeculator, +) from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator class _TestSpeculator(AutoRegressiveSpeculator): def load_draft_model(self, target_model, target_attn_layer_names): - raise NotImplementedError + return self.test_draft_model class _DraftModel(torch.nn.Module): @@ -61,6 +65,19 @@ def embed_input_ids( raise AssertionError("embed_input_ids should not be called during loading") +def _mock_base_model_load(monkeypatch): + monkeypatch.setattr( + base_spec_module, + "get_layers_from_vllm_config", + lambda *args, **kwargs: {}, + ) + monkeypatch.setattr( + DraftModelSpeculator, + "_validate_local_argmax_reduction", + lambda self: None, + ) + + def _make_speculator( monkeypatch, output: torch.Tensor | tuple[torch.Tensor, torch.Tensor], @@ -83,17 +100,21 @@ def _make_speculator( return speculator -def test_mm_support_uses_target_config(monkeypatch): +def test_mm_support_configured_after_model_load(monkeypatch): target_model_config = object() draft_model_config = object() vllm_config = SimpleNamespace(model_config=target_model_config) + draft_model = _MultimodalDraftModel() def init_base(speculator, vllm_config, device): + speculator.vllm_config = vllm_config + speculator.device = device speculator.max_num_tokens = 4 speculator.max_num_reqs = 2 speculator.hidden_size = 3 speculator.dtype = torch.float32 speculator.draft_model_config = draft_model_config + speculator.supports_mm_inputs = False checked_configs = [] @@ -102,46 +123,68 @@ def supports_multimodal_inputs(model_config): return True monkeypatch.setattr(DraftModelSpeculator, "__init__", init_base) + _mock_base_model_load(monkeypatch) monkeypatch.setattr( - spec_module.MULTIMODAL_REGISTRY, + base_spec_module.MULTIMODAL_REGISTRY, "supports_multimodal_inputs", supports_multimodal_inputs, ) speculator = _TestSpeculator(vllm_config, torch.device("cpu")) + assert checked_configs == [] + assert not speculator.supports_mm_inputs + assert speculator.inputs_embeds is None + + speculator.test_draft_model = draft_model + speculator.load_model(torch.nn.Module()) + assert checked_configs == [target_model_config] assert speculator.supports_mm_inputs + assert speculator.inputs_embeds is not None assert speculator.inputs_embeds.shape == (4, 3) def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch): speculator = object.__new__(_TestSpeculator) - speculator.supports_mm_inputs = True + speculator.supports_mm_inputs = False + speculator.inputs_embeds = None + speculator.vllm_config = SimpleNamespace(model_config=object()) + speculator.max_num_tokens = 4 + speculator.hidden_size = 3 + speculator.dtype = torch.float32 + speculator.device = torch.device("cpu") draft_model = _MultimodalDraftModel() + speculator.test_draft_model = draft_model + _mock_base_model_load(monkeypatch) monkeypatch.setattr( - DraftModelSpeculator, - "load_model", - lambda self, target_model: setattr(self, "model", draft_model), + base_spec_module.MULTIMODAL_REGISTRY, + "supports_multimodal_inputs", + lambda model_config: True, ) speculator.load_model(torch.nn.Module()) assert speculator.supports_mm_inputs + assert speculator.inputs_embeds is not None def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch): speculator = object.__new__(_TestSpeculator) - speculator.supports_mm_inputs = True + speculator.supports_mm_inputs = False + speculator.inputs_embeds = None + speculator.vllm_config = SimpleNamespace(model_config=object()) draft_model = _TextOnlyDraftModel() + speculator.test_draft_model = draft_model warning_messages = [] + _mock_base_model_load(monkeypatch) monkeypatch.setattr( - DraftModelSpeculator, - "load_model", - lambda self, target_model: setattr(self, "model", draft_model), + base_spec_module.MULTIMODAL_REGISTRY, + "supports_multimodal_inputs", + lambda model_config: True, ) monkeypatch.setattr( - spec_module.logger, + base_spec_module.logger, "warning_once", lambda message, *args: warning_messages.append(message % args), ) @@ -156,6 +199,40 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch): ] +def test_multi_module_mm_support_configured_after_model_load(monkeypatch): + speculator = object.__new__(MultiModuleMTPSpeculator) + speculator.supports_mm_inputs = False + speculator.inputs_embeds = None + speculator.cached_draft_input_embeds = None + speculator.vllm_config = SimpleNamespace(model_config=object()) + speculator.max_num_tokens = 4 + speculator.max_num_reqs = 2 + speculator.num_speculative_steps = 3 + speculator.hidden_size = 3 + speculator.dtype = torch.float32 + speculator.device = torch.device("cpu") + draft_model = _MultimodalDraftModel() + _mock_base_model_load(monkeypatch) + monkeypatch.setattr( + MultiModuleMTPSpeculator, + "load_draft_model", + lambda self, target_model, target_attn_layer_names: draft_model, + ) + monkeypatch.setattr( + base_spec_module.MULTIMODAL_REGISTRY, + "supports_multimodal_inputs", + lambda model_config: True, + ) + + speculator.load_model(torch.nn.Module()) + + assert speculator.supports_mm_inputs + assert speculator.inputs_embeds is not None + assert speculator.inputs_embeds.shape == (4, 3) + assert speculator.cached_draft_input_embeds is not None + assert speculator.cached_draft_input_embeds.shape == (2, 2, 3) + + @pytest.mark.parametrize( ("model_cls", "expected"), [ diff --git a/vllm/models/inkling/amd/mtp.py b/vllm/models/inkling/amd/mtp.py index 34cde020ed38..5a6ba23b4f51 100644 --- a/vllm/models/inkling/amd/mtp.py +++ b/vllm/models/inkling/amd/mtp.py @@ -29,6 +29,7 @@ is_mtp_completeness_check_enabled, ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings from vllm.model_executor.models.utils import maybe_prefix from vllm.sequence import IntermediateTensors @@ -213,7 +214,7 @@ def forward( return hidden -class InklingMTP(nn.Module): +class InklingMTP(nn.Module, SupportsMultiModalEmbeddings): def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: super().__init__() assert vllm_config.speculative_config is not None diff --git a/vllm/models/inkling/nvidia/mtp.py b/vllm/models/inkling/nvidia/mtp.py index 197894beb915..f9cba37f3c71 100644 --- a/vllm/models/inkling/nvidia/mtp.py +++ b/vllm/models/inkling/nvidia/mtp.py @@ -33,6 +33,7 @@ is_mtp_completeness_check_enabled, ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings from vllm.model_executor.models.utils import maybe_prefix from vllm.sequence import IntermediateTensors @@ -242,7 +243,7 @@ def forward( return hidden -class InklingMTP(nn.Module): +class InklingMTP(nn.Module, SupportsMultiModalEmbeddings): def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: super().__init__() assert vllm_config.speculative_config is not None diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index b275985acd8c..7dbd81125325 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -9,8 +9,6 @@ from vllm.config.compilation import CUDAGraphMode from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.logger import init_logger -from vllm.model_executor.models import supports_multimodal_embeddings -from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.triton_utils import tl, triton from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer from vllm.v1.worker.gpu.cudagraph_utils import ( @@ -39,15 +37,7 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): self.max_num_reqs, dtype=torch.int64, device=device ) - # Draft heads reuse embeddings produced by the target's multimodal encoder - # and commonly have text-only checkpoint configs. - self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs( - vllm_config.model_config - ) - if self.supports_mm_inputs: - self.inputs_embeds = torch.zeros( - self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device - ) + self.inputs_embeds: torch.Tensor | None = None self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None @@ -57,14 +47,12 @@ def load_model(self, target_model: nn.Module) -> None: if not self.supports_mm_inputs: return - if not supports_multimodal_embeddings(self.model): - logger.warning_once( - "Draft model %s does not support external multimodal embeddings. " - "Embeddings from the target model will not be passed to the " - "drafter; using text-only draft inputs instead.", - type(self.model).__name__, - ) - self.supports_mm_inputs = False + self.inputs_embeds = torch.zeros( + self.max_num_tokens, + self.hidden_size, + dtype=self.dtype, + device=self.device, + ) @property def advance_draft_positions(self) -> bool: @@ -313,6 +301,7 @@ def _run_model( ): inputs_embeds = None if self.supports_mm_inputs: + assert self.inputs_embeds is not None # Merge multimodal embeddings with input ids. mm_embeds, is_mm_embed = mm_inputs or (None, None) num_input_tokens = ( diff --git a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py index 7fda70a8019a..4be22b4474c6 100644 --- a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py @@ -9,7 +9,6 @@ from vllm.config.compilation import CUDAGraphMode from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.logger import init_logger -from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.utils import PAD_SLOT_ID from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer @@ -39,25 +38,7 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): self.max_num_reqs, dtype=torch.int64, device=device ) - self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs( - self.draft_model_config - ) - # HACK: the Inkling MTP draft has no MM processor of its own (its draft - # config is flattened text-only), but it consumes the target's merged - # embeddings at draft prefill — treat it as MM-capable whenever the - # target is. - if ( - not self.supports_mm_inputs - and self.draft_model_config.hf_config.model_type == "inkling_mtp" - ): - self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs( - vllm_config.model_config - ) self.inputs_embeds: torch.Tensor | None = None - if self.supports_mm_inputs: - self.inputs_embeds = torch.zeros( - self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device - ) # Input id overrides for the last num_speculative_steps - 1 draft steps. # Used by chunked-prefilling requests to swap in the future prefill token @@ -80,14 +61,6 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): device=self.device, ) self.cached_draft_input_embeds: torch.Tensor | None = None - if self.supports_mm_inputs: - self.cached_draft_input_embeds = torch.zeros( - self.max_num_reqs, - self.num_speculative_steps - 1, - self.hidden_size, - dtype=self.dtype, - device=self.device, - ) self.cached_target_hidden_states = torch.zeros( self.max_num_reqs, self.num_speculative_steps - 1, @@ -105,6 +78,25 @@ def load_draft_model( ) -> nn.Module: return load_eagle_model(target_model, self.vllm_config) + def load_model(self, target_model: nn.Module) -> None: + super().load_model(target_model) + if not self.supports_mm_inputs: + return + + self.inputs_embeds = torch.zeros( + self.max_num_tokens, + self.hidden_size, + dtype=self.dtype, + device=self.device, + ) + self.cached_draft_input_embeds = torch.zeros( + self.max_num_reqs, + self.num_speculative_steps - 1, + self.hidden_size, + dtype=self.dtype, + device=self.device, + ) + def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: # TODO(TheEpicDolphin): Support piecewise cudagraph for multi-module MTP. if cudagraph_mode.has_piecewise_cudagraphs(): diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index 34f9925940c8..71eb412ba201 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -13,6 +13,8 @@ from vllm.distributed.eplb.eplb_state import EplbState from vllm.logger import init_logger from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.model_executor.models import supports_multimodal_embeddings +from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.attn_utils import ( build_attn_metadata, @@ -137,6 +139,8 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): device=device, ) + self.supports_mm_inputs = False + @abstractmethod def load_draft_model( self, @@ -164,6 +168,19 @@ def load_model(self, target_model: nn.Module) -> None: ) self.draft_attn_layer_names = all_attn_layers - target_attn_layer_names + target_supports_mm = MULTIMODAL_REGISTRY.supports_multimodal_inputs( + self.vllm_config.model_config + ) + draft_supports_mm = supports_multimodal_embeddings(self.model) + self.supports_mm_inputs = target_supports_mm and draft_supports_mm + if target_supports_mm and not draft_supports_mm: + logger.warning_once( + "Draft model %s does not support external multimodal embeddings. " + "Embeddings from the target model will not be passed to the " + "drafter; using text-only draft inputs instead.", + type(self.model).__name__, + ) + def set_eplb_state(self, eplb_state: EplbState) -> None: """Inject EPLB state after construction.""" self.eplb_state = eplb_state