Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
200 changes: 199 additions & 1 deletion tests/v1/worker/test_gpu_autoregressive_speculator.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,32 @@
from contextlib import nullcontext
from types import SimpleNamespace

import pytest
import torch

from vllm.config.compilation import CUDAGraphMode
from vllm.model_executor.models import supports_multimodal_embeddings
from vllm.model_executor.models.exaone4_5_mtp import Exaone4_5_MTP
from vllm.model_executor.models.llama4_eagle import EagleLlama4ForCausalLM
from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
from vllm.model_executor.models.mistral_eagle import EagleMistralForCausalLM
from vllm.model_executor.models.mistral_large_3_eagle import (
EagleMistralLarge3ForCausalLM,
)
from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module
from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import (
MultiModuleMTPSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator


class _TestSpeculator(AutoRegressiveSpeculator):
def load_draft_model(self, target_model, target_attn_layer_names):
raise NotImplementedError
return self.test_draft_model


class _DraftModel(torch.nn.Module):
Expand All @@ -27,6 +41,43 @@ def forward(self, **kwargs):
return self.output


class _MultimodalDraftModel(torch.nn.Module):
supports_multimodal_embeddings = True

def embed_input_ids(
self,
input_ids,
multimodal_embeddings=None,
*,
is_multimodal=None,
):
raise AssertionError("embed_input_ids should not be called during loading")


class _TextOnlyDraftModel(torch.nn.Module):
def embed_input_ids(
self,
input_ids,
multimodal_embeddings=None,
*,
is_multimodal=None,
):
raise AssertionError("embed_input_ids should not be called during loading")


def _mock_base_model_load(monkeypatch):
monkeypatch.setattr(
base_spec_module,
"get_layers_from_vllm_config",
lambda *args, **kwargs: {},
)
monkeypatch.setattr(
DraftModelSpeculator,
"_validate_local_argmax_reduction",
lambda self: None,
)


def _make_speculator(
monkeypatch,
output: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
Expand All @@ -49,6 +100,153 @@ def _make_speculator(
return speculator


def test_mm_support_configured_after_model_load(monkeypatch):
target_model_config = object()
draft_model_config = object()
vllm_config = SimpleNamespace(model_config=target_model_config)
draft_model = _MultimodalDraftModel()

def init_base(speculator, vllm_config, device):
speculator.vllm_config = vllm_config
speculator.device = device
speculator.max_num_tokens = 4
speculator.max_num_reqs = 2
speculator.hidden_size = 3
speculator.dtype = torch.float32
speculator.draft_model_config = draft_model_config
speculator.supports_mm_inputs = False

checked_configs = []

def supports_multimodal_inputs(model_config):
checked_configs.append(model_config)
return True

monkeypatch.setattr(DraftModelSpeculator, "__init__", init_base)
_mock_base_model_load(monkeypatch)
monkeypatch.setattr(
base_spec_module.MULTIMODAL_REGISTRY,
"supports_multimodal_inputs",
supports_multimodal_inputs,
)

speculator = _TestSpeculator(vllm_config, torch.device("cpu"))

assert checked_configs == []
assert not speculator.supports_mm_inputs
assert speculator.inputs_embeds is None

speculator.test_draft_model = draft_model
speculator.load_model(torch.nn.Module())

assert checked_configs == [target_model_config]
assert speculator.supports_mm_inputs
assert speculator.inputs_embeds is not None
assert speculator.inputs_embeds.shape == (4, 3)


def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch):
speculator = object.__new__(_TestSpeculator)
speculator.supports_mm_inputs = False
speculator.inputs_embeds = None
speculator.vllm_config = SimpleNamespace(model_config=object())
speculator.max_num_tokens = 4
speculator.hidden_size = 3
speculator.dtype = torch.float32
speculator.device = torch.device("cpu")
draft_model = _MultimodalDraftModel()
speculator.test_draft_model = draft_model
_mock_base_model_load(monkeypatch)
monkeypatch.setattr(
base_spec_module.MULTIMODAL_REGISTRY,
"supports_multimodal_inputs",
lambda model_config: True,
)

speculator.load_model(torch.nn.Module())

assert speculator.supports_mm_inputs
assert speculator.inputs_embeds is not None


def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch):
speculator = object.__new__(_TestSpeculator)
speculator.supports_mm_inputs = False
speculator.inputs_embeds = None
speculator.vllm_config = SimpleNamespace(model_config=object())
draft_model = _TextOnlyDraftModel()
speculator.test_draft_model = draft_model
warning_messages = []
_mock_base_model_load(monkeypatch)
monkeypatch.setattr(
base_spec_module.MULTIMODAL_REGISTRY,
"supports_multimodal_inputs",
lambda model_config: True,
)
monkeypatch.setattr(
base_spec_module.logger,
"warning_once",
lambda message, *args: warning_messages.append(message % args),
)

speculator.load_model(torch.nn.Module())

assert not speculator.supports_mm_inputs
assert warning_messages == [
"Draft model _TextOnlyDraftModel does not support external multimodal "
"embeddings. Embeddings from the target model will not be passed to the "
"drafter; using text-only draft inputs instead."
]


def test_multi_module_mm_support_configured_after_model_load(monkeypatch):
speculator = object.__new__(MultiModuleMTPSpeculator)
speculator.supports_mm_inputs = False
speculator.inputs_embeds = None
speculator.cached_draft_input_embeds = None
speculator.vllm_config = SimpleNamespace(model_config=object())
speculator.max_num_tokens = 4
speculator.max_num_reqs = 2
speculator.num_speculative_steps = 3
speculator.hidden_size = 3
speculator.dtype = torch.float32
speculator.device = torch.device("cpu")
draft_model = _MultimodalDraftModel()
_mock_base_model_load(monkeypatch)
monkeypatch.setattr(
MultiModuleMTPSpeculator,
"load_draft_model",
lambda self, target_model, target_attn_layer_names: draft_model,
)
monkeypatch.setattr(
base_spec_module.MULTIMODAL_REGISTRY,
"supports_multimodal_inputs",
lambda model_config: True,
)

speculator.load_model(torch.nn.Module())

assert speculator.supports_mm_inputs
assert speculator.inputs_embeds is not None
assert speculator.inputs_embeds.shape == (4, 3)
assert speculator.cached_draft_input_embeds is not None
assert speculator.cached_draft_input_embeds.shape == (2, 2, 3)


@pytest.mark.parametrize(
("model_cls", "expected"),
[
(EagleLlama4ForCausalLM, True),
(EagleMistralForCausalLM, True),
(EagleMistralLarge3ForCausalLM, True),
(Exaone4_5_MTP, True),
(Eagle3LlamaForCausalLM, False),
],
)
def test_draft_model_multimodal_embedding_capability(model_cls, expected):
assert supports_multimodal_embeddings(model_cls) is expected


def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch):
logits_hidden = torch.full((4, 3), 1.0)
feedback_hidden = torch.full((4, 3), 2.0)
Expand Down
4 changes: 4 additions & 0 deletions vllm/model_executor/models/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,14 @@
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
SupportsMultiModalEmbeddings,
SupportsPP,
SupportsTranscription,
has_inner_state,
supports_lora,
supports_mrope,
supports_multimodal,
supports_multimodal_embeddings,
supports_pp,
supports_transcription,
)
Expand All @@ -35,6 +37,8 @@
"supports_lora",
"SupportsMultiModal",
"supports_multimodal",
"SupportsMultiModalEmbeddings",
"supports_multimodal_embeddings",
"SupportsMRoPE",
"supports_mrope",
"SupportsPP",
Expand Down
35 changes: 34 additions & 1 deletion vllm/model_executor/models/interfaces.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,22 @@ def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:


@runtime_checkable
class SupportsMultiModal(Protocol):
class SupportsMultiModalEmbeddings(Protocol):
"""The interface for models that can merge external multimodal embeddings."""

supports_multimodal_embeddings: ClassVar[Literal[True]] = True

def embed_input_ids(
self,
input_ids: Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: Tensor | None = None,
) -> Tensor: ...


@runtime_checkable
class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
"""The interface required for all multi-modal models."""

supports_multimodal: ClassVar[Literal[True]] = True
Expand Down Expand Up @@ -488,6 +503,24 @@ def supports_multimodal(
return getattr(model, "supports_multimodal", False)


@overload
def supports_multimodal_embeddings(
model: type[object],
) -> TypeIs[type[SupportsMultiModalEmbeddings]]: ...


@overload
def supports_multimodal_embeddings(
model: object,
) -> TypeIs[SupportsMultiModalEmbeddings]: ...


def supports_multimodal_embeddings(
model: type[object] | object,
) -> TypeIs[type[SupportsMultiModalEmbeddings]] | TypeIs[SupportsMultiModalEmbeddings]:
return getattr(model, "supports_multimodal_embeddings", False)


def supports_multimodal_raw_input_only(model: type[object] | object) -> bool:
return getattr(model, "supports_multimodal_raw_input_only", False)

Expand Down
4 changes: 2 additions & 2 deletions vllm/model_executor/models/llama4_eagle.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM
from vllm.model_executor.models.utils import extract_layer_index

from .interfaces import SupportsMultiModal
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
from .utils import (
AutoWeightsLoader,
WeightsMapper,
Expand Down Expand Up @@ -152,7 +152,7 @@ def pad_layer_name(layer: str) -> str:
}


class EagleLlama4ForCausalLM(Llama4ForCausalLM):
class EagleLlama4ForCausalLM(Llama4ForCausalLM, SupportsMultiModalEmbeddings):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self)
self.config = vllm_config.speculative_config.draft_model_config.hf_config
Expand Down
7 changes: 5 additions & 2 deletions vllm/model_executor/models/mistral_eagle.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models.interfaces import MultiModalEmbeddings
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsMultiModalEmbeddings,
)
from vllm.model_executor.models.llama import LlamaConfig
from vllm.model_executor.models.mistral import (
MistralDecoderLayer,
Expand Down Expand Up @@ -109,7 +112,7 @@ def forward(
return hidden_states, hidden_states


class EagleMistralForCausalLM(MistralForCausalLM):
class EagleMistralForCausalLM(MistralForCausalLM, SupportsMultiModalEmbeddings):
mistral_mapping = MistralForCausalLM.mistral_mapping | {
"eagle_linear": "model.fc",
}
Expand Down
6 changes: 4 additions & 2 deletions vllm/model_executor/models/mistral_large_3_eagle.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
)
from vllm.model_executor.models.mistral_large_3 import MistralLarge3ForCausalLM

from .interfaces import SupportsMultiModal
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
from .utils import WeightsMapper, make_empty_intermediate_tensors_factory, maybe_prefix

logger = init_logger(__name__)
Expand Down Expand Up @@ -107,7 +107,9 @@ def forward(
return output


class EagleMistralLarge3ForCausalLM(MistralLarge3ForCausalLM):
class EagleMistralLarge3ForCausalLM(
MistralLarge3ForCausalLM, SupportsMultiModalEmbeddings
):
hf_to_vllm_mapper = MistralLarge3ForCausalLM.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_regex={
regex.compile(r"\Aeagle_linear\.weight\Z"): r"model.fc.weight",
Expand Down
3 changes: 2 additions & 1 deletion vllm/models/inkling/amd/mtp.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
from vllm.model_executor.models.utils import maybe_prefix
from vllm.sequence import IntermediateTensors

Expand Down Expand Up @@ -213,7 +214,7 @@ def forward(
return hidden


class InklingMTP(nn.Module):
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
Expand Down
3 changes: 2 additions & 1 deletion vllm/models/inkling/nvidia/mtp.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
from vllm.model_executor.models.utils import maybe_prefix
from vllm.sequence import IntermediateTensors

Expand Down Expand Up @@ -242,7 +243,7 @@ def forward(
return hidden


class InklingMTP(nn.Module):
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
Expand Down
Loading
Loading