diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 0b69ac8f4753..28c72d20d511 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -489,6 +489,7 @@ Some models are supported only via the [Transformers modeling backend](#transfor | `Jais2ForCausalLM` | Jais2 | `inceptionai/Jais-2-8B-Chat`, `inceptionai/Jais-2-70B-Chat`, etc. | ✅︎ | ✅︎ | | `Kolibri1ForCausalLM` | Kolibri 1 | `Aleph-Alpha/Kolibri-1`, `Aleph-Alpha/Kolibri-1-BF16`, etc. | ✅︎ | ✅︎ | | `MellumForCausalLM` | Mellum 2 | `JetBrains/Mellum2-12B-A2.5B-Base`, etc. | ✅︎ | ✅︎ | +| `ModernBertDecoderForCausalLM` | Ettin (decoder) | `jhu-clsp/ettin-decoder-17m`, `jhu-clsp/ettin-decoder-150m`, `jhu-clsp/ettin-decoder-1b`, etc. | ✅︎ | ✅︎ | | `NanbeigeForCausalLM` | Nanbeige4.2 | `Nanbeige/Nanbeige4.2-3B`, etc. | ✅︎ | ✅︎ | | `OlmoForCausalLM` | OLMo | `allenai/OLMo-1B-hf`, `allenai/OLMo-7B-hf`, etc. | ✅︎ | ✅︎ | | `Olmo2ForCausalLM` | OLMo2 | `allenai/OLMo-2-0425-1B`, etc. | ✅︎ | ✅︎ | diff --git a/tests/models/registry.py b/tests/models/registry.py index dbb97991a247..18866500f218 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -443,6 +443,9 @@ def check_available_online( "tiny-random": "TitanML/tiny-mixtral", }, ), + "ModernBertDecoderForCausalLM": _HfExamplesInfo( + "jhu-clsp/ettin-decoder-150m", {"tiny": "jhu-clsp/ettin-decoder-17m"} + ), "NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"), "NemotronHForCausalLM": _HfExamplesInfo( "nvidia/Nemotron-H-8B-Base-8K", trust_remote_code=True diff --git a/tests/models/transformers/test_backend.py b/tests/models/transformers/test_backend.py index cce66d73aa1a..4d6ed21efdc3 100644 --- a/tests/models/transformers/test_backend.py +++ b/tests/models/transformers/test_backend.py @@ -18,6 +18,10 @@ from vllm.config import ModelConfig, VllmConfig from vllm.model_executor.models.interfaces import SupportsMultiModal from vllm.model_executor.models.transformers.base import Base +from vllm.model_executor.models.transformers.causal import ( + LM_HEAD_TRANSFORM_ATTR, + CausalMixin, +) from vllm.model_executor.models.transformers.fusers import AttentionFuser from vllm.model_executor.models.transformers.fusers.attention import VLLM_ATTN_IMPL from vllm.model_executor.models.transformers.multimodal import MultiModalMixin @@ -134,6 +138,25 @@ def test_models( ) +def test_output_head_transform( + hf_runner: type[HfRunner], + vllm_runner: type[VllmRunner], + example_prompts: list[str], +) -> None: + """ModernBERT decoder passes the hidden states through a head of its own + (dense -> act -> norm) before its output embeddings, which have a bias. + + Compared in float32 because the tiny model has near-tied logits in bfloat16.""" + check_implementation( + hf_runner, + vllm_runner, + example_prompts, + "jhu-clsp/ettin-decoder-17m", + num_fused=(0, 1), # `Wi` already holds the input and gate projections + dtype="float32", + ) + + def test_hybrid_attention(vllm_runner: type[VllmRunner]) -> None: prompts, _, _ = prep_prompts(4, (800, 801)) kwargs_ref = {"max_model_len": 8192, "enforce_eager": True} @@ -629,6 +652,44 @@ def test_marking_skipped_without_tokenizer(): assert not isinstance(vision_tower, StageMissingLayer) +class OutputHeadStub: + """Just enough of the backend to find a model's output head.""" + + config: PreTrainedConfig + model_config: Any + model: nn.Module + _output_head = CausalMixin._output_head + _rename_output_head = CausalMixin._rename_output_head + + +@pytest.mark.parametrize( + ("model_type", "embeddings", "transform"), + [ + ("llama", "lm_head", None), # does not override `get_output_embeddings` + ("gpt_neox", "lm_head", None), # overrides it, but the head is plain + ("modernbert-decoder", "decoder", "lm_head"), # dense -> act -> norm + ], +) +def test_output_head(model_type: str, embeddings: str, transform: str | None): + """Output heads which transform the hidden states are found and renamed.""" + stub = OutputHeadStub() + stub.config = AutoConfig.for_model(model_type, num_hidden_layers=1) + stub.model_config = SimpleNamespace(dtype=torch.float32) + with torch.device("meta"): + stub.model = AutoModel.from_config(stub.config) + + output_head = stub._output_head + assert (output_head.embeddings, output_head.transform) == (embeddings, transform) + assert stub._rename_output_head(f"{embeddings}.bias") == "lm_head.bias" + if transform is not None: + assert isinstance(output_head.transform_module, nn.Module) + renamed = stub._rename_output_head(f"{transform}.dense.weight") + assert renamed == f"{LM_HEAD_TRANSFORM_ATTR}.dense.weight" + assert stub._rename_output_head("model.layers.0.mlp.Wo.weight") == ( + "model.layers.0.mlp.Wo.weight" + ) + + NUM_LAYERS = 4 diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 23e67c9f506c..52e3bbeccfbf 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -717,6 +717,7 @@ "Jais2ForCausalLM": ("transformers", "TransformersForCausalLM"), "Kolibri1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"), "MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"), + "ModernBertDecoderForCausalLM": ("transformers", "TransformersForCausalLM"), "NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"), "OlmoForCausalLM": ("transformers", "TransformersForCausalLM"), "Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"), diff --git a/vllm/model_executor/models/transformers/causal.py b/vllm/model_executor/models/transformers/causal.py index 9124784d3621..178c7ba75575 100644 --- a/vllm/model_executor/models/transformers/causal.py +++ b/vllm/model_executor/models/transformers/causal.py @@ -17,10 +17,15 @@ """Transformers modeling backend mixin for causal language models.""" from collections.abc import Iterable -from typing import TYPE_CHECKING +from functools import cached_property +from typing import TYPE_CHECKING, NamedTuple import torch +from torch import nn +from transformers import AutoModelForCausalLM, PreTrainedModel +from transformers.models.auto.auto_factory import _get_model_class +from vllm.logger import init_logger from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -34,6 +39,21 @@ if TYPE_CHECKING: from vllm.config import VllmConfig +logger = init_logger(__name__) + +LM_HEAD_TRANSFORM_ATTR = "lm_head_transform" +"""Attribute in which to store the module that transforms the hidden states before the +output embeddings, for models which have one (see `CausalMixin._output_head`).""" + + +class OutputHead(NamedTuple): + embeddings: str = "lm_head" + """Name of the output embeddings in the Transformers `ForCausalLM` class.""" + transform: str | None = None + """Name of the module that transforms the hidden states before the output + embeddings, if the model has one.""" + transform_module: nn.Module | None = None + class CausalMixin(VllmModelForTextGeneration, Base): def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): @@ -62,16 +82,81 @@ def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): scale=getattr(self.text_config, "logit_scale", 1.0), soft_cap=getattr(self.text_config, "final_logit_softcapping", None), ) + + if (transform := self._output_head.transform_module) is not None: + self.init_parameters(transform) + setattr(self, LM_HEAD_TRANSFORM_ATTR, transform) else: self.lm_head = PPMissingLayer() + @cached_property + def _output_head(self) -> OutputHead: + """Find the output head of the model's Transformers `ForCausalLM` class. + + Most models project the hidden states with an output embedding called + `lm_head`. Some (e.g. ModernBERT decoder) first pass them through a module + of their own (dense -> activation -> norm) and name the output embeddings + differently. Only classes which override `get_output_embeddings` can do + this, so the `ForCausalLM` class is only built (on `meta`) for those. + """ + mapping = AutoModelForCausalLM._model_mapping + if type(self.config) not in mapping: + return OutputHead() + model_class = _get_model_class(self.config, mapping) + if model_class.get_output_embeddings is PreTrainedModel.get_output_embeddings: + return OutputHead() + + with torch.device("meta"): + causal_lm = model_class._from_config( + self.config, dtype=self.model_config.dtype + ) + output_embeddings = causal_lm.get_output_embeddings() + children = {m: n for n, m in causal_lm.named_children()} + if output_embeddings not in children: + return OutputHead() + + # Modules which are neither the base model nor the output embeddings + base_model_class = type(self.model) + others = [ + (name, module) + for name, module in causal_lm.named_children() + if module is not output_embeddings + and not isinstance(module, base_model_class) + and next(module.parameters(), None) is not None + ] + if len(others) > 1: + logger.warning_once( + "%s has more than one module besides its base model and output " + "embeddings (%s). The Transformers modeling backend cannot tell the " + "order in which they are applied, so it will ignore them.", + model_class.__name__, + ", ".join(name for name, _ in others), + ) + others = [] + transform, transform_module = others[0] if others else (None, None) + output_head = OutputHead( + children[output_embeddings], transform, transform_module + ) + logger.debug("Identified output head as: %s", output_head) + return output_head + + def _rename_output_head(self, name: str) -> str: + """Rename the output head weights to match this module's attributes.""" + embeddings, transform, _ = self._output_head + if transform is not None and name.startswith(f"{transform}."): + return LM_HEAD_TRANSFORM_ATTR + name.removeprefix(transform) + if embeddings != "lm_head" and name.startswith(f"{embeddings}."): + return "lm_head" + name.removeprefix(embeddings) + return name + def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]: - """A thin wrapper around `Base.load_weights` to handle the lm_head bias.""" + """A thin wrapper around `Base.load_weights` to handle the output head.""" lm_head_bias = set() load_device = self.vllm_config.load_config.device or self.device_config.device def auto_load_lm_head_bias(weights): for name, weight in weights: + name = self._rename_output_head(name) if name.endswith("lm_head.bias") and self.pp_group.is_last_rank: # load_weights is called outside vLLM's device context, # so we must enter it again when registering the bias. @@ -85,5 +170,7 @@ def auto_load_lm_head_bias(weights): return super().load_weights(auto_load_lm_head_bias(weights)) | lm_head_bias def compute_logits(self, hidden_states: "torch.Tensor") -> "torch.Tensor | None": + if (transform := getattr(self, LM_HEAD_TRANSFORM_ATTR, None)) is not None: + hidden_states = transform(hidden_states) logits = self.logits_processor(self.lm_head, hidden_states, self.lm_head.bias) return logits