From 181a07b08747adcef66a0d0e524210125ab7f215 Mon Sep 17 00:00:00 2001 From: Thore Koritzius Date: Mon, 12 Oct 2026 17:29:56 +0200 Subject: [PATCH 1/2] [Model] Support ModernBERT decoder (Ettin) via the Transformers modeling backend Some Transformers `ForCausalLM` classes pass the hidden states through a module of their own before the output embeddings, and name the output embeddings something other than `lm_head`. ModernBERT decoder (the Ettin decoders) does both: `decoder(lm_head(hidden_states))`, where `lm_head` is dense -> act -> norm and `decoder` is the tied output embedding with a bias. The Transformers modeling backend only builds the base model plus a `ParallelLMHead`, so loading failed with "There is no module or parameter named 'lm_head.dense'". `CausalMixin` now finds the output head from the `ForCausalLM` class: for classes which override `get_output_embeddings` (31 of 178 in Transformers 5.19; the others are skipped without building anything), it builds the class on `meta`, renames the output embeddings' weights to `lm_head` and applies a single extra module, if any, to the hidden states in `compute_logits`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G6Bm8s1uyizXtz9t5ZCZAw Signed-off-by: Thore Koritzius --- docs/models/supported_models.md | 1 + tests/models/registry.py | 3 + tests/models/transformers/test_backend.py | 61 +++++++++++++ vllm/model_executor/models/registry.py | 1 + .../models/transformers/causal.py | 91 ++++++++++++++++++- 5 files changed, 155 insertions(+), 2 deletions(-) diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index bbcb94f32cf5..9db2c970009c 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -443,6 +443,7 @@ th { | `MistralForCausalLM` | Ministral-3, Mistral, Mistral-Instruct | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc. | ✅︎ | ✅︎ | | `MistralLarge3ForCausalLM` | Mistral-Large-3-675B-Base-2512, Mistral-Large-3-675B-Instruct-2512 | `mistralai/Mistral-Large-3-675B-Base-2512`, `mistralai/Mistral-Large-3-675B-Instruct-2512`, etc. | ✅︎ | ✅︎ | | `MixtralForCausalLM` | Mixtral-8x7B, Mixtral-8x7B-Instruct | `mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, `mistral-community/Mixtral-8x22B-v0.1`, etc. | ✅︎ | ✅︎ | +| `ModernBertDecoderForCausalLM` | Ettin (decoder) | `jhu-clsp/ettin-decoder-17m`, `jhu-clsp/ettin-decoder-150m`, `jhu-clsp/ettin-decoder-1b`, etc. | ✅︎ | ✅︎ | | `NemotronForCausalLM` | Nemotron-3, Nemotron-4, Minitron | `nvidia/Minitron-8B-Base`, `mgoin/Nemotron-4-340B-Base-hf-FP8`, etc. | ✅︎ | ✅︎ | | `NemotronHForCausalLM` | Nemotron-H | `nvidia/Nemotron-H-8B-Base-8K`, `nvidia/Nemotron-H-47B-Base-8K`, `nvidia/Nemotron-H-56B-Base-8K`, etc. | ✅︎ | ✅︎ | | `OlmoHybridForCausalLM` | OLMo Hybrid | `allenai/Olmo-Hybrid-7B` | ✅︎ | ✅︎ | diff --git a/tests/models/registry.py b/tests/models/registry.py index 0a7027461ef3..bcb74b670b79 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -436,6 +436,9 @@ def check_available_online( "tiny-random": "TitanML/tiny-mixtral", }, ), + "ModernBertDecoderForCausalLM": _HfExamplesInfo( + "jhu-clsp/ettin-decoder-150m", {"tiny": "jhu-clsp/ettin-decoder-17m"} + ), "NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"), "NemotronHForCausalLM": _HfExamplesInfo( "nvidia/Nemotron-H-8B-Base-8K", trust_remote_code=True diff --git a/tests/models/transformers/test_backend.py b/tests/models/transformers/test_backend.py index cce66d73aa1a..4d6ed21efdc3 100644 --- a/tests/models/transformers/test_backend.py +++ b/tests/models/transformers/test_backend.py @@ -18,6 +18,10 @@ from vllm.config import ModelConfig, VllmConfig from vllm.model_executor.models.interfaces import SupportsMultiModal from vllm.model_executor.models.transformers.base import Base +from vllm.model_executor.models.transformers.causal import ( + LM_HEAD_TRANSFORM_ATTR, + CausalMixin, +) from vllm.model_executor.models.transformers.fusers import AttentionFuser from vllm.model_executor.models.transformers.fusers.attention import VLLM_ATTN_IMPL from vllm.model_executor.models.transformers.multimodal import MultiModalMixin @@ -134,6 +138,25 @@ def test_models( ) +def test_output_head_transform( + hf_runner: type[HfRunner], + vllm_runner: type[VllmRunner], + example_prompts: list[str], +) -> None: + """ModernBERT decoder passes the hidden states through a head of its own + (dense -> act -> norm) before its output embeddings, which have a bias. + + Compared in float32 because the tiny model has near-tied logits in bfloat16.""" + check_implementation( + hf_runner, + vllm_runner, + example_prompts, + "jhu-clsp/ettin-decoder-17m", + num_fused=(0, 1), # `Wi` already holds the input and gate projections + dtype="float32", + ) + + def test_hybrid_attention(vllm_runner: type[VllmRunner]) -> None: prompts, _, _ = prep_prompts(4, (800, 801)) kwargs_ref = {"max_model_len": 8192, "enforce_eager": True} @@ -629,6 +652,44 @@ def test_marking_skipped_without_tokenizer(): assert not isinstance(vision_tower, StageMissingLayer) +class OutputHeadStub: + """Just enough of the backend to find a model's output head.""" + + config: PreTrainedConfig + model_config: Any + model: nn.Module + _output_head = CausalMixin._output_head + _rename_output_head = CausalMixin._rename_output_head + + +@pytest.mark.parametrize( + ("model_type", "embeddings", "transform"), + [ + ("llama", "lm_head", None), # does not override `get_output_embeddings` + ("gpt_neox", "lm_head", None), # overrides it, but the head is plain + ("modernbert-decoder", "decoder", "lm_head"), # dense -> act -> norm + ], +) +def test_output_head(model_type: str, embeddings: str, transform: str | None): + """Output heads which transform the hidden states are found and renamed.""" + stub = OutputHeadStub() + stub.config = AutoConfig.for_model(model_type, num_hidden_layers=1) + stub.model_config = SimpleNamespace(dtype=torch.float32) + with torch.device("meta"): + stub.model = AutoModel.from_config(stub.config) + + output_head = stub._output_head + assert (output_head.embeddings, output_head.transform) == (embeddings, transform) + assert stub._rename_output_head(f"{embeddings}.bias") == "lm_head.bias" + if transform is not None: + assert isinstance(output_head.transform_module, nn.Module) + renamed = stub._rename_output_head(f"{transform}.dense.weight") + assert renamed == f"{LM_HEAD_TRANSFORM_ATTR}.dense.weight" + assert stub._rename_output_head("model.layers.0.mlp.Wo.weight") == ( + "model.layers.0.mlp.Wo.weight" + ) + + NUM_LAYERS = 4 diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 36f9bd6e6fcc..9b2ded8770cc 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -718,6 +718,7 @@ "HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"), "Jais2ForCausalLM": ("transformers", "TransformersForCausalLM"), "MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"), + "ModernBertDecoderForCausalLM": ("transformers", "TransformersForCausalLM"), "NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"), "OlmoForCausalLM": ("transformers", "TransformersForCausalLM"), "Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"), diff --git a/vllm/model_executor/models/transformers/causal.py b/vllm/model_executor/models/transformers/causal.py index 9124784d3621..178c7ba75575 100644 --- a/vllm/model_executor/models/transformers/causal.py +++ b/vllm/model_executor/models/transformers/causal.py @@ -17,10 +17,15 @@ """Transformers modeling backend mixin for causal language models.""" from collections.abc import Iterable -from typing import TYPE_CHECKING +from functools import cached_property +from typing import TYPE_CHECKING, NamedTuple import torch +from torch import nn +from transformers import AutoModelForCausalLM, PreTrainedModel +from transformers.models.auto.auto_factory import _get_model_class +from vllm.logger import init_logger from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -34,6 +39,21 @@ if TYPE_CHECKING: from vllm.config import VllmConfig +logger = init_logger(__name__) + +LM_HEAD_TRANSFORM_ATTR = "lm_head_transform" +"""Attribute in which to store the module that transforms the hidden states before the +output embeddings, for models which have one (see `CausalMixin._output_head`).""" + + +class OutputHead(NamedTuple): + embeddings: str = "lm_head" + """Name of the output embeddings in the Transformers `ForCausalLM` class.""" + transform: str | None = None + """Name of the module that transforms the hidden states before the output + embeddings, if the model has one.""" + transform_module: nn.Module | None = None + class CausalMixin(VllmModelForTextGeneration, Base): def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): @@ -62,16 +82,81 @@ def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): scale=getattr(self.text_config, "logit_scale", 1.0), soft_cap=getattr(self.text_config, "final_logit_softcapping", None), ) + + if (transform := self._output_head.transform_module) is not None: + self.init_parameters(transform) + setattr(self, LM_HEAD_TRANSFORM_ATTR, transform) else: self.lm_head = PPMissingLayer() + @cached_property + def _output_head(self) -> OutputHead: + """Find the output head of the model's Transformers `ForCausalLM` class. + + Most models project the hidden states with an output embedding called + `lm_head`. Some (e.g. ModernBERT decoder) first pass them through a module + of their own (dense -> activation -> norm) and name the output embeddings + differently. Only classes which override `get_output_embeddings` can do + this, so the `ForCausalLM` class is only built (on `meta`) for those. + """ + mapping = AutoModelForCausalLM._model_mapping + if type(self.config) not in mapping: + return OutputHead() + model_class = _get_model_class(self.config, mapping) + if model_class.get_output_embeddings is PreTrainedModel.get_output_embeddings: + return OutputHead() + + with torch.device("meta"): + causal_lm = model_class._from_config( + self.config, dtype=self.model_config.dtype + ) + output_embeddings = causal_lm.get_output_embeddings() + children = {m: n for n, m in causal_lm.named_children()} + if output_embeddings not in children: + return OutputHead() + + # Modules which are neither the base model nor the output embeddings + base_model_class = type(self.model) + others = [ + (name, module) + for name, module in causal_lm.named_children() + if module is not output_embeddings + and not isinstance(module, base_model_class) + and next(module.parameters(), None) is not None + ] + if len(others) > 1: + logger.warning_once( + "%s has more than one module besides its base model and output " + "embeddings (%s). The Transformers modeling backend cannot tell the " + "order in which they are applied, so it will ignore them.", + model_class.__name__, + ", ".join(name for name, _ in others), + ) + others = [] + transform, transform_module = others[0] if others else (None, None) + output_head = OutputHead( + children[output_embeddings], transform, transform_module + ) + logger.debug("Identified output head as: %s", output_head) + return output_head + + def _rename_output_head(self, name: str) -> str: + """Rename the output head weights to match this module's attributes.""" + embeddings, transform, _ = self._output_head + if transform is not None and name.startswith(f"{transform}."): + return LM_HEAD_TRANSFORM_ATTR + name.removeprefix(transform) + if embeddings != "lm_head" and name.startswith(f"{embeddings}."): + return "lm_head" + name.removeprefix(embeddings) + return name + def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]: - """A thin wrapper around `Base.load_weights` to handle the lm_head bias.""" + """A thin wrapper around `Base.load_weights` to handle the output head.""" lm_head_bias = set() load_device = self.vllm_config.load_config.device or self.device_config.device def auto_load_lm_head_bias(weights): for name, weight in weights: + name = self._rename_output_head(name) if name.endswith("lm_head.bias") and self.pp_group.is_last_rank: # load_weights is called outside vLLM's device context, # so we must enter it again when registering the bias. @@ -85,5 +170,7 @@ def auto_load_lm_head_bias(weights): return super().load_weights(auto_load_lm_head_bias(weights)) | lm_head_bias def compute_logits(self, hidden_states: "torch.Tensor") -> "torch.Tensor | None": + if (transform := getattr(self, LM_HEAD_TRANSFORM_ATTR, None)) is not None: + hidden_states = transform(hidden_states) logits = self.logits_processor(self.lm_head, hidden_states, self.lm_head.bias) return logits From 5a1ed1c4fdff8341d0c9b101591a8dfde919f7ea Mon Sep 17 00:00:00 2001 From: Thore Koritzius Date: Mon, 12 Oct 2026 21:30:29 +0200 Subject: [PATCH 2/2] chore: move ModernBertDecoderForCausalLM to transformers models table Signed-off-by: Thore Koritzius --- docs/models/supported_models.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 9db2c970009c..d243c03187ab 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -443,7 +443,6 @@ th { | `MistralForCausalLM` | Ministral-3, Mistral, Mistral-Instruct | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc. | ✅︎ | ✅︎ | | `MistralLarge3ForCausalLM` | Mistral-Large-3-675B-Base-2512, Mistral-Large-3-675B-Instruct-2512 | `mistralai/Mistral-Large-3-675B-Base-2512`, `mistralai/Mistral-Large-3-675B-Instruct-2512`, etc. | ✅︎ | ✅︎ | | `MixtralForCausalLM` | Mixtral-8x7B, Mixtral-8x7B-Instruct | `mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, `mistral-community/Mixtral-8x22B-v0.1`, etc. | ✅︎ | ✅︎ | -| `ModernBertDecoderForCausalLM` | Ettin (decoder) | `jhu-clsp/ettin-decoder-17m`, `jhu-clsp/ettin-decoder-150m`, `jhu-clsp/ettin-decoder-1b`, etc. | ✅︎ | ✅︎ | | `NemotronForCausalLM` | Nemotron-3, Nemotron-4, Minitron | `nvidia/Minitron-8B-Base`, `mgoin/Nemotron-4-340B-Base-hf-FP8`, etc. | ✅︎ | ✅︎ | | `NemotronHForCausalLM` | Nemotron-H | `nvidia/Nemotron-H-8B-Base-8K`, `nvidia/Nemotron-H-47B-Base-8K`, `nvidia/Nemotron-H-56B-Base-8K`, etc. | ✅︎ | ✅︎ | | `OlmoHybridForCausalLM` | OLMo Hybrid | `allenai/Olmo-Hybrid-7B` | ✅︎ | ✅︎ | @@ -488,6 +487,7 @@ Some models are supported only via the [Transformers modeling backend](#transfor | `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ | | `Jais2ForCausalLM` | Jais2 | `inceptionai/Jais-2-8B-Chat`, `inceptionai/Jais-2-70B-Chat`, etc. | ✅︎ | ✅︎ | | `MellumForCausalLM` | Mellum 2 | `JetBrains/Mellum2-12B-A2.5B-Base`, etc. | ✅︎ | ✅︎ | +| `ModernBertDecoderForCausalLM` | Ettin (decoder) | `jhu-clsp/ettin-decoder-17m`, `jhu-clsp/ettin-decoder-150m`, `jhu-clsp/ettin-decoder-1b`, etc. | ✅︎ | ✅︎ | | `NanbeigeForCausalLM` | Nanbeige4.2 | `Nanbeige/Nanbeige4.2-3B`, etc. | ✅︎ | ✅︎ | | `OlmoForCausalLM` | OLMo | `allenai/OLMo-1B-hf`, `allenai/OLMo-7B-hf`, etc. | ✅︎ | ✅︎ | | `Olmo2ForCausalLM` | OLMo2 | `allenai/OLMo-2-0425-1B`, etc. | ✅︎ | ✅︎ |