Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/models/supported_models.md
Comment thread
hmellor marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,7 @@ Some models are supported only via the [Transformers modeling backend](#transfor
| `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ |
| `Jais2ForCausalLM` | Jais2 | `inceptionai/Jais-2-8B-Chat`, `inceptionai/Jais-2-70B-Chat`, etc. | ✅︎ | ✅︎ |
| `MellumForCausalLM` | Mellum 2 | `JetBrains/Mellum2-12B-A2.5B-Base`, etc. | ✅︎ | ✅︎ |
| `ModernBertDecoderForCausalLM` | Ettin (decoder) | `jhu-clsp/ettin-decoder-17m`, `jhu-clsp/ettin-decoder-150m`, `jhu-clsp/ettin-decoder-1b`, etc. | ✅︎ | ✅︎ |
| `NanbeigeForCausalLM` | Nanbeige4.2 | `Nanbeige/Nanbeige4.2-3B`, etc. | ✅︎ | ✅︎ |
| `OlmoForCausalLM` | OLMo | `allenai/OLMo-1B-hf`, `allenai/OLMo-7B-hf`, etc. | ✅︎ | ✅︎ |
| `Olmo2ForCausalLM` | OLMo2 | `allenai/OLMo-2-0425-1B`, etc. | ✅︎ | ✅︎ |
Expand Down
3 changes: 3 additions & 0 deletions tests/models/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,9 @@ def check_available_online(
"tiny-random": "TitanML/tiny-mixtral",
},
),
"ModernBertDecoderForCausalLM": _HfExamplesInfo(
"jhu-clsp/ettin-decoder-150m", {"tiny": "jhu-clsp/ettin-decoder-17m"}
),
"NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"),
"NemotronHForCausalLM": _HfExamplesInfo(
"nvidia/Nemotron-H-8B-Base-8K", trust_remote_code=True
Expand Down
61 changes: 61 additions & 0 deletions tests/models/transformers/test_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
from vllm.config import ModelConfig, VllmConfig
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.transformers.base import Base
from vllm.model_executor.models.transformers.causal import (
LM_HEAD_TRANSFORM_ATTR,
CausalMixin,
)
from vllm.model_executor.models.transformers.fusers import AttentionFuser
from vllm.model_executor.models.transformers.fusers.attention import VLLM_ATTN_IMPL
from vllm.model_executor.models.transformers.multimodal import MultiModalMixin
Expand Down Expand Up @@ -134,6 +138,25 @@ def test_models(
)


def test_output_head_transform(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
example_prompts: list[str],
) -> None:
"""ModernBERT decoder passes the hidden states through a head of its own
(dense -> act -> norm) before its output embeddings, which have a bias.

Compared in float32 because the tiny model has near-tied logits in bfloat16."""
check_implementation(
hf_runner,
vllm_runner,
example_prompts,
"jhu-clsp/ettin-decoder-17m",
num_fused=(0, 1), # `Wi` already holds the input and gate projections
dtype="float32",
)


def test_hybrid_attention(vllm_runner: type[VllmRunner]) -> None:
prompts, _, _ = prep_prompts(4, (800, 801))
kwargs_ref = {"max_model_len": 8192, "enforce_eager": True}
Expand Down Expand Up @@ -629,6 +652,44 @@ def test_marking_skipped_without_tokenizer():
assert not isinstance(vision_tower, StageMissingLayer)


class OutputHeadStub:
"""Just enough of the backend to find a model's output head."""

config: PreTrainedConfig
model_config: Any
model: nn.Module
_output_head = CausalMixin._output_head
_rename_output_head = CausalMixin._rename_output_head


@pytest.mark.parametrize(
("model_type", "embeddings", "transform"),
[
("llama", "lm_head", None), # does not override `get_output_embeddings`
("gpt_neox", "lm_head", None), # overrides it, but the head is plain
("modernbert-decoder", "decoder", "lm_head"), # dense -> act -> norm
],
)
def test_output_head(model_type: str, embeddings: str, transform: str | None):
"""Output heads which transform the hidden states are found and renamed."""
stub = OutputHeadStub()
stub.config = AutoConfig.for_model(model_type, num_hidden_layers=1)
stub.model_config = SimpleNamespace(dtype=torch.float32)
with torch.device("meta"):
stub.model = AutoModel.from_config(stub.config)

output_head = stub._output_head
assert (output_head.embeddings, output_head.transform) == (embeddings, transform)
assert stub._rename_output_head(f"{embeddings}.bias") == "lm_head.bias"
if transform is not None:
assert isinstance(output_head.transform_module, nn.Module)
renamed = stub._rename_output_head(f"{transform}.dense.weight")
assert renamed == f"{LM_HEAD_TRANSFORM_ATTR}.dense.weight"
assert stub._rename_output_head("model.layers.0.mlp.Wo.weight") == (
"model.layers.0.mlp.Wo.weight"
)


NUM_LAYERS = 4


Expand Down
1 change: 1 addition & 0 deletions vllm/model_executor/models/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -718,6 +718,7 @@
"HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"Jais2ForCausalLM": ("transformers", "TransformersForCausalLM"),
"MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"ModernBertDecoderForCausalLM": ("transformers", "TransformersForCausalLM"),
"NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"),
"OlmoForCausalLM": ("transformers", "TransformersForCausalLM"),
"Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"),
Expand Down
91 changes: 89 additions & 2 deletions vllm/model_executor/models/transformers/causal.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,15 @@
"""Transformers modeling backend mixin for causal language models."""

from collections.abc import Iterable
from typing import TYPE_CHECKING
from functools import cached_property
from typing import TYPE_CHECKING, NamedTuple

import torch
from torch import nn
from transformers import AutoModelForCausalLM, PreTrainedModel
from transformers.models.auto.auto_factory import _get_model_class

from vllm.logger import init_logger
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
Expand All @@ -34,6 +39,21 @@
if TYPE_CHECKING:
from vllm.config import VllmConfig

logger = init_logger(__name__)

LM_HEAD_TRANSFORM_ATTR = "lm_head_transform"
"""Attribute in which to store the module that transforms the hidden states before the
output embeddings, for models which have one (see `CausalMixin._output_head`)."""


class OutputHead(NamedTuple):
embeddings: str = "lm_head"
"""Name of the output embeddings in the Transformers `ForCausalLM` class."""
transform: str | None = None
"""Name of the module that transforms the hidden states before the output
embeddings, if the model has one."""
transform_module: nn.Module | None = None


class CausalMixin(VllmModelForTextGeneration, Base):
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
Expand Down Expand Up @@ -62,16 +82,81 @@ def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
scale=getattr(self.text_config, "logit_scale", 1.0),
soft_cap=getattr(self.text_config, "final_logit_softcapping", None),
)

if (transform := self._output_head.transform_module) is not None:
self.init_parameters(transform)
setattr(self, LM_HEAD_TRANSFORM_ATTR, transform)
else:
self.lm_head = PPMissingLayer()

@cached_property
def _output_head(self) -> OutputHead:
"""Find the output head of the model's Transformers `ForCausalLM` class.

Most models project the hidden states with an output embedding called
`lm_head`. Some (e.g. ModernBERT decoder) first pass them through a module
of their own (dense -> activation -> norm) and name the output embeddings
differently. Only classes which override `get_output_embeddings` can do
this, so the `ForCausalLM` class is only built (on `meta`) for those.
"""
mapping = AutoModelForCausalLM._model_mapping
if type(self.config) not in mapping:
return OutputHead()
model_class = _get_model_class(self.config, mapping)
if model_class.get_output_embeddings is PreTrainedModel.get_output_embeddings:
return OutputHead()

with torch.device("meta"):
causal_lm = model_class._from_config(
self.config, dtype=self.model_config.dtype
)
output_embeddings = causal_lm.get_output_embeddings()
children = {m: n for n, m in causal_lm.named_children()}
if output_embeddings not in children:
return OutputHead()

# Modules which are neither the base model nor the output embeddings
base_model_class = type(self.model)
others = [
(name, module)
for name, module in causal_lm.named_children()
if module is not output_embeddings
and not isinstance(module, base_model_class)
and next(module.parameters(), None) is not None
]
if len(others) > 1:
logger.warning_once(
"%s has more than one module besides its base model and output "
"embeddings (%s). The Transformers modeling backend cannot tell the "
"order in which they are applied, so it will ignore them.",
model_class.__name__,
", ".join(name for name, _ in others),
)
others = []
transform, transform_module = others[0] if others else (None, None)
output_head = OutputHead(
children[output_embeddings], transform, transform_module
)
logger.debug("Identified output head as: %s", output_head)
return output_head

def _rename_output_head(self, name: str) -> str:
"""Rename the output head weights to match this module's attributes."""
embeddings, transform, _ = self._output_head
if transform is not None and name.startswith(f"{transform}."):
return LM_HEAD_TRANSFORM_ATTR + name.removeprefix(transform)
if embeddings != "lm_head" and name.startswith(f"{embeddings}."):
return "lm_head" + name.removeprefix(embeddings)
return name

def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]:
"""A thin wrapper around `Base.load_weights` to handle the lm_head bias."""
"""A thin wrapper around `Base.load_weights` to handle the output head."""
lm_head_bias = set()
load_device = self.vllm_config.load_config.device or self.device_config.device

def auto_load_lm_head_bias(weights):
for name, weight in weights:
name = self._rename_output_head(name)
if name.endswith("lm_head.bias") and self.pp_group.is_last_rank:
# load_weights is called outside vLLM's device context,
# so we must enter it again when registering the bias.
Expand All @@ -85,5 +170,7 @@ def auto_load_lm_head_bias(weights):
return super().load_weights(auto_load_lm_head_bias(weights)) | lm_head_bias

def compute_logits(self, hidden_states: "torch.Tensor") -> "torch.Tensor | None":
if (transform := getattr(self, LM_HEAD_TRANSFORM_ATTR, None)) is not None:
hidden_states = transform(hidden_states)
logits = self.logits_processor(self.lm_head, hidden_states, self.lm_head.bias)
return logits
Loading