From 4c659771ee97632b493837010babdf9bd6652dcc Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 2 Dec 2025 13:47:39 +0100 Subject: [PATCH 001/131] Add return_dict to get_text_features methods to allow returning 'BaseModelOutputWithPooling' Added to all architectures except blip-2, which has a much different structure here. It uses 'Blip2TextModelWithProjection' to get these embeddings/features, but this class isn't as simple to use --- .../models/aimv2/modeling_aimv2.py | 14 +++++++++++-- .../models/align/modeling_align.py | 14 +++++++++++-- .../models/altclip/modeling_altclip.py | 14 +++++++++++-- src/transformers/models/blip/modeling_blip.py | 16 ++++++++++++--- .../chinese_clip/modeling_chinese_clip.py | 14 +++++++++++-- src/transformers/models/clap/modeling_clap.py | 12 ++++++++++- src/transformers/models/clip/modeling_clip.py | 12 ++++++++++- .../models/clipseg/modeling_clipseg.py | 12 ++++++++++- src/transformers/models/clvp/modeling_clvp.py | 16 ++++++++++++--- .../models/flava/modeling_flava.py | 11 +++++++++- .../models/groupvit/modeling_groupvit.py | 16 +++++++++++++-- .../models/metaclip_2/modeling_metaclip_2.py | 9 ++++++++- .../models/metaclip_2/modular_metaclip_2.py | 2 ++ .../models/owlv2/modeling_owlv2.py | 14 +++++++++++-- .../models/owlvit/modeling_owlvit.py | 14 +++++++++++-- src/transformers/models/sam3/modeling_sam3.py | 20 ++++++++++++++++--- .../models/siglip/modeling_siglip.py | 9 ++++++++- .../models/siglip2/modeling_siglip2.py | 9 ++++++++- .../modeling_vision_text_dual_encoder.py | 15 ++++++++++++-- .../models/x_clip/modeling_x_clip.py | 16 +++++++++++++-- 20 files changed, 225 insertions(+), 34 deletions(-) diff --git a/src/transformers/models/aimv2/modeling_aimv2.py b/src/transformers/models/aimv2/modeling_aimv2.py index 2ce372c1040a..80f8f95ae1d4 100644 --- a/src/transformers/models/aimv2/modeling_aimv2.py +++ b/src/transformers/models/aimv2/modeling_aimv2.py @@ -23,7 +23,7 @@ import math from collections.abc import Callable from dataclasses import dataclass -from typing import Any, Optional +from typing import Any, Optional, Union import torch import torch.nn.functional as F @@ -607,8 +607,12 @@ def get_text_features( input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`Aimv2TextModel`]. @@ -635,6 +639,12 @@ def get_text_features( pooled_output = text_outputs.pooler_output text_features = self.text_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) + return text_features @filter_out_non_signature_kwargs() diff --git a/src/transformers/models/align/modeling_align.py b/src/transformers/models/align/modeling_align.py index d136a1001075..ac9afdee57f4 100644 --- a/src/transformers/models/align/modeling_align.py +++ b/src/transformers/models/align/modeling_align.py @@ -1095,8 +1095,12 @@ def get_text_features( token_type_ids: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, inputs_embeds: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`AlignTextModel`]. @@ -1114,7 +1118,7 @@ def get_text_features( >>> with torch.inference_mode(): ... text_features = model.get_text_features(**inputs) ```""" - text_outputs = self.text_model( + text_outputs: BaseModelOutputWithPooling = self.text_model( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, @@ -1124,6 +1128,12 @@ def get_text_features( last_hidden_state = text_outputs[0][:, 0, :] text_features = self.text_projection(last_hidden_state) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) + return text_features @filter_out_non_signature_kwargs() diff --git a/src/transformers/models/altclip/modeling_altclip.py b/src/transformers/models/altclip/modeling_altclip.py index 254a4fc294d5..542c92609a40 100755 --- a/src/transformers/models/altclip/modeling_altclip.py +++ b/src/transformers/models/altclip/modeling_altclip.py @@ -1155,8 +1155,12 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, token_type_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`AltCLIPTextModel`]. @@ -1174,7 +1178,7 @@ def get_text_features( >>> with torch.inference_mode(): ... text_features = model.get_text_features(**inputs) ```""" - text_outputs = self.text_model( + text_outputs: BaseModelOutputWithPoolingAndProjection = self.text_model( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, @@ -1183,6 +1187,12 @@ def get_text_features( pooled_output = text_outputs.pooler_output text_features = self.text_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) + return text_features @filter_out_non_signature_kwargs() diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index e0268b73f692..6563c9c60996 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -26,7 +26,7 @@ from ...activations import ACT2FN from ...generation import GenerationMixin from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, BaseModelOutputWithPoolingAndCrossAttentions from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, logging, torch_int @@ -573,8 +573,12 @@ def get_text_features( input_ids: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`BlipTextModel`]. @@ -590,7 +594,7 @@ def get_text_features( >>> inputs = processor(text=["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> text_features = model.get_text_features(**inputs) ```""" - text_outputs = self.text_model( + text_outputs: BaseModelOutputWithPoolingAndCrossAttentions = self.text_model( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, @@ -599,6 +603,12 @@ def get_text_features( pooled_output = text_outputs[1] text_features = self.text_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) + return text_features @auto_docstring diff --git a/src/transformers/models/chinese_clip/modeling_chinese_clip.py b/src/transformers/models/chinese_clip/modeling_chinese_clip.py index 957615c923ec..48dd5e98fff0 100644 --- a/src/transformers/models/chinese_clip/modeling_chinese_clip.py +++ b/src/transformers/models/chinese_clip/modeling_chinese_clip.py @@ -1004,8 +1004,12 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, token_type_ids: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the final [CLS] hidden state of Text-Transformer. @@ -1024,7 +1028,7 @@ def get_text_features( ... text_features = model.get_text_features(**inputs) >>> text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True) ```""" - text_outputs = self.text_model( + text_outputs: BaseModelOutputWithPooling = self.text_model( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, @@ -1034,6 +1038,12 @@ def get_text_features( pooled_output = text_outputs[0][:, 0, :] text_features = self.text_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) + return text_features @filter_out_non_signature_kwargs() diff --git a/src/transformers/models/clap/modeling_clap.py b/src/transformers/models/clap/modeling_clap.py index 1421bdb83c9d..4ceec47ff042 100644 --- a/src/transformers/models/clap/modeling_clap.py +++ b/src/transformers/models/clap/modeling_clap.py @@ -1548,8 +1548,12 @@ def get_text_features( input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`ClapTextModel`]. @@ -1573,6 +1577,12 @@ def get_text_features( text_features = self.text_projection(text_outputs.pooler_output) text_features = F.normalize(text_features, dim=-1) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) + return text_features @filter_out_non_signature_kwargs() diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index 05109a841cfa..4164d296e777 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -788,8 +788,12 @@ def get_text_features( input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`CLIPTextModel`]. @@ -816,6 +820,12 @@ def get_text_features( pooled_output = text_outputs.pooler_output text_features = self.text_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) + return text_features @filter_out_non_signature_kwargs() diff --git a/src/transformers/models/clipseg/modeling_clipseg.py b/src/transformers/models/clipseg/modeling_clipseg.py index af7f0324142a..5dbdc9b86a4f 100644 --- a/src/transformers/models/clipseg/modeling_clipseg.py +++ b/src/transformers/models/clipseg/modeling_clipseg.py @@ -853,8 +853,12 @@ def get_text_features( input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`CLIPSegTextModel`]. @@ -880,6 +884,12 @@ def get_text_features( pooled_output = text_outputs.pooler_output text_features = self.text_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) + return text_features @filter_out_non_signature_kwargs() diff --git a/src/transformers/models/clvp/modeling_clvp.py b/src/transformers/models/clvp/modeling_clvp.py index 835275f4e20c..7acb9095469a 100644 --- a/src/transformers/models/clvp/modeling_clvp.py +++ b/src/transformers/models/clvp/modeling_clvp.py @@ -1475,7 +1475,8 @@ def get_text_features( input_ids: Optional[torch.LongTensor] = None, text_encoder_inputs_embeds: Optional[torch.FloatTensor] = None, attention_mask: Optional[torch.LongTensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" This method can be used to extract text_embeds from a text. The text embeddings obtained by applying the projection layer to the pooled output of the CLVP text encoder model. @@ -1495,6 +1496,8 @@ def get_text_features( - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `torch.FloatTensor` of shape `(batch_size, output_dim)`: @@ -1519,13 +1522,20 @@ def get_text_features( ``` """ - outputs = self.text_encoder_model( + text_outputs: ClvpEncoderOutput = self.text_encoder_model( input_ids=input_ids, inputs_embeds=text_encoder_inputs_embeds, attention_mask=attention_mask, ) + text_features = text_outputs.embeds - return outputs[0] + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) + + return text_features def get_speech_features( self, diff --git a/src/transformers/models/flava/modeling_flava.py b/src/transformers/models/flava/modeling_flava.py index c6ffb49febd0..1fdcd1d79272 100644 --- a/src/transformers/models/flava/modeling_flava.py +++ b/src/transformers/models/flava/modeling_flava.py @@ -998,7 +998,8 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, token_type_ids: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" input_ids (`torch.LongTensor` of shape `(batch_size, text_seq_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See @@ -1010,6 +1011,8 @@ def get_text_features( - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by @@ -1040,6 +1043,12 @@ def get_text_features( pooled_output = text_outputs.last_hidden_state text_features = self.text_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) + return text_features @filter_out_non_signature_kwargs() diff --git a/src/transformers/models/groupvit/modeling_groupvit.py b/src/transformers/models/groupvit/modeling_groupvit.py index dfb43d94bd49..37daababe750 100644 --- a/src/transformers/models/groupvit/modeling_groupvit.py +++ b/src/transformers/models/groupvit/modeling_groupvit.py @@ -1228,8 +1228,12 @@ def get_text_features( input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`GroupViTTextModel`]. @@ -1252,7 +1256,15 @@ def get_text_features( attention_mask=attention_mask, position_ids=position_ids, ) - text_features = self.text_projection(text_outputs.pooler_output) + pooled_output = text_outputs.pooler_output + text_features = self.text_projection(pooled_output) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) + return text_features @filter_out_non_signature_kwargs() diff --git a/src/transformers/models/metaclip_2/modeling_metaclip_2.py b/src/transformers/models/metaclip_2/modeling_metaclip_2.py index b8dc9663364a..784234cdfe10 100644 --- a/src/transformers/models/metaclip_2/modeling_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modeling_metaclip_2.py @@ -803,7 +803,8 @@ def get_text_features( input_ids: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by @@ -828,6 +829,12 @@ def get_text_features( pooled_output = text_outputs.pooler_output text_features = self.text_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) + return text_features @filter_out_non_signature_kwargs() diff --git a/src/transformers/models/metaclip_2/modular_metaclip_2.py b/src/transformers/models/metaclip_2/modular_metaclip_2.py index 49914ab1e6b3..7f44bc9b20be 100644 --- a/src/transformers/models/metaclip_2/modular_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modular_metaclip_2.py @@ -560,6 +560,7 @@ def get_text_features( input_ids: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, + return_dict: bool = False, ): r""" Returns: @@ -581,6 +582,7 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + return_dict=return_dict, ) def get_image_features( diff --git a/src/transformers/models/owlv2/modeling_owlv2.py b/src/transformers/models/owlv2/modeling_owlv2.py index 5426d37b73c3..bf918817e162 100644 --- a/src/transformers/models/owlv2/modeling_owlv2.py +++ b/src/transformers/models/owlv2/modeling_owlv2.py @@ -974,12 +974,15 @@ def get_text_features( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by @@ -1000,7 +1003,14 @@ def get_text_features( ```""" # Get embeddings for all text queries in all batch samples text_outputs: BaseModelOutputWithPooling = self.text_model(input_ids=input_ids, attention_mask=attention_mask) - text_features = self.text_projection(text_outputs.pooler_output) + pooled_output = text_outputs.pooler_output + text_features = self.text_projection(pooled_output) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) return text_features diff --git a/src/transformers/models/owlvit/modeling_owlvit.py b/src/transformers/models/owlvit/modeling_owlvit.py index a4619e6b11f1..7cbea96d256a 100644 --- a/src/transformers/models/owlvit/modeling_owlvit.py +++ b/src/transformers/models/owlvit/modeling_owlvit.py @@ -955,12 +955,15 @@ def get_text_features( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by @@ -981,7 +984,14 @@ def get_text_features( ```""" # Get embeddings for all text queries in all batch samples text_outputs: BaseModelOutputWithPooling = self.text_model(input_ids=input_ids, attention_mask=attention_mask) - text_features = self.text_projection(text_outputs.pooler_output) + pooled_output = text_outputs.pooler_output + text_features = self.text_projection(pooled_output) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) return text_features diff --git a/src/transformers/models/sam3/modeling_sam3.py b/src/transformers/models/sam3/modeling_sam3.py index 98e616bbef3b..fe3f9ff2757d 100644 --- a/src/transformers/models/sam3/modeling_sam3.py +++ b/src/transformers/models/sam3/modeling_sam3.py @@ -35,6 +35,7 @@ from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ( BaseModelOutput, + BaseModelOutputWithPooling, ModelOutput, ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel @@ -2134,9 +2135,14 @@ def get_text_features( self, input_ids: torch.LongTensor, attention_mask: Optional[torch.Tensor] = None, + return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> torch.FloatTensor: + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Text embeddings that can be passed as `text_embeds` to the forward method. @@ -2164,8 +2170,16 @@ def get_text_features( """ text_features = self.text_encoder( input_ids=input_ids, attention_mask=attention_mask, **kwargs - ).last_hidden_state - text_features = self.text_projection(text_features) + ) + last_hidden_state = text_features.last_hidden_state + text_features = self.text_projection(last_hidden_state) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=text_features, + ) + return text_features @auto_docstring diff --git a/src/transformers/models/siglip/modeling_siglip.py b/src/transformers/models/siglip/modeling_siglip.py index 64ee423c9294..cb585d2a55cb 100644 --- a/src/transformers/models/siglip/modeling_siglip.py +++ b/src/transformers/models/siglip/modeling_siglip.py @@ -781,8 +781,12 @@ def get_text_features( input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`SiglipTextModel`]. @@ -808,6 +812,9 @@ def get_text_features( ) pooled_output = text_outputs.pooler_output + if return_dict: + return text_outputs + return pooled_output @filter_out_non_signature_kwargs() diff --git a/src/transformers/models/siglip2/modeling_siglip2.py b/src/transformers/models/siglip2/modeling_siglip2.py index b29f892d9358..36d7f5c13cfe 100644 --- a/src/transformers/models/siglip2/modeling_siglip2.py +++ b/src/transformers/models/siglip2/modeling_siglip2.py @@ -838,8 +838,12 @@ def get_text_features( input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`Siglip2TextModel`]. @@ -865,6 +869,9 @@ def get_text_features( ) pooled_output = text_outputs.pooler_output + if return_dict: + return text_outputs + return pooled_output @filter_out_non_signature_kwargs() diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index ee1ddfa0f6e4..aab794aafa54 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -112,8 +112,12 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, token_type_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`CLIPTextModel`]. @@ -137,7 +141,14 @@ def get_text_features( position_ids=position_ids, token_type_ids=token_type_ids, ) - text_features = self.text_projection(text_outputs.pooler_output) + pooled_output = text_outputs.pooler_output + text_features = self.text_projection(pooled_output) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) return text_features diff --git a/src/transformers/models/x_clip/modeling_x_clip.py b/src/transformers/models/x_clip/modeling_x_clip.py index 81f540460228..731fd21c0d18 100644 --- a/src/transformers/models/x_clip/modeling_x_clip.py +++ b/src/transformers/models/x_clip/modeling_x_clip.py @@ -1205,8 +1205,12 @@ def get_text_features( input_ids: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`XCLIPTextModel`]. @@ -1229,7 +1233,15 @@ def get_text_features( attention_mask=attention_mask, position_ids=position_ids, ) - text_features = self.text_projection(text_outputs.pooler_output) + pooled_output = text_outputs.pooler_output + text_features = self.text_projection(pooled_output) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + pooler_output=text_features, + ) + return text_features @filter_out_non_signature_kwargs() From 47c2418b3ee4b5b57b38ae30810e646b07176a00 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 2 Dec 2025 16:27:50 +0100 Subject: [PATCH 002/131] Add return_dict to get_image_features methods to allow returning 'BaseModelOutputWithPooling' Well, the architectures supporting get_image_features are all extremely different, with wildly different outputs for the get_image_features methods. 2d outputs, 3d outputs, lists of 2d outputs (due to non-matching shapes), existing 'return_attentions' resulting in returning 2-tuple, existing 'return_dict' resulting in returning 3-tuples (???), high quality image embeddings, low quality image embeddings, deepstack image embeddings, etc. etc. etc. And I only went through like 70-80% of all architectures with get_image_features before I gave up. Standardisation of all of these sounds like a lost cause. --- .../models/aimv2/modeling_aimv2.py | 12 ++++++- .../models/align/modeling_align.py | 14 ++++++-- .../models/altclip/modeling_altclip.py | 12 ++++++- src/transformers/models/aria/modeling_aria.py | 24 ++++++++++++-- src/transformers/models/aria/modular_aria.py | 18 +++++++++-- .../models/aya_vision/modeling_aya_vision.py | 13 +++++++- .../models/aya_vision/modular_aya_vision.py | 12 +++++++ src/transformers/models/blip/modeling_blip.py | 17 +++++++--- .../models/blip_2/modeling_blip_2.py | 15 +++++++-- .../models/chameleon/modeling_chameleon.py | 26 +++++++++++---- .../chinese_clip/modeling_chinese_clip.py | 13 ++++++-- src/transformers/models/clip/modeling_clip.py | 12 ++++++- .../models/clipseg/modeling_clipseg.py | 12 ++++++- .../cohere2_vision/modeling_cohere2_vision.py | 15 ++++++--- .../cohere2_vision/modular_cohere2_vision.py | 14 ++++++-- .../deepseek_vl/modeling_deepseek_vl.py | 17 +++++++--- .../modeling_deepseek_vl_hybrid.py | 17 +++++++--- .../modular_deepseek_vl_hybrid.py | 14 ++++++-- src/transformers/models/emu3/modeling_emu3.py | 1 + src/transformers/models/emu3/modular_emu3.py | 3 +- .../models/flava/modeling_flava.py | 11 ++++++- .../models/florence2/modeling_florence2.py | 17 +++++++--- .../models/florence2/modular_florence2.py | 17 +++++++--- src/transformers/models/fuyu/modeling_fuyu.py | 1 + .../models/gemma3/modeling_gemma3.py | 30 +++++++++++++---- .../models/gemma3/modular_gemma3.py | 19 ++++++++--- .../models/gemma3n/modeling_gemma3n.py | 32 +++++++++++++------ .../models/gemma3n/modular_gemma3n.py | 24 ++++++++++---- .../models/glm46v/modeling_glm46v.py | 1 + .../models/glm4v/modeling_glm4v.py | 1 + .../models/glm4v_moe/modeling_glm4v_moe.py | 1 + .../models/got_ocr2/modeling_got_ocr2.py | 23 ++++++++++--- .../models/got_ocr2/modular_got_ocr2.py | 22 ++++++++++--- .../models/groupvit/modeling_groupvit.py | 9 +++++- .../models/idefics2/modeling_idefics2.py | 27 ++++++++++------ .../models/idefics3/modeling_idefics3.py | 21 ++++++++---- .../instructblip/modeling_instructblip.py | 12 +++++-- .../modeling_instructblipvideo.py | 1 + .../modular_instructblipvideo.py | 1 + .../models/internvl/modeling_internvl.py | 18 +++++++++-- .../models/internvl/modular_internvl.py | 18 +++++++++-- .../models/janus/modeling_janus.py | 15 ++++++--- .../models/janus/modular_janus.py | 17 +++++++--- .../models/kosmos2/modeling_kosmos2.py | 16 +++++++--- .../models/lfm2_vl/modeling_lfm2_vl.py | 20 +++++++++--- .../models/lfm2_vl/modular_lfm2_vl.py | 19 +++++++++-- .../models/llava/modeling_llava.py | 10 +++++- .../models/llava_next/modeling_llava_next.py | 21 +++++++++--- .../modeling_llava_next_video.py | 18 ++++++++--- .../modular_llava_next_video.py | 15 +++++++-- .../modeling_llava_onevision.py | 18 ++++++++--- .../modular_llava_onevision.py | 15 +++++++-- .../models/metaclip_2/modeling_metaclip_2.py | 12 ++++++- .../models/metaclip_2/modular_metaclip_2.py | 5 +++ .../models/mistral3/modeling_mistral3.py | 13 +++++++- .../models/mistral3/modular_mistral3.py | 12 +++++++ .../models/ovis2/modeling_ovis2.py | 1 + .../models/ovis2/modular_ovis2.py | 1 + .../models/owlv2/modeling_owlv2.py | 12 ++++++- .../models/owlvit/modeling_owlvit.py | 12 ++++++- .../models/paligemma/modeling_paligemma.py | 18 ++++++++--- .../perception_lm/modeling_perception_lm.py | 19 ++++++++--- .../perception_lm/modular_perception_lm.py | 18 +++++++++-- .../qwen2_5_omni/modeling_qwen2_5_omni.py | 17 ++++++++-- .../qwen2_5_omni/modular_qwen2_5_omni.py | 12 +++++-- .../models/qwen2_5_vl/modeling_qwen2_5_vl.py | 1 + .../models/qwen2_vl/modeling_qwen2_vl.py | 1 + .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 16 +++++++++- .../models/smolvlm/modeling_smolvlm.py | 4 +-- 69 files changed, 766 insertions(+), 179 deletions(-) diff --git a/src/transformers/models/aimv2/modeling_aimv2.py b/src/transformers/models/aimv2/modeling_aimv2.py index 80f8f95ae1d4..aa72b6eb34f5 100644 --- a/src/transformers/models/aimv2/modeling_aimv2.py +++ b/src/transformers/models/aimv2/modeling_aimv2.py @@ -653,8 +653,12 @@ def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`Aimv2VisionModel`]. @@ -684,6 +688,12 @@ def get_image_features( pooled_output = vision_outputs.pooler_output image_features = self.visual_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @auto_docstring diff --git a/src/transformers/models/align/modeling_align.py b/src/transformers/models/align/modeling_align.py index ac9afdee57f4..54b4e1d64244 100644 --- a/src/transformers/models/align/modeling_align.py +++ b/src/transformers/models/align/modeling_align.py @@ -1138,8 +1138,11 @@ def get_text_features( @filter_out_non_signature_kwargs() @auto_docstring - def get_image_features(self, pixel_values: torch.FloatTensor) -> torch.FloatTensor: + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`AlignVisionModel`]. @@ -1161,8 +1164,15 @@ def get_image_features(self, pixel_values: torch.FloatTensor) -> torch.FloatTens >>> with torch.inference_mode(): ... image_features = model.get_image_features(**inputs) ```""" - vision_outputs = self.vision_model(pixel_values=pixel_values) + vision_outputs: BaseModelOutputWithPoolingAndNoAttention = self.vision_model(pixel_values=pixel_values) image_features = vision_outputs.pooler_output + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @can_return_tuple diff --git a/src/transformers/models/altclip/modeling_altclip.py b/src/transformers/models/altclip/modeling_altclip.py index 542c92609a40..cf8baf04c8ea 100755 --- a/src/transformers/models/altclip/modeling_altclip.py +++ b/src/transformers/models/altclip/modeling_altclip.py @@ -1201,8 +1201,12 @@ def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`AltCLIPVisionModel`]. @@ -1231,6 +1235,12 @@ def get_image_features( pooled_output = vision_outputs.pooler_output image_features = self.visual_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @auto_docstring diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index 714585c17424..1424a6071343 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -33,7 +33,12 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, ModelOutput +from ...modeling_outputs import ( + BaseModelOutputWithPast, + BaseModelOutputWithPooling, + CausalLMOutputWithPast, + ModelOutput, +) from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -918,8 +923,9 @@ def get_image_features( pixel_values: torch.FloatTensor, pixel_mask: Optional[torch.FloatTensor] = None, vision_feature_layer: int = -1, - ): - """ + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + r""" Obtains image last hidden states from the vision tower and apply multimodal projection. Args: @@ -931,6 +937,9 @@ def get_image_features( The index of the layer to select the vision feature. If multiple indices are provided, the vision feature of the corresponding indices will be concatenated to form the vision features. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -948,6 +957,13 @@ def get_image_features( selected_image_feature = image_outputs.hidden_states[vision_feature_layer] image_features = self.multi_modal_projector(selected_image_feature, attn_mask=image_attn_mask) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( @@ -1077,11 +1093,13 @@ def get_image_features( pixel_values: torch.FloatTensor, pixel_mask: Optional[torch.FloatTensor] = None, vision_feature_layer: int = -1, + return_dict: bool = False, ): return self.model.get_image_features( pixel_values=pixel_values, pixel_mask=pixel_mask, vision_feature_layer=vision_feature_layer, + return_dict=return_dict, ) @can_return_tuple diff --git a/src/transformers/models/aria/modular_aria.py b/src/transformers/models/aria/modular_aria.py index 1001e647803b..06fa5c0db308 100644 --- a/src/transformers/models/aria/modular_aria.py +++ b/src/transformers/models/aria/modular_aria.py @@ -38,6 +38,7 @@ validate_preprocess_arguments, ) from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_outputs import BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel from ...processing_utils import ImagesKwargs, MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack from ...tokenization_python import PreTokenizedInput, TextInput @@ -1278,8 +1279,9 @@ def get_image_features( pixel_values: torch.FloatTensor, pixel_mask: Optional[torch.FloatTensor] = None, vision_feature_layer: int = -1, - ): - """ + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + r""" Obtains image last hidden states from the vision tower and apply multimodal projection. Args: @@ -1291,6 +1293,9 @@ def get_image_features( The index of the layer to select the vision feature. If multiple indices are provided, the vision feature of the corresponding indices will be concatenated to form the vision features. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -1308,6 +1313,13 @@ def get_image_features( selected_image_feature = image_outputs.hidden_states[vision_feature_layer] image_features = self.multi_modal_projector(selected_image_feature, attn_mask=image_attn_mask) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def forward( @@ -1374,11 +1386,13 @@ def get_image_features( pixel_values: torch.FloatTensor, pixel_mask: Optional[torch.FloatTensor] = None, vision_feature_layer: int = -1, + return_dict: bool = False, ): return self.model.get_image_features( pixel_values=pixel_values, pixel_mask=pixel_mask, vision_feature_layer=vision_feature_layer, + return_dict=return_dict, ) @can_return_tuple diff --git a/src/transformers/models/aya_vision/modeling_aya_vision.py b/src/transformers/models/aya_vision/modeling_aya_vision.py index 41ac2cc76529..caa5577b39bc 100644 --- a/src/transformers/models/aya_vision/modeling_aya_vision.py +++ b/src/transformers/models/aya_vision/modeling_aya_vision.py @@ -28,7 +28,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -186,6 +186,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, **kwargs, ): """ @@ -201,6 +202,9 @@ def get_image_features( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -234,6 +238,13 @@ def get_image_features( selected_image_feature = torch.cat(hs_pool, dim=-1) image_features = self.multi_modal_projector(selected_image_feature) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( diff --git a/src/transformers/models/aya_vision/modular_aya_vision.py b/src/transformers/models/aya_vision/modular_aya_vision.py index f76e046e0b94..1d1577bd0ac2 100644 --- a/src/transformers/models/aya_vision/modular_aya_vision.py +++ b/src/transformers/models/aya_vision/modular_aya_vision.py @@ -30,6 +30,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import auto_docstring, logging from ...utils.generic import check_model_inputs @@ -112,6 +113,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, **kwargs, ): """ @@ -127,6 +129,9 @@ def get_image_features( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -160,6 +165,13 @@ def get_image_features( selected_image_feature = torch.cat(hs_pool, dim=-1) image_features = self.multi_modal_projector(selected_image_feature) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @check_model_inputs diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 6563c9c60996..dace6ee5a64c 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -616,8 +616,12 @@ def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`BlipVisionModel`]. @@ -640,14 +644,19 @@ def get_image_features( >>> image_features = model.get_image_features(**inputs) ```""" - vision_outputs = self.vision_model( + vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, ) - - pooled_output = vision_outputs[1] # pooled_output + pooled_output = vision_outputs.pooler_output image_features = self.visual_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @auto_docstring diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 4107f448717b..f20d0ebe3d1e 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -1136,8 +1136,12 @@ def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, - ) -> Union[torch.FloatTensor, CausalLMOutputWithPast]: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: vision_outputs (`torch.FloatTensor`): The vision model's last layer pooled logits. @@ -1163,8 +1167,15 @@ def get_image_features( interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, ) + image_features = vision_outputs.pooler_output + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) - return vision_outputs.pooler_output + return image_features @filter_out_non_signature_kwargs() @auto_docstring diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 2e816ad63b4d..76cfa3411e07 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -28,7 +28,7 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -853,7 +853,7 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor): bpe_toks = bpe_toks.view(batch_size, -1) return bpe_toks - def get_image_features(self, pixel_values: torch.FloatTensor): + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): """ Tokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layer @@ -861,10 +861,21 @@ def get_image_features(self, pixel_values: torch.FloatTensor): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): The tensors corresponding to the input images. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ image_tokens = self.get_image_tokens(pixel_values) - vision_embeddings = self.get_input_embeddings()(image_tokens) - return vision_embeddings + image_embeddings = self.get_input_embeddings()(image_tokens) + image_features = image_embeddings.mean(dim=1) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_embeddings, + pooler_output=image_features, + ) + + # NOTE: @Tom Not easily converted to the standard format, this one breaks backward compatibility + return image_features def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -926,7 +937,8 @@ def forward( inputs_embeds = self.embed_tokens(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values) + image_tokens = self.get_image_tokens(pixel_values) + image_embeds = self.get_input_embeddings()(image_tokens) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds ) @@ -1022,8 +1034,8 @@ def __init__(self, config): def get_image_tokens(self, pixel_values): return self.model.get_image_tokens(pixel_values) - def get_image_features(self, pixel_values): - return self.model.get_image_features(pixel_values) + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): + return self.model.get_image_features(pixel_values, return_dict=return_dict) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/chinese_clip/modeling_chinese_clip.py b/src/transformers/models/chinese_clip/modeling_chinese_clip.py index 48dd5e98fff0..af6d4e6d5252 100644 --- a/src/transformers/models/chinese_clip/modeling_chinese_clip.py +++ b/src/transformers/models/chinese_clip/modeling_chinese_clip.py @@ -1052,8 +1052,12 @@ def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the final [CLS] hidden state of Vision-Transformer. @@ -1081,10 +1085,15 @@ def get_image_features( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, ) - pooled_output = vision_outputs.pooler_output image_features = self.visual_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @can_return_tuple diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index 4164d296e777..9994d7915598 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -834,8 +834,12 @@ def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`CLIPVisionModel`]. @@ -865,6 +869,12 @@ def get_image_features( pooled_output = vision_outputs.pooler_output image_features = self.visual_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @can_return_tuple diff --git a/src/transformers/models/clipseg/modeling_clipseg.py b/src/transformers/models/clipseg/modeling_clipseg.py index 5dbdc9b86a4f..0d492167ab3f 100644 --- a/src/transformers/models/clipseg/modeling_clipseg.py +++ b/src/transformers/models/clipseg/modeling_clipseg.py @@ -898,8 +898,12 @@ def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = True, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`CLIPSegVisionModel`]. @@ -929,6 +933,12 @@ def get_image_features( pooled_output = vision_outputs.pooler_output image_features = self.visual_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @auto_docstring diff --git a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py index a6edbbf128eb..c7447634f83f 100644 --- a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py @@ -27,7 +27,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring @@ -167,7 +167,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.FloatTensor): + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -182,6 +182,13 @@ def get_image_features(self, pixel_values: torch.FloatTensor): image_features = self.vision_tower(pixel_values, output_hidden_states=True) selected_image_feature = image_features.last_hidden_state image_features = self.multi_modal_projector(selected_image_feature) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=selected_image_feature, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( @@ -279,8 +286,8 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head - def get_image_features(self, pixel_values: torch.FloatTensor): - return self.model.get_image_features(pixel_values=pixel_values) + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): + return self.model.get_image_features(pixel_values=pixel_values, return_dict=return_dict) @check_model_inputs @auto_docstring diff --git a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py index 5f417075a931..e7ef177a27ff 100644 --- a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py @@ -34,6 +34,7 @@ from ...image_processing_utils import BatchFeature from ...image_utils import ImageInput from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import ImagesKwargs, Unpack from ...utils import TransformersKwargs, auto_docstring, logging from ...utils.generic import check_model_inputs @@ -97,7 +98,7 @@ class Cohere2VisionPreTrainedModel(AyaVisionPreTrainedModel): class Cohere2VisionModel(AyaVisionModel): _checkpoint_conversion_mapping = {} - def get_image_features(self, pixel_values: torch.FloatTensor): + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -112,6 +113,13 @@ def get_image_features(self, pixel_values: torch.FloatTensor): image_features = self.vision_tower(pixel_values, output_hidden_states=True) selected_image_feature = image_features.last_hidden_state image_features = self.multi_modal_projector(selected_image_feature) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=selected_image_feature, + pooler_output=image_features, + ) + return image_features @check_model_inputs @@ -164,8 +172,8 @@ def forward( class Cohere2VisionForConditionalGeneration(AyaVisionForConditionalGeneration): _checkpoint_conversion_mapping = {} - def get_image_features(self, pixel_values: torch.FloatTensor): - return self.model.get_image_features(pixel_values=pixel_values) + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): + return self.model.get_image_features(pixel_values=pixel_values, return_dict=return_dict) @check_model_inputs @auto_docstring diff --git a/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py b/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py index f568477303c0..0d5735bcf2a8 100644 --- a/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py +++ b/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py @@ -26,7 +26,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import ModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -153,10 +153,17 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values): - image_embeds = self.vision_model(pixel_values) - image_embeds = self.aligner(image_embeds.last_hidden_state) - return image_embeds + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): + vision_outputs = self.vision_model(pixel_values) + image_features = self.aligner(vision_outputs.last_hidden_state) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + + return image_features def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor diff --git a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py index c0204fe94382..9e12fd2a9b41 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py @@ -27,7 +27,7 @@ from ... import initialization as init from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import ModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -269,11 +269,20 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values, high_res_pixel_values): + def get_image_features( + self, pixel_values: torch.FloatTensor, high_res_pixel_values: torch.FloatTensor, return_dict: bool = False + ): vision_encodings = self.get_low_res_image_features(pixel_values) high_res_vision_encodings = self.get_high_res_image_features(high_res_pixel_values) - images_embeds = self.aligner(vision_encodings, high_res_vision_encodings) - return images_embeds + image_features = self.aligner(vision_encodings, high_res_vision_encodings) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_encodings, + pooler_output=image_features, + ) + + return image_features def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor diff --git a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py index 4501ed7810d2..afb8d309e26d 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py @@ -43,6 +43,7 @@ valid_images, validate_preprocess_arguments, ) +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import ImagesKwargs, Unpack from ...tokenization_utils_base import ( PreTokenizedInput, @@ -276,11 +277,18 @@ def get_high_res_image_features(self, pixel_values): return output - def get_image_features(self, pixel_values, high_res_pixel_values): + def get_image_features(self, pixel_values: torch.FloatTensor, high_res_pixel_values: torch.FloatTensor, return_dict: bool = False): vision_encodings = self.get_low_res_image_features(pixel_values) high_res_vision_encodings = self.get_high_res_image_features(high_res_pixel_values) - images_embeds = self.aligner(vision_encodings, high_res_vision_encodings) - return images_embeds + image_features = self.aligner(vision_encodings, high_res_vision_encodings) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_encodings, + pooler_output=image_features, + ) + + return image_features @can_return_tuple @auto_docstring(custom_args=DEEPSEEK_VL_COMMON_CUSTOM_ARGS) diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index 6c87e7442733..fb466c2984a4 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -1389,6 +1389,7 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_sizes: torch ] image_features = self.get_input_embeddings()(image_tokens) image_features = torch.split(image_features, split_sizes) + # NOTE: @Tom Not easily converted to the standard format return image_features @torch.no_grad() diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index bd6b3013c644..5484b742c480 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -25,7 +25,7 @@ from ... import initialization as init from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPooling, CausalLMOutputWithPast from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import auto_docstring, can_return_tuple, logging @@ -941,6 +941,7 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_sizes: torch ] image_features = self.get_input_embeddings()(image_tokens) image_features = torch.split(image_features, split_sizes) + # NOTE: @Tom Not easily converted to the standard format return image_features @torch.no_grad() diff --git a/src/transformers/models/flava/modeling_flava.py b/src/transformers/models/flava/modeling_flava.py index 1fdcd1d79272..e1192e056357 100644 --- a/src/transformers/models/flava/modeling_flava.py +++ b/src/transformers/models/flava/modeling_flava.py @@ -1059,10 +1059,13 @@ def get_image_features( bool_masked_pos: Optional[torch.BoolTensor] = None, interpolate_pos_encoding: Optional[bool] = None, attention_mask: Optional[torch.Tensor] = None, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, image_num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by @@ -1096,6 +1099,12 @@ def get_image_features( pooled_output = image_outputs.last_hidden_state image_features = self.image_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @auto_docstring diff --git a/src/transformers/models/florence2/modeling_florence2.py b/src/transformers/models/florence2/modeling_florence2.py index 9212cd5d8f5f..b2ae6980774a 100644 --- a/src/transformers/models/florence2/modeling_florence2.py +++ b/src/transformers/models/florence2/modeling_florence2.py @@ -29,7 +29,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import Seq2SeqLMOutput, Seq2SeqModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling, Seq2SeqLMOutput, Seq2SeqModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torch_available, logging @@ -652,7 +652,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.Tensor, **kwargs): + def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False, **kwargs): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -662,9 +662,16 @@ def get_image_features(self, pixel_values: torch.Tensor, **kwargs): Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - image_features = self.vision_tower(pixel_values, **kwargs) - image_embeds = self.multi_modal_projector(image_features) - return image_embeds + last_hidden_states = self.vision_tower(pixel_values, **kwargs) + image_features = self.multi_modal_projector(last_hidden_states) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_states=last_hidden_states, + pooler_output=image_features, + ) + + return image_features def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor diff --git a/src/transformers/models/florence2/modular_florence2.py b/src/transformers/models/florence2/modular_florence2.py index 74ace79943e7..2e7d62a2e45d 100644 --- a/src/transformers/models/florence2/modular_florence2.py +++ b/src/transformers/models/florence2/modular_florence2.py @@ -27,7 +27,7 @@ from ...configuration_utils import PreTrainedConfig from ...feature_extraction_utils import BatchFeature from ...image_utils import ImageInput -from ...modeling_outputs import Seq2SeqLMOutput, Seq2SeqModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling, Seq2SeqLMOutput, Seq2SeqModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import MultiModalData, ProcessorMixin, Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput @@ -1519,7 +1519,7 @@ def get_encoder(self, modality=None): else: return super().get_encoder(modality=modality) - def get_image_features(self, pixel_values: torch.Tensor, **kwargs): + def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False, **kwargs): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -1529,9 +1529,16 @@ def get_image_features(self, pixel_values: torch.Tensor, **kwargs): Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - image_features = self.vision_tower(pixel_values, **kwargs) - image_embeds = self.multi_modal_projector(image_features) - return image_embeds + last_hidden_states = self.vision_tower(pixel_values, **kwargs) + image_features = self.multi_modal_projector(last_hidden_states) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_states=last_hidden_states, + pooler_output=image_features, + ) + + return image_features @can_return_tuple @auto_docstring diff --git a/src/transformers/models/fuyu/modeling_fuyu.py b/src/transformers/models/fuyu/modeling_fuyu.py index 2bd58b0e6f38..816c70f3777c 100644 --- a/src/transformers/models/fuyu/modeling_fuyu.py +++ b/src/transformers/models/fuyu/modeling_fuyu.py @@ -128,6 +128,7 @@ def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs): self.vision_embed_tokens(patch.to(self.vision_embed_tokens.weight.dtype)).squeeze(0) for patch in pixel_values ] + # NOTE: @Tom Not easily converted to the standard format return patch_embeddings def get_placeholder_mask( diff --git a/src/transformers/models/gemma3/modeling_gemma3.py b/src/transformers/models/gemma3/modeling_gemma3.py index 9919cb0b0888..b6d867b098f5 100644 --- a/src/transformers/models/gemma3/modeling_gemma3.py +++ b/src/transformers/models/gemma3/modeling_gemma3.py @@ -35,7 +35,12 @@ from ...masking_utils import create_causal_mask, create_masks_for_generate, create_sliding_window_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GenericForSequenceClassification, GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast +from ...modeling_outputs import ( + BaseModelOutputWithPast, + BaseModelOutputWithPooling, + CausalLMOutputWithPast, + SequenceClassifierOutputWithPast, +) from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -866,18 +871,31 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.Tensor) -> torch.Tensor: + def get_image_features( + self, pixel_values: torch.FloatTensor, return_dict: bool = False + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) The tensors corresponding to the input images. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - vision_outputs = self.vision_tower(pixel_values=pixel_values).last_hidden_state - image_features = self.multi_modal_projector(vision_outputs) + vision_outputs = self.vision_tower(pixel_values=pixel_values) + last_hidden_state = vision_outputs.last_hidden_state + image_features = self.multi_modal_projector(last_hidden_state) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( @@ -1050,8 +1068,8 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) - def get_image_features(self, pixel_values): - return self.model.get_image_features(pixel_values) + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): + return self.model.get_image_features(pixel_values, return_dict=return_dict) @auto_docstring def forward( diff --git a/src/transformers/models/gemma3/modular_gemma3.py b/src/transformers/models/gemma3/modular_gemma3.py index 2d489d77f5c7..f485a63a87b4 100644 --- a/src/transformers/models/gemma3/modular_gemma3.py +++ b/src/transformers/models/gemma3/modular_gemma3.py @@ -25,7 +25,7 @@ from ...masking_utils import create_causal_mask, create_masks_for_generate, create_sliding_window_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GenericForSequenceClassification, GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, SequenceClassifierOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, SequenceClassifierOutputWithPast from ...modeling_rope_utils import ( ROPE_INIT_FUNCTIONS, RopeParameters, @@ -825,18 +825,29 @@ def __init__(self, config: Gemma3Config): super().__init__(config) del self.text_config_dtype - def get_image_features(self, pixel_values: torch.Tensor) -> torch.Tensor: + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) The tensors corresponding to the input images. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - vision_outputs = self.vision_tower(pixel_values=pixel_values).last_hidden_state - image_features = self.multi_modal_projector(vision_outputs) + vision_outputs = self.vision_tower(pixel_values=pixel_values) + last_hidden_state = vision_outputs.last_hidden_state + image_features = self.multi_modal_projector(last_hidden_state) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=image_features, + ) + return image_features @can_return_tuple diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index e61a32fb4c96..bb78a627e34c 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -35,7 +35,7 @@ from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -2091,30 +2091,42 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.Tensor) -> torch.Tensor: + def get_image_features( + self, pixel_values: torch.FloatTensor, return_dict: bool = False + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) The tensors corresponding to the input images. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - vision_outputs = self.vision_tower( - pixel_values=pixel_values, do_pooling=False, return_dict=True - ).last_hidden_state + vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, return_dict=True) + last_hidden_state = vision_outputs.last_hidden_state # Convert from (batch, channels, height, width) to (batch, height * width, channels) where: # height == width and height * width == Gemma3nConfig.vision_soft_tokens_per_image. - vision_outputs = vision_outputs.reshape( - vision_outputs.shape[0], + vision_outputs = last_hidden_state.reshape( + last_hidden_state.shape[0], self.config.vision_config.hidden_size, self.config.vision_soft_tokens_per_image, ).permute(0, 2, 1) # Normalize and embed the soft tokens into language model space. vision_outputs *= self.config.vision_config.hidden_size**0.5 - return self.embed_vision(inputs_embeds=vision_outputs) + image_features = self.embed_vision(inputs_embeds=vision_outputs) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=image_features, + ) + + # NOTE: @Tom I'm not sure the image_features here is 2-dimensional as we'd like + return image_features def get_placeholder_mask( self, @@ -2341,8 +2353,8 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) - def get_image_features(self, pixel_values): - return self.model.get_image_features(pixel_values) + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): + return self.model.get_image_features(pixel_values, return_dict=return_dict) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index a97ef8bceeeb..dbbc4f996a11 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -27,7 +27,7 @@ from ...configuration_utils import PreTrainedConfig, layer_type_validation from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling from ...modeling_rope_utils import RopeParameters from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -2200,30 +2200,42 @@ def __init__(self, config: Gemma3nConfig): self.embed_vision = Gemma3nMultimodalEmbedder(config.vision_config, config.text_config) self.embed_audio = Gemma3nMultimodalEmbedder(config.audio_config, config.text_config) - def get_image_features(self, pixel_values: torch.Tensor) -> torch.Tensor: + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) The tensors corresponding to the input images. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ vision_outputs = self.vision_tower( pixel_values=pixel_values, do_pooling=False, return_dict=True - ).last_hidden_state + ) + last_hidden_state = vision_outputs.last_hidden_state # Convert from (batch, channels, height, width) to (batch, height * width, channels) where: # height == width and height * width == Gemma3nConfig.vision_soft_tokens_per_image. - vision_outputs = vision_outputs.reshape( - vision_outputs.shape[0], + vision_outputs = last_hidden_state.reshape( + last_hidden_state.shape[0], self.config.vision_config.hidden_size, self.config.vision_soft_tokens_per_image, ).permute(0, 2, 1) # Normalize and embed the soft tokens into language model space. vision_outputs *= self.config.vision_config.hidden_size**0.5 - return self.embed_vision(inputs_embeds=vision_outputs) + image_features = self.embed_vision(inputs_embeds=vision_outputs) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=image_features, + ) + + # NOTE: @Tom I'm not sure the image_features here is 2-dimensional as we'd like + return image_features def get_placeholder_mask( self, diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 5950d5d7a2ab..1328f724912a 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -328,6 +328,7 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) + # NOTE: @Tom Not easily converted to the standard format return image_embeds def get_placeholder_mask( diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index 6b83d1f011f3..223df1299e40 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1176,6 +1176,7 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) + # NOTE: @Tom Not easily converted to the standard format return image_embeds def get_placeholder_mask( diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 11c80fc502a3..1675d30c910f 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1385,6 +1385,7 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) + # NOTE: @Tom Not easily converted to the standard format return image_embeds def get_placeholder_mask( diff --git a/src/transformers/models/got_ocr2/modeling_got_ocr2.py b/src/transformers/models/got_ocr2/modeling_got_ocr2.py index a05bb0e25985..145afb66245e 100644 --- a/src/transformers/models/got_ocr2/modeling_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modeling_got_ocr2.py @@ -35,7 +35,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -554,17 +554,30 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, - ): + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + pixel_values (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`) + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - image_outputs = self.vision_tower(pixel_values).last_hidden_state - return self.multi_modal_projector(image_outputs) + image_outputs = self.vision_tower(pixel_values) + last_hidden_state = image_outputs.last_hidden_state + image_features = self.multi_modal_projector(last_hidden_state) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=image_features, + ) + + return image_features def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor diff --git a/src/transformers/models/got_ocr2/modular_got_ocr2.py b/src/transformers/models/got_ocr2/modular_got_ocr2.py index e4fa64918bf6..200545038641 100644 --- a/src/transformers/models/got_ocr2/modular_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modular_got_ocr2.py @@ -22,6 +22,7 @@ from ... import initialization as init from ...cache_utils import Cache from ...configuration_utils import PreTrainedConfig +from ...modeling_outputs import BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import auto_docstring, can_return_tuple, logging @@ -310,17 +311,30 @@ def __init__(self, config: GotOcr2Config): def get_image_features( self, pixel_values: torch.FloatTensor, - ): + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + pixel_values (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`) + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - image_outputs = self.vision_tower(pixel_values).last_hidden_state - return self.multi_modal_projector(image_outputs) + image_outputs = self.vision_tower(pixel_values) + last_hidden_state = image_outputs.last_hidden_state + image_features = self.multi_modal_projector(last_hidden_state) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=image_features, + ) + + return image_features def forward( self, diff --git a/src/transformers/models/groupvit/modeling_groupvit.py b/src/transformers/models/groupvit/modeling_groupvit.py index 37daababe750..327c7472d8f4 100644 --- a/src/transformers/models/groupvit/modeling_groupvit.py +++ b/src/transformers/models/groupvit/modeling_groupvit.py @@ -1269,7 +1269,7 @@ def get_text_features( @filter_out_non_signature_kwargs() @auto_docstring - def get_image_features(self, pixel_values: torch.Tensor) -> torch.FloatTensor: + def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by @@ -1295,6 +1295,13 @@ def get_image_features(self, pixel_values: torch.Tensor) -> torch.FloatTensor: ```""" vision_outputs: BaseModelOutputWithPooling = self.vision_model(pixel_values) image_features = self.visual_projection(vision_outputs.pooler_output) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @auto_docstring diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index 1a394de39153..74ce1b751bbe 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -28,7 +28,7 @@ from ...modeling_attn_mask_utils import _prepare_4d_attention_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, ModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -822,8 +822,8 @@ def inputs_merger( return inputs_embeds def get_image_features( - self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None - ): + self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -832,6 +832,8 @@ def get_image_features( The tensors corresponding to the input images. pixel_attention_mask (`torch.LongTensor`, *optional*): The attention mask indicating padded regions in the image. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ batch_size, num_images, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility @@ -863,11 +865,18 @@ def get_image_features( image_hidden_states = image_hidden_states.last_hidden_state # Modality projection & resampling - image_hidden_states = self.connector( + image_features = self.connector( image_hidden_states, attention_mask=patch_attention_mask.view(pixel_values.size(0), -1) ) - image_hidden_states = image_hidden_states.view(-1, image_hidden_states.shape[-1]) - return image_hidden_states + image_features = image_features.view(-1, image_features.shape[-1]) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_hidden_states, + pooler_output=image_features, + ) + + return image_features @can_return_tuple @auto_docstring( @@ -986,9 +995,9 @@ def set_input_embeddings(self, value): self.model.text_model.set_input_embeddings(value) def get_image_features( - self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None - ): - return self.model.get_image_features(pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask) + self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + return self.model.get_image_features(pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask, return_dict=return_dict) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index 9efb9287d9c6..a28a5cacb119 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -27,7 +27,7 @@ from ...masking_utils import create_bidirectional_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, ModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -567,8 +567,8 @@ def inputs_merger( return inputs_embeds def get_image_features( - self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None - ): + self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, return_dict: bool = False + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -577,6 +577,8 @@ def get_image_features( The tensors corresponding to the input images. pixel_attention_mask (`torch.LongTensor`, *optional*): The attention mask indicating padded regions in the image. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ batch_size, num_images, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility @@ -606,11 +608,18 @@ def get_image_features( # Get sequence from the vision encoder image_hidden_states = self.vision_model(pixel_values=pixel_values, patch_attention_mask=patch_attention_mask) - image_hidden_states.last_hidden_state + image_hidden_states = image_hidden_states.last_hidden_state # Modality projection & resampling - image_hidden_states = self.connector(image_hidden_states.last_hidden_state) - return image_hidden_states + image_features = self.connector(image_hidden_states) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_hidden_states, + pooler_output=image_features, + ) + + return image_features @can_return_tuple @auto_docstring( diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index eb97ea42623f..3a4a7a392706 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -1216,10 +1216,16 @@ def get_image_features( query_output = query_outputs[0][:, : query_tokens.size(1), :] # step 3: use the language model, conditioned on the query outputs and the prompt - language_model_inputs = self.language_projection(query_output) + image_features = self.language_projection(query_output) + + # NOTE: @Tom backwards incompatibility if return_dict: - return language_model_inputs, vision_outputs, query_outputs - return language_model_inputs + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + + return image_features def get_placeholder_mask(self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor): """ diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 42f6a7aea58e..2c0b791551cd 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -1216,6 +1216,7 @@ def get_image_features( pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): The tensors corresponding to the input images. """ + # NOTE: @Tom perhaps we should just raise an error here instead? pass def get_placeholder_mask(self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor): diff --git a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py index 591cff7e692b..01e279e8caf4 100644 --- a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py @@ -353,6 +353,7 @@ def get_image_features( interpolate_pos_encoding: Optional[bool] = False, return_dict: Optional[bool] = False, ): + # NOTE: @Tom perhaps we should just raise an error here instead? pass def get_placeholder_mask(self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor): diff --git a/src/transformers/models/internvl/modeling_internvl.py b/src/transformers/models/internvl/modeling_internvl.py index e2ed5fa9b375..4a436dbae8eb 100644 --- a/src/transformers/models/internvl/modeling_internvl.py +++ b/src/transformers/models/internvl/modeling_internvl.py @@ -554,8 +554,9 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, **kwargs, - ): + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -564,6 +565,9 @@ def get_image_features( The tensors corresponding to the input images. vision_feature_layer (`int` or `list[int]`): Layer index or list of layer indices to extract features from. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: vision_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`. """ @@ -578,10 +582,11 @@ def get_image_features( pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio + vision_outputs = self.vision_tower(pixel_values=pixel_values) if vision_feature_layer == -1: - vision_features = self.vision_tower(pixel_values=pixel_values).last_hidden_state + vision_features = vision_outputs.last_hidden_state else: - vision_features = self.vision_model(pixel_values=pixel_values).hidden_states[vision_feature_layer] + vision_features = vision_outputs.hidden_states[vision_feature_layer] if vision_feature_select_strategy == "default": vision_features = vision_features[:, 1:, :] @@ -601,6 +606,13 @@ def get_image_features( # Project features through multi-modal projector vision_features = self.multi_modal_projector(vision_features) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=vision_features, + ) + return vision_features def get_placeholder_mask( diff --git a/src/transformers/models/internvl/modular_internvl.py b/src/transformers/models/internvl/modular_internvl.py index d7a75e5d1677..6b162f42a26b 100644 --- a/src/transformers/models/internvl/modular_internvl.py +++ b/src/transformers/models/internvl/modular_internvl.py @@ -497,8 +497,9 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, **kwargs, - ): + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -507,6 +508,9 @@ def get_image_features( The tensors corresponding to the input images. vision_feature_layer (`int` or `list[int]`): Layer index or list of layer indices to extract features from. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: vision_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`. """ @@ -521,10 +525,11 @@ def get_image_features( pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio + vision_outputs = self.vision_tower(pixel_values=pixel_values) if vision_feature_layer == -1: - vision_features = self.vision_tower(pixel_values=pixel_values).last_hidden_state + vision_features = vision_outputs.last_hidden_state else: - vision_features = self.vision_model(pixel_values=pixel_values).hidden_states[vision_feature_layer] + vision_features = vision_outputs.hidden_states[vision_feature_layer] if vision_feature_select_strategy == "default": vision_features = vision_features[:, 1:, :] @@ -544,6 +549,13 @@ def get_image_features( # Project features through multi-modal projector vision_features = self.multi_modal_projector(vision_features) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=vision_features, + ) + return vision_features @can_return_tuple diff --git a/src/transformers/models/janus/modeling_janus.py b/src/transformers/models/janus/modeling_janus.py index 8bf106074dda..98ebda983e5d 100644 --- a/src/transformers/models/janus/modeling_janus.py +++ b/src/transformers/models/janus/modeling_janus.py @@ -1082,10 +1082,17 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values): - image_embeds = self.vision_model(pixel_values) - image_embeds = self.aligner(image_embeds.last_hidden_state) - return image_embeds + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): + vision_outputs = self.vision_model(pixel_values) + image_features = self.aligner(vision_outputs.last_hidden_state) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + + return image_features def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor diff --git a/src/transformers/models/janus/modular_janus.py b/src/transformers/models/janus/modular_janus.py index 4aa52f274687..ed229148d9d3 100644 --- a/src/transformers/models/janus/modular_janus.py +++ b/src/transformers/models/janus/modular_janus.py @@ -45,7 +45,7 @@ valid_images, validate_preprocess_arguments, ) -from ...modeling_outputs import ModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import ImagesKwargs, Unpack from ...utils import ( @@ -898,10 +898,17 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values): - image_embeds = self.vision_model(pixel_values) - image_embeds = self.aligner(image_embeds.last_hidden_state) - return image_embeds + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): + vision_outputs = self.vision_model(pixel_values) + image_features = self.aligner(vision_outputs.last_hidden_state) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + + return image_features def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor diff --git a/src/transformers/models/kosmos2/modeling_kosmos2.py b/src/transformers/models/kosmos2/modeling_kosmos2.py index e5c62aa7e1c1..f55cf80fb257 100644 --- a/src/transformers/models/kosmos2/modeling_kosmos2.py +++ b/src/transformers/models/kosmos2/modeling_kosmos2.py @@ -1482,9 +1482,10 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, - return_attentions: Optional[bool] = False, + return_attentions: Optional[bool] = False, # TODO: @Tom neatly deprecate this interpolate_pos_encoding: Optional[bool] = False, - ): + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1495,6 +1496,8 @@ def get_image_features( Whether to return `projection_attentions` or not. interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): Whether to interpolate positional embeddings or not. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ vision_model_output = self.vision_model( pixel_values=pixel_values, @@ -1506,8 +1509,13 @@ def get_image_features( image_embeds = nn.functional.normalize(image_embeds, dim=-1) image_embeds, projection_attentions = self.image_to_text_projection(image_embeds) - if return_attentions: - return image_embeds, projection_attentions + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_model_output.last_hidden_state, + pooler_output=image_embeds, + attentions=projection_attentions, # TODO: @Tom does this match expectations? + ) + return image_embeds @can_return_tuple diff --git a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py index 4ef98a251bfa..6bc7497d4d90 100755 --- a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py @@ -28,7 +28,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -166,6 +166,7 @@ def get_image_features( pixel_values: torch.FloatTensor, spatial_shapes: torch.Tensor, pixel_attention_mask: torch.Tensor, + return_dict: bool = False, **kwargs, ) -> list[torch.Tensor]: """ @@ -178,6 +179,9 @@ def get_image_features( The spatial shapes of the input images. pixel_attention_mask (`torch.Tensor` of shape `(batch_size, height, width)`): The pixel attention mask of the input images. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`list[torch.Tensor]`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -185,13 +189,14 @@ def get_image_features( pixel_values=pixel_values, spatial_shapes=spatial_shapes, pixel_attention_mask=pixel_attention_mask, - ).last_hidden_state + ) + last_hidden_state = image_outputs.last_hidden_state img_feature_lengths = pixel_attention_mask.sum(dim=1) image_features = [] - for img_idx in range(image_outputs.size(0)): - feature = image_outputs[img_idx] + for img_idx in range(last_hidden_state.size(0)): + feature = last_hidden_state[img_idx] # unpad the image representation feature = feature[: img_feature_lengths[img_idx], :].unsqueeze(0) @@ -206,6 +211,13 @@ def get_image_features( img_embedding = img_embedding.reshape(-1, img_embedding.size(-1)) image_features.append(img_embedding) + # NOTE: @Tom Not easily converted to the standard format due to variable lengths + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( diff --git a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py index 4115a27cbf9c..ae3f3eb647a3 100644 --- a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py @@ -21,6 +21,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from ..llava.modeling_llava import ( @@ -97,6 +98,7 @@ def get_image_features( pixel_values: torch.FloatTensor, spatial_shapes: torch.Tensor, pixel_attention_mask: torch.Tensor, + return_dict: bool = False, **kwargs, ) -> list[torch.Tensor]: """ @@ -109,6 +111,9 @@ def get_image_features( The spatial shapes of the input images. pixel_attention_mask (`torch.Tensor` of shape `(batch_size, height, width)`): The pixel attention mask of the input images. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`list[torch.Tensor]`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -116,13 +121,14 @@ def get_image_features( pixel_values=pixel_values, spatial_shapes=spatial_shapes, pixel_attention_mask=pixel_attention_mask, - ).last_hidden_state + ) + last_hidden_state = image_outputs.last_hidden_state img_feature_lengths = pixel_attention_mask.sum(dim=1) image_features = [] - for img_idx in range(image_outputs.size(0)): - feature = image_outputs[img_idx] + for img_idx in range(last_hidden_state.size(0)): + feature = last_hidden_state[img_idx] # unpad the image representation feature = feature[: img_feature_lengths[img_idx], :].unsqueeze(0) @@ -137,6 +143,13 @@ def get_image_features( img_embedding = img_embedding.reshape(-1, img_embedding.size(-1)) image_features.append(img_embedding) + # NOTE: @Tom Not easily converted to the standard format due to variable lengths + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index df80dd6716d2..e04806e0f806 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -23,7 +23,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -152,6 +152,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, **kwargs, ): """ @@ -209,6 +210,13 @@ def get_image_features( image_features = torch.split(image_features.squeeze(0), split_sizes) else: image_features = list(image_features) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index cf0ebf1ce869..8000023ad436 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -28,7 +28,7 @@ from ...generation import GenerationMixin from ...image_processing_utils import select_best_resolution from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -352,6 +352,7 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -368,6 +369,9 @@ def get_image_features( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (list[`torch.Tensor`]): List of image feature tensor, each contains all the visual feature of all patches and are of shape `(num_patches, image_length, embed_dim)`). @@ -398,13 +402,13 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_features = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): - selected_image_feature = image_features.hidden_states[vision_feature_layer] + selected_image_feature = image_outputs.hidden_states[vision_feature_layer] else: - hs_pool = [image_features.hidden_states[layer_idx] for layer_idx in vision_feature_layer] + hs_pool = [image_outputs.hidden_states[layer_idx] for layer_idx in vision_feature_layer] selected_image_feature = torch.cat(hs_pool, dim=-1) if vision_feature_select_strategy == "default": @@ -420,6 +424,13 @@ def get_image_features( vision_feature_select_strategy=vision_feature_select_strategy, image_newline=self.image_newline, ) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( @@ -570,12 +581,14 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, ): return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, + return_dict=return_dict, ) @can_return_tuple diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index eed31b38096d..e9b02b7c158b 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -33,7 +33,7 @@ from ...generation import GenerationMixin from ...image_processing_utils import select_best_resolution from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -404,6 +404,7 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -450,13 +451,13 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_features = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): - selected_image_feature = image_features.hidden_states[vision_feature_layer] + selected_image_feature = image_outputs.hidden_states[vision_feature_layer] else: - hs_pool = [image_features.hidden_states[layer_idx] for layer_idx in vision_feature_layer] + hs_pool = [image_outputs.hidden_states[layer_idx] for layer_idx in vision_feature_layer] selected_image_feature = torch.cat(hs_pool, dim=-1) if vision_feature_select_strategy == "default": @@ -470,6 +471,13 @@ def get_image_features( vision_feature_select_strategy, image_newline=self.image_newline, ) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( @@ -709,12 +717,14 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, ): return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, + return_dict=return_dict, ) @can_return_tuple diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index 61b4b5fdf920..6270b2bcdd45 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -32,6 +32,7 @@ from ...cache_utils import Cache from ...configuration_utils import PreTrainedConfig +from ...modeling_outputs import BaseModelOutputWithPooling from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...processing_utils import Unpack from ...utils import logging @@ -278,6 +279,7 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -324,13 +326,13 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_features = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): - selected_image_feature = image_features.hidden_states[vision_feature_layer] + selected_image_feature = image_outputs.hidden_states[vision_feature_layer] else: - hs_pool = [image_features.hidden_states[layer_idx] for layer_idx in vision_feature_layer] + hs_pool = [image_outputs.hidden_states[layer_idx] for layer_idx in vision_feature_layer] selected_image_feature = torch.cat(hs_pool, dim=-1) if vision_feature_select_strategy == "default": @@ -344,6 +346,13 @@ def get_image_features( vision_feature_select_strategy, image_newline=self.image_newline, ) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_video_features( diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index 260177a63796..345a84b6584a 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -33,7 +33,7 @@ from ...generation import GenerationMixin from ...image_processing_utils import select_best_resolution from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -363,6 +363,7 @@ def get_image_features( vision_feature_select_strategy: Optional[str] = None, vision_aspect_ratio: Optional[str] = None, batch_num_images: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -421,13 +422,13 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_features = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): - selected_image_feature = image_features.hidden_states[vision_feature_layer] + selected_image_feature = image_outputs.hidden_states[vision_feature_layer] else: - hs_pool = [image_features.hidden_states[layer_idx] for layer_idx in vision_feature_layer] + hs_pool = [image_outputs.hidden_states[layer_idx] for layer_idx in vision_feature_layer] selected_image_feature = torch.cat(hs_pool, dim=-1) if vision_feature_select_strategy == "default": @@ -441,6 +442,13 @@ def get_image_features( image_newline=self.image_newline, vision_aspect_ratio=vision_aspect_ratio, ) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( @@ -697,12 +705,14 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, ): return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, + return_dict=return_dict, ) @can_return_tuple diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index dd714def07c2..52ce2771def0 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -45,6 +45,7 @@ SizeDict, get_image_size, ) +from ...modeling_outputs import BaseModelOutputWithPooling from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...processing_utils import Unpack from ...utils import ( @@ -322,6 +323,7 @@ def get_image_features( vision_feature_select_strategy: Optional[str] = None, vision_aspect_ratio: Optional[str] = None, batch_num_images: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -380,13 +382,13 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_features = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): - selected_image_feature = image_features.hidden_states[vision_feature_layer] + selected_image_feature = image_outputs.hidden_states[vision_feature_layer] else: - hs_pool = [image_features.hidden_states[layer_idx] for layer_idx in vision_feature_layer] + hs_pool = [image_outputs.hidden_states[layer_idx] for layer_idx in vision_feature_layer] selected_image_feature = torch.cat(hs_pool, dim=-1) if vision_feature_select_strategy == "default": @@ -400,6 +402,13 @@ def get_image_features( image_newline=self.image_newline, vision_aspect_ratio=vision_aspect_ratio, ) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_video_features( diff --git a/src/transformers/models/metaclip_2/modeling_metaclip_2.py b/src/transformers/models/metaclip_2/modeling_metaclip_2.py index 784234cdfe10..2f3e8bd59e1a 100644 --- a/src/transformers/models/metaclip_2/modeling_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modeling_metaclip_2.py @@ -843,8 +843,12 @@ def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`MetaClip2VisionModel`]. @@ -873,6 +877,12 @@ def get_image_features( pooled_output = vision_outputs.pooler_output image_features = self.visual_projection(pooled_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @can_return_tuple diff --git a/src/transformers/models/metaclip_2/modular_metaclip_2.py b/src/transformers/models/metaclip_2/modular_metaclip_2.py index 7f44bc9b20be..c5665e42083e 100644 --- a/src/transformers/models/metaclip_2/modular_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modular_metaclip_2.py @@ -589,8 +589,12 @@ def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, + return_dict: bool = False, ): r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`MetaClip2VisionModel`]. @@ -615,6 +619,7 @@ def get_image_features( return super().get_image_features( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=return_dict, ) diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index a944ff4f9055..2bfa05c39244 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -29,7 +29,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...integrations import use_kernel_forward_from_hub -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -218,6 +218,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, + return_dict: bool = False, **kwargs, ): """ @@ -232,6 +233,9 @@ def get_image_features( vision features. image_sizes (`torch.Tensor`, *optional*): Tensor containing the image sizes as returned by the processor. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -254,6 +258,13 @@ def get_image_features( downsample_ratio = self.vision_tower.patch_size * self.config.spatial_merge_size split_sizes = [(height // downsample_ratio) * (width // downsample_ratio) for height, width in image_sizes] image_features = torch.split(image_features.squeeze(0), split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( diff --git a/src/transformers/models/mistral3/modular_mistral3.py b/src/transformers/models/mistral3/modular_mistral3.py index 6bc499d21453..acef601186f5 100644 --- a/src/transformers/models/mistral3/modular_mistral3.py +++ b/src/transformers/models/mistral3/modular_mistral3.py @@ -20,6 +20,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import logging from ..llava.modeling_llava import ( @@ -123,6 +124,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, + return_dict: bool = False, **kwargs, ): """ @@ -137,6 +139,9 @@ def get_image_features( vision features. image_sizes (`torch.Tensor`, *optional*): Tensor containing the image sizes as returned by the processor. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -159,6 +164,13 @@ def get_image_features( downsample_ratio = self.vision_tower.patch_size * self.config.spatial_merge_size split_sizes = [(height // downsample_ratio) * (width // downsample_ratio) for height, width in image_sizes] image_features = torch.split(image_features.squeeze(0), split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def forward( diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index c7480492d381..f9c63bd7a637 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -557,6 +557,7 @@ def get_image_features( ).to(image_features.device) visual_indicator_features = self.visual_embeddings_table(visual_indicator) + # NOTE: @Tom Not easily converted to the standard format return image_features, visual_indicator_features def get_placeholder_mask( diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index 6faf887eb333..80847f4ccfea 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -249,6 +249,7 @@ def get_image_features( ).to(image_features.device) visual_indicator_features = self.visual_embeddings_table(visual_indicator) + # NOTE: @Tom Not easily converted to the standard format return image_features, visual_indicator_features @can_return_tuple diff --git a/src/transformers/models/owlv2/modeling_owlv2.py b/src/transformers/models/owlv2/modeling_owlv2.py index bf918817e162..4f3f6682e025 100644 --- a/src/transformers/models/owlv2/modeling_owlv2.py +++ b/src/transformers/models/owlv2/modeling_owlv2.py @@ -1020,8 +1020,12 @@ def get_image_features( self, pixel_values: torch.Tensor, interpolate_pos_encoding: bool = False, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`Owlv2VisionModel`]. @@ -1048,6 +1052,12 @@ def get_image_features( ) image_features = self.visual_projection(vision_outputs.pooler_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @auto_docstring diff --git a/src/transformers/models/owlvit/modeling_owlvit.py b/src/transformers/models/owlvit/modeling_owlvit.py index 7cbea96d256a..5e755bf73f3b 100644 --- a/src/transformers/models/owlvit/modeling_owlvit.py +++ b/src/transformers/models/owlvit/modeling_owlvit.py @@ -1001,8 +1001,12 @@ def get_image_features( self, pixel_values: torch.Tensor, interpolate_pos_encoding: bool = False, - ) -> torch.FloatTensor: + return_dict: bool = False, + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`OwlViTVisionModel`]. @@ -1029,6 +1033,12 @@ def get_image_features( ) image_features = self.visual_projection(vision_outputs.pooler_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @auto_docstring diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index d147f0b953ac..965412df4ec7 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -26,7 +26,7 @@ from ...generation import GenerationMixin from ...masking_utils import create_masks_for_generate from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import ( @@ -257,13 +257,16 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.FloatTensor): + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): """ Obtains image last hidden states from the vision tower and apply multimodal projection. Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) The tensors corresponding to the input images. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -271,6 +274,13 @@ def get_image_features(self, pixel_values: torch.FloatTensor): selected_image_feature = image_outputs.last_hidden_state image_features = self.multi_modal_projector(selected_image_feature) image_features = image_features / (self.config.text_config.hidden_size**0.5) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=selected_image_feature, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( @@ -444,8 +454,8 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) - def get_image_features(self, pixel_values): - return self.model.get_image_features(pixel_values) + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): + return self.model.get_image_features(pixel_values, return_dict=return_dict) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/perception_lm/modeling_perception_lm.py b/src/transformers/models/perception_lm/modeling_perception_lm.py index cb41dfbb86aa..6902b3fd2827 100644 --- a/src/transformers/models/perception_lm/modeling_perception_lm.py +++ b/src/transformers/models/perception_lm/modeling_perception_lm.py @@ -28,7 +28,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...utils import auto_docstring, can_return_tuple from ..auto import AutoModel @@ -183,6 +183,7 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, + return_dict: bool = False, **kwargs, ): """ @@ -191,14 +192,24 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_tiles, channels, height, width)`) The tensors corresponding to the input images. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_tiles, num_patches, embed_dim)`). """ image_outputs = self.vision_tower(pixel_values.flatten(0, 1)) - image_outputs = image_outputs.last_hidden_state + last_hidden_state = image_outputs.last_hidden_state if self.config.vision_use_cls_token: - image_outputs = image_outputs[:, 1:, :] - image_features = self.multi_modal_projector(image_outputs) + last_hidden_state = last_hidden_state[:, 1:, :] + image_features = self.multi_modal_projector(last_hidden_state) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( diff --git a/src/transformers/models/perception_lm/modular_perception_lm.py b/src/transformers/models/perception_lm/modular_perception_lm.py index 823163e23749..77485b38832b 100644 --- a/src/transformers/models/perception_lm/modular_perception_lm.py +++ b/src/transformers/models/perception_lm/modular_perception_lm.py @@ -21,6 +21,7 @@ from torch import nn from ...cache_utils import Cache +from ...modeling_outputs import BaseModelOutputWithPooling from ...utils import ( auto_docstring, can_return_tuple, @@ -148,6 +149,7 @@ def __init__(self, config: PerceptionLMConfig): def get_image_features( self, pixel_values: torch.FloatTensor, + return_dict: bool = False, **kwargs, ): """ @@ -156,14 +158,24 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_tiles, channels, height, width)`) The tensors corresponding to the input images. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_tiles, num_patches, embed_dim)`). """ image_outputs = self.vision_tower(pixel_values.flatten(0, 1)) - image_outputs = image_outputs.last_hidden_state + last_hidden_state = image_outputs.last_hidden_state if self.config.vision_use_cls_token: - image_outputs = image_outputs[:, 1:, :] - image_features = self.multi_modal_projector(image_outputs) + last_hidden_state = last_hidden_state[:, 1:, :] + image_features = self.multi_modal_projector(last_hidden_state) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index ef5e846c83fc..64c66f7b5589 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -37,7 +37,7 @@ from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -1732,7 +1732,12 @@ def get_video_features( video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw) return video_embeds - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, + ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1744,6 +1749,14 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op """ pixel_values = pixel_values.type(self.visual.dtype) image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_embeds, + # pooler_output=image_features, # NOTE: @Tom no pooled embeddings here + ) + + # NOTE: @Tom Not easily converted to the standard format return image_embeds def get_audio_features( diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index 2d179285d5f9..2f92667006ec 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -44,7 +44,7 @@ from ...cache_utils import Cache from ...configuration_utils import PreTrainedConfig, layer_type_validation from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutput, ModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput from ...modeling_rope_utils import RopeParameters from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...processing_utils import Unpack @@ -2080,7 +2080,7 @@ def get_video_features( video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw) return video_embeds - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -2092,6 +2092,14 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op """ pixel_values = pixel_values.type(self.visual.dtype) image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_embeds, + # pooler_output=image_features, # NOTE: @Tom no pooled embeddings here + ) + + # NOTE: @Tom Not easily converted to the standard format return image_embeds def get_audio_features( diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index e5d1ca7e1943..222622c61528 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -1190,6 +1190,7 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) + # NOTE: @Tom Not easily converted to the standard format return image_embeds def get_placeholder_mask( diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index cbe2848d8f7b..c2f9f0a7b8d6 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -1125,6 +1125,7 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) + # NOTE: @Tom Not easily converted to the standard format return image_embeds def get_placeholder_mask( diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index a04ddc842e0c..eaa15ce53e3d 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -42,6 +42,7 @@ from ...modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPast, + BaseModelOutputWithPooling, CausalLMOutputWithPast, MoeCausalLMOutputWithPast, MoeModelOutputWithPast, @@ -1924,7 +1925,12 @@ def get_video_features( video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw) return video_embeds - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, + ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1936,6 +1942,14 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op """ pixel_values = pixel_values.type(self.visual.dtype) image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_embeds, + # pooler_output=image_features, # NOTE: @Tom no pooled embeddings here + ) + + # NOTE: @Tom Not easily converted to the standard format return image_embeds def get_audio_features( diff --git a/src/transformers/models/smolvlm/modeling_smolvlm.py b/src/transformers/models/smolvlm/modeling_smolvlm.py index e85048037c33..84c32a5413c5 100644 --- a/src/transformers/models/smolvlm/modeling_smolvlm.py +++ b/src/transformers/models/smolvlm/modeling_smolvlm.py @@ -32,7 +32,7 @@ from ...masking_utils import create_bidirectional_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, ModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import ( @@ -519,7 +519,7 @@ def inputs_merger( def get_image_features( self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None - ): + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. From b6d6df3b8baae15438fbba0654a54758bf4f2132 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 2 Dec 2025 16:52:09 +0100 Subject: [PATCH 003/131] make fixup --- src/transformers/models/align/modeling_align.py | 4 +++- src/transformers/models/blip/modeling_blip.py | 6 +++++- .../modular_deepseek_vl_hybrid.py | 4 +++- src/transformers/models/emu3/modular_emu3.py | 2 +- src/transformers/models/gemma3/modular_gemma3.py | 4 +++- src/transformers/models/gemma3n/modular_gemma3n.py | 8 ++++---- .../models/groupvit/modeling_groupvit.py | 4 +++- .../models/idefics2/modeling_idefics2.py | 14 +++++++++++--- .../models/idefics3/modeling_idefics3.py | 5 ++++- .../models/kosmos2/modeling_kosmos2.py | 2 +- .../llava_next_video/modular_llava_next_video.py | 2 +- .../llava_onevision/modular_llava_onevision.py | 2 +- .../models/qwen2_5_omni/modular_qwen2_5_omni.py | 7 ++++++- src/transformers/models/sam3/modeling_sam3.py | 5 +---- 14 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/transformers/models/align/modeling_align.py b/src/transformers/models/align/modeling_align.py index 54b4e1d64244..1d4015eaf017 100644 --- a/src/transformers/models/align/modeling_align.py +++ b/src/transformers/models/align/modeling_align.py @@ -1138,7 +1138,9 @@ def get_text_features( @filter_out_non_signature_kwargs() @auto_docstring - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + def get_image_features( + self, pixel_values: torch.FloatTensor, return_dict: bool = False + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" return_dict (`bool`, *optional*, default to `False`): Whether to return a `ModelOutput` instead of a pooled embedding. diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index dace6ee5a64c..d145055b1f4a 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -26,7 +26,11 @@ from ...activations import ACT2FN from ...generation import GenerationMixin from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, BaseModelOutputWithPoolingAndCrossAttentions +from ...modeling_outputs import ( + BaseModelOutput, + BaseModelOutputWithPooling, + BaseModelOutputWithPoolingAndCrossAttentions, +) from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, logging, torch_int diff --git a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py index afb8d309e26d..834786e42dfa 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py @@ -277,7 +277,9 @@ def get_high_res_image_features(self, pixel_values): return output - def get_image_features(self, pixel_values: torch.FloatTensor, high_res_pixel_values: torch.FloatTensor, return_dict: bool = False): + def get_image_features( + self, pixel_values: torch.FloatTensor, high_res_pixel_values: torch.FloatTensor, return_dict: bool = False + ): vision_encodings = self.get_low_res_image_features(pixel_values) high_res_vision_encodings = self.get_high_res_image_features(high_res_pixel_values) image_features = self.aligner(vision_encodings, high_res_vision_encodings) diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index 5484b742c480..e2cad252c2af 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -25,7 +25,7 @@ from ... import initialization as init from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPooling, CausalLMOutputWithPast +from ...modeling_outputs import CausalLMOutputWithPast from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import auto_docstring, can_return_tuple, logging diff --git a/src/transformers/models/gemma3/modular_gemma3.py b/src/transformers/models/gemma3/modular_gemma3.py index f485a63a87b4..7aa7b24ec6e0 100644 --- a/src/transformers/models/gemma3/modular_gemma3.py +++ b/src/transformers/models/gemma3/modular_gemma3.py @@ -825,7 +825,9 @@ def __init__(self, config: Gemma3Config): super().__init__(config) del self.text_config_dtype - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + def get_image_features( + self, pixel_values: torch.FloatTensor, return_dict: bool = False + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index dbbc4f996a11..352d8658b4de 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -2200,7 +2200,9 @@ def __init__(self, config: Gemma3nConfig): self.embed_vision = Gemma3nMultimodalEmbedder(config.vision_config, config.text_config) self.embed_audio = Gemma3nMultimodalEmbedder(config.audio_config, config.text_config) - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + def get_image_features( + self, pixel_values: torch.FloatTensor, return_dict: bool = False + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. @@ -2213,9 +2215,7 @@ def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - vision_outputs = self.vision_tower( - pixel_values=pixel_values, do_pooling=False, return_dict=True - ) + vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, return_dict=True) last_hidden_state = vision_outputs.last_hidden_state # Convert from (batch, channels, height, width) to (batch, height * width, channels) where: # height == width and height * width == Gemma3nConfig.vision_soft_tokens_per_image. diff --git a/src/transformers/models/groupvit/modeling_groupvit.py b/src/transformers/models/groupvit/modeling_groupvit.py index 327c7472d8f4..6ef3d9121e26 100644 --- a/src/transformers/models/groupvit/modeling_groupvit.py +++ b/src/transformers/models/groupvit/modeling_groupvit.py @@ -1269,7 +1269,9 @@ def get_text_features( @filter_out_non_signature_kwargs() @auto_docstring - def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + def get_image_features( + self, pixel_values: torch.Tensor, return_dict: bool = False + ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index 74ce1b751bbe..dff7952e293d 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -822,7 +822,10 @@ def inputs_merger( return inputs_embeds def get_image_features( - self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, return_dict: bool = False, + self, + pixel_values: torch.FloatTensor, + pixel_attention_mask: Optional[torch.LongTensor] = None, + return_dict: bool = False, ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -995,9 +998,14 @@ def set_input_embeddings(self, value): self.model.text_model.set_input_embeddings(value) def get_image_features( - self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, return_dict: bool = False, + self, + pixel_values: torch.FloatTensor, + pixel_attention_mask: Optional[torch.LongTensor] = None, + return_dict: bool = False, ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: - return self.model.get_image_features(pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask, return_dict=return_dict) + return self.model.get_image_features( + pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask, return_dict=return_dict + ) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index a28a5cacb119..7967b8ef9dd6 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -567,7 +567,10 @@ def inputs_merger( return inputs_embeds def get_image_features( - self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, return_dict: bool = False + self, + pixel_values: torch.FloatTensor, + pixel_attention_mask: Optional[torch.LongTensor] = None, + return_dict: bool = False, ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/kosmos2/modeling_kosmos2.py b/src/transformers/models/kosmos2/modeling_kosmos2.py index f55cf80fb257..78847329174f 100644 --- a/src/transformers/models/kosmos2/modeling_kosmos2.py +++ b/src/transformers/models/kosmos2/modeling_kosmos2.py @@ -1482,7 +1482,7 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, - return_attentions: Optional[bool] = False, # TODO: @Tom neatly deprecate this + return_attentions: Optional[bool] = False, # TODO: @Tom neatly deprecate this interpolate_pos_encoding: Optional[bool] = False, return_dict: bool = False, ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index 6270b2bcdd45..5f13e7981f38 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -32,8 +32,8 @@ from ...cache_utils import Cache from ...configuration_utils import PreTrainedConfig -from ...modeling_outputs import BaseModelOutputWithPooling from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import logging from ..auto import CONFIG_MAPPING, AutoConfig diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index 52ce2771def0..f84246d9c12f 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -45,8 +45,8 @@ SizeDict, get_image_size, ) -from ...modeling_outputs import BaseModelOutputWithPooling from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import ( TensorType, diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index 2f92667006ec..106e668b0cbe 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -2080,7 +2080,12 @@ def get_video_features( video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw) return video_embeds - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False): + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, + ): """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/sam3/modeling_sam3.py b/src/transformers/models/sam3/modeling_sam3.py index fe3f9ff2757d..da7b8b8d25fd 100644 --- a/src/transformers/models/sam3/modeling_sam3.py +++ b/src/transformers/models/sam3/modeling_sam3.py @@ -2138,7 +2138,6 @@ def get_text_features( return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: - r""" return_dict (`bool`, *optional*, default to `False`): Whether to return a `ModelOutput` instead of a pooled embedding. @@ -2168,9 +2167,7 @@ def get_text_features( >>> outputs = model(pixel_values=img_inputs.pixel_values, text_embeds=text_embeds) ``` """ - text_features = self.text_encoder( - input_ids=input_ids, attention_mask=attention_mask, **kwargs - ) + text_features = self.text_encoder(input_ids=input_ids, attention_mask=attention_mask, **kwargs) last_hidden_state = text_features.last_hidden_state text_features = self.text_projection(last_hidden_state) From aa5141971dc1ce931f8935fac808c84833542968 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 4 Dec 2025 11:08:58 +0100 Subject: [PATCH 004/131] Ignore discrepancies for pooler_output, focus on last_hidden_state --- .../models/chameleon/modeling_chameleon.py | 14 ++----- src/transformers/models/emu3/modeling_emu3.py | 18 ++++++--- src/transformers/models/emu3/modular_emu3.py | 16 +++++--- .../models/gemma3n/modeling_gemma3n.py | 1 - .../models/gemma3n/modular_gemma3n.py | 1 - .../models/glm46v/modeling_glm46v.py | 23 ++++++++--- .../models/glm4v/modeling_glm4v.py | 23 ++++++++--- .../models/glm4v_moe/modeling_glm4v_moe.py | 23 ++++++++--- .../modeling_instructblipvideo.py | 1 - .../modular_instructblipvideo.py | 1 - .../models/lfm2_vl/modeling_lfm2_vl.py | 1 - .../models/lfm2_vl/modular_lfm2_vl.py | 1 - .../qwen2_5_omni/modeling_qwen2_5_omni.py | 30 +++++++------- .../qwen2_5_omni/modular_qwen2_5_omni.py | 28 ++++++------- .../models/qwen2_5_vl/modeling_qwen2_5_vl.py | 39 +++++++++++++++---- .../models/qwen2_5_vl/modular_qwen2_5_vl.py | 15 +++++-- .../models/qwen2_vl/modeling_qwen2_vl.py | 29 +++++++++++--- .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 14 ++----- 18 files changed, 184 insertions(+), 94 deletions(-) diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 76cfa3411e07..b4097854376a 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -859,23 +859,18 @@ def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool them with text embeddings layer Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): The tensors corresponding to the input images. return_dict (`bool`, *optional*, default to `False`): Whether to return a `ModelOutput` instead of a pooled embedding. """ image_tokens = self.get_image_tokens(pixel_values) image_embeddings = self.get_input_embeddings()(image_tokens) - image_features = image_embeddings.mean(dim=1) if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_embeddings, - pooler_output=image_features, - ) + return BaseModelOutputWithPooling(last_hidden_state=image_embeddings) - # NOTE: @Tom Not easily converted to the standard format, this one breaks backward compatibility - return image_features + return image_embeddings def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -937,8 +932,7 @@ def forward( inputs_embeds = self.embed_tokens(input_ids) if pixel_values is not None: - image_tokens = self.get_image_tokens(pixel_values) - image_embeds = self.get_input_embeddings()(image_tokens) + image_embeds = self.get_image_features(pixel_values) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds ) diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index fb466c2984a4..e2d0136bbf3f 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -36,7 +36,7 @@ from ...integrations import use_kernel_forward_from_hub, use_kernel_func_from_hub from ...masking_utils import create_causal_mask from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -1373,7 +1373,9 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.L bpe_tokens = torch.cat(bpe_tokens_list) return bpe_tokens - def get_image_features(self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor): + def get_image_features( + self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, return_dict: bool = False + ): """ Tokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layer @@ -1387,9 +1389,15 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_sizes: torch (height // self.vqmodel.vision_spatial_factor) * (width // self.vqmodel.vision_spatial_factor + 1) for height, width in image_sizes ] - image_features = self.get_input_embeddings()(image_tokens) - image_features = torch.split(image_features, split_sizes) - # NOTE: @Tom Not easily converted to the standard format + image_embeddings = self.get_input_embeddings()(image_tokens) + image_features = torch.split(image_embeddings, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_embeddings, + pooler_output=image_features, + ) + return image_features @torch.no_grad() diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index e2cad252c2af..5a137cec3329 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -25,7 +25,7 @@ from ... import initialization as init from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPooling, CausalLMOutputWithPast from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import auto_docstring, can_return_tuple, logging @@ -925,7 +925,7 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.L bpe_tokens = torch.cat(bpe_tokens_list) return bpe_tokens - def get_image_features(self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor): + def get_image_features(self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, return_dict: bool = False): """ Tokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layer @@ -939,9 +939,15 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_sizes: torch (height // self.vqmodel.vision_spatial_factor) * (width // self.vqmodel.vision_spatial_factor + 1) for height, width in image_sizes ] - image_features = self.get_input_embeddings()(image_tokens) - image_features = torch.split(image_features, split_sizes) - # NOTE: @Tom Not easily converted to the standard format + image_embeddings = self.get_input_embeddings()(image_tokens) + image_features = torch.split(image_embeddings, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_embeddings, + pooler_output=image_features, + ) + return image_features @torch.no_grad() diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index bb78a627e34c..7ca876dfae6f 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -2125,7 +2125,6 @@ def get_image_features( pooler_output=image_features, ) - # NOTE: @Tom I'm not sure the image_features here is 2-dimensional as we'd like return image_features def get_placeholder_mask( diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index 352d8658b4de..87610e2304d0 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -2234,7 +2234,6 @@ def get_image_features( pooler_output=image_features, ) - # NOTE: @Tom I'm not sure the image_features here is 2-dimensional as we'd like return image_features def get_placeholder_mask( diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 1328f724912a..eddf2b68aef8 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -28,7 +28,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import ModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling @@ -314,7 +314,12 @@ def get_video_features( video_embeds = torch.split(video_embeds, split_sizes) return video_embeds - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, + ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -323,12 +328,20 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - image_embeds = torch.split(image_embeds, split_sizes) - # NOTE: @Tom Not easily converted to the standard format + image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_embeds, + ) + return image_embeds def get_placeholder_mask( diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index 223df1299e40..074226220647 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -35,7 +35,7 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -1162,7 +1162,12 @@ def get_video_features( video_embeds = torch.split(video_embeds, split_sizes) return video_embeds - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, + ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1171,12 +1176,20 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - image_embeds = torch.split(image_embeds, split_sizes) - # NOTE: @Tom Not easily converted to the standard format + image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_embeds, + ) + return image_embeds def get_placeholder_mask( diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 1675d30c910f..162d546bb13a 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -36,7 +36,7 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import ModelOutput, MoeModelOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput, MoeModelOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -1371,7 +1371,12 @@ def get_video_features( video_embeds = torch.split(video_embeds, split_sizes) return video_embeds - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, + ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1380,12 +1385,20 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - image_embeds = torch.split(image_embeds, split_sizes) - # NOTE: @Tom Not easily converted to the standard format + image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_embeds, + ) + return image_embeds def get_placeholder_mask( diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 2c0b791551cd..42f6a7aea58e 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -1216,7 +1216,6 @@ def get_image_features( pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): The tensors corresponding to the input images. """ - # NOTE: @Tom perhaps we should just raise an error here instead? pass def get_placeholder_mask(self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor): diff --git a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py index 01e279e8caf4..591cff7e692b 100644 --- a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py @@ -353,7 +353,6 @@ def get_image_features( interpolate_pos_encoding: Optional[bool] = False, return_dict: Optional[bool] = False, ): - # NOTE: @Tom perhaps we should just raise an error here instead? pass def get_placeholder_mask(self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor): diff --git a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py index 6bc7497d4d90..1b78231161af 100755 --- a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py @@ -211,7 +211,6 @@ def get_image_features( img_embedding = img_embedding.reshape(-1, img_embedding.size(-1)) image_features.append(img_embedding) - # NOTE: @Tom Not easily converted to the standard format due to variable lengths if return_dict: return BaseModelOutputWithPooling( last_hidden_state=image_outputs.last_hidden_state, diff --git a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py index ae3f3eb647a3..c3c37e4cc45f 100644 --- a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py @@ -143,7 +143,6 @@ def get_image_features( img_embedding = img_embedding.reshape(-1, img_embedding.size(-1)) image_features.append(img_embedding) - # NOTE: @Tom Not easily converted to the standard format due to variable lengths if return_dict: return BaseModelOutputWithPooling( last_hidden_state=image_outputs.last_hidden_state, diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index 64c66f7b5589..4b3860c9a448 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -1174,13 +1174,17 @@ def get_window_index(self, grid_thw): return window_index, cu_window_seqlens - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor: + def forward( + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + ) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -1227,9 +1231,16 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) rotary_pos_emb=rotary_pos_emb, **kwargs, ) - hidden_states = self.merger(hidden_states) + + merged_hidden_states = self.merger(hidden_states) reverse_indices = torch.argsort(window_index) - hidden_states = hidden_states[reverse_indices, :] + merged_hidden_states = merged_hidden_states[reverse_indices, :] + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) return hidden_states @@ -1746,18 +1757,11 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_embeds, - # pooler_output=image_features, # NOTE: @Tom no pooled embeddings here - ) - - # NOTE: @Tom Not easily converted to the standard format - return image_embeds + return self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=return_dict) def get_audio_features( self, diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index 106e668b0cbe..94924d2cfbd3 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -1926,13 +1926,15 @@ def __init__(self, config: Qwen2_5OmniVisionEncoderConfig, *inputs, **kwargs) -> super().__init__(config, *inputs, **kwargs) self.blocks = nn.ModuleList([Qwen2_5OmniVisionBlock(config) for _ in range(config.depth)]) - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor: + def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -1979,9 +1981,16 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) rotary_pos_emb=rotary_pos_emb, **kwargs, ) - hidden_states = self.merger(hidden_states) + + merged_hidden_states = self.merger(hidden_states) reverse_indices = torch.argsort(window_index) - hidden_states = hidden_states[reverse_indices, :] + merged_hidden_states = merged_hidden_states[reverse_indices, :] + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) return hidden_states @@ -2094,18 +2103,11 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_embeds, - # pooler_output=image_features, # NOTE: @Tom no pooled embeddings here - ) - - # NOTE: @Tom Not easily converted to the standard format - return image_embeds + return self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=return_dict) def get_audio_features( self, diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index 222622c61528..3e8af2c19b03 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -38,7 +38,7 @@ from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -405,13 +405,17 @@ def get_window_index(self, grid_thw): return window_index, cu_window_seqlens - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor: + def forward( + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + ) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -459,9 +463,15 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) **kwargs, ) - hidden_states = self.merger(hidden_states) + merged_hidden_states = self.merger(hidden_states) reverse_indices = torch.argsort(window_index) - hidden_states = hidden_states[reverse_indices, :] + merged_hidden_states = merged_hidden_states[reverse_indices, :] + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) return hidden_states @@ -1176,7 +1186,12 @@ def get_video_features( video_embeds = torch.split(video_embeds, split_sizes) return video_embeds - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, + ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1185,12 +1200,20 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - image_embeds = torch.split(image_embeds, split_sizes) - # NOTE: @Tom Not easily converted to the standard format + image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_embeds, + ) + return image_embeds def get_placeholder_mask( diff --git a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py index a8d88a73d9bc..81391301662d 100644 --- a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py @@ -49,6 +49,7 @@ from ...image_utils import ImageInput from ...modeling_flash_attention_utils import is_flash_attn_available from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import MultiModalData, ProcessingKwargs, Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput from ...utils import logging @@ -277,13 +278,15 @@ def get_window_index(self, grid_thw): return window_index, cu_window_seqlens - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor: + def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -331,9 +334,15 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) **kwargs, ) - hidden_states = self.merger(hidden_states) + merged_hidden_states = self.merger(hidden_states) reverse_indices = torch.argsort(window_index) - hidden_states = hidden_states[reverse_indices, :] + merged_hidden_states = merged_hidden_states[reverse_indices, :] + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) return hidden_states diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index c2f9f0a7b8d6..14592a1b607a 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -34,7 +34,7 @@ from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -733,11 +733,14 @@ def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, + return_dict: bool = False, **kwargs, ) -> torch.Tensor: r""" grid_thw (`torch.LongTensor` of shape `(num_images, 3)`): The temporal, height and width dimensions of feature shape for each image. Each row contains [t, h, w] values. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a `ModelOutput` instead of exclusively the merged hidden states. """ hidden_states = self.patch_embed(hidden_states) rotary_pos_emb = self.rot_pos_emb(grid_thw) @@ -762,6 +765,14 @@ def forward( **kwargs, ) + merged_hidden_states = self.merger(hidden_states) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) + return self.merger(hidden_states) @@ -1111,7 +1122,7 @@ def get_video_features( video_embeds = torch.split(video_embeds, split_sizes) return video_embeds - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1120,12 +1131,20 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - image_embeds = torch.split(image_embeds, split_sizes) - # NOTE: @Tom Not easily converted to the standard format + image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_embeds, + ) + return image_embeds def get_placeholder_mask( diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index eaa15ce53e3d..e8da6efbd87c 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -42,7 +42,6 @@ from ...modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPast, - BaseModelOutputWithPooling, CausalLMOutputWithPast, MoeCausalLMOutputWithPast, MoeModelOutputWithPast, @@ -1939,18 +1938,11 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_embeds, - # pooler_output=image_features, # NOTE: @Tom no pooled embeddings here - ) - - # NOTE: @Tom Not easily converted to the standard format - return image_embeds + return self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=return_dict) def get_audio_features( self, From 278b0686492f988a61944a4a898bca3407c73fbe Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 12 Dec 2025 10:43:12 +0100 Subject: [PATCH 005/131] Update get_image_features for the missing architectures --- .../models/edgetam/modeling_edgetam.py | 43 ++++++++++++++ .../edgetam_video/modeling_edgetam_video.py | 43 ++++++++++++++ .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 41 +++++++++++-- .../qwen3_omni_moe/modular_qwen3_omni_moe.py | 24 +++++++- .../models/qwen3_vl/modeling_qwen3_vl.py | 58 ++++++++++++++++--- .../models/qwen3_vl/modular_qwen3_vl.py | 49 ++++++++++++++-- .../qwen3_vl_moe/modeling_qwen3_vl_moe.py | 58 ++++++++++++++++--- src/transformers/models/sam2/modeling_sam2.py | 43 ++++++++++++++ src/transformers/models/sam2/modular_sam2.py | 43 ++++++++++++++ .../models/sam2_video/modeling_sam2_video.py | 43 ++++++++++++++ .../sam3_tracker/modeling_sam3_tracker.py | 43 ++++++++++++++ .../modeling_sam3_tracker_video.py | 47 ++++++++++++++- .../modular_sam3_tracker_video.py | 44 ++++++++++++++ .../models/siglip/modeling_siglip.py | 7 +++ .../models/siglip2/modeling_siglip2.py | 6 ++ .../models/siglip2/modular_siglip2.py | 6 ++ .../models/smolvlm/modeling_smolvlm.py | 19 +++++- .../models/smolvlm/modular_smolvlm.py | 17 +++++- .../models/t5gemma2/modeling_t5gemma2.py | 21 +++++-- .../models/t5gemma2/modular_t5gemma2.py | 21 +++++-- .../video_llama_3/modeling_video_llama_3.py | 18 ++++-- .../video_llama_3/modular_video_llama_3.py | 18 ++++-- .../video_llava/modeling_video_llava.py | 14 ++++- .../models/vipllava/modeling_vipllava.py | 26 +++++++-- .../models/vipllava/modular_vipllava.py | 17 +++++- .../modeling_vision_text_dual_encoder.py | 11 +++- 26 files changed, 722 insertions(+), 58 deletions(-) diff --git a/src/transformers/models/edgetam/modeling_edgetam.py b/src/transformers/models/edgetam/modeling_edgetam.py index 1e0126d0ffbb..bfaa631c3923 100644 --- a/src/transformers/models/edgetam/modeling_edgetam.py +++ b/src/transformers/models/edgetam/modeling_edgetam.py @@ -463,6 +463,38 @@ def forward( ) +@dataclass +class BaseModelOutputWithFeatureMaps(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + feature_maps (`list[torch.Tensor]`): + List of feature maps from different layers of the model. + feature_maps_position_embeddings (`list[torch.Tensor]`): + List of position embeddings corresponding to the feature maps. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + attentions: Optional[tuple[torch.FloatTensor, ...]] = None + feature_maps: Optional[list[torch.Tensor]] = None + feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None + + @dataclass @auto_docstring(custom_intro="Base class for the EdgeTam model's output.") class EdgeTamImageSegmentationOutput(ModelOutput): @@ -1190,6 +1222,7 @@ def forward( def get_image_features( self, pixel_values: torch.FloatTensor, + return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -1203,6 +1236,8 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -1232,6 +1267,14 @@ def get_image_features( for feature_map_position_embedding in feature_maps_position_embeddings ] + if return_dict: + return BaseModelOutputWithFeatureMaps( + last_hidden_state=vision_outputs.last_hidden_state, + attentions=vision_outputs.attentions, + feature_maps=feature_maps, + feature_maps_position_embeddings=feature_maps_position_embeddings, + ) + return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions diff --git a/src/transformers/models/edgetam_video/modeling_edgetam_video.py b/src/transformers/models/edgetam_video/modeling_edgetam_video.py index 756a14397d0d..3b8306aafeaa 100644 --- a/src/transformers/models/edgetam_video/modeling_edgetam_video.py +++ b/src/transformers/models/edgetam_video/modeling_edgetam_video.py @@ -1547,6 +1547,38 @@ class EdgeTamVideoSegmentationOutput(ModelOutput): frame_idx: Optional[int] = None +@dataclass +class BaseModelOutputWithFeatureMaps(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + feature_maps (`list[torch.Tensor]`): + List of feature maps from different layers of the model. + feature_maps_position_embeddings (`list[torch.Tensor]`): + List of position embeddings corresponding to the feature maps. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + attentions: Optional[tuple[torch.FloatTensor, ...]] = None + feature_maps: Optional[list[torch.Tensor]] = None + feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None + + class EdgeTamVideoPositionalEmbedding(nn.Module): def __init__(self, config: EdgeTamVideoPromptEncoderConfig): super().__init__() @@ -2210,6 +2242,7 @@ def forward( def get_image_features( self, pixel_values: torch.FloatTensor, + return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -2223,6 +2256,8 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -2252,6 +2287,14 @@ def get_image_features( for feature_map_position_embedding in feature_maps_position_embeddings ] + if return_dict: + return BaseModelOutputWithFeatureMaps( + last_hidden_state=vision_outputs.last_hidden_state, + attentions=vision_outputs.attentions, + feature_maps=feature_maps, + feature_maps_position_embeddings=feature_maps_position_embeddings, + ) + return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions def _prepare_vision_features( diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index e8da6efbd87c..bd638ac7dc24 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -49,7 +49,7 @@ from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import auto_docstring, can_return_tuple +from ...utils import ModelOutput, auto_docstring, can_return_tuple from ...utils.generic import OutputRecorder, TransformersKwargs, check_model_inputs from .configuration_qwen3_omni_moe import ( Qwen3OmniMoeAudioEncoderConfig, @@ -64,6 +64,28 @@ ) +@dataclass +class BaseModelOutputWithDeepstackFeatures(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + deepstack_features (`List[torch.FloatTensor]`, *optional*): + List of hidden-states (feature maps) from deepstack layers. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + deepstack_features: Optional[list[torch.FloatTensor]] = None + + @auto_docstring class Qwen3OmniMoePreTrainedModel(PreTrainedModel): config: Qwen3OmniMoeConfig @@ -1171,13 +1193,17 @@ def fast_pos_embed_interpolate(self, grid_thw): patch_pos_embeds = torch.cat(patch_pos_embeds_permute) return patch_pos_embeds - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor: + def forward( + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + ) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. Returns: `torch.Tensor`: hidden_states. @@ -1219,9 +1245,16 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) ) deepstack_feature_lists.append(deepstack_feature) - hidden_states = self.merger(hidden_states) + merged_hidden_states = self.merger(hidden_states) + + if return_dict: + return BaseModelOutputWithDeepstackFeatures( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + deepstack_features=deepstack_feature_lists, + ) - return hidden_states, deepstack_feature_lists + return merged_hidden_states, deepstack_feature_lists @property def deepstack_merger_list(self): diff --git a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py index 26cd94112e44..ef0896939199 100644 --- a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py @@ -46,7 +46,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import ProcessorMixin, Unpack from ...tokenization_utils_base import TextInput -from ...utils import auto_docstring, can_return_tuple, logging +from ...utils import ModelOutput, auto_docstring, can_return_tuple, logging from ...utils.generic import OutputRecorder, TransformersKwargs, check_model_inputs from ...video_utils import VideoInput, make_batched_videos from ..mimi.modeling_mimi import MimiLayerScale @@ -97,6 +97,28 @@ logger = logging.get_logger(__name__) +@dataclass +class BaseModelOutputWithDeepstackFeatures(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + deepstack_features (`List[torch.FloatTensor]`, *optional*): + List of hidden-states (feature maps) from deepstack layers. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + deepstack_features: Optional[list[torch.FloatTensor]] = None + + def _get_feat_extract_output_lengths(input_lengths): """ Computes the output length of the convolutional layers and the output length of the audio encoder diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index 86500d1e8e24..7e2fe04f84c7 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -34,15 +34,37 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, is_torchdynamo_compiling +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, is_torchdynamo_compiling from ...utils.generic import check_model_inputs from .configuration_qwen3_vl import Qwen3VLConfig, Qwen3VLTextConfig, Qwen3VLVisionConfig +@dataclass +class BaseModelOutputWithDeepstackFeatures(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + deepstack_features (`List[torch.FloatTensor]`, *optional*): + List of hidden-states (feature maps) from deepstack layers. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + deepstack_features: Optional[list[torch.FloatTensor]] = None + + class Qwen3VLVisionMLP(nn.Module): def __init__(self, config): super().__init__() @@ -731,13 +753,17 @@ def fast_pos_embed_interpolate(self, grid_thw): patch_pos_embeds = torch.cat(patch_pos_embeds_permute) return patch_pos_embeds - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor: + def forward( + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + ) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. Returns: `torch.Tensor`: hidden_states. @@ -779,9 +805,16 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) ) deepstack_feature_lists.append(deepstack_feature) - hidden_states = self.merger(hidden_states) + merged_hidden_states = self.merger(hidden_states) - return hidden_states, deepstack_feature_lists + if return_dict: + return BaseModelOutputWithDeepstackFeatures( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + deepstack_features=deepstack_feature_lists, + ) + + return merged_hidden_states, deepstack_feature_lists @auto_docstring( @@ -1073,7 +1106,12 @@ def get_video_features( # Same implementation as for images return self.get_image_features(pixel_values_videos, video_grid_thw) - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, + ): """ Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1082,9 +1120,15 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. """ pixel_values = pixel_values.type(self.visual.dtype) - image_embeds, deepstack_image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + vision_output = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=return_dict) + if return_dict: + return vision_output + + image_embeds, deepstack_image_embeds = vision_output split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) return image_embeds, deepstack_image_embeds diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index bd9368a0c443..d0226d93b287 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -15,6 +15,7 @@ """PyTorch Qwen3-VL model.""" from collections.abc import Callable +from dataclasses import dataclass from typing import Any, Optional, Union import numpy as np @@ -34,7 +35,7 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...processing_utils import ProcessingKwargs, Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput -from ...utils import auto_docstring, is_torchdynamo_compiling, logging +from ...utils import ModelOutput, auto_docstring, is_torchdynamo_compiling, logging from ...utils.generic import check_model_inputs from ...video_utils import VideoInput from ..llama.modeling_llama import LlamaRotaryEmbedding @@ -64,6 +65,27 @@ logger = logging.get_logger(__name__) +@dataclass +class BaseModelOutputWithDeepstackFeatures(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + deepstack_features (`List[torch.FloatTensor]`, *optional*): + List of hidden-states (feature maps) from deepstack layers. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + deepstack_features: Optional[list[torch.FloatTensor]] = None + class Qwen3VLVisionConfig(PreTrainedConfig): model_type = "qwen3_vl" @@ -627,13 +649,15 @@ def fast_pos_embed_interpolate(self, grid_thw): patch_pos_embeds = torch.cat(patch_pos_embeds_permute) return patch_pos_embeds - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor: + def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. Returns: `torch.Tensor`: hidden_states. @@ -675,9 +699,16 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) ) deepstack_feature_lists.append(deepstack_feature) - hidden_states = self.merger(hidden_states) + merged_hidden_states = self.merger(hidden_states) + + if return_dict: + return BaseModelOutputWithDeepstackFeatures( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + deepstack_features=deepstack_feature_lists, + ) - return hidden_states, deepstack_feature_lists + return merged_hidden_states, deepstack_feature_lists @auto_docstring( @@ -930,7 +961,7 @@ def get_rope_index( return position_ids, mrope_position_deltas - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False): """ Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -939,9 +970,15 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. """ pixel_values = pixel_values.type(self.visual.dtype) - image_embeds, deepstack_image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + vision_output = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=return_dict) + if return_dict: + return vision_output + + image_embeds, deepstack_image_embeds = vision_output split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) return image_embeds, deepstack_image_embeds diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index be87e08c8a24..e55e6dc501ed 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -35,11 +35,11 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, is_torchdynamo_compiling +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, is_torchdynamo_compiling from ...utils.generic import OutputRecorder, check_model_inputs from .configuration_qwen3_vl_moe import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig, Qwen3VLMoeVisionConfig @@ -420,6 +420,28 @@ def _init_weights(self, module): init.normal_(module.down_proj, mean=0.0, std=std) +@dataclass +class BaseModelOutputWithDeepstackFeatures(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + deepstack_features (`List[torch.FloatTensor]`, *optional*): + List of hidden-states (feature maps) from deepstack layers. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + deepstack_features: Optional[list[torch.FloatTensor]] = None + + class Qwen3VLMoeVisionMLP(nn.Module): def __init__(self, config): super().__init__() @@ -745,13 +767,17 @@ def fast_pos_embed_interpolate(self, grid_thw): patch_pos_embeds = torch.cat(patch_pos_embeds_permute) return patch_pos_embeds - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor: + def forward( + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + ) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. Returns: `torch.Tensor`: hidden_states. @@ -793,9 +819,16 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) ) deepstack_feature_lists.append(deepstack_feature) - hidden_states = self.merger(hidden_states) + merged_hidden_states = self.merger(hidden_states) - return hidden_states, deepstack_feature_lists + if return_dict: + return BaseModelOutputWithDeepstackFeatures( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + deepstack_features=deepstack_feature_lists, + ) + + return merged_hidden_states, deepstack_feature_lists class Qwen3VLMoeTextRotaryEmbedding(nn.Module): @@ -1230,7 +1263,12 @@ def get_video_features( # Same implementation as for images return self.get_image_features(pixel_values_videos, video_grid_thw) - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, + ): """ Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1239,9 +1277,15 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. """ pixel_values = pixel_values.type(self.visual.dtype) - image_embeds, deepstack_image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + vision_output = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=return_dict) + if return_dict: + return vision_output + + image_embeds, deepstack_image_embeds = vision_output split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) return image_embeds, deepstack_image_embeds diff --git a/src/transformers/models/sam2/modeling_sam2.py b/src/transformers/models/sam2/modeling_sam2.py index a55ceb59d9a8..018309ff179a 100644 --- a/src/transformers/models/sam2/modeling_sam2.py +++ b/src/transformers/models/sam2/modeling_sam2.py @@ -51,6 +51,38 @@ ) +@dataclass +class BaseModelOutputWithFeatureMaps(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + feature_maps (`list[torch.Tensor]`): + List of feature maps from different layers of the model. + feature_maps_position_embeddings (`list[torch.Tensor]`): + List of position embeddings corresponding to the feature maps. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + attentions: Optional[tuple[torch.FloatTensor, ...]] = None + feature_maps: Optional[list[torch.Tensor]] = None + feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None + + @dataclass @auto_docstring(custom_intro="Base class for the vision encoder's outputs.") class Sam2VisionEncoderOutput(ModelOutput): @@ -1549,6 +1581,7 @@ def forward( def get_image_features( self, pixel_values: torch.FloatTensor, + return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -1562,6 +1595,8 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -1591,6 +1626,14 @@ def get_image_features( for feature_map_position_embedding in feature_maps_position_embeddings ] + if return_dict: + return BaseModelOutputWithFeatureMaps( + last_hidden_state=vision_outputs.last_hidden_state, + attentions=vision_outputs.attentions, + feature_maps=feature_maps, + feature_maps_position_embeddings=feature_maps_position_embeddings, + ) + return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions diff --git a/src/transformers/models/sam2/modular_sam2.py b/src/transformers/models/sam2/modular_sam2.py index 92f878ba8492..e2b457333d44 100644 --- a/src/transformers/models/sam2/modular_sam2.py +++ b/src/transformers/models/sam2/modular_sam2.py @@ -72,6 +72,38 @@ logger = logging.get_logger(__name__) +@dataclass +class BaseModelOutputWithFeatureMaps(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + feature_maps (`list[torch.Tensor]`): + List of feature maps from different layers of the model. + feature_maps_position_embeddings (`list[torch.Tensor]`): + List of position embeddings corresponding to the feature maps. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + attentions: Optional[tuple[torch.FloatTensor, ...]] = None + feature_maps: Optional[list[torch.Tensor]] = None + feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None + + class Sam2FastImageProcessorKwargs(ImagesKwargs, total=False): r""" mask_size (`dict[str, int]`, *optional*): @@ -1237,6 +1269,7 @@ def get_image_embeddings( def get_image_features( self, pixel_values: torch.FloatTensor, + return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -1250,6 +1283,8 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -1279,6 +1314,14 @@ def get_image_features( for feature_map_position_embedding in feature_maps_position_embeddings ] + if return_dict: + return BaseModelOutputWithFeatureMaps( + last_hidden_state=vision_outputs.last_hidden_state, + attentions=vision_outputs.attentions, + feature_maps=feature_maps, + feature_maps_position_embeddings=feature_maps_position_embeddings, + ) + return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions @check_model_inputs diff --git a/src/transformers/models/sam2_video/modeling_sam2_video.py b/src/transformers/models/sam2_video/modeling_sam2_video.py index d491e10d5692..4e7954793df6 100644 --- a/src/transformers/models/sam2_video/modeling_sam2_video.py +++ b/src/transformers/models/sam2_video/modeling_sam2_video.py @@ -1101,6 +1101,38 @@ def forward( return vision_features, vision_pos_enc +@dataclass +class BaseModelOutputWithFeatureMaps(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + feature_maps (`list[torch.Tensor]`): + List of feature maps from different layers of the model. + feature_maps_position_embeddings (`list[torch.Tensor]`): + List of position embeddings corresponding to the feature maps. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + attentions: Optional[tuple[torch.FloatTensor, ...]] = None + feature_maps: Optional[list[torch.Tensor]] = None + feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None + + @dataclass @auto_docstring(custom_intro="Base class for the vision encoder's outputs.") class Sam2VideoVisionEncoderOutput(ModelOutput): @@ -1818,6 +1850,7 @@ def forward( def get_image_features( self, pixel_values: torch.FloatTensor, + return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -1831,6 +1864,8 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -1860,6 +1895,14 @@ def get_image_features( for feature_map_position_embedding in feature_maps_position_embeddings ] + if return_dict: + return BaseModelOutputWithFeatureMaps( + last_hidden_state=vision_outputs.last_hidden_state, + attentions=vision_outputs.attentions, + feature_maps=feature_maps, + feature_maps_position_embeddings=feature_maps_position_embeddings, + ) + return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions def _prepare_vision_features( diff --git a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py index c3baaf7e3a07..f772291b4228 100644 --- a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py +++ b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py @@ -719,6 +719,38 @@ def _dynamic_multimask_via_stability(self, all_mask_logits, all_iou_scores): return mask_logits_out, iou_scores_out +@dataclass +class BaseModelOutputWithFeatureMaps(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + feature_maps (`list[torch.Tensor]`): + List of feature maps from different layers of the model. + feature_maps_position_embeddings (`list[torch.Tensor]`): + List of position embeddings corresponding to the feature maps. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + attentions: Optional[tuple[torch.FloatTensor, ...]] = None + feature_maps: Optional[list[torch.Tensor]] = None + feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None + + @dataclass @auto_docstring(custom_intro="Base class for the vision encoder's outputs.") class Sam3TrackerVisionEncoderOutput(ModelOutput): @@ -1046,6 +1078,7 @@ def forward( def get_image_features( self, pixel_values: torch.FloatTensor, + return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -1059,6 +1092,8 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -1088,6 +1123,14 @@ def get_image_features( for feature_map_position_embedding in feature_maps_position_embeddings ] + if return_dict: + return BaseModelOutputWithFeatureMaps( + last_hidden_state=vision_outputs.last_hidden_state, + attentions=vision_outputs.attentions, + feature_maps=feature_maps, + feature_maps_position_embeddings=feature_maps_position_embeddings, + ) + return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions diff --git a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py index 125b86bacea0..061e5d7796a6 100644 --- a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py @@ -36,11 +36,11 @@ from ...activations import ACT2FN from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput +from ...modeling_outputs import BaseModelOutput, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import compile_compatible_method_lru_cache -from ...utils import ModelOutput, auto_docstring +from ...utils import auto_docstring from ...utils.generic import OutputRecorder, TransformersKwargs from ..auto import AutoModel from .configuration_sam3_tracker_video import ( @@ -50,6 +50,38 @@ ) +@dataclass +class BaseModelOutputWithFeatureMaps(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + feature_maps (`list[torch.Tensor]`): + List of feature maps from different layers of the model. + feature_maps_position_embeddings (`list[torch.Tensor]`): + List of position embeddings corresponding to the feature maps. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + attentions: Optional[tuple[torch.FloatTensor, ...]] = None + feature_maps: Optional[list[torch.Tensor]] = None + feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None + + class Sam3TrackerVideoInferenceCache: """Cache for vision features and model constants.""" @@ -1837,6 +1869,7 @@ def forward( def get_image_features( self, pixel_values: torch.FloatTensor, + return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -1850,6 +1883,8 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -1879,6 +1914,14 @@ def get_image_features( for feature_map_position_embedding in feature_maps_position_embeddings[:-1] ] + if return_dict: + return BaseModelOutputWithFeatureMaps( + last_hidden_state=vision_outputs.last_hidden_state, + attentions=vision_outputs.attentions, + feature_maps=feature_maps, + feature_maps_position_embeddings=feature_maps_position_embeddings, + ) + return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions def _prepare_vision_features( diff --git a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py index 12e4bc8854e0..8d65402d63a2 100644 --- a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py @@ -13,11 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from dataclasses import dataclass from typing import Optional import torch from ...configuration_utils import PreTrainedConfig +from ...modeling_outputs import ModelOutput from ...processing_utils import Unpack from ...utils.generic import TransformersKwargs from ..auto import CONFIG_MAPPING, AutoConfig, AutoModel @@ -53,6 +55,37 @@ from ..sam2_video.processing_sam2_video import Sam2VideoProcessor +@dataclass +class BaseModelOutputWithFeatureMaps(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + feature_maps (`list[torch.Tensor]`): + List of feature maps from different layers of the model. + feature_maps_position_embeddings (`list[torch.Tensor]`): + List of position embeddings corresponding to the feature maps. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + attentions: Optional[tuple[torch.FloatTensor, ...]] = None + feature_maps: Optional[list[torch.Tensor]] = None + feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None + class Sam3TrackerVideoPromptEncoderConfig(Sam2VideoPromptEncoderConfig): r""" This is the configuration class to store the configuration of a [`Sam3TrackerVideoPromptEncoder`]. The [`Sam3TrackerVideoPromptEncoder`] @@ -526,6 +559,7 @@ def __init__(self, config: Sam3TrackerVideoConfig, remove_vision_encoder: bool = def get_image_features( self, pixel_values: torch.FloatTensor, + return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -539,6 +573,8 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -568,6 +604,14 @@ def get_image_features( for feature_map_position_embedding in feature_maps_position_embeddings[:-1] ] + if return_dict: + return BaseModelOutputWithFeatureMaps( + last_hidden_state=vision_outputs.last_hidden_state, + attentions=vision_outputs.attentions, + feature_maps=feature_maps, + feature_maps_position_embeddings=feature_maps_position_embeddings, + ) + return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions diff --git a/src/transformers/models/siglip/modeling_siglip.py b/src/transformers/models/siglip/modeling_siglip.py index cb585d2a55cb..2acae505d572 100644 --- a/src/transformers/models/siglip/modeling_siglip.py +++ b/src/transformers/models/siglip/modeling_siglip.py @@ -823,9 +823,13 @@ def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, + return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`SiglipVisionModel`]. @@ -855,6 +859,9 @@ def get_image_features( ) pooled_output = vision_outputs.pooler_output + if return_dict: + return vision_outputs + return pooled_output # NOTE: SiglipModel uses Pretrained backbones, so we don't need to add `check_model_inputs` here diff --git a/src/transformers/models/siglip2/modeling_siglip2.py b/src/transformers/models/siglip2/modeling_siglip2.py index 36d7f5c13cfe..3540ad878535 100644 --- a/src/transformers/models/siglip2/modeling_siglip2.py +++ b/src/transformers/models/siglip2/modeling_siglip2.py @@ -881,12 +881,15 @@ def get_image_features( pixel_values: Optional[torch.FloatTensor] = None, pixel_attention_mask: Optional[torch.Tensor] = None, spatial_shapes: Optional[torch.LongTensor] = None, + return_dict: bool = False, ) -> torch.FloatTensor: r""" pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): Mask to avoid performing attention on padding pixel indices. spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): Tensor containing the spatial dimensions (height, width) of the input images. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by @@ -918,6 +921,9 @@ def get_image_features( ) pooled_output = vision_outputs.pooler_output + if return_dict: + return vision_outputs + return pooled_output # NOTE: Siglip2Model uses Pretrained backbones, so we don't need to add `check_model_inputs` here diff --git a/src/transformers/models/siglip2/modular_siglip2.py b/src/transformers/models/siglip2/modular_siglip2.py index 1b0a06f666c2..782cdeab8bfa 100644 --- a/src/transformers/models/siglip2/modular_siglip2.py +++ b/src/transformers/models/siglip2/modular_siglip2.py @@ -368,12 +368,15 @@ def get_image_features( pixel_values: Optional[torch.FloatTensor] = None, pixel_attention_mask: Optional[torch.Tensor] = None, spatial_shapes: Optional[torch.LongTensor] = None, + return_dict: bool = False, ) -> torch.FloatTensor: r""" pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): Mask to avoid performing attention on padding pixel indices. spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): Tensor containing the spatial dimensions (height, width) of the input images. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by @@ -405,6 +408,9 @@ def get_image_features( ) pooled_output = vision_outputs.pooler_output + if return_dict: + return vision_outputs + return pooled_output # Update: add `spatial_shapes` and `pixel_attention_mask` diff --git a/src/transformers/models/smolvlm/modeling_smolvlm.py b/src/transformers/models/smolvlm/modeling_smolvlm.py index 84c32a5413c5..72e7c7b7a944 100644 --- a/src/transformers/models/smolvlm/modeling_smolvlm.py +++ b/src/transformers/models/smolvlm/modeling_smolvlm.py @@ -518,7 +518,10 @@ def inputs_merger( return merged_embeds def get_image_features( - self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None + self, + pixel_values: torch.FloatTensor, + pixel_attention_mask: Optional[torch.LongTensor] = None, + return_dict: bool = False, ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -528,6 +531,9 @@ def get_image_features( The tensors corresponding to the input images. pixel_attention_mask (`torch.LongTensor`, *optional*): The attention mask indicating padded regions in the image. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + """ batch_size, num_images, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility @@ -563,8 +569,15 @@ def get_image_features( image_hidden_states = image_hidden_states.last_hidden_state # Modality projection & resampling - image_hidden_states = self.connector(image_hidden_states) - return image_hidden_states + image_features = self.connector(image_hidden_states) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_hidden_states, + pooler_output=image_features, + ) + + return image_features @can_return_tuple @auto_docstring( diff --git a/src/transformers/models/smolvlm/modular_smolvlm.py b/src/transformers/models/smolvlm/modular_smolvlm.py index 960d249c6260..2b49382214e0 100644 --- a/src/transformers/models/smolvlm/modular_smolvlm.py +++ b/src/transformers/models/smolvlm/modular_smolvlm.py @@ -21,6 +21,7 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationConfig from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import auto_docstring, can_return_tuple, logging from ..idefics3.configuration_idefics3 import Idefics3Config, Idefics3VisionConfig @@ -193,7 +194,7 @@ def inputs_merger( return merged_embeds def get_image_features( - self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None + self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, return_dict: bool = False ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -203,6 +204,9 @@ def get_image_features( The tensors corresponding to the input images. pixel_attention_mask (`torch.LongTensor`, *optional*): The attention mask indicating padded regions in the image. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + """ batch_size, num_images, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility @@ -238,8 +242,15 @@ def get_image_features( image_hidden_states = image_hidden_states.last_hidden_state # Modality projection & resampling - image_hidden_states = self.connector(image_hidden_states) - return image_hidden_states + image_features = self.connector(image_hidden_states) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_hidden_states, + pooler_output=image_features, + ) + + return image_features @can_return_tuple @auto_docstring( diff --git a/src/transformers/models/t5gemma2/modeling_t5gemma2.py b/src/transformers/models/t5gemma2/modeling_t5gemma2.py index cdbee5cb2ea7..a55a26aabe5c 100644 --- a/src/transformers/models/t5gemma2/modeling_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modeling_t5gemma2.py @@ -37,6 +37,7 @@ from ...modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPooling, Seq2SeqLMOutput, Seq2SeqModelOutput, SequenceClassifierOutput, @@ -800,12 +801,24 @@ def __init__( # Initialize weights and apply final processing self.post_init() - def get_image_features(self, pixel_values: torch.Tensor) -> torch.Tensor: - """Convert pixel image to image features via the encoder and projector.""" + def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False) -> torch.Tensor: + """ + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + + Convert pixel image to image features via the encoder and projector. + """ # pixel_values: (batch_size, channels, height, width) # image_features: Image feature tensor of shape (num_images, image_length, embed_dim). vision_outputs = self.vision_tower(pixel_values=pixel_values).last_hidden_state image_features = self.multi_modal_projector(vision_outputs) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_image_placeholder_mask( @@ -1210,8 +1223,8 @@ def get_encoder(self): def get_decoder(self): return self.model.get_decoder() - def get_image_features(self, pixel_values): - return self.get_encoder().get_image_features(pixel_values) + def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False): + return self.get_encoder().get_image_features(pixel_values, return_dict=return_dict) @property def vision_tower(self): diff --git a/src/transformers/models/t5gemma2/modular_t5gemma2.py b/src/transformers/models/t5gemma2/modular_t5gemma2.py index 67c749f58427..78b85d6d710e 100644 --- a/src/transformers/models/t5gemma2/modular_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modular_t5gemma2.py @@ -29,6 +29,7 @@ from ...modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPooling, Seq2SeqLMOutput, Seq2SeqModelOutput, SequenceClassifierOutput, @@ -579,12 +580,24 @@ def __init__( # Initialize weights and apply final processing self.post_init() - def get_image_features(self, pixel_values: torch.Tensor) -> torch.Tensor: - """Convert pixel image to image features via the encoder and projector.""" + def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False) -> torch.Tensor: + """ + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + + Convert pixel image to image features via the encoder and projector. + """ # pixel_values: (batch_size, channels, height, width) # image_features: Image feature tensor of shape (num_images, image_length, embed_dim). vision_outputs = self.vision_tower(pixel_values=pixel_values).last_hidden_state image_features = self.multi_modal_projector(vision_outputs) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_image_placeholder_mask( @@ -976,8 +989,8 @@ def get_encoder(self): def get_decoder(self): return self.model.get_decoder() - def get_image_features(self, pixel_values): - return self.get_encoder().get_image_features(pixel_values) + def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False): + return self.get_encoder().get_image_features(pixel_values, return_dict=return_dict) @property def vision_tower(self): diff --git a/src/transformers/models/video_llama_3/modeling_video_llama_3.py b/src/transformers/models/video_llama_3/modeling_video_llama_3.py index d635977ca151..dbec71962581 100644 --- a/src/transformers/models/video_llama_3/modeling_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modeling_video_llama_3.py @@ -29,7 +29,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, ModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -561,6 +561,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor, image_merge_sizes: torch.LongTensor, + return_dict: bool = False, ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -572,18 +573,27 @@ def get_image_features( The temporal, height and width of feature shape of each image in LLM. image_merge_sizes (`torch.Tensor` of shape `(num_images,)`): The spatial downsampling ratio of each image feature. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ - image_embeds = self.vision_model( + vision_outputs = self.vision_model( pixel_values=pixel_values, grid_thw=image_grid_thw, merge_sizes=image_merge_sizes, return_dict=True, - ).last_hidden_state - image_embeds = self.projector(image_embeds) + ) + last_hidden_state = vision_outputs.last_hidden_state + image_embeds = self.projector(last_hidden_state) split_sizes = image_grid_thw.prod(dim=1) // (image_merge_sizes**2) image_embeds = torch.split(image_embeds, split_sizes.tolist()) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=image_embeds, + ) + return image_embeds def get_placeholder_mask( diff --git a/src/transformers/models/video_llama_3/modular_video_llama_3.py b/src/transformers/models/video_llama_3/modular_video_llama_3.py index 9389b9112ff9..eb4e6ad45969 100644 --- a/src/transformers/models/video_llama_3/modular_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modular_video_llama_3.py @@ -36,7 +36,7 @@ valid_images, validate_preprocess_arguments, ) -from ...modeling_outputs import BaseModelOutput, ModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput @@ -607,6 +607,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor, image_merge_sizes: torch.LongTensor, + return_dict: bool = False, ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -618,18 +619,27 @@ def get_image_features( The temporal, height and width of feature shape of each image in LLM. image_merge_sizes (`torch.Tensor` of shape `(num_images,)`): The spatial downsampling ratio of each image feature. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ - image_embeds = self.vision_model( + vision_outputs = self.vision_model( pixel_values=pixel_values, grid_thw=image_grid_thw, merge_sizes=image_merge_sizes, return_dict=True, - ).last_hidden_state - image_embeds = self.projector(image_embeds) + ) + last_hidden_state = vision_outputs.last_hidden_state + image_embeds = self.projector(last_hidden_state) split_sizes = image_grid_thw.prod(dim=1) // (image_merge_sizes**2) image_embeds = torch.split(image_embeds, split_sizes.tolist()) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=image_embeds, + ) + return image_embeds @can_return_tuple diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index dfa2c6559f8c..0be70ce86903 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -25,7 +25,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import ModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -177,6 +177,7 @@ def get_image_features( pixel_values_images: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -191,6 +192,9 @@ def get_image_features( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -223,6 +227,12 @@ def get_image_features( image_features = self.multi_modal_projector(image_outputs) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_video_features( @@ -431,11 +441,13 @@ def get_image_features( pixel_values_images: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, ): return self.model.get_image_features( pixel_values_images=pixel_values_images, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, + return_dict=return_dict, ) @can_return_tuple diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index dcd50dcb02b6..e4691dc3bb5c 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -28,7 +28,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...utils import auto_docstring, can_return_tuple from ..auto import AutoModel @@ -151,7 +151,10 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) def get_image_features( - self, pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None + self, + pixel_values: torch.FloatTensor, + vision_feature_layers: Optional[Union[int, list[int]]] = None, + return_dict: bool = False, ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -162,6 +165,9 @@ def get_image_features( vision_feature_layers (`Union[int, list[int]]`): The vision feature layer, or the list of indexes of the layers to select the vision feature. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -179,6 +185,13 @@ def get_image_features( image_features = [image_outputs.hidden_states[index][:, 1:] for index in vision_feature_layers] image_features = torch.cat(image_features, dim=-1) image_features = self.multi_modal_projector(image_features) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features def get_placeholder_mask( @@ -305,9 +318,14 @@ def get_output_embeddings(self) -> nn.Module: return self.lm_head def get_image_features( - self, pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None + self, + pixel_values: torch.FloatTensor, + vision_feature_layers: Optional[Union[int, list[int]]] = None, + return_dict: bool = False, ): - return self.model.get_image_features(pixel_values=pixel_values, vision_feature_layers=vision_feature_layers) + return self.model.get_image_features( + pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, return_dict=return_dict + ) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/vipllava/modular_vipllava.py b/src/transformers/models/vipllava/modular_vipllava.py index ec12cc6ee1bf..e7a6282ac358 100644 --- a/src/transformers/models/vipllava/modular_vipllava.py +++ b/src/transformers/models/vipllava/modular_vipllava.py @@ -28,6 +28,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache +from ...modeling_outputs import BaseModelOutputWithPooling from ...utils import auto_docstring, logging from .configuration_vipllava import VipLlavaConfig @@ -73,7 +74,7 @@ class VipLlavaPreTrainedModel(LlavaPreTrainedModel): class VipLlavaModel(LlavaModel): def get_image_features( - self, pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None + self, pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None, return_dict: bool = False ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -84,6 +85,9 @@ def get_image_features( vision_feature_layers (`Union[int, list[int]]`): The vision feature layer, or the list of indexes of the layers to select the vision feature. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -101,6 +105,13 @@ def get_image_features( image_features = [image_outputs.hidden_states[index][:, 1:] for index in vision_feature_layers] image_features = torch.cat(image_features, dim=-1) image_features = self.multi_modal_projector(image_features) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @auto_docstring @@ -175,9 +186,9 @@ def forward( class VipLlavaForConditionalGeneration(LlavaForConditionalGeneration): def get_image_features( - self, pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None + self, pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None, return_dict: bool = False ): - return self.model.get_image_features(pixel_values=pixel_values, vision_feature_layers=vision_feature_layers) + return self.model.get_image_features(pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, return_dict=return_dict) def forward( self, diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index aab794aafa54..bfb1ebd119aa 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -154,8 +154,11 @@ def get_text_features( @filter_out_non_signature_kwargs() @auto_docstring - def get_image_features(self, pixel_values: torch.Tensor) -> torch.FloatTensor: + def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False) -> torch.FloatTensor: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`CLIPVisionModel`]. @@ -181,6 +184,12 @@ def get_image_features(self, pixel_values: torch.Tensor) -> torch.FloatTensor: vision_outputs = self.vision_model(pixel_values=pixel_values) image_features = self.visual_projection(vision_outputs.pooler_output) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=image_features, + ) + return image_features @auto_docstring From 3b140453cdfde7987125408206daac8137be83c9 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 12 Dec 2025 11:44:43 +0100 Subject: [PATCH 006/131] Update all get_audio_features --- .../audioflamingo3/modeling_audioflamingo3.py | 13 +++++++++++-- .../audioflamingo3/modular_audioflamingo3.py | 13 +++++++++++-- src/transformers/models/clap/modeling_clap.py | 9 +++++++++ .../models/gemma3n/modeling_gemma3n.py | 12 ++++++++++-- .../models/gemma3n/modular_gemma3n.py | 12 ++++++++++-- .../granite_speech/modeling_granite_speech.py | 11 +++++++++-- .../models/qwen2_5_omni/modeling_qwen2_5_omni.py | 6 ++++++ .../models/qwen2_5_omni/modular_qwen2_5_omni.py | 6 ++++++ .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 6 ++++++ .../qwen3_omni_moe/modular_qwen3_omni_moe.py | 6 ++++++ .../models/voxtral/modeling_voxtral.py | 16 ++++++++++++++-- .../models/voxtral/modular_voxtral.py | 16 ++++++++++++++-- 12 files changed, 112 insertions(+), 14 deletions(-) diff --git a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py index 72f3356c1544..bc4553ad61c6 100644 --- a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py @@ -33,7 +33,7 @@ from ...masking_utils import create_bidirectional_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, CausalLMOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -443,7 +443,7 @@ def get_decoder(self): return self.language_model.get_decoder() def get_audio_features( - self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor + self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor, return_dict: bool = False ) -> torch.FloatTensor: """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. @@ -456,6 +456,8 @@ def get_audio_features( and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`): Mask to avoid performing attention on padded feature indices. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `torch.FloatTensor`: @@ -470,6 +472,13 @@ def get_audio_features( post_lengths = (input_features_mask.sum(-1) - 2) // 2 + 1 valid_mask = torch.arange(audio_embeds.shape[1], device=post_lengths.device)[None, :] < post_lengths[:, None] audio_embeds = audio_embeds[valid_mask.to(audio_embeds.device)] + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=encoder_output.last_hidden_state, + pooler_output=audio_embeds, + ) + return audio_embeds @can_return_tuple diff --git a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py index f8535ebbe439..513e79ed8389 100644 --- a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py @@ -22,7 +22,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...masking_utils import create_bidirectional_mask -from ...modeling_outputs import BaseModelOutput, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, CausalLMOutputWithPast from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from ..qwen2_audio.modeling_qwen2_audio import ( @@ -140,7 +140,7 @@ def __init__(self, config): super().__init__(config) def get_audio_features( - self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor + self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor, return_dict: bool = False ) -> torch.FloatTensor: """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. @@ -153,6 +153,8 @@ def get_audio_features( and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`): Mask to avoid performing attention on padded feature indices. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `torch.FloatTensor`: @@ -167,6 +169,13 @@ def get_audio_features( post_lengths = (input_features_mask.sum(-1) - 2) // 2 + 1 valid_mask = torch.arange(audio_embeds.shape[1], device=post_lengths.device)[None, :] < post_lengths[:, None] audio_embeds = audio_embeds[valid_mask.to(audio_embeds.device)] + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=encoder_output.last_hidden_state, + pooler_output=audio_embeds, + ) + return audio_embeds def get_audio_embeds(self): diff --git a/src/transformers/models/clap/modeling_clap.py b/src/transformers/models/clap/modeling_clap.py index 4ceec47ff042..ca3e9c0d110b 100644 --- a/src/transformers/models/clap/modeling_clap.py +++ b/src/transformers/models/clap/modeling_clap.py @@ -1592,11 +1592,14 @@ def get_audio_features( input_features: torch.Tensor, is_longer: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, + return_dict: bool = False ) -> torch.FloatTensor: r""" is_longer (`torch.FloatTensor`, of shape `(batch_size, 1)`, *optional*): Whether the audio clip is longer than `max_length`. If `True`, a feature fusion will be enabled to enhance the features. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: audio_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The audio embeddings obtained by @@ -1622,6 +1625,12 @@ def get_audio_features( audio_features = self.audio_projection(audio_outputs.pooler_output) audio_features = F.normalize(audio_features, dim=-1) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=audio_outputs.last_hidden_state, + pooler_output=audio_features, + ) + return audio_features @can_return_tuple diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index 7ca876dfae6f..dc54b18133a5 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -2312,7 +2312,7 @@ def forward( ) def get_audio_features( - self, input_features: torch.Tensor, input_features_mask: torch.Tensor + self, input_features: torch.Tensor, input_features_mask: torch.Tensor, return_dict: bool = False ) -> tuple[torch.Tensor, torch.Tensor]: """ Projects the last hidden state from the audio encoder into language model space. @@ -2327,7 +2327,15 @@ def get_audio_features( audio_features (`torch.Tensor`): Audio feature tensor of shape `(num_images, audio_length, embed_dim)`). """ audio_outputs, audio_mask = self.audio_tower(input_features, input_features_mask) - return self.embed_audio(inputs_embeds=audio_outputs), audio_mask + audio_embeds = self.embed_audio(inputs_embeds=audio_outputs) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=audio_outputs.last_hidden_state, + pooler_output=audio_embeds, + ) + + return audio_embeds, audio_mask @auto_docstring( diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index 87610e2304d0..2aa6f820705e 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -2421,7 +2421,7 @@ def forward( ) def get_audio_features( - self, input_features: torch.Tensor, input_features_mask: torch.Tensor + self, input_features: torch.Tensor, input_features_mask: torch.Tensor, return_dict: bool = False ) -> tuple[torch.Tensor, torch.Tensor]: """ Projects the last hidden state from the audio encoder into language model space. @@ -2436,7 +2436,15 @@ def get_audio_features( audio_features (`torch.Tensor`): Audio feature tensor of shape `(num_images, audio_length, embed_dim)`). """ audio_outputs, audio_mask = self.audio_tower(input_features, input_features_mask) - return self.embed_audio(inputs_embeds=audio_outputs), audio_mask + audio_embeds = self.embed_audio(inputs_embeds=audio_outputs) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=audio_outputs.last_hidden_state, + pooler_output=audio_embeds, + ) + + return audio_embeds, audio_mask @auto_docstring( diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index 524da4b16f1d..ff90ca1669fc 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -24,7 +24,7 @@ from ... import initialization as init from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import ModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...utils import auto_docstring, is_peft_available, logging from ..auto import AutoModel, AutoModelForCausalLM @@ -334,10 +334,17 @@ def get_input_embeddings(self): def get_output_embeddings(self): return self.language_model.get_output_embeddings() - def get_audio_features(self, input_features: torch.Tensor) -> torch.Tensor: + def get_audio_features(self, input_features: torch.Tensor, return_dict: bool = False) -> torch.Tensor: """Get the audio features to merged into the multimodal embeddings.""" encoder_embeds = self.encoder(input_features) projected_embeds = self.projector(encoder_embeds) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=encoder_embeds, + pooler_output=projected_embeds, + ) + return projected_embeds @auto_docstring diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index 4b3860c9a448..f42523db8b86 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -1768,6 +1768,7 @@ def get_audio_features( input_features: torch.FloatTensor, feature_attention_mask: Optional[torch.LongTensor] = None, audio_feature_lengths: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes audios into continuous embeddings that can be forwarded to the language model. @@ -1779,6 +1780,8 @@ def get_audio_features( Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): The length of feature shape of each audio in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) @@ -1800,6 +1803,9 @@ def get_audio_features( if audio_features.shape[0] != sum(audio_output_lengths.tolist()): raise ValueError("length of audio_features should match audio_output_lengths") + if return_dict: + return audio_outputs + return audio_features def get_placeholder_mask( diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index 94924d2cfbd3..f4a4185c2301 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -2114,6 +2114,7 @@ def get_audio_features( input_features: torch.FloatTensor, feature_attention_mask: Optional[torch.LongTensor] = None, audio_feature_lengths: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes audios into continuous embeddings that can be forwarded to the language model. @@ -2125,6 +2126,8 @@ def get_audio_features( Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): The length of feature shape of each audio in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) @@ -2146,6 +2149,9 @@ def get_audio_features( if audio_features.shape[0] != sum(audio_output_lengths.tolist()): raise ValueError("length of audio_features should match audio_output_lengths") + if return_dict: + return audio_outputs + return audio_features def get_placeholder_mask( diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index bd638ac7dc24..5ca7e6279a73 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -1982,6 +1982,7 @@ def get_audio_features( input_features: torch.FloatTensor, feature_attention_mask: Optional[torch.LongTensor] = None, audio_feature_lengths: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes audios into continuous embeddings that can be forwarded to the language model. @@ -1993,6 +1994,8 @@ def get_audio_features( Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): The length of feature shape of each audio in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) @@ -2007,6 +2010,9 @@ def get_audio_features( ) audio_features = audio_outputs.last_hidden_state + if return_dict: + return audio_outputs + return audio_features def get_placeholder_mask( diff --git a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py index ef0896939199..842aedc3fb2a 100644 --- a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py @@ -1424,6 +1424,7 @@ def get_audio_features( input_features: torch.FloatTensor, feature_attention_mask: Optional[torch.LongTensor] = None, audio_feature_lengths: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes audios into continuous embeddings that can be forwarded to the language model. @@ -1435,6 +1436,8 @@ def get_audio_features( Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): The length of feature shape of each audio in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) @@ -1449,6 +1452,9 @@ def get_audio_features( ) audio_features = audio_outputs.last_hidden_state + if return_dict: + return audio_outputs + return audio_features @can_return_tuple diff --git a/src/transformers/models/voxtral/modeling_voxtral.py b/src/transformers/models/voxtral/modeling_voxtral.py index f61148769b48..5d7331951f6b 100644 --- a/src/transformers/models/voxtral/modeling_voxtral.py +++ b/src/transformers/models/voxtral/modeling_voxtral.py @@ -31,7 +31,12 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_outputs import ( + BaseModelOutput, + BaseModelOutputWithPast, + BaseModelOutputWithPooling, + CausalLMOutputWithPast, +) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -400,7 +405,7 @@ def set_decoder(self, decoder): def get_decoder(self): return self.language_model.get_decoder() - def get_audio_features(self, input_features: torch.FloatTensor): + def get_audio_features(self, input_features: torch.FloatTensor, return_dict: bool = False): """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. Args: @@ -419,6 +424,13 @@ def get_audio_features(self, input_features: torch.FloatTensor): audio_hidden_states = audio_outputs.last_hidden_state audio_hidden_states = audio_hidden_states.reshape(-1, self.config.audio_config.intermediate_size) audio_embeds = self.multi_modal_projector(audio_hidden_states) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=audio_outputs.last_hidden_state, + pooler_output=audio_embeds, + ) + return audio_embeds def get_audio_embeds(self, input_features: torch.FloatTensor): diff --git a/src/transformers/models/voxtral/modular_voxtral.py b/src/transformers/models/voxtral/modular_voxtral.py index de638c08a991..23817e70fa0f 100644 --- a/src/transformers/models/voxtral/modular_voxtral.py +++ b/src/transformers/models/voxtral/modular_voxtral.py @@ -22,7 +22,12 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_outputs import ( + BaseModelOutput, + BaseModelOutputWithPast, + BaseModelOutputWithPooling, + CausalLMOutputWithPast, +) from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ...utils.generic import check_model_inputs @@ -162,7 +167,7 @@ def set_decoder(self, decoder): def get_decoder(self): return self.language_model.get_decoder() - def get_audio_features(self, input_features: torch.FloatTensor): + def get_audio_features(self, input_features: torch.FloatTensor, return_dict: bool = False): """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. Args: @@ -181,6 +186,13 @@ def get_audio_features(self, input_features: torch.FloatTensor): audio_hidden_states = audio_outputs.last_hidden_state audio_hidden_states = audio_hidden_states.reshape(-1, self.config.audio_config.intermediate_size) audio_embeds = self.multi_modal_projector(audio_hidden_states) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=audio_outputs.last_hidden_state, + pooler_output=audio_embeds, + ) + return audio_embeds def get_audio_embeds(self, input_features: torch.FloatTensor): From b7e0d66de20e474a60d4a241fe7250ea39c04da3 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 12 Dec 2025 12:51:41 +0100 Subject: [PATCH 007/131] Update get_video_features, except instructblipvideo Should be fine though, as that 'get_video_features' doesn't live on the AutoModel class, but the AutoModelForConditionalGeneration class --- .../models/glm46v/modeling_glm46v.py | 18 +++++++++-- .../models/glm4v/modeling_glm4v.py | 18 +++++++++-- .../models/glm4v/modular_glm4v.py | 17 +++++++--- .../models/glm4v_moe/modeling_glm4v_moe.py | 18 +++++++++-- .../modeling_llava_next_video.py | 12 +++++++ .../modular_llava_next_video.py | 12 +++++++ .../modeling_llava_onevision.py | 18 +++++++++-- .../modular_llava_onevision.py | 16 ++++++++-- .../qwen2_5_omni/modeling_qwen2_5_omni.py | 10 ++++-- .../qwen2_5_omni/modular_qwen2_5_omni.py | 7 ++-- .../models/qwen2_5_vl/modeling_qwen2_5_vl.py | 18 +++++++++-- .../models/qwen2_vl/modeling_qwen2_vl.py | 15 +++++++-- .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 10 ++++-- .../models/qwen3_vl/modeling_qwen3_vl.py | 9 ++++-- .../models/qwen3_vl/modular_qwen3_vl.py | 6 ++-- .../qwen3_vl_moe/modeling_qwen3_vl_moe.py | 9 ++++-- .../video_llama_3/modeling_video_llama_3.py | 10 +++++- .../video_llama_3/modular_video_llama_3.py | 5 ++- .../video_llava/modeling_video_llava.py | 32 +++++++++++++++++++ .../models/x_clip/modeling_x_clip.py | 7 ++++ 20 files changed, 225 insertions(+), 42 deletions(-) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index eddf2b68aef8..b99c4b5c6aa4 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -291,7 +291,10 @@ def get_rope_index( return position_ids, mrope_position_deltas def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -301,6 +304,8 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames @@ -309,9 +314,16 @@ def get_video_features( repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) temp_frames_hw.append(repeated_row) flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) - video_embeds = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw) + vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, return_dict=True) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - video_embeds = torch.split(video_embeds, split_sizes) + video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=video_embeds, + ) + return video_embeds def get_image_features( diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index 074226220647..10ac895207a3 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1139,7 +1139,10 @@ def get_rope_index( return position_ids, mrope_position_deltas def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1149,6 +1152,8 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames @@ -1157,9 +1162,16 @@ def get_video_features( repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) temp_frames_hw.append(repeated_row) flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) - video_embeds = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw) + vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, return_dict=True) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - video_embeds = torch.split(video_embeds, split_sizes) + video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=video_embeds, + ) + return video_embeds def get_image_features( diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 8cabe1793ab7..cbc7cd7233bb 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -30,7 +30,7 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling from ...modeling_rope_utils import RopeParameters from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...processing_utils import Unpack @@ -1131,7 +1131,7 @@ def get_rope_index( return position_ids, mrope_position_deltas def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1141,6 +1141,8 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames @@ -1149,9 +1151,16 @@ def get_video_features( repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) temp_frames_hw.append(repeated_row) flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) - video_embeds = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw) + vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, return_dict=True) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - video_embeds = torch.split(video_embeds, split_sizes) + video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=video_embeds, + ) + return video_embeds def get_placeholder_mask( diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 162d546bb13a..cc110917fb6f 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1348,7 +1348,10 @@ def get_rope_index( return position_ids, mrope_position_deltas def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1358,6 +1361,8 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames @@ -1366,9 +1371,16 @@ def get_video_features( repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) temp_frames_hw.append(repeated_row) flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) - video_embeds = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw) + vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, return_dict=True) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - video_embeds = torch.split(video_embeds, split_sizes) + video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=video_embeds, + ) + return video_embeds def get_image_features( diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index e9b02b7c158b..1579bb0cdf88 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -624,6 +624,7 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, ): """ Obtains video last hidden states from the vision tower and apply multimodal projection. @@ -638,6 +639,8 @@ def get_video_features( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches and are of shape `(num_videos, video_length, embed_dim)`). @@ -670,6 +673,13 @@ def get_video_features( video_features = self.vision_resampler(selected_video_features) video_features = self.multi_modal_projector(video_features) video_features = torch.split(video_features, frames, dim=0) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=video_features.last_hidden_state, + pooler_output=video_features, + ) + return video_features @@ -961,11 +971,13 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, ): return self.model.get_video_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, + return_dict=return_dict, ) diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index 5f13e7981f38..77d1e982f376 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -360,6 +360,7 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, ): """ Obtains video last hidden states from the vision tower and apply multimodal projection. @@ -374,6 +375,8 @@ def get_video_features( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. Returns: video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches and are of shape `(num_videos, video_length, embed_dim)`). @@ -406,6 +409,13 @@ def get_video_features( video_features = self.vision_resampler(selected_video_features) video_features = self.multi_modal_projector(video_features) video_features = torch.split(video_features, frames, dim=0) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=video_features.last_hidden_state, + pooler_output=video_features, + ) + return video_features def get_placeholder_mask( @@ -546,11 +556,13 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, ): return self.model.get_video_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, + return_dict=return_dict, ) def forward( diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index 345a84b6584a..6944e06cd431 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -607,6 +607,7 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: Union[int, list[int]], vision_feature_select_strategy: str, + return_dict: bool = False, ): """ Obtains video last hidden states from the vision tower, apply multimodal projection and pooling. @@ -621,20 +622,23 @@ def get_video_features( vision_feature_select_strategy (`str`): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches and are of shape `(num_videos, video_length, embed_dim)`). """ batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) - video_features = self.vision_tower(pixel_values, output_hidden_states=True) + vision_outputs = self.vision_tower(pixel_values, output_hidden_states=True) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): - selected_video_feature = video_features.hidden_states[vision_feature_layer] + selected_video_feature = vision_outputs.hidden_states[vision_feature_layer] else: - hs_pool = [video_features.hidden_states[layer_idx] for layer_idx in vision_feature_layer] + hs_pool = [vision_outputs.hidden_states[layer_idx] for layer_idx in vision_feature_layer] selected_video_feature = torch.cat(hs_pool, dim=-1) if vision_feature_select_strategy == "default": @@ -644,6 +648,12 @@ def get_video_features( video_features = self.apply_pooling(video_features) video_features = video_features.reshape(batch_size, frames * video_features.shape[1], -1) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=video_features, + ) + return video_features def apply_pooling(self, image_features): @@ -940,11 +950,13 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, + return_dict: bool = False, ): return self.model.get_video_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, + return_dict=return_dict, ) diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index f84246d9c12f..603604a1f1b1 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -416,6 +416,7 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: Union[int, list[int]], vision_feature_select_strategy: str, + return_dict: bool = False, ): """ Obtains video last hidden states from the vision tower, apply multimodal projection and pooling. @@ -430,20 +431,23 @@ def get_video_features( vision_feature_select_strategy (`str`): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches and are of shape `(num_videos, video_length, embed_dim)`). """ batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) - video_features = self.vision_tower(pixel_values, output_hidden_states=True) + vision_outputs = self.vision_tower(pixel_values, output_hidden_states=True) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): - selected_video_feature = video_features.hidden_states[vision_feature_layer] + selected_video_feature = vision_outputs.hidden_states[vision_feature_layer] else: - hs_pool = [video_features.hidden_states[layer_idx] for layer_idx in vision_feature_layer] + hs_pool = [vision_outputs.hidden_states[layer_idx] for layer_idx in vision_feature_layer] selected_video_feature = torch.cat(hs_pool, dim=-1) if vision_feature_select_strategy == "default": @@ -453,6 +457,12 @@ def get_video_features( video_features = self.apply_pooling(video_features) video_features = video_features.reshape(batch_size, frames * video_features.shape[1], -1) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=video_features, + ) + return video_features def forward( diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index f42523db8b86..18871532673f 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -1728,7 +1728,10 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1738,10 +1741,11 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw) - return video_embeds + return self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=return_dict) def get_image_features( self, diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index f4a4185c2301..2c7014110c53 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -2074,7 +2074,7 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -2084,10 +2084,11 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw) - return video_embeds + return self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=return_dict) def get_image_features( self, diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index 3e8af2c19b03..3e080b26dbdf 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -1169,7 +1169,10 @@ def get_rope_index( return position_ids, mrope_position_deltas def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1179,11 +1182,20 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw) + vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=True) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - video_embeds = torch.split(video_embeds, split_sizes) + video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=video_embeds, + ) + return video_embeds def get_image_features( diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 14592a1b607a..1056a54452b9 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -1105,7 +1105,7 @@ def get_rope_index( return position_ids, mrope_position_deltas def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1115,11 +1115,20 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw) + vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=True) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - video_embeds = torch.split(video_embeds, split_sizes) + video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=video_embeds, + ) + return video_embeds def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False): diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 5ca7e6279a73..4c2b73c97de2 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -1942,7 +1942,10 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1952,10 +1955,11 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw) - return video_embeds + return self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=return_dict) def get_image_features( self, diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index 7e2fe04f84c7..ad08e9d5dffd 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -1092,7 +1092,10 @@ def get_rope_index( return position_ids, mrope_position_deltas def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1102,9 +1105,11 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. """ # Same implementation as for images - return self.get_image_features(pixel_values_videos, video_grid_thw) + return self.get_image_features(pixel_values_videos, video_grid_thw, return_dict=return_dict) def get_image_features( self, diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index d0226d93b287..8c1ac6f0e7cc 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -984,7 +984,7 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op return image_embeds, deepstack_image_embeds def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -994,9 +994,11 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. """ # Same implementation as for images - return self.get_image_features(pixel_values_videos, video_grid_thw) + return self.get_image_features(pixel_values_videos, video_grid_thw, return_dict=return_dict) @auto_docstring @check_model_inputs diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index e55e6dc501ed..06a9d9f962da 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -1249,7 +1249,10 @@ def get_rope_index( return position_ids, mrope_position_deltas def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1259,9 +1262,11 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. """ # Same implementation as for images - return self.get_image_features(pixel_values_videos, video_grid_thw) + return self.get_image_features(pixel_values_videos, video_grid_thw, return_dict=return_dict) def get_image_features( self, diff --git a/src/transformers/models/video_llama_3/modeling_video_llama_3.py b/src/transformers/models/video_llama_3/modeling_video_llama_3.py index dbec71962581..1fd7c3799720 100644 --- a/src/transformers/models/video_llama_3/modeling_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modeling_video_llama_3.py @@ -542,6 +542,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor, video_merge_sizes: torch.LongTensor, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -553,8 +554,15 @@ def get_video_features( The temporal, height and width of feature shape of each video in LLM. video_merge_sizes (`torch.Tensor` of shape `(num_videos,)`): The spatial downsampling ratio of each video feature. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ - return self.get_image_features(pixel_values_videos, video_grid_thw, video_merge_sizes) + return self.get_image_features( + pixel_values=pixel_values_videos, + image_grid_thw=video_grid_thw, + image_merge_sizes=video_merge_sizes, + return_dict=return_dict, + ) def get_image_features( self, diff --git a/src/transformers/models/video_llama_3/modular_video_llama_3.py b/src/transformers/models/video_llama_3/modular_video_llama_3.py index eb4e6ad45969..b91b0644a0d9 100644 --- a/src/transformers/models/video_llama_3/modular_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modular_video_llama_3.py @@ -588,6 +588,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor, video_merge_sizes: torch.LongTensor, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -599,8 +600,10 @@ def get_video_features( The temporal, height and width of feature shape of each video in LLM. video_merge_sizes (`torch.Tensor` of shape `(num_videos,)`): The spatial downsampling ratio of each video feature. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ - return self.get_image_features(pixel_values_videos, video_grid_thw, video_merge_sizes) + return self.get_image_features(pixel_values=pixel_values_videos, image_grid_thw=video_grid_thw, image_merge_sizes=video_merge_sizes, return_dict=return_dict) def get_image_features( self, diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index 0be70ce86903..67d82823f10c 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -36,6 +36,27 @@ logger = logging.get_logger(__name__) +@dataclass +class BaseModelOutputWithNumFrames(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + num_frames (`int`): + Number of frames the videos have. + """ + last_hidden_state: torch.FloatTensor + pooler_output: torch.FloatTensor + num_frames: int + + @dataclass @auto_docstring( custom_intro=""" @@ -239,6 +260,7 @@ def get_video_features( self, pixel_values_videos: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, + return_dict: bool = False, ): """ Obtains video last hidden states from the vision tower and apply multimodal projection. @@ -250,6 +272,9 @@ def get_video_features( The index of the layer to select the vision feature. If multiple indices are provided, the vision feature of the corresponding indices will be concatenated to form the vision features. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: video_features (`torch.Tensor`): Video feature tensor of shape `(num_videos * num_frames, image_length, embed_dim)`). frames (`int`): Number of frames the videos have. @@ -273,6 +298,13 @@ def get_video_features( video_features = self.multi_modal_projector(video_features) + if return_dict: + return BaseModelOutputWithNumFrames( + last_hidden_state=video_outputs.last_hidden_state, + pooler_output=video_features, + num_frames=num_frames, + ) + return video_features, num_frames def get_placeholder_mask( diff --git a/src/transformers/models/x_clip/modeling_x_clip.py b/src/transformers/models/x_clip/modeling_x_clip.py index 731fd21c0d18..3d0c54680ec8 100644 --- a/src/transformers/models/x_clip/modeling_x_clip.py +++ b/src/transformers/models/x_clip/modeling_x_clip.py @@ -1249,8 +1249,12 @@ def get_text_features( def get_video_features( self, pixel_values: torch.Tensor, + return_dict: bool = False, ) -> torch.FloatTensor: r""" + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: video_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The video embeddings obtained by applying the projection layer to the pooled output of [`XCLIPVisionModel`] and @@ -1338,6 +1342,9 @@ def get_video_features( mit_outputs: BaseModelOutputWithPooling = self.mit(cls_features) video_embeds = mit_outputs.pooler_output + if return_dict: + return mit_outputs + return video_embeds @auto_docstring From 7eb89b615ee5dbe6ffedd484aa4ed0c4838c7988 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 12 Dec 2025 14:02:50 +0100 Subject: [PATCH 008/131] Run ruff formatting --- src/transformers/models/clap/modeling_clap.py | 2 +- src/transformers/models/emu3/modular_emu3.py | 4 +++- src/transformers/models/glm4v/modular_glm4v.py | 5 ++++- .../paddleocr_vl/modeling_paddleocr_vl.py | 18 +++++++++++++++--- .../qwen2_5_omni/modular_qwen2_5_omni.py | 9 +++++++-- .../models/qwen2_5_vl/modular_qwen2_5_vl.py | 4 +++- .../models/qwen2_vl/modeling_qwen2_vl.py | 12 ++++++++++-- .../models/qwen3_vl/modular_qwen3_vl.py | 17 ++++++++++++++--- .../modular_sam3_tracker_video.py | 1 + .../models/smolvlm/modular_smolvlm.py | 5 ++++- .../video_llama_3/modular_video_llama_3.py | 7 ++++++- .../models/video_llava/modeling_video_llava.py | 1 + .../models/vipllava/modular_vipllava.py | 14 +++++++++++--- 13 files changed, 80 insertions(+), 19 deletions(-) diff --git a/src/transformers/models/clap/modeling_clap.py b/src/transformers/models/clap/modeling_clap.py index 4036ba38f2d7..0849f21ed78d 100644 --- a/src/transformers/models/clap/modeling_clap.py +++ b/src/transformers/models/clap/modeling_clap.py @@ -1594,7 +1594,7 @@ def get_audio_features( input_features: torch.Tensor, is_longer: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - return_dict: bool = False + return_dict: bool = False, ) -> torch.FloatTensor: r""" is_longer (`torch.FloatTensor`, of shape `(batch_size, 1)`, *optional*): diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index 5a137cec3329..08e15816c96a 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -925,7 +925,9 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.L bpe_tokens = torch.cat(bpe_tokens_list) return bpe_tokens - def get_image_features(self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, return_dict: bool = False): + def get_image_features( + self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, return_dict: bool = False + ): """ Tokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layer diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 5b7cd6859797..96daf3ff3abc 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -1133,7 +1133,10 @@ def get_rope_index( return position_ids, mrope_position_deltas def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index ebd48e8d9c69..dcf36d79b933 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -1185,7 +1185,10 @@ def get_rope_index( return position_ids, mrope_position_deltas def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1195,11 +1198,20 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw) + vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=True) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - video_embeds = torch.split(video_embeds, split_sizes) + video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=video_embeds, + ) + return video_embeds def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index c8a5d2a95b26..631ba455c558 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -1928,7 +1928,9 @@ def __init__(self, config: Qwen2_5OmniVisionEncoderConfig, *inputs, **kwargs) -> super().__init__(config, *inputs, **kwargs) self.blocks = nn.ModuleList([Qwen2_5OmniVisionBlock(config) for _ in range(config.depth)]) - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs) -> torch.Tensor: + def forward( + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + ) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): @@ -2076,7 +2078,10 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False, + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py index a36609ce830e..7f79a7b38634 100644 --- a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py @@ -278,7 +278,9 @@ def get_window_index(self, grid_thw): return window_index, cu_window_seqlens - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs) -> torch.Tensor: + def forward( + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + ) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 71dd3041f132..f4111212dddc 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -1105,7 +1105,10 @@ def get_rope_index( return position_ids, mrope_position_deltas def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1131,7 +1134,12 @@ def get_video_features( return video_embeds - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False): + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, + ): """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index dfdc9983a201..9950d219f94f 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -65,6 +65,7 @@ logger = logging.get_logger(__name__) + @dataclass class BaseModelOutputWithDeepstackFeatures(ModelOutput): """ @@ -649,7 +650,9 @@ def fast_pos_embed_interpolate(self, grid_thw): patch_pos_embeds = torch.cat(patch_pos_embeds_permute) return patch_pos_embeds - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs) -> torch.Tensor: + def forward( + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + ) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): @@ -961,7 +964,12 @@ def get_rope_index( return position_ids, mrope_position_deltas - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False): + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, + ): """ Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -984,7 +992,10 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op return image_embeds, deepstack_image_embeds def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, return_dict: bool = False + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. diff --git a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py index 8d65402d63a2..e603a1ab0fb6 100644 --- a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py @@ -86,6 +86,7 @@ class BaseModelOutputWithFeatureMaps(ModelOutput): feature_maps: Optional[list[torch.Tensor]] = None feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None + class Sam3TrackerVideoPromptEncoderConfig(Sam2VideoPromptEncoderConfig): r""" This is the configuration class to store the configuration of a [`Sam3TrackerVideoPromptEncoder`]. The [`Sam3TrackerVideoPromptEncoder`] diff --git a/src/transformers/models/smolvlm/modular_smolvlm.py b/src/transformers/models/smolvlm/modular_smolvlm.py index 2b49382214e0..98457d4a1bf7 100644 --- a/src/transformers/models/smolvlm/modular_smolvlm.py +++ b/src/transformers/models/smolvlm/modular_smolvlm.py @@ -194,7 +194,10 @@ def inputs_merger( return merged_embeds def get_image_features( - self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, return_dict: bool = False + self, + pixel_values: torch.FloatTensor, + pixel_attention_mask: Optional[torch.LongTensor] = None, + return_dict: bool = False, ): """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/video_llama_3/modular_video_llama_3.py b/src/transformers/models/video_llama_3/modular_video_llama_3.py index b91b0644a0d9..8478155f771a 100644 --- a/src/transformers/models/video_llama_3/modular_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modular_video_llama_3.py @@ -603,7 +603,12 @@ def get_video_features( return_dict (`bool`, *optional*, default to `False`): Whether to return a `ModelOutput` instead of a pooled embedding. """ - return self.get_image_features(pixel_values=pixel_values_videos, image_grid_thw=video_grid_thw, image_merge_sizes=video_merge_sizes, return_dict=return_dict) + return self.get_image_features( + pixel_values=pixel_values_videos, + image_grid_thw=video_grid_thw, + image_merge_sizes=video_merge_sizes, + return_dict=return_dict, + ) def get_image_features( self, diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index 67d82823f10c..93b6ec59ad97 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -52,6 +52,7 @@ class BaseModelOutputWithNumFrames(ModelOutput): num_frames (`int`): Number of frames the videos have. """ + last_hidden_state: torch.FloatTensor pooler_output: torch.FloatTensor num_frames: int diff --git a/src/transformers/models/vipllava/modular_vipllava.py b/src/transformers/models/vipllava/modular_vipllava.py index e7a6282ac358..81b7235990bc 100644 --- a/src/transformers/models/vipllava/modular_vipllava.py +++ b/src/transformers/models/vipllava/modular_vipllava.py @@ -74,7 +74,10 @@ class VipLlavaPreTrainedModel(LlavaPreTrainedModel): class VipLlavaModel(LlavaModel): def get_image_features( - self, pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None, return_dict: bool = False + self, + pixel_values: torch.FloatTensor, + vision_feature_layers: Optional[Union[int, list[int]]] = None, + return_dict: bool = False, ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -186,9 +189,14 @@ def forward( class VipLlavaForConditionalGeneration(LlavaForConditionalGeneration): def get_image_features( - self, pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None, return_dict: bool = False + self, + pixel_values: torch.FloatTensor, + vision_feature_layers: Optional[Union[int, list[int]]] = None, + return_dict: bool = False, ): - return self.model.get_image_features(pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, return_dict=return_dict) + return self.model.get_image_features( + pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, return_dict=return_dict + ) def forward( self, From 57af63d3448100877d9089c739f33e7622a16b57 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 12 Dec 2025 14:49:42 +0100 Subject: [PATCH 009/131] Patch Glm4v VisionModel forward with BaseModelOutputWithPooling --- src/transformers/models/glm4v/modeling_glm4v.py | 17 ++++++++++++++--- src/transformers/models/glm4v/modular_glm4v.py | 17 ++++++++++++++--- .../models/glm4v_moe/modeling_glm4v_moe.py | 17 ++++++++++++++--- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index e10b206ed06b..dc8fbc42c1ef 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -768,13 +768,17 @@ def rot_pos_emb(self, grid_thw): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb, pos_ids - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor: + def forward( + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + ) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -813,8 +817,15 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) hidden_states = hidden_states.permute(0, 3, 1, 2) hidden_states = self.downsample(hidden_states).view(-1, self.config.out_hidden_size) - hidden_states = self.merger(hidden_states) - return hidden_states + merged_hidden_states = self.merger(hidden_states) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) + + return merged_hidden_states @auto_docstring diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 96daf3ff3abc..3dd09a481b0d 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -788,13 +788,17 @@ def rot_pos_emb(self, grid_thw): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb, pos_ids - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor: + def forward( + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + ) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -833,8 +837,15 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) hidden_states = hidden_states.permute(0, 3, 1, 2) hidden_states = self.downsample(hidden_states).view(-1, self.config.out_hidden_size) - hidden_states = self.merger(hidden_states) - return hidden_states + merged_hidden_states = self.merger(hidden_states) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) + + return merged_hidden_states class Glm4vTextModel(Qwen2_5_VLTextModel): diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 09f79917fc72..2fab4c75be7b 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -976,13 +976,17 @@ def rot_pos_emb(self, grid_thw): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb, pos_ids - def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor: + def forward( + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + ) -> torch.Tensor: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. + return_dict (`bool`, *optional*, defaults to `False`): + Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -1021,8 +1025,15 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) hidden_states = hidden_states.permute(0, 3, 1, 2) hidden_states = self.downsample(hidden_states).view(-1, self.config.out_hidden_size) - hidden_states = self.merger(hidden_states) - return hidden_states + merged_hidden_states = self.merger(hidden_states) + + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) + + return merged_hidden_states @auto_docstring From 7285187cd547a850e69ca90fe42731afa51c6592 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 12 Dec 2025 15:16:17 +0100 Subject: [PATCH 010/131] Patch instructblip, although backwards incompatibility stands --- .../instructblip/modeling_instructblip.py | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index 3a4a7a392706..4cf4864bb056 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -45,6 +45,31 @@ logger = logging.get_logger(__name__) +@dataclass +class BaseModelOutputWithVisionQformerOutputs(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + vision_outputs (`BaseModelOutputWithPooling`): + Outputs of the vision encoder. + qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): + Outputs of the Q-Former (Querying Transformer). + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + vision_outputs: Optional[torch.FloatTensor] = None + qformer_outputs: Optional[tuple[torch.FloatTensor]] = None + + @dataclass @auto_docstring( custom_intro=""" @@ -1220,9 +1245,11 @@ def get_image_features( # NOTE: @Tom backwards incompatibility if return_dict: - return BaseModelOutputWithPooling( + return BaseModelOutputWithVisionQformerOutputs( last_hidden_state=vision_outputs.last_hidden_state, pooler_output=image_features, + vision_outputs=vision_outputs, + qformer_outputs=query_outputs, ) return image_features @@ -1319,13 +1346,16 @@ def forward( The unusual aspect of this image is that a man is ironing clothes on the back of a yellow SUV, which is parked in the middle of a busy city street. This is an unconventional approach to ironing clothes, as it requires the man to balance himself and his ironing equipment on top of the vehicle while navigating through traffic. Additionally, the presence of taxis and other vehicles in the scene further emphasizes the unusual nature of this situation. ```""" - language_model_inputs, vision_outputs, query_outputs = self.get_image_features( + image_features = self.get_image_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, ) + language_model_inputs = image_features.pooler_output + vision_outputs = image_features.vision_outputs + query_outputs = image_features.qformer_outputs if inputs_embeds is None: inputs_embeds = self.get_input_embeddings()(input_ids) @@ -1410,12 +1440,12 @@ def generate( self._preprocess_accelerate() batch_size = pixel_values.shape[0] - language_model_inputs, vision_outputs, query_outputs = self.get_image_features( + language_model_inputs = self.get_image_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, interpolate_pos_encoding=interpolate_pos_encoding, - return_dict=True, + return_dict=False, ) if inputs_embeds is None: From fd7be5272778801853399b98e535d6173c5e82ef Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 12 Dec 2025 16:01:56 +0100 Subject: [PATCH 011/131] Patch Kosmos2 and Ovis2 --- .../models/kosmos2/modeling_kosmos2.py | 45 ++++++++++-- .../models/ovis2/modeling_ovis2.py | 70 +++++++++++++++---- .../models/ovis2/modular_ovis2.py | 67 ++++++++++++++---- 3 files changed, 148 insertions(+), 34 deletions(-) diff --git a/src/transformers/models/kosmos2/modeling_kosmos2.py b/src/transformers/models/kosmos2/modeling_kosmos2.py index 88756b6d4c2c..79377b172566 100644 --- a/src/transformers/models/kosmos2/modeling_kosmos2.py +++ b/src/transformers/models/kosmos2/modeling_kosmos2.py @@ -15,6 +15,7 @@ """PyTorch KOSMOS-2 model.""" import math +import warnings from collections.abc import Callable from dataclasses import dataclass from typing import Any, Optional, Union @@ -74,6 +75,32 @@ def _make_causal_mask( return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) +@dataclass +class BaseModelOutputWithProjectionAttentions(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + projection_attentions (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights given by `Kosmos2ImageToTextProjection`, after the attention softmax, used to compute + the weighted average in the self-attention heads. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + projection_attentions: Optional[tuple[torch.FloatTensor]] = None + + @dataclass @auto_docstring( custom_intro=""" @@ -1483,9 +1510,9 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, - return_attentions: Optional[bool] = False, # TODO: @Tom neatly deprecate this interpolate_pos_encoding: Optional[bool] = False, return_dict: bool = False, + return_attentions: Optional[bool] = False, # TODO: @Tom neatly deprecate this ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1493,13 +1520,21 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): The tensors corresponding to the input images. - return_attentions (`bool`, *optional*, defaults to `False`): - Whether to return `projection_attentions` or not. interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): Whether to interpolate positional embeddings or not. return_dict (`bool`, *optional*, default to `False`): Whether to return a `ModelOutput` instead of a pooled embedding. """ + + # NOTE: @Tom backwards incompatibility + if return_attentions: + warnings.warn( + "`return_attentions` is deprecated and will be removed in a future version. Please use `return_dict`" + " and access `projection_attentions` from the returned `ModelOutput` instead.", + FutureWarning, + ) + return_dict = True + vision_model_output = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, @@ -1511,10 +1546,10 @@ def get_image_features( image_embeds, projection_attentions = self.image_to_text_projection(image_embeds) if return_dict: - return BaseModelOutputWithPooling( + return BaseModelOutputWithProjectionAttentions( last_hidden_state=vision_model_output.last_hidden_state, pooler_output=image_embeds, - attentions=projection_attentions, # TODO: @Tom does this match expectations? + projection_attentions=projection_attentions, ) return image_embeds diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index f9c63bd7a637..44f319b068df 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -32,14 +32,36 @@ from ...generation import GenerationMixin from ...integrations import use_kernel_forward_from_hub from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ..auto import AutoModel from .configuration_ovis2 import Ovis2Config, Ovis2VisionConfig +@dataclass +class BaseModelOutputWithVisualIndicatorFeatures(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + visual_indicator_features (`torch.FloatTensor` of shape `(batch_size, visual_indicator_size)`): + Visual indicator features extracted from the model, which can be used for auxiliary tasks or further processing. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + visual_indicator_features: Optional[torch.FloatTensor] = None + + @dataclass @auto_docstring( custom_intro=""" @@ -457,7 +479,9 @@ def __init__(self, config: Ovis2VisionConfig): ) self.head_norm = nn.LayerNorm(self.vocab_size - self.num_visual_indicator_tokens) - def forward(self, pixel_values: torch.FloatTensor, **kwargs) -> tuple[torch.Tensor, torch.Tensor]: + def forward( + self, pixel_values: torch.FloatTensor, return_dict: bool = False, **kwargs + ) -> tuple[torch.Tensor, torch.Tensor]: outputs = self.transformer(pixel_values, **kwargs) last_hidden_state = outputs[0] if self.config.hidden_stride > 1: @@ -490,6 +514,12 @@ def forward(self, pixel_values: torch.FloatTensor, **kwargs) -> tuple[torch.Tens elif self.config.tokenize_function == "softmax": prob_token = nn.functional.softmax(logits, dim=-1) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=prob_token, + ) + return prob_token @@ -521,6 +551,7 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, + return_dict: bool = False, ) -> torch.FloatTensor: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -538,7 +569,8 @@ def get_image_features( Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - image_features = self.vision_tower(pixel_values) + image_outputs = self.vision_tower(pixel_values, return_dict=True) + image_features = image_outputs.pooler_output batch_size, img_seq_len, _ = image_features.shape padding_tensor = torch.zeros( (batch_size, img_seq_len, self.vision_tower.num_visual_indicator_tokens), @@ -550,15 +582,21 @@ def get_image_features( image_features = torch.cat([image_features, padding_tensor], dim=2) image_features = self.visual_embeddings_table(image_features) - visual_indicator = torch.arange( - self.visual_vocab_size - self.vision_tower.num_visual_indicator_tokens, - self.visual_vocab_size, - dtype=torch.long, - ).to(image_features.device) - visual_indicator_features = self.visual_embeddings_table(visual_indicator) + if return_dict: + visual_indicator = torch.arange( + self.visual_vocab_size - self.vision_tower.num_visual_indicator_tokens, + self.visual_vocab_size, + dtype=torch.long, + ).to(image_features.device) + visual_indicator_features = self.visual_embeddings_table(visual_indicator) + + return BaseModelOutputWithVisualIndicatorFeatures( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + visual_indicator_features=visual_indicator_features, + ) - # NOTE: @Tom Not easily converted to the standard format - return image_features, visual_indicator_features + return image_features def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -615,7 +653,9 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_features, visual_indicator_features = self.get_image_features(pixel_values=pixel_values) + image_outputs = self.get_image_features(pixel_values=pixel_values, return_dict=True) + image_features = image_outputs.pooler_output + visual_indicator_features = image_outputs.visual_indicator_features special_image_mask = self.get_placeholder_mask( input_ids, @@ -683,8 +723,8 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head - def get_image_features(self, pixel_values: torch.FloatTensor): - return self.model.get_image_features(pixel_values=pixel_values) + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False) -> torch.FloatTensor: + return self.model.get_image_features(pixel_values=pixel_values, return_dict=return_dict) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index 80847f4ccfea..38a866792307 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -14,6 +14,7 @@ # limitations under the License. import math +from dataclasses import dataclass from typing import Optional, Union import torch @@ -21,7 +22,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -44,6 +45,28 @@ def hard_softmax(logits: torch.Tensor, dim: int): return ret +@dataclass +class BaseModelOutputWithVisualIndicatorFeatures(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + visual_indicator_features (`torch.FloatTensor` of shape `(batch_size, visual_indicator_size)`): + Visual indicator features extracted from the model, which can be used for auxiliary tasks or further processing. + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + visual_indicator_features: Optional[torch.FloatTensor] = None + + class Ovis2ModelOutputWithPast(LlavaNextModelOutputWithPast): pass @@ -176,7 +199,7 @@ def __init__(self, config: Ovis2VisionConfig): ) self.head_norm = nn.LayerNorm(self.vocab_size - self.num_visual_indicator_tokens) - def forward(self, pixel_values: torch.FloatTensor, **kwargs) -> tuple[torch.Tensor, torch.Tensor]: + def forward(self, pixel_values: torch.FloatTensor, return_dict: bool = False, **kwargs) -> tuple[torch.Tensor, torch.Tensor]: outputs = self.transformer(pixel_values, **kwargs) last_hidden_state = outputs[0] if self.config.hidden_stride > 1: @@ -209,6 +232,12 @@ def forward(self, pixel_values: torch.FloatTensor, **kwargs) -> tuple[torch.Tens elif self.config.tokenize_function == "softmax": prob_token = nn.functional.softmax(logits, dim=-1) + if return_dict: + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=prob_token, + ) + return prob_token @@ -229,8 +258,10 @@ def __init__(self, config: Ovis2Config): def get_image_features( self, pixel_values: torch.FloatTensor, + return_dict: bool = False, ) -> torch.FloatTensor: - image_features = self.vision_tower(pixel_values) + image_outputs = self.vision_tower(pixel_values, return_dict=True) + image_features = image_outputs.pooler_output batch_size, img_seq_len, _ = image_features.shape padding_tensor = torch.zeros( (batch_size, img_seq_len, self.vision_tower.num_visual_indicator_tokens), @@ -242,15 +273,21 @@ def get_image_features( image_features = torch.cat([image_features, padding_tensor], dim=2) image_features = self.visual_embeddings_table(image_features) - visual_indicator = torch.arange( - self.visual_vocab_size - self.vision_tower.num_visual_indicator_tokens, - self.visual_vocab_size, - dtype=torch.long, - ).to(image_features.device) - visual_indicator_features = self.visual_embeddings_table(visual_indicator) + if return_dict: + visual_indicator = torch.arange( + self.visual_vocab_size - self.vision_tower.num_visual_indicator_tokens, + self.visual_vocab_size, + dtype=torch.long, + ).to(image_features.device) + visual_indicator_features = self.visual_embeddings_table(visual_indicator) + + return BaseModelOutputWithVisualIndicatorFeatures( + last_hidden_state=image_outputs.last_hidden_state, + pooler_output=image_features, + visual_indicator_features=visual_indicator_features, + ) - # NOTE: @Tom Not easily converted to the standard format - return image_features, visual_indicator_features + return image_features @can_return_tuple @auto_docstring @@ -283,7 +320,9 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_features, visual_indicator_features = self.get_image_features(pixel_values=pixel_values) + image_outputs = self.get_image_features(pixel_values=pixel_values, return_dict=True) + image_features = image_outputs.pooler_output + visual_indicator_features = image_outputs.visual_indicator_features special_image_mask = self.get_placeholder_mask( input_ids, @@ -339,8 +378,8 @@ def __init__(self, config: Ovis2Config): super().__init__(config) self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - def get_image_features(self, pixel_values: torch.FloatTensor): - return self.model.get_image_features(pixel_values=pixel_values) + def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False) -> torch.FloatTensor: + return self.model.get_image_features(pixel_values=pixel_values, return_dict=return_dict) @can_return_tuple @auto_docstring From 3f183fd419376fb1f94dd6b7aab2851af847f02c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 12 Dec 2025 16:46:48 +0100 Subject: [PATCH 012/131] Reformat Ovis2 --- src/transformers/models/ovis2/modular_ovis2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index 38a866792307..ef0eebc642ef 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -199,7 +199,9 @@ def __init__(self, config: Ovis2VisionConfig): ) self.head_norm = nn.LayerNorm(self.vocab_size - self.num_visual_indicator_tokens) - def forward(self, pixel_values: torch.FloatTensor, return_dict: bool = False, **kwargs) -> tuple[torch.Tensor, torch.Tensor]: + def forward( + self, pixel_values: torch.FloatTensor, return_dict: bool = False, **kwargs + ) -> tuple[torch.Tensor, torch.Tensor]: outputs = self.transformer(pixel_values, **kwargs) last_hidden_state = outputs[0] if self.config.hidden_stride > 1: From 391aac9307681ffa931da6f8e11705a71de7c5f0 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 12 Dec 2025 17:15:22 +0100 Subject: [PATCH 013/131] Avoid now-deprecated return_attentions --- src/transformers/models/kosmos2/modeling_kosmos2.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/kosmos2/modeling_kosmos2.py b/src/transformers/models/kosmos2/modeling_kosmos2.py index 79377b172566..d7ea18587fb3 100644 --- a/src/transformers/models/kosmos2/modeling_kosmos2.py +++ b/src/transformers/models/kosmos2/modeling_kosmos2.py @@ -1624,9 +1624,11 @@ def forward( if image_embeds is None: if pixel_values is None: raise ValueError("You have to specify either `pixel_values` or `image_embeds`.") - image_embeds, projection_attentions = self.get_image_features( - pixel_values, return_attentions=True, interpolate_pos_encoding=interpolate_pos_encoding + image_features = self.get_image_features( + pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True ) + image_embeds = image_features.pooler_output + projection_attentions = image_features.projection_attentions outputs = self.text_model( input_ids=input_ids, From f8c887ff4b364a6eb2fb00ceca85eb5901998b86 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 16 Dec 2025 09:32:23 +0100 Subject: [PATCH 014/131] Remove NumFrames --- .../video_llava/modeling_video_llava.py | 36 ++++--------------- 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index 93b6ec59ad97..92608503ba29 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -36,28 +36,6 @@ logger = logging.get_logger(__name__) -@dataclass -class BaseModelOutputWithNumFrames(ModelOutput): - """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - num_frames (`int`): - Number of frames the videos have. - """ - - last_hidden_state: torch.FloatTensor - pooler_output: torch.FloatTensor - num_frames: int - - @dataclass @auto_docstring( custom_intro=""" @@ -287,7 +265,7 @@ def get_video_features( batch_size_vid, num_frames, channels, height, width = pixel_values_videos.shape pixel_values = pixel_values_videos.reshape(batch_size_vid * num_frames, channels, height, width) - video_outputs = self.video_tower(pixel_values, output_hidden_states=True) + video_outputs = self.video_tower(pixel_values, output_hidden_states=True, return_dict=True) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them @@ -300,13 +278,11 @@ def get_video_features( video_features = self.multi_modal_projector(video_features) if return_dict: - return BaseModelOutputWithNumFrames( - last_hidden_state=video_outputs.last_hidden_state, - pooler_output=video_features, - num_frames=num_frames, - ) + video_outputs.pooler_output = video_features + return video_outputs - return video_features, num_frames + # NOTE: @Tom backwards incompatibility + return video_features def get_placeholder_mask( self, @@ -407,7 +383,7 @@ def forward( inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features) if pixel_values_videos is not None: - video_features, num_frames = self.get_video_features( + video_features = self.get_video_features( pixel_values_videos=pixel_values_videos, vision_feature_layer=vision_feature_layer ) video_features = video_features.to(inputs_embeds.device, inputs_embeds.dtype) From 9a251ce03753d4a85f3b7845f383f1eed5d2baac Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 16 Dec 2025 10:57:19 +0100 Subject: [PATCH 015/131] Proposal to simplify get_..._features via TransformersKwargs & check_model_inputs The changes in check_model_inputs aren't the clearest/prettiest, but they work well for now. --- src/transformers/models/clip/modeling_clip.py | 36 ++++++------------- src/transformers/utils/generic.py | 16 +++++++-- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index 9994d7915598..84b89fe5669c 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -781,19 +781,16 @@ def __init__(self, config: CLIPConfig): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @check_model_inputs(tie_last_hidden_states=False, default_return_pooled=True) @auto_docstring def get_text_features( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`CLIPTextModel`]. @@ -816,30 +813,22 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + **kwargs, ) pooled_output = text_outputs.pooler_output - text_features = self.text_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) + text_outputs.pooler_output = self.text_projection(pooled_output) - return text_features + return text_outputs - @filter_out_non_signature_kwargs() + @check_model_inputs(tie_last_hidden_states=False, default_return_pooled=True) @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`CLIPVisionModel`]. @@ -865,17 +854,12 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + **kwargs, ) pooled_output = vision_outputs.pooler_output - image_features = self.visual_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + vision_outputs.pooler_output = self.visual_projection(pooled_output) - return image_features + return vision_outputs @can_return_tuple @auto_docstring diff --git a/src/transformers/utils/generic.py b/src/transformers/utils/generic.py index 5b7a10ad7186..fba43fadf7b5 100644 --- a/src/transformers/utils/generic.py +++ b/src/transformers/utils/generic.py @@ -814,7 +814,7 @@ class OutputRecorder: class_name: str | None = None -def check_model_inputs(func=None, *, tie_last_hidden_states=True): +def check_model_inputs(func=None, *, tie_last_hidden_states=True, default_return_pooled=False): """ Decorator to intercept specific layer outputs without using hooks. Compatible with torch.compile (Dynamo tracing). @@ -825,6 +825,10 @@ def check_model_inputs(func=None, *, tie_last_hidden_states=True): This is true for all language models and should be toggled off only if `out.hidden_states[-1]` has to be the hidden state before last layer norm, which is needed for some vision models (e.g. CLIP, SigLIP) + default_return_pooled (`bool`, *optional*, defaults to `False`): + Whether the default behavior is to return pooled outputs. If `True`, and `return_dict` + is not specified in the function call, then `return_dict` will be set to `False` instead + of reading from `self.config.return_dict`. """ def wrapped_fn(func): @@ -859,7 +863,10 @@ def wrapper(self, *args, **kwargs): else: kwargs["use_cache"] = use_cache - return_dict = kwargs.pop("return_dict", None) + # User-provided kwargs will always take precedence, but if the wrapped method should return pooled outputs + # by default, then we set return_dict to False instead of reading from the model config. + default_return_dict = False if default_return_pooled else None + return_dict = kwargs.pop("return_dict", default_return_dict) if return_dict is None: return_dict = getattr(self.config, "return_dict", True) @@ -983,7 +990,10 @@ def wrapped_forward(*args, **kwargs): else: outputs[key] = collected_outputs[key] if return_dict is False: - outputs = outputs.to_tuple() + if default_return_pooled: + outputs = outputs.pooler_output + else: + outputs = outputs.to_tuple() return outputs return wrapper From 858d9d4260ab90dfb1cf19f1fb254565c49999af Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 16 Dec 2025 13:22:13 +0100 Subject: [PATCH 016/131] Revert check_model_inputs, adopt can_return_tuple, accept BC on get_..._features methods This commit updates all get_text_features methods, even blip_2, which was previously not yet attempted --- .../models/aimv2/modeling_aimv2.py | 38 +++++----------- .../models/align/modeling_align.py | 39 +++++----------- .../models/altclip/modeling_altclip.py | 39 +++++----------- src/transformers/models/blip/modeling_blip.py | 37 +++++----------- .../models/blip_2/modeling_blip_2.py | 29 ++++-------- .../chinese_clip/modeling_chinese_clip.py | 42 ++++++------------ src/transformers/models/clap/modeling_clap.py | 41 ++++++----------- src/transformers/models/clip/modeling_clip.py | 5 +-- .../models/clipseg/modeling_clipseg.py | 39 +++++----------- src/transformers/models/clvp/modeling_clvp.py | 23 ++++------ .../models/flava/modeling_flava.py | 41 ++++++----------- .../models/glm46v/modeling_glm46v.py | 17 +++---- .../models/glm4v/modeling_glm4v.py | 35 ++++++--------- .../models/glm4v/modular_glm4v.py | 35 ++++++--------- .../models/glm4v_moe/modeling_glm4v_moe.py | 35 ++++++--------- .../models/groupvit/modeling_groupvit.py | 38 ++++++---------- .../models/metaclip_2/modeling_metaclip_2.py | 44 +++++-------------- .../models/metaclip_2/modular_metaclip_2.py | 17 +++---- .../models/owlv2/modeling_owlv2.py | 44 +++++++------------ .../models/owlvit/modeling_owlvit.py | 44 +++++++------------ .../models/siglip/modeling_siglip.py | 31 +++---------- .../models/siglip2/modeling_siglip2.py | 42 +++++------------- .../models/siglip2/modular_siglip2.py | 28 ++++-------- .../modeling_vision_text_dual_encoder.py | 42 ++++++------------ .../models/x_clip/modeling_x_clip.py | 40 +++++------------ src/transformers/utils/generic.py | 16 ++----- 26 files changed, 288 insertions(+), 593 deletions(-) diff --git a/src/transformers/models/aimv2/modeling_aimv2.py b/src/transformers/models/aimv2/modeling_aimv2.py index aa72b6eb34f5..5a6c03a227aa 100644 --- a/src/transformers/models/aimv2/modeling_aimv2.py +++ b/src/transformers/models/aimv2/modeling_aimv2.py @@ -37,7 +37,7 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, filter_out_non_signature_kwargs +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple from ...utils.generic import check_model_inputs from .configuration_aimv2 import Aimv2Config, Aimv2TextConfig, Aimv2VisionConfig @@ -600,19 +600,16 @@ def __init__(self, config: Aimv2Config): self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`Aimv2TextModel`]. @@ -635,30 +632,22 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + **kwargs, ) pooled_output = text_outputs.pooler_output - text_features = self.text_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) + text_outputs.pooler_output = self.text_projection(pooled_output) - return text_features + return text_outputs - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`Aimv2VisionModel`]. @@ -684,17 +673,12 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + **kwargs, ) pooled_output = vision_outputs.pooler_output - image_features = self.visual_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + vision_outputs.pooler_output = self.visual_projection(pooled_output) - return image_features + return vision_outputs @auto_docstring @can_return_tuple diff --git a/src/transformers/models/align/modeling_align.py b/src/transformers/models/align/modeling_align.py index 8171d6398953..57f81977d1c3 100644 --- a/src/transformers/models/align/modeling_align.py +++ b/src/transformers/models/align/modeling_align.py @@ -32,8 +32,9 @@ BaseModelOutputWithPoolingAndNoAttention, ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from ...processing_utils import Unpack from ...pytorch_utils import apply_chunking_to_forward -from ...utils import ModelOutput, auto_docstring, can_return_tuple, filter_out_non_signature_kwargs, logging +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, logging from .configuration_align import AlignConfig, AlignTextConfig, AlignVisionConfig @@ -1088,7 +1089,7 @@ def __init__(self, config: AlignConfig): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, @@ -1097,12 +1098,9 @@ def get_text_features( token_type_ids: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, inputs_embeds: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`AlignTextModel`]. @@ -1126,27 +1124,19 @@ def get_text_features( token_type_ids=token_type_ids, position_ids=position_ids, inputs_embeds=inputs_embeds, + **kwargs, ) last_hidden_state = text_outputs[0][:, 0, :] - text_features = self.text_projection(last_hidden_state) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) + text_outputs.pooler_output = self.text_projection(last_hidden_state) - return text_features + return text_outputs - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_image_features( - self, pixel_values: torch.FloatTensor, return_dict: bool = False + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`AlignVisionModel`]. @@ -1168,16 +1158,7 @@ def get_image_features( >>> with torch.inference_mode(): ... image_features = model.get_image_features(**inputs) ```""" - vision_outputs: BaseModelOutputWithPoolingAndNoAttention = self.vision_model(pixel_values=pixel_values) - image_features = vision_outputs.pooler_output - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return self.vision_model(pixel_values=pixel_values, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/altclip/modeling_altclip.py b/src/transformers/models/altclip/modeling_altclip.py index 9a9568ed8485..42faa1eea0bd 100755 --- a/src/transformers/models/altclip/modeling_altclip.py +++ b/src/transformers/models/altclip/modeling_altclip.py @@ -32,8 +32,9 @@ BaseModelOutputWithPoolingAndProjection, ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from ...processing_utils import Unpack from ...pytorch_utils import apply_chunking_to_forward -from ...utils import ModelOutput, auto_docstring, can_return_tuple, filter_out_non_signature_kwargs, logging, torch_int +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, logging, torch_int from .configuration_altclip import AltCLIPConfig, AltCLIPTextConfig, AltCLIPVisionConfig @@ -1150,7 +1151,7 @@ def __init__(self, config: AltCLIPConfig): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, @@ -1158,12 +1159,9 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, token_type_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`AltCLIPTextModel`]. @@ -1186,30 +1184,22 @@ def get_text_features( attention_mask=attention_mask, position_ids=position_ids, token_type_ids=token_type_ids, + **kwargs, ) pooled_output = text_outputs.pooler_output - text_features = self.text_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) + text_outputs.pooler_output = self.text_projection(pooled_output) - return text_features + return text_outputs - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`AltCLIPVisionModel`]. @@ -1234,17 +1224,12 @@ def get_image_features( vision_outputs = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + **kwargs, ) pooled_output = vision_outputs.pooler_output - image_features = self.visual_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + vision_outputs.pooler_output = self.visual_projection(pooled_output) - return image_features + return vision_outputs @auto_docstring def forward( diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index d145055b1f4a..43b459dc5019 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -571,18 +571,16 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.text_model.set_input_embeddings(value) + @can_return_tuple @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`BlipTextModel`]. @@ -602,30 +600,22 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + **kwargs, ) + pooled_output = text_outputs.pooler_output + text_outputs.pooler_output = self.text_projection(pooled_output) - pooled_output = text_outputs[1] - text_features = self.text_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) - - return text_features + return text_outputs + @can_return_tuple @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`BlipVisionModel`]. @@ -651,17 +641,12 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + **kwargs, ) pooled_output = vision_outputs.pooler_output - image_features = self.visual_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + vision_outputs.pooler_output = self.visual_projection(pooled_output) - return image_features + return vision_outputs @auto_docstring def get_multimodal_features( diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 7bf8a7e09c20..cdc19a1e2cf0 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -1066,7 +1066,7 @@ def get_encoder(self, modality=None): else: return super().get_encoder(modality=modality) - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, @@ -1075,6 +1075,7 @@ def get_text_features( decoder_input_ids: Optional[torch.Tensor] = None, decoder_attention_mask: Optional[torch.Tensor] = None, labels: Optional[torch.Tensor] = None, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, CausalLMOutputWithPast]: r""" decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): @@ -1115,7 +1116,7 @@ def get_text_features( text_outputs: CausalLMOutputWithPast = self.language_model( input_ids=input_ids, attention_mask=attention_mask, - return_dict=True, + **kwargs, ) else: inputs_embeds = self.language_model.get_input_embeddings()(input_ids) @@ -1125,23 +1126,20 @@ def get_text_features( decoder_input_ids=decoder_input_ids, decoder_attention_mask=decoder_attention_mask, labels=labels, - return_dict=True, + **kwargs, ) - return text_outputs.logits + return text_outputs - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: vision_outputs (`torch.FloatTensor`): The vision model's last layer pooled logits. @@ -1162,20 +1160,11 @@ def get_image_features( >>> with torch.inference_mode(): ... image_outputs = model.get_image_features(**inputs) ```""" - vision_outputs = self.vision_model( + return self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, - return_dict=True, + **kwargs, ) - image_features = vision_outputs.pooler_output - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features @filter_out_non_signature_kwargs() @auto_docstring diff --git a/src/transformers/models/chinese_clip/modeling_chinese_clip.py b/src/transformers/models/chinese_clip/modeling_chinese_clip.py index 58cafd1cd436..4c360d87cc3a 100644 --- a/src/transformers/models/chinese_clip/modeling_chinese_clip.py +++ b/src/transformers/models/chinese_clip/modeling_chinese_clip.py @@ -31,8 +31,9 @@ BaseModelOutputWithPoolingAndCrossAttentions, ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from ...processing_utils import Unpack from ...pytorch_utils import apply_chunking_to_forward -from ...utils import ModelOutput, auto_docstring, can_return_tuple, filter_out_non_signature_kwargs, logging, torch_int +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, logging, torch_int from .configuration_chinese_clip import ChineseCLIPConfig, ChineseCLIPTextConfig, ChineseCLIPVisionConfig @@ -998,7 +999,7 @@ def __init__(self, config: ChineseCLIPConfig): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, @@ -1006,12 +1007,9 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, token_type_ids: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the final [CLS] hidden state of Text-Transformer. @@ -1035,31 +1033,22 @@ def get_text_features( attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, + **kwargs, ) + pooled_output = text_outputs.last_hidden_state[:, 0, :] + text_outputs.pooler_output = self.text_projection(pooled_output) - pooled_output = text_outputs[0][:, 0, :] - text_features = self.text_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) - - return text_features + return text_outputs - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the final [CLS] hidden state of Vision-Transformer. @@ -1086,17 +1075,12 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + **kwargs, ) pooled_output = vision_outputs.pooler_output - image_features = self.visual_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + vision_outputs.pooler_output = self.visual_projection(pooled_output) - return image_features + return vision_outputs @can_return_tuple @auto_docstring diff --git a/src/transformers/models/clap/modeling_clap.py b/src/transformers/models/clap/modeling_clap.py index 0849f21ed78d..31697fb55fb5 100644 --- a/src/transformers/models/clap/modeling_clap.py +++ b/src/transformers/models/clap/modeling_clap.py @@ -33,8 +33,9 @@ BaseModelOutputWithPoolingAndCrossAttentions, ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from ...processing_utils import Unpack from ...pytorch_utils import apply_chunking_to_forward, meshgrid -from ...utils import ModelOutput, auto_docstring, can_return_tuple, filter_out_non_signature_kwargs, logging, torch_int +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, logging, torch_int from .configuration_clap import ClapAudioConfig, ClapConfig, ClapTextConfig @@ -1543,19 +1544,16 @@ def __init__(self, config: ClapConfig): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`ClapTextModel`]. @@ -1574,34 +1572,29 @@ def get_text_features( ... text_features = model.get_text_features(**inputs) ```""" text_outputs: BaseModelOutputWithPooling = self.text_model( - input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + **kwargs, ) text_features = self.text_projection(text_outputs.pooler_output) - text_features = F.normalize(text_features, dim=-1) + text_outputs.pooler_output = F.normalize(text_features, dim=-1) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) - - return text_features + return text_outputs - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_audio_features( self, input_features: torch.Tensor, is_longer: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: r""" is_longer (`torch.FloatTensor`, of shape `(batch_size, 1)`, *optional*): Whether the audio clip is longer than `max_length`. If `True`, a feature fusion will be enabled to enhance the features. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: audio_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The audio embeddings obtained by @@ -1625,15 +1618,9 @@ def get_audio_features( input_features=input_features, is_longer=is_longer ) audio_features = self.audio_projection(audio_outputs.pooler_output) - audio_features = F.normalize(audio_features, dim=-1) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=audio_outputs.last_hidden_state, - pooler_output=audio_features, - ) + audio_outputs.pooler_output = F.normalize(audio_features, dim=-1) - return audio_features + return audio_outputs @can_return_tuple @auto_docstring diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index 84b89fe5669c..5741f329379a 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -33,7 +33,6 @@ TransformersKwargs, auto_docstring, can_return_tuple, - filter_out_non_signature_kwargs, logging, torch_int, ) @@ -781,7 +780,7 @@ def __init__(self, config: CLIPConfig): # Initialize weights and apply final processing self.post_init() - @check_model_inputs(tie_last_hidden_states=False, default_return_pooled=True) + @can_return_tuple @auto_docstring def get_text_features( self, @@ -820,7 +819,7 @@ def get_text_features( return text_outputs - @check_model_inputs(tie_last_hidden_states=False, default_return_pooled=True) + @can_return_tuple @auto_docstring def get_image_features( self, diff --git a/src/transformers/models/clipseg/modeling_clipseg.py b/src/transformers/models/clipseg/modeling_clipseg.py index bff8eb86b816..d6f25dc97b15 100644 --- a/src/transformers/models/clipseg/modeling_clipseg.py +++ b/src/transformers/models/clipseg/modeling_clipseg.py @@ -29,7 +29,8 @@ from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...utils import ModelOutput, auto_docstring, can_return_tuple, filter_out_non_signature_kwargs, logging, torch_int +from ...processing_utils import Unpack +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, logging, torch_int from .configuration_clipseg import CLIPSegConfig, CLIPSegTextConfig, CLIPSegVisionConfig @@ -848,19 +849,16 @@ def __init__(self, config: CLIPSegConfig): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`CLIPSegTextModel`]. @@ -882,30 +880,22 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + **kwargs, ) pooled_output = text_outputs.pooler_output - text_features = self.text_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) + text_outputs.pooler_output = self.text_projection(pooled_output) - return text_features + return text_outputs - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = True, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`CLIPSegVisionModel`]. @@ -931,17 +921,12 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + **kwargs, ) pooled_output = vision_outputs.pooler_output - image_features = self.visual_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + vision_outputs.pooler_output = self.visual_projection(pooled_output) - return image_features + return vision_outputs @auto_docstring def forward( diff --git a/src/transformers/models/clvp/modeling_clvp.py b/src/transformers/models/clvp/modeling_clvp.py index 05bab78266b3..6d9b124150d8 100644 --- a/src/transformers/models/clvp/modeling_clvp.py +++ b/src/transformers/models/clvp/modeling_clvp.py @@ -37,10 +37,13 @@ CausalLMOutputWithCrossAttentions, ) from ...modeling_utils import PreTrainedModel +from ...processing_utils import Unpack from ...pytorch_utils import Conv1D, isin_mps_friendly from ...utils import ( ModelOutput, + TransformersKwargs, auto_docstring, + can_return_tuple, logging, ) from .configuration_clvp import ( @@ -1474,13 +1477,14 @@ def fix_speech_decoder_output(self, speech_ids: torch.LongTensor) -> torch.LongT return speech_ids + @can_return_tuple def get_text_features( self, input_ids: Optional[torch.LongTensor] = None, text_encoder_inputs_embeds: Optional[torch.FloatTensor] = None, attention_mask: Optional[torch.LongTensor] = None, - return_dict: bool = False, - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + **kwargs: Unpack[TransformersKwargs], + ) -> Union[torch.FloatTensor, ClvpEncoderOutput]: r""" This method can be used to extract text_embeds from a text. The text embeddings obtained by applying the projection layer to the pooled output of the CLVP text encoder model. @@ -1500,8 +1504,6 @@ def get_text_features( - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `torch.FloatTensor` of shape `(batch_size, output_dim)`: @@ -1525,21 +1527,12 @@ def get_text_features( >>> text_embeds = model.get_text_features(input_ids=processor_output["input_ids"]) ``` """ - - text_outputs: ClvpEncoderOutput = self.text_encoder_model( + return self.text_encoder_model( input_ids=input_ids, inputs_embeds=text_encoder_inputs_embeds, attention_mask=attention_mask, + **kwargs, ) - text_features = text_outputs.embeds - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) - - return text_features def get_speech_features( self, diff --git a/src/transformers/models/flava/modeling_flava.py b/src/transformers/models/flava/modeling_flava.py index bcceeafa29f8..17288646cb97 100644 --- a/src/transformers/models/flava/modeling_flava.py +++ b/src/transformers/models/flava/modeling_flava.py @@ -28,7 +28,8 @@ from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel -from ...utils import ModelOutput, auto_docstring, filter_out_non_signature_kwargs, logging, torch_int +from ...processing_utils import Unpack +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, logging, torch_int from .configuration_flava import ( FlavaConfig, FlavaImageCodebookConfig, @@ -993,7 +994,7 @@ def __init__(self, config: FlavaConfig): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, @@ -1001,7 +1002,7 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, token_type_ids: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" input_ids (`torch.LongTensor` of shape `(batch_size, text_seq_length)`): @@ -1014,8 +1015,6 @@ def get_text_features( - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by @@ -1042,19 +1041,14 @@ def get_text_features( attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, + **kwargs, ) - pooled_output = text_outputs.last_hidden_state - text_features = self.text_projection(pooled_output) + last_hidden_state = text_outputs.last_hidden_state + text_outputs.pooler_output = self.text_projection(last_hidden_state) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) - - return text_features + return text_outputs - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_image_features( self, @@ -1062,13 +1056,11 @@ def get_image_features( bool_masked_pos: Optional[torch.BoolTensor] = None, interpolate_pos_encoding: Optional[bool] = None, attention_mask: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, image_num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by @@ -1098,17 +1090,12 @@ def get_image_features( bool_masked_pos=bool_masked_pos, attention_mask=attention_mask, interpolate_pos_encoding=interpolate_pos_encoding, + **kwargs, ) - pooled_output = image_outputs.last_hidden_state - image_features = self.image_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) + last_hidden_state = image_outputs.last_hidden_state + image_outputs.pooler_output = self.image_projection(last_hidden_state) - return image_features + return image_outputs @auto_docstring def forward( diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index b99c4b5c6aa4..ad3c35078784 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -290,11 +290,13 @@ def get_rope_index( return position_ids, mrope_position_deltas + @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -304,8 +306,6 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames @@ -314,17 +314,12 @@ def get_video_features( repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) temp_frames_hw.append(repeated_row) flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) - vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, return_dict=True) + vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, **kwargs) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + vision_outputs.pooler_output = video_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=video_embeds, - ) - - return video_embeds + return vision_outputs def get_image_features( self, diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index dc8fbc42c1ef..f4a5711881c7 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -768,8 +768,10 @@ def rot_pos_emb(self, grid_thw): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb, pos_ids + @check_model_inputs(tie_last_hidden_states=False) + @auto_docstring def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> torch.Tensor: """ Args: @@ -777,8 +779,6 @@ def forward( The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -807,6 +807,7 @@ def forward( hidden_states, cu_seqlens=cu_seqlens, position_embeddings=position_embeddings, + **kwargs, ) hidden_states = self.post_layernorm(hidden_states) @@ -819,13 +820,10 @@ def forward( merged_hidden_states = self.merger(hidden_states) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=hidden_states, - pooler_output=merged_hidden_states, - ) - - return merged_hidden_states + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) @auto_docstring @@ -1149,11 +1147,13 @@ def get_rope_index( return position_ids, mrope_position_deltas + @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1163,8 +1163,6 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames @@ -1173,17 +1171,12 @@ def get_video_features( repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) temp_frames_hw.append(repeated_row) flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) - vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, return_dict=True) + vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, **kwargs) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + vision_outputs.pooler_output = video_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=video_embeds, - ) - - return video_embeds + return vision_outputs def get_image_features( self, diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 3dd09a481b0d..e57cb0938326 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -788,8 +788,10 @@ def rot_pos_emb(self, grid_thw): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb, pos_ids + @check_model_inputs(tie_last_hidden_states=False) + @auto_docstring def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> torch.Tensor: """ Args: @@ -797,8 +799,6 @@ def forward( The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -827,6 +827,7 @@ def forward( hidden_states, cu_seqlens=cu_seqlens, position_embeddings=position_embeddings, + **kwargs, ) hidden_states = self.post_layernorm(hidden_states) @@ -839,13 +840,10 @@ def forward( merged_hidden_states = self.merger(hidden_states) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=hidden_states, - pooler_output=merged_hidden_states, - ) - - return merged_hidden_states + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) class Glm4vTextModel(Qwen2_5_VLTextModel): @@ -1143,11 +1141,13 @@ def get_rope_index( return position_ids, mrope_position_deltas + @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1157,8 +1157,6 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames @@ -1167,17 +1165,12 @@ def get_video_features( repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) temp_frames_hw.append(repeated_row) flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) - vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, return_dict=True) + vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, **kwargs) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + vision_outputs.pooler_output = video_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=video_embeds, - ) - - return video_embeds + return vision_outputs def get_placeholder_mask( self, diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 2fab4c75be7b..198f9731012c 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -976,8 +976,10 @@ def rot_pos_emb(self, grid_thw): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb, pos_ids + @check_model_inputs(tie_last_hidden_states=False) + @auto_docstring def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> torch.Tensor: """ Args: @@ -985,8 +987,6 @@ def forward( The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -1015,6 +1015,7 @@ def forward( hidden_states, cu_seqlens=cu_seqlens, position_embeddings=position_embeddings, + **kwargs, ) hidden_states = self.post_layernorm(hidden_states) @@ -1027,13 +1028,10 @@ def forward( merged_hidden_states = self.merger(hidden_states) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=hidden_states, - pooler_output=merged_hidden_states, - ) - - return merged_hidden_states + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) @auto_docstring @@ -1359,11 +1357,13 @@ def get_rope_index( return position_ids, mrope_position_deltas + @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1373,8 +1373,6 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames @@ -1383,17 +1381,12 @@ def get_video_features( repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) temp_frames_hw.append(repeated_row) flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) - vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, return_dict=True) + vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, **kwargs) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + vision_outputs.pooler_output = video_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=video_embeds, - ) - - return video_embeds + return vision_outputs def get_image_features( self, diff --git a/src/transformers/models/groupvit/modeling_groupvit.py b/src/transformers/models/groupvit/modeling_groupvit.py index b18069eb5ca6..b9d64c0a584b 100644 --- a/src/transformers/models/groupvit/modeling_groupvit.py +++ b/src/transformers/models/groupvit/modeling_groupvit.py @@ -28,7 +28,8 @@ from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel -from ...utils import ModelOutput, auto_docstring, filter_out_non_signature_kwargs, logging, torch_int +from ...processing_utils import Unpack +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_dict, logging, torch_int from .configuration_groupvit import GroupViTConfig, GroupViTTextConfig, GroupViTVisionConfig @@ -1223,19 +1224,16 @@ def __init__(self, config: GroupViTConfig): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @can_return_dict @auto_docstring def get_text_features( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`GroupViTTextModel`]. @@ -1259,20 +1257,16 @@ def get_text_features( position_ids=position_ids, ) pooled_output = text_outputs.pooler_output - text_features = self.text_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) + text_outputs.pooler_output = self.text_projection(pooled_output) - return text_features + return text_outputs - @filter_out_non_signature_kwargs() + @can_return_dict @auto_docstring def get_image_features( - self, pixel_values: torch.Tensor, return_dict: bool = False + self, + pixel_values: torch.Tensor, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" Returns: @@ -1297,16 +1291,10 @@ def get_image_features( >>> with torch.inference_mode(): ... image_features = model.get_image_features(**inputs) ```""" - vision_outputs: BaseModelOutputWithPooling = self.vision_model(pixel_values) - image_features = self.visual_projection(vision_outputs.pooler_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + vision_outputs: BaseModelOutputWithPooling = self.vision_model(pixel_values, **kwargs) + vision_outputs.pooler_output = self.visual_projection(vision_outputs.pooler_output) - return image_features + return vision_outputs @auto_docstring def forward( diff --git a/src/transformers/models/metaclip_2/modeling_metaclip_2.py b/src/transformers/models/metaclip_2/modeling_metaclip_2.py index 2f3e8bd59e1a..5d1c47581e79 100644 --- a/src/transformers/models/metaclip_2/modeling_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modeling_metaclip_2.py @@ -18,14 +18,7 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - ModelOutput, - TransformersKwargs, - auto_docstring, - can_return_tuple, - filter_out_non_signature_kwargs, - torch_int, -) +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, torch_int from ...utils.generic import check_model_inputs from .configuration_metaclip_2 import MetaClip2Config, MetaClip2TextConfig, MetaClip2VisionConfig @@ -796,14 +789,14 @@ def __init__(self, config: MetaClip2Config): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, - input_ids: Optional[torch.Tensor] = None, + input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" Returns: @@ -825,30 +818,22 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + **kwargs, ) pooled_output = text_outputs.pooler_output - text_features = self.text_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) + text_outputs.pooler_output = self.text_projection(pooled_output) - return text_features + return text_outputs - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`MetaClip2VisionModel`]. @@ -873,17 +858,12 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + **kwargs, ) pooled_output = vision_outputs.pooler_output - image_features = self.visual_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + vision_outputs.pooler_output = self.visual_projection(pooled_output) - return image_features + return vision_outputs @can_return_tuple @auto_docstring diff --git a/src/transformers/models/metaclip_2/modular_metaclip_2.py b/src/transformers/models/metaclip_2/modular_metaclip_2.py index c5665e42083e..3d3f8f5a135e 100644 --- a/src/transformers/models/metaclip_2/modular_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modular_metaclip_2.py @@ -555,12 +555,14 @@ def forward( **kwargs, ) + @can_return_tuple + @auto_docstring def get_text_features( self, - input_ids: Optional[torch.Tensor] = None, + input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): r""" Returns: @@ -582,19 +584,18 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, - return_dict=return_dict, + **kwargs, ) + @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`MetaClip2VisionModel`]. @@ -619,7 +620,7 @@ def get_image_features( return super().get_image_features( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, - return_dict=return_dict, + **kwargs, ) diff --git a/src/transformers/models/owlv2/modeling_owlv2.py b/src/transformers/models/owlv2/modeling_owlv2.py index ab4b1a412a52..b942b5e7ccee 100644 --- a/src/transformers/models/owlv2/modeling_owlv2.py +++ b/src/transformers/models/owlv2/modeling_owlv2.py @@ -27,10 +27,12 @@ from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel +from ...processing_utils import Unpack from ...utils import ( ModelOutput, + TransformersKwargs, auto_docstring, - filter_out_non_signature_kwargs, + can_return_tuple, is_vision_available, logging, torch_int, @@ -970,21 +972,19 @@ def __init__(self, config: Owlv2Config): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by @@ -1004,30 +1004,25 @@ def get_text_features( ... text_features = model.get_text_features(**inputs) ```""" # Get embeddings for all text queries in all batch samples - text_outputs: BaseModelOutputWithPooling = self.text_model(input_ids=input_ids, attention_mask=attention_mask) + text_outputs: BaseModelOutputWithPooling = self.text_model( + input_ids=input_ids, + attention_mask=attention_mask, + **kwargs, + ) pooled_output = text_outputs.pooler_output - text_features = self.text_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) + text_outputs.pooler_output = self.text_projection(pooled_output) - return text_features + return text_outputs - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_image_features( self, pixel_values: torch.Tensor, interpolate_pos_encoding: bool = False, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`Owlv2VisionModel`]. @@ -1051,16 +1046,11 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + **kwargs, ) - image_features = self.visual_projection(vision_outputs.pooler_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + vision_outputs.pooler_output = self.visual_projection(vision_outputs.pooler_output) - return image_features + return vision_outputs @auto_docstring def forward( diff --git a/src/transformers/models/owlvit/modeling_owlvit.py b/src/transformers/models/owlvit/modeling_owlvit.py index 00dda51da6e6..6d12ba581f1b 100644 --- a/src/transformers/models/owlvit/modeling_owlvit.py +++ b/src/transformers/models/owlvit/modeling_owlvit.py @@ -27,10 +27,12 @@ from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel +from ...processing_utils import Unpack from ...utils import ( ModelOutput, + TransformersKwargs, auto_docstring, - filter_out_non_signature_kwargs, + can_return_tuple, is_vision_available, logging, torch_int, @@ -951,21 +953,19 @@ def __init__(self, config: OwlViTConfig): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by @@ -985,30 +985,25 @@ def get_text_features( ... text_features = model.get_text_features(**inputs) ```""" # Get embeddings for all text queries in all batch samples - text_outputs: BaseModelOutputWithPooling = self.text_model(input_ids=input_ids, attention_mask=attention_mask) + text_outputs: BaseModelOutputWithPooling = self.text_model( + input_ids=input_ids, + attention_mask=attention_mask, + **kwargs, + ) pooled_output = text_outputs.pooler_output - text_features = self.text_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) + text_outputs.pooler_output = self.text_projection(pooled_output) - return text_features + return text_outputs - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_image_features( self, pixel_values: torch.Tensor, interpolate_pos_encoding: bool = False, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`OwlViTVisionModel`]. @@ -1032,16 +1027,11 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + **kwargs, ) - image_features = self.visual_projection(vision_outputs.pooler_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + vision_outputs.pooler_output = self.visual_projection(vision_outputs.pooler_output) - return image_features + return vision_outputs @auto_docstring def forward( diff --git a/src/transformers/models/siglip/modeling_siglip.py b/src/transformers/models/siglip/modeling_siglip.py index 2acae505d572..e239fbed6f29 100644 --- a/src/transformers/models/siglip/modeling_siglip.py +++ b/src/transformers/models/siglip/modeling_siglip.py @@ -36,7 +36,6 @@ TransformersKwargs, auto_docstring, can_return_tuple, - filter_out_non_signature_kwargs, torch_int, ) from ...utils.generic import check_model_inputs @@ -774,19 +773,16 @@ def __init__(self, config: SiglipConfig): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`SiglipTextModel`]. @@ -805,31 +801,22 @@ def get_text_features( >>> with torch.no_grad(): ... text_features = model.get_text_features(**inputs) ```""" - text_outputs: BaseModelOutputWithPooling = self.text_model( + return self.text_model( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + **kwargs, ) - pooled_output = text_outputs.pooler_output - - if return_dict: - return text_outputs - return pooled_output - - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, - return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`SiglipVisionModel`]. @@ -852,17 +839,11 @@ def get_image_features( >>> with torch.no_grad(): ... image_features = model.get_image_features(**inputs) ```""" - vision_outputs: BaseModelOutputWithPooling = self.vision_model( + return self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, **kwargs, ) - pooled_output = vision_outputs.pooler_output - - if return_dict: - return vision_outputs - - return pooled_output # NOTE: SiglipModel uses Pretrained backbones, so we don't need to add `check_model_inputs` here @can_return_tuple diff --git a/src/transformers/models/siglip2/modeling_siglip2.py b/src/transformers/models/siglip2/modeling_siglip2.py index 12b207421bd2..e4a779232e21 100644 --- a/src/transformers/models/siglip2/modeling_siglip2.py +++ b/src/transformers/models/siglip2/modeling_siglip2.py @@ -36,7 +36,7 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, filter_out_non_signature_kwargs +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple from ...utils.generic import check_model_inputs from .configuration_siglip2 import Siglip2Config, Siglip2TextConfig, Siglip2VisionConfig @@ -752,16 +752,14 @@ def __init__(self, config: Siglip2VisionConfig): def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding - @check_model_inputs(tie_last_hidden_states=False) + @can_return_tuple @auto_docstring def forward( self, pixel_values: torch.FloatTensor, pixel_attention_mask: torch.Tensor, spatial_shapes: torch.LongTensor, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPooling: r""" pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): @@ -792,8 +790,7 @@ def forward( pixel_values=pixel_values, attention_mask=pixel_attention_mask, spatial_shapes=spatial_shapes, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, + **kwargs, ) @@ -833,19 +830,16 @@ def __init__(self, config: Siglip2Config): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`Siglip2TextModel`]. @@ -864,34 +858,27 @@ def get_text_features( >>> with torch.no_grad(): ... text_features = model.get_text_features(**inputs) ```""" - text_outputs: BaseModelOutputWithPooling = self.text_model( + return self.text_model( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + **kwargs, ) - pooled_output = text_outputs.pooler_output - if return_dict: - return text_outputs - - return pooled_output - - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, pixel_attention_mask: Optional[torch.Tensor] = None, spatial_shapes: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: r""" pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): Mask to avoid performing attention on padding pixel indices. spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): Tensor containing the spatial dimensions (height, width) of the input images. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by @@ -916,17 +903,12 @@ def get_image_features( ... image_features = model.get_image_features(**inputs) ``` """ - vision_outputs: BaseModelOutputWithPooling = self.vision_model( + return self.vision_model( pixel_values=pixel_values, attention_mask=pixel_attention_mask, spatial_shapes=spatial_shapes, + **kwargs, ) - pooled_output = vision_outputs.pooler_output - - if return_dict: - return vision_outputs - - return pooled_output # NOTE: Siglip2Model uses Pretrained backbones, so we don't need to add `check_model_inputs` here @can_return_tuple diff --git a/src/transformers/models/siglip2/modular_siglip2.py b/src/transformers/models/siglip2/modular_siglip2.py index c64383c92f1c..8e3c9ddb6501 100644 --- a/src/transformers/models/siglip2/modular_siglip2.py +++ b/src/transformers/models/siglip2/modular_siglip2.py @@ -36,8 +36,8 @@ ) from ...modeling_attn_mask_utils import _prepare_4d_attention_mask -from ...utils import auto_docstring, filter_out_non_signature_kwargs -from ...utils.generic import check_model_inputs +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple class Siglip2TextConfig(SiglipTextConfig): @@ -316,16 +316,14 @@ def forward(self, hidden_state: torch.Tensor, attention_mask: Optional[torch.Ten class Siglip2VisionModel(SiglipVisionModel): # Update: add `spatial_shapes` and `pixel_attention_mask` - @check_model_inputs(tie_last_hidden_states=False) + @can_return_tuple @auto_docstring def forward( self, pixel_values: torch.FloatTensor, pixel_attention_mask: torch.Tensor, spatial_shapes: torch.LongTensor, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPooling: r""" pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): @@ -356,29 +354,26 @@ def forward( pixel_values=pixel_values, attention_mask=pixel_attention_mask, spatial_shapes=spatial_shapes, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, + **kwargs, ) class Siglip2Model(SiglipModel): # Update: add `spatial_shapes` and `pixel_attention_mask` - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, pixel_attention_mask: Optional[torch.Tensor] = None, spatial_shapes: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: r""" pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): Mask to avoid performing attention on padding pixel indices. spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): Tensor containing the spatial dimensions (height, width) of the input images. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by @@ -403,17 +398,12 @@ def get_image_features( ... image_features = model.get_image_features(**inputs) ``` """ - vision_outputs: BaseModelOutputWithPooling = self.vision_model( + return self.vision_model( pixel_values=pixel_values, attention_mask=pixel_attention_mask, spatial_shapes=spatial_shapes, + **kwargs, ) - pooled_output = vision_outputs.pooler_output - - if return_dict: - return vision_outputs - - return pooled_output # Update: add `spatial_shapes` and `pixel_attention_mask` def forward( diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index cec4e42957f0..a9105ac20bc7 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -21,7 +21,8 @@ from ...modeling_outputs import BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel -from ...utils import auto_docstring, filter_out_non_signature_kwargs, logging +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from ..auto.configuration_auto import AutoConfig from ..auto.modeling_auto import AutoModel from ..clip.modeling_clip import CLIPOutput, CLIPVisionConfig, CLIPVisionModel @@ -104,7 +105,7 @@ def __init__( self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, @@ -112,12 +113,9 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, token_type_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`CLIPTextModel`]. @@ -140,25 +138,19 @@ def get_text_features( attention_mask=attention_mask, position_ids=position_ids, token_type_ids=token_type_ids, + **kwargs, ) pooled_output = text_outputs.pooler_output - text_features = self.text_projection(pooled_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) + text_outputs.pooler_output = self.text_projection(pooled_output) - return text_features + return text_outputs - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring - def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False) -> torch.FloatTensor: + def get_image_features( + self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> torch.FloatTensor: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`CLIPVisionModel`]. @@ -181,16 +173,10 @@ def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = Fal >>> with torch.inference_mode(): ... image_features = model.get_image_features(**inputs) ```""" - vision_outputs = self.vision_model(pixel_values=pixel_values) - image_features = self.visual_projection(vision_outputs.pooler_output) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + vision_outputs = self.vision_model(pixel_values=pixel_values, **kwargs) + vision_outputs.pooler_output = self.visual_projection(vision_outputs.pooler_output) - return image_features + return vision_outputs @auto_docstring def forward( diff --git a/src/transformers/models/x_clip/modeling_x_clip.py b/src/transformers/models/x_clip/modeling_x_clip.py index 80734a0fb81e..e3763aafab9f 100644 --- a/src/transformers/models/x_clip/modeling_x_clip.py +++ b/src/transformers/models/x_clip/modeling_x_clip.py @@ -28,11 +28,12 @@ from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from ...processing_utils import Unpack from ...utils import ( ModelOutput, + TransformersKwargs, auto_docstring, can_return_tuple, - filter_out_non_signature_kwargs, logging, torch_int, ) @@ -1200,19 +1201,16 @@ def __init__(self, config: XCLIPConfig): # Initialize weights and apply final processing self.post_init() - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`XCLIPTextModel`]. @@ -1234,29 +1232,21 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + **kwargs, ) pooled_output = text_outputs.pooler_output - text_features = self.text_projection(pooled_output) + text_outputs.pooler_output = self.text_projection(pooled_output) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=text_outputs.last_hidden_state, - pooler_output=text_features, - ) + return text_outputs - return text_features - - @filter_out_non_signature_kwargs() + @can_return_tuple @auto_docstring def get_video_features( self, pixel_values: torch.Tensor, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: video_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The video embeddings obtained by applying the projection layer to the pooled output of [`XCLIPVisionModel`] and @@ -1334,20 +1324,14 @@ def get_video_features( batch_size, num_frames, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.reshape(-1, num_channels, height, width) - vision_outputs: BaseModelOutputWithPooling = self.vision_model(pixel_values=pixel_values) - + vision_outputs: BaseModelOutputWithPooling = self.vision_model(pixel_values=pixel_values, **kwargs) video_embeds = vision_outputs.pooler_output video_embeds = self.visual_projection(video_embeds) cls_features = video_embeds.view(batch_size, num_frames, -1) + mit_outputs: BaseModelOutputWithPooling = self.mit(cls_features, **kwargs) - mit_outputs: BaseModelOutputWithPooling = self.mit(cls_features) - video_embeds = mit_outputs.pooler_output - - if return_dict: - return mit_outputs - - return video_embeds + return mit_outputs @auto_docstring def forward( diff --git a/src/transformers/utils/generic.py b/src/transformers/utils/generic.py index fba43fadf7b5..5b7a10ad7186 100644 --- a/src/transformers/utils/generic.py +++ b/src/transformers/utils/generic.py @@ -814,7 +814,7 @@ class OutputRecorder: class_name: str | None = None -def check_model_inputs(func=None, *, tie_last_hidden_states=True, default_return_pooled=False): +def check_model_inputs(func=None, *, tie_last_hidden_states=True): """ Decorator to intercept specific layer outputs without using hooks. Compatible with torch.compile (Dynamo tracing). @@ -825,10 +825,6 @@ def check_model_inputs(func=None, *, tie_last_hidden_states=True, default_return This is true for all language models and should be toggled off only if `out.hidden_states[-1]` has to be the hidden state before last layer norm, which is needed for some vision models (e.g. CLIP, SigLIP) - default_return_pooled (`bool`, *optional*, defaults to `False`): - Whether the default behavior is to return pooled outputs. If `True`, and `return_dict` - is not specified in the function call, then `return_dict` will be set to `False` instead - of reading from `self.config.return_dict`. """ def wrapped_fn(func): @@ -863,10 +859,7 @@ def wrapper(self, *args, **kwargs): else: kwargs["use_cache"] = use_cache - # User-provided kwargs will always take precedence, but if the wrapped method should return pooled outputs - # by default, then we set return_dict to False instead of reading from the model config. - default_return_dict = False if default_return_pooled else None - return_dict = kwargs.pop("return_dict", default_return_dict) + return_dict = kwargs.pop("return_dict", None) if return_dict is None: return_dict = getattr(self.config, "return_dict", True) @@ -990,10 +983,7 @@ def wrapped_forward(*args, **kwargs): else: outputs[key] = collected_outputs[key] if return_dict is False: - if default_return_pooled: - outputs = outputs.pooler_output - else: - outputs = outputs.to_tuple() + outputs = outputs.to_tuple() return outputs return wrapper From 2a643038817ba741b1ddaa068ea5867e36af6c7e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 16 Dec 2025 13:29:35 +0100 Subject: [PATCH 017/131] Fix typo: can_return_dict -> can_return_tuple --- src/transformers/models/groupvit/modeling_groupvit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/groupvit/modeling_groupvit.py b/src/transformers/models/groupvit/modeling_groupvit.py index b9d64c0a584b..2ad575e2c212 100644 --- a/src/transformers/models/groupvit/modeling_groupvit.py +++ b/src/transformers/models/groupvit/modeling_groupvit.py @@ -29,7 +29,7 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_dict, logging, torch_int +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, logging, torch_int from .configuration_groupvit import GroupViTConfig, GroupViTTextConfig, GroupViTVisionConfig @@ -1224,7 +1224,7 @@ def __init__(self, config: GroupViTConfig): # Initialize weights and apply final processing self.post_init() - @can_return_dict + @can_return_tuple @auto_docstring def get_text_features( self, @@ -1261,7 +1261,7 @@ def get_text_features( return text_outputs - @can_return_dict + @can_return_tuple @auto_docstring def get_image_features( self, From fc8ee939e7b932dc9646ec2e06d9684a53d5b686 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 16 Dec 2025 15:34:56 +0100 Subject: [PATCH 018/131] Adopt can_return_tuple for many get_image_features A handful of outliers that aren't updated yet, e.g. if there's 2+ ModelOutput classes that are viable, or the vq-based ones For context, the other modeling file classes haven't been updated with the new get_..._features format, nor have the tests --- src/transformers/models/aria/modeling_aria.py | 35 +++------- src/transformers/models/aria/modular_aria.py | 35 +++------- .../models/aya_vision/modeling_aya_vision.py | 20 ++---- .../models/aya_vision/modular_aya_vision.py | 20 ++---- .../models/chameleon/modeling_chameleon.py | 14 ++-- .../cohere2_vision/modeling_cohere2_vision.py | 26 +++---- .../cohere2_vision/modular_cohere2_vision.py | 25 +++---- .../deepseek_vl/modeling_deepseek_vl.py | 18 ++--- .../modeling_deepseek_vl_hybrid.py | 4 ++ .../modular_deepseek_vl_hybrid.py | 2 + .../models/edgetam/modeling_edgetam.py | 57 +++------------- .../edgetam_video/modeling_edgetam_video.py | 57 +++------------- src/transformers/models/emu3/modeling_emu3.py | 2 + src/transformers/models/emu3/modular_emu3.py | 2 + .../models/fast_vlm/modeling_fast_vlm.py | 9 +-- .../models/fast_vlm/modular_fast_vlm.py | 10 +-- .../models/florence2/modeling_florence2.py | 28 ++++---- .../models/florence2/modular_florence2.py | 28 ++++---- src/transformers/models/fuyu/modeling_fuyu.py | 2 +- .../models/gemma3/modeling_gemma3.py | 21 ++---- .../models/gemma3/modular_gemma3.py | 17 ++--- .../models/gemma3n/modeling_gemma3n.py | 27 +++----- .../models/gemma3n/modular_gemma3n.py | 23 +++---- .../models/glm46v/modeling_glm46v.py | 36 +++++----- .../models/glm4v/modeling_glm4v.py | 34 ++++++---- .../models/glm4v_moe/modeling_glm4v_moe.py | 34 ++++++---- .../models/got_ocr2/modeling_got_ocr2.py | 19 ++---- .../models/got_ocr2/modular_got_ocr2.py | 17 ++--- .../models/idefics2/modeling_idefics2.py | 25 +++---- .../models/idefics3/modeling_idefics3.py | 18 +++-- .../instructblip/modeling_instructblip.py | 4 +- .../modeling_instructblipvideo.py | 2 +- .../modular_instructblipvideo.py | 2 +- .../models/internvl/modeling_internvl.py | 19 ++---- .../models/internvl/modular_internvl.py | 17 ++--- .../models/janus/modeling_janus.py | 16 ++--- .../models/janus/modular_janus.py | 18 ++--- .../models/kosmos2/modeling_kosmos2.py | 35 +++------- .../models/lfm2_vl/modeling_lfm2_vl.py | 20 ++---- .../models/lfm2_vl/modular_lfm2_vl.py | 19 ++---- .../models/llama4/modeling_llama4.py | 9 +-- .../models/llava/modeling_llava.py | 18 ++--- .../models/llava_next/modeling_llava_next.py | 20 +++--- .../modeling_llava_next_video.py | 18 ++--- .../modular_llava_next_video.py | 16 ++--- .../modeling_llava_onevision.py | 18 ++--- .../modular_llava_onevision.py | 14 ++-- .../models/mistral3/modeling_mistral3.py | 20 ++---- .../models/mistral3/modular_mistral3.py | 18 ++--- .../models/ovis2/modeling_ovis2.py | 34 +++++----- .../models/ovis2/modular_ovis2.py | 34 +++++----- .../paddleocr_vl/modeling_paddleocr_vl.py | 46 ++++++++----- .../paddleocr_vl/modular_paddleocr_vl.py | 13 +++- .../models/paligemma/modeling_paligemma.py | 22 +++--- .../perception_lm/modeling_perception_lm.py | 22 +++--- .../perception_lm/modular_perception_lm.py | 20 ++---- .../qwen2_5_omni/modeling_qwen2_5_omni.py | 30 +++----- .../qwen2_5_omni/modular_qwen2_5_omni.py | 29 +++----- .../models/qwen2_5_vl/modeling_qwen2_5_vl.py | 49 +++++++------ .../models/qwen2_vl/modeling_qwen2_vl.py | 49 +++++++------ .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 37 +++++----- .../models/qwen3_vl/modeling_qwen3_vl.py | 68 ++++++++++--------- .../models/qwen3_vl/modular_qwen3_vl.py | 48 ++++++------- .../qwen3_vl_moe/modeling_qwen3_vl_moe.py | 68 ++++++++++--------- src/transformers/models/sam2/modeling_sam2.py | 25 +++---- src/transformers/models/sam2/modular_sam2.py | 24 +++---- .../models/sam2_video/modeling_sam2_video.py | 57 +++------------- .../sam3_tracker/modeling_sam3_tracker.py | 57 +++------------- .../modeling_sam3_tracker_video.py | 27 +++----- .../modular_sam3_tracker_video.py | 25 +++---- .../models/smolvlm/modeling_smolvlm.py | 21 +++--- .../models/smolvlm/modular_smolvlm.py | 24 +++---- .../models/t5gemma2/modeling_t5gemma2.py | 25 +++---- .../models/t5gemma2/modular_t5gemma2.py | 25 +++---- .../video_llama_3/modeling_video_llama_3.py | 43 ++++++------ .../video_llama_3/modular_video_llama_3.py | 25 +++---- .../video_llava/modeling_video_llava.py | 43 +++++------- .../models/vipllava/modeling_vipllava.py | 25 +++---- .../models/vipllava/modular_vipllava.py | 24 +++---- 79 files changed, 774 insertions(+), 1233 deletions(-) diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index 34ae17434e71..349147d6ac6e 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -918,28 +918,18 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, pixel_mask: Optional[torch.FloatTensor] = None, vision_feature_layer: int = -1, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" Obtains image last hidden states from the vision tower and apply multimodal projection. - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - pixel_mask (`torch.FloatTensor]`, *optional*): - The tensors corresponding to the input image mask. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -948,7 +938,10 @@ def get_image_features( ) patch_attention_mask = self._create_patch_attention_mask(pixel_mask) image_outputs = self.vision_tower( - pixel_values, patch_attention_mask=patch_attention_mask, output_hidden_states=True + pixel_values, + patch_attention_mask=patch_attention_mask, + output_hidden_states=True, + **kwargs, ) image_attn_mask = None if patch_attention_mask is not None: @@ -956,15 +949,9 @@ def get_image_features( image_attn_mask = torch.logical_not(flattened_mask) selected_image_feature = image_outputs.hidden_states[vision_feature_layer] - image_features = self.multi_modal_projector(selected_image_feature, attn_mask=image_attn_mask) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) + image_outputs.pooler_output = self.multi_modal_projector(selected_image_feature, attn_mask=image_attn_mask) - return image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -1093,13 +1080,13 @@ def get_image_features( pixel_values: torch.FloatTensor, pixel_mask: Optional[torch.FloatTensor] = None, vision_feature_layer: int = -1, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, pixel_mask=pixel_mask, vision_feature_layer=vision_feature_layer, - return_dict=return_dict, + **kwargs, ) @can_return_tuple diff --git a/src/transformers/models/aria/modular_aria.py b/src/transformers/models/aria/modular_aria.py index 06fa5c0db308..156c67f14493 100644 --- a/src/transformers/models/aria/modular_aria.py +++ b/src/transformers/models/aria/modular_aria.py @@ -1274,28 +1274,18 @@ def _create_patch_attention_mask(self, pixel_mask): ) return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool() + @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, pixel_mask: Optional[torch.FloatTensor] = None, vision_feature_layer: int = -1, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: r""" Obtains image last hidden states from the vision tower and apply multimodal projection. - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - pixel_mask (`torch.FloatTensor]`, *optional*): - The tensors corresponding to the input image mask. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ @@ -1304,7 +1294,10 @@ def get_image_features( ) patch_attention_mask = self._create_patch_attention_mask(pixel_mask) image_outputs = self.vision_tower( - pixel_values, patch_attention_mask=patch_attention_mask, output_hidden_states=True + pixel_values, + patch_attention_mask=patch_attention_mask, + output_hidden_states=True, + **kwargs, ) image_attn_mask = None if patch_attention_mask is not None: @@ -1312,15 +1305,9 @@ def get_image_features( image_attn_mask = torch.logical_not(flattened_mask) selected_image_feature = image_outputs.hidden_states[vision_feature_layer] - image_features = self.multi_modal_projector(selected_image_feature, attn_mask=image_attn_mask) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) + image_outputs.pooler_output = self.multi_modal_projector(selected_image_feature, attn_mask=image_attn_mask) - return image_features + return image_outputs def forward( self, @@ -1386,13 +1373,13 @@ def get_image_features( pixel_values: torch.FloatTensor, pixel_mask: Optional[torch.FloatTensor] = None, vision_feature_layer: int = -1, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, pixel_mask=pixel_mask, vision_feature_layer=vision_feature_layer, - return_dict=return_dict, + **kwargs, ) @can_return_tuple diff --git a/src/transformers/models/aya_vision/modeling_aya_vision.py b/src/transformers/models/aya_vision/modeling_aya_vision.py index caa5577b39bc..8667963383a1 100644 --- a/src/transformers/models/aya_vision/modeling_aya_vision.py +++ b/src/transformers/models/aya_vision/modeling_aya_vision.py @@ -28,7 +28,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -181,13 +181,13 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -202,8 +202,6 @@ def get_image_features( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). @@ -237,15 +235,9 @@ def get_image_features( hs_pool = [hs[:, 1:] for hs in hs_pool] selected_image_feature = torch.cat(hs_pool, dim=-1) - image_features = self.multi_modal_projector(selected_image_feature) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) + image_outputs.pooler_output = self.multi_modal_projector(selected_image_feature) - return image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -367,7 +359,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, diff --git a/src/transformers/models/aya_vision/modular_aya_vision.py b/src/transformers/models/aya_vision/modular_aya_vision.py index 1d1577bd0ac2..8905beaeec6d 100644 --- a/src/transformers/models/aya_vision/modular_aya_vision.py +++ b/src/transformers/models/aya_vision/modular_aya_vision.py @@ -25,14 +25,12 @@ LlavaModel, LlavaModelOutputWithPast, LlavaPreTrainedModel, - TransformersKwargs, ) from ...activations import ACT2FN from ...cache_utils import Cache -from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack -from ...utils import auto_docstring, logging +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from ...utils.generic import check_model_inputs from .configuration_aya_vision import AyaVisionConfig @@ -108,13 +106,13 @@ class AyaVisionModelOutputWithPast(LlavaModelOutputWithPast): class AyaVisionModel(LlavaModel): # Unlike LLaVA, the model doesn't have to deal with Pixtral-style image states + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -129,8 +127,6 @@ def get_image_features( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). @@ -164,15 +160,9 @@ def get_image_features( hs_pool = [hs[:, 1:] for hs in hs_pool] selected_image_feature = torch.cat(hs_pool, dim=-1) - image_features = self.multi_modal_projector(selected_image_feature) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) + image_outputs.pooler_output = self.multi_modal_projector(selected_image_feature) - return image_features + return image_outputs @check_model_inputs @auto_docstring diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 5a73017c4f12..b79f8a2971d1 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -28,7 +28,7 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -854,7 +854,7 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor): bpe_toks = bpe_toks.view(batch_size, -1) return bpe_toks - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): + def get_image_features(self, pixel_values: torch.FloatTensor): """ Tokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layer @@ -862,15 +862,9 @@ def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): The tensors corresponding to the input images. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ image_tokens = self.get_image_tokens(pixel_values) image_embeddings = self.get_input_embeddings()(image_tokens) - - if return_dict: - return BaseModelOutputWithPooling(last_hidden_state=image_embeddings) - return image_embeddings def get_placeholder_mask( @@ -1029,8 +1023,8 @@ def __init__(self, config): def get_image_tokens(self, pixel_values): return self.model.get_image_tokens(pixel_values) - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): - return self.model.get_image_features(pixel_values, return_dict=return_dict) + def get_image_features(self, pixel_values: torch.FloatTensor): + return self.model.get_image_features(pixel_values) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py index c7447634f83f..df1d3950d7f3 100644 --- a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py @@ -27,10 +27,10 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_cohere2_vision import Cohere2VisionConfig @@ -167,7 +167,8 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): + @can_return_tuple + def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -178,18 +179,11 @@ def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool image_features (List[`torch.Tensor`]): List of image feature tensor, each contains all the visual feature of all patches and are of shape `(num_patches, image_length, embed_dim)`). """ + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) + selected_image_feature = image_outputs.last_hidden_state + image_outputs.pooler_output = self.multi_modal_projector(selected_image_feature) - image_features = self.vision_tower(pixel_values, output_hidden_states=True) - selected_image_feature = image_features.last_hidden_state - image_features = self.multi_modal_projector(selected_image_feature) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=selected_image_feature, - pooler_output=image_features, - ) - - return image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -286,8 +280,8 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): - return self.model.get_image_features(pixel_values=pixel_values, return_dict=return_dict) + def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + return self.model.get_image_features(pixel_values=pixel_values, **kwargs) @check_model_inputs @auto_docstring diff --git a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py index 2c77aa30d279..c5b23f95c4a6 100644 --- a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py @@ -34,9 +34,8 @@ from ...image_processing_utils import BatchFeature from ...image_utils import ImageInput from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import ImagesKwargs, Unpack -from ...utils import TransformersKwargs, auto_docstring, logging +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from ...utils.generic import check_model_inputs from .configuration_cohere2_vision import Cohere2VisionConfig @@ -98,7 +97,8 @@ class Cohere2VisionPreTrainedModel(AyaVisionPreTrainedModel): class Cohere2VisionModel(AyaVisionModel): _checkpoint_conversion_mapping = {} - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): + @can_return_tuple + def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -109,18 +109,11 @@ def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool image_features (List[`torch.Tensor`]): List of image feature tensor, each contains all the visual feature of all patches and are of shape `(num_patches, image_length, embed_dim)`). """ + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) + selected_image_feature = image_outputs.last_hidden_state + image_outputs.pooler_output = self.multi_modal_projector(selected_image_feature) - image_features = self.vision_tower(pixel_values, output_hidden_states=True) - selected_image_feature = image_features.last_hidden_state - image_features = self.multi_modal_projector(selected_image_feature) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=selected_image_feature, - pooler_output=image_features, - ) - - return image_features + return image_outputs @check_model_inputs @auto_docstring @@ -172,8 +165,8 @@ def forward( class Cohere2VisionForConditionalGeneration(AyaVisionForConditionalGeneration): _checkpoint_conversion_mapping = {} - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): - return self.model.get_image_features(pixel_values=pixel_values, return_dict=return_dict) + def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + return self.model.get_image_features(pixel_values=pixel_values, **kwargs) @check_model_inputs @auto_docstring diff --git a/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py b/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py index 0d5735bcf2a8..f54fb668b7f2 100644 --- a/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py +++ b/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py @@ -26,7 +26,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -153,17 +153,13 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): - vision_outputs = self.vision_model(pixel_values) - image_features = self.aligner(vision_outputs.last_hidden_state) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + @can_return_tuple + @auto_docstring + def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + vision_outputs = self.vision_model(pixel_values, **kwargs) + vision_outputs.pooler_output = self.aligner(vision_outputs.last_hidden_state) - return image_features + return vision_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor diff --git a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py index 9e12fd2a9b41..2421b8409da9 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py @@ -269,6 +269,8 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, high_res_pixel_values: torch.FloatTensor, return_dict: bool = False ): @@ -276,6 +278,8 @@ def get_image_features( high_res_vision_encodings = self.get_high_res_image_features(high_res_pixel_values) image_features = self.aligner(vision_encodings, high_res_vision_encodings) + # TODO: @Tom to revisit with the new can_return_tuple approach + # Presumably something like a fresh ModelOutput subclass that holds both if return_dict: return BaseModelOutputWithPooling( last_hidden_state=vision_encodings, diff --git a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py index 834786e42dfa..0b8521d52120 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py @@ -284,6 +284,8 @@ def get_image_features( high_res_vision_encodings = self.get_high_res_image_features(high_res_pixel_values) image_features = self.aligner(vision_encodings, high_res_vision_encodings) + # TODO: @Tom to revisit with the new can_return_tuple approach + # Presumably something like a fresh ModelOutput subclass that holds both if return_dict: return BaseModelOutputWithPooling( last_hidden_state=vision_encodings, diff --git a/src/transformers/models/edgetam/modeling_edgetam.py b/src/transformers/models/edgetam/modeling_edgetam.py index 71a00b0f2e4f..8b4464c9545b 100644 --- a/src/transformers/models/edgetam/modeling_edgetam.py +++ b/src/transformers/models/edgetam/modeling_edgetam.py @@ -38,7 +38,7 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import compile_compatible_method_lru_cache -from ...utils import ModelOutput, auto_docstring +from ...utils import ModelOutput, auto_docstring, can_return_tuple from ..auto import AutoModel from .configuration_edgetam import ( EdgeTamConfig, @@ -463,38 +463,6 @@ def forward( ) -@dataclass -class BaseModelOutputWithFeatureMaps(ModelOutput): - """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, - sequence_length)`. - - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention - heads. - feature_maps (`list[torch.Tensor]`): - List of feature maps from different layers of the model. - feature_maps_position_embeddings (`list[torch.Tensor]`): - List of position embeddings corresponding to the feature maps. - """ - - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None - attentions: Optional[tuple[torch.FloatTensor, ...]] = None - feature_maps: Optional[list[torch.Tensor]] = None - feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None - - @dataclass @auto_docstring(custom_intro="Base class for the EdgeTam model's output.") class EdgeTamImageSegmentationOutput(ModelOutput): @@ -1219,10 +1187,10 @@ def forward( vision_attentions=vision_attentions, ) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -1236,8 +1204,6 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -1246,10 +1212,7 @@ def get_image_features( - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. """ - vision_outputs: EdgeTamVisionEncoderOutput = self.vision_encoder( - pixel_values, - **kwargs, - ) + vision_outputs: EdgeTamVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding @@ -1266,16 +1229,12 @@ def get_image_features( feature_map_position_embedding.flatten(2).permute(2, 0, 1) for feature_map_position_embedding in feature_maps_position_embeddings ] + vision_outputs.fpn_hidden_states = feature_maps + vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - if return_dict: - return BaseModelOutputWithFeatureMaps( - last_hidden_state=vision_outputs.last_hidden_state, - attentions=vision_outputs.attentions, - feature_maps=feature_maps, - feature_maps_position_embeddings=feature_maps_position_embeddings, - ) - - return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions + # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the + # fpn hidden states/position encoding order, still have to double-check + return vision_outputs __all__ = ["EdgeTamModel", "EdgeTamVisionModel", "EdgeTamPreTrainedModel"] diff --git a/src/transformers/models/edgetam_video/modeling_edgetam_video.py b/src/transformers/models/edgetam_video/modeling_edgetam_video.py index 15dd498f4fc9..122faa22b165 100644 --- a/src/transformers/models/edgetam_video/modeling_edgetam_video.py +++ b/src/transformers/models/edgetam_video/modeling_edgetam_video.py @@ -42,7 +42,7 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import compile_compatible_method_lru_cache -from ...utils import ModelOutput, auto_docstring +from ...utils import ModelOutput, auto_docstring, can_return_tuple from ...utils.generic import TransformersKwargs from ..auto import AutoModel from .configuration_edgetam_video import ( @@ -1547,38 +1547,6 @@ class EdgeTamVideoSegmentationOutput(ModelOutput): frame_idx: Optional[int] = None -@dataclass -class BaseModelOutputWithFeatureMaps(ModelOutput): - """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, - sequence_length)`. - - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention - heads. - feature_maps (`list[torch.Tensor]`): - List of feature maps from different layers of the model. - feature_maps_position_embeddings (`list[torch.Tensor]`): - List of position embeddings corresponding to the feature maps. - """ - - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None - attentions: Optional[tuple[torch.FloatTensor, ...]] = None - feature_maps: Optional[list[torch.Tensor]] = None - feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None - - class EdgeTamVideoPositionalEmbedding(nn.Module): def __init__(self, config: EdgeTamVideoPromptEncoderConfig): super().__init__() @@ -2240,10 +2208,10 @@ def forward( frame_idx=frame_idx, ) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -2257,8 +2225,6 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -2267,10 +2233,7 @@ def get_image_features( - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. """ - vision_outputs: EdgeTamVideoVisionEncoderOutput = self.vision_encoder( - pixel_values, - **kwargs, - ) + vision_outputs: EdgeTamVideoVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding @@ -2287,16 +2250,12 @@ def get_image_features( feature_map_position_embedding.flatten(2).permute(2, 0, 1) for feature_map_position_embedding in feature_maps_position_embeddings ] + vision_outputs.fpn_hidden_states = feature_maps + vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - if return_dict: - return BaseModelOutputWithFeatureMaps( - last_hidden_state=vision_outputs.last_hidden_state, - attentions=vision_outputs.attentions, - feature_maps=feature_maps, - feature_maps_position_embeddings=feature_maps_position_embeddings, - ) - - return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions + # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the + # fpn hidden states/position encoding order, still have to double-check + return vision_outputs def _prepare_vision_features( self, diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index a0b7fd318766..dae92ab1ffc5 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -1392,6 +1392,8 @@ def get_image_features( image_embeddings = self.get_input_embeddings()(image_tokens) image_features = torch.split(image_embeddings, split_sizes) + # TODO: @Tom to revisit with the new can_return_tuple approach. + # Presumably requires updating the `self.vqmodel.encode` method to return a BaseModelOutput if return_dict: return BaseModelOutputWithPooling( last_hidden_state=image_embeddings, diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index 08e15816c96a..3b8fc5ddb144 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -944,6 +944,8 @@ def get_image_features( image_embeddings = self.get_input_embeddings()(image_tokens) image_features = torch.split(image_embeddings, split_sizes) + # TODO: @Tom to revisit with the new can_return_tuple approach. + # Presumably requires updating the `self.vqmodel.encode` method to return a BaseModelOutput if return_dict: return BaseModelOutputWithPooling( last_hidden_state=image_embeddings, diff --git a/src/transformers/models/fast_vlm/modeling_fast_vlm.py b/src/transformers/models/fast_vlm/modeling_fast_vlm.py index 577dd0653690..f7acc901d2fa 100644 --- a/src/transformers/models/fast_vlm/modeling_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modeling_fast_vlm.py @@ -114,12 +114,13 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -144,14 +145,14 @@ def get_image_features( else self.config.vision_feature_select_strategy ) - kwargs = {k: v for k, v in kwargs.items() if v is not None} image_outputs = self.vision_tower(pixel_values, **kwargs) # since the vision tower is hybrid in FastVLM, its output needs to be handled differently from Llava selected_image_feature = image_outputs.last_hidden_state selected_image_feature = selected_image_feature.flatten(2).permute(0, 2, 1) image_features = self.multi_modal_projector(selected_image_feature) - image_features = list(image_features) + image_outputs.pooler_output = list(image_features) + return image_features def get_placeholder_mask( @@ -306,7 +307,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, diff --git a/src/transformers/models/fast_vlm/modular_fast_vlm.py b/src/transformers/models/fast_vlm/modular_fast_vlm.py index 516a40bd48dc..87173c060048 100644 --- a/src/transformers/models/fast_vlm/modular_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modular_fast_vlm.py @@ -19,7 +19,8 @@ from ...activations import ACT2FN from ...configuration_utils import PreTrainedConfig -from ...utils import auto_docstring +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ..auto import CONFIG_MAPPING from ..llava.configuration_llava import LlavaConfig from ..llava.modeling_llava import ( @@ -162,12 +163,13 @@ class FastVlmModel(LlavaModel): def __init__(self, config: FastVlmConfig): super().__init__(config) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -192,14 +194,14 @@ def get_image_features( else self.config.vision_feature_select_strategy ) - kwargs = {k: v for k, v in kwargs.items() if v is not None} image_outputs = self.vision_tower(pixel_values, **kwargs) # since the vision tower is hybrid in FastVLM, its output needs to be handled differently from Llava selected_image_feature = image_outputs.last_hidden_state selected_image_feature = selected_image_feature.flatten(2).permute(0, 2, 1) image_features = self.multi_modal_projector(selected_image_feature) - image_features = list(image_features) + image_outputs.pooler_output = list(image_features) + return image_features def forward(self, **super_kwargs): diff --git a/src/transformers/models/florence2/modeling_florence2.py b/src/transformers/models/florence2/modeling_florence2.py index 40fb7a0e79d6..aa061c3b932b 100644 --- a/src/transformers/models/florence2/modeling_florence2.py +++ b/src/transformers/models/florence2/modeling_florence2.py @@ -33,6 +33,7 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torch_available, logging +from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_florence2 import Florence2Config, Florence2VisionConfig @@ -541,12 +542,16 @@ def __init__(self, config: Florence2VisionConfig): # Initialize weights and apply final processing self.post_init() - def forward(self, hidden_states: torch.Tensor, **kwargs): + @check_model_inputs(tie_last_hidden_states=False) + def forward(self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> BaseModelOutputWithPooling: for conv, block in zip(self.convs, self.blocks): hidden_states = conv(hidden_states) for layer in block: hidden_states = layer(hidden_states) - return hidden_states + + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + ) class Florence2MultiModalProjector(nn.Module): @@ -652,7 +657,8 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False, **kwargs): + @can_return_tuple + def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -662,16 +668,10 @@ def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = Fal Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - last_hidden_states = self.vision_tower(pixel_values, **kwargs) - image_features = self.multi_modal_projector(last_hidden_states) + image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs.pooler_output = self.multi_modal_projector(image_outputs.last_hidden_state) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_states=last_hidden_states, - pooler_output=image_features, - ) - - return image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -826,7 +826,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head - def get_image_features(self, pixel_values: torch.Tensor, **kwargs): + def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]): return self.model.get_image_features(pixel_values=pixel_values, **kwargs) @can_return_tuple @@ -908,7 +908,7 @@ def forward( output_hidden_states=output_hidden_states, return_dict=True, cache_position=cache_position, - # **kwargs, ## TODO: add back when Bart attention is refactored and takes kwargs + **kwargs, ) hidden_states = outputs[0] diff --git a/src/transformers/models/florence2/modular_florence2.py b/src/transformers/models/florence2/modular_florence2.py index e21f22915abf..e02b1326dbc5 100644 --- a/src/transformers/models/florence2/modular_florence2.py +++ b/src/transformers/models/florence2/modular_florence2.py @@ -32,6 +32,7 @@ from ...processing_utils import MultiModalData, ProcessorMixin, Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torch_available, logging +from ...utils.generic import check_model_inputs from ..auto import CONFIG_MAPPING, AutoConfig from ..bart.modeling_bart import eager_attention_forward, shift_tokens_right from ..beit.modeling_beit import BeitDropPath @@ -1422,12 +1423,16 @@ def __init__(self, config: Florence2VisionConfig): # Initialize weights and apply final processing self.post_init() - def forward(self, hidden_states: torch.Tensor, **kwargs): + @check_model_inputs(tie_last_hidden_states=False) + def forward(self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> BaseModelOutputWithPooling: for conv, block in zip(self.convs, self.blocks): hidden_states = conv(hidden_states) for layer in block: hidden_states = layer(hidden_states) - return hidden_states + + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + ) class Florence2MultiModalProjector(nn.Module): @@ -1519,7 +1524,8 @@ def get_encoder(self, modality=None): else: return super().get_encoder(modality=modality) - def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False, **kwargs): + @can_return_tuple + def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -1529,16 +1535,10 @@ def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = Fal Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - last_hidden_states = self.vision_tower(pixel_values, **kwargs) - image_features = self.multi_modal_projector(last_hidden_states) + image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs.pooler_output = self.multi_modal_projector(image_outputs.last_hidden_state) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_states=last_hidden_states, - pooler_output=image_features, - ) - - return image_features + return image_outputs @can_return_tuple @auto_docstring @@ -1632,7 +1632,7 @@ class Florence2ForConditionalGeneration(LlavaForConditionalGeneration): "lm_head.weight": "model.language_model.shared.weight", } - def get_image_features(self, pixel_values: torch.Tensor, **kwargs): + def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]): return self.model.get_image_features(pixel_values=pixel_values, **kwargs) @can_return_tuple @@ -1714,7 +1714,7 @@ def forward( output_hidden_states=output_hidden_states, return_dict=True, cache_position=cache_position, - # **kwargs, ## TODO: add back when Bart attention is refactored and takes kwargs + **kwargs, ) hidden_states = outputs[0] diff --git a/src/transformers/models/fuyu/modeling_fuyu.py b/src/transformers/models/fuyu/modeling_fuyu.py index 816c70f3777c..99dcdc2f3cb0 100644 --- a/src/transformers/models/fuyu/modeling_fuyu.py +++ b/src/transformers/models/fuyu/modeling_fuyu.py @@ -116,7 +116,7 @@ def gather_continuous_embeddings( ) return output_embeddings - def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs): + def get_image_features(self, pixel_values: torch.FloatTensor): """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/gemma3/modeling_gemma3.py b/src/transformers/models/gemma3/modeling_gemma3.py index c1b0b46c75b5..0427adbaaaf2 100644 --- a/src/transformers/models/gemma3/modeling_gemma3.py +++ b/src/transformers/models/gemma3/modeling_gemma3.py @@ -828,8 +828,9 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_image_features( - self, pixel_values: torch.FloatTensor, return_dict: bool = False + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. @@ -837,23 +838,15 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) The tensors corresponding to the input images. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - vision_outputs = self.vision_tower(pixel_values=pixel_values) + vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) last_hidden_state = vision_outputs.last_hidden_state - image_features = self.multi_modal_projector(last_hidden_state) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=last_hidden_state, - pooler_output=image_features, - ) + vision_outputs.pooler_output = self.multi_modal_projector(last_hidden_state) - return image_features + return vision_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -1014,8 +1007,8 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): - return self.model.get_image_features(pixel_values, return_dict=return_dict) + def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + return self.model.get_image_features(pixel_values, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/gemma3/modular_gemma3.py b/src/transformers/models/gemma3/modular_gemma3.py index 07797450218d..ff92b84724ae 100644 --- a/src/transformers/models/gemma3/modular_gemma3.py +++ b/src/transformers/models/gemma3/modular_gemma3.py @@ -783,8 +783,9 @@ def __init__(self, config: Gemma3Config): super().__init__(config) del self.text_config_dtype + @can_return_tuple def get_image_features( - self, pixel_values: torch.FloatTensor, return_dict: bool = False + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. @@ -792,23 +793,15 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) The tensors corresponding to the input images. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - vision_outputs = self.vision_tower(pixel_values=pixel_values) + vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) last_hidden_state = vision_outputs.last_hidden_state - image_features = self.multi_modal_projector(last_hidden_state) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=last_hidden_state, - pooler_output=image_features, - ) + vision_outputs.pooler_output = self.multi_modal_projector(last_hidden_state) - return image_features + return vision_outputs @can_return_tuple @auto_docstring diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index 8ddeba180770..8e881e8fd036 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -1900,8 +1900,11 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_image_features( - self, pixel_values: torch.FloatTensor, return_dict: bool = False + self, + pixel_values: torch.FloatTensor, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. @@ -1909,32 +1912,24 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) The tensors corresponding to the input images. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, return_dict=True) + vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, **kwargs) last_hidden_state = vision_outputs.last_hidden_state # Convert from (batch, channels, height, width) to (batch, height * width, channels) where: # height == width and height * width == Gemma3nConfig.vision_soft_tokens_per_image. - vision_outputs = last_hidden_state.reshape( + last_hidden_state = last_hidden_state.reshape( last_hidden_state.shape[0], self.config.vision_config.hidden_size, self.config.vision_soft_tokens_per_image, ).permute(0, 2, 1) # Normalize and embed the soft tokens into language model space. - vision_outputs *= self.config.vision_config.hidden_size**0.5 - image_features = self.embed_vision(inputs_embeds=vision_outputs) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=last_hidden_state, - pooler_output=image_features, - ) + last_hidden_state *= self.config.vision_config.hidden_size**0.5 + vision_outputs.pooler_output = self.embed_vision(inputs_embeds=last_hidden_state) - return image_features + return vision_outputs def get_placeholder_mask( self, @@ -2169,8 +2164,8 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): - return self.model.get_image_features(pixel_values, return_dict=return_dict) + def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + return self.model.get_image_features(pixel_values, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index 5b59d6e101c9..b9e8f895ed9c 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -2158,8 +2158,11 @@ def __init__(self, config: Gemma3nConfig): self.embed_vision = Gemma3nMultimodalEmbedder(config.vision_config, config.text_config) self.embed_audio = Gemma3nMultimodalEmbedder(config.audio_config, config.text_config) + @can_return_tuple def get_image_features( - self, pixel_values: torch.FloatTensor, return_dict: bool = False + self, + pixel_values: torch.FloatTensor, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. @@ -2167,32 +2170,24 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) The tensors corresponding to the input images. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, return_dict=True) + vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, **kwargs) last_hidden_state = vision_outputs.last_hidden_state # Convert from (batch, channels, height, width) to (batch, height * width, channels) where: # height == width and height * width == Gemma3nConfig.vision_soft_tokens_per_image. - vision_outputs = last_hidden_state.reshape( + last_hidden_state = last_hidden_state.reshape( last_hidden_state.shape[0], self.config.vision_config.hidden_size, self.config.vision_soft_tokens_per_image, ).permute(0, 2, 1) # Normalize and embed the soft tokens into language model space. - vision_outputs *= self.config.vision_config.hidden_size**0.5 - image_features = self.embed_vision(inputs_embeds=vision_outputs) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=last_hidden_state, - pooler_output=image_features, - ) + last_hidden_state *= self.config.vision_config.hidden_size**0.5 + vision_outputs.pooler_output = self.embed_vision(inputs_embeds=last_hidden_state) - return image_features + return vision_outputs def get_placeholder_mask( self, diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index ad3c35078784..ecc1259b8bfa 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -28,7 +28,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling @@ -321,11 +321,12 @@ def get_video_features( return vision_outputs + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -335,21 +336,14 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + vision_outputs.pooler_output = image_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_embeds, - ) - - return image_embeds + return vision_outputs def get_placeholder_mask( self, @@ -549,12 +543,22 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], ): - return self.model.get_video_features(pixel_values_videos, video_grid_thw) + return self.model.get_video_features( + pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs + ) - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): - return self.model.get_image_features(pixel_values, image_grid_thw) + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ): + return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index f4a5711881c7..18bf5bc89d30 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1178,11 +1178,12 @@ def get_video_features( return vision_outputs + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1192,21 +1193,14 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + vision_outputs.pooler_output = image_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_embeds, - ) - - return image_embeds + return vision_outputs def get_placeholder_mask( self, @@ -1406,12 +1400,22 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], ): - return self.model.get_video_features(pixel_values_videos, video_grid_thw) + return self.model.get_video_features( + pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs + ) - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): - return self.model.get_image_features(pixel_values, image_grid_thw) + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ): + return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 198f9731012c..89eba39ca94b 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1388,11 +1388,12 @@ def get_video_features( return vision_outputs + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1402,21 +1403,14 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + vision_outputs.pooler_output = image_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_embeds, - ) - - return image_embeds + return vision_outputs def get_placeholder_mask( self, @@ -1669,12 +1663,22 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], ): - return self.model.get_video_features(pixel_values_videos, video_grid_thw) + return self.model.get_video_features( + pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs + ) - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): - return self.model.get_image_features(pixel_values, image_grid_thw) + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ): + return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @auto_docstring @check_model_inputs diff --git a/src/transformers/models/got_ocr2/modeling_got_ocr2.py b/src/transformers/models/got_ocr2/modeling_got_ocr2.py index 145afb66245e..17c020436ead 100644 --- a/src/transformers/models/got_ocr2/modeling_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modeling_got_ocr2.py @@ -551,33 +551,26 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`) - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - image_outputs = self.vision_tower(pixel_values) + image_outputs = self.vision_tower(pixel_values, **kwargs) last_hidden_state = image_outputs.last_hidden_state - image_features = self.multi_modal_projector(last_hidden_state) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=last_hidden_state, - pooler_output=image_features, - ) + image_outputs.pooler_output = self.multi_modal_projector(last_hidden_state) - return image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -696,7 +689,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, diff --git a/src/transformers/models/got_ocr2/modular_got_ocr2.py b/src/transformers/models/got_ocr2/modular_got_ocr2.py index 200545038641..5d8820788619 100644 --- a/src/transformers/models/got_ocr2/modular_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modular_got_ocr2.py @@ -308,33 +308,26 @@ def __init__(self, config: GotOcr2Config): super().__init__(config) self.vision_tower = GotOcr2VisionEncoder(config.vision_config) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`) - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - image_outputs = self.vision_tower(pixel_values) + image_outputs = self.vision_tower(pixel_values, **kwargs) last_hidden_state = image_outputs.last_hidden_state - image_features = self.multi_modal_projector(last_hidden_state) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=last_hidden_state, - pooler_output=image_features, - ) + image_outputs.pooler_output = self.multi_modal_projector(last_hidden_state) - return image_features + return image_outputs def forward( self, diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index dff7952e293d..faa178865eb9 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -821,11 +821,12 @@ def inputs_merger( inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_hidden_states) return inputs_embeds + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -835,8 +836,6 @@ def get_image_features( The tensors corresponding to the input images. pixel_attention_mask (`torch.LongTensor`, *optional*): The attention mask indicating padded regions in the image. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ batch_size, num_images, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility @@ -864,22 +863,18 @@ def get_image_features( patches_subgrid = patches_subgrid.unfold(dimension=2, size=patch_size, step=patch_size) patch_attention_mask = (patches_subgrid.sum(dim=(-1, -2)) == patch_size * patch_size).bool() # Get sequence from the vision encoder - image_hidden_states = self.vision_model(pixel_values=pixel_values, patch_attention_mask=patch_attention_mask) - image_hidden_states = image_hidden_states.last_hidden_state + image_outputs = self.vision_model( + pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, **kwargs + ) + image_hidden_states = image_outputs.last_hidden_state # Modality projection & resampling image_features = self.connector( image_hidden_states, attention_mask=patch_attention_mask.view(pixel_values.size(0), -1) ) - image_features = image_features.view(-1, image_features.shape[-1]) + image_outputs.pooler_output = image_features.view(-1, image_features.shape[-1]) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_hidden_states, - pooler_output=image_features, - ) - - return image_features + return image_outputs @can_return_tuple @auto_docstring( @@ -1001,10 +996,10 @@ def get_image_features( self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: return self.model.get_image_features( - pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask, return_dict=return_dict + pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask, **kwargs ) @can_return_tuple diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index 7967b8ef9dd6..84597c37782f 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -566,11 +566,12 @@ def inputs_merger( inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_hidden_states) return inputs_embeds + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -610,19 +611,16 @@ def get_image_features( patch_attention_mask = (patches_subgrid.sum(dim=(-1, -2)) > 0).bool() # Get sequence from the vision encoder - image_hidden_states = self.vision_model(pixel_values=pixel_values, patch_attention_mask=patch_attention_mask) - image_hidden_states = image_hidden_states.last_hidden_state + image_outputs = self.vision_model( + pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, **kwargs + ) + image_hidden_states = image_outputs.last_hidden_state # Modality projection & resampling image_features = self.connector(image_hidden_states) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_hidden_states, - pooler_output=image_features, - ) - - return image_features + return image_outputs @can_return_tuple @auto_docstring( diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index 4cf4864bb056..51d04e9189cb 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -1243,7 +1243,9 @@ def get_image_features( # step 3: use the language model, conditioned on the query outputs and the prompt image_features = self.language_projection(query_output) - # NOTE: @Tom backwards incompatibility + # NOTE: @Tom the easy solution with the new format would be to grab vision_outputs, + # override the pooler_output with image_features and return that, but we'd lose the + # query_outputs if return_dict: return BaseModelOutputWithVisionQformerOutputs( last_hidden_state=vision_outputs.last_hidden_state, diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 42f6a7aea58e..f73cedfb0368 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -1207,7 +1207,6 @@ def get_image_features( qformer_input_ids: torch.LongTensor, qformer_attention_mask: Optional[torch.LongTensor] = None, interpolate_pos_encoding: Optional[bool] = False, - return_dict: Optional[bool] = False, ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1217,6 +1216,7 @@ def get_image_features( The tensors corresponding to the input images. """ pass + # TODO: @Tom Let's fully remove if not supported def get_placeholder_mask(self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor): """ diff --git a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py index 591cff7e692b..b5c41db3d25a 100644 --- a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py @@ -351,9 +351,9 @@ def get_image_features( qformer_input_ids: torch.LongTensor, qformer_attention_mask: Optional[torch.LongTensor] = None, interpolate_pos_encoding: Optional[bool] = False, - return_dict: Optional[bool] = False, ): pass + # TODO: @Tom Let's fully remove if not supported def get_placeholder_mask(self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor): """ diff --git a/src/transformers/models/internvl/modeling_internvl.py b/src/transformers/models/internvl/modeling_internvl.py index 33b3cebfefee..094504ba6f04 100644 --- a/src/transformers/models/internvl/modeling_internvl.py +++ b/src/transformers/models/internvl/modeling_internvl.py @@ -547,13 +547,13 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -563,8 +563,6 @@ def get_image_features( The tensors corresponding to the input images. vision_feature_layer (`int` or `list[int]`): Layer index or list of layer indices to extract features from. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: vision_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`. @@ -580,7 +578,7 @@ def get_image_features( pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio - vision_outputs = self.vision_tower(pixel_values=pixel_values) + vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) if vision_feature_layer == -1: vision_features = vision_outputs.last_hidden_state else: @@ -604,14 +602,9 @@ def get_image_features( # Project features through multi-modal projector vision_features = self.multi_modal_projector(vision_features) + vision_outputs.pooler_output = vision_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=vision_features, - ) - - return vision_features + return vision_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -796,7 +789,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, diff --git a/src/transformers/models/internvl/modular_internvl.py b/src/transformers/models/internvl/modular_internvl.py index 103aadb2c70b..c1eaf66427fb 100644 --- a/src/transformers/models/internvl/modular_internvl.py +++ b/src/transformers/models/internvl/modular_internvl.py @@ -490,13 +490,13 @@ def pixel_shuffle(self, vision_features: torch.Tensor, scale_factor: float = 0.5 return vision_features + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -506,8 +506,6 @@ def get_image_features( The tensors corresponding to the input images. vision_feature_layer (`int` or `list[int]`): Layer index or list of layer indices to extract features from. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: vision_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`. @@ -523,7 +521,7 @@ def get_image_features( pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio - vision_outputs = self.vision_tower(pixel_values=pixel_values) + vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) if vision_feature_layer == -1: vision_features = vision_outputs.last_hidden_state else: @@ -547,14 +545,9 @@ def get_image_features( # Project features through multi-modal projector vision_features = self.multi_modal_projector(vision_features) + vision_outputs.pooler_output = vision_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=vision_features, - ) - - return vision_features + return vision_outputs @can_return_tuple @auto_docstring diff --git a/src/transformers/models/janus/modeling_janus.py b/src/transformers/models/janus/modeling_janus.py index 0fff4867b9b8..3d3d4b00294e 100644 --- a/src/transformers/models/janus/modeling_janus.py +++ b/src/transformers/models/janus/modeling_janus.py @@ -1083,17 +1083,13 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): - vision_outputs = self.vision_model(pixel_values) - image_features = self.aligner(vision_outputs.last_hidden_state) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + @can_return_tuple + @auto_docstring + def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + vision_outputs = self.vision_model(pixel_values, **kwargs) + vision_outputs.pooler_output = self.aligner(vision_outputs.last_hidden_state) - return image_features + return vision_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor diff --git a/src/transformers/models/janus/modular_janus.py b/src/transformers/models/janus/modular_janus.py index 0f93643c4139..0c8ebcab99a9 100644 --- a/src/transformers/models/janus/modular_janus.py +++ b/src/transformers/models/janus/modular_janus.py @@ -45,7 +45,7 @@ valid_images, validate_preprocess_arguments, ) -from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import ImagesKwargs, Unpack from ...utils import ( @@ -899,17 +899,13 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): - vision_outputs = self.vision_model(pixel_values) - image_features = self.aligner(vision_outputs.last_hidden_state) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + @can_return_tuple + @auto_docstring + def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + vision_outputs = self.vision_model(pixel_values, **kwargs) + vision_outputs.pooler_output = self.aligner(vision_outputs.last_hidden_state) - return image_features + return vision_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor diff --git a/src/transformers/models/kosmos2/modeling_kosmos2.py b/src/transformers/models/kosmos2/modeling_kosmos2.py index d7ea18587fb3..b9449ed110a6 100644 --- a/src/transformers/models/kosmos2/modeling_kosmos2.py +++ b/src/transformers/models/kosmos2/modeling_kosmos2.py @@ -1507,50 +1507,33 @@ def get_input_embeddings(self) -> nn.Module: def set_input_embeddings(self, value): self.text_model.model.embed_tokens = value + @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: Optional[bool] = False, - return_dict: bool = False, - return_attentions: Optional[bool] = False, # TODO: @Tom neatly deprecate this + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate positional embeddings or not. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - """ - - # NOTE: @Tom backwards incompatibility - if return_attentions: + if "return_attentions" in kwargs: warnings.warn( "`return_attentions` is deprecated and will be removed in a future version. Please use `return_dict`" " and access `projection_attentions` from the returned `ModelOutput` instead.", FutureWarning, ) - return_dict = True - vision_model_output = self.vision_model( + vision_output = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + **kwargs, ) # The whole `last_hidden_state` through `post_layernorm` instead of just `pooled_output`. - image_embeds = self.vision_model.model.post_layernorm(vision_model_output[0]) + image_embeds = self.vision_model.model.post_layernorm(vision_output[0]) # normalized features image_embeds = nn.functional.normalize(image_embeds, dim=-1) image_embeds, projection_attentions = self.image_to_text_projection(image_embeds) - - if return_dict: - return BaseModelOutputWithProjectionAttentions( - last_hidden_state=vision_model_output.last_hidden_state, - pooler_output=image_embeds, - projection_attentions=projection_attentions, - ) + vision_output.pooler_output = image_embeds + # NOTE: @Tom we need to add projection_attentions to the output return image_embeds diff --git a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py index 1b78231161af..e2e83ee8a7eb 100755 --- a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py @@ -28,7 +28,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -161,13 +161,13 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, spatial_shapes: torch.Tensor, pixel_attention_mask: torch.Tensor, - return_dict: bool = False, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ) -> list[torch.Tensor]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -179,8 +179,6 @@ def get_image_features( The spatial shapes of the input images. pixel_attention_mask (`torch.Tensor` of shape `(batch_size, height, width)`): The pixel attention mask of the input images. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`list[torch.Tensor]`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). @@ -189,6 +187,7 @@ def get_image_features( pixel_values=pixel_values, spatial_shapes=spatial_shapes, pixel_attention_mask=pixel_attention_mask, + **kwargs, ) last_hidden_state = image_outputs.last_hidden_state @@ -211,13 +210,8 @@ def get_image_features( img_embedding = img_embedding.reshape(-1, img_embedding.size(-1)) image_features.append(img_embedding) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + image_outputs.pooler_output = image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -334,7 +328,7 @@ def get_image_features( pixel_values: torch.FloatTensor, spatial_shapes: torch.Tensor, pixel_attention_mask: torch.Tensor, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, diff --git a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py index c3c37e4cc45f..438d3d928a8c 100644 --- a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py @@ -21,7 +21,6 @@ from ...activations import ACT2FN from ...cache_utils import Cache -from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from ..llava.modeling_llava import ( @@ -93,13 +92,13 @@ class Lfm2VlModel(LlavaModel): def __init__(self, config: Lfm2VlConfig): super().__init__(config) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, spatial_shapes: torch.Tensor, pixel_attention_mask: torch.Tensor, - return_dict: bool = False, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ) -> list[torch.Tensor]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -111,8 +110,6 @@ def get_image_features( The spatial shapes of the input images. pixel_attention_mask (`torch.Tensor` of shape `(batch_size, height, width)`): The pixel attention mask of the input images. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`list[torch.Tensor]`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). @@ -121,6 +118,7 @@ def get_image_features( pixel_values=pixel_values, spatial_shapes=spatial_shapes, pixel_attention_mask=pixel_attention_mask, + **kwargs, ) last_hidden_state = image_outputs.last_hidden_state @@ -143,13 +141,8 @@ def get_image_features( img_embedding = img_embedding.reshape(-1, img_embedding.size(-1)) image_features.append(img_embedding) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + image_outputs.pooler_output = image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -245,7 +238,7 @@ def get_image_features( pixel_values: torch.FloatTensor, spatial_shapes: torch.Tensor, pixel_attention_mask: torch.Tensor, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, diff --git a/src/transformers/models/llama4/modeling_llama4.py b/src/transformers/models/llama4/modeling_llama4.py index 9d45fa3f6c08..d8fcd0d6f203 100644 --- a/src/transformers/models/llama4/modeling_llama4.py +++ b/src/transformers/models/llama4/modeling_llama4.py @@ -1203,7 +1203,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_select_strategy: str, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply al projection. @@ -1218,11 +1218,8 @@ def get_image_features( image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ if vision_feature_select_strategy not in ["default", "full"]: - raise ValueError(f"Unexpected select feature strategy: {self.vision_feature_select_strategy}") - kwargs = {k: v for k, v in kwargs.items() if v is not None} - image_outputs = self.vision_model(pixel_values, output_hidden_states=False, **kwargs) - hidden_state = image_outputs.last_hidden_state - return hidden_state + raise ValueError(f"Unexpected select feature strategy: {vision_feature_select_strategy}") + return self.vision_model(pixel_values, **kwargs) def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index e04806e0f806..d5bb97ac0b4b 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -23,7 +23,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -147,13 +147,13 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -183,7 +183,6 @@ def get_image_features( if vision_feature_select_strategy not in ["default", "full"]: raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}") - kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) @@ -210,14 +209,9 @@ def get_image_features( image_features = torch.split(image_features.squeeze(0), split_sizes) else: image_features = list(image_features) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -339,7 +333,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index 8000023ad436..592a6d13d5f8 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -28,7 +28,7 @@ from ...generation import GenerationMixin from ...image_processing_utils import select_best_resolution from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -346,13 +346,14 @@ def pack_image_features(self, image_features, image_sizes, vision_feature_select feature_lens = torch.tensor(feature_lens, dtype=torch.long, device=image_features[0].device) return new_image_features, feature_lens + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -402,7 +403,7 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -424,14 +425,9 @@ def get_image_features( vision_feature_select_strategy=vision_feature_select_strategy, image_newline=self.image_newline, ) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -581,14 +577,14 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - return_dict=return_dict, + **kwargs, ) @can_return_tuple diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 1579bb0cdf88..b7b011382460 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -398,13 +398,14 @@ def pack_image_features(self, image_features, image_sizes, vision_feature_select feature_lens = torch.tensor(feature_lens, dtype=torch.long, device=image_features[0].device) return new_image_features, feature_lens + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -451,7 +452,7 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -471,14 +472,9 @@ def get_image_features( vision_feature_select_strategy, image_newline=self.image_newline, ) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return image_outputs def get_placeholder_mask( self, @@ -727,14 +723,14 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - return_dict=return_dict, + **kwargs, ) @can_return_tuple diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index 77d1e982f376..0b3c8f26a3eb 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -35,7 +35,7 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack -from ...utils import logging +from ...utils import can_return_tuple, logging from ..auto import CONFIG_MAPPING, AutoConfig @@ -273,13 +273,14 @@ def __init__(self, config: LlavaNextVideoConfig, **super_kwargs): self.vision_resampler = LlavaNextVideoPooler(config) self.post_init() + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -326,7 +327,7 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -346,14 +347,9 @@ def get_image_features( vision_feature_select_strategy, image_newline=self.image_newline, ) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return image_outputs def get_video_features( self, diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index 6944e06cd431..cab379df064f 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -355,6 +355,7 @@ def pack_image_features(self, image_features, image_sizes, image_newline=None, v feature_lens = torch.tensor(feature_lens, dtype=torch.long, device=image_features[0].device) return new_image_features, feature_lens + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, @@ -363,7 +364,7 @@ def get_image_features( vision_feature_select_strategy: Optional[str] = None, vision_aspect_ratio: Optional[str] = None, batch_num_images: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -422,7 +423,7 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -442,14 +443,9 @@ def get_image_features( image_newline=self.image_newline, vision_aspect_ratio=vision_aspect_ratio, ) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return image_outputs def get_placeholder_mask( self, @@ -715,14 +711,14 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - return_dict=return_dict, + **kwargs, ) @can_return_tuple diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index 603604a1f1b1..62890a7a6b6f 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -315,6 +315,7 @@ def apply_pooling(self, image_features): image_features = image_features.view(batch_frames, -1, dim) return image_features + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, @@ -323,7 +324,7 @@ def get_image_features( vision_feature_select_strategy: Optional[str] = None, vision_aspect_ratio: Optional[str] = None, batch_num_images: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -382,7 +383,7 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -402,14 +403,9 @@ def get_image_features( image_newline=self.image_newline, vision_aspect_ratio=vision_aspect_ratio, ) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return image_outputs def get_video_features( self, diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index 2bfa05c39244..c67eaa4d3d86 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -29,7 +29,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...integrations import use_kernel_forward_from_hub -from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -213,13 +213,13 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, - return_dict: bool = False, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -233,8 +233,6 @@ def get_image_features( vision features. image_sizes (`torch.Tensor`, *optional*): Tensor containing the image sizes as returned by the processor. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). @@ -243,7 +241,6 @@ def get_image_features( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) - kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, output_hidden_states=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, @@ -258,14 +255,9 @@ def get_image_features( downsample_ratio = self.vision_tower.patch_size * self.config.spatial_merge_size split_sizes = [(height // downsample_ratio) * (width // downsample_ratio) for height, width in image_sizes] image_features = torch.split(image_features.squeeze(0), split_sizes) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -393,7 +385,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, diff --git a/src/transformers/models/mistral3/modular_mistral3.py b/src/transformers/models/mistral3/modular_mistral3.py index acef601186f5..48b7ee3a9daa 100644 --- a/src/transformers/models/mistral3/modular_mistral3.py +++ b/src/transformers/models/mistral3/modular_mistral3.py @@ -20,7 +20,6 @@ from ...activations import ACT2FN from ...cache_utils import Cache -from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import logging from ..llava.modeling_llava import ( @@ -124,8 +123,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, - return_dict: bool = False, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -139,8 +137,6 @@ def get_image_features( vision features. image_sizes (`torch.Tensor`, *optional*): Tensor containing the image sizes as returned by the processor. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). @@ -149,7 +145,6 @@ def get_image_features( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) - kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, output_hidden_states=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, @@ -164,14 +159,9 @@ def get_image_features( downsample_ratio = self.vision_tower.patch_size * self.config.spatial_merge_size split_sizes = [(height // downsample_ratio) * (width // downsample_ratio) for height, width in image_sizes] image_features = torch.split(image_features.squeeze(0), split_sizes) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return image_outputs def forward( self, @@ -245,7 +235,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values=pixel_values, diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index 44f319b068df..794999d1d8d3 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -548,10 +548,11 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -569,7 +570,7 @@ def get_image_features( Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - image_outputs = self.vision_tower(pixel_values, return_dict=True) + image_outputs = self.vision_tower(pixel_values, **kwargs) image_features = image_outputs.pooler_output batch_size, img_seq_len, _ = image_features.shape padding_tensor = torch.zeros( @@ -582,21 +583,16 @@ def get_image_features( image_features = torch.cat([image_features, padding_tensor], dim=2) image_features = self.visual_embeddings_table(image_features) - if return_dict: - visual_indicator = torch.arange( - self.visual_vocab_size - self.vision_tower.num_visual_indicator_tokens, - self.visual_vocab_size, - dtype=torch.long, - ).to(image_features.device) - visual_indicator_features = self.visual_embeddings_table(visual_indicator) - - return BaseModelOutputWithVisualIndicatorFeatures( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - visual_indicator_features=visual_indicator_features, - ) + visual_indicator = torch.arange( + self.visual_vocab_size - self.vision_tower.num_visual_indicator_tokens, + self.visual_vocab_size, + dtype=torch.long, + ).to(image_features.device) + visual_indicator_features = self.visual_embeddings_table(visual_indicator) + image_outputs.pooler_output = image_features + # NOTE: @Tom let's readd visual_indicator_features to the output - return image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -723,8 +719,10 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False) -> torch.FloatTensor: - return self.model.get_image_features(pixel_values=pixel_values, return_dict=return_dict) + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> torch.FloatTensor: + return self.model.get_image_features(pixel_values=pixel_values, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index ef0eebc642ef..a44db4045ef9 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -257,12 +257,13 @@ def __init__(self, config: Ovis2Config): self.language_model = AutoModel.from_config(config.text_config) del self.multi_modal_projector + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: - image_outputs = self.vision_tower(pixel_values, return_dict=True) + image_outputs = self.vision_tower(pixel_values, **kwargs) image_features = image_outputs.pooler_output batch_size, img_seq_len, _ = image_features.shape padding_tensor = torch.zeros( @@ -275,21 +276,16 @@ def get_image_features( image_features = torch.cat([image_features, padding_tensor], dim=2) image_features = self.visual_embeddings_table(image_features) - if return_dict: - visual_indicator = torch.arange( - self.visual_vocab_size - self.vision_tower.num_visual_indicator_tokens, - self.visual_vocab_size, - dtype=torch.long, - ).to(image_features.device) - visual_indicator_features = self.visual_embeddings_table(visual_indicator) - - return BaseModelOutputWithVisualIndicatorFeatures( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - visual_indicator_features=visual_indicator_features, - ) + visual_indicator = torch.arange( + self.visual_vocab_size - self.vision_tower.num_visual_indicator_tokens, + self.visual_vocab_size, + dtype=torch.long, + ).to(image_features.device) + visual_indicator_features = self.visual_embeddings_table(visual_indicator) + image_outputs.pooler_output = image_features + # NOTE: @Tom let's readd visual_indicator_features to the output - return image_features + return image_outputs @can_return_tuple @auto_docstring @@ -380,8 +376,10 @@ def __init__(self, config: Ovis2Config): super().__init__(config) self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False) -> torch.FloatTensor: - return self.model.get_image_features(pixel_values=pixel_values, return_dict=return_dict) + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> torch.FloatTensor: + return self.model.get_image_features(pixel_values=pixel_values, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index dcf36d79b933..03748e557b78 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -1188,7 +1188,7 @@ def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1198,23 +1198,22 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=True) + vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + vision_outputs.pooler_output = video_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=video_embeds, - ) - - return video_embeds + return vision_outputs - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + @can_return_tuple + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1238,10 +1237,13 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op pixel_values=pixel_values, image_grid_thw=image_grid_thw, cu_seqlens=cu_seqlens, + **kwargs, ) image_embeds = vision_outputs.last_hidden_state image_embeds = self.projector(image_embeds, image_grid_thw) - return image_embeds + vision_outputs.pooler_output = image_embeds + + return vision_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -1361,12 +1363,22 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], ): - return self.model.get_video_features(pixel_values_videos, video_grid_thw) + return self.model.get_video_features( + pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs + ) - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): - return self.model.get_image_features(pixel_values, image_grid_thw) + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ): + return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py index 79e6fc45d8bf..7cfefc38edde 100644 --- a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py @@ -1103,7 +1103,13 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.embed_tokens = value - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + @can_return_tuple + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1127,10 +1133,13 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op pixel_values=pixel_values, image_grid_thw=image_grid_thw, cu_seqlens=cu_seqlens, + **kwargs, ) image_embeds = vision_outputs.last_hidden_state image_embeds = self.projector(image_embeds, image_grid_thw) - return image_embeds + vision_outputs.pooler_output = image_embeds + + return vision_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index 965412df4ec7..c218a50c0c0b 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -26,7 +26,7 @@ from ...generation import GenerationMixin from ...masking_utils import create_masks_for_generate from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling +from ...modeling_outputs import BaseModelOutputWithPast from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import ( @@ -257,31 +257,25 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): + @can_return_tuple + def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): """ Obtains image last hidden states from the vision tower and apply multimodal projection. Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) The tensors corresponding to the input images. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ - image_outputs = self.vision_tower(pixel_values) + image_outputs = self.vision_tower(pixel_values, **kwargs) selected_image_feature = image_outputs.last_hidden_state image_features = self.multi_modal_projector(selected_image_feature) image_features = image_features / (self.config.text_config.hidden_size**0.5) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=selected_image_feature, - pooler_output=image_features, - ) - - return image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -454,8 +448,8 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.FloatTensor, return_dict: bool = False): - return self.model.get_image_features(pixel_values, return_dict=return_dict) + def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + return self.model.get_image_features(pixel_values, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/perception_lm/modeling_perception_lm.py b/src/transformers/models/perception_lm/modeling_perception_lm.py index 6902b3fd2827..d82668708723 100644 --- a/src/transformers/models/perception_lm/modeling_perception_lm.py +++ b/src/transformers/models/perception_lm/modeling_perception_lm.py @@ -28,9 +28,10 @@ from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import auto_docstring, can_return_tuple +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ..auto import AutoModel from .configuration_perception_lm import PerceptionLMConfig @@ -180,11 +181,11 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -192,25 +193,18 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_tiles, channels, height, width)`) The tensors corresponding to the input images. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_tiles, num_patches, embed_dim)`). """ - image_outputs = self.vision_tower(pixel_values.flatten(0, 1)) + image_outputs = self.vision_tower(pixel_values.flatten(0, 1), **kwargs) last_hidden_state = image_outputs.last_hidden_state if self.config.vision_use_cls_token: last_hidden_state = last_hidden_state[:, 1:, :] image_features = self.multi_modal_projector(last_hidden_state) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return image_outputs def get_placeholder_mask( self, diff --git a/src/transformers/models/perception_lm/modular_perception_lm.py b/src/transformers/models/perception_lm/modular_perception_lm.py index 77485b38832b..5f4f20b1bebe 100644 --- a/src/transformers/models/perception_lm/modular_perception_lm.py +++ b/src/transformers/models/perception_lm/modular_perception_lm.py @@ -21,8 +21,9 @@ from torch import nn from ...cache_utils import Cache -from ...modeling_outputs import BaseModelOutputWithPooling +from ...processing_utils import Unpack from ...utils import ( + TransformersKwargs, auto_docstring, can_return_tuple, logging, @@ -146,11 +147,11 @@ def __init__(self, config: PerceptionLMConfig): self.multi_modal_projector = PerceptionLMMultiModalProjector(config) self.language_model = AutoModel.from_config(config.text_config) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -158,25 +159,18 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_tiles, channels, height, width)`) The tensors corresponding to the input images. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_tiles, num_patches, embed_dim)`). """ - image_outputs = self.vision_tower(pixel_values.flatten(0, 1)) + image_outputs = self.vision_tower(pixel_values.flatten(0, 1), **kwargs) last_hidden_state = image_outputs.last_hidden_state if self.config.vision_use_cls_token: last_hidden_state = last_hidden_state[:, 1:, :] image_features = self.multi_modal_projector(last_hidden_state) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return image_outputs def get_placeholder_mask( self, diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index 161e8ca08114..bb5e37c87a7e 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -41,7 +41,7 @@ from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, check_torch_load_is_safe, logging +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, check_torch_load_is_safe, logging from ...utils.deprecation import deprecate_kwarg from ...utils.generic import maybe_autocast from ...utils.hub import cached_file @@ -1728,11 +1728,12 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @can_return_tuple def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1742,17 +1743,16 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - return self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=return_dict) + return self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1762,18 +1762,17 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - return self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=return_dict) + return self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) + @can_return_tuple def get_audio_features( self, input_features: torch.FloatTensor, feature_attention_mask: Optional[torch.LongTensor] = None, audio_feature_lengths: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes audios into continuous embeddings that can be forwarded to the language model. @@ -1785,8 +1784,6 @@ def get_audio_features( Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): The length of feature shape of each audio in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) @@ -1803,15 +1800,10 @@ def get_audio_features( feature_lens=feature_lens, aftercnn_lens=audio_feat_lengths, ) - audio_features = audio_outputs.last_hidden_state - - if audio_features.shape[0] != sum(audio_output_lengths.tolist()): + if audio_outputs.last_hidden_state.shape[0] != sum(audio_output_lengths.tolist()): raise ValueError("length of audio_features should match audio_output_lengths") - if return_dict: - return audio_outputs - - return audio_features + return audio_outputs def get_placeholder_mask( self, diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index 631ba455c558..318964d161f3 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -51,6 +51,7 @@ from ...utils import ( TransformersKwargs, auto_docstring, + can_return_tuple, check_torch_load_is_safe, logging, ) @@ -2077,11 +2078,12 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @can_return_tuple def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -2091,17 +2093,16 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - return self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=return_dict) + return self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -2111,18 +2112,17 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - return self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=return_dict) + return self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) + @can_return_tuple def get_audio_features( self, input_features: torch.FloatTensor, feature_attention_mask: Optional[torch.LongTensor] = None, audio_feature_lengths: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes audios into continuous embeddings that can be forwarded to the language model. @@ -2134,8 +2134,6 @@ def get_audio_features( Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): The length of feature shape of each audio in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) @@ -2152,15 +2150,10 @@ def get_audio_features( feature_lens=feature_lens, aftercnn_lens=audio_feat_lengths, ) - audio_features = audio_outputs.last_hidden_state - - if audio_features.shape[0] != sum(audio_output_lengths.tolist()): + if audio_outputs.last_hidden_state.shape[0] != sum(audio_output_lengths.tolist()): raise ValueError("length of audio_features should match audio_output_lengths") - if return_dict: - return audio_outputs - - return audio_features + return audio_outputs def get_placeholder_mask( self, diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index 4f8466b82239..62926da32bd0 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -1173,7 +1173,7 @@ def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1183,27 +1183,21 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=True) + vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + vision_outputs.pooler_output = video_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=video_embeds, - ) - - return video_embeds + return vision_outputs + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1213,21 +1207,14 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + vision_outputs.pooler_output = image_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_embeds, - ) - - return image_embeds + return vision_outputs def get_placeholder_mask( self, @@ -1421,12 +1408,22 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], ): - return self.model.get_video_features(pixel_values_videos, video_grid_thw) + return self.model.get_video_features( + pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs + ) - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): - return self.model.get_image_features(pixel_values, image_grid_thw) + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ): + return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index f4111212dddc..96e4a0c50ff1 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -1108,7 +1108,7 @@ def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1118,27 +1118,21 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=True) + vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + vision_outputs.pooler_output = video_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=video_embeds, - ) - - return video_embeds + return vision_outputs + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1148,21 +1142,14 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) + vision_outputs.pooler_output = image_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_embeds, - ) - - return image_embeds + return vision_outputs def get_placeholder_mask( self, @@ -1319,12 +1306,22 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], ): - return self.model.get_video_features(pixel_values_videos, video_grid_thw) + return self.model.get_video_features( + pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs + ) - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): - return self.model.get_image_features(pixel_values, image_grid_thw) + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ): + return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 94f6713b96fb..65994ecdde86 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -48,7 +48,7 @@ ) from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...processing_utils import Unpack +from ...processing_utils import ProcessingKwargs, Unpack from ...utils import ModelOutput, auto_docstring, can_return_tuple from ...utils.generic import OutputRecorder, TransformersKwargs, check_model_inputs, maybe_autocast from .configuration_qwen3_omni_moe import ( @@ -1194,17 +1194,16 @@ def fast_pos_embed_interpolate(self, grid_thw): patch_pos_embeds = torch.cat(patch_pos_embeds_permute) return patch_pos_embeds + @check_model_inputs(tie_last_hidden_states=False) def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs - ) -> torch.Tensor: + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[ProcessingKwargs] + ) -> BaseModelOutputWithDeepstackFeatures: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. Returns: `torch.Tensor`: hidden_states. @@ -1248,14 +1247,11 @@ def forward( merged_hidden_states = self.merger(hidden_states) - if return_dict: - return BaseModelOutputWithDeepstackFeatures( - last_hidden_state=hidden_states, - pooler_output=merged_hidden_states, - deepstack_features=deepstack_feature_lists, - ) - - return merged_hidden_states, deepstack_feature_lists + return BaseModelOutputWithDeepstackFeatures( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + deepstack_features=deepstack_feature_lists, + ) @property def deepstack_merger_list(self): @@ -1942,11 +1938,12 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @can_return_tuple def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1956,17 +1953,16 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - return self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=return_dict) + return self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1976,12 +1972,11 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ pixel_values = pixel_values.type(self.visual.dtype) - return self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=return_dict) + return self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) + @can_return_tuple def get_audio_features( self, input_features: torch.FloatTensor, diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index 1417f6d4adab..9bed63e837e9 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -37,7 +37,7 @@ from ...modeling_outputs import BaseModelOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...processing_utils import Unpack +from ...processing_utils import ProcessingKwargs, Unpack from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple from ...utils.generic import check_model_inputs, maybe_autocast from .configuration_qwen3_vl import Qwen3VLConfig, Qwen3VLTextConfig, Qwen3VLVisionConfig @@ -753,17 +753,16 @@ def fast_pos_embed_interpolate(self, grid_thw): patch_pos_embeds = torch.cat(patch_pos_embeds_permute) return patch_pos_embeds + @check_model_inputs(tie_last_hidden_states=False) def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs - ) -> torch.Tensor: + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[ProcessingKwargs] + ) -> BaseModelOutputWithDeepstackFeatures: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. Returns: `torch.Tensor`: hidden_states. @@ -807,14 +806,11 @@ def forward( merged_hidden_states = self.merger(hidden_states) - if return_dict: - return BaseModelOutputWithDeepstackFeatures( - last_hidden_state=hidden_states, - pooler_output=merged_hidden_states, - deepstack_features=deepstack_feature_lists, - ) - - return merged_hidden_states, deepstack_feature_lists + return BaseModelOutputWithDeepstackFeatures( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + deepstack_features=deepstack_feature_lists, + ) @auto_docstring( @@ -1095,8 +1091,8 @@ def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, - ): + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithDeepstackFeatures: """ Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1105,18 +1101,17 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. """ # Same implementation as for images - return self.get_image_features(pixel_values_videos, video_grid_thw, return_dict=return_dict) + return self.get_image_features(pixel_values_videos, video_grid_thw, **kwargs) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, - ): + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithDeepstackFeatures: """ Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1125,18 +1120,17 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_output = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=return_dict) - if return_dict: - return vision_output - - image_embeds, deepstack_image_embeds = vision_output + vision_output: BaseModelOutputWithDeepstackFeatures = self.visual( + pixel_values, grid_thw=image_grid_thw, **kwargs + ) + image_embeds = vision_output.last_hidden_state split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) - return image_embeds, deepstack_image_embeds + vision_output.pooler_output = image_embeds + + return vision_output def get_placeholder_mask( self, @@ -1339,12 +1333,22 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], ): - return self.model.get_video_features(pixel_values_videos, video_grid_thw) + return self.model.get_video_features( + pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs + ) - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): - return self.model.get_image_features(pixel_values, image_grid_thw) + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ): + return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple def forward( diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index 9950d219f94f..311949ce52eb 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -650,17 +650,16 @@ def fast_pos_embed_interpolate(self, grid_thw): patch_pos_embeds = torch.cat(patch_pos_embeds_permute) return patch_pos_embeds + @check_model_inputs(tie_last_hidden_states=False) def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs - ) -> torch.Tensor: + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[ProcessingKwargs] + ) -> BaseModelOutputWithDeepstackFeatures: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. Returns: `torch.Tensor`: hidden_states. @@ -704,14 +703,11 @@ def forward( merged_hidden_states = self.merger(hidden_states) - if return_dict: - return BaseModelOutputWithDeepstackFeatures( - last_hidden_state=hidden_states, - pooler_output=merged_hidden_states, - deepstack_features=deepstack_feature_lists, - ) - - return merged_hidden_states, deepstack_feature_lists + return BaseModelOutputWithDeepstackFeatures( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + deepstack_features=deepstack_feature_lists, + ) @auto_docstring( @@ -964,12 +960,13 @@ def get_rope_index( return position_ids, mrope_position_deltas + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, - ): + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithDeepstackFeatures: """ Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -978,25 +975,24 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_output = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=return_dict) - if return_dict: - return vision_output - - image_embeds, deepstack_image_embeds = vision_output + vision_output: BaseModelOutputWithDeepstackFeatures = self.visual( + pixel_values, grid_thw=image_grid_thw, **kwargs + ) + image_embeds = vision_output.last_hidden_state split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) - return image_embeds, deepstack_image_embeds + vision_output.pooler_output = image_embeds + + return vision_output def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, - ): + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithDeepstackFeatures: """ Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1005,11 +1001,9 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. """ # Same implementation as for images - return self.get_image_features(pixel_values_videos, video_grid_thw, return_dict=return_dict) + return self.get_image_features(pixel_values_videos, video_grid_thw, **kwargs) @auto_docstring @check_model_inputs diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index eee208d384d6..47e7b79f5705 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -38,7 +38,7 @@ from ...modeling_outputs import BaseModelOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...processing_utils import Unpack +from ...processing_utils import ProcessingKwargs, Unpack from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple from ...utils.generic import OutputRecorder, check_model_inputs, maybe_autocast from .configuration_qwen3_vl_moe import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig, Qwen3VLMoeVisionConfig @@ -767,17 +767,16 @@ def fast_pos_embed_interpolate(self, grid_thw): patch_pos_embeds = torch.cat(patch_pos_embeds_permute) return patch_pos_embeds + @check_model_inputs(tie_last_hidden_states=False) def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs - ) -> torch.Tensor: + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[ProcessingKwargs] + ) -> BaseModelOutputWithDeepstackFeatures: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. Returns: `torch.Tensor`: hidden_states. @@ -821,14 +820,11 @@ def forward( merged_hidden_states = self.merger(hidden_states) - if return_dict: - return BaseModelOutputWithDeepstackFeatures( - last_hidden_state=hidden_states, - pooler_output=merged_hidden_states, - deepstack_features=deepstack_feature_lists, - ) - - return merged_hidden_states, deepstack_feature_lists + return BaseModelOutputWithDeepstackFeatures( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + deepstack_features=deepstack_feature_lists, + ) class Qwen3VLMoeTextRotaryEmbedding(nn.Module): @@ -1252,8 +1248,8 @@ def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, - ): + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithDeepstackFeatures: """ Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1262,18 +1258,17 @@ def get_video_features( The tensors corresponding to the input videos. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. """ # Same implementation as for images - return self.get_image_features(pixel_values_videos, video_grid_thw, return_dict=return_dict) + return self.get_image_features(pixel_values_videos, video_grid_thw, **kwargs) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, - return_dict: bool = False, - ): + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithDeepstackFeatures: """ Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1282,18 +1277,17 @@ def get_image_features( The tensors corresponding to the input images. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a [`BaseModelOutputWithDeepstackFeatures`] instead of a plain tuple. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_output = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=return_dict) - if return_dict: - return vision_output - - image_embeds, deepstack_image_embeds = vision_output + vision_output: BaseModelOutputWithDeepstackFeatures = self.visual( + pixel_values, grid_thw=image_grid_thw, **kwargs + ) + image_embeds = vision_output.last_hidden_state split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) - return image_embeds, deepstack_image_embeds + vision_output.pooler_output = image_embeds + + return vision_output def get_placeholder_mask( self, @@ -1549,12 +1543,22 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], ): - return self.model.get_video_features(pixel_values_videos, video_grid_thw) + return self.model.get_video_features( + pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs + ) - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): - return self.model.get_image_features(pixel_values, image_grid_thw) + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ): + return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple def forward( diff --git a/src/transformers/models/sam2/modeling_sam2.py b/src/transformers/models/sam2/modeling_sam2.py index 9023ea497723..fee83133d696 100644 --- a/src/transformers/models/sam2/modeling_sam2.py +++ b/src/transformers/models/sam2/modeling_sam2.py @@ -39,7 +39,7 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import compile_compatible_method_lru_cache -from ...utils import ModelOutput, auto_docstring +from ...utils import ModelOutput, auto_docstring, can_return_tuple from ...utils.generic import TransformersKwargs, check_model_inputs from ..auto import AutoModel from .configuration_sam2 import ( @@ -1578,10 +1578,10 @@ def forward( vision_attentions=vision_attentions, ) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -1595,8 +1595,6 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -1605,10 +1603,7 @@ def get_image_features( - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. """ - vision_outputs: Sam2VisionEncoderOutput = self.vision_encoder( - pixel_values, - **kwargs, - ) + vision_outputs: Sam2VisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding @@ -1625,16 +1620,12 @@ def get_image_features( feature_map_position_embedding.flatten(2).permute(2, 0, 1) for feature_map_position_embedding in feature_maps_position_embeddings ] + vision_outputs.fpn_hidden_states = feature_maps + vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - if return_dict: - return BaseModelOutputWithFeatureMaps( - last_hidden_state=vision_outputs.last_hidden_state, - attentions=vision_outputs.attentions, - feature_maps=feature_maps, - feature_maps_position_embeddings=feature_maps_position_embeddings, - ) - - return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions + # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the + # fpn hidden states/position encoding order, still have to double-check + return vision_outputs __all__ = ["Sam2Model", "Sam2VisionModel", "Sam2PreTrainedModel", "Sam2HieraDetModel"] diff --git a/src/transformers/models/sam2/modular_sam2.py b/src/transformers/models/sam2/modular_sam2.py index 725b11049326..2d93a47a3174 100644 --- a/src/transformers/models/sam2/modular_sam2.py +++ b/src/transformers/models/sam2/modular_sam2.py @@ -43,6 +43,7 @@ ModelOutput, TensorType, auto_docstring, + can_return_tuple, logging, ) from ...utils.generic import TransformersKwargs, check_model_inputs @@ -1266,10 +1267,10 @@ def get_image_embeddings( return image_embeddings + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -1283,8 +1284,6 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -1293,10 +1292,7 @@ def get_image_features( - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. """ - vision_outputs: Sam2VisionEncoderOutput = self.vision_encoder( - pixel_values, - **kwargs, - ) + vision_outputs: Sam2VisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding @@ -1313,16 +1309,12 @@ def get_image_features( feature_map_position_embedding.flatten(2).permute(2, 0, 1) for feature_map_position_embedding in feature_maps_position_embeddings ] + vision_outputs.fpn_hidden_states = feature_maps + vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - if return_dict: - return BaseModelOutputWithFeatureMaps( - last_hidden_state=vision_outputs.last_hidden_state, - attentions=vision_outputs.attentions, - feature_maps=feature_maps, - feature_maps_position_embeddings=feature_maps_position_embeddings, - ) - - return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions + # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the + # fpn hidden states/position encoding order, still have to double-check + return vision_outputs @check_model_inputs @auto_docstring diff --git a/src/transformers/models/sam2_video/modeling_sam2_video.py b/src/transformers/models/sam2_video/modeling_sam2_video.py index e6a6534f45f2..c72b4e66ad86 100644 --- a/src/transformers/models/sam2_video/modeling_sam2_video.py +++ b/src/transformers/models/sam2_video/modeling_sam2_video.py @@ -40,7 +40,7 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import compile_compatible_method_lru_cache -from ...utils import ModelOutput, auto_docstring +from ...utils import ModelOutput, auto_docstring, can_return_tuple from ...utils.generic import OutputRecorder, TransformersKwargs from ..auto import AutoModel from .configuration_sam2_video import Sam2VideoConfig, Sam2VideoMaskDecoderConfig, Sam2VideoPromptEncoderConfig @@ -1101,38 +1101,6 @@ def forward( return vision_features, vision_pos_enc -@dataclass -class BaseModelOutputWithFeatureMaps(ModelOutput): - """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, - sequence_length)`. - - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention - heads. - feature_maps (`list[torch.Tensor]`): - List of feature maps from different layers of the model. - feature_maps_position_embeddings (`list[torch.Tensor]`): - List of position embeddings corresponding to the feature maps. - """ - - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None - attentions: Optional[tuple[torch.FloatTensor, ...]] = None - feature_maps: Optional[list[torch.Tensor]] = None - feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None - - @dataclass @auto_docstring(custom_intro="Base class for the vision encoder's outputs.") class Sam2VideoVisionEncoderOutput(ModelOutput): @@ -1848,10 +1816,10 @@ def forward( frame_idx=frame_idx, ) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -1865,8 +1833,6 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -1875,10 +1841,7 @@ def get_image_features( - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. """ - vision_outputs: Sam2VideoVisionEncoderOutput = self.vision_encoder( - pixel_values, - **kwargs, - ) + vision_outputs: Sam2VideoVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding @@ -1895,16 +1858,12 @@ def get_image_features( feature_map_position_embedding.flatten(2).permute(2, 0, 1) for feature_map_position_embedding in feature_maps_position_embeddings ] + vision_outputs.fpn_hidden_states = feature_maps + vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - if return_dict: - return BaseModelOutputWithFeatureMaps( - last_hidden_state=vision_outputs.last_hidden_state, - attentions=vision_outputs.attentions, - feature_maps=feature_maps, - feature_maps_position_embeddings=feature_maps_position_embeddings, - ) - - return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions + # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the + # fpn hidden states/position encoding order, still have to double-check + return vision_outputs def _prepare_vision_features( self, diff --git a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py index bfb73261e3c0..2c2e6be99dff 100644 --- a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py +++ b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py @@ -37,7 +37,7 @@ from ...modeling_outputs import BaseModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ModelOutput, auto_docstring +from ...utils import ModelOutput, auto_docstring, can_return_tuple from ...utils.generic import TransformersKwargs, check_model_inputs from ..auto import AutoModel from .configuration_sam3_tracker import Sam3TrackerConfig, Sam3TrackerMaskDecoderConfig, Sam3TrackerPromptEncoderConfig @@ -724,38 +724,6 @@ def _dynamic_multimask_via_stability(self, all_mask_logits, all_iou_scores): return mask_logits_out, iou_scores_out -@dataclass -class BaseModelOutputWithFeatureMaps(ModelOutput): - """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, - sequence_length)`. - - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention - heads. - feature_maps (`list[torch.Tensor]`): - List of feature maps from different layers of the model. - feature_maps_position_embeddings (`list[torch.Tensor]`): - List of position embeddings corresponding to the feature maps. - """ - - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None - attentions: Optional[tuple[torch.FloatTensor, ...]] = None - feature_maps: Optional[list[torch.Tensor]] = None - feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None - - @dataclass @auto_docstring(custom_intro="Base class for the vision encoder's outputs.") class Sam3TrackerVisionEncoderOutput(ModelOutput): @@ -1080,10 +1048,10 @@ def forward( vision_attentions=vision_attentions, ) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -1097,8 +1065,6 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -1107,10 +1073,7 @@ def get_image_features( - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. """ - vision_outputs: Sam3TrackerVisionEncoderOutput = self.vision_encoder( - pixel_values, - **kwargs, - ) + vision_outputs: Sam3TrackerVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding @@ -1127,16 +1090,12 @@ def get_image_features( feature_map_position_embedding.flatten(2).permute(2, 0, 1) for feature_map_position_embedding in feature_maps_position_embeddings ] + vision_outputs.fpn_hidden_states = feature_maps + vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - if return_dict: - return BaseModelOutputWithFeatureMaps( - last_hidden_state=vision_outputs.last_hidden_state, - attentions=vision_outputs.attentions, - feature_maps=feature_maps, - feature_maps_position_embeddings=feature_maps_position_embeddings, - ) - - return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions + # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the + # fpn hidden states/position encoding order, still have to double-check + return vision_outputs __all__ = ["Sam3TrackerModel", "Sam3TrackerPreTrainedModel"] diff --git a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py index 20b9bcfaaa0f..cf5a33ba8c60 100644 --- a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py @@ -40,8 +40,8 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import compile_compatible_method_lru_cache -from ...utils import auto_docstring -from ...utils.generic import OutputRecorder, TransformersKwargs +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils.generic import OutputRecorder from ..auto import AutoModel from .configuration_sam3_tracker_video import ( Sam3TrackerVideoConfig, @@ -1867,10 +1867,10 @@ def forward( frame_idx=frame_idx, ) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -1884,8 +1884,6 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -1894,10 +1892,7 @@ def get_image_features( - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. """ - vision_outputs: Sam3TrackerVideoVisionEncoderOutput = self.vision_encoder( - pixel_values, - **kwargs, - ) + vision_outputs: Sam3TrackerVideoVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding @@ -1914,16 +1909,12 @@ def get_image_features( feature_map_position_embedding.flatten(2).permute(2, 0, 1) for feature_map_position_embedding in feature_maps_position_embeddings[:-1] ] + vision_outputs.fpn_hidden_states = feature_maps + vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - if return_dict: - return BaseModelOutputWithFeatureMaps( - last_hidden_state=vision_outputs.last_hidden_state, - attentions=vision_outputs.attentions, - feature_maps=feature_maps, - feature_maps_position_embeddings=feature_maps_position_embeddings, - ) - - return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions + # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the + # fpn hidden states/position encoding order, still have to double-check + return vision_outputs def _prepare_vision_features( self, diff --git a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py index e603a1ab0fb6..02cae27015a6 100644 --- a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py @@ -21,7 +21,7 @@ from ...configuration_utils import PreTrainedConfig from ...modeling_outputs import ModelOutput from ...processing_utils import Unpack -from ...utils.generic import TransformersKwargs +from ...utils import TransformersKwargs, can_return_tuple from ..auto import CONFIG_MAPPING, AutoConfig, AutoModel from ..sam2_video.configuration_sam2_video import Sam2VideoMaskDecoderConfig, Sam2VideoPromptEncoderConfig from ..sam2_video.modeling_sam2_video import ( @@ -557,10 +557,10 @@ def __init__(self, config: Sam3TrackerVideoConfig, remove_vision_encoder: bool = self.post_init() + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, - return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple[ list[torch.Tensor], @@ -574,8 +574,6 @@ def get_image_features( Args: pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `tuple`: A tuple containing: @@ -584,10 +582,7 @@ def get_image_features( - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. """ - vision_outputs: Sam3TrackerVideoVisionEncoderOutput = self.vision_encoder( - pixel_values, - **kwargs, - ) + vision_outputs: Sam3TrackerVideoVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding @@ -604,16 +599,12 @@ def get_image_features( feature_map_position_embedding.flatten(2).permute(2, 0, 1) for feature_map_position_embedding in feature_maps_position_embeddings[:-1] ] + vision_outputs.fpn_hidden_states = feature_maps + vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - if return_dict: - return BaseModelOutputWithFeatureMaps( - last_hidden_state=vision_outputs.last_hidden_state, - attentions=vision_outputs.attentions, - feature_maps=feature_maps, - feature_maps_position_embeddings=feature_maps_position_embeddings, - ) - - return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions + # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the + # fpn hidden states/position encoding order, still have to double-check + return vision_outputs __all__ = [ diff --git a/src/transformers/models/smolvlm/modeling_smolvlm.py b/src/transformers/models/smolvlm/modeling_smolvlm.py index 72e7c7b7a944..adac23d26098 100644 --- a/src/transformers/models/smolvlm/modeling_smolvlm.py +++ b/src/transformers/models/smolvlm/modeling_smolvlm.py @@ -517,11 +517,12 @@ def inputs_merger( merged_embeds = torch.where(image_mask.unsqueeze(-1), image_embeds, inputs_embeds) return merged_embeds + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -531,9 +532,6 @@ def get_image_features( The tensors corresponding to the input images. pixel_attention_mask (`torch.LongTensor`, *optional*): The attention mask indicating padded regions in the image. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - """ batch_size, num_images, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility @@ -565,19 +563,16 @@ def get_image_features( patch_attention_mask = (patches_subgrid.sum(dim=(-1, -2)) > 0).bool() # Get sequence from the vision encoder - image_hidden_states = self.vision_model(pixel_values=pixel_values, patch_attention_mask=patch_attention_mask) - image_hidden_states = image_hidden_states.last_hidden_state + image_outputs = self.vision_model( + pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, **kwargs + ) + image_hidden_states = image_outputs.last_hidden_state # Modality projection & resampling image_features = self.connector(image_hidden_states) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_hidden_states, - pooler_output=image_features, - ) - - return image_features + return image_outputs @can_return_tuple @auto_docstring( diff --git a/src/transformers/models/smolvlm/modular_smolvlm.py b/src/transformers/models/smolvlm/modular_smolvlm.py index 98457d4a1bf7..6b78ea7ea181 100644 --- a/src/transformers/models/smolvlm/modular_smolvlm.py +++ b/src/transformers/models/smolvlm/modular_smolvlm.py @@ -21,9 +21,8 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationConfig from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack -from ...utils import auto_docstring, can_return_tuple, logging +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from ..idefics3.configuration_idefics3 import Idefics3Config, Idefics3VisionConfig from ..idefics3.image_processing_idefics3 import Idefics3ImageProcessor from ..idefics3.image_processing_idefics3_fast import Idefics3ImageProcessorFast @@ -193,11 +192,12 @@ def inputs_merger( merged_embeds = torch.where(image_mask.unsqueeze(-1), image_embeds, inputs_embeds) return merged_embeds + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -207,9 +207,6 @@ def get_image_features( The tensors corresponding to the input images. pixel_attention_mask (`torch.LongTensor`, *optional*): The attention mask indicating padded regions in the image. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - """ batch_size, num_images, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility @@ -241,19 +238,16 @@ def get_image_features( patch_attention_mask = (patches_subgrid.sum(dim=(-1, -2)) > 0).bool() # Get sequence from the vision encoder - image_hidden_states = self.vision_model(pixel_values=pixel_values, patch_attention_mask=patch_attention_mask) - image_hidden_states = image_hidden_states.last_hidden_state + image_outputs = self.vision_model( + pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, **kwargs + ) + image_hidden_states = image_outputs.last_hidden_state # Modality projection & resampling image_features = self.connector(image_hidden_states) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_hidden_states, - pooler_output=image_features, - ) - - return image_features + return image_outputs @can_return_tuple @auto_docstring( diff --git a/src/transformers/models/t5gemma2/modeling_t5gemma2.py b/src/transformers/models/t5gemma2/modeling_t5gemma2.py index 66e5d0bdcde8..6b6a512af3ea 100644 --- a/src/transformers/models/t5gemma2/modeling_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modeling_t5gemma2.py @@ -37,7 +37,6 @@ from ...modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions, - BaseModelOutputWithPooling, Seq2SeqLMOutput, Seq2SeqModelOutput, SequenceClassifierOutput, @@ -801,25 +800,19 @@ def __init__( # Initialize weights and apply final processing self.post_init() - def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False) -> torch.Tensor: + @can_return_tuple + def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> torch.Tensor: """ - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Convert pixel image to image features via the encoder and projector. """ # pixel_values: (batch_size, channels, height, width) # image_features: Image feature tensor of shape (num_images, image_length, embed_dim). - vision_outputs = self.vision_tower(pixel_values=pixel_values).last_hidden_state - image_features = self.multi_modal_projector(vision_outputs) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) + last_hidden_state = vision_outputs.last_hidden_state + image_features = self.multi_modal_projector(last_hidden_state) + vision_outputs.pooler_output = image_features - return image_features + return vision_outputs def get_image_placeholder_mask( self, @@ -1223,8 +1216,8 @@ def get_encoder(self): def get_decoder(self): return self.model.get_decoder() - def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False): - return self.get_encoder().get_image_features(pixel_values, return_dict=return_dict) + def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> torch.Tensor: + return self.get_encoder().get_image_features(pixel_values, **kwargs) @property def vision_tower(self): diff --git a/src/transformers/models/t5gemma2/modular_t5gemma2.py b/src/transformers/models/t5gemma2/modular_t5gemma2.py index 78b85d6d710e..e8d0447d82aa 100644 --- a/src/transformers/models/t5gemma2/modular_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modular_t5gemma2.py @@ -29,7 +29,6 @@ from ...modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions, - BaseModelOutputWithPooling, Seq2SeqLMOutput, Seq2SeqModelOutput, SequenceClassifierOutput, @@ -580,25 +579,19 @@ def __init__( # Initialize weights and apply final processing self.post_init() - def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False) -> torch.Tensor: + @can_return_tuple + def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> torch.Tensor: """ - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Convert pixel image to image features via the encoder and projector. """ # pixel_values: (batch_size, channels, height, width) # image_features: Image feature tensor of shape (num_images, image_length, embed_dim). - vision_outputs = self.vision_tower(pixel_values=pixel_values).last_hidden_state - image_features = self.multi_modal_projector(vision_outputs) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - ) + vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) + last_hidden_state = vision_outputs.last_hidden_state + image_features = self.multi_modal_projector(last_hidden_state) + vision_outputs.pooler_output = image_features - return image_features + return vision_outputs def get_image_placeholder_mask( self, @@ -989,8 +982,8 @@ def get_encoder(self): def get_decoder(self): return self.model.get_decoder() - def get_image_features(self, pixel_values: torch.Tensor, return_dict: bool = False): - return self.get_encoder().get_image_features(pixel_values, return_dict=return_dict) + def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> torch.Tensor: + return self.get_encoder().get_image_features(pixel_values, **kwargs) @property def vision_tower(self): diff --git a/src/transformers/models/video_llama_3/modeling_video_llama_3.py b/src/transformers/models/video_llama_3/modeling_video_llama_3.py index 1fd7c3799720..59f8f8e99d4e 100644 --- a/src/transformers/models/video_llama_3/modeling_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modeling_video_llama_3.py @@ -29,7 +29,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutput, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -537,12 +537,13 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor, video_merge_sizes: torch.LongTensor, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -554,22 +555,21 @@ def get_video_features( The temporal, height and width of feature shape of each video in LLM. video_merge_sizes (`torch.Tensor` of shape `(num_videos,)`): The spatial downsampling ratio of each video feature. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ return self.get_image_features( pixel_values=pixel_values_videos, image_grid_thw=video_grid_thw, image_merge_sizes=video_merge_sizes, - return_dict=return_dict, + **kwargs, ) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor, image_merge_sizes: torch.LongTensor, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -581,28 +581,21 @@ def get_image_features( The temporal, height and width of feature shape of each image in LLM. image_merge_sizes (`torch.Tensor` of shape `(num_images,)`): The spatial downsampling ratio of each image feature. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ vision_outputs = self.vision_model( pixel_values=pixel_values, grid_thw=image_grid_thw, merge_sizes=image_merge_sizes, - return_dict=True, + **kwargs, ) last_hidden_state = vision_outputs.last_hidden_state image_embeds = self.projector(last_hidden_state) split_sizes = image_grid_thw.prod(dim=1) // (image_merge_sizes**2) image_embeds = torch.split(image_embeds, split_sizes.tolist()) + vision_outputs.pooler_output = image_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=last_hidden_state, - pooler_output=image_embeds, - ) - - return image_embeds + return vision_outputs def get_placeholder_mask( self, @@ -774,12 +767,22 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], ): - return self.model.get_video_features(pixel_values_videos, video_grid_thw) + return self.model.get_video_features( + pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs + ) - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): - return self.model.get_image_features(pixel_values, image_grid_thw) + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ): + return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/video_llama_3/modular_video_llama_3.py b/src/transformers/models/video_llama_3/modular_video_llama_3.py index 8478155f771a..04d23b7a0231 100644 --- a/src/transformers/models/video_llama_3/modular_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modular_video_llama_3.py @@ -36,7 +36,7 @@ valid_images, validate_preprocess_arguments, ) -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutput, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput @@ -583,12 +583,13 @@ def __init__(self, config: VideoLlama3Config): def get_rope_index(self): raise AttributeError("Not needed for VideoLLaMA3") + @can_return_tuple def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor, video_merge_sizes: torch.LongTensor, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -600,22 +601,21 @@ def get_video_features( The temporal, height and width of feature shape of each video in LLM. video_merge_sizes (`torch.Tensor` of shape `(num_videos,)`): The spatial downsampling ratio of each video feature. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ return self.get_image_features( pixel_values=pixel_values_videos, image_grid_thw=video_grid_thw, image_merge_sizes=video_merge_sizes, - return_dict=return_dict, + **kwargs, ) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor, image_merge_sizes: torch.LongTensor, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -627,28 +627,21 @@ def get_image_features( The temporal, height and width of feature shape of each image in LLM. image_merge_sizes (`torch.Tensor` of shape `(num_images,)`): The spatial downsampling ratio of each image feature. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ vision_outputs = self.vision_model( pixel_values=pixel_values, grid_thw=image_grid_thw, merge_sizes=image_merge_sizes, - return_dict=True, + **kwargs, ) last_hidden_state = vision_outputs.last_hidden_state image_embeds = self.projector(last_hidden_state) split_sizes = image_grid_thw.prod(dim=1) // (image_merge_sizes**2) image_embeds = torch.split(image_embeds, split_sizes.tolist()) + vision_outputs.pooler_output = image_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=last_hidden_state, - pooler_output=image_embeds, - ) - - return image_embeds + return vision_outputs @can_return_tuple def forward( diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index 92608503ba29..faea5fdba6cd 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -25,7 +25,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -172,12 +172,13 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_image_features( self, pixel_values_images: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -192,8 +193,6 @@ def get_image_features( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). @@ -210,36 +209,32 @@ def get_image_features( if vision_feature_select_strategy not in ["default", "full"]: raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}") - image_outputs = self.image_tower(pixel_values_images, output_hidden_states=True) + image_outputs = self.image_tower(pixel_values_images, output_hidden_states=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): - image_outputs = image_outputs.hidden_states[vision_feature_layer] + selected_hidden_state = image_outputs.hidden_states[vision_feature_layer] if vision_feature_select_strategy == "default": - image_outputs = image_outputs[:, 1:] + selected_hidden_state = selected_hidden_state[:, 1:] else: hs_pool = [image_outputs.hidden_states[layer_idx] for layer_idx in vision_feature_layer] # For default; crop CLS from each hidden state in the hidden state pool if vision_feature_select_strategy == "default": hs_pool = [hs[:, 1:] for hs in hs_pool] - image_outputs = torch.cat(hs_pool, dim=-1) + selected_hidden_state = torch.cat(hs_pool, dim=-1) - image_features = self.multi_modal_projector(image_outputs) + image_features = self.multi_modal_projector(selected_hidden_state) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return image_outputs + @can_return_tuple def get_video_features( self, pixel_values_videos: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains video last hidden states from the vision tower and apply multimodal projection. @@ -251,8 +246,6 @@ def get_video_features( The index of the layer to select the vision feature. If multiple indices are provided, the vision feature of the corresponding indices will be concatenated to form the vision features. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: video_features (`torch.Tensor`): Video feature tensor of shape `(num_videos * num_frames, image_length, embed_dim)`). @@ -276,13 +269,9 @@ def get_video_features( video_features = torch.cat(hs_pool, dim=-1) video_features = self.multi_modal_projector(video_features) + video_outputs.pooler_output = video_features - if return_dict: - video_outputs.pooler_output = video_features - return video_outputs - - # NOTE: @Tom backwards incompatibility - return video_features + return video_outputs def get_placeholder_mask( self, @@ -450,13 +439,13 @@ def get_image_features( pixel_values_images: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( pixel_values_images=pixel_values_images, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - return_dict=return_dict, + **kwargs, ) @can_return_tuple diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index e4691dc3bb5c..e8b4489d478e 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -28,9 +28,10 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import auto_docstring, can_return_tuple +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ..auto import AutoModel from .configuration_vipllava import VipLlavaConfig @@ -150,11 +151,12 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -165,8 +167,6 @@ def get_image_features( vision_feature_layers (`Union[int, list[int]]`): The vision feature layer, or the list of indexes of the layers to select the vision feature. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). @@ -174,7 +174,7 @@ def get_image_features( vision_feature_layers = ( vision_feature_layers if vision_feature_layers is not None else self.config.vision_feature_layers ) - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) # If multiple feature layers are provided (which is usually the case) # then the image features are concatenated after the CLS is removed. @@ -185,14 +185,9 @@ def get_image_features( image_features = [image_outputs.hidden_states[index][:, 1:] for index in vision_feature_layers] image_features = torch.cat(image_features, dim=-1) image_features = self.multi_modal_projector(image_features) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -321,10 +316,10 @@ def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( - pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, return_dict=return_dict + pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, **kwargs ) @can_return_tuple diff --git a/src/transformers/models/vipllava/modular_vipllava.py b/src/transformers/models/vipllava/modular_vipllava.py index 81b7235990bc..faea2f645ff5 100644 --- a/src/transformers/models/vipllava/modular_vipllava.py +++ b/src/transformers/models/vipllava/modular_vipllava.py @@ -28,8 +28,8 @@ from ...activations import ACT2FN from ...cache_utils import Cache -from ...modeling_outputs import BaseModelOutputWithPooling -from ...utils import auto_docstring, logging +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from .configuration_vipllava import VipLlavaConfig @@ -73,11 +73,12 @@ class VipLlavaPreTrainedModel(LlavaPreTrainedModel): class VipLlavaModel(LlavaModel): + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -88,8 +89,6 @@ def get_image_features( vision_feature_layers (`Union[int, list[int]]`): The vision feature layer, or the list of indexes of the layers to select the vision feature. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). @@ -97,7 +96,7 @@ def get_image_features( vision_feature_layers = ( vision_feature_layers if vision_feature_layers is not None else self.config.vision_feature_layers ) - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) # If multiple feature layers are provided (which is usually the case) # then the image features are concatenated after the CLS is removed. @@ -108,14 +107,9 @@ def get_image_features( image_features = [image_outputs.hidden_states[index][:, 1:] for index in vision_feature_layers] image_features = torch.cat(image_features, dim=-1) image_features = self.multi_modal_projector(image_features) + image_outputs.pooler_output = image_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_outputs.last_hidden_state, - pooler_output=image_features, - ) - - return image_features + return image_outputs @auto_docstring def forward( @@ -192,10 +186,10 @@ def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_image_features( - pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, return_dict=return_dict + pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, **kwargs ) def forward( From 00aa0f5dc1ccd968335257cb871d41ee9c0bc21e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 16 Dec 2025 16:15:34 +0100 Subject: [PATCH 019/131] Update all get_audio_features, some edge cases handled (e.g. gemma3n) --- .../audioflamingo3/modeling_audioflamingo3.py | 23 +++---- .../audioflamingo3/modular_audioflamingo3.py | 23 +++---- .../models/gemma3n/modeling_gemma3n.py | 64 ++++++++++++++---- .../models/gemma3n/modular_gemma3n.py | 65 +++++++++++++++---- .../granite_speech/modeling_granite_speech.py | 26 ++++---- .../qwen2_5_omni/modeling_qwen2_5_omni.py | 28 +++----- .../qwen2_5_omni/modular_qwen2_5_omni.py | 27 +++----- .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 12 ++-- .../qwen3_omni_moe/modular_qwen3_omni_moe.py | 12 ++-- .../models/voxtral/modeling_voxtral.py | 22 ++----- .../models/voxtral/modular_voxtral.py | 16 ++--- 11 files changed, 177 insertions(+), 141 deletions(-) diff --git a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py index 9406116cf87e..9a234ae0ab7f 100644 --- a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py @@ -33,7 +33,7 @@ from ...masking_utils import create_bidirectional_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutput, CausalLMOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -443,8 +443,12 @@ def set_decoder(self, decoder): def get_decoder(self): return self.language_model.get_decoder() + @can_return_tuple def get_audio_features( - self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor, return_dict: bool = False + self, + input_features: torch.FloatTensor, + input_features_mask: torch.Tensor, + **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. @@ -457,8 +461,6 @@ def get_audio_features( and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`): Mask to avoid performing attention on padded feature indices. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `torch.FloatTensor`: @@ -466,21 +468,16 @@ def get_audio_features( """ # Encode audio - encoder_output = self.audio_tower(input_features, input_features_mask=input_features_mask) - audio_embeds = self.multi_modal_projector(encoder_output.last_hidden_state) + audio_output = self.audio_tower(input_features, input_features_mask=input_features_mask) + audio_embeds = self.multi_modal_projector(audio_output.last_hidden_state) # Mask according to avg pooling (which is after attention blocks) post_lengths = (input_features_mask.sum(-1) - 2) // 2 + 1 valid_mask = torch.arange(audio_embeds.shape[1], device=post_lengths.device)[None, :] < post_lengths[:, None] audio_embeds = audio_embeds[valid_mask.to(audio_embeds.device)] + audio_output.pooler_output = audio_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=encoder_output.last_hidden_state, - pooler_output=audio_embeds, - ) - - return audio_embeds + return audio_output @can_return_tuple @auto_docstring diff --git a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py index 8c60ca29f965..c01778549424 100644 --- a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py @@ -22,7 +22,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...masking_utils import create_bidirectional_mask -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutput, CausalLMOutputWithPast from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from ..qwen2_audio.modeling_qwen2_audio import ( @@ -140,8 +140,12 @@ class AudioFlamingo3ForConditionalGeneration(VoxtralForConditionalGeneration): def __init__(self, config): super().__init__(config) + @can_return_tuple def get_audio_features( - self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor, return_dict: bool = False + self, + input_features: torch.FloatTensor, + input_features_mask: torch.Tensor, + **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. @@ -154,8 +158,6 @@ def get_audio_features( and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`): Mask to avoid performing attention on padded feature indices. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: `torch.FloatTensor`: @@ -163,21 +165,16 @@ def get_audio_features( """ # Encode audio - encoder_output = self.audio_tower(input_features, input_features_mask=input_features_mask) - audio_embeds = self.multi_modal_projector(encoder_output.last_hidden_state) + audio_output = self.audio_tower(input_features, input_features_mask=input_features_mask) + audio_embeds = self.multi_modal_projector(audio_output.last_hidden_state) # Mask according to avg pooling (which is after attention blocks) post_lengths = (input_features_mask.sum(-1) - 2) // 2 + 1 valid_mask = torch.arange(audio_embeds.shape[1], device=post_lengths.device)[None, :] < post_lengths[:, None] audio_embeds = audio_embeds[valid_mask.to(audio_embeds.device)] + audio_output.pooler_output = audio_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=encoder_output.last_hidden_state, - pooler_output=audio_embeds, - ) - - return audio_embeds + return audio_output def get_audio_embeds(self): raise NotImplementedError("This method is not supported for AudioFlamingo3ForConditionalGeneration.") diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index 8e881e8fd036..bb8faacc4570 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -45,6 +45,40 @@ from .configuration_gemma3n import Gemma3nAudioConfig, Gemma3nConfig, Gemma3nTextConfig, Gemma3nVisionConfig +@dataclass +class Gemma3nAudioEncoderModelOutput(BaseModelOutputWithPooling): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, audio_soft_tokens_per_image, hidden_size)`): + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + audio_mel_mask (`torch.FloatTensor`, *optional*): + A torch.BoolTensor of shape `(batch_size, num_frames)` + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + hidden_states: Optional[tuple[torch.FloatTensor, ...]] = None + attentions: Optional[tuple[torch.FloatTensor, ...]] = None + audio_mel_mask: Optional[torch.BoolTensor] = None + + @dataclass @auto_docstring( custom_intro=""" @@ -920,8 +954,9 @@ def __init__(self, config: Gemma3nAudioConfig): [Gemma3nAudioConformerBlock(config) for _ in range(config.conf_num_hidden_layers)] ) + @check_model_inputs(tie_last_hidden_states=False) def forward( - self, audio_mel: torch.Tensor, audio_mel_mask: torch.BoolTensor, **kwargs + self, audio_mel: torch.Tensor, audio_mel_mask: torch.BoolTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple[torch.Tensor, torch.BoolTensor]: """Encodes a batch of MELs. @@ -973,7 +1008,10 @@ def forward( current_mask = current_mask[:, :: self.config.conf_reduction_factor] audio_encodings = audio_encodings.masked_fill(current_mask.unsqueeze(-1), 0.0) - return audio_encodings, current_mask + return Gemma3nAudioEncoderModelOutput( + last_hidden_state=audio_encodings, + audio_mel_mask=current_mask, + ) class Gemma3nTextScaledWordEmbedding(nn.Embedding): @@ -2069,7 +2107,9 @@ def forward( # Merge text and audio if input_features is not None and input_features_mask is not None: - audio_features, audio_mask = self.get_audio_features(input_features, ~input_features_mask) + audio_features = self.get_audio_features(input_features, ~input_features_mask) + audio_features = audio_features.pooler_output + audio_mask = audio_features.audio_mel_mask # The Gemma3nProcessor expects all audio will be 30s in length and inserts 188 audio soft tokens into the # text to account for this. However, the audio preprocessing and encoder do not gurarantee they will @@ -2116,7 +2156,10 @@ def forward( ) def get_audio_features( - self, input_features: torch.Tensor, input_features_mask: torch.Tensor, return_dict: bool = False + self, + input_features: torch.Tensor, + input_features_mask: torch.Tensor, + **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: """ Projects the last hidden state from the audio encoder into language model space. @@ -2130,16 +2173,11 @@ def get_audio_features( Returns: audio_features (`torch.Tensor`): Audio feature tensor of shape `(num_images, audio_length, embed_dim)`). """ - audio_outputs, audio_mask = self.audio_tower(input_features, input_features_mask) - audio_embeds = self.embed_audio(inputs_embeds=audio_outputs) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=audio_outputs.last_hidden_state, - pooler_output=audio_embeds, - ) + audio_outputs: Gemma3nAudioEncoderModelOutput = self.audio_tower(input_features, input_features_mask) + audio_embeds = self.embed_audio(inputs_embeds=audio_outputs.last_hidden_state) + audio_outputs.pooler_output = audio_embeds - return audio_embeds, audio_mask + return audio_outputs @auto_docstring( diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index b9e8f895ed9c..d185f3685823 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -15,6 +15,7 @@ # limitations under the License. import math from collections.abc import Callable, Sequence +from dataclasses import dataclass from typing import Any, Optional, Union import torch @@ -622,6 +623,40 @@ def __init__( self.initializer_range = initializer_range +@dataclass +class Gemma3nAudioEncoderModelOutput(BaseModelOutputWithPooling): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, audio_soft_tokens_per_image, hidden_size)`): + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state of the first token of the sequence (classification token) after further processing + through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns + the classification token after processing through a linear layer and a tanh activation function. The linear + layer weights are trained from the next sentence prediction (classification) objective during pretraining. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + audio_mel_mask (`torch.FloatTensor`, *optional*): + A torch.BoolTensor of shape `(batch_size, num_frames)` + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + pooler_output: Optional[torch.FloatTensor] = None + hidden_states: Optional[tuple[torch.FloatTensor, ...]] = None + attentions: Optional[tuple[torch.FloatTensor, ...]] = None + audio_mel_mask: Optional[torch.BoolTensor] = None + + class Gemma3nModelOutputWithPast(PaligemmaModelOutputWithPast): r""" past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): @@ -1473,8 +1508,9 @@ def __init__(self, config: Gemma3nAudioConfig): [Gemma3nAudioConformerBlock(config) for _ in range(config.conf_num_hidden_layers)] ) + @check_model_inputs(tie_last_hidden_states=False) def forward( - self, audio_mel: torch.Tensor, audio_mel_mask: torch.BoolTensor, **kwargs + self, audio_mel: torch.Tensor, audio_mel_mask: torch.BoolTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple[torch.Tensor, torch.BoolTensor]: """Encodes a batch of MELs. @@ -1526,7 +1562,10 @@ def forward( current_mask = current_mask[:, :: self.config.conf_reduction_factor] audio_encodings = audio_encodings.masked_fill(current_mask.unsqueeze(-1), 0.0) - return audio_encodings, current_mask + return Gemma3nAudioEncoderModelOutput( + last_hidden_state=audio_encodings, + audio_mel_mask=current_mask, + ) # ==== Language Model ==== @@ -2327,7 +2366,9 @@ def forward( # Merge text and audio if input_features is not None and input_features_mask is not None: - audio_features, audio_mask = self.get_audio_features(input_features, ~input_features_mask) + audio_features = self.get_audio_features(input_features, ~input_features_mask) + audio_features = audio_features.pooler_output + audio_mask = audio_features.audio_mel_mask # The Gemma3nProcessor expects all audio will be 30s in length and inserts 188 audio soft tokens into the # text to account for this. However, the audio preprocessing and encoder do not gurarantee they will @@ -2374,7 +2415,10 @@ def forward( ) def get_audio_features( - self, input_features: torch.Tensor, input_features_mask: torch.Tensor, return_dict: bool = False + self, + input_features: torch.Tensor, + input_features_mask: torch.Tensor, + **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: """ Projects the last hidden state from the audio encoder into language model space. @@ -2388,16 +2432,11 @@ def get_audio_features( Returns: audio_features (`torch.Tensor`): Audio feature tensor of shape `(num_images, audio_length, embed_dim)`). """ - audio_outputs, audio_mask = self.audio_tower(input_features, input_features_mask) - audio_embeds = self.embed_audio(inputs_embeds=audio_outputs) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=audio_outputs.last_hidden_state, - pooler_output=audio_embeds, - ) + audio_outputs: Gemma3nAudioEncoderModelOutput = self.audio_tower(input_features, input_features_mask) + audio_embeds = self.embed_audio(inputs_embeds=audio_outputs.last_hidden_state) + audio_outputs.pooler_output = audio_embeds - return audio_embeds, audio_mask + return audio_outputs @auto_docstring( diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index ff90ca1669fc..d7d40a93dec7 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -26,7 +26,9 @@ from ...generation import GenerationMixin from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import auto_docstring, is_peft_available, logging +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_peft_available, logging +from ...utils.generic import check_model_inputs from ..auto import AutoModel, AutoModelForCausalLM from .configuration_granite_speech import GraniteSpeechConfig, GraniteSpeechEncoderConfig @@ -267,7 +269,8 @@ def __init__(self, config: GraniteSpeechEncoderConfig): self.out_mid = nn.Linear(config.output_dim, config.hidden_dim, bias=True) self.num_layers = config.num_layers - def forward(self, hidden_states: torch.Tensor): + @check_model_inputs(tie_last_hidden_states=False) + def forward(self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> BaseModelOutputWithPooling: hidden_states = self.input_linear(hidden_states) for idx, layer in enumerate(self.layers, start=1): hidden_states = layer(hidden_states, attention_dists=self.attention_dists) @@ -276,7 +279,8 @@ def forward(self, hidden_states: torch.Tensor): hidden_states_mid = hidden_states.clone() hidden_states_mid = self.out(hidden_states_mid) hidden_states += self.out_mid(nn.Softmax(dim=-1)(hidden_states_mid)) - return hidden_states + + return BaseModelOutputWithPooling(last_hidden_state=hidden_states) @auto_docstring @@ -334,18 +338,14 @@ def get_input_embeddings(self): def get_output_embeddings(self): return self.language_model.get_output_embeddings() - def get_audio_features(self, input_features: torch.Tensor, return_dict: bool = False) -> torch.Tensor: + @can_return_tuple + def get_audio_features(self, input_features: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> torch.Tensor: """Get the audio features to merged into the multimodal embeddings.""" - encoder_embeds = self.encoder(input_features) - projected_embeds = self.projector(encoder_embeds) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=encoder_embeds, - pooler_output=projected_embeds, - ) + audio_outputs = self.encoder(input_features, **kwargs) + projected_embeds = self.projector(audio_outputs.last_hidden_state) + audio_outputs.pooler_output = projected_embeds - return projected_embeds + return audio_outputs @auto_docstring def forward( diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index bb5e37c87a7e..aae33b172211 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -43,7 +43,7 @@ from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, check_torch_load_is_safe, logging from ...utils.deprecation import deprecate_kwarg -from ...utils.generic import maybe_autocast +from ...utils.generic import check_model_inputs, maybe_autocast from ...utils.hub import cached_file from ..qwen2.modeling_qwen2 import Qwen2RMSNorm from .configuration_qwen2_5_omni import ( @@ -765,14 +765,9 @@ def _prepare_attention_mask(self, inputs_tensor: torch.Tensor, cu_seqlens: torch attention_mask[..., cu_seqlens[i - 1] : cu_seqlens[i], cu_seqlens[i - 1] : cu_seqlens[i]] = 0 return attention_mask + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring - def forward( - self, - input_features, - feature_lens=None, - aftercnn_lens=None, - **kwargs, - ): + def forward(self, input_features, feature_lens=None, aftercnn_lens=None, **kwargs: Unpack[TransformersKwargs]): r""" feature_lens (`torch.LongTensor` of shape `(batch_size,)`): mel length @@ -1175,8 +1170,9 @@ def get_window_index(self, grid_thw): return window_index, cu_window_seqlens + @check_model_inputs(tie_last_hidden_states=False) def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> torch.Tensor: """ Args: @@ -1184,8 +1180,6 @@ def forward( The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -1237,13 +1231,10 @@ def forward( reverse_indices = torch.argsort(window_index) merged_hidden_states = merged_hidden_states[reverse_indices, :] - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=hidden_states, - pooler_output=merged_hidden_states, - ) - - return hidden_states + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) class Qwen2_5OmniRotaryEmbedding(nn.Module): @@ -1799,6 +1790,7 @@ def get_audio_features( input_features, feature_lens=feature_lens, aftercnn_lens=audio_feat_lengths, + **kwargs, ) if audio_outputs.last_hidden_state.shape[0] != sum(audio_output_lengths.tolist()): raise ValueError("length of audio_features should match audio_output_lengths") diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index 318964d161f3..270588b01a72 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -56,6 +56,7 @@ logging, ) from ...utils.deprecation import deprecate_kwarg +from ...utils.generic import check_model_inputs from ...utils.hub import cached_file @@ -1690,14 +1691,9 @@ def _prepare_attention_mask(self, inputs_tensor: torch.Tensor, cu_seqlens: torch attention_mask[..., cu_seqlens[i - 1] : cu_seqlens[i], cu_seqlens[i - 1] : cu_seqlens[i]] = 0 return attention_mask + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring - def forward( - self, - input_features, - feature_lens=None, - aftercnn_lens=None, - **kwargs, - ): + def forward(self, input_features, feature_lens=None, aftercnn_lens=None, **kwargs: Unpack[TransformersKwargs]): r""" feature_lens (`torch.LongTensor` of shape `(batch_size,)`): mel length @@ -1929,8 +1925,9 @@ def __init__(self, config: Qwen2_5OmniVisionEncoderConfig, *inputs, **kwargs) -> super().__init__(config, *inputs, **kwargs) self.blocks = nn.ModuleList([Qwen2_5OmniVisionBlock(config) for _ in range(config.depth)]) + @check_model_inputs(tie_last_hidden_states=False) def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> torch.Tensor: """ Args: @@ -1938,8 +1935,6 @@ def forward( The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -1991,13 +1986,10 @@ def forward( reverse_indices = torch.argsort(window_index) merged_hidden_states = merged_hidden_states[reverse_indices, :] - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=hidden_states, - pooler_output=merged_hidden_states, - ) - - return hidden_states + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) class Qwen2_5OmniRotaryEmbedding(Qwen2VLRotaryEmbedding): @@ -2149,6 +2141,7 @@ def get_audio_features( input_features, feature_lens=feature_lens, aftercnn_lens=audio_feat_lengths, + **kwargs, ) if audio_outputs.last_hidden_state.shape[0] != sum(audio_output_lengths.tolist()): raise ValueError("length of audio_features should match audio_output_lengths") diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 65994ecdde86..653c067046e5 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -732,6 +732,7 @@ def _prepare_attention_mask(self, inputs_tensor: torch.Tensor, cu_seqlens: torch attention_mask[..., cu_seqlens[i - 1] : cu_seqlens[i], cu_seqlens[i - 1] : cu_seqlens[i]] = 0 return attention_mask + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -1982,7 +1983,7 @@ def get_audio_features( input_features: torch.FloatTensor, feature_attention_mask: Optional[torch.LongTensor] = None, audio_feature_lengths: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes audios into continuous embeddings that can be forwarded to the language model. @@ -1994,8 +1995,6 @@ def get_audio_features( Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): The length of feature shape of each audio in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) @@ -2007,13 +2006,10 @@ def get_audio_features( audio_outputs = self.audio_tower( input_features, feature_lens=feature_lens, + **kwargs, ) - audio_features = audio_outputs.last_hidden_state - - if return_dict: - return audio_outputs - return audio_features + return audio_outputs def get_placeholder_mask( self, diff --git a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py index d43fad470fdc..32a101ba31cc 100644 --- a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py @@ -1420,12 +1420,13 @@ def __init__(self, config): self.num_experts_per_tok = config.text_config.num_experts_per_tok self.router_aux_loss_coef = config.text_config.router_aux_loss_coef + @can_return_tuple def get_audio_features( self, input_features: torch.FloatTensor, feature_attention_mask: Optional[torch.LongTensor] = None, audio_feature_lengths: Optional[torch.LongTensor] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes audios into continuous embeddings that can be forwarded to the language model. @@ -1437,8 +1438,6 @@ def get_audio_features( Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): The length of feature shape of each audio in LLM. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) @@ -1450,13 +1449,10 @@ def get_audio_features( audio_outputs = self.audio_tower( input_features, feature_lens=feature_lens, + **kwargs, ) - audio_features = audio_outputs.last_hidden_state - - if return_dict: - return audio_outputs - return audio_features + return audio_outputs @can_return_tuple @auto_docstring diff --git a/src/transformers/models/voxtral/modeling_voxtral.py b/src/transformers/models/voxtral/modeling_voxtral.py index 5d7331951f6b..f649364d21be 100644 --- a/src/transformers/models/voxtral/modeling_voxtral.py +++ b/src/transformers/models/voxtral/modeling_voxtral.py @@ -31,12 +31,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import ( - BaseModelOutput, - BaseModelOutputWithPast, - BaseModelOutputWithPooling, - CausalLMOutputWithPast, -) +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -405,9 +400,11 @@ def set_decoder(self, decoder): def get_decoder(self): return self.language_model.get_decoder() - def get_audio_features(self, input_features: torch.FloatTensor, return_dict: bool = False): + @can_return_tuple + def get_audio_features(self, input_features: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. + Args: input_features (`torch.FloatTensor`): Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be @@ -420,18 +417,13 @@ def get_audio_features(self, input_features: torch.FloatTensor, return_dict: boo `torch.FloatTensor`: The audio embeddings. """ - audio_outputs = self.audio_tower(input_features) + audio_outputs = self.audio_tower(input_features, **kwargs) audio_hidden_states = audio_outputs.last_hidden_state audio_hidden_states = audio_hidden_states.reshape(-1, self.config.audio_config.intermediate_size) audio_embeds = self.multi_modal_projector(audio_hidden_states) + audio_outputs.pooler_output = audio_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=audio_outputs.last_hidden_state, - pooler_output=audio_embeds, - ) - - return audio_embeds + return audio_outputs def get_audio_embeds(self, input_features: torch.FloatTensor): warnings.warn( diff --git a/src/transformers/models/voxtral/modular_voxtral.py b/src/transformers/models/voxtral/modular_voxtral.py index 23817e70fa0f..42ace2c52b05 100644 --- a/src/transformers/models/voxtral/modular_voxtral.py +++ b/src/transformers/models/voxtral/modular_voxtral.py @@ -25,7 +25,6 @@ from ...modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPast, - BaseModelOutputWithPooling, CausalLMOutputWithPast, ) from ...processing_utils import Unpack @@ -167,9 +166,11 @@ def set_decoder(self, decoder): def get_decoder(self): return self.language_model.get_decoder() - def get_audio_features(self, input_features: torch.FloatTensor, return_dict: bool = False): + @can_return_tuple + def get_audio_features(self, input_features: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. + Args: input_features (`torch.FloatTensor`): Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be @@ -182,18 +183,13 @@ def get_audio_features(self, input_features: torch.FloatTensor, return_dict: boo `torch.FloatTensor`: The audio embeddings. """ - audio_outputs = self.audio_tower(input_features) + audio_outputs = self.audio_tower(input_features, **kwargs) audio_hidden_states = audio_outputs.last_hidden_state audio_hidden_states = audio_hidden_states.reshape(-1, self.config.audio_config.intermediate_size) audio_embeds = self.multi_modal_projector(audio_hidden_states) + audio_outputs.pooler_output = audio_embeds - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=audio_outputs.last_hidden_state, - pooler_output=audio_embeds, - ) - - return audio_embeds + return audio_outputs def get_audio_embeds(self, input_features: torch.FloatTensor): warnings.warn( From 1ccbf5a3c9ad3e1aaef9c656bb7a938c4eacae70 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 16 Dec 2025 16:31:53 +0100 Subject: [PATCH 020/131] Update most get_video_features, some edge case remain, e.g. instructblipvideo --- .../modeling_instructblipvideo.py | 3 +++ .../modular_instructblipvideo.py | 3 +++ .../modeling_llava_next_video.py | 27 ++++++++----------- .../modular_llava_next_video.py | 26 +++++++----------- .../modeling_llava_onevision.py | 22 ++++++--------- .../modular_llava_onevision.py | 17 ++++-------- .../paddleocr_vl/modeling_paddleocr_vl.py | 1 + .../models/qwen2_5_vl/modeling_qwen2_5_vl.py | 1 + .../models/qwen2_vl/modeling_qwen2_vl.py | 1 + .../models/qwen3_vl/modeling_qwen3_vl.py | 1 + .../qwen3_vl_moe/modeling_qwen3_vl_moe.py | 1 + 11 files changed, 45 insertions(+), 58 deletions(-) diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index f73cedfb0368..806ea7127946 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -1504,6 +1504,9 @@ def get_video_features( # unbatch inputs back, each video-frame gets `num_query_tokens` seq length language_model_inputs = language_model_inputs.reshape(batch_size, self.config.num_query_tokens * frames, -1) + # NOTE: @Tom the easy solution with the new format would be to grab vision_outputs, + # override the pooler_output with language_model_inputs and return that, but we'd lose the + # query_outputs if return_dict: return language_model_inputs, vision_outputs, query_outputs return language_model_inputs diff --git a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py index b5c41db3d25a..55d734fabb34 100644 --- a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py @@ -340,6 +340,9 @@ def get_video_features( # unbatch inputs back, each video-frame gets `num_query_tokens` seq length language_model_inputs = language_model_inputs.reshape(batch_size, self.config.num_query_tokens * frames, -1) + # NOTE: @Tom the easy solution with the new format would be to grab vision_outputs, + # override the pooler_output with language_model_inputs and return that, but we'd lose the + # query_outputs if return_dict: return language_model_inputs, vision_outputs, query_outputs return language_model_inputs diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index b7b011382460..6028c75e9d34 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -33,7 +33,7 @@ from ...generation import GenerationMixin from ...image_processing_utils import select_best_resolution from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -615,12 +615,13 @@ def forward( video_hidden_states=video_features if pixel_values_videos is not None else None, ) + @can_return_tuple def get_video_features( self, pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains video last hidden states from the vision tower and apply multimodal projection. @@ -635,8 +636,7 @@ def get_video_features( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches and are of shape `(num_videos, video_length, embed_dim)`). @@ -652,14 +652,14 @@ def get_video_features( batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channels, height, width) - video_features = self.vision_tower(pixel_values, output_hidden_states=True) + video_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): - selected_video_features = video_features.hidden_states[vision_feature_layer] + selected_video_features = video_outputs.hidden_states[vision_feature_layer] else: - hs_pool = [video_features.hidden_states[layer_idx] for layer_idx in vision_feature_layer] + hs_pool = [video_outputs.hidden_states[layer_idx] for layer_idx in vision_feature_layer] selected_video_features = torch.cat(hs_pool, dim=-1) if vision_feature_select_strategy == "default": @@ -669,14 +669,9 @@ def get_video_features( video_features = self.vision_resampler(selected_video_features) video_features = self.multi_modal_projector(video_features) video_features = torch.split(video_features, frames, dim=0) + video_outputs.pooler_output = video_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=video_features.last_hidden_state, - pooler_output=video_features, - ) - - return video_features + return video_outputs @auto_docstring( @@ -967,13 +962,13 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_video_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - return_dict=return_dict, + **kwargs, ) diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index 0b3c8f26a3eb..9313dba516c4 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -33,7 +33,6 @@ from ...cache_utils import Cache from ...configuration_utils import PreTrainedConfig from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import can_return_tuple, logging from ..auto import CONFIG_MAPPING, AutoConfig @@ -351,12 +350,13 @@ def get_image_features( return image_outputs + @can_return_tuple def get_video_features( self, pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains video last hidden states from the vision tower and apply multimodal projection. @@ -371,8 +371,7 @@ def get_video_features( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. + Returns: video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches and are of shape `(num_videos, video_length, embed_dim)`). @@ -388,14 +387,14 @@ def get_video_features( batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channels, height, width) - video_features = self.vision_tower(pixel_values, output_hidden_states=True) + video_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): - selected_video_features = video_features.hidden_states[vision_feature_layer] + selected_video_features = video_outputs.hidden_states[vision_feature_layer] else: - hs_pool = [video_features.hidden_states[layer_idx] for layer_idx in vision_feature_layer] + hs_pool = [video_outputs.hidden_states[layer_idx] for layer_idx in vision_feature_layer] selected_video_features = torch.cat(hs_pool, dim=-1) if vision_feature_select_strategy == "default": @@ -405,14 +404,9 @@ def get_video_features( video_features = self.vision_resampler(selected_video_features) video_features = self.multi_modal_projector(video_features) video_features = torch.split(video_features, frames, dim=0) + video_outputs.pooler_output = video_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=video_features.last_hidden_state, - pooler_output=video_features, - ) - - return video_features + return video_outputs def get_placeholder_mask( self, @@ -552,13 +546,13 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_video_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - return_dict=return_dict, + **kwargs, ) def forward( diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index cab379df064f..351d7612c5ad 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -33,7 +33,7 @@ from ...generation import GenerationMixin from ...image_processing_utils import select_best_resolution from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -598,12 +598,13 @@ def forward( video_hidden_states=video_features if pixel_values_videos is not None else None, ) + @can_return_tuple def get_video_features( self, pixel_values: torch.FloatTensor, vision_feature_layer: Union[int, list[int]], vision_feature_select_strategy: str, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains video last hidden states from the vision tower, apply multimodal projection and pooling. @@ -618,8 +619,6 @@ def get_video_features( vision_feature_select_strategy (`str`): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches @@ -627,7 +626,7 @@ def get_video_features( """ batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) - vision_outputs = self.vision_tower(pixel_values, output_hidden_states=True) + vision_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them @@ -643,14 +642,9 @@ def get_video_features( video_features = self.apply_pooling(video_features) video_features = video_features.reshape(batch_size, frames * video_features.shape[1], -1) + vision_outputs.pooler_output = video_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=video_features, - ) - - return video_features + return vision_outputs def apply_pooling(self, image_features): height = width = self.config.vision_config.image_size // self.config.vision_config.patch_size @@ -946,13 +940,13 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): return self.model.get_video_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - return_dict=return_dict, + **kwargs, ) diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index 62890a7a6b6f..56fdaeb22a26 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -46,7 +46,6 @@ get_image_size, ) from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import ( TensorType, @@ -407,12 +406,13 @@ def get_image_features( return image_outputs + @can_return_tuple def get_video_features( self, pixel_values: torch.FloatTensor, vision_feature_layer: Union[int, list[int]], vision_feature_select_strategy: str, - return_dict: bool = False, + **kwargs: Unpack[TransformersKwargs], ): """ Obtains video last hidden states from the vision tower, apply multimodal projection and pooling. @@ -427,8 +427,6 @@ def get_video_features( vision_feature_select_strategy (`str`): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches @@ -436,7 +434,7 @@ def get_video_features( """ batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) - vision_outputs = self.vision_tower(pixel_values, output_hidden_states=True) + vision_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them @@ -452,14 +450,9 @@ def get_video_features( video_features = self.apply_pooling(video_features) video_features = video_features.reshape(batch_size, frames * video_features.shape[1], -1) + vision_outputs.pooler_output = video_features - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=video_features, - ) - - return video_features + return vision_outputs def forward( self, diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index 03748e557b78..fde95d05c2e5 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -1184,6 +1184,7 @@ def get_rope_index( return position_ids, mrope_position_deltas + @can_return_tuple def get_video_features( self, pixel_values_videos: torch.FloatTensor, diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index 62926da32bd0..2d49b0ef675a 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -1169,6 +1169,7 @@ def get_rope_index( return position_ids, mrope_position_deltas + @can_return_tuple def get_video_features( self, pixel_values_videos: torch.FloatTensor, diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 96e4a0c50ff1..e162efa73753 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -1104,6 +1104,7 @@ def get_rope_index( return position_ids, mrope_position_deltas + @can_return_tuple def get_video_features( self, pixel_values_videos: torch.FloatTensor, diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index 9bed63e837e9..a83f682d68ef 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -1087,6 +1087,7 @@ def get_rope_index( return position_ids, mrope_position_deltas + @can_return_tuple def get_video_features( self, pixel_values_videos: torch.FloatTensor, diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index 47e7b79f5705..fb79ceb74602 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -1244,6 +1244,7 @@ def get_rope_index( return position_ids, mrope_position_deltas + @can_return_tuple def get_video_features( self, pixel_values_videos: torch.FloatTensor, From 78fa904f4dc48fbd06938a8c6fbd5f8e1cca7c86 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 16 Dec 2025 16:50:14 +0100 Subject: [PATCH 021/131] Patch Fuyu, just return BaseModelOutputWithPooling without pooler The Fuyu architecture doesn't have an image encoder: > Architecturally, Fuyu is a vanilla decoder-only transformer - there is no image encoder. --- src/transformers/models/fuyu/modeling_fuyu.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/fuyu/modeling_fuyu.py b/src/transformers/models/fuyu/modeling_fuyu.py index 99dcdc2f3cb0..8e541c296d9e 100644 --- a/src/transformers/models/fuyu/modeling_fuyu.py +++ b/src/transformers/models/fuyu/modeling_fuyu.py @@ -21,10 +21,11 @@ from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPooling, CausalLMOutputWithPast from ...modeling_utils import PreTrainedModel from ...models.auto.modeling_auto import AutoModel -from ...utils import auto_docstring, can_return_tuple, logging +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from .configuration_fuyu import FuyuConfig @@ -116,7 +117,8 @@ def gather_continuous_embeddings( ) return output_embeddings - def get_image_features(self, pixel_values: torch.FloatTensor): + @can_return_tuple + def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -128,8 +130,7 @@ def get_image_features(self, pixel_values: torch.FloatTensor): self.vision_embed_tokens(patch.to(self.vision_embed_tokens.weight.dtype)).squeeze(0) for patch in pixel_values ] - # NOTE: @Tom Not easily converted to the standard format - return patch_embeddings + return BaseModelOutputWithPooling(last_hidden_state=patch_embeddings) def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor From f082a8e8ddccc05581c966b044e2b03a1170a477 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 16 Dec 2025 17:58:18 +0100 Subject: [PATCH 022/131] Introduce ModelOutput subclass for Chameleon, patch get_image_features --- .../models/chameleon/modeling_chameleon.py | 59 ++++++++++++++----- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index b79f8a2971d1..75dab6bce28f 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -15,6 +15,7 @@ """PyTorch Chameleon model.""" from collections.abc import Callable +from dataclasses import dataclass from functools import cached_property from typing import Optional, Union @@ -28,7 +29,7 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -38,13 +39,30 @@ can_return_tuple, logging, ) -from ...utils.generic import maybe_autocast +from ...utils.generic import check_model_inputs, maybe_autocast from .configuration_chameleon import ChameleonConfig, ChameleonVQVAEConfig logger = logging.get_logger(__name__) +@dataclass +@auto_docstring +class ChameleonVQVAEModelOutput(BaseModelOutputWithPooling): + r""" + image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):` + Indices of the image tokens predicted by the VQ-VAE model. + quantized_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + Quantized last hidden state from the VQ-VAE model. + embedding_loss (`torch.FloatTensor`): + The embedding loss computed during quantization. + """ + + quantized_last_hidden_state: Optional[torch.FloatTensor] = None + image_tokens: Optional[torch.FloatTensor] = None + embedding_loss: Optional[torch.FloatTensor] = None + + # Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Chameleon class ChameleonRMSNorm(nn.Module): def __init__(self, hidden_size, eps=1e-6): @@ -810,11 +828,19 @@ def __init__(self, config: ChameleonVQVAEConfig): self.post_quant_conv = torch.nn.Conv2d(config.embed_dim, config.latent_channels, 1) self.eval() # Chameleon's VQ model is frozen - def encode(self, pixel_values: torch.LongTensor): + @check_model_inputs(tie_last_hidden_states=False) + def encode( + self, pixel_values: torch.LongTensor, **kwargs: Unpack[TransformersKwargs] + ) -> ChameleonVQVAEModelOutput: hidden_states = self.encoder(pixel_values) - hidden_states = self.quant_conv(hidden_states) - quant, emb_loss, indices = self.quantize(hidden_states) - return quant, emb_loss, indices + conv_hidden_states = self.quant_conv(hidden_states) + quantized_last_hidden_state, emb_loss, indices = self.quantize(conv_hidden_states) + return ChameleonVQVAEModelOutput( + last_hidden_state=hidden_states, + quantized_last_hidden_state=quantized_last_hidden_state, + image_tokens=indices, + embedding_loss=emb_loss, + ) @auto_docstring @@ -849,12 +875,15 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor): The tensors corresponding to the input images. """ batch_size = pixel_values.shape[0] - _, _, image_toks = self.vqmodel.encode(pixel_values) - bpe_toks = self.vocabulary_mapping.convert_img2bpe(image_toks) + vqmodel_outputs: ChameleonVQVAEModelOutput = self.vqmodel.encode(pixel_values, return_dict=True) + bpe_toks = self.vocabulary_mapping.convert_img2bpe(vqmodel_outputs.image_tokens) bpe_toks = bpe_toks.view(batch_size, -1) return bpe_toks - def get_image_features(self, pixel_values: torch.FloatTensor): + @can_return_tuple + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> BaseModelOutputWithPooling: """ Tokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layer @@ -863,9 +892,9 @@ def get_image_features(self, pixel_values: torch.FloatTensor): pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): The tensors corresponding to the input images. """ - image_tokens = self.get_image_tokens(pixel_values) - image_embeddings = self.get_input_embeddings()(image_tokens) - return image_embeddings + vqmodel_outputs: ChameleonVQVAEModelOutput = self.vqmodel.encode(pixel_values, **kwargs) + vqmodel_outputs.pooler_output = self.get_input_embeddings()(vqmodel_outputs.image_tokens) + return vqmodel_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -927,11 +956,11 @@ def forward( inputs_embeds = self.embed_tokens(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values).pooler_output special_image_mask = self.get_placeholder_mask( - input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds + input_ids, inputs_embeds=inputs_embeds, image_features=image_features ) - inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_embeds) + inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features) # torch.jit.trace() doesn't support cache objects in the output if use_cache and past_key_values is None and not torch.jit.is_tracing(): From 9ddd3b43a4eea6b18c4e2dc620499709bb02fe53 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 17 Dec 2025 16:39:28 +0100 Subject: [PATCH 023/131] Update modeling files with new output formats for get_..._features --- src/transformers/models/aria/modeling_aria.py | 3 +- src/transformers/models/aria/modular_aria.py | 5 ++- .../audioflamingo3/modeling_audioflamingo3.py | 2 +- .../audioflamingo3/modular_audioflamingo3.py | 2 +- .../models/aya_vision/modeling_aya_vision.py | 3 +- .../models/aya_vision/modular_aya_vision.py | 3 +- .../models/chameleon/modeling_chameleon.py | 6 +-- src/transformers/models/clap/modeling_clap.py | 2 +- .../models/clipseg/modeling_clipseg.py | 4 +- .../cohere2_vision/modeling_cohere2_vision.py | 2 +- .../cohere2_vision/modular_cohere2_vision.py | 2 +- .../deepseek_vl/modeling_deepseek_vl.py | 2 +- .../models/edgetam/modeling_edgetam.py | 11 +++-- .../edgetam_video/modeling_edgetam_video.py | 15 ++++--- .../models/fast_vlm/modeling_fast_vlm.py | 3 +- .../models/florence2/modeling_florence2.py | 4 +- .../models/florence2/modular_florence2.py | 4 +- src/transformers/models/fuyu/modeling_fuyu.py | 2 +- .../models/gemma3/modeling_gemma3.py | 2 +- .../models/gemma3/modular_gemma3.py | 2 +- .../models/gemma3n/modeling_gemma3n.py | 4 +- .../models/gemma3n/modular_gemma3n.py | 4 +- .../models/glm46v/modeling_glm46v.py | 4 +- .../models/glm4v/modeling_glm4v.py | 4 +- .../models/glm4v/modular_glm4v.py | 4 +- .../models/glm4v_moe/modeling_glm4v_moe.py | 4 +- .../models/got_ocr2/modeling_got_ocr2.py | 4 +- .../models/got_ocr2/modular_got_ocr2.py | 4 +- .../granite_speech/modeling_granite_speech.py | 2 +- .../models/idefics2/modeling_idefics2.py | 4 +- .../models/idefics3/modeling_idefics3.py | 4 +- .../models/internvl/modeling_internvl.py | 3 +- .../models/internvl/modular_internvl.py | 3 +- .../models/janus/modeling_janus.py | 39 ++++++++++++++---- .../models/janus/modular_janus.py | 8 ++-- .../models/kosmos2/modeling_kosmos2.py | 2 +- .../models/lfm2_vl/modeling_lfm2_vl.py | 3 +- .../models/lfm2_vl/modular_lfm2_vl.py | 3 +- .../models/llama4/modeling_llama4.py | 3 +- .../models/llava/modeling_llava.py | 5 ++- .../models/llava_next/modeling_llava_next.py | 3 +- .../modeling_llava_next_video.py | 6 ++- .../modular_llava_next_video.py | 6 ++- .../modeling_llava_onevision.py | 6 ++- .../modular_llava_onevision.py | 6 ++- .../models/mistral3/modeling_mistral3.py | 3 +- .../models/mistral3/modular_mistral3.py | 3 +- .../models/ovis2/modeling_ovis2.py | 41 +++++++------------ .../models/ovis2/modular_ovis2.py | 39 ++++++------------ .../paddleocr_vl/modeling_paddleocr_vl.py | 5 +-- .../paddleocr_vl/modular_paddleocr_vl.py | 5 +-- .../models/paligemma/modeling_paligemma.py | 2 +- .../perception_lm/modeling_perception_lm.py | 4 +- .../perception_lm/modular_perception_lm.py | 4 +- .../qwen2_5_omni/modeling_qwen2_5_omni.py | 7 ++-- .../qwen2_5_omni/modular_qwen2_5_omni.py | 7 ++-- .../models/qwen2_5_vl/modeling_qwen2_5_vl.py | 4 +- .../models/qwen2_5_vl/modular_qwen2_5_vl.py | 4 +- .../models/qwen2_vl/modeling_qwen2_vl.py | 23 ++++------- .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 13 ++++-- .../qwen3_omni_moe/modular_qwen3_omni_moe.py | 13 ++++-- .../models/qwen3_vl/modeling_qwen3_vl.py | 16 ++++++-- .../models/qwen3_vl/modular_qwen3_vl.py | 16 ++++++-- .../qwen3_vl_moe/modeling_qwen3_vl_moe.py | 16 ++++++-- src/transformers/models/sam2/modeling_sam2.py | 11 ++--- src/transformers/models/sam2/modular_sam2.py | 11 ++--- .../models/sam2_video/modeling_sam2_video.py | 15 ++++--- .../models/sam2_video/modular_sam2_video.py | 12 +++--- .../sam3_tracker/modeling_sam3_tracker.py | 11 +++-- .../modeling_sam3_tracker_video.py | 15 ++++--- .../models/smolvlm/modeling_smolvlm.py | 5 ++- .../models/smolvlm/modular_smolvlm.py | 5 ++- .../models/t5gemma2/modeling_t5gemma2.py | 2 +- .../models/t5gemma2/modular_t5gemma2.py | 2 +- .../video_llama_3/modeling_video_llama_3.py | 8 +++- .../video_llama_3/modular_video_llama_3.py | 8 +++- .../video_llava/modeling_video_llava.py | 7 ++-- .../models/vipllava/modeling_vipllava.py | 4 +- .../models/vipllava/modular_vipllava.py | 4 +- .../models/voxtral/modeling_voxtral.py | 2 +- .../models/voxtral/modular_voxtral.py | 2 +- 81 files changed, 334 insertions(+), 232 deletions(-) diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index 349147d6ac6e..9ef9aa0e9b19 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -1001,7 +1001,8 @@ def forward( pixel_values=pixel_values, pixel_mask=pixel_mask, vision_feature_layer=self.config.vision_feature_layer, - ) + return_dict=True, + ).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/aria/modular_aria.py b/src/transformers/models/aria/modular_aria.py index 156c67f14493..9b3a5a6b3810 100644 --- a/src/transformers/models/aria/modular_aria.py +++ b/src/transformers/models/aria/modular_aria.py @@ -1309,6 +1309,8 @@ def get_image_features( return image_outputs + @can_return_tuple + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1331,7 +1333,8 @@ def forward( pixel_values=pixel_values, pixel_mask=pixel_mask, vision_feature_layer=self.config.vision_feature_layer, - ) + return_dict=True, + ).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py index 9a234ae0ab7f..d48ff0aed491 100644 --- a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py @@ -563,7 +563,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if input_features is not None and input_ids is not None: - audio_embeds = self.get_audio_features(input_features, input_features_mask) + audio_embeds = self.get_audio_features(input_features, input_features_mask, return_dict=True).pooler_output # replace text-audio token placeholders with audio embeddings audio_token_mask = (input_ids == self.config.audio_token_id).unsqueeze(-1) diff --git a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py index c01778549424..2efe7ae0690a 100644 --- a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py @@ -263,7 +263,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if input_features is not None and input_ids is not None: - audio_embeds = self.get_audio_features(input_features, input_features_mask) + audio_embeds = self.get_audio_features(input_features, input_features_mask, return_dict=True).pooler_output # replace text-audio token placeholders with audio embeddings audio_token_mask = (input_ids == self.config.audio_token_id).unsqueeze(-1) diff --git a/src/transformers/models/aya_vision/modeling_aya_vision.py b/src/transformers/models/aya_vision/modeling_aya_vision.py index 8667963383a1..efbcdcd60f02 100644 --- a/src/transformers/models/aya_vision/modeling_aya_vision.py +++ b/src/transformers/models/aya_vision/modeling_aya_vision.py @@ -299,7 +299,8 @@ def forward( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - ) + return_dict=True, + ).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/aya_vision/modular_aya_vision.py b/src/transformers/models/aya_vision/modular_aya_vision.py index 8905beaeec6d..982346ae74ab 100644 --- a/src/transformers/models/aya_vision/modular_aya_vision.py +++ b/src/transformers/models/aya_vision/modular_aya_vision.py @@ -200,7 +200,8 @@ def forward( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - ) + return_dict=True, + ).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 75dab6bce28f..6b68f67958e9 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -956,7 +956,7 @@ def forward( inputs_embeds = self.embed_tokens(input_ids) if pixel_values is not None: - image_features = self.get_image_features(pixel_values).pooler_output + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features ) @@ -1052,8 +1052,8 @@ def __init__(self, config): def get_image_tokens(self, pixel_values): return self.model.get_image_tokens(pixel_values) - def get_image_features(self, pixel_values: torch.FloatTensor): - return self.model.get_image_features(pixel_values) + def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + return self.model.get_image_features(pixel_values, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/clap/modeling_clap.py b/src/transformers/models/clap/modeling_clap.py index 31697fb55fb5..c96f86070d5e 100644 --- a/src/transformers/models/clap/modeling_clap.py +++ b/src/transformers/models/clap/modeling_clap.py @@ -1615,7 +1615,7 @@ def get_audio_features( ... audio_features = model.get_audio_features(**inputs) ```""" audio_outputs: BaseModelOutputWithPooling = self.audio_model( - input_features=input_features, is_longer=is_longer + input_features=input_features, is_longer=is_longer, **kwargs ) audio_features = self.audio_projection(audio_outputs.pooler_output) audio_outputs.pooler_output = F.normalize(audio_features, dim=-1) diff --git a/src/transformers/models/clipseg/modeling_clipseg.py b/src/transformers/models/clipseg/modeling_clipseg.py index d6f25dc97b15..0b76281e6585 100644 --- a/src/transformers/models/clipseg/modeling_clipseg.py +++ b/src/transformers/models/clipseg/modeling_clipseg.py @@ -1220,13 +1220,13 @@ def get_conditional_embeddings( with torch.no_grad(): conditional_embeddings = self.clip.get_text_features( input_ids, attention_mask=attention_mask, position_ids=position_ids - ) + ).pooler_output elif conditional_pixel_values is not None: # compute conditional embeddings from images if len(conditional_pixel_values) != batch_size: raise ValueError("Make sure to pass as many prompt images as there are query images") with torch.no_grad(): - conditional_embeddings = self.clip.get_image_features(conditional_pixel_values) + conditional_embeddings = self.clip.get_image_features(conditional_pixel_values).pooler_output else: raise ValueError( "Invalid conditional, should be either provided as `input_ids` or `conditional_pixel_values`" diff --git a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py index df1d3950d7f3..6cbc4d6169a0 100644 --- a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py @@ -230,7 +230,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_features = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py index c5b23f95c4a6..7ae21954efea 100644 --- a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py @@ -136,7 +136,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_features = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py b/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py index f54fb668b7f2..b0dc803b709d 100644 --- a/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py +++ b/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py @@ -208,7 +208,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values) + image_embeds = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_embeds.reshape(-1, inputs_embeds.shape[-1]) image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) image_attention_mask = self.get_placeholder_mask( diff --git a/src/transformers/models/edgetam/modeling_edgetam.py b/src/transformers/models/edgetam/modeling_edgetam.py index 8b4464c9545b..fd1a06f53172 100644 --- a/src/transformers/models/edgetam/modeling_edgetam.py +++ b/src/transformers/models/edgetam/modeling_edgetam.py @@ -968,7 +968,8 @@ def get_image_embeddings( Input pixel values """ batch_size = pixel_values.shape[0] - feature_maps, _, _, _ = self.get_image_features(pixel_values, **kwargs) + image_outputs = self.get_image_features(pixel_values, return_dict=True, **kwargs) + feature_maps = image_outputs.fpn_hidden_states # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding @@ -1124,10 +1125,12 @@ def forward( vision_hidden_states = None if pixel_values is not None: - feature_maps, _, vision_hidden_states, vision_attentions = self.get_image_features( - pixel_values, - **kwargs, + image_outputs: EdgeTamVisionEncoderOutput = self.get_image_features( + pixel_values, return_dict=True, **kwargs ) + feature_maps = image_outputs.fpn_hidden_states + vision_hidden_states = image_outputs.hidden_states + vision_attentions = image_outputs.attentions # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding diff --git a/src/transformers/models/edgetam_video/modeling_edgetam_video.py b/src/transformers/models/edgetam_video/modeling_edgetam_video.py index 122faa22b165..65f98c09cd27 100644 --- a/src/transformers/models/edgetam_video/modeling_edgetam_video.py +++ b/src/transformers/models/edgetam_video/modeling_edgetam_video.py @@ -2063,7 +2063,8 @@ def get_image_embeddings( Input pixel values """ batch_size = pixel_values.shape[0] - feature_maps, _, _, _ = self.get_image_features(pixel_values, **kwargs) + image_outputs = self.get_image_features(pixel_values, return_dict=True, **kwargs) + feature_maps = image_outputs.fpn_hidden_states # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding @@ -2272,7 +2273,9 @@ def _prepare_vision_features( else: # Compute features using image encoder image_batch = inference_session.get_frame(frame_idx).unsqueeze(0) # Add batch dimension - vision_feats, vision_pos_embeds, _, _ = self.get_image_features(image_batch) + image_outputs = self.get_image_features(image_batch, return_dict=True) + vision_feats = image_outputs.fpn_hidden_states + vision_pos_embeds = image_outputs.fpn_position_embeddings # Cache features inference_session.cache.cache_vision_features( frame_idx, {"vision_feats": vision_feats, "vision_pos_embeds": vision_pos_embeds} @@ -2377,10 +2380,10 @@ def _single_frame_forward( vision_hidden_states = None if pixel_values is not None: - feature_maps, _, vision_hidden_states, vision_attentions = self.get_image_features( - pixel_values, - **kwargs, - ) + image_outputs = self.get_image_features(pixel_values, return_dict=True, **kwargs) + feature_maps = image_outputs.fpn_hidden_states + vision_hidden_states = image_outputs.hidden_states + vision_attentions = image_outputs.attentions # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding diff --git a/src/transformers/models/fast_vlm/modeling_fast_vlm.py b/src/transformers/models/fast_vlm/modeling_fast_vlm.py index f7acc901d2fa..de098951ca3d 100644 --- a/src/transformers/models/fast_vlm/modeling_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modeling_fast_vlm.py @@ -223,7 +223,8 @@ def forward( vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, image_sizes=image_sizes, - ) + return_dict=True, + ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/florence2/modeling_florence2.py b/src/transformers/models/florence2/modeling_florence2.py index aa061c3b932b..c3e7d4f2eb22 100644 --- a/src/transformers/models/florence2/modeling_florence2.py +++ b/src/transformers/models/florence2/modeling_florence2.py @@ -731,7 +731,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_features = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features @@ -987,7 +987,7 @@ def _prepare_encoder_decoder_kwargs_for_generation( inputs_embeds = self.get_input_embeddings()(inputs_tensor) if pixel_values is not None: - image_features = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( inputs_tensor, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/florence2/modular_florence2.py b/src/transformers/models/florence2/modular_florence2.py index e02b1326dbc5..8bb67ed9973e 100644 --- a/src/transformers/models/florence2/modular_florence2.py +++ b/src/transformers/models/florence2/modular_florence2.py @@ -1574,7 +1574,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_features = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features @@ -1763,7 +1763,7 @@ def _prepare_encoder_decoder_kwargs_for_generation( inputs_embeds = self.get_input_embeddings()(inputs_tensor) if pixel_values is not None: - image_features = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( inputs_tensor, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/fuyu/modeling_fuyu.py b/src/transformers/models/fuyu/modeling_fuyu.py index 8e541c296d9e..16b0d2839f43 100644 --- a/src/transformers/models/fuyu/modeling_fuyu.py +++ b/src/transformers/models/fuyu/modeling_fuyu.py @@ -209,7 +209,7 @@ def forward( inputs_embeds = self.language_model.get_input_embeddings()(input_ids) if image_patches is not None: - patch_embeddings = self.get_image_features(image_patches) + patch_embeddings = self.get_image_features(image_patches, return_dict=True).last_hidden_state patch_embeddings = torch.cat(patch_embeddings, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=patch_embeddings diff --git a/src/transformers/models/gemma3/modeling_gemma3.py b/src/transformers/models/gemma3/modeling_gemma3.py index 0427adbaaaf2..56407d6d80b2 100644 --- a/src/transformers/models/gemma3/modeling_gemma3.py +++ b/src/transformers/models/gemma3/modeling_gemma3.py @@ -937,7 +937,7 @@ def forward( # Merge text and images if pixel_values is not None: - image_features = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/gemma3/modular_gemma3.py b/src/transformers/models/gemma3/modular_gemma3.py index ff92b84724ae..5e785a8c8dde 100644 --- a/src/transformers/models/gemma3/modular_gemma3.py +++ b/src/transformers/models/gemma3/modular_gemma3.py @@ -841,7 +841,7 @@ def forward( # Merge text and images if pixel_values is not None: - image_features = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index bb8faacc4570..965035be91f7 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -2098,7 +2098,7 @@ def forward( # Merge text and images if pixel_values is not None: - image_features = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features @@ -2107,7 +2107,7 @@ def forward( # Merge text and audio if input_features is not None and input_features_mask is not None: - audio_features = self.get_audio_features(input_features, ~input_features_mask) + audio_features = self.get_audio_features(input_features, ~input_features_mask, return_dict=True) audio_features = audio_features.pooler_output audio_mask = audio_features.audio_mel_mask diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index d185f3685823..a0f6d28e7d3e 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -2357,7 +2357,7 @@ def forward( # Merge text and images if pixel_values is not None: - image_features = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features @@ -2366,7 +2366,7 @@ def forward( # Merge text and audio if input_features is not None and input_features_mask is not None: - audio_features = self.get_audio_features(input_features, ~input_features_mask) + audio_features = self.get_audio_features(input_features, ~input_features_mask, return_dict=True) audio_features = audio_features.pooler_output audio_mask = audio_features.audio_mel_mask diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index ecc1259b8bfa..d74dcd332b9a 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -418,13 +418,13 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = self.get_image_features(pixel_values, image_grid_thw, return_dict=True).pooler_output image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask(input_ids, inputs_embeds, image_features=image_embeds) inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw, return_dict=True).pooler_output video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask = self.get_placeholder_mask(input_ids, inputs_embeds, video_features=video_embeds) inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index 18bf5bc89d30..59ce2528f5b4 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1275,13 +1275,13 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = self.get_image_features(pixel_values, image_grid_thw, return_dict=True).pooler_output image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask(input_ids, inputs_embeds, image_features=image_embeds) inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw, return_dict=True).pooler_output video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask = self.get_placeholder_mask(input_ids, inputs_embeds, video_features=video_embeds) inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index e57cb0938326..d4e31b6fdfa1 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -1245,13 +1245,13 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = self.get_image_features(pixel_values, image_grid_thw, return_dict=True).pooler_output image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask(input_ids, inputs_embeds, image_features=image_embeds) inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw, return_dict=True).pooler_output video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask = self.get_placeholder_mask(input_ids, inputs_embeds, video_features=video_embeds) inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 89eba39ca94b..cfc063307bac 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1485,13 +1485,13 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = self.get_image_features(pixel_values, image_grid_thw, return_dict=True).pooler_output image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask(input_ids, inputs_embeds, image_features=image_embeds) inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw, return_dict=True).pooler_output video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask = self.get_placeholder_mask(input_ids, inputs_embeds, video_features=video_embeds) inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) diff --git a/src/transformers/models/got_ocr2/modeling_got_ocr2.py b/src/transformers/models/got_ocr2/modeling_got_ocr2.py index 17c020436ead..2fb23f51a625 100644 --- a/src/transformers/models/got_ocr2/modeling_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modeling_got_ocr2.py @@ -626,7 +626,9 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_features = self.get_image_features(pixel_values=pixel_values.to(inputs_embeds.dtype)) + image_features = self.get_image_features( + pixel_values=pixel_values.to(inputs_embeds.dtype), return_dict=True + ).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/got_ocr2/modular_got_ocr2.py b/src/transformers/models/got_ocr2/modular_got_ocr2.py index 5d8820788619..3eb27472cc15 100644 --- a/src/transformers/models/got_ocr2/modular_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modular_got_ocr2.py @@ -357,7 +357,9 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_features = self.get_image_features(pixel_values=pixel_values.to(inputs_embeds.dtype)) + image_features = self.get_image_features( + pixel_values=pixel_values.to(inputs_embeds.dtype), return_dict=True + ).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index d7d40a93dec7..c6919516b9f4 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -402,7 +402,7 @@ def forward( if input_features.dtype != self.dtype: input_features = input_features.to(self.dtype) # Get the audio features from the encoder / projector - audio_embeds = self.get_audio_features(input_features) + audio_embeds = self.get_audio_features(input_features, return_dict=True).pooler_output # Merge the audio features into the LLM embeddings inputs_embeds = self.get_merged_audio_embeddings( diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index faa178865eb9..6cda89105630 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -934,7 +934,9 @@ def forward( if pixel_values is not None and image_hidden_states is not None: raise ValueError("You cannot specify both pixel_values and image_hidden_states at the same time") elif pixel_values is not None: - image_hidden_states = self.get_image_features(pixel_values, pixel_attention_mask) + image_hidden_states = self.get_image_features( + pixel_values, pixel_attention_mask, return_dict=True + ).pooler_output elif image_hidden_states is not None: image_hidden_states = image_hidden_states.to(dtype=self.dtype, device=input_ids.device) diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index 84597c37782f..592222fe35cf 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -688,7 +688,9 @@ def forward( if pixel_values is not None and image_hidden_states is not None: raise ValueError("You cannot specify both pixel_values and image_hidden_states at the same time") elif pixel_values is not None: - image_hidden_states = self.get_image_features(pixel_values, pixel_attention_mask) + image_hidden_states = self.get_image_features( + pixel_values, pixel_attention_mask, return_dict=True + ).pooler_output elif image_hidden_states is not None: image_hidden_states = image_hidden_states.to(dtype=self.dtype, device=input_ids.device) diff --git a/src/transformers/models/internvl/modeling_internvl.py b/src/transformers/models/internvl/modeling_internvl.py index 094504ba6f04..8a930f776c15 100644 --- a/src/transformers/models/internvl/modeling_internvl.py +++ b/src/transformers/models/internvl/modeling_internvl.py @@ -665,7 +665,8 @@ def forward( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - ) + return_dict=True, + ).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/internvl/modular_internvl.py b/src/transformers/models/internvl/modular_internvl.py index c1eaf66427fb..ac3b9ac3014e 100644 --- a/src/transformers/models/internvl/modular_internvl.py +++ b/src/transformers/models/internvl/modular_internvl.py @@ -584,7 +584,8 @@ def forward( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - ) + return_dict=True, + ).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/janus/modeling_janus.py b/src/transformers/models/janus/modeling_janus.py index 3d3d4b00294e..a38e37224c9a 100644 --- a/src/transformers/models/janus/modeling_janus.py +++ b/src/transformers/models/janus/modeling_janus.py @@ -946,6 +946,23 @@ def forward(self, hidden_state: torch.FloatTensor) -> torch.FloatTensor: return hidden_state +@dataclass +@auto_docstring +class JanusVQVAEModelOutput(BaseModelOutputWithPooling): + r""" + image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):` + Indices of the image tokens predicted by the VQ-VAE model. + quantized_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + Quantized last hidden state from the VQ-VAE model. + embedding_loss (`torch.FloatTensor`): + The embedding loss computed during quantization. + """ + + quantized_last_hidden_state: Optional[torch.FloatTensor] = None + image_tokens: Optional[torch.FloatTensor] = None + embedding_loss: Optional[torch.FloatTensor] = None + + @auto_docstring( custom_intro=""" The VQ-VAE model used in Janus for encoding/decoding images into discrete tokens. @@ -977,11 +994,17 @@ def __init__(self, config: JanusVQVAEConfig): # Initialize the VQVAE model. self.post_init() - def encode(self, pixel_values: torch.LongTensor): + @check_model_inputs(tie_last_hidden_states=False) + def encode(self, pixel_values: torch.LongTensor, **kwargs: Unpack[TransformersKwargs]) -> JanusVQVAEModelOutput: hidden_states = self.encoder(pixel_values) - hidden_states = self.quant_conv(hidden_states) - quant, emb_loss, indices = self.quantize(hidden_states) - return quant, emb_loss, indices + conv_hidden_states = self.quant_conv(hidden_states) + quantized_last_hidden_state, emb_loss, indices = self.quantize(conv_hidden_states) + return JanusVQVAEModelOutput( + last_hidden_state=hidden_states, + quantized_last_hidden_state=quantized_last_hidden_state, + image_tokens=indices, + embedding_loss=emb_loss, + ) def decode(self, image_tokens: torch.LongTensor) -> torch.FloatTensor: """ @@ -1010,10 +1033,10 @@ def forward( **kwargs, ) -> tuple[torch.FloatTensor, torch.FloatTensor]: batch_size = pixel_values.shape[0] - quant, embedding_loss, indices = self.encode(pixel_values) - decoded_pixel_values = self.decode(indices.view(batch_size, -1)) + encode_outputs = self.encode(pixel_values, return_dict=True, **kwargs) + decoded_pixel_values = self.decode(encode_outputs.image_tokens.view(batch_size, -1)) - return JanusVQVAEOutput(decoded_pixel_values, embedding_loss) + return JanusVQVAEOutput(decoded_pixel_values, encode_outputs.embedding_loss) class JanusVQVAEAlignerMLP(nn.Module): @@ -1138,7 +1161,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values) + image_embeds = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_embeds.reshape(-1, inputs_embeds.shape[-1]) image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) image_attention_mask = self.get_placeholder_mask( diff --git a/src/transformers/models/janus/modular_janus.py b/src/transformers/models/janus/modular_janus.py index 0c8ebcab99a9..5030b630f1fb 100644 --- a/src/transformers/models/janus/modular_janus.py +++ b/src/transformers/models/janus/modular_janus.py @@ -826,10 +826,10 @@ def forward( **kwargs, ) -> tuple[torch.FloatTensor, torch.FloatTensor]: batch_size = pixel_values.shape[0] - quant, embedding_loss, indices = self.encode(pixel_values) - decoded_pixel_values = self.decode(indices.view(batch_size, -1)) + encode_outputs = self.encode(pixel_values, return_dict=True, **kwargs) + decoded_pixel_values = self.decode(encode_outputs.image_tokens.view(batch_size, -1)) - return JanusVQVAEOutput(decoded_pixel_values, embedding_loss) + return JanusVQVAEOutput(decoded_pixel_values, encode_outputs.embedding_loss) class JanusVQVAEAlignerMLP(nn.Module): @@ -954,7 +954,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values) + image_embeds = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_embeds.reshape(-1, inputs_embeds.shape[-1]) image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) image_attention_mask = self.get_placeholder_mask( diff --git a/src/transformers/models/kosmos2/modeling_kosmos2.py b/src/transformers/models/kosmos2/modeling_kosmos2.py index b9449ed110a6..f59544238678 100644 --- a/src/transformers/models/kosmos2/modeling_kosmos2.py +++ b/src/transformers/models/kosmos2/modeling_kosmos2.py @@ -1535,7 +1535,7 @@ def get_image_features( vision_output.pooler_output = image_embeds # NOTE: @Tom we need to add projection_attentions to the output - return image_embeds + return vision_output @can_return_tuple @auto_docstring diff --git a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py index e2e83ee8a7eb..f5b5b05e261e 100755 --- a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py @@ -271,7 +271,8 @@ def forward( pixel_values=pixel_values, spatial_shapes=spatial_shapes, pixel_attention_mask=pixel_attention_mask, - ) + return_dict=True, + ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids=input_ids, diff --git a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py index 438d3d928a8c..3eec87167588 100644 --- a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py @@ -202,7 +202,8 @@ def forward( pixel_values=pixel_values, spatial_shapes=spatial_shapes, pixel_attention_mask=pixel_attention_mask, - ) + return_dict=True, + ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids=input_ids, diff --git a/src/transformers/models/llama4/modeling_llama4.py b/src/transformers/models/llama4/modeling_llama4.py index d8fcd0d6f203..fe11289fcc8e 100644 --- a/src/transformers/models/llama4/modeling_llama4.py +++ b/src/transformers/models/llama4/modeling_llama4.py @@ -1317,7 +1317,8 @@ def forward( image_features = self.get_image_features( pixel_values=pixel_values, vision_feature_select_strategy=vision_feature_select_strategy, - ) + return_dict=True, + ).pooler_output vision_flat = image_features.view(-1, image_features.size(-1)) projected_vision_flat = self.multi_modal_projector(vision_flat).to( diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index d5bb97ac0b4b..a6bb70670b12 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -201,7 +201,7 @@ def get_image_features( image_features = self.multi_modal_projector(selected_image_feature) - if "image_sizes" in kwargs: + if kwargs.get("image_sizes") is not None: split_sizes = [ (height // self.vision_tower.patch_size) * (width // self.vision_tower.patch_size) for height, width in kwargs["image_sizes"] @@ -274,7 +274,8 @@ def forward( vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, image_sizes=image_sizes, - ) + return_dict=True, + ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index 592a6d13d5f8..2286c038f541 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -504,7 +504,8 @@ def forward( image_sizes, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - ) + return_dict=True, + ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 6028c75e9d34..dfa4322499c8 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -569,7 +569,8 @@ def forward( image_sizes, vision_feature_layer=self.vision_feature_layer, vision_feature_select_strategy=self.vision_feature_select_strategy, - ) + return_dict=True, + ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features @@ -581,7 +582,8 @@ def forward( pixel_values_videos, vision_feature_layer=self.vision_feature_layer, vision_feature_select_strategy=self.vision_feature_select_strategy, - ) + return_dict=True, + ).pooler_output video_features = [feature.flatten(0, 1) for feature in video_features] video_feature_lens = [feature.size(0) for feature in video_features] video_features = torch.cat(video_features, dim=0) diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index 9313dba516c4..ac727c1ea2d3 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -493,7 +493,8 @@ def forward( image_sizes, vision_feature_layer=self.vision_feature_layer, vision_feature_select_strategy=self.vision_feature_select_strategy, - ) + return_dict=True, + ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features @@ -505,7 +506,8 @@ def forward( pixel_values_videos, vision_feature_layer=self.vision_feature_layer, vision_feature_select_strategy=self.vision_feature_select_strategy, - ) + return_dict=True, + ).pooler_output video_features = [feature.flatten(0, 1) for feature in video_features] video_feature_lens = [feature.size(0) for feature in video_features] video_features = torch.cat(video_features, dim=0) diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index 351d7612c5ad..713f2da6f870 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -551,7 +551,8 @@ def forward( vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, batch_num_images=batch_num_images, - ) + return_dict=True, + ).pooler_output image_features = torch.cat(image_features, dim=0) image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask, _ = self.get_placeholder_mask( @@ -565,7 +566,8 @@ def forward( pixel_values_videos, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - ) + return_dict=True, + ).pooler_output image_newline = ( self.image_newline[None, None, :].repeat(video_features.shape[0], 1, 1).to(video_features.device) ) diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index 56fdaeb22a26..829dd60af533 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -516,7 +516,8 @@ def forward( vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, batch_num_images=batch_num_images, - ) + return_dict=True, + ).pooler_output image_features = torch.cat(image_features, dim=0) image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask, _ = self.get_placeholder_mask( @@ -530,7 +531,8 @@ def forward( pixel_values_videos, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - ) + return_dict=True, + ).pooler_output image_newline = ( self.image_newline[None, None, :].repeat(video_features.shape[0], 1, 1).to(video_features.device) ) diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index c67eaa4d3d86..ea10c28e427d 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -322,7 +322,8 @@ def forward( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, image_sizes=image_sizes, - ) + return_dict=True, + ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/mistral3/modular_mistral3.py b/src/transformers/models/mistral3/modular_mistral3.py index 48b7ee3a9daa..aa84e56d8c4f 100644 --- a/src/transformers/models/mistral3/modular_mistral3.py +++ b/src/transformers/models/mistral3/modular_mistral3.py @@ -200,7 +200,8 @@ def forward( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, image_sizes=image_sizes, - ) + return_dict=True, + ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index 794999d1d8d3..379e0bdf144d 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -32,33 +32,23 @@ from ...generation import GenerationMixin from ...integrations import use_kernel_forward_from_hub from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple +from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_ovis2 import Ovis2Config, Ovis2VisionConfig @dataclass -class BaseModelOutputWithVisualIndicatorFeatures(ModelOutput): +@auto_docstring +class BaseModelOutputWithVisualIndicatorFeatures(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - visual_indicator_features (`torch.FloatTensor` of shape `(batch_size, visual_indicator_size)`): - Visual indicator features extracted from the model, which can be used for auxiliary tasks or further processing. + visual_indicator_features (`torch.FloatTensor` of shape `(batch_size, visual_indicator_size)`): + Visual indicator features extracted from the model, which can be used for auxiliary tasks or further processing. """ - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None visual_indicator_features: Optional[torch.FloatTensor] = None @@ -479,8 +469,9 @@ def __init__(self, config: Ovis2VisionConfig): ) self.head_norm = nn.LayerNorm(self.vocab_size - self.num_visual_indicator_tokens) + @check_model_inputs(tie_last_hidden_states=False) def forward( - self, pixel_values: torch.FloatTensor, return_dict: bool = False, **kwargs + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple[torch.Tensor, torch.Tensor]: outputs = self.transformer(pixel_values, **kwargs) last_hidden_state = outputs[0] @@ -514,13 +505,10 @@ def forward( elif self.config.tokenize_function == "softmax": prob_token = nn.functional.softmax(logits, dim=-1) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=last_hidden_state, - pooler_output=prob_token, - ) - - return prob_token + return BaseModelOutputWithVisualIndicatorFeatures( + last_hidden_state=last_hidden_state, + pooler_output=prob_token, + ) @auto_docstring( @@ -588,9 +576,8 @@ def get_image_features( self.visual_vocab_size, dtype=torch.long, ).to(image_features.device) - visual_indicator_features = self.visual_embeddings_table(visual_indicator) image_outputs.pooler_output = image_features - # NOTE: @Tom let's readd visual_indicator_features to the output + image_outputs.visual_indicator_features = self.visual_embeddings_table(visual_indicator) return image_outputs diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index a44db4045ef9..0f7520be8d97 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -22,10 +22,11 @@ from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils.generic import check_model_inputs from ..aimv2.modeling_aimv2 import Aimv2Attention, Aimv2EncoderLayer from ..auto import AutoModel from ..llama.modeling_llama import LlamaMLP, LlamaRMSNorm @@ -46,24 +47,13 @@ def hard_softmax(logits: torch.Tensor, dim: int): @dataclass -class BaseModelOutputWithVisualIndicatorFeatures(ModelOutput): +@auto_docstring +class BaseModelOutputWithVisualIndicatorFeatures(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - visual_indicator_features (`torch.FloatTensor` of shape `(batch_size, visual_indicator_size)`): - Visual indicator features extracted from the model, which can be used for auxiliary tasks or further processing. + visual_indicator_features (`torch.FloatTensor` of shape `(batch_size, visual_indicator_size)`): + Visual indicator features extracted from the model, which can be used for auxiliary tasks or further processing. """ - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None visual_indicator_features: Optional[torch.FloatTensor] = None @@ -199,8 +189,9 @@ def __init__(self, config: Ovis2VisionConfig): ) self.head_norm = nn.LayerNorm(self.vocab_size - self.num_visual_indicator_tokens) + @check_model_inputs(tie_last_hidden_states=False) def forward( - self, pixel_values: torch.FloatTensor, return_dict: bool = False, **kwargs + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple[torch.Tensor, torch.Tensor]: outputs = self.transformer(pixel_values, **kwargs) last_hidden_state = outputs[0] @@ -234,13 +225,10 @@ def forward( elif self.config.tokenize_function == "softmax": prob_token = nn.functional.softmax(logits, dim=-1) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=last_hidden_state, - pooler_output=prob_token, - ) - - return prob_token + return BaseModelOutputWithVisualIndicatorFeatures( + last_hidden_state=last_hidden_state, + pooler_output=prob_token, + ) class Ovis2Model(LlavaModel): @@ -281,9 +269,8 @@ def get_image_features( self.visual_vocab_size, dtype=torch.long, ).to(image_features.device) - visual_indicator_features = self.visual_embeddings_table(visual_indicator) image_outputs.pooler_output = image_features - # NOTE: @Tom let's readd visual_indicator_features to the output + image_outputs.visual_indicator_features = self.visual_embeddings_table(visual_indicator) return image_outputs diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index fde95d05c2e5..55f2cf896bbc 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -1295,9 +1295,8 @@ def forward( inputs_embeds = self.language_model.embed_tokens(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw).to( - inputs_embeds.device, inputs_embeds.dtype - ) + image_embeds = self.get_image_features(pixel_values, image_grid_thw, return_dict=True).pooler_output + image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype) image_mask = self.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds) inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) diff --git a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py index 7cfefc38edde..163585346c5e 100644 --- a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py @@ -1190,9 +1190,8 @@ def forward( inputs_embeds = self.language_model.embed_tokens(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw).to( - inputs_embeds.device, inputs_embeds.dtype - ) + image_embeds = self.get_image_features(pixel_values, image_grid_thw, return_dict=True).pooler_output + image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype) image_mask = self.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds) inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index c218a50c0c0b..5bac0dc480b0 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -379,7 +379,7 @@ def forward( # Merge text and images if pixel_values is not None: - image_features = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/perception_lm/modeling_perception_lm.py b/src/transformers/models/perception_lm/modeling_perception_lm.py index d82668708723..f157bf1c135a 100644 --- a/src/transformers/models/perception_lm/modeling_perception_lm.py +++ b/src/transformers/models/perception_lm/modeling_perception_lm.py @@ -280,7 +280,7 @@ def forward( image_features = None if pixel_values is not None: - image_features = self.get_image_features(pixel_values=pixel_values) + image_features = self.get_image_features(pixel_values=pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, dtype=inputs_embeds.dtype) special_image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features @@ -289,7 +289,7 @@ def forward( video_features = None if pixel_values_videos is not None: - video_features = self.get_image_features(pixel_values=pixel_values_videos) + video_features = self.get_image_features(pixel_values=pixel_values_videos, return_dict=True).pooler_output video_features = video_features.to(inputs_embeds.device, dtype=inputs_embeds.dtype) _, special_video_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_features diff --git a/src/transformers/models/perception_lm/modular_perception_lm.py b/src/transformers/models/perception_lm/modular_perception_lm.py index 5f4f20b1bebe..25d426334efd 100644 --- a/src/transformers/models/perception_lm/modular_perception_lm.py +++ b/src/transformers/models/perception_lm/modular_perception_lm.py @@ -246,7 +246,7 @@ def forward( image_features = None if pixel_values is not None: - image_features = self.get_image_features(pixel_values=pixel_values) + image_features = self.get_image_features(pixel_values=pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, dtype=inputs_embeds.dtype) special_image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features @@ -255,7 +255,7 @@ def forward( video_features = None if pixel_values_videos is not None: - video_features = self.get_image_features(pixel_values=pixel_values_videos) + video_features = self.get_image_features(pixel_values=pixel_values_videos, return_dict=True).pooler_output video_features = video_features.to(inputs_embeds.device, dtype=inputs_embeds.dtype) _, special_video_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_features diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index aae33b172211..3c0a1c3416ca 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -1943,13 +1943,14 @@ def forward( input_features, feature_attention_mask=feature_attention_mask, audio_feature_lengths=audio_feature_lengths, - ) + return_dict=True, + ).last_hidden_state audio_features = audio_features.to(inputs_embeds.device, inputs_embeds.dtype) _, _, audio_mask = self.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds) inputs_embeds = inputs_embeds.masked_scatter(audio_mask, audio_features) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = self.get_image_features(pixel_values, image_grid_thw, return_dict=True).pooler_output image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -1957,7 +1958,7 @@ def forward( inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw, return_dict=True).pooler_output video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index 270588b01a72..c353ca9d381f 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -2294,13 +2294,14 @@ def forward( input_features, feature_attention_mask=feature_attention_mask, audio_feature_lengths=audio_feature_lengths, - ) + return_dict=True, + ).last_hidden_state audio_features = audio_features.to(inputs_embeds.device, inputs_embeds.dtype) _, _, audio_mask = self.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds) inputs_embeds = inputs_embeds.masked_scatter(audio_mask, audio_features) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = self.get_image_features(pixel_values, image_grid_thw, return_dict=True).pooler_output image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -2308,7 +2309,7 @@ def forward( inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw, return_dict=True).pooler_output video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index 2d49b0ef675a..13f63f2c1fe9 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -1299,7 +1299,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = self.get_image_features(pixel_values, image_grid_thw, return_dict=True).pooler_output image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -1307,7 +1307,7 @@ def forward( inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw, return_dict=True).pooler_output video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds diff --git a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py index 7f79a7b38634..607598bb6c87 100644 --- a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py @@ -590,7 +590,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = self.get_image_features(pixel_values, image_grid_thw, return_dict=True).pooler_output image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -598,7 +598,7 @@ def forward( inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw, return_dict=True).pooler_output video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index e162efa73753..5ffeb4265328 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -44,7 +44,7 @@ can_return_tuple, logging, ) -from ...utils.generic import maybe_autocast +from ...utils.generic import check_model_inputs, maybe_autocast from ..qwen2.modeling_qwen2 import ( Qwen2RMSNorm, ) @@ -728,19 +728,17 @@ def rot_pos_emb(self, grid_thw): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, - return_dict: bool = False, - **kwargs, + **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: r""" grid_thw (`torch.LongTensor` of shape `(num_images, 3)`): The temporal, height and width dimensions of feature shape for each image. Each row contains [t, h, w] values. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a `ModelOutput` instead of exclusively the merged hidden states. """ hidden_states = self.patch_embed(hidden_states) rotary_pos_emb = self.rot_pos_emb(grid_thw) @@ -767,13 +765,10 @@ def forward( merged_hidden_states = self.merger(hidden_states) - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=hidden_states, - pooler_output=merged_hidden_states, - ) - - return self.merger(hidden_states) + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) @auto_docstring @@ -1231,7 +1226,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = self.get_image_features(pixel_values, image_grid_thw, return_dict=True).pooler_output image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -1239,7 +1234,7 @@ def forward( inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw, return_dict=True).pooler_output video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 653c067046e5..e335593192af 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -2156,13 +2156,18 @@ def forward( input_features, feature_attention_mask=feature_attention_mask, audio_feature_lengths=audio_feature_lengths, - ) + return_dict=True, + ).last_hidden_state audio_features = audio_features.to(inputs_embeds.device, inputs_embeds.dtype) _, _, audio_mask = self.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds) inputs_embeds = inputs_embeds.masked_scatter(audio_mask, audio_features) if pixel_values is not None: - image_embeds, image_embeds_multiscale = self.get_image_features(pixel_values, image_grid_thw) + image_outputs: BaseModelOutputWithDeepstackFeatures = self.get_image_features( + pixel_values, image_grid_thw, return_dict=True + ) + image_embeds = image_outputs.pooler_output + image_embeds_multiscale = image_outputs.deepstack_features image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -2170,7 +2175,9 @@ def forward( inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds, video_embeds_multiscale = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds, video_embeds_multiscale = self.get_video_features( + pixel_values_videos, video_grid_thw, return_dict=True + ).pooler_output video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask, _ = self.get_placeholder_mask( diff --git a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py index 32a101ba31cc..0fd7766665ce 100644 --- a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py @@ -1496,13 +1496,18 @@ def forward( input_features, feature_attention_mask=feature_attention_mask, audio_feature_lengths=audio_feature_lengths, - ) + return_dict=True, + ).last_hidden_state audio_features = audio_features.to(inputs_embeds.device, inputs_embeds.dtype) _, _, audio_mask = self.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds) inputs_embeds = inputs_embeds.masked_scatter(audio_mask, audio_features) if pixel_values is not None: - image_embeds, image_embeds_multiscale = self.get_image_features(pixel_values, image_grid_thw) + image_outputs: BaseModelOutputWithDeepstackFeatures = self.get_image_features( + pixel_values, image_grid_thw, return_dict=True + ) + image_embeds = image_outputs.pooler_output + image_embeds_multiscale = image_outputs.deepstack_features image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -1510,7 +1515,9 @@ def forward( inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds, video_embeds_multiscale = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds, video_embeds_multiscale = self.get_video_features( + pixel_values_videos, video_grid_thw, return_dict=True + ).pooler_output video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask, _ = self.get_placeholder_mask( diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index a83f682d68ef..9ea3e1791088 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -44,7 +44,9 @@ @dataclass -class BaseModelOutputWithDeepstackFeatures(ModelOutput): +class BaseModelOutputWithDeepstackFeatures( + ModelOutput +): # NOTE: @Tom Should we subclass from BaseModelOutputWithPooler instead? """ Base class for model's outputs that also contains a pooling of the last hidden states. @@ -1205,7 +1207,11 @@ def forward( video_mask = None if pixel_values is not None: - image_embeds, deepstack_image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_outputs: BaseModelOutputWithDeepstackFeatures = self.get_image_features( + pixel_values, image_grid_thw, return_dict=True + ) + image_embeds = image_outputs.pooler_output + deepstack_image_embeds = image_outputs.deepstack_features image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -1213,7 +1219,11 @@ def forward( inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds, deepstack_video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_outputs: BaseModelOutputWithDeepstackFeatures = self.get_video_features( + pixel_values_videos, video_grid_thw, return_dict=True + ) + video_embeds = video_outputs.pooler_output + deepstack_video_embeds = video_outputs.deepstack_features video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index 311949ce52eb..63299a8524d3 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -67,7 +67,9 @@ @dataclass -class BaseModelOutputWithDeepstackFeatures(ModelOutput): +class BaseModelOutputWithDeepstackFeatures( + ModelOutput +): # NOTE: @Tom Should we subclass from BaseModelOutputWithPooler instead? """ Base class for model's outputs that also contains a pooling of the last hidden states. @@ -1037,7 +1039,11 @@ def forward( video_mask = None if pixel_values is not None: - image_embeds, deepstack_image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_outputs: BaseModelOutputWithDeepstackFeatures = self.get_image_features( + pixel_values, image_grid_thw, return_dict=True + ) + image_embeds = image_outputs.pooler_output + deepstack_image_embeds = image_outputs.deepstack_features image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -1045,7 +1051,11 @@ def forward( inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds, deepstack_video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_outputs: BaseModelOutputWithDeepstackFeatures = self.get_video_features( + pixel_values_videos, video_grid_thw, return_dict=True + ) + video_embeds = video_outputs.pooler_output + deepstack_video_embeds = video_outputs.deepstack_features video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index fb79ceb74602..fb9f3b02f478 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -421,7 +421,9 @@ def _init_weights(self, module): @dataclass -class BaseModelOutputWithDeepstackFeatures(ModelOutput): +class BaseModelOutputWithDeepstackFeatures( + ModelOutput +): # NOTE: @Tom Should we subclass from BaseModelOutputWithPooler instead? """ Base class for model's outputs that also contains a pooling of the last hidden states. @@ -1362,7 +1364,11 @@ def forward( video_mask = None if pixel_values is not None: - image_embeds, deepstack_image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_outputs: BaseModelOutputWithDeepstackFeatures = self.get_image_features( + pixel_values, image_grid_thw, return_dict=True + ) + image_embeds = image_outputs.pooler_output + deepstack_image_embeds = image_outputs.deepstack_features image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -1370,7 +1376,11 @@ def forward( inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds, deepstack_video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_outputs: BaseModelOutputWithDeepstackFeatures = self.get_video_features( + pixel_values_videos, video_grid_thw, return_dict=True + ) + video_embeds = video_outputs.pooler_output + deepstack_video_embeds = video_outputs.deepstack_features video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds diff --git a/src/transformers/models/sam2/modeling_sam2.py b/src/transformers/models/sam2/modeling_sam2.py index fee83133d696..f620869a0a5b 100644 --- a/src/transformers/models/sam2/modeling_sam2.py +++ b/src/transformers/models/sam2/modeling_sam2.py @@ -1359,7 +1359,8 @@ def get_image_embeddings( Input pixel values """ batch_size = pixel_values.shape[0] - feature_maps, _, _, _ = self.get_image_features(pixel_values, **kwargs) + image_outputs = self.get_image_features(pixel_values, return_dict=True, **kwargs) + feature_maps = image_outputs.fpn_hidden_states # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding @@ -1515,10 +1516,10 @@ def forward( vision_hidden_states = None if pixel_values is not None: - feature_maps, _, vision_hidden_states, vision_attentions = self.get_image_features( - pixel_values, - **kwargs, - ) + image_outputs: Sam2VisionEncoderOutput = self.get_image_features(pixel_values, return_dict=True, **kwargs) + feature_maps = image_outputs.fpn_hidden_states + vision_hidden_states = image_outputs.hidden_states + vision_attentions = image_outputs.attentions # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding diff --git a/src/transformers/models/sam2/modular_sam2.py b/src/transformers/models/sam2/modular_sam2.py index 2d93a47a3174..39b0e27c1cfe 100644 --- a/src/transformers/models/sam2/modular_sam2.py +++ b/src/transformers/models/sam2/modular_sam2.py @@ -1254,7 +1254,8 @@ def get_image_embeddings( Input pixel values """ batch_size = pixel_values.shape[0] - feature_maps, _, _, _ = self.get_image_features(pixel_values, **kwargs) + image_outputs = self.get_image_features(pixel_values, return_dict=True, **kwargs) + feature_maps = image_outputs.fpn_hidden_states # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding @@ -1426,10 +1427,10 @@ def forward( vision_hidden_states = None if pixel_values is not None: - feature_maps, _, vision_hidden_states, vision_attentions = self.get_image_features( - pixel_values, - **kwargs, - ) + image_outputs: Sam2VisionEncoderOutput = self.get_image_features(pixel_values, return_dict=True, **kwargs) + feature_maps = image_outputs.fpn_hidden_states + vision_hidden_states = image_outputs.hidden_states + vision_attentions = image_outputs.attentions # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding diff --git a/src/transformers/models/sam2_video/modeling_sam2_video.py b/src/transformers/models/sam2_video/modeling_sam2_video.py index c72b4e66ad86..fc633a77a93c 100644 --- a/src/transformers/models/sam2_video/modeling_sam2_video.py +++ b/src/transformers/models/sam2_video/modeling_sam2_video.py @@ -1645,7 +1645,8 @@ def get_image_embeddings( Input pixel values """ batch_size = pixel_values.shape[0] - feature_maps, _, _, _ = self.get_image_features(pixel_values, **kwargs) + image_outputs = self.get_image_features(pixel_values, return_dict=True, **kwargs) + feature_maps = image_outputs.fpn_hidden_states # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding @@ -1880,7 +1881,9 @@ def _prepare_vision_features( else: # Compute features using image encoder image_batch = inference_session.get_frame(frame_idx).unsqueeze(0) # Add batch dimension - vision_feats, vision_pos_embeds, _, _ = self.get_image_features(image_batch) + image_outputs = self.get_image_features(image_batch, return_dict=True) + vision_feats = image_outputs.fpn_hidden_states + vision_pos_embeds = image_outputs.fpn_position_embeddings # Cache features inference_session.cache.cache_vision_features( frame_idx, {"vision_feats": vision_feats, "vision_pos_embeds": vision_pos_embeds} @@ -1985,10 +1988,10 @@ def _single_frame_forward( vision_hidden_states = None if pixel_values is not None: - feature_maps, _, vision_hidden_states, vision_attentions = self.get_image_features( - pixel_values, - **kwargs, - ) + image_outputs = self.get_image_features(pixel_values, return_dict=True, **kwargs) + feature_maps = image_outputs.fpn_hidden_states + vision_hidden_states = image_outputs.hidden_states + vision_attentions = image_outputs.attentions # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding diff --git a/src/transformers/models/sam2_video/modular_sam2_video.py b/src/transformers/models/sam2_video/modular_sam2_video.py index e876ee5bda5b..e4728d0e4e2c 100644 --- a/src/transformers/models/sam2_video/modular_sam2_video.py +++ b/src/transformers/models/sam2_video/modular_sam2_video.py @@ -1540,7 +1540,9 @@ def _prepare_vision_features( else: # Compute features using image encoder image_batch = inference_session.get_frame(frame_idx).unsqueeze(0) # Add batch dimension - vision_feats, vision_pos_embeds, _, _ = self.get_image_features(image_batch) + image_outputs = self.get_image_features(image_batch, return_dict=True) + vision_feats = image_outputs.fpn_hidden_states + vision_pos_embeds = image_outputs.fpn_position_embeddings # Cache features inference_session.cache.cache_vision_features( frame_idx, {"vision_feats": vision_feats, "vision_pos_embeds": vision_pos_embeds} @@ -1645,10 +1647,10 @@ def _single_frame_forward( vision_hidden_states = None if pixel_values is not None: - feature_maps, _, vision_hidden_states, vision_attentions = self.get_image_features( - pixel_values, - **kwargs, - ) + image_outputs = self.get_image_features(pixel_values, return_dict=True, **kwargs) + feature_maps = image_outputs.fpn_hidden_states + vision_hidden_states = image_outputs.hidden_states + vision_attentions = image_outputs.attentions # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding diff --git a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py index 2c2e6be99dff..546592c0c3a7 100644 --- a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py +++ b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py @@ -829,7 +829,8 @@ def get_image_embeddings( Input pixel values """ batch_size = pixel_values.shape[0] - feature_maps, _, _, _ = self.get_image_features(pixel_values, **kwargs) + image_outputs = self.get_image_features(pixel_values, return_dict=True, **kwargs) + feature_maps = image_outputs.fpn_hidden_states # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding @@ -985,10 +986,12 @@ def forward( vision_hidden_states = None if pixel_values is not None: - feature_maps, _, vision_hidden_states, vision_attentions = self.get_image_features( - pixel_values, - **kwargs, + image_outputs: Sam3TrackerVisionEncoderOutput = self.get_image_features( + pixel_values, return_dict=True, **kwargs ) + feature_maps = image_outputs.fpn_hidden_states + vision_hidden_states = image_outputs.hidden_states + vision_attentions = image_outputs.attentions # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding diff --git a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py index cf5a33ba8c60..b7340b3d5339 100644 --- a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py @@ -1696,7 +1696,8 @@ def get_image_embeddings( Input pixel values """ batch_size = pixel_values.shape[0] - feature_maps, _, _, _ = self.get_image_features(pixel_values, **kwargs) + image_outputs = self.get_image_features(pixel_values, return_dict=True, **kwargs) + feature_maps = image_outputs.fpn_hidden_states # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding @@ -1931,7 +1932,9 @@ def _prepare_vision_features( else: # Compute features using image encoder image_batch = inference_session.get_frame(frame_idx).unsqueeze(0) # Add batch dimension - vision_feats, vision_pos_embeds, _, _ = self.get_image_features(image_batch) + image_outputs = self.get_image_features(image_batch, return_dict=True) + vision_feats = image_outputs.fpn_hidden_states + vision_pos_embeds = image_outputs.fpn_position_embeddings # Cache features inference_session.cache.cache_vision_features( frame_idx, {"vision_feats": vision_feats, "vision_pos_embeds": vision_pos_embeds} @@ -2036,10 +2039,10 @@ def _single_frame_forward( vision_hidden_states = None if pixel_values is not None: - feature_maps, _, vision_hidden_states, vision_attentions = self.get_image_features( - pixel_values, - **kwargs, - ) + image_outputs = self.get_image_features(pixel_values, return_dict=True, **kwargs) + feature_maps = image_outputs.fpn_hidden_states + vision_hidden_states = image_outputs.hidden_states + vision_attentions = image_outputs.attentions # add no memory embedding to the last feature map feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding diff --git a/src/transformers/models/smolvlm/modeling_smolvlm.py b/src/transformers/models/smolvlm/modeling_smolvlm.py index adac23d26098..31e903411b43 100644 --- a/src/transformers/models/smolvlm/modeling_smolvlm.py +++ b/src/transformers/models/smolvlm/modeling_smolvlm.py @@ -641,7 +641,10 @@ def forward( raise ValueError("You cannot specify both pixel_values and image_hidden_states at the same time") if pixel_values is not None: - image_hidden_states = self.get_image_features(pixel_values, pixel_attention_mask).to(inputs_embeds.device) + image_hidden_states = self.get_image_features( + pixel_values, pixel_attention_mask, return_dict=True + ).pooler_output + image_hidden_states = image_hidden_states.to(inputs_embeds.device) elif image_hidden_states is not None: image_hidden_states = image_hidden_states.to(dtype=self.dtype, device=inputs_embeds.device) diff --git a/src/transformers/models/smolvlm/modular_smolvlm.py b/src/transformers/models/smolvlm/modular_smolvlm.py index 6b78ea7ea181..341dfd7bd1c4 100644 --- a/src/transformers/models/smolvlm/modular_smolvlm.py +++ b/src/transformers/models/smolvlm/modular_smolvlm.py @@ -310,7 +310,10 @@ def forward( raise ValueError("You cannot specify both pixel_values and image_hidden_states at the same time") if pixel_values is not None: - image_hidden_states = self.get_image_features(pixel_values, pixel_attention_mask).to(inputs_embeds.device) + image_hidden_states = self.get_image_features( + pixel_values, pixel_attention_mask, return_dict=True + ).pooler_output + image_hidden_states = image_hidden_states.to(inputs_embeds.device) elif image_hidden_states is not None: image_hidden_states = image_hidden_states.to(dtype=self.dtype, device=inputs_embeds.device) diff --git a/src/transformers/models/t5gemma2/modeling_t5gemma2.py b/src/transformers/models/t5gemma2/modeling_t5gemma2.py index 6b6a512af3ea..9f22f0ba902a 100644 --- a/src/transformers/models/t5gemma2/modeling_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modeling_t5gemma2.py @@ -851,7 +851,7 @@ def preprocess_image_features( inputs_embeds: Optional[torch.FloatTensor] = None, ): """Convert pixel images to image features and merge into input embeds.""" - image_features = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) image_mask = self.get_image_placeholder_mask( diff --git a/src/transformers/models/t5gemma2/modular_t5gemma2.py b/src/transformers/models/t5gemma2/modular_t5gemma2.py index e8d0447d82aa..a37d070a12e8 100644 --- a/src/transformers/models/t5gemma2/modular_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modular_t5gemma2.py @@ -630,7 +630,7 @@ def preprocess_image_features( inputs_embeds: Optional[torch.FloatTensor] = None, ): """Convert pixel images to image features and merge into input embeds.""" - image_features = self.get_image_features(pixel_values) + image_features = self.get_image_features(pixel_values, return_dict=True).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) image_mask = self.get_image_placeholder_mask( diff --git a/src/transformers/models/video_llama_3/modeling_video_llama_3.py b/src/transformers/models/video_llama_3/modeling_video_llama_3.py index 59f8f8e99d4e..30ea3d9b31c1 100644 --- a/src/transformers/models/video_llama_3/modeling_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modeling_video_llama_3.py @@ -674,7 +674,9 @@ def forward( image_embeds = None if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw, image_merge_sizes) + image_embeds = self.get_image_features( + pixel_values, image_grid_thw, image_merge_sizes, return_dict=True + ).pooler_output image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -683,7 +685,9 @@ def forward( video_embeds = None if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw, video_merge_sizes) + video_embeds = self.get_video_features( + pixel_values_videos, video_grid_thw, video_merge_sizes, return_dict=True + ).pooler_output video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) if video_compression_mask is not None: video_embeds = video_embeds[video_compression_mask.to(video_embeds.device)] diff --git a/src/transformers/models/video_llama_3/modular_video_llama_3.py b/src/transformers/models/video_llama_3/modular_video_llama_3.py index 04d23b7a0231..a6f4b7af8b04 100644 --- a/src/transformers/models/video_llama_3/modular_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modular_video_llama_3.py @@ -680,7 +680,9 @@ def forward( image_embeds = None if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw, image_merge_sizes) + image_embeds = self.get_image_features( + pixel_values, image_grid_thw, image_merge_sizes, return_dict=True + ).pooler_output image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -689,7 +691,9 @@ def forward( video_embeds = None if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw, video_merge_sizes) + video_embeds = self.get_video_features( + pixel_values_videos, video_grid_thw, video_merge_sizes, return_dict=True + ).pooler_output video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) if video_compression_mask is not None: video_embeds = video_embeds[video_compression_mask.to(video_embeds.device)] diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index faea5fdba6cd..d1dc00e5ba89 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -364,7 +364,8 @@ def forward( pixel_values_images, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - ) + return_dict=True, + ).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features @@ -373,8 +374,8 @@ def forward( if pixel_values_videos is not None: video_features = self.get_video_features( - pixel_values_videos=pixel_values_videos, vision_feature_layer=vision_feature_layer - ) + pixel_values_videos=pixel_values_videos, vision_feature_layer=vision_feature_layer, return_dict=True + ).pooler_output video_features = video_features.to(inputs_embeds.device, inputs_embeds.dtype) _, special_video_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_features diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index e8b4489d478e..b8532f7cab99 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -252,8 +252,8 @@ def forward( if pixel_values is not None: image_features = self.get_image_features( - pixel_values=pixel_values, vision_feature_layers=vision_feature_layers - ) + pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, return_dict=True + ).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/vipllava/modular_vipllava.py b/src/transformers/models/vipllava/modular_vipllava.py index faea2f645ff5..a0f638c232b5 100644 --- a/src/transformers/models/vipllava/modular_vipllava.py +++ b/src/transformers/models/vipllava/modular_vipllava.py @@ -150,8 +150,8 @@ def forward( if pixel_values is not None: image_features = self.get_image_features( - pixel_values=pixel_values, vision_feature_layers=vision_feature_layers - ) + pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, return_dict=True + ).pooler_output image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/voxtral/modeling_voxtral.py b/src/transformers/models/voxtral/modeling_voxtral.py index f649364d21be..8d1f1c649c57 100644 --- a/src/transformers/models/voxtral/modeling_voxtral.py +++ b/src/transformers/models/voxtral/modeling_voxtral.py @@ -484,7 +484,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if input_features is not None and input_ids is not None: - audio_embeds = self.get_audio_features(input_features) + audio_embeds = self.get_audio_features(input_features, return_dict=True).pooler_output # replace text-audio token placeholders with audio embeddings audio_token_mask = (input_ids == self.config.audio_token_id).unsqueeze(-1) diff --git a/src/transformers/models/voxtral/modular_voxtral.py b/src/transformers/models/voxtral/modular_voxtral.py index 42ace2c52b05..30bec55e69ce 100644 --- a/src/transformers/models/voxtral/modular_voxtral.py +++ b/src/transformers/models/voxtral/modular_voxtral.py @@ -250,7 +250,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if input_features is not None and input_ids is not None: - audio_embeds = self.get_audio_features(input_features) + audio_embeds = self.get_audio_features(input_features, return_dict=True).pooler_output # replace text-audio token placeholders with audio embeddings audio_token_mask = (input_ids == self.config.audio_token_id).unsqueeze(-1) From 006b2a54471afd6ec58bc5b0ebed461e20f6a33a Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 17 Dec 2025 18:31:38 +0100 Subject: [PATCH 024/131] Update fast_vlm modeling forward from modular llava to remove image_sizes --- .../models/fast_vlm/modeling_fast_vlm.py | 6 +- .../models/fast_vlm/modular_fast_vlm.py | 134 +++++++++++++++++- 2 files changed, 130 insertions(+), 10 deletions(-) diff --git a/src/transformers/models/fast_vlm/modeling_fast_vlm.py b/src/transformers/models/fast_vlm/modeling_fast_vlm.py index de098951ca3d..138fc047d2bd 100644 --- a/src/transformers/models/fast_vlm/modeling_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modeling_fast_vlm.py @@ -153,7 +153,7 @@ def get_image_features( image_features = self.multi_modal_projector(selected_image_feature) image_outputs.pooler_output = list(image_features) - return image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -192,7 +192,6 @@ def forward( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, cache_position: Optional[torch.LongTensor] = None, - image_sizes: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], ) -> Union[tuple, FastVlmModelOutputWithPast]: r""" @@ -222,7 +221,6 @@ def forward( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, - image_sizes=image_sizes, return_dict=True, ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) @@ -332,7 +330,6 @@ def forward( labels: Optional[torch.LongTensor] = None, cache_position: Optional[torch.LongTensor] = None, logits_to_keep: Union[int, torch.Tensor] = 0, - image_sizes: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], ) -> Union[tuple, FastVlmCausalLMOutputWithPast]: r""" @@ -399,7 +396,6 @@ def forward( vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, cache_position=cache_position, - image_sizes=image_sizes, **kwargs, ) diff --git a/src/transformers/models/fast_vlm/modular_fast_vlm.py b/src/transformers/models/fast_vlm/modular_fast_vlm.py index 87173c060048..b641b4a6e102 100644 --- a/src/transformers/models/fast_vlm/modular_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modular_fast_vlm.py @@ -18,14 +18,17 @@ from torch import nn from ...activations import ACT2FN +from ...cache_utils import Cache from ...configuration_utils import PreTrainedConfig from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ..auto import CONFIG_MAPPING from ..llava.configuration_llava import LlavaConfig from ..llava.modeling_llava import ( + LlavaCausalLMOutputWithPast, LlavaForConditionalGeneration, LlavaModel, + LlavaModelOutputWithPast, LlavaMultiModalProjector, LlavaPreTrainedModel, ) @@ -157,6 +160,10 @@ class FastVlmPreTrainedModel(LlavaPreTrainedModel): pass +class FastVlmModelOutputWithPast(LlavaModelOutputWithPast): + pass + + class FastVlmModel(LlavaModel): _checkpoint_conversion_mapping = {} @@ -202,9 +209,23 @@ def get_image_features( image_features = self.multi_modal_projector(selected_image_feature) image_outputs.pooler_output = list(image_features) - return image_features + return image_outputs - def forward(self, **super_kwargs): + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + vision_feature_layer: Optional[Union[int, list[int]]] = None, + vision_feature_select_strategy: Optional[str] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, FastVlmModelOutputWithPast]: r""" vision_feature_layer (`Union[int, list[int], NoneType]`, *optional*): The index of the layer to select the vision feature. If multiple indices are provided, the vision feature of the @@ -212,7 +233,54 @@ def forward(self, **super_kwargs): vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Only "full" supported. """ - super().forward(**super_kwargs) + vision_feature_layer = ( + vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer + ) + vision_feature_select_strategy = ( + vision_feature_select_strategy + if vision_feature_select_strategy is not None + else self.config.vision_feature_select_strategy + ) + + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds = self.get_input_embeddings()(input_ids) + + if pixel_values is not None: + image_features = self.get_image_features( + pixel_values=pixel_values, + vision_feature_layer=vision_feature_layer, + vision_feature_select_strategy=vision_feature_select_strategy, + return_dict=True, + ).pooler_output + image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) + special_image_mask = self.get_placeholder_mask( + input_ids, inputs_embeds=inputs_embeds, image_features=image_features + ) + inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features) + + outputs = self.language_model( + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + cache_position=cache_position, + **kwargs, + ) + + return FastVlmModelOutputWithPast( + last_hidden_state=outputs.last_hidden_state, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + image_hidden_states=image_features if pixel_values is not None else None, + ) + + +class FastVlmCausalLMOutputWithPast(LlavaCausalLMOutputWithPast): + pass @auto_docstring( @@ -223,7 +291,23 @@ def forward(self, **super_kwargs): class FastVlmForConditionalGeneration(LlavaForConditionalGeneration): _checkpoint_conversion_mapping = {} - def forward(self, **super_kwargs): + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + vision_feature_layer: Optional[Union[int, list[int]]] = None, + vision_feature_select_strategy: Optional[str] = None, + labels: Optional[torch.LongTensor] = None, + cache_position: Optional[torch.LongTensor] = None, + logits_to_keep: Union[int, torch.Tensor] = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, FastVlmCausalLMOutputWithPast]: r""" vision_feature_layer (`Union[int, list[int], NoneType]`, *optional*): The index of the layer to select the vision feature. If multiple indices are provided, the vision feature of the @@ -269,7 +353,47 @@ def forward(self, **super_kwargs): >>> print(processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]) system\n You are a helpful assistant.\n user\n What are these?\n assistant\n The image depicts a traditional Chinese street... ```""" - super().forward(**super_kwargs) + vision_feature_layer = ( + vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer + ) + vision_feature_select_strategy = ( + vision_feature_select_strategy + if vision_feature_select_strategy is not None + else self.config.vision_feature_select_strategy + ) + + outputs = self.model( + input_ids=input_ids, + pixel_values=pixel_values, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + vision_feature_layer=vision_feature_layer, + vision_feature_select_strategy=vision_feature_select_strategy, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = outputs[0] + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep + logits = self.lm_head(hidden_states[:, slice_indices, :]) + + loss = None + if labels is not None: + loss = self.loss_function( + logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size, **kwargs + ) + + return FastVlmCausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + image_hidden_states=outputs.image_hidden_states, + ) __all__ = ["FastVlmForConditionalGeneration", "FastVlmModel", "FastVlmPreTrainedModel", "FastVlmConfig"] From 1d6639b7e187eaaf062659ae856b48f34e9de141 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 18 Dec 2025 14:19:39 +0100 Subject: [PATCH 025/131] Update colqwen2 its self.vlm.model.visual call to expect BaseModelOutput --- src/transformers/models/colqwen2/modeling_colqwen2.py | 4 +++- src/transformers/models/colqwen2/modular_colqwen2.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/colqwen2/modeling_colqwen2.py b/src/transformers/models/colqwen2/modeling_colqwen2.py index b611797a9346..819b1272f2a2 100644 --- a/src/transformers/models/colqwen2/modeling_colqwen2.py +++ b/src/transformers/models/colqwen2/modeling_colqwen2.py @@ -175,7 +175,9 @@ def forward( inputs_embeds = self.vlm.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.vlm.model.visual(pixel_values, grid_thw=image_grid_thw) + image_embeds = self.vlm.model.visual( + pixel_values, grid_thw=image_grid_thw, return_dict=True + ).pooler_output image_mask = ( (input_ids == self.config.vlm_config.image_token_id).unsqueeze(-1).expand_as(inputs_embeds) ) diff --git a/src/transformers/models/colqwen2/modular_colqwen2.py b/src/transformers/models/colqwen2/modular_colqwen2.py index 6f98205bf376..30f1981b2f0a 100644 --- a/src/transformers/models/colqwen2/modular_colqwen2.py +++ b/src/transformers/models/colqwen2/modular_colqwen2.py @@ -356,7 +356,9 @@ def forward( inputs_embeds = self.vlm.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.vlm.model.visual(pixel_values, grid_thw=image_grid_thw) + image_embeds = self.vlm.model.visual( + pixel_values, grid_thw=image_grid_thw, return_dict=True + ).pooler_output image_mask = ( (input_ids == self.config.vlm_config.image_token_id).unsqueeze(-1).expand_as(inputs_embeds) ) From d52def3716424f71df800f7a3e1d95d2a7d1557e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 18 Dec 2025 14:29:44 +0100 Subject: [PATCH 026/131] Replace prior return_dict with check_model_inputs on qwen2_5_vl its VisionTransformer --- .../models/qwen2_5_vl/modeling_qwen2_5_vl.py | 18 +++++++----------- .../models/qwen2_5_vl/modular_qwen2_5_vl.py | 17 +++++++---------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index 4e56475ce4e9..86ea24e9f41a 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -43,7 +43,7 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging -from ...utils.generic import maybe_autocast +from ...utils.generic import check_model_inputs, maybe_autocast from ..qwen2.modeling_qwen2 import Qwen2RMSNorm from .configuration_qwen2_5_vl import Qwen2_5_VLConfig, Qwen2_5_VLTextConfig, Qwen2_5_VLVisionConfig @@ -408,8 +408,9 @@ def get_window_index(self, grid_thw): return window_index, cu_window_seqlens + @check_model_inputs(tie_last_hidden_states=False) def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> torch.Tensor: """ Args: @@ -417,8 +418,6 @@ def forward( The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -470,13 +469,10 @@ def forward( reverse_indices = torch.argsort(window_index) merged_hidden_states = merged_hidden_states[reverse_indices, :] - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=hidden_states, - pooler_output=merged_hidden_states, - ) - - return hidden_states + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) @dataclass diff --git a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py index de835cb1e9b8..8e7584b35e4f 100644 --- a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py @@ -53,6 +53,7 @@ from ...processing_utils import MultiModalData, ProcessingKwargs, Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput from ...utils import logging +from ...utils.generic import check_model_inputs from ...video_utils import VideoInput @@ -280,8 +281,9 @@ def get_window_index(self, grid_thw): return window_index, cu_window_seqlens + @check_model_inputs(tie_last_hidden_states=False) def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, return_dict: bool = False, **kwargs + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> torch.Tensor: """ Args: @@ -289,8 +291,6 @@ def forward( The final hidden states of the model. grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): The temporal, height and width of feature shape of each image in LLM. - return_dict (`bool`, *optional*, defaults to `False`): - Whether to return a `ModelOutput` instead of exclusively the merged hidden states. Returns: `torch.Tensor`: hidden_states. @@ -342,13 +342,10 @@ def forward( reverse_indices = torch.argsort(window_index) merged_hidden_states = merged_hidden_states[reverse_indices, :] - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=hidden_states, - pooler_output=merged_hidden_states, - ) - - return hidden_states + return BaseModelOutputWithPooling( + last_hidden_state=hidden_states, + pooler_output=merged_hidden_states, + ) class Qwen2_5_VLModelOutputWithPast(Qwen2VLModelOutputWithPast): From ff676635d66d2fd854fa7bb3ed703000d2d4a44e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 18 Dec 2025 14:35:45 +0100 Subject: [PATCH 027/131] Use BaseModelOutputWithProjectionAttentions for Kosmos2 to allow returning the projection attentions --- .../models/kosmos2/modeling_kosmos2.py | 35 +++++++------------ 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/src/transformers/models/kosmos2/modeling_kosmos2.py b/src/transformers/models/kosmos2/modeling_kosmos2.py index f59544238678..e634cc03b949 100644 --- a/src/transformers/models/kosmos2/modeling_kosmos2.py +++ b/src/transformers/models/kosmos2/modeling_kosmos2.py @@ -76,28 +76,17 @@ def _make_causal_mask( @dataclass -class BaseModelOutputWithProjectionAttentions(ModelOutput): +@auto_docstring +class BaseModelOutputWithProjectionAttentions(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. + projection_attentions (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - projection_attentions (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, - sequence_length)`. - - Attentions weights given by `Kosmos2ImageToTextProjection`, after the attention softmax, used to compute - the weighted average in the self-attention heads. + Attentions weights given by `Kosmos2ImageToTextProjection`, after the attention softmax, used to compute + the weighted average in the self-attention heads. """ - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None projection_attentions: Optional[tuple[torch.FloatTensor]] = None @@ -518,7 +507,7 @@ def forward( if output_hidden_states: encoder_states = encoder_states + (hidden_states,) - return BaseModelOutput( + return BaseModelOutputWithProjectionAttentions( last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions ) @@ -1227,7 +1216,7 @@ def forward( interpolate_pos_encoding: bool = False, return_dict: Optional[bool] = None, **kwargs, - ) -> Union[tuple, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithProjectionAttentions]: return self.model( pixel_values=pixel_values, output_attentions=output_attentions, @@ -1514,7 +1503,7 @@ def get_image_features( pixel_values: torch.FloatTensor, interpolate_pos_encoding: Optional[bool] = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[torch.FloatTensor, BaseModelOutputWithProjectionAttentions]: if "return_attentions" in kwargs: warnings.warn( "`return_attentions` is deprecated and will be removed in a future version. Please use `return_dict`" @@ -1522,7 +1511,7 @@ def get_image_features( FutureWarning, ) - vision_output = self.vision_model( + vision_output: BaseModelOutputWithProjectionAttentions = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, **kwargs, @@ -1533,7 +1522,7 @@ def get_image_features( image_embeds = nn.functional.normalize(image_embeds, dim=-1) image_embeds, projection_attentions = self.image_to_text_projection(image_embeds) vision_output.pooler_output = image_embeds - # NOTE: @Tom we need to add projection_attentions to the output + vision_output.projection_attentions = projection_attentions return vision_output From 22522c45a0f2d209928ab700b62295d95f6ab5f5 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 18 Dec 2025 14:47:56 +0100 Subject: [PATCH 028/131] Update Emu akin to Chameleon --- .../models/chameleon/modeling_chameleon.py | 4 +- src/transformers/models/emu3/modular_emu3.py | 62 ++++++++++++------- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 13e81c55f667..86a35d2169ea 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -50,10 +50,10 @@ @auto_docstring class ChameleonVQVAEModelOutput(BaseModelOutputWithPooling): r""" - image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):` - Indices of the image tokens predicted by the VQ-VAE model. quantized_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): Quantized last hidden state from the VQ-VAE model. + image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):` + Indices of the image tokens predicted by the VQ-VAE model. embedding_loss (`torch.FloatTensor`): The embedding loss computed during quantization. """ diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index 3b8fc5ddb144..844ae1f05c3f 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -15,6 +15,7 @@ # limitations under the License. import math +from dataclasses import dataclass from functools import cached_property from typing import Optional, Union @@ -29,6 +30,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import auto_docstring, can_return_tuple, logging +from ...utils.generic import check_model_inputs from ..chameleon.modeling_chameleon import ( ChameleonPreTrainedModel, ChameleonVQVAEEncoderConvDownsample, @@ -41,6 +43,17 @@ logger = logging.get_logger(__name__) +@dataclass +@auto_docstring +class Emu3VQVAEModelOutput(BaseModelOutputWithPooling): + r""" + image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):` + Indices of the image tokens predicted by the VQ-VAE model. + """ + + image_tokens: Optional[torch.FloatTensor] = None + + class Emu3Attention(LlamaAttention): pass @@ -733,7 +746,10 @@ def __init__(self, config: Emu3VQVAEConfig): self.post_init() - def encode(self, pixel_values: torch.Tensor, image_sizes: torch.Tensor): + @check_model_inputs(tie_last_hidden_states=False) + def encode( + self, pixel_values: torch.Tensor, image_sizes: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> Emu3VQVAEModelOutput: is_image = pixel_values.ndim == 4 if is_image: temporal = self.config.temporal_downsample_factor @@ -745,12 +761,11 @@ def encode(self, pixel_values: torch.Tensor, image_sizes: torch.Tensor): hidden_states = self.encoder(pixel_values) # b t c h w -> b c t h w - hidden_states = hidden_states.permute(0, 2, 1, 3, 4) - hidden_states = self.quant_conv(hidden_states) + conv_hidden_states = self.quant_conv(hidden_states.permute(0, 2, 1, 3, 4)) # b c t h w -> b t c h w - hidden_states = hidden_states.permute(0, 2, 1, 3, 4) - codes = self.quantize(hidden_states) + conv_hidden_states = conv_hidden_states.permute(0, 2, 1, 3, 4) + codes = self.quantize(conv_hidden_states) image_tokens = codes.squeeze(1) if is_image else codes @@ -759,7 +774,10 @@ def encode(self, pixel_values: torch.Tensor, image_sizes: torch.Tensor): for single_image, size in zip(image_tokens, image_sizes) ] - return image_tokens + return Emu3VQVAEModelOutput( + last_hidden_state=hidden_states, + image_tokens=image_tokens, + ) def decode(self, hidden_states: torch.Tensor): is_image = hidden_states.ndim == 3 @@ -920,13 +938,16 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.L image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`): The sizes of the images in the batch, being (height, width) for each image. """ - image_tokens_list = self.vqmodel.encode(pixel_values, image_sizes) - bpe_tokens_list = [self.vocabulary_mapping.convert_img2bpe(tokens).flatten() for tokens in image_tokens_list] + image_tokens_list: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, return_dict=True) + bpe_tokens_list = [ + self.vocabulary_mapping.convert_img2bpe(tokens).flatten() for tokens in image_tokens_list.image_tokens + ] bpe_tokens = torch.cat(bpe_tokens_list) return bpe_tokens + @can_return_tuple def get_image_features( - self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, return_dict: bool = False + self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs] ): """ Tokenizes images into discrete tokens with VQGAN module and embeds @@ -936,23 +957,16 @@ def get_image_features( pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): The tensors corresponding to the input images. """ - image_tokens = self.get_image_tokens(pixel_values, image_sizes) + vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, return_dict=True) split_sizes = [ (height // self.vqmodel.vision_spatial_factor) * (width // self.vqmodel.vision_spatial_factor + 1) for height, width in image_sizes ] - image_embeddings = self.get_input_embeddings()(image_tokens) + image_embeddings = self.get_input_embeddings()(vqmodel_outputs.image_tokens) image_features = torch.split(image_embeddings, split_sizes) + vqmodel_outputs.pooler_output = image_features - # TODO: @Tom to revisit with the new can_return_tuple approach. - # Presumably requires updating the `self.vqmodel.encode` method to return a BaseModelOutput - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_embeddings, - pooler_output=image_features, - ) - - return image_features + return vqmodel_outputs @torch.no_grad() def decode_image_tokens(self, image_tokens: torch.LongTensor, height: int, width: int): @@ -1027,12 +1041,12 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_sizes) - image_embeds = torch.cat(image_embeds, dim=0) + image_features = self.get_image_features(pixel_values, image_sizes).pooler_output + image_features = torch.cat(image_features, dim=0) special_image_mask = self.get_placeholder_mask( - input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds + input_ids, inputs_embeds=inputs_embeds, image_features=image_features ) - inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_embeds) + inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features) # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) outputs = self.text_model( From 37a53c38a03e61528581cf9b871b5089c4ca0d6f Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 18 Dec 2025 16:11:10 +0100 Subject: [PATCH 029/131] Update the blip architectures with a naive fix A better solution might be to remove the qformer etc. calls from the get_image/video_features and run those separately in the forward methods. --- .../models/blip_2/modeling_blip_2.py | 43 +++++++---- .../instructblip/modeling_instructblip.py | 67 +++++++---------- .../modeling_instructblipvideo.py | 72 +++++++++---------- .../modular_instructblipvideo.py | 57 ++++++++------- tests/models/blip/test_modeling_blip.py | 4 +- tests/models/blip_2/test_modeling_blip_2.py | 4 +- 6 files changed, 124 insertions(+), 123 deletions(-) diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index cdc19a1e2cf0..b345156c5605 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -55,6 +55,17 @@ logger = logging.get_logger(__name__) +@dataclass +@auto_docstring +class BaseModelOutputWithQformerOutputs(BaseModelOutputWithPooling): + """ + qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): + Outputs of the Q-Former (Querying Transformer). + """ + + qformer_outputs: Optional[tuple[torch.FloatTensor]] = None + + @dataclass @auto_docstring( custom_intro=""" @@ -492,7 +503,7 @@ def forward( pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[tuple, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithQformerOutputs]: if pixel_values is None: raise ValueError("You have to specify pixel_values") @@ -509,7 +520,7 @@ def forward( pooled_output = last_hidden_state[:, 0, :] pooled_output = self.post_layernorm(pooled_output) - return BaseModelOutputWithPooling( + return BaseModelOutputWithQformerOutputs( last_hidden_state=last_hidden_state, pooler_output=pooled_output, ) @@ -1607,11 +1618,12 @@ def _preprocess_accelerate(self): if hasattr(self.language_model, "_hf_hook"): self.language_model._hf_hook.io_same_device = True # For `generate` compatibility + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: Optional[bool] = False, - return_dict: Optional[bool] = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1622,10 +1634,11 @@ def get_image_features( """ # step 1: forward the images through the vision encoder, # to get image embeddings of shape (batch_size, seq_len, hidden_size) - vision_outputs = self.vision_model( + vision_outputs: BaseModelOutputWithQformerOutputs = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, + **kwargs, ) image_embeds = vision_outputs[0] @@ -1633,23 +1646,24 @@ def get_image_features( image_attention_mask = torch.ones(image_embeds.size()[:-1], dtype=torch.long, device=image_embeds.device) query_tokens = self.query_tokens.expand(image_embeds.shape[0], -1, -1) - query_outputs = self.qformer( + qformer_outputs = self.qformer( query_embeds=query_tokens, encoder_hidden_states=image_embeds, encoder_attention_mask=image_attention_mask, return_dict=True, ) - query_output = query_outputs[0] + vision_outputs.qformer_outputs = qformer_outputs + query_output = qformer_outputs[0] # Qformer is kept in fp32, we downcast the output back if needed if query_output.dtype != image_embeds.dtype: query_output = query_output.to(image_embeds.dtype) # step 3: use the language model, conditioned on the query outputs and the prompt - language_model_inputs = self.language_projection(query_output) - if return_dict: - return language_model_inputs, vision_outputs, query_outputs - return language_model_inputs + image_features = self.language_projection(query_output) + vision_outputs.pooler_output = image_features + + return vision_outputs def get_placeholder_mask(self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor): """ @@ -1754,9 +1768,14 @@ def forward( two ```""" - language_model_inputs, vision_outputs, query_outputs = self.get_image_features( + image_features: BaseModelOutputWithQformerOutputs = self.get_image_features( pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True ) + language_model_inputs = image_features.pooler_output + qformer_outputs = image_features.qformer_outputs + vision_outputs = image_features + vision_outputs.qformer_outputs = None # to avoid redundancy in the output + if inputs_embeds is None: inputs_embeds = self.get_input_embeddings()(input_ids) @@ -1806,7 +1825,7 @@ def forward( loss=loss, logits=logits, vision_outputs=vision_outputs, - qformer_outputs=query_outputs, + qformer_outputs=qformer_outputs, language_model_outputs=outputs, ) diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index 51d04e9189cb..98d8643ab58c 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -46,27 +46,13 @@ @dataclass -class BaseModelOutputWithVisionQformerOutputs(ModelOutput): +@auto_docstring +class BaseModelOutputWithQformerOutputs(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - vision_outputs (`BaseModelOutputWithPooling`): - Outputs of the vision encoder. - qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): - Outputs of the Q-Former (Querying Transformer). + qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): + Outputs of the Q-Former (Querying Transformer). """ - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None - vision_outputs: Optional[torch.FloatTensor] = None qformer_outputs: Optional[tuple[torch.FloatTensor]] = None @@ -423,7 +409,7 @@ def forward( pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[tuple, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithQformerOutputs]: if pixel_values is None: raise ValueError("You have to specify pixel_values") @@ -440,7 +426,7 @@ def forward( pooled_output = last_hidden_state[:, 0, :] pooled_output = self.post_layernorm(pooled_output) - return BaseModelOutputWithPooling( + return BaseModelOutputWithQformerOutputs( last_hidden_state=last_hidden_state, pooler_output=pooled_output, ) @@ -1197,13 +1183,14 @@ def _preprocess_accelerate(self): if hasattr(self.language_model, "_hf_hook"): self.language_model._hf_hook.io_same_device = True # For `generate` compatibility + @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, qformer_input_ids: torch.LongTensor, qformer_attention_mask: Optional[torch.LongTensor] = None, interpolate_pos_encoding: Optional[bool] = False, - return_dict: Optional[bool] = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1214,10 +1201,11 @@ def get_image_features( """ # step 1: forward the images through the vision encoder, # to get image embeddings of shape (batch_size, seq_len, hidden_size) - vision_outputs = self.vision_model( + vision_outputs: BaseModelOutputWithQformerOutputs = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, + **kwargs, ) image_embeds = vision_outputs[0] @@ -1230,31 +1218,23 @@ def get_image_features( if qformer_attention_mask is None: qformer_attention_mask = torch.ones_like(qformer_input_ids) qformer_attention_mask = torch.cat([query_attention_mask, qformer_attention_mask], dim=1) - query_outputs = self.qformer( + qformer_outputs = self.qformer( input_ids=qformer_input_ids, attention_mask=qformer_attention_mask, query_embeds=query_tokens, encoder_hidden_states=image_embeds, encoder_attention_mask=image_attention_mask, return_dict=True, + **kwargs, ) - query_output = query_outputs[0][:, : query_tokens.size(1), :] + vision_outputs.qformer_outputs = qformer_outputs + query_output = qformer_outputs[0][:, : query_tokens.size(1), :] # step 3: use the language model, conditioned on the query outputs and the prompt image_features = self.language_projection(query_output) + vision_outputs.pooler_output = image_features - # NOTE: @Tom the easy solution with the new format would be to grab vision_outputs, - # override the pooler_output with image_features and return that, but we'd lose the - # query_outputs - if return_dict: - return BaseModelOutputWithVisionQformerOutputs( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=image_features, - vision_outputs=vision_outputs, - qformer_outputs=query_outputs, - ) - - return image_features + return vision_outputs def get_placeholder_mask(self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor): """ @@ -1348,7 +1328,8 @@ def forward( The unusual aspect of this image is that a man is ironing clothes on the back of a yellow SUV, which is parked in the middle of a busy city street. This is an unconventional approach to ironing clothes, as it requires the man to balance himself and his ironing equipment on top of the vehicle while navigating through traffic. Additionally, the presence of taxis and other vehicles in the scene further emphasizes the unusual nature of this situation. ```""" - image_features = self.get_image_features( + # NOTE: @Tom this is not particularly clean, e.g. having to manually remove qformer outputs from vision outputs + image_features: BaseModelOutputWithQformerOutputs = self.get_image_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, @@ -1356,8 +1337,9 @@ def forward( return_dict=True, ) language_model_inputs = image_features.pooler_output - vision_outputs = image_features.vision_outputs - query_outputs = image_features.qformer_outputs + qformer_outputs = image_features.qformer_outputs + vision_outputs = image_features + vision_outputs.qformer_outputs = None # to avoid redundancy in the output if inputs_embeds is None: inputs_embeds = self.get_input_embeddings()(input_ids) @@ -1399,7 +1381,7 @@ def forward( loss=loss, logits=logits, vision_outputs=vision_outputs, - qformer_outputs=query_outputs, + qformer_outputs=qformer_outputs, language_model_outputs=outputs, ) @@ -1442,13 +1424,14 @@ def generate( self._preprocess_accelerate() batch_size = pixel_values.shape[0] - language_model_inputs = self.get_image_features( + image_features: BaseModelOutputWithQformerOutputs = self.get_image_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, interpolate_pos_encoding=interpolate_pos_encoding, - return_dict=False, + return_dict=True, ) + language_model_inputs = image_features.pooler_output if inputs_embeds is None: if input_ids is None: diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 806ea7127946..917e65e99646 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -160,6 +160,17 @@ def _init_weights(self, module): init.zeros_(module.query_tokens) +@dataclass +@auto_docstring +class BaseModelOutputWithQformerOutputs(BaseModelOutputWithPooling): + """ + qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): + Outputs of the Q-Former (Querying Transformer). + """ + + qformer_outputs: Optional[tuple[torch.FloatTensor]] = None + + # Adapted from transformers.models.siglip.modeling_siglip.eager_attention_forward -> InstructBlipVideo doesn't cast attn weights to fp32 def eager_attention_forward( module: nn.Module, @@ -365,7 +376,7 @@ def forward( pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[tuple, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithQformerOutputs]: if pixel_values is None: raise ValueError("You have to specify pixel_values") @@ -382,7 +393,7 @@ def forward( pooled_output = last_hidden_state[:, 0, :] pooled_output = self.post_layernorm(pooled_output) - return BaseModelOutputWithPooling( + return BaseModelOutputWithQformerOutputs( last_hidden_state=last_hidden_state, pooler_output=pooled_output, ) @@ -1201,23 +1212,6 @@ def _preprocess_accelerate(self): if hasattr(self.language_model, "_hf_hook"): self.language_model._hf_hook.io_same_device = True # For `generate` compatibility - def get_image_features( - self, - pixel_values: torch.FloatTensor, - qformer_input_ids: torch.LongTensor, - qformer_attention_mask: Optional[torch.LongTensor] = None, - interpolate_pos_encoding: Optional[bool] = False, - ): - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - """ - pass - # TODO: @Tom Let's fully remove if not supported - def get_placeholder_mask(self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor): """ Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`. @@ -1318,15 +1312,19 @@ def forward( ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict - language_model_inputs, vision_outputs, query_outputs = self.get_video_features( + # NOTE: @Tom this is not particularly clean, e.g. having to manually remove qformer outputs from vision outputs + video_features: BaseModelOutputWithQformerOutputs = self.get_video_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, + **kwargs, ) - vision_outputs = vision_outputs.to_tuple() if not return_dict else vision_outputs - query_outputs = query_outputs.to_tuple() if not return_dict else query_outputs + language_model_inputs = video_features.pooler_output + qformer_outputs = video_features.qformer_outputs + vision_outputs = video_features + vision_outputs.qformer_outputs = None # to avoid redundancy in the output if inputs_embeds is None: inputs_embeds = self.get_input_embeddings()(input_ids) @@ -1375,7 +1373,7 @@ def forward( loss=loss, logits=logits, vision_outputs=vision_outputs, - qformer_outputs=query_outputs, + qformer_outputs=qformer_outputs, language_model_outputs=outputs, ) @@ -1418,13 +1416,14 @@ def generate( self._preprocess_accelerate() batch_size = pixel_values.shape[0] - language_model_inputs, vision_outputs, query_outputs = self.get_video_features( + video_features: BaseModelOutputWithQformerOutputs = self.get_video_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, ) + language_model_inputs = video_features.pooler_output if inputs_embeds is None: if input_ids is None: @@ -1449,13 +1448,14 @@ def generate( return outputs + @can_return_tuple def get_video_features( self, pixel_values: torch.FloatTensor, qformer_input_ids: torch.LongTensor, qformer_attention_mask: Optional[torch.LongTensor] = None, interpolate_pos_encoding: Optional[bool] = False, - return_dict: Optional[bool] = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1469,10 +1469,11 @@ def get_video_features( batch_size, frames, channel, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channel, height, width) - vision_outputs = self.vision_model( + vision_outputs: BaseModelOutputWithQformerOutputs = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, + **kwargs, ) image_embeds = vision_outputs[0] @@ -1489,27 +1490,26 @@ def get_video_features( qformer_input_ids = qformer_input_ids.repeat_interleave(frames, dim=0) qformer_attention_mask = qformer_attention_mask.repeat_interleave(frames, dim=0) qformer_attention_mask = torch.cat([query_attention_mask, qformer_attention_mask], dim=1) - query_outputs = self.qformer( + qformer_outputs = self.qformer( input_ids=qformer_input_ids, attention_mask=qformer_attention_mask, query_embeds=query_tokens, encoder_hidden_states=image_embeds, encoder_attention_mask=image_attention_mask, return_dict=True, + **kwargs, ) - query_output = query_outputs[0][:, : query_tokens.size(1), :] + vision_outputs.qformer_outputs = qformer_outputs + query_output = qformer_outputs[0][:, : query_tokens.size(1), :] # step 3: use the language model, conditioned on the query outputs and the prompt - language_model_inputs = self.language_projection(query_output) + video_features = self.language_projection(query_output) # unbatch inputs back, each video-frame gets `num_query_tokens` seq length - language_model_inputs = language_model_inputs.reshape(batch_size, self.config.num_query_tokens * frames, -1) - # NOTE: @Tom the easy solution with the new format would be to grab vision_outputs, - # override the pooler_output with language_model_inputs and return that, but we'd lose the - # query_outputs - if return_dict: - return language_model_inputs, vision_outputs, query_outputs - return language_model_inputs + video_features = video_features.reshape(batch_size, self.config.num_query_tokens * frames, -1) + vision_outputs.pooler_output = video_features + + return vision_outputs __all__ = [ diff --git a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py index 55d734fabb34..511d380113f9 100644 --- a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py @@ -22,6 +22,7 @@ InstructBlipVisionConfig, ) from transformers.models.instructblip.modeling_instructblip import ( + BaseModelOutputWithQformerOutputs, InstructBlipForConditionalGeneration, InstructBlipForConditionalGenerationModelOutput, InstructBlipModel, @@ -35,7 +36,7 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES from ...processing_utils import Unpack -from ...utils import logging +from ...utils import can_return_tuple, logging from ..auto import CONFIG_MAPPING, AutoConfig @@ -285,13 +286,14 @@ def forward( class InstructBlipVideoForConditionalGeneration(InstructBlipForConditionalGeneration): + @can_return_tuple def get_video_features( self, pixel_values: torch.FloatTensor, qformer_input_ids: torch.LongTensor, qformer_attention_mask: Optional[torch.LongTensor] = None, interpolate_pos_encoding: Optional[bool] = False, - return_dict: Optional[bool] = False, + **kwargs: Unpack[TransformersKwargs], ): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -305,10 +307,11 @@ def get_video_features( batch_size, frames, channel, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channel, height, width) - vision_outputs = self.vision_model( + vision_outputs: BaseModelOutputWithQformerOutputs = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, + **kwargs, ) image_embeds = vision_outputs[0] @@ -325,38 +328,29 @@ def get_video_features( qformer_input_ids = qformer_input_ids.repeat_interleave(frames, dim=0) qformer_attention_mask = qformer_attention_mask.repeat_interleave(frames, dim=0) qformer_attention_mask = torch.cat([query_attention_mask, qformer_attention_mask], dim=1) - query_outputs = self.qformer( + qformer_outputs = self.qformer( input_ids=qformer_input_ids, attention_mask=qformer_attention_mask, query_embeds=query_tokens, encoder_hidden_states=image_embeds, encoder_attention_mask=image_attention_mask, return_dict=True, + **kwargs, ) - query_output = query_outputs[0][:, : query_tokens.size(1), :] + vision_outputs.qformer_outputs = qformer_outputs + query_output = qformer_outputs[0][:, : query_tokens.size(1), :] # step 3: use the language model, conditioned on the query outputs and the prompt - language_model_inputs = self.language_projection(query_output) + video_features = self.language_projection(query_output) # unbatch inputs back, each video-frame gets `num_query_tokens` seq length - language_model_inputs = language_model_inputs.reshape(batch_size, self.config.num_query_tokens * frames, -1) - # NOTE: @Tom the easy solution with the new format would be to grab vision_outputs, - # override the pooler_output with language_model_inputs and return that, but we'd lose the - # query_outputs - if return_dict: - return language_model_inputs, vision_outputs, query_outputs - return language_model_inputs - - # Model supports only videos - def get_image_features( - self, - pixel_values: torch.FloatTensor, - qformer_input_ids: torch.LongTensor, - qformer_attention_mask: Optional[torch.LongTensor] = None, - interpolate_pos_encoding: Optional[bool] = False, - ): - pass - # TODO: @Tom Let's fully remove if not supported + video_features = video_features.reshape(batch_size, self.config.num_query_tokens * frames, -1) + vision_outputs.pooler_output = video_features + + return vision_outputs + + def get_image_features(**super_kwargs): + raise AttributeError("No need to inherit as this architecture only supports videos.") def get_placeholder_mask(self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor): """ @@ -456,15 +450,19 @@ def forward( ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict - language_model_inputs, vision_outputs, query_outputs = self.get_video_features( + # NOTE: @Tom this is not particularly clean, e.g. having to manually remove qformer outputs from vision outputs + video_features: BaseModelOutputWithQformerOutputs = self.get_video_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, + **kwargs, ) - vision_outputs = vision_outputs.to_tuple() if not return_dict else vision_outputs - query_outputs = query_outputs.to_tuple() if not return_dict else query_outputs + language_model_inputs = video_features.pooler_output + qformer_outputs = video_features.qformer_outputs + vision_outputs = video_features + vision_outputs.qformer_outputs = None # to avoid redundancy in the output if inputs_embeds is None: inputs_embeds = self.get_input_embeddings()(input_ids) @@ -513,7 +511,7 @@ def forward( loss=loss, logits=logits, vision_outputs=vision_outputs, - qformer_outputs=query_outputs, + qformer_outputs=qformer_outputs, language_model_outputs=outputs, ) @@ -556,13 +554,14 @@ def generate( self._preprocess_accelerate() batch_size = pixel_values.shape[0] - language_model_inputs, vision_outputs, query_outputs = self.get_video_features( + video_features: BaseModelOutputWithQformerOutputs = self.get_video_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, ) + language_model_inputs = video_features.pooler_output if inputs_embeds is None: if input_ids is None: diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 5a81b45f71a7..fdc66dc8321e 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -483,7 +483,7 @@ def test_get_image_features(self): model.eval() image_features = model.get_image_features(**inputs_dict) self.assertEqual( - image_features.shape, + image_features.pooler_output.shape, ( self.model_tester.batch_size, model.projection_dim, @@ -502,7 +502,7 @@ def test_get_text_features(self): model.eval() text_features = model.get_text_features(**inputs_dict) self.assertEqual( - text_features.shape, + text_features.logits.shape, ( self.model_tester.batch_size, model.projection_dim, diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index 2448d3221cf7..55deab49b469 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -951,7 +951,7 @@ def test_get_text_features(self): model = Blip2Model(config).to(torch_device) model.eval() text_features = model.get_text_features(**inputs_dict) - self.assertEqual(text_features[0].shape, (10, config.text_config.vocab_size)) + self.assertEqual(text_features.logits[0].shape, (10, config.text_config.vocab_size)) def test_get_image_features(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -965,7 +965,7 @@ def test_get_image_features(self): model.eval() image_features = model.get_image_features(**inputs_dict) self.assertEqual( - image_features[0].shape, + image_features.pooler_output[0].shape, (config.vision_config.hidden_size,), ) From 440914b6b0f83217547b3fbbcc9716f078912cdc Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 18 Dec 2025 16:21:47 +0100 Subject: [PATCH 030/131] Convert remaining modulars (emu3, janus), patch emu3 --- src/transformers/models/emu3/modeling_emu3.py | 65 ++++++++++++------- src/transformers/models/emu3/modular_emu3.py | 10 ++- .../models/janus/modeling_janus.py | 19 ++++-- 3 files changed, 63 insertions(+), 31 deletions(-) diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index dae92ab1ffc5..ef40790762eb 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -22,6 +22,7 @@ import math from collections.abc import Callable +from dataclasses import dataclass from functools import cached_property from typing import Optional, Union @@ -45,6 +46,17 @@ from .configuration_emu3 import Emu3Config, Emu3TextConfig, Emu3VQVAEConfig +@dataclass +@auto_docstring +class Emu3VQVAEModelOutput(BaseModelOutputWithPooling): + r""" + image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):` + Indices of the image tokens predicted by the VQ-VAE model. + """ + + image_tokens: Optional[torch.FloatTensor] = None + + def rotate_half(x): """Rotates half the hidden dims of the input.""" x1 = x[..., : x.shape[-1] // 2] @@ -985,7 +997,10 @@ def __init__(self, config: Emu3VQVAEConfig): self.post_init() - def encode(self, pixel_values: torch.Tensor, image_sizes: torch.Tensor): + @check_model_inputs(tie_last_hidden_states=False) + def encode( + self, pixel_values: torch.Tensor, image_sizes: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> Emu3VQVAEModelOutput: is_image = pixel_values.ndim == 4 if is_image: temporal = self.config.temporal_downsample_factor @@ -997,12 +1012,11 @@ def encode(self, pixel_values: torch.Tensor, image_sizes: torch.Tensor): hidden_states = self.encoder(pixel_values) # b t c h w -> b c t h w - hidden_states = hidden_states.permute(0, 2, 1, 3, 4) - hidden_states = self.quant_conv(hidden_states) + conv_hidden_states = self.quant_conv(hidden_states.permute(0, 2, 1, 3, 4)) # b c t h w -> b t c h w - hidden_states = hidden_states.permute(0, 2, 1, 3, 4) - codes = self.quantize(hidden_states) + conv_hidden_states = conv_hidden_states.permute(0, 2, 1, 3, 4) + codes = self.quantize(conv_hidden_states) image_tokens = codes.squeeze(1) if is_image else codes @@ -1011,7 +1025,10 @@ def encode(self, pixel_values: torch.Tensor, image_sizes: torch.Tensor): for single_image, size in zip(image_tokens, image_sizes) ] - return image_tokens + return Emu3VQVAEModelOutput( + last_hidden_state=hidden_states, + image_tokens=image_tokens, + ) def decode(self, hidden_states: torch.Tensor): is_image = hidden_states.ndim == 3 @@ -1368,13 +1385,16 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.L image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`): The sizes of the images in the batch, being (height, width) for each image. """ - image_tokens_list = self.vqmodel.encode(pixel_values, image_sizes) - bpe_tokens_list = [self.vocabulary_mapping.convert_img2bpe(tokens).flatten() for tokens in image_tokens_list] + vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, return_dict=True) + bpe_tokens_list = [ + self.vocabulary_mapping.convert_img2bpe(tokens).flatten() for tokens in vqmodel_outputs.image_tokens + ] bpe_tokens = torch.cat(bpe_tokens_list) return bpe_tokens + @can_return_tuple def get_image_features( - self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, return_dict: bool = False + self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs] ): """ Tokenizes images into discrete tokens with VQGAN module and embeds @@ -1384,23 +1404,20 @@ def get_image_features( pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): The tensors corresponding to the input images. """ - image_tokens = self.get_image_tokens(pixel_values, image_sizes) + vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, return_dict=True) split_sizes = [ (height // self.vqmodel.vision_spatial_factor) * (width // self.vqmodel.vision_spatial_factor + 1) for height, width in image_sizes ] - image_embeddings = self.get_input_embeddings()(image_tokens) + bpe_tokens_list = [ + self.vocabulary_mapping.convert_img2bpe(tokens).flatten() for tokens in vqmodel_outputs.image_tokens + ] + bpe_tokens = torch.cat(bpe_tokens_list) + image_embeddings = self.get_input_embeddings()(bpe_tokens) image_features = torch.split(image_embeddings, split_sizes) + vqmodel_outputs.pooler_output = image_features - # TODO: @Tom to revisit with the new can_return_tuple approach. - # Presumably requires updating the `self.vqmodel.encode` method to return a BaseModelOutput - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=image_embeddings, - pooler_output=image_features, - ) - - return image_features + return vqmodel_outputs @torch.no_grad() def decode_image_tokens(self, image_tokens: torch.LongTensor, height: int, width: int): @@ -1475,12 +1492,12 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_sizes) - image_embeds = torch.cat(image_embeds, dim=0) + image_features = self.get_image_features(pixel_values, image_sizes).pooler_output + image_features = torch.cat(image_features, dim=0) special_image_mask = self.get_placeholder_mask( - input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds + input_ids, inputs_embeds=inputs_embeds, image_features=image_features ) - inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_embeds) + inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features) # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) outputs = self.text_model( diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index 844ae1f05c3f..c1b97da7a9ac 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -938,9 +938,9 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.L image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`): The sizes of the images in the batch, being (height, width) for each image. """ - image_tokens_list: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, return_dict=True) + vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, return_dict=True) bpe_tokens_list = [ - self.vocabulary_mapping.convert_img2bpe(tokens).flatten() for tokens in image_tokens_list.image_tokens + self.vocabulary_mapping.convert_img2bpe(tokens).flatten() for tokens in vqmodel_outputs.image_tokens ] bpe_tokens = torch.cat(bpe_tokens_list) return bpe_tokens @@ -962,7 +962,11 @@ def get_image_features( (height // self.vqmodel.vision_spatial_factor) * (width // self.vqmodel.vision_spatial_factor + 1) for height, width in image_sizes ] - image_embeddings = self.get_input_embeddings()(vqmodel_outputs.image_tokens) + bpe_tokens_list = [ + self.vocabulary_mapping.convert_img2bpe(tokens).flatten() for tokens in vqmodel_outputs.image_tokens + ] + bpe_tokens = torch.cat(bpe_tokens_list) + image_embeddings = self.get_input_embeddings()(bpe_tokens) image_features = torch.split(image_embeddings, split_sizes) vqmodel_outputs.pooler_output = image_features diff --git a/src/transformers/models/janus/modeling_janus.py b/src/transformers/models/janus/modeling_janus.py index eb932e2047f9..442de4bd63a8 100644 --- a/src/transformers/models/janus/modeling_janus.py +++ b/src/transformers/models/janus/modeling_janus.py @@ -420,6 +420,17 @@ def forward( return BaseModelOutput(last_hidden_state=hidden_states) +@dataclass +@auto_docstring +class BaseModelOutputWithQformerOutputs(BaseModelOutputWithPooling): + """ + qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): + Outputs of the Q-Former (Querying Transformer). + """ + + qformer_outputs: Optional[tuple[torch.FloatTensor]] = None + + class JanusAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper""" @@ -570,7 +581,7 @@ def forward( pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[tuple, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithQformerOutputs]: if pixel_values is None: raise ValueError("You have to specify pixel_values") @@ -587,7 +598,7 @@ def forward( pooled_output = last_hidden_state[:, 0, :] pooled_output = self.post_layernorm(pooled_output) - return BaseModelOutputWithPooling( + return BaseModelOutputWithQformerOutputs( last_hidden_state=last_hidden_state, pooler_output=pooled_output, ) @@ -950,10 +961,10 @@ def forward(self, hidden_state: torch.FloatTensor) -> torch.FloatTensor: @auto_docstring class JanusVQVAEModelOutput(BaseModelOutputWithPooling): r""" - image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):` - Indices of the image tokens predicted by the VQ-VAE model. quantized_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): Quantized last hidden state from the VQ-VAE model. + image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):` + Indices of the image tokens predicted by the VQ-VAE model. embedding_loss (`torch.FloatTensor`): The embedding loss computed during quantization. """ From 48353a5421f0b69cf8f6289c2aafad160c214f79 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 18 Dec 2025 16:43:38 +0100 Subject: [PATCH 031/131] Patch blip test --- tests/models/blip/test_modeling_blip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index fdc66dc8321e..3d91cef4f593 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -502,7 +502,7 @@ def test_get_text_features(self): model.eval() text_features = model.get_text_features(**inputs_dict) self.assertEqual( - text_features.logits.shape, + text_features.pooler_output.shape, ( self.model_tester.batch_size, model.projection_dim, From 531321c8ded3419a05fde83885e82a450a71efd2 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 18 Dec 2025 17:25:54 +0100 Subject: [PATCH 032/131] Update deepseek_vl using a new BaseModelOutputWithHighResVisionEncodings --- .../modeling_deepseek_vl_hybrid.py | 76 ++++++++++++------ .../modular_deepseek_vl_hybrid.py | 77 +++++++++++++------ 2 files changed, 104 insertions(+), 49 deletions(-) diff --git a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py index 0ba9a83ef3dc..6a47cca44af9 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py @@ -35,6 +35,30 @@ from .configuration_deepseek_vl_hybrid import DeepseekVLHybridConfig +@dataclass +@auto_docstring +class BaseModelOutputWithHighResVisionEncodings(BaseModelOutputWithPooling): + r""" + high_res_vision_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the high resolution vision model. + high_res_vision_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the high resolution vision model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the high resolution vision model at the output of each layer plus the optional initial embedding outputs. + high_res_vision_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)` from the high resolution vision model. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + high_res_vision_last_hidden_state: Optional[torch.FloatTensor] = None + high_res_vision_hidden_states: Optional[tuple[torch.FloatTensor]] = None + high_res_vision_attentions: Optional[tuple[torch.FloatTensor]] = None + + @dataclass @auto_docstring( custom_intro=""" @@ -270,23 +294,25 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple - @auto_docstring def get_image_features( - self, pixel_values: torch.FloatTensor, high_res_pixel_values: torch.FloatTensor, return_dict: bool = False + self, + pixel_values: torch.FloatTensor, + high_res_pixel_values: torch.FloatTensor, + **kwargs: Unpack[TransformersKwargs], ): - vision_encodings = self.get_low_res_image_features(pixel_values) - high_res_vision_encodings = self.get_high_res_image_features(high_res_pixel_values) - image_features = self.aligner(vision_encodings, high_res_vision_encodings) - - # TODO: @Tom to revisit with the new can_return_tuple approach - # Presumably something like a fresh ModelOutput subclass that holds both - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_encodings, - pooler_output=image_features, - ) - - return image_features + low_res_outputs = self.get_low_res_image_features(pixel_values, **kwargs) + high_res_outputs = self.get_high_res_image_features(high_res_pixel_values, **kwargs) + image_features = self.aligner(low_res_outputs.last_hidden_state, high_res_outputs.last_hidden_state) + + return BaseModelOutputWithHighResVisionEncodings( + last_hidden_state=low_res_outputs.last_hidden_state, + pooler_output=image_features, + hidden_states=low_res_outputs.hidden_states, + attentions=low_res_outputs.attentions, + high_res_vision_last_hidden_state=high_res_outputs.last_hidden_state, + high_res_vision_hidden_states=high_res_outputs.hidden_states, + high_res_vision_attentions=high_res_outputs.attentions, + ) def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -349,7 +375,7 @@ def forward( image_attention_mask = input_ids == self.config.image_token_id image_attention_mask = image_attention_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) - image_embeds = self.get_image_features(pixel_values, high_res_pixel_values) + image_embeds = self.get_image_features(pixel_values, high_res_pixel_values, return_dict=True).pooler_output image_features = image_embeds.reshape(-1, inputs_embeds.shape[-1]) image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) inputs_embeds = inputs_embeds.masked_scatter(image_attention_mask, image_features) @@ -373,21 +399,20 @@ def forward( image_hidden_states=image_embeds if pixel_values is not None else None, ) - def get_low_res_image_features(self, pixel_values): - output = self.vision_model(pixel_values) - output = output[0] - return output + def get_low_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + return self.vision_model(pixel_values, return_dict=True, **kwargs) - def get_high_res_image_features(self, pixel_values): - output = self.high_res_vision_model( + def get_high_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + high_res_outputs = self.high_res_vision_model( pixel_values=pixel_values, output_hidden_states=True, return_dict=True, + **kwargs, ) - last_hidden_state = output.last_hidden_state + last_hidden_state = high_res_outputs.last_hidden_state last_hidden_state = self.high_res_vision_proj(last_hidden_state) - hidden_states = output.hidden_states + hidden_states = high_res_outputs.hidden_states global_hidden_state = hidden_states[self.global_attn_index + 1] # +1 for embedding layer global_hidden_state = self.high_res_vision_neck(global_hidden_state) global_hidden_state = self.high_res_vision_proj(global_hidden_state) @@ -397,8 +422,9 @@ def get_high_res_image_features(self, pixel_values): # batch_size, hidden_size, height, width -> batch_size, seq_len, hidden_size output = output.permute(0, 2, 3, 1) output = output.reshape(output.shape[0], -1, output.shape[-1]) + high_res_outputs.last_hidden_state = output - return output + return high_res_outputs class DeepseekVLHybridForConditionalGeneration(DeepseekVLHybridPreTrainedModel, GenerationMixin): diff --git a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py index 5c7bb223d045..6f1604596ac5 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from dataclasses import dataclass from typing import Optional, Union import torch @@ -145,6 +146,30 @@ def __init__( ) +@dataclass +@auto_docstring +class BaseModelOutputWithHighResVisionEncodings(BaseModelOutputWithPooling): + r""" + high_res_vision_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the high resolution vision model. + high_res_vision_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the high resolution vision model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the high resolution vision model at the output of each layer plus the optional initial embedding outputs. + high_res_vision_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)` from the high resolution vision model. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + high_res_vision_last_hidden_state: Optional[torch.FloatTensor] = None + high_res_vision_hidden_states: Optional[tuple[torch.FloatTensor]] = None + high_res_vision_attentions: Optional[tuple[torch.FloatTensor]] = None + + class DeepseekVLHybridBaseModelOutputWithPast(IdeficsBaseModelOutputWithPast): pass @@ -250,21 +275,20 @@ def __init__(self, config): super().__init__(config) - def get_low_res_image_features(self, pixel_values): - output = self.vision_model(pixel_values) - output = output[0] - return output + def get_low_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + return self.vision_model(pixel_values, return_dict=True, **kwargs) - def get_high_res_image_features(self, pixel_values): - output = self.high_res_vision_model( + def get_high_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + high_res_outputs = self.high_res_vision_model( pixel_values=pixel_values, output_hidden_states=True, return_dict=True, + **kwargs, ) - last_hidden_state = output.last_hidden_state + last_hidden_state = high_res_outputs.last_hidden_state last_hidden_state = self.high_res_vision_proj(last_hidden_state) - hidden_states = output.hidden_states + hidden_states = high_res_outputs.hidden_states global_hidden_state = hidden_states[self.global_attn_index + 1] # +1 for embedding layer global_hidden_state = self.high_res_vision_neck(global_hidden_state) global_hidden_state = self.high_res_vision_proj(global_hidden_state) @@ -274,25 +298,30 @@ def get_high_res_image_features(self, pixel_values): # batch_size, hidden_size, height, width -> batch_size, seq_len, hidden_size output = output.permute(0, 2, 3, 1) output = output.reshape(output.shape[0], -1, output.shape[-1]) + high_res_outputs.last_hidden_state = output - return output + return high_res_outputs + @can_return_tuple def get_image_features( - self, pixel_values: torch.FloatTensor, high_res_pixel_values: torch.FloatTensor, return_dict: bool = False + self, + pixel_values: torch.FloatTensor, + high_res_pixel_values: torch.FloatTensor, + **kwargs: Unpack[TransformersKwargs], ): - vision_encodings = self.get_low_res_image_features(pixel_values) - high_res_vision_encodings = self.get_high_res_image_features(high_res_pixel_values) - image_features = self.aligner(vision_encodings, high_res_vision_encodings) - - # TODO: @Tom to revisit with the new can_return_tuple approach - # Presumably something like a fresh ModelOutput subclass that holds both - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=vision_encodings, - pooler_output=image_features, - ) - - return image_features + low_res_outputs = self.get_low_res_image_features(pixel_values, **kwargs) + high_res_outputs = self.get_high_res_image_features(high_res_pixel_values, **kwargs) + image_features = self.aligner(low_res_outputs.last_hidden_state, high_res_outputs.last_hidden_state) + + return BaseModelOutputWithHighResVisionEncodings( + last_hidden_state=low_res_outputs.last_hidden_state, + pooler_output=image_features, + hidden_states=low_res_outputs.hidden_states, + attentions=low_res_outputs.attentions, + high_res_vision_last_hidden_state=high_res_outputs.last_hidden_state, + high_res_vision_hidden_states=high_res_outputs.hidden_states, + high_res_vision_attentions=high_res_outputs.attentions, + ) @can_return_tuple @auto_docstring(custom_args=DEEPSEEK_VL_COMMON_CUSTOM_ARGS) @@ -331,7 +360,7 @@ def forward( image_attention_mask = input_ids == self.config.image_token_id image_attention_mask = image_attention_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) - image_embeds = self.get_image_features(pixel_values, high_res_pixel_values) + image_embeds = self.get_image_features(pixel_values, high_res_pixel_values, return_dict=True).pooler_output image_features = image_embeds.reshape(-1, inputs_embeds.shape[-1]) image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) inputs_embeds = inputs_embeds.masked_scatter(image_attention_mask, image_features) From 70577d2b1cdf506b29e525855f289c7bcb4403d8 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 18 Dec 2025 17:34:56 +0100 Subject: [PATCH 033/131] Remove 'copied' for blip_2, instructblip and kosmos2 as they required custom changes --- src/transformers/models/blip_2/modeling_blip_2.py | 1 - src/transformers/models/instructblip/modeling_instructblip.py | 1 - src/transformers/models/kosmos2/modeling_kosmos2.py | 1 - 3 files changed, 3 deletions(-) diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index d6cbb56e433c..5f0be497c258 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -477,7 +477,6 @@ def forward( @auto_docstring -# Copied from transformers.models.blip.modeling_blip.BlipVisionModel with Blip->Blip2, BLIP->BLIP_2 class Blip2VisionModel(Blip2PreTrainedModel): main_input_name = "pixel_values" input_modalities = ("image",) diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index 3fc5d23563d2..484964674023 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -383,7 +383,6 @@ def forward( return BaseModelOutput(last_hidden_state=hidden_states) -# Copied from transformers.models.blip.modeling_blip.BlipVisionModel with Blip->InstructBlip, BLIP->INSTRUCTBLIP class InstructBlipVisionModel(InstructBlipPreTrainedModel): main_input_name = "pixel_values" input_modalities = ("image",) diff --git a/src/transformers/models/kosmos2/modeling_kosmos2.py b/src/transformers/models/kosmos2/modeling_kosmos2.py index 193e04bfe654..ca35fb0e1c55 100644 --- a/src/transformers/models/kosmos2/modeling_kosmos2.py +++ b/src/transformers/models/kosmos2/modeling_kosmos2.py @@ -424,7 +424,6 @@ def forward( return outputs -# Copied from transformers.models.altclip.modeling_altclip.AltCLIPEncoder with AltCLIP->Kosmos2Vision class Kosmos2VisionEncoder(nn.Module): """ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a From f6f90d67da191495476dcdbbf8bead6512ae0b8b Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 18 Dec 2025 18:10:13 +0100 Subject: [PATCH 034/131] Patch qwen3_vl and qwen3_vl_moe, where I used last_hidden_state instead of pooler_output --- .../models/qwen3_vl/modeling_qwen3_vl.py | 31 ++++++------------- .../models/qwen3_vl/modular_qwen3_vl.py | 29 +++++------------ .../qwen3_vl_moe/modeling_qwen3_vl_moe.py | 31 ++++++------------- 3 files changed, 26 insertions(+), 65 deletions(-) diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index 4128598ed8df..6444dd6c87dd 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -35,36 +35,23 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...processing_utils import ProcessingKwargs, Unpack -from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ...utils.generic import check_model_inputs, maybe_autocast from .configuration_qwen3_vl import Qwen3VLConfig, Qwen3VLTextConfig, Qwen3VLVisionConfig @dataclass -class BaseModelOutputWithDeepstackFeatures( - ModelOutput -): # NOTE: @Tom Should we subclass from BaseModelOutputWithPooler instead? +@auto_docstring +class BaseModelOutputWithDeepstackFeatures(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - deepstack_features (`List[torch.FloatTensor]`, *optional*): - List of hidden-states (feature maps) from deepstack layers. + deepstack_features (`List[torch.FloatTensor]`, *optional*): + List of hidden-states (feature maps) from deepstack layers. """ - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None deepstack_features: Optional[list[torch.FloatTensor]] = None @@ -768,7 +755,7 @@ def fast_pos_embed_interpolate(self, grid_thw): @check_model_inputs(tie_last_hidden_states=False) def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[ProcessingKwargs] + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> BaseModelOutputWithDeepstackFeatures: """ Args: @@ -1139,7 +1126,7 @@ def get_image_features( vision_output: BaseModelOutputWithDeepstackFeatures = self.visual( pixel_values, grid_thw=image_grid_thw, **kwargs ) - image_embeds = vision_output.last_hidden_state + image_embeds = vision_output.pooler_output split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) vision_output.pooler_output = image_embeds diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index 4d4bc5e5b0c4..7b17ca23e000 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -31,12 +31,12 @@ from ...image_utils import ImageInput from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling from ...modeling_rope_utils import RopeParameters, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import ProcessingKwargs, Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput -from ...utils import ModelOutput, auto_docstring, can_return_tuple, logging +from ...utils import auto_docstring, can_return_tuple, logging from ...utils.generic import check_model_inputs, maybe_autocast from ...video_utils import VideoInput from ..llama.modeling_llama import LlamaRotaryEmbedding @@ -68,26 +68,13 @@ @dataclass -class BaseModelOutputWithDeepstackFeatures( - ModelOutput -): # NOTE: @Tom Should we subclass from BaseModelOutputWithPooler instead? +@auto_docstring +class BaseModelOutputWithDeepstackFeatures(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - deepstack_features (`List[torch.FloatTensor]`, *optional*): - List of hidden-states (feature maps) from deepstack layers. + deepstack_features (`List[torch.FloatTensor]`, *optional*): + List of hidden-states (feature maps) from deepstack layers. """ - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None deepstack_features: Optional[list[torch.FloatTensor]] = None @@ -663,7 +650,7 @@ def fast_pos_embed_interpolate(self, grid_thw): @check_model_inputs(tie_last_hidden_states=False) def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[ProcessingKwargs] + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> BaseModelOutputWithDeepstackFeatures: """ Args: @@ -991,7 +978,7 @@ def get_image_features( vision_output: BaseModelOutputWithDeepstackFeatures = self.visual( pixel_values, grid_thw=image_grid_thw, **kwargs ) - image_embeds = vision_output.last_hidden_state + image_embeds = vision_output.pooler_output split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) vision_output.pooler_output = image_embeds diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index 4b9b46fec8d5..25e7c5e2549b 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -35,11 +35,11 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...processing_utils import ProcessingKwargs, Unpack -from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ...utils.generic import OutputRecorder, check_model_inputs, maybe_autocast from .configuration_qwen3_vl_moe import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig, Qwen3VLMoeVisionConfig @@ -440,26 +440,13 @@ def forward(self, seqlen: int) -> torch.Tensor: @dataclass -class BaseModelOutputWithDeepstackFeatures( - ModelOutput -): # NOTE: @Tom Should we subclass from BaseModelOutputWithPooler instead? +@auto_docstring +class BaseModelOutputWithDeepstackFeatures(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - deepstack_features (`List[torch.FloatTensor]`, *optional*): - List of hidden-states (feature maps) from deepstack layers. + deepstack_features (`List[torch.FloatTensor]`, *optional*): + List of hidden-states (feature maps) from deepstack layers. """ - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None deepstack_features: Optional[list[torch.FloatTensor]] = None @@ -778,7 +765,7 @@ def fast_pos_embed_interpolate(self, grid_thw): @check_model_inputs(tie_last_hidden_states=False) def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[ProcessingKwargs] + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> BaseModelOutputWithDeepstackFeatures: """ Args: @@ -1292,7 +1279,7 @@ def get_image_features( vision_output: BaseModelOutputWithDeepstackFeatures = self.visual( pixel_values, grid_thw=image_grid_thw, **kwargs ) - image_embeds = vision_output.last_hidden_state + image_embeds = vision_output.pooler_output split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) vision_output.pooler_output = image_embeds From 7af0b665ca8800808f9ffbf2977afd1434b66dd2 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 18 Dec 2025 18:20:30 +0100 Subject: [PATCH 035/131] Run repo-consistency --- .../models/qwen3_omni_moe/modeling_qwen3_omni_moe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index bba4fd2d05e2..a3aaed696134 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -48,7 +48,7 @@ ) from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...processing_utils import ProcessingKwargs, Unpack +from ...processing_utils import Unpack from ...utils import ModelOutput, auto_docstring, can_return_tuple from ...utils.generic import OutputRecorder, TransformersKwargs, check_model_inputs, maybe_autocast from .configuration_qwen3_omni_moe import ( @@ -1217,7 +1217,7 @@ def fast_pos_embed_interpolate(self, grid_thw): @check_model_inputs(tie_last_hidden_states=False) def forward( - self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[ProcessingKwargs] + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> BaseModelOutputWithDeepstackFeatures: """ Args: From cbe007b64f3ba90ae8efc6b095ad13818b0995a7 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 31 Dec 2025 12:35:42 +0100 Subject: [PATCH 036/131] Use kwargs["output_hidden_states"] = True to hardcode output_hidden_states where needed --- src/transformers/models/aria/modeling_aria.py | 2 +- src/transformers/models/aria/modular_aria.py | 2 +- .../audioflamingo3/modeling_audioflamingo3.py | 2 +- .../audioflamingo3/modular_audioflamingo3.py | 2 +- .../models/aya_vision/modeling_aya_vision.py | 3 ++- .../models/aya_vision/modular_aya_vision.py | 3 ++- .../modeling_deepseek_vl_hybrid.py | 2 +- .../modular_deepseek_vl_hybrid.py | 2 +- src/transformers/models/dpt/modeling_dpt.py | 10 ++++----- .../modeling_ernie4_5_vl_moe.py | 21 +++++++++++++++---- .../models/glmasr/modeling_glmasr.py | 3 ++- .../models/llava/modeling_llava.py | 3 ++- .../models/llava_next/modeling_llava_next.py | 3 ++- .../modeling_llava_next_video.py | 6 ++++-- .../modular_llava_next_video.py | 6 ++++-- .../modeling_llava_onevision.py | 6 ++++-- .../modular_llava_onevision.py | 6 ++++-- .../models/mistral3/modeling_mistral3.py | 3 ++- .../models/mistral3/modular_mistral3.py | 3 ++- .../video_llava/modeling_video_llava.py | 3 ++- .../models/vipllava/modeling_vipllava.py | 3 ++- .../models/vipllava/modular_vipllava.py | 3 ++- 22 files changed, 64 insertions(+), 33 deletions(-) diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index ce5fa39f3de8..2646b38dbea4 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -937,10 +937,10 @@ def get_image_features( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) patch_attention_mask = self._create_patch_attention_mask(pixel_mask) + kwargs["output_hidden_states"] = True image_outputs = self.vision_tower( pixel_values, patch_attention_mask=patch_attention_mask, - output_hidden_states=True, **kwargs, ) image_attn_mask = None diff --git a/src/transformers/models/aria/modular_aria.py b/src/transformers/models/aria/modular_aria.py index 892b212fae67..cad8a0368541 100644 --- a/src/transformers/models/aria/modular_aria.py +++ b/src/transformers/models/aria/modular_aria.py @@ -1293,10 +1293,10 @@ def get_image_features( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) patch_attention_mask = self._create_patch_attention_mask(pixel_mask) + kwargs["output_hidden_states"] = True image_outputs = self.vision_tower( pixel_values, patch_attention_mask=patch_attention_mask, - output_hidden_states=True, **kwargs, ) image_attn_mask = None diff --git a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py index d48ff0aed491..4f18c849dc41 100644 --- a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py @@ -468,7 +468,7 @@ def get_audio_features( """ # Encode audio - audio_output = self.audio_tower(input_features, input_features_mask=input_features_mask) + audio_output = self.audio_tower(input_features, input_features_mask=input_features_mask, **kwargs) audio_embeds = self.multi_modal_projector(audio_output.last_hidden_state) # Mask according to avg pooling (which is after attention blocks) diff --git a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py index 2efe7ae0690a..81a545a4fbf4 100644 --- a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py @@ -165,7 +165,7 @@ def get_audio_features( """ # Encode audio - audio_output = self.audio_tower(input_features, input_features_mask=input_features_mask) + audio_output = self.audio_tower(input_features, input_features_mask=input_features_mask, **kwargs) audio_embeds = self.multi_modal_projector(audio_output.last_hidden_state) # Mask according to avg pooling (which is after attention blocks) diff --git a/src/transformers/models/aya_vision/modeling_aya_vision.py b/src/transformers/models/aya_vision/modeling_aya_vision.py index aaee53db976c..5b3a8ba45132 100644 --- a/src/transformers/models/aya_vision/modeling_aya_vision.py +++ b/src/transformers/models/aya_vision/modeling_aya_vision.py @@ -220,7 +220,8 @@ def get_image_features( kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + image_outputs = self.vision_tower(pixel_values, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/aya_vision/modular_aya_vision.py b/src/transformers/models/aya_vision/modular_aya_vision.py index 982346ae74ab..bd6d5736ecc9 100644 --- a/src/transformers/models/aya_vision/modular_aya_vision.py +++ b/src/transformers/models/aya_vision/modular_aya_vision.py @@ -145,7 +145,8 @@ def get_image_features( kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + image_outputs = self.vision_tower(pixel_values, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py index 1982649564d8..e0400551dbd4 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py @@ -403,9 +403,9 @@ def get_low_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: return self.vision_model(pixel_values, return_dict=True, **kwargs) def get_high_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + kwargs["output_hidden_states"] = True high_res_outputs = self.high_res_vision_model( pixel_values=pixel_values, - output_hidden_states=True, return_dict=True, **kwargs, ) diff --git a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py index 8b6821e676ca..80b385e1e693 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py @@ -279,9 +279,9 @@ def get_low_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: return self.vision_model(pixel_values, return_dict=True, **kwargs) def get_high_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + kwargs["output_hidden_states"] = True high_res_outputs = self.high_res_vision_model( pixel_values=pixel_values, - output_hidden_states=True, return_dict=True, **kwargs, ) diff --git a/src/transformers/models/dpt/modeling_dpt.py b/src/transformers/models/dpt/modeling_dpt.py index fb871c90d1e3..f3daf6b4fdf6 100755 --- a/src/transformers/models/dpt/modeling_dpt.py +++ b/src/transformers/models/dpt/modeling_dpt.py @@ -992,11 +992,12 @@ def forward( if labels is not None: raise NotImplementedError("Training is not implemented yet") + kwargs["output_hidden_states"] = True if self.backbone is not None: - outputs = self.backbone.forward_with_filtered_kwargs(pixel_values, output_hidden_states=True, **kwargs) + outputs = self.backbone.forward_with_filtered_kwargs(pixel_values, **kwargs) hidden_states = outputs.feature_maps else: - outputs = self.dpt(pixel_values, output_hidden_states=True, **kwargs) + outputs = self.dpt(pixel_values, **kwargs) hidden_states = outputs.hidden_states # only keep certain features based on config.backbone_out_indices # note that the hidden_states also include the initial embeddings @@ -1125,9 +1126,8 @@ def forward( if labels is not None and self.config.num_labels == 1: raise ValueError("The number of labels should be greater than one") - outputs: BaseModelOutputWithPoolingAndIntermediateActivations = self.dpt( - pixel_values, output_hidden_states=True, **kwargs - ) + kwargs["output_hidden_states"] = True + outputs: BaseModelOutputWithPoolingAndIntermediateActivations = self.dpt(pixel_values, **kwargs) hidden_states = outputs.hidden_states # only keep certain features based on config.backbone_out_indices diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index 905fbb2377ba..e4aebc827258 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -924,6 +924,7 @@ def rot_pos_emb(self, grid_thw): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -1260,6 +1261,7 @@ def get_rope_index( return position_ids, mrope_position_deltas + @can_return_tuple def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None ): @@ -1282,6 +1284,7 @@ def get_video_features( video_embeds = torch.split(video_embeds, split_sizes) return video_embeds + @can_return_tuple def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1595,12 +1598,22 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], ): - return self.model.get_video_features(pixel_values_videos, video_grid_thw) + return self.model.get_video_features( + pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs + ) - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): - return self.model.get_image_features(pixel_values, image_grid_thw) + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ): + return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @auto_docstring @can_return_tuple diff --git a/src/transformers/models/glmasr/modeling_glmasr.py b/src/transformers/models/glmasr/modeling_glmasr.py index 121f13cc91b7..045b9920bb28 100644 --- a/src/transformers/models/glmasr/modeling_glmasr.py +++ b/src/transformers/models/glmasr/modeling_glmasr.py @@ -383,6 +383,7 @@ def set_decoder(self, decoder): def get_decoder(self): return self.language_model.get_decoder() + @can_return_tuple def get_audio_features( self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor ) -> torch.FloatTensor: @@ -469,7 +470,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if input_features is not None and input_ids is not None: - audio_embeds = self.get_audio_features(input_features, input_features_mask) + audio_embeds = self.get_audio_features(input_features, input_features_mask, return_dict=True).pooler_output # replace text-audio token placeholders with audio embeddings audio_token_mask = (input_ids == self.config.audio_token_id).unsqueeze(-1) diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 17f7a190cb4c..b20abc8fef95 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -184,7 +184,8 @@ def get_image_features( raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}") # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + image_outputs = self.vision_tower(pixel_values, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index bee76aa0b22c..fec6fe262a82 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -403,7 +403,8 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + image_outputs = self.vision_tower(pixel_values, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 67de285fd52b..bdcf9a0f85de 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -452,7 +452,8 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + image_outputs = self.vision_tower(pixel_values, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -654,7 +655,8 @@ def get_video_features( batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channels, height, width) - video_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + video_outputs = self.vision_tower(pixel_values, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index fb6575c0715f..11594cbae8fe 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -326,7 +326,8 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + image_outputs = self.vision_tower(pixel_values, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -387,7 +388,8 @@ def get_video_features( batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channels, height, width) - video_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + video_outputs = self.vision_tower(pixel_values, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index 183acf17c867..f3d551454de0 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -423,7 +423,8 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + image_outputs = self.vision_tower(pixel_values, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -628,7 +629,8 @@ def get_video_features( """ batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) - vision_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + vision_outputs = self.vision_tower(pixel_values, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index a6de367325d3..3ed341d39260 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -381,7 +381,8 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + image_outputs = self.vision_tower(pixel_values, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -433,7 +434,8 @@ def get_video_features( """ batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) - vision_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + vision_outputs = self.vision_tower(pixel_values, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index ddb67dd3c6c8..4dc609405d1a 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -242,7 +242,8 @@ def get_image_features( ) # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. - image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): diff --git a/src/transformers/models/mistral3/modular_mistral3.py b/src/transformers/models/mistral3/modular_mistral3.py index 14ede88daa63..80a628483fcd 100644 --- a/src/transformers/models/mistral3/modular_mistral3.py +++ b/src/transformers/models/mistral3/modular_mistral3.py @@ -146,7 +146,8 @@ def get_image_features( ) # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. - image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index cfeb800b26ed..15b0f0292996 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -209,7 +209,8 @@ def get_image_features( if vision_feature_select_strategy not in ["default", "full"]: raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}") - image_outputs = self.image_tower(pixel_values_images, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + image_outputs = self.image_tower(pixel_values_images, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index 7282efb1cb6a..c0803b9fdfd1 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -174,7 +174,8 @@ def get_image_features( vision_feature_layers = ( vision_feature_layers if vision_feature_layers is not None else self.config.vision_feature_layers ) - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + image_outputs = self.vision_tower(pixel_values, **kwargs) # If multiple feature layers are provided (which is usually the case) # then the image features are concatenated after the CLS is removed. diff --git a/src/transformers/models/vipllava/modular_vipllava.py b/src/transformers/models/vipllava/modular_vipllava.py index a0f638c232b5..e99f74a2df0d 100644 --- a/src/transformers/models/vipllava/modular_vipllava.py +++ b/src/transformers/models/vipllava/modular_vipllava.py @@ -96,7 +96,8 @@ def get_image_features( vision_feature_layers = ( vision_feature_layers if vision_feature_layers is not None else self.config.vision_feature_layers ) - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True, **kwargs) + kwargs["output_hidden_states"] = True + image_outputs = self.vision_tower(pixel_values, **kwargs) # If multiple feature layers are provided (which is usually the case) # then the image features are concatenated after the CLS is removed. From 7c34c6ec1c2654fb58f36679dd6341d6249b0975 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 31 Dec 2025 13:24:55 +0100 Subject: [PATCH 037/131] Update new GlmAsr get_audio_features on ForConditionalGeneration --- src/transformers/models/glmasr/modeling_glmasr.py | 12 ++++++++---- src/transformers/models/glmasr/modular_glmasr.py | 9 +++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/glmasr/modeling_glmasr.py b/src/transformers/models/glmasr/modeling_glmasr.py index 045b9920bb28..8e883f28c32c 100644 --- a/src/transformers/models/glmasr/modeling_glmasr.py +++ b/src/transformers/models/glmasr/modeling_glmasr.py @@ -385,7 +385,10 @@ def get_decoder(self): @can_return_tuple def get_audio_features( - self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor + self, + input_features: torch.FloatTensor, + input_features_mask: torch.Tensor, + **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. @@ -403,7 +406,7 @@ def get_audio_features( `torch.FloatTensor`: The audio embeddings. """ - audio_outputs = self.audio_tower(input_features) + audio_outputs = self.audio_tower(input_features, **kwargs) audio_hidden_states = audio_outputs.last_hidden_state audio_hidden_states = audio_hidden_states.reshape( input_features.shape[0], -1, self.config.audio_config.intermediate_size @@ -417,8 +420,9 @@ def get_audio_features( post_lengths = (audio_lengths - merge_factor) // merge_factor + 1 valid_mask = torch.arange(audio_embeds.shape[1], device=post_lengths.device)[None, :] < post_lengths[:, None] - audio_embeds = audio_embeds[valid_mask.to(audio_embeds.device)] - return audio_embeds + audio_outputs.pooler_output = audio_embeds[valid_mask.to(audio_embeds.device)] + + return audio_outputs @can_return_tuple @auto_docstring diff --git a/src/transformers/models/glmasr/modular_glmasr.py b/src/transformers/models/glmasr/modular_glmasr.py index 35fe2986b2a1..4bac1b60a875 100644 --- a/src/transformers/models/glmasr/modular_glmasr.py +++ b/src/transformers/models/glmasr/modular_glmasr.py @@ -354,9 +354,9 @@ def __init__(self, config: GlmAsrConfig): ) class GlmAsrForConditionalGeneration(AudioFlamingo3ForConditionalGeneration): def get_audio_features( - self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor + self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> torch.FloatTensor: - audio_outputs = self.audio_tower(input_features) + audio_outputs = self.audio_tower(input_features, **kwargs) audio_hidden_states = audio_outputs.last_hidden_state audio_hidden_states = audio_hidden_states.reshape( input_features.shape[0], -1, self.config.audio_config.intermediate_size @@ -370,8 +370,9 @@ def get_audio_features( post_lengths = (audio_lengths - merge_factor) // merge_factor + 1 valid_mask = torch.arange(audio_embeds.shape[1], device=post_lengths.device)[None, :] < post_lengths[:, None] - audio_embeds = audio_embeds[valid_mask.to(audio_embeds.device)] - return audio_embeds + audio_outputs.pooler_output = audio_embeds[valid_mask.to(audio_embeds.device)] + + return audio_outputs def forward( self, From d9edd994ca5bf63f3cec3bc6e62545c54920fea1 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 31 Dec 2025 13:31:07 +0100 Subject: [PATCH 038/131] Run make style --- src/transformers/models/glmasr/modular_glmasr.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/glmasr/modular_glmasr.py b/src/transformers/models/glmasr/modular_glmasr.py index 4bac1b60a875..d58538028721 100644 --- a/src/transformers/models/glmasr/modular_glmasr.py +++ b/src/transformers/models/glmasr/modular_glmasr.py @@ -354,7 +354,10 @@ def __init__(self, config: GlmAsrConfig): ) class GlmAsrForConditionalGeneration(AudioFlamingo3ForConditionalGeneration): def get_audio_features( - self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + self, + input_features: torch.FloatTensor, + input_features_mask: torch.Tensor, + **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: audio_outputs = self.audio_tower(input_features, **kwargs) audio_hidden_states = audio_outputs.last_hidden_state From 763ddf6970e2d430e980de979394f59bbd396350 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 31 Dec 2025 14:55:34 +0100 Subject: [PATCH 039/131] Try to add _can_record_outputs to florence2 --- src/transformers/models/florence2/modeling_florence2.py | 1 + src/transformers/models/florence2/modular_florence2.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/transformers/models/florence2/modeling_florence2.py b/src/transformers/models/florence2/modeling_florence2.py index d2341e37ddae..6702e288cd23 100644 --- a/src/transformers/models/florence2/modeling_florence2.py +++ b/src/transformers/models/florence2/modeling_florence2.py @@ -493,6 +493,7 @@ class Florence2VisionPreTrainedModel(PreTrainedModel): _supports_flex_attn = True _can_compile_fullgraph = True + _can_record_outputs = {"hidden_states": Florence2VisionBlock} @auto_docstring diff --git a/src/transformers/models/florence2/modular_florence2.py b/src/transformers/models/florence2/modular_florence2.py index 0df5b6c3f2d6..5ff6a99005a0 100644 --- a/src/transformers/models/florence2/modular_florence2.py +++ b/src/transformers/models/florence2/modular_florence2.py @@ -1374,6 +1374,7 @@ class Florence2VisionPreTrainedModel(PreTrainedModel): _supports_flex_attn = True _can_compile_fullgraph = True + _can_record_outputs = {"hidden_states": Florence2VisionBlock} @auto_docstring From 8420640346709eafd29a27a16441a30f517fed6c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 31 Dec 2025 14:56:36 +0100 Subject: [PATCH 040/131] Override JanusVisionModel.forward to avoid bad q-former copy from Blip2 --- .../models/janus/modeling_janus.py | 15 ++-------- .../models/janus/modular_janus.py | 29 ++++++++++++++++++- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/transformers/models/janus/modeling_janus.py b/src/transformers/models/janus/modeling_janus.py index 605226b5c367..da2b39ad7474 100644 --- a/src/transformers/models/janus/modeling_janus.py +++ b/src/transformers/models/janus/modeling_janus.py @@ -426,17 +426,6 @@ def forward( return BaseModelOutput(last_hidden_state=hidden_states) -@dataclass -@auto_docstring -class BaseModelOutputWithQformerOutputs(BaseModelOutputWithPooling): - """ - qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): - Outputs of the Q-Former (Querying Transformer). - """ - - qformer_outputs: Optional[tuple[torch.FloatTensor]] = None - - class JanusAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper""" @@ -587,7 +576,7 @@ def forward( pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[tuple, BaseModelOutputWithQformerOutputs]: + ) -> Union[tuple, BaseModelOutputWithPooling]: if pixel_values is None: raise ValueError("You have to specify pixel_values") @@ -604,7 +593,7 @@ def forward( pooled_output = last_hidden_state[:, 0, :] pooled_output = self.post_layernorm(pooled_output) - return BaseModelOutputWithQformerOutputs( + return BaseModelOutputWithPooling( last_hidden_state=last_hidden_state, pooler_output=pooled_output, ) diff --git a/src/transformers/models/janus/modular_janus.py b/src/transformers/models/janus/modular_janus.py index 08e88b6f7bc2..b4fab6ddbcc8 100644 --- a/src/transformers/models/janus/modular_janus.py +++ b/src/transformers/models/janus/modular_janus.py @@ -44,7 +44,7 @@ valid_images, validate_preprocess_arguments, ) -from ...modeling_outputs import ModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import ImagesKwargs, Unpack from ...utils import ( @@ -558,6 +558,33 @@ def __init__(self, config: JanusVisionConfig): super().__init__(config) self.encoder = JanusVisionEncoder(config) + def forward( + self, + pixel_values: Optional[torch.FloatTensor] = None, + interpolate_pos_encoding: bool = False, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, BaseModelOutputWithPooling]: + if pixel_values is None: + raise ValueError("You have to specify pixel_values") + + hidden_states = self.embeddings(pixel_values, interpolate_pos_encoding=interpolate_pos_encoding) + + encoder_outputs: BaseModelOutput = self.encoder( + inputs_embeds=hidden_states, + **kwargs, + ) + + last_hidden_state = encoder_outputs.last_hidden_state + last_hidden_state = self.post_layernorm(last_hidden_state) + + pooled_output = last_hidden_state[:, 0, :] + pooled_output = self.post_layernorm(pooled_output) + + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + ) + class JanusVisionAlignerMLP(nn.Module): def __init__(self, config: JanusVisionConfig): From e0ea300331c9ac5e57ac72fc0b5ca9a287e4a064 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 31 Dec 2025 15:04:42 +0100 Subject: [PATCH 041/131] Import missing BaseModelOutput --- src/transformers/models/janus/modular_janus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/janus/modular_janus.py b/src/transformers/models/janus/modular_janus.py index b4fab6ddbcc8..e91ce93e0744 100644 --- a/src/transformers/models/janus/modular_janus.py +++ b/src/transformers/models/janus/modular_janus.py @@ -44,7 +44,7 @@ valid_images, validate_preprocess_arguments, ) -from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import ImagesKwargs, Unpack from ...utils import ( From 78bd0d01c1c574a14e233de39b6c12c538f3ea69 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 31 Dec 2025 15:07:36 +0100 Subject: [PATCH 042/131] Pop deprecated 'return_attentions', setting 'return_dict' won't be useful iiuc --- src/transformers/models/kosmos2/modeling_kosmos2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/kosmos2/modeling_kosmos2.py b/src/transformers/models/kosmos2/modeling_kosmos2.py index 10e8aead2ca9..28fc9f1ef3bc 100644 --- a/src/transformers/models/kosmos2/modeling_kosmos2.py +++ b/src/transformers/models/kosmos2/modeling_kosmos2.py @@ -1521,6 +1521,7 @@ def get_image_features( " and access `projection_attentions` from the returned `ModelOutput` instead.", FutureWarning, ) + kwargs.pop("return_attentions", None) vision_output: BaseModelOutputWithProjectionAttentions = self.vision_model( pixel_values=pixel_values, From d348d935530d53e727256eb77df9085fec5d45df Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 31 Dec 2025 15:58:54 +0100 Subject: [PATCH 043/131] Reintroduce kwargs filtering in llava etc. for safety re. image_sizes We also don't need to incorporate code cleanup etc. in this PR, we should keep it as minimal as possible and leave these kinds of lines intact. --- src/transformers/models/fast_vlm/modeling_fast_vlm.py | 1 + src/transformers/models/fast_vlm/modular_fast_vlm.py | 1 + src/transformers/models/llama4/modeling_llama4.py | 1 + src/transformers/models/llava/modeling_llava.py | 1 + src/transformers/models/mistral3/modeling_mistral3.py | 1 + src/transformers/models/mistral3/modular_mistral3.py | 1 + 6 files changed, 6 insertions(+) diff --git a/src/transformers/models/fast_vlm/modeling_fast_vlm.py b/src/transformers/models/fast_vlm/modeling_fast_vlm.py index 671ea561c80c..68a0b2487a43 100644 --- a/src/transformers/models/fast_vlm/modeling_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modeling_fast_vlm.py @@ -145,6 +145,7 @@ def get_image_features( else self.config.vision_feature_select_strategy ) + kwargs = {k: v for k, v in kwargs.items() if v is not None} image_outputs = self.vision_tower(pixel_values, **kwargs) # since the vision tower is hybrid in FastVLM, its output needs to be handled differently from Llava diff --git a/src/transformers/models/fast_vlm/modular_fast_vlm.py b/src/transformers/models/fast_vlm/modular_fast_vlm.py index b641b4a6e102..88d9606d42a6 100644 --- a/src/transformers/models/fast_vlm/modular_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modular_fast_vlm.py @@ -201,6 +201,7 @@ def get_image_features( else self.config.vision_feature_select_strategy ) + kwargs = {k: v for k, v in kwargs.items() if v is not None} image_outputs = self.vision_tower(pixel_values, **kwargs) # since the vision tower is hybrid in FastVLM, its output needs to be handled differently from Llava diff --git a/src/transformers/models/llama4/modeling_llama4.py b/src/transformers/models/llama4/modeling_llama4.py index acdb6e45538d..88a8f26648a4 100644 --- a/src/transformers/models/llama4/modeling_llama4.py +++ b/src/transformers/models/llama4/modeling_llama4.py @@ -1219,6 +1219,7 @@ def get_image_features( """ if vision_feature_select_strategy not in ["default", "full"]: raise ValueError(f"Unexpected select feature strategy: {vision_feature_select_strategy}") + kwargs = {k: v for k, v in kwargs.items() if v is not None} return self.vision_model(pixel_values, **kwargs) def get_placeholder_mask( diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index b20abc8fef95..01d08d5fef1b 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -183,6 +183,7 @@ def get_image_features( if vision_feature_select_strategy not in ["default", "full"]: raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}") + kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. kwargs["output_hidden_states"] = True image_outputs = self.vision_tower(pixel_values, **kwargs) diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index 4dc609405d1a..7aa2a65037b6 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -241,6 +241,7 @@ def get_image_features( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) + kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. kwargs["output_hidden_states"] = True image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, **kwargs) diff --git a/src/transformers/models/mistral3/modular_mistral3.py b/src/transformers/models/mistral3/modular_mistral3.py index 80a628483fcd..f5b99a373898 100644 --- a/src/transformers/models/mistral3/modular_mistral3.py +++ b/src/transformers/models/mistral3/modular_mistral3.py @@ -145,6 +145,7 @@ def get_image_features( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) + kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. kwargs["output_hidden_states"] = True image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, **kwargs) From 71ea85a2cf8c9a317007cc04c408fda78bc829e6 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 31 Dec 2025 16:11:35 +0100 Subject: [PATCH 044/131] Use BaseModelOutputWithPooling superclass consistently for custom get_..._features outputs --- .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 22 ++++-------- .../qwen3_omni_moe/modular_qwen3_omni_moe.py | 22 ++++-------- src/transformers/models/sam2/modeling_sam2.py | 32 ++++------------- src/transformers/models/sam2/modular_sam2.py | 31 ++++------------- .../modeling_sam3_tracker_video.py | 34 +++++-------------- .../modular_sam3_tracker_video.py | 34 +++++-------------- 6 files changed, 42 insertions(+), 133 deletions(-) diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 7bcdf4abf512..a5a6a2406697 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -42,6 +42,7 @@ from ...modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPast, + BaseModelOutputWithPooling, CausalLMOutputWithPast, MoeCausalLMOutputWithPast, MoeModelOutputWithPast, @@ -49,7 +50,7 @@ from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ModelOutput, auto_docstring, can_return_tuple +from ...utils import auto_docstring, can_return_tuple from ...utils.generic import OutputRecorder, TransformersKwargs, check_model_inputs, maybe_autocast from .configuration_qwen3_omni_moe import ( Qwen3OmniMoeAudioEncoderConfig, @@ -65,24 +66,13 @@ @dataclass -class BaseModelOutputWithDeepstackFeatures(ModelOutput): +@auto_docstring +class BaseModelOutputWithDeepstackFeatures(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - deepstack_features (`List[torch.FloatTensor]`, *optional*): - List of hidden-states (feature maps) from deepstack layers. + deepstack_features (`List[torch.FloatTensor]`, *optional*): + List of hidden-states (feature maps) from deepstack layers. """ - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None deepstack_features: Optional[list[torch.FloatTensor]] = None diff --git a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py index fe6763611f44..abc1d4a3659a 100644 --- a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py @@ -38,6 +38,7 @@ from ...modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPast, + BaseModelOutputWithPooling, CausalLMOutputWithPast, MoeCausalLMOutputWithPast, MoeModelOutputWithPast, @@ -46,7 +47,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import ProcessorMixin, Unpack from ...tokenization_utils_base import TextInput -from ...utils import ModelOutput, auto_docstring, can_return_tuple, logging +from ...utils import auto_docstring, can_return_tuple, logging from ...utils.generic import OutputRecorder, TransformersKwargs, check_model_inputs from ...video_utils import VideoInput, make_batched_videos from ..mimi.modeling_mimi import MimiLayerScale @@ -103,24 +104,13 @@ @dataclass -class BaseModelOutputWithDeepstackFeatures(ModelOutput): +@auto_docstring +class BaseModelOutputWithDeepstackFeatures(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - deepstack_features (`List[torch.FloatTensor]`, *optional*): - List of hidden-states (feature maps) from deepstack layers. + deepstack_features (`List[torch.FloatTensor]`, *optional*): + List of hidden-states (feature maps) from deepstack layers. """ - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None deepstack_features: Optional[list[torch.FloatTensor]] = None diff --git a/src/transformers/models/sam2/modeling_sam2.py b/src/transformers/models/sam2/modeling_sam2.py index f837dd1520e8..ee636655ece6 100644 --- a/src/transformers/models/sam2/modeling_sam2.py +++ b/src/transformers/models/sam2/modeling_sam2.py @@ -35,7 +35,7 @@ from ... import initialization as init from ...activations import ACT2FN from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import compile_compatible_method_lru_cache @@ -52,33 +52,15 @@ @dataclass -class BaseModelOutputWithFeatureMaps(ModelOutput): +@auto_docstring +class BaseModelOutputWithFeatureMaps(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, - sequence_length)`. - - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention - heads. - feature_maps (`list[torch.Tensor]`): - List of feature maps from different layers of the model. - feature_maps_position_embeddings (`list[torch.Tensor]`): - List of position embeddings corresponding to the feature maps. + feature_maps (`list[torch.Tensor]`): + List of feature maps from different layers of the model. + feature_maps_position_embeddings (`list[torch.Tensor]`): + List of position embeddings corresponding to the feature maps. """ - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None - attentions: Optional[tuple[torch.FloatTensor, ...]] = None feature_maps: Optional[list[torch.Tensor]] = None feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None diff --git a/src/transformers/models/sam2/modular_sam2.py b/src/transformers/models/sam2/modular_sam2.py index df3b3f95c0dd..4934d284861d 100644 --- a/src/transformers/models/sam2/modular_sam2.py +++ b/src/transformers/models/sam2/modular_sam2.py @@ -37,6 +37,7 @@ pil_torch_interpolation_mapping, ) from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import ImagesKwargs, Unpack from ...utils import ( @@ -74,33 +75,15 @@ @dataclass -class BaseModelOutputWithFeatureMaps(ModelOutput): +@auto_docstring +class BaseModelOutputWithFeatureMaps(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, - sequence_length)`. - - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention - heads. - feature_maps (`list[torch.Tensor]`): - List of feature maps from different layers of the model. - feature_maps_position_embeddings (`list[torch.Tensor]`): - List of position embeddings corresponding to the feature maps. + feature_maps (`list[torch.Tensor]`): + List of feature maps from different layers of the model. + feature_maps_position_embeddings (`list[torch.Tensor]`): + List of position embeddings corresponding to the feature maps. """ - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None - attentions: Optional[tuple[torch.FloatTensor, ...]] = None feature_maps: Optional[list[torch.Tensor]] = None feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None diff --git a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py index 62431fc01e66..a2d26eb17603 100644 --- a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py @@ -36,11 +36,11 @@ from ...activations import ACT2FN from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, ModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import compile_compatible_method_lru_cache -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple from ...utils.generic import OutputRecorder from ..auto import AutoModel from .configuration_sam3_tracker_video import ( @@ -51,33 +51,15 @@ @dataclass -class BaseModelOutputWithFeatureMaps(ModelOutput): +@auto_docstring +class BaseModelOutputWithFeatureMaps(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, - sequence_length)`. - - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention - heads. - feature_maps (`list[torch.Tensor]`): - List of feature maps from different layers of the model. - feature_maps_position_embeddings (`list[torch.Tensor]`): - List of position embeddings corresponding to the feature maps. + feature_maps (`list[torch.Tensor]`): + List of feature maps from different layers of the model. + feature_maps_position_embeddings (`list[torch.Tensor]`): + List of position embeddings corresponding to the feature maps. """ - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None - attentions: Optional[tuple[torch.FloatTensor, ...]] = None feature_maps: Optional[list[torch.Tensor]] = None feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None diff --git a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py index 745fb6538761..a623190f95be 100644 --- a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py @@ -19,9 +19,9 @@ import torch from ...configuration_utils import PreTrainedConfig -from ...modeling_outputs import ModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack -from ...utils import TransformersKwargs, can_return_tuple +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ..auto import CONFIG_MAPPING, AutoConfig, AutoModel from ..sam2_video.configuration_sam2_video import Sam2VideoMaskDecoderConfig, Sam2VideoPromptEncoderConfig from ..sam2_video.modeling_sam2_video import ( @@ -56,33 +56,15 @@ @dataclass -class BaseModelOutputWithFeatureMaps(ModelOutput): +@auto_docstring +class BaseModelOutputWithFeatureMaps(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, - sequence_length)`. - - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention - heads. - feature_maps (`list[torch.Tensor]`): - List of feature maps from different layers of the model. - feature_maps_position_embeddings (`list[torch.Tensor]`): - List of position embeddings corresponding to the feature maps. + feature_maps (`list[torch.Tensor]`): + List of feature maps from different layers of the model. + feature_maps_position_embeddings (`list[torch.Tensor]`): + List of position embeddings corresponding to the feature maps. """ - last_hidden_state: Optional[torch.FloatTensor] = None - pooler_output: Optional[torch.FloatTensor] = None - attentions: Optional[tuple[torch.FloatTensor, ...]] = None feature_maps: Optional[list[torch.Tensor]] = None feature_maps_position_embeddings: Optional[list[torch.Tensor]] = None From 8c59e9513362e701c94ffe68edbe376d853f6d93 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 13:13:49 +0100 Subject: [PATCH 045/131] Update Blip-2 family and its BaseModelOutputWithVisionQformerOutputs To use both a vision_outputs and qformer_outputs as keys in the BaseModelOutputWithPooling subclass, despite some duplication. --- .../models/blip_2/modeling_blip_2.py | 24 +++++++--- .../instructblip/modeling_instructblip.py | 27 +++++++---- .../modeling_instructblipvideo.py | 47 +++++++++++-------- .../modular_instructblipvideo.py | 21 ++++++--- 4 files changed, 77 insertions(+), 42 deletions(-) diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 5f0be497c258..7fa878d5b8a1 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -57,12 +57,15 @@ @dataclass @auto_docstring -class BaseModelOutputWithQformerOutputs(BaseModelOutputWithPooling): +class BaseModelOutputWithVisionQformerOutputs(BaseModelOutputWithPooling): """ + vision_outputs (`BaseModelOutputWithPooling`): + Outputs of the vision encoder. qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): Outputs of the Q-Former (Querying Transformer). """ + vision_outputs: Optional[torch.FloatTensor] = None qformer_outputs: Optional[tuple[torch.FloatTensor]] = None @@ -504,7 +507,7 @@ def forward( pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[tuple, BaseModelOutputWithQformerOutputs]: + ) -> Union[tuple, BaseModelOutputWithPooling]: if pixel_values is None: raise ValueError("You have to specify pixel_values") @@ -521,7 +524,7 @@ def forward( pooled_output = last_hidden_state[:, 0, :] pooled_output = self.post_layernorm(pooled_output) - return BaseModelOutputWithQformerOutputs( + return BaseModelOutputWithPooling( last_hidden_state=last_hidden_state, pooler_output=pooled_output, ) @@ -1635,12 +1638,20 @@ def get_image_features( """ # step 1: forward the images through the vision encoder, # to get image embeddings of shape (batch_size, seq_len, hidden_size) - vision_outputs: BaseModelOutputWithQformerOutputs = self.vision_model( + vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, **kwargs, ) + vision_outputs = BaseModelOutputWithVisionQformerOutputs( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=vision_outputs.pooler_output, + hidden_states=vision_outputs.hidden_states, + attentions=vision_outputs.attentions, + vision_outputs=vision_outputs, + qformer_outputs=None, + ) image_embeds = vision_outputs[0] # step 2: forward the query tokens through the QFormer, using the image embeddings for cross-attention @@ -1769,13 +1780,12 @@ def forward( two ```""" - image_features: BaseModelOutputWithQformerOutputs = self.get_image_features( + image_features: BaseModelOutputWithVisionQformerOutputs = self.get_image_features( pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True ) language_model_inputs = image_features.pooler_output qformer_outputs = image_features.qformer_outputs - vision_outputs = image_features - vision_outputs.qformer_outputs = None # to avoid redundancy in the output + vision_outputs = image_features.vision_outputs if inputs_embeds is None: inputs_embeds = self.get_input_embeddings()(input_ids) diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index 484964674023..b7c138559a73 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -47,12 +47,15 @@ @dataclass @auto_docstring -class BaseModelOutputWithQformerOutputs(BaseModelOutputWithPooling): +class BaseModelOutputWithVisionQformerOutputs(BaseModelOutputWithPooling): """ + vision_outputs (`BaseModelOutputWithPooling`): + Outputs of the vision encoder. qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): Outputs of the Q-Former (Querying Transformer). """ + vision_outputs: Optional[torch.FloatTensor] = None qformer_outputs: Optional[tuple[torch.FloatTensor]] = None @@ -410,7 +413,7 @@ def forward( pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[tuple, BaseModelOutputWithQformerOutputs]: + ) -> Union[tuple, BaseModelOutputWithPooling]: if pixel_values is None: raise ValueError("You have to specify pixel_values") @@ -427,7 +430,7 @@ def forward( pooled_output = last_hidden_state[:, 0, :] pooled_output = self.post_layernorm(pooled_output) - return BaseModelOutputWithQformerOutputs( + return BaseModelOutputWithPooling( last_hidden_state=last_hidden_state, pooler_output=pooled_output, ) @@ -1202,12 +1205,20 @@ def get_image_features( """ # step 1: forward the images through the vision encoder, # to get image embeddings of shape (batch_size, seq_len, hidden_size) - vision_outputs: BaseModelOutputWithQformerOutputs = self.vision_model( + vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, **kwargs, ) + vision_outputs = BaseModelOutputWithVisionQformerOutputs( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=vision_outputs.pooler_output, + hidden_states=vision_outputs.hidden_states, + attentions=vision_outputs.attentions, + vision_outputs=vision_outputs, + qformer_outputs=None, + ) image_embeds = vision_outputs[0] # step 2: forward the query tokens through the QFormer, using the image embeddings for cross-attention @@ -1329,8 +1340,7 @@ def forward( The unusual aspect of this image is that a man is ironing clothes on the back of a yellow SUV, which is parked in the middle of a busy city street. This is an unconventional approach to ironing clothes, as it requires the man to balance himself and his ironing equipment on top of the vehicle while navigating through traffic. Additionally, the presence of taxis and other vehicles in the scene further emphasizes the unusual nature of this situation. ```""" - # NOTE: @Tom this is not particularly clean, e.g. having to manually remove qformer outputs from vision outputs - image_features: BaseModelOutputWithQformerOutputs = self.get_image_features( + image_features: BaseModelOutputWithVisionQformerOutputs = self.get_image_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, @@ -1339,8 +1349,7 @@ def forward( ) language_model_inputs = image_features.pooler_output qformer_outputs = image_features.qformer_outputs - vision_outputs = image_features - vision_outputs.qformer_outputs = None # to avoid redundancy in the output + vision_outputs = image_features.vision_outputs if inputs_embeds is None: inputs_embeds = self.get_input_embeddings()(input_ids) @@ -1425,7 +1434,7 @@ def generate( self._preprocess_accelerate() batch_size = pixel_values.shape[0] - image_features: BaseModelOutputWithQformerOutputs = self.get_image_features( + image_features: BaseModelOutputWithVisionQformerOutputs = self.get_image_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index c69877a3b3bd..4af413c969b9 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -212,17 +212,6 @@ def _init_weights(self, module): init.copy_(module.position_ids, torch.arange(module.position_ids.shape[-1]).expand((1, -1))) -@dataclass -@auto_docstring -class BaseModelOutputWithQformerOutputs(BaseModelOutputWithPooling): - """ - qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): - Outputs of the Q-Former (Querying Transformer). - """ - - qformer_outputs: Optional[tuple[torch.FloatTensor]] = None - - # Adapted from transformers.models.siglip.modeling_siglip.eager_attention_forward -> InstructBlipVideo doesn't cast attn weights to fp32 def eager_attention_forward( module: nn.Module, @@ -428,7 +417,7 @@ def forward( pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[tuple, BaseModelOutputWithQformerOutputs]: + ) -> Union[tuple, BaseModelOutputWithPooling]: if pixel_values is None: raise ValueError("You have to specify pixel_values") @@ -445,7 +434,7 @@ def forward( pooled_output = last_hidden_state[:, 0, :] pooled_output = self.post_layernorm(pooled_output) - return BaseModelOutputWithQformerOutputs( + return BaseModelOutputWithPooling( last_hidden_state=last_hidden_state, pooler_output=pooled_output, ) @@ -1131,6 +1120,20 @@ def forward( ) +@dataclass +@auto_docstring +class BaseModelOutputWithVisionQformerOutputs(BaseModelOutputWithPooling): + """ + vision_outputs (`BaseModelOutputWithPooling`): + Outputs of the vision encoder. + qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): + Outputs of the Q-Former (Querying Transformer). + """ + + vision_outputs: Optional[torch.FloatTensor] = None + qformer_outputs: Optional[tuple[torch.FloatTensor]] = None + + @auto_docstring( custom_intro=""" InstructBlipVideo Model for generating text given an image and an optional text prompt. The model consists of a vision @@ -1314,8 +1317,7 @@ def forward( ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict - # NOTE: @Tom this is not particularly clean, e.g. having to manually remove qformer outputs from vision outputs - video_features: BaseModelOutputWithQformerOutputs = self.get_video_features( + video_features: BaseModelOutputWithVisionQformerOutputs = self.get_video_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, @@ -1325,8 +1327,7 @@ def forward( ) language_model_inputs = video_features.pooler_output qformer_outputs = video_features.qformer_outputs - vision_outputs = video_features - vision_outputs.qformer_outputs = None # to avoid redundancy in the output + vision_outputs = video_features.vision_outputs if inputs_embeds is None: inputs_embeds = self.get_input_embeddings()(input_ids) @@ -1418,7 +1419,7 @@ def generate( self._preprocess_accelerate() batch_size = pixel_values.shape[0] - video_features: BaseModelOutputWithQformerOutputs = self.get_video_features( + video_features: BaseModelOutputWithVisionQformerOutputs = self.get_video_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, @@ -1471,12 +1472,20 @@ def get_video_features( batch_size, frames, channel, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channel, height, width) - vision_outputs: BaseModelOutputWithQformerOutputs = self.vision_model( + vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, **kwargs, ) + vision_outputs = BaseModelOutputWithVisionQformerOutputs( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=vision_outputs.pooler_output, + hidden_states=vision_outputs.hidden_states, + attentions=vision_outputs.attentions, + vision_outputs=vision_outputs, + qformer_outputs=None, + ) image_embeds = vision_outputs[0] # step 2: forward the query tokens through the QFormer, using the image embeddings for cross-attention diff --git a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py index 511d380113f9..3d07defaeb26 100644 --- a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py @@ -22,7 +22,7 @@ InstructBlipVisionConfig, ) from transformers.models.instructblip.modeling_instructblip import ( - BaseModelOutputWithQformerOutputs, + BaseModelOutputWithVisionQformerOutputs, InstructBlipForConditionalGeneration, InstructBlipForConditionalGenerationModelOutput, InstructBlipModel, @@ -34,6 +34,7 @@ from ...configuration_utils import PreTrainedConfig from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_outputs import BaseModelOutputWithPooling from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES from ...processing_utils import Unpack from ...utils import can_return_tuple, logging @@ -307,12 +308,20 @@ def get_video_features( batch_size, frames, channel, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channel, height, width) - vision_outputs: BaseModelOutputWithQformerOutputs = self.vision_model( + vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, return_dict=True, **kwargs, ) + vision_outputs = BaseModelOutputWithVisionQformerOutputs( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=vision_outputs.pooler_output, + hidden_states=vision_outputs.hidden_states, + attentions=vision_outputs.attentions, + vision_outputs=vision_outputs, + qformer_outputs=None, + ) image_embeds = vision_outputs[0] # step 2: forward the query tokens through the QFormer, using the image embeddings for cross-attention @@ -450,8 +459,7 @@ def forward( ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict - # NOTE: @Tom this is not particularly clean, e.g. having to manually remove qformer outputs from vision outputs - video_features: BaseModelOutputWithQformerOutputs = self.get_video_features( + video_features: BaseModelOutputWithVisionQformerOutputs = self.get_video_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, @@ -461,8 +469,7 @@ def forward( ) language_model_inputs = video_features.pooler_output qformer_outputs = video_features.qformer_outputs - vision_outputs = video_features - vision_outputs.qformer_outputs = None # to avoid redundancy in the output + vision_outputs = video_features.vision_outputs if inputs_embeds is None: inputs_embeds = self.get_input_embeddings()(input_ids) @@ -554,7 +561,7 @@ def generate( self._preprocess_accelerate() batch_size = pixel_values.shape[0] - video_features: BaseModelOutputWithQformerOutputs = self.get_video_features( + video_features: BaseModelOutputWithVisionQformerOutputs = self.get_video_features( pixel_values, qformer_input_ids=qformer_input_ids, qformer_attention_mask=qformer_attention_mask, From 3f4c34bb3ea07bb53ac7060e30e92a806b4fcaef Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 13:28:28 +0100 Subject: [PATCH 046/131] Update glm4v _can_record_outputs --- src/transformers/models/glm4v/modeling_glm4v.py | 4 ++++ src/transformers/models/glm4v/modular_glm4v.py | 4 ++++ src/transformers/models/glm4v_moe/modeling_glm4v_moe.py | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index 91f71860e29c..291b9d7bdabe 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -718,6 +718,10 @@ class Glm4vVisionModel(Glm4vPreTrainedModel): config: Glm4vVisionConfig input_modalities = ("image", "video") _no_split_modules = ["Glm4vVisionBlock"] + _can_record_outputs = { + "hidden_states": Glm4vVisionBlock, + "attentions": Glm4vVisionAttention, + } def __init__(self, config) -> None: super().__init__(config) diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 395390b3df75..239c581eca94 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -736,6 +736,10 @@ class Glm4vVisionModel(Glm4vPreTrainedModel): config: Glm4vVisionConfig input_modalities = ("image", "video") _no_split_modules = ["Glm4vVisionBlock"] + _can_record_outputs = { + "hidden_states": Glm4vVisionBlock, + "attentions": Glm4vVisionAttention, + } def __init__(self, config) -> None: super().__init__(config) diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 16c02f9b4600..28d55d8d41e4 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -932,6 +932,10 @@ class Glm4vMoeVisionModel(Glm4vMoePreTrainedModel): config: Glm4vMoeVisionConfig input_modalities = ("image", "video") _no_split_modules = ["Glm4vMoeVisionBlock"] + _can_record_outputs = { + "hidden_states": Glm4vMoeVisionBlock, + "attentions": Glm4vMoeVisionAttention, + } def __init__(self, config) -> None: super().__init__(config) From b39b6d1c9c073394dd368f2bf3c32ab89086cbdd Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 13:40:18 +0100 Subject: [PATCH 047/131] Remove check_model_inputs in granite_speech I could also use can_return_tuple, but this might be problematic if `return_dict=False` in the config --- .../models/granite_speech/modeling_granite_speech.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index 58de2710e4c8..76c168b62406 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -269,7 +269,6 @@ def __init__(self, config: GraniteSpeechEncoderConfig): self.out_mid = nn.Linear(config.output_dim, config.hidden_dim, bias=True) self.num_layers = config.num_layers - @check_model_inputs(tie_last_hidden_states=False) def forward(self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> BaseModelOutputWithPooling: hidden_states = self.input_linear(hidden_states) for idx, layer in enumerate(self.layers, start=1): From af0ccb107349a746dfcdb11f37f4ed6c1c64a014 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 13:47:31 +0100 Subject: [PATCH 048/131] Run make style --- .../models/granite_speech/modeling_granite_speech.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index 76c168b62406..fbae189a11a0 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -28,7 +28,6 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_peft_available, logging -from ...utils.generic import check_model_inputs from ..auto import AutoModel, AutoModelForCausalLM from .configuration_granite_speech import GraniteSpeechConfig, GraniteSpeechEncoderConfig From f8e08d974111c8b29610f85e11982da16aecfca3 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 14:00:10 +0100 Subject: [PATCH 049/131] Add _can_record_outputs to Ovis2VisionModel --- src/transformers/models/ovis2/modeling_ovis2.py | 4 ++++ src/transformers/models/ovis2/modular_ovis2.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index f75d1e27187b..a32b744b49bf 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -461,6 +461,10 @@ def hard_softmax(logits: torch.Tensor, dim: int): class Ovis2VisionModel(Ovis2PreTrainedModel): config: Ovis2VisionConfig + _can_record_outputs = { + "hidden_states": Ovis2VisionEncoderLayer, + "attentions": Ovis2VisionAttention, + } def __init__(self, config: Ovis2VisionConfig): super().__init__(config) diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index 69757261ef5a..5d289546b312 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -181,6 +181,10 @@ def _init_weights(self, module): class Ovis2VisionModel(Ovis2PreTrainedModel): config: Ovis2VisionConfig + _can_record_outputs = { + "hidden_states": Ovis2VisionEncoderLayer, + "attentions": Ovis2VisionAttention, + } def __init__(self, config: Ovis2VisionConfig): super().__init__(config) From 2d747d9478158d8672f65b15836359bcc5e04980 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 15:01:28 +0100 Subject: [PATCH 050/131] Update get_text_features/get_video_features from pe_video --- .../models/pe_video/modeling_pe_video.py | 55 ++++++++++-------- .../models/pe_video/modular_pe_video.py | 56 +++++++++++-------- 2 files changed, 65 insertions(+), 46 deletions(-) diff --git a/src/transformers/models/pe_video/modeling_pe_video.py b/src/transformers/models/pe_video/modeling_pe_video.py index 65ccf45af24a..4e3a941891c8 100644 --- a/src/transformers/models/pe_video/modeling_pe_video.py +++ b/src/transformers/models/pe_video/modeling_pe_video.py @@ -20,7 +20,7 @@ # limitations under the License. from collections.abc import Callable from dataclasses import dataclass -from typing import Any, Optional +from typing import Any, Optional, Union import torch import torch.nn as nn @@ -35,8 +35,7 @@ from ...modeling_outputs import BaseModelOutputWithPooling, MaskedLMOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...processing_utils import Unpack -from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple +from ...utils import ModelOutput, TransformersKwargs, Unpack, auto_docstring, can_return_tuple from ...utils.generic import check_model_inputs, maybe_autocast from ..auto import AutoModel, AutoModelForImageClassification from .configuration_pe_video import PeVideoConfig, PeVideoEncoderConfig @@ -571,27 +570,37 @@ def __init__(self, config: PeVideoConfig): self.post_init() - def get_text_features(self, input_ids, attention_mask=None): - # TODO: should it be named feature or embeds - text_outputs: MaskedLMOutput = self.text_model( - input_ids=input_ids, - attention_mask=attention_mask, - return_dict=True, - ) - - text_features = text_outputs.last_hidden_state - text_features = self.text_video_head(text_features) - return text_features + @can_return_tuple + @auto_docstring + def get_text_features( + self, + input_ids: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, BaseModelOutputWithPooling]: + text_outputs: BaseModelOutputWithPooling = self.text_model( + input_ids=input_ids, + attention_mask=attention_mask, + **kwargs, + ) + text_outputs.pooler_output = self.text_video_head(text_outputs.last_hidden_state) + return text_outputs - def get_video_features(self, pixel_values_videos, padding_mask_videos=None): - # TODO: should it be named feature or embeds - video_outputs: BaseModelOutputWithPooling = self.video_encoder( - pixel_values_videos=pixel_values_videos, - padding_mask_videos=padding_mask_videos, - return_dict=True, - ) - video_features = self.video_head(video_outputs.pooler_output) - return video_features + @can_return_tuple + @auto_docstring + def get_video_features( + self, + pixel_values_videos: torch.Tensor, + padding_mask_videos: Optional[torch.Tensor] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, BaseModelOutputWithPooling]: + video_outputs: BaseModelOutputWithPooling = self.video_encoder( + pixel_values_videos=pixel_values_videos, + padding_mask_videos=padding_mask_videos, + **kwargs, + ) + video_outputs.pooler_output = self.video_head(video_outputs.pooler_output) + return video_outputs @can_return_tuple def forward( diff --git a/src/transformers/models/pe_video/modular_pe_video.py b/src/transformers/models/pe_video/modular_pe_video.py index 65f1c6b6367b..c72c75f176ba 100644 --- a/src/transformers/models/pe_video/modular_pe_video.py +++ b/src/transformers/models/pe_video/modular_pe_video.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass -from typing import Any, Optional +from typing import Any, Optional, Union import torch import torch.nn as nn @@ -21,7 +21,7 @@ from ...modeling_attn_mask_utils import _prepare_4d_attention_mask from ...modeling_outputs import BaseModelOutputWithPooling, MaskedLMOutput -from ...utils import ModelOutput, auto_docstring, can_return_tuple +from ...utils import ModelOutput, TransformersKwargs, Unpack, auto_docstring, can_return_tuple from ...utils.generic import check_model_inputs from ..auto import AutoModel, AutoModelForImageClassification from ..pe_audio_video.modeling_pe_audio_video import ( @@ -151,27 +151,37 @@ def __init__(self, config: PeVideoConfig): self.post_init() - def get_text_features(self, input_ids, attention_mask=None): - # TODO: should it be named feature or embeds - text_outputs: MaskedLMOutput = self.text_model( - input_ids=input_ids, - attention_mask=attention_mask, - return_dict=True, - ) - - text_features = text_outputs.last_hidden_state - text_features = self.text_video_head(text_features) - return text_features - - def get_video_features(self, pixel_values_videos, padding_mask_videos=None): - # TODO: should it be named feature or embeds - video_outputs: BaseModelOutputWithPooling = self.video_encoder( - pixel_values_videos=pixel_values_videos, - padding_mask_videos=padding_mask_videos, - return_dict=True, - ) - video_features = self.video_head(video_outputs.pooler_output) - return video_features + @can_return_tuple + @auto_docstring + def get_text_features( + self, + input_ids: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, BaseModelOutputWithPooling]: + text_outputs: BaseModelOutputWithPooling = self.text_model( + input_ids=input_ids, + attention_mask=attention_mask, + **kwargs, + ) + text_outputs.pooler_output = self.text_video_head(text_outputs.last_hidden_state) + return text_outputs + + @can_return_tuple + @auto_docstring + def get_video_features( + self, + pixel_values_videos: torch.Tensor, + padding_mask_videos: Optional[torch.Tensor] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, BaseModelOutputWithPooling]: + video_outputs: BaseModelOutputWithPooling = self.video_encoder( + pixel_values_videos=pixel_values_videos, + padding_mask_videos=padding_mask_videos, + **kwargs, + ) + video_outputs.pooler_output = self.video_head(video_outputs.pooler_output) + return video_outputs @can_return_tuple def forward( From 008e15d37b21df021f3aed55efb4705dd448bd88 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 15:01:56 +0100 Subject: [PATCH 051/131] Update missing case on sam3 --- src/transformers/models/sam3/modeling_sam3.py | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/transformers/models/sam3/modeling_sam3.py b/src/transformers/models/sam3/modeling_sam3.py index b102c489a5d1..c2790782339d 100644 --- a/src/transformers/models/sam3/modeling_sam3.py +++ b/src/transformers/models/sam3/modeling_sam3.py @@ -40,7 +40,7 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import compile_compatible_method_lru_cache -from ...utils import auto_docstring, logging +from ...utils import auto_docstring, can_return_tuple, logging from ...utils.generic import TransformersKwargs, check_model_inputs from ..auto import AutoModel from .configuration_sam3 import ( @@ -2150,6 +2150,7 @@ def __init__(self, config: Sam3Config): self.post_init() + @can_return_tuple @auto_docstring def get_text_features( self, @@ -2157,11 +2158,8 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. - Returns: text_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Text embeddings that can be passed as `text_embeds` to the forward method. @@ -2187,17 +2185,11 @@ def get_text_features( >>> outputs = model(pixel_values=img_inputs.pixel_values, text_embeds=text_embeds) ``` """ - text_features = self.text_encoder(input_ids=input_ids, attention_mask=attention_mask, **kwargs) - last_hidden_state = text_features.last_hidden_state - text_features = self.text_projection(last_hidden_state) - - if return_dict: - return BaseModelOutputWithPooling( - last_hidden_state=last_hidden_state, - pooler_output=text_features, - ) + text_outputs = self.text_encoder(input_ids=input_ids, attention_mask=attention_mask, **kwargs) + last_hidden_state = text_outputs.last_hidden_state + text_outputs.pooler_output = self.text_projection(last_hidden_state) - return text_features + return text_outputs @auto_docstring def get_vision_features( From e92efb9c8b6fd379e221e51a15dc87642665e68c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 15:13:14 +0100 Subject: [PATCH 052/131] Update get_text_features type hints to Union[tuple, BaseModelOutputWithPooling] Blip-2 and Clvp are the only exceptions --- src/transformers/models/aimv2/modeling_aimv2.py | 2 +- src/transformers/models/align/modeling_align.py | 2 +- src/transformers/models/altclip/modeling_altclip.py | 2 +- src/transformers/models/blip/modeling_blip.py | 2 +- src/transformers/models/blip_2/modeling_blip_2.py | 2 +- src/transformers/models/chinese_clip/modeling_chinese_clip.py | 2 +- src/transformers/models/clap/modeling_clap.py | 2 +- src/transformers/models/clip/modeling_clip.py | 2 +- src/transformers/models/clipseg/modeling_clipseg.py | 2 +- src/transformers/models/clvp/modeling_clvp.py | 2 +- src/transformers/models/flava/modeling_flava.py | 2 +- src/transformers/models/groupvit/modeling_groupvit.py | 2 +- src/transformers/models/metaclip_2/modeling_metaclip_2.py | 2 +- src/transformers/models/metaclip_2/modular_metaclip_2.py | 4 ++-- src/transformers/models/owlv2/modeling_owlv2.py | 2 +- src/transformers/models/owlvit/modeling_owlvit.py | 2 +- src/transformers/models/siglip/modeling_siglip.py | 2 +- src/transformers/models/siglip2/modeling_siglip2.py | 2 +- .../modeling_vision_text_dual_encoder.py | 2 +- src/transformers/models/x_clip/modeling_x_clip.py | 2 +- 20 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/transformers/models/aimv2/modeling_aimv2.py b/src/transformers/models/aimv2/modeling_aimv2.py index 54b922be7582..389d7b59f287 100644 --- a/src/transformers/models/aimv2/modeling_aimv2.py +++ b/src/transformers/models/aimv2/modeling_aimv2.py @@ -612,7 +612,7 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/align/modeling_align.py b/src/transformers/models/align/modeling_align.py index d42f4c3e0a75..a4dc2d10b53b 100644 --- a/src/transformers/models/align/modeling_align.py +++ b/src/transformers/models/align/modeling_align.py @@ -1104,7 +1104,7 @@ def get_text_features( position_ids: Optional[torch.Tensor] = None, inputs_embeds: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/altclip/modeling_altclip.py b/src/transformers/models/altclip/modeling_altclip.py index cbee667f7051..510ba492efff 100755 --- a/src/transformers/models/altclip/modeling_altclip.py +++ b/src/transformers/models/altclip/modeling_altclip.py @@ -1164,7 +1164,7 @@ def get_text_features( position_ids: Optional[torch.Tensor] = None, token_type_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index b9ec7b58eb63..c6f18b239047 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -581,7 +581,7 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 7fa878d5b8a1..bdf75b3b0bcd 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -1091,7 +1091,7 @@ def get_text_features( decoder_attention_mask: Optional[torch.Tensor] = None, labels: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, CausalLMOutputWithPast]: + ) -> Union[tuple, BaseModelOutput]: r""" decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. diff --git a/src/transformers/models/chinese_clip/modeling_chinese_clip.py b/src/transformers/models/chinese_clip/modeling_chinese_clip.py index 000cf57c3393..288cded83128 100644 --- a/src/transformers/models/chinese_clip/modeling_chinese_clip.py +++ b/src/transformers/models/chinese_clip/modeling_chinese_clip.py @@ -1011,7 +1011,7 @@ def get_text_features( token_type_ids: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/clap/modeling_clap.py b/src/transformers/models/clap/modeling_clap.py index 6d45d6e32fe5..cdcb72b293ce 100644 --- a/src/transformers/models/clap/modeling_clap.py +++ b/src/transformers/models/clap/modeling_clap.py @@ -1562,7 +1562,7 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index 9acc608d63e7..8ab75b0e64d0 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -790,7 +790,7 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/clipseg/modeling_clipseg.py b/src/transformers/models/clipseg/modeling_clipseg.py index a2649a6d9e30..643c180d4c6e 100644 --- a/src/transformers/models/clipseg/modeling_clipseg.py +++ b/src/transformers/models/clipseg/modeling_clipseg.py @@ -859,7 +859,7 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/clvp/modeling_clvp.py b/src/transformers/models/clvp/modeling_clvp.py index 4a0ff4b77720..de44210daadc 100644 --- a/src/transformers/models/clvp/modeling_clvp.py +++ b/src/transformers/models/clvp/modeling_clvp.py @@ -1495,7 +1495,7 @@ def get_text_features( text_encoder_inputs_embeds: Optional[torch.FloatTensor] = None, attention_mask: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, ClvpEncoderOutput]: + ) -> Union[tuple, ClvpEncoderOutput]: r""" This method can be used to extract text_embeds from a text. The text embeddings obtained by applying the projection layer to the pooled output of the CLVP text encoder model. diff --git a/src/transformers/models/flava/modeling_flava.py b/src/transformers/models/flava/modeling_flava.py index 54520ccfbc2d..3d4869fdbf24 100644 --- a/src/transformers/models/flava/modeling_flava.py +++ b/src/transformers/models/flava/modeling_flava.py @@ -1006,7 +1006,7 @@ def get_text_features( token_type_ids: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" input_ids (`torch.LongTensor` of shape `(batch_size, text_seq_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See diff --git a/src/transformers/models/groupvit/modeling_groupvit.py b/src/transformers/models/groupvit/modeling_groupvit.py index fb47e5ed7f30..89515481a070 100644 --- a/src/transformers/models/groupvit/modeling_groupvit.py +++ b/src/transformers/models/groupvit/modeling_groupvit.py @@ -1237,7 +1237,7 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/metaclip_2/modeling_metaclip_2.py b/src/transformers/models/metaclip_2/modeling_metaclip_2.py index f2b4fa73e012..84b9c646cbbb 100644 --- a/src/transformers/models/metaclip_2/modeling_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modeling_metaclip_2.py @@ -799,7 +799,7 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/metaclip_2/modular_metaclip_2.py b/src/transformers/models/metaclip_2/modular_metaclip_2.py index fb1de6ca9749..8db42d94b701 100644 --- a/src/transformers/models/metaclip_2/modular_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modular_metaclip_2.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, Union import torch from torch import nn @@ -565,7 +565,7 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/owlv2/modeling_owlv2.py b/src/transformers/models/owlv2/modeling_owlv2.py index 7b892a5f8b00..0ac70bb97f2b 100644 --- a/src/transformers/models/owlv2/modeling_owlv2.py +++ b/src/transformers/models/owlv2/modeling_owlv2.py @@ -982,7 +982,7 @@ def get_text_features( input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See diff --git a/src/transformers/models/owlvit/modeling_owlvit.py b/src/transformers/models/owlvit/modeling_owlvit.py index a8030394f01c..5aef274063f8 100644 --- a/src/transformers/models/owlvit/modeling_owlvit.py +++ b/src/transformers/models/owlvit/modeling_owlvit.py @@ -963,7 +963,7 @@ def get_text_features( input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See diff --git a/src/transformers/models/siglip/modeling_siglip.py b/src/transformers/models/siglip/modeling_siglip.py index b46e347e9d91..7f0eceb02bf4 100644 --- a/src/transformers/models/siglip/modeling_siglip.py +++ b/src/transformers/models/siglip/modeling_siglip.py @@ -797,7 +797,7 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/siglip2/modeling_siglip2.py b/src/transformers/models/siglip2/modeling_siglip2.py index 8a1b746958d4..6f250de981b6 100644 --- a/src/transformers/models/siglip2/modeling_siglip2.py +++ b/src/transformers/models/siglip2/modeling_siglip2.py @@ -854,7 +854,7 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index a9105ac20bc7..f142788fa4ee 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -114,7 +114,7 @@ def get_text_features( position_ids: Optional[torch.Tensor] = None, token_type_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by diff --git a/src/transformers/models/x_clip/modeling_x_clip.py b/src/transformers/models/x_clip/modeling_x_clip.py index 7373f03b121d..e49192e5f76f 100644 --- a/src/transformers/models/x_clip/modeling_x_clip.py +++ b/src/transformers/models/x_clip/modeling_x_clip.py @@ -1211,7 +1211,7 @@ def get_text_features( attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by From b06a2d2e3f4ff1613b201c277c56c77c9febb26a Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 17:15:44 +0100 Subject: [PATCH 053/131] Add _can_record_inputs to qwen2_5_omni and qwen2_5_vl --- .../qwen2_5_omni/modeling_qwen2_5_omni.py | 6 +- .../qwen2_5_omni/modular_qwen2_5_omni.py | 6 +- .../models/qwen2_5_vl/modeling_qwen2_5_vl.py | 282 +++++++++--------- .../models/qwen2_5_vl/modular_qwen2_5_vl.py | 9 + 4 files changed, 162 insertions(+), 141 deletions(-) diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index 4e1dd6a49886..2c323403826b 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -1142,6 +1142,10 @@ class Qwen2_5OmniVisionEncoder(Qwen2_5OmniPreTrainedModel): config: Qwen2_5OmniVisionEncoderConfig _no_split_modules = ["Qwen2_5OmniVisionBlock"] _input_embed_layer = "patch_embed" + _can_record_inputs = { + "hidden_states": Qwen2_5OmniVisionBlock, + "attentions": Qwen2_5OmniVisionAttention, + } input_modalities = ("image", "video") def __init__(self, config: Qwen2_5OmniVisionEncoderConfig, *inputs, **kwargs) -> None: @@ -1815,7 +1819,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index 9f3ef5b121d4..cf50a6c6eb8f 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -1945,6 +1945,10 @@ class Qwen2_5OmniVisionEncoder(Qwen2_5_VisionTransformerPretrainedModel): input_modalities = ("image", "video") _no_split_modules = ["Qwen2_5OmniVisionBlock"] _input_embed_layer = "patch_embed" + _can_record_inputs = { + "hidden_states": Qwen2_5OmniVisionBlock, + "attentions": Qwen2_5OmniVisionAttention, + } def __init__(self, config: Qwen2_5OmniVisionEncoderConfig, *inputs, **kwargs) -> None: super().__init__(config, *inputs, **kwargs) @@ -2120,7 +2124,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index ef421c5cbb1b..c9544d24d6aa 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -314,10 +314,151 @@ def _init_weights(self, module): init.copy_(module.inv_freq, inv_freq) +def apply_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section, unsqueeze_dim=1): + """Applies Rotary Position Embedding with Multimodal Sections to the query and key tensors (https://qwenlm.github.io/blog/qwen2-vl/). + + Explanation: + Multimodal 3D rotary position embedding is an extension to 1D rotary position embedding. The input embedding + sequence contains vision (images / videos) embedding and text embedding or just contains text embedding. For + vision embedding part, we apply rotary position embedding on temporal, height and width dimension separately. + Here we split the channel dimension to 3 chunks for the temporal, height and width rotary position embedding. + For text embedding part, we just apply 1D rotary position embedding. The three rotary position index (temporal, + height and width) of text embedding is always the same, so the text embedding rotary position embedding has no + difference with modern LLMs. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`): + The position indices of the tokens corresponding to the query and key tensors. For example, this can be + used to pass offsetted position ids when working with a KV-cache. + mrope_section(`List(int)`): + Multimodal rope section is for channel dimension of temporal, height and width in rope calculation. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + mrope_section = mrope_section * 2 + cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section, dim=-1))], dim=-1).unsqueeze( + unsqueeze_dim + ) + sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section, dim=-1))], dim=-1).unsqueeze( + unsqueeze_dim + ) + + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +class Qwen2_5_VLAttention(nn.Module): + """ + Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer + and "Generating Long Sequences with Sparse Transformers". + """ + + def __init__(self, config: Qwen2_5_VLTextConfig, layer_idx: Optional[int] = None): + super().__init__() + self.config = config + self.layer_idx = layer_idx + if layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will " + "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.is_causal = True + self.attention_dropout = config.attention_dropout + self.rope_parameters = config.rope_parameters + self.scaling = self.head_dim**-0.5 + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + self.layer_type = config.layer_types[layer_idx] if hasattr(config, "layer_types") else None + self.sliding_window = config.sliding_window if self.layer_type == "sliding_attention" else None + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + + cos, sin = position_embeddings + query_states, key_states = apply_multimodal_rotary_pos_emb( + query_states, key_states, cos, sin, self.config.rope_parameters["mrope_section"] + ) + + if past_key_values is not None: + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + sliding_window=self.sliding_window, + position_ids=position_ids, # pass positions for FA2 + **kwargs, + ) + + attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + class Qwen2_5_VisionTransformerPretrainedModel(Qwen2_5_VLPreTrainedModel): config: Qwen2_5_VLVisionConfig _no_split_modules = ["Qwen2_5_VLVisionBlock"] _input_embed_layer = "patch_embed" + _can_record_inputs = { + "hidden_states": Qwen2_5_VLVisionBlock, + "attentions": Qwen2_5_VLAttention, + } def __init__(self, config, *inputs, **kwargs) -> None: super().__init__(config, *inputs, **kwargs) @@ -590,143 +731,6 @@ def forward(self, x): return down_proj -def apply_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section, unsqueeze_dim=1): - """Applies Rotary Position Embedding with Multimodal Sections to the query and key tensors (https://qwenlm.github.io/blog/qwen2-vl/). - - Explanation: - Multimodal 3D rotary position embedding is an extension to 1D rotary position embedding. The input embedding - sequence contains vision (images / videos) embedding and text embedding or just contains text embedding. For - vision embedding part, we apply rotary position embedding on temporal, height and width dimension separately. - Here we split the channel dimension to 3 chunks for the temporal, height and width rotary position embedding. - For text embedding part, we just apply 1D rotary position embedding. The three rotary position index (temporal, - height and width) of text embedding is always the same, so the text embedding rotary position embedding has no - difference with modern LLMs. - - Args: - q (`torch.Tensor`): The query tensor. - k (`torch.Tensor`): The key tensor. - cos (`torch.Tensor`): The cosine part of the rotary embedding. - sin (`torch.Tensor`): The sine part of the rotary embedding. - position_ids (`torch.Tensor`): - The position indices of the tokens corresponding to the query and key tensors. For example, this can be - used to pass offsetted position ids when working with a KV-cache. - mrope_section(`List(int)`): - Multimodal rope section is for channel dimension of temporal, height and width in rope calculation. - unsqueeze_dim (`int`, *optional*, defaults to 1): - The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and - sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note - that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and - k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes - cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have - the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. - Returns: - `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. - """ - mrope_section = mrope_section * 2 - cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section, dim=-1))], dim=-1).unsqueeze( - unsqueeze_dim - ) - sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section, dim=-1))], dim=-1).unsqueeze( - unsqueeze_dim - ) - - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - return q_embed, k_embed - - -class Qwen2_5_VLAttention(nn.Module): - """ - Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer - and "Generating Long Sequences with Sparse Transformers". - """ - - def __init__(self, config: Qwen2_5_VLTextConfig, layer_idx: Optional[int] = None): - super().__init__() - self.config = config - self.layer_idx = layer_idx - if layer_idx is None: - logger.warning_once( - f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will " - "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` " - "when creating this class." - ) - - self.hidden_size = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.hidden_size // self.num_heads - self.num_key_value_heads = config.num_key_value_heads - self.num_key_value_groups = self.num_heads // self.num_key_value_heads - self.is_causal = True - self.attention_dropout = config.attention_dropout - self.rope_parameters = config.rope_parameters - self.scaling = self.head_dim**-0.5 - - if (self.head_dim * self.num_heads) != self.hidden_size: - raise ValueError( - f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" - f" and `num_heads`: {self.num_heads})." - ) - self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True) - self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) - self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) - self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) - self.layer_type = config.layer_types[layer_idx] if hasattr(config, "layer_types") else None - self.sliding_window = config.sliding_window if self.layer_type == "sliding_attention" else None - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[Cache] = None, - output_attentions: bool = False, - use_cache: bool = False, - cache_position: Optional[torch.LongTensor] = None, - position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, - **kwargs: Unpack[FlashAttentionKwargs], - ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - - cos, sin = position_embeddings - query_states, key_states = apply_multimodal_rotary_pos_emb( - query_states, key_states, cos, sin, self.config.rope_parameters["mrope_section"] - ) - - if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) - - attention_interface: Callable = eager_attention_forward - if self.config._attn_implementation != "eager": - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - - attn_output, attn_weights = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask, - dropout=0.0 if not self.training else self.attention_dropout, - scaling=self.scaling, - sliding_window=self.sliding_window, - position_ids=position_ids, # pass positions for FA2 - **kwargs, - ) - - attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() - attn_output = self.o_proj(attn_output) - return attn_output, attn_weights - - class Qwen2_5_VLDecoderLayer(GradientCheckpointingLayer): def __init__(self, config: Qwen2_5_VLTextConfig, layer_idx: int): super().__init__() @@ -1206,7 +1210,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1430,7 +1434,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py index b8843fc41754..4cbe1e34fe2d 100644 --- a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py @@ -50,6 +50,7 @@ Qwen2VLModel, Qwen2VLModelOutputWithPast, Qwen2VLPreTrainedModel, + Qwen2VLAttention, TransformersKwargs, VisionAttention, VisionRotaryEmbedding, @@ -178,10 +179,18 @@ def _init_weights(self, module): init.copy_(module.inv_freq, inv_freq) + +class Qwen2_5_VLAttention(Qwen2VLAttention): + pass + class Qwen2_5_VisionTransformerPretrainedModel(Qwen2_5_VLPreTrainedModel): config: Qwen2_5_VLVisionConfig _no_split_modules = ["Qwen2_5_VLVisionBlock"] _input_embed_layer = "patch_embed" + _can_record_inputs = { + "hidden_states": Qwen2_5_VLVisionBlock, + "attentions": Qwen2_5_VLAttention, + } def __init__(self, config, *inputs, **kwargs) -> None: super().__init__(config, *inputs, **kwargs) From 4a573afc5da78d926393fb0c9449ff57ff264c0b Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 17:19:49 +0100 Subject: [PATCH 054/131] Update get_image_features and get_video_features on ernie4_5_vl_moe Can we even use BaseModelOutputWithPooling for these? It's a MoE model --- .../modeling_ernie4_5_vl_moe.py | 46 +++++++++++-------- .../modular_ernie4_5_vl_moe.py | 46 +++++++++++-------- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index dcd299ef6149..a23d38ab12eb 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -35,7 +35,7 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import MoeCausalLMOutputWithPast, MoeModelOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPooling, MoeCausalLMOutputWithPast, MoeModelOutputWithPast from ...modeling_rope_utils import dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -926,13 +926,9 @@ def rot_pos_emb(self, grid_thw): return rotary_pos_emb @check_model_inputs(tie_last_hidden_states=False) - @auto_docstring def forward( - self, - hidden_states: torch.Tensor, - grid_thw: torch.Tensor, - **kwargs, - ) -> torch.Tensor: + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" grid_thw (`torch.LongTensor` of shape `(num_images, 3)`): The temporal, height and width dimensions of feature shape for each image. Each row contains [t, h, w] values. @@ -960,7 +956,7 @@ def forward( **kwargs, ) hidden_states = self.ln(hidden_states) - return hidden_states + return BaseModelOutputWithPooling(last_hidden_state=hidden_states) class Ernie4_5_VL_MoeVisionMLP(nn.Module): @@ -1264,8 +1260,11 @@ def get_rope_index( @can_return_tuple def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None - ): + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1275,18 +1274,24 @@ def get_video_features( video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. """ - video_embeds = self.vision_tower(pixel_values_videos, video_grid_thw) - video_embeds = self.resampler_model(video_embeds, video_grid_thw) + video_outputs = self.vision_tower(pixel_values_videos, video_grid_thw, **kwargs) + video_embeds = self.resampler_model(video_outputs.last_hidden_state, video_grid_thw) split_sizes = ( video_grid_thw.prod(-1) // self.vision_tower.spatial_merge_size**2 // self.resampler_model.temporal_merge_size ).tolist() video_embeds = torch.split(video_embeds, split_sizes) - return video_embeds + video_outputs.pooler_output = video_embeds + return video_outputs @can_return_tuple - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1296,11 +1301,12 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. """ - image_embeds = self.vision_tower(pixel_values, image_grid_thw) - image_embeds = self.resampler_model(image_embeds, image_grid_thw) + image_outputs = self.vision_tower(pixel_values, image_grid_thw, **kwargs) + image_embeds = self.resampler_model(image_outputs.last_hidden_state, image_grid_thw) split_sizes = (image_grid_thw.prod(-1) // self.vision_tower.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) - return image_embeds + image_outputs.pooler_output = image_embeds + return image_outputs def get_placeholder_mask( self, @@ -1378,7 +1384,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = self.get_image_features(pixel_values, image_grid_thw, return_dict=True).pooler_output image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -1386,7 +1392,7 @@ def forward( inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw, return_dict=True).pooler_output video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds @@ -1613,7 +1619,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @auto_docstring diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index 863719409d5a..78d1c0564f13 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -49,7 +49,7 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import MoeCausalLMOutputWithPast, MoeModelOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPooling, MoeCausalLMOutputWithPast, MoeModelOutputWithPast from ...modeling_rope_utils import dynamic_rope_update from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack @@ -869,12 +869,10 @@ def get_dtype(self): def get_device(self): raise AttributeError("Ernie 4.5 VL Moe does not need this!") + @check_model_inputs(tie_last_hidden_states=False) def forward( - self, - hidden_states: torch.Tensor, - grid_thw: torch.Tensor, - **kwargs, - ) -> torch.Tensor: + self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: hidden_states = self.patch_embed(hidden_states) rotary_pos_emb = self.rot_pos_emb(grid_thw) emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) @@ -898,7 +896,7 @@ def forward( **kwargs, ) hidden_states = self.ln(hidden_states) - return hidden_states + return BaseModelOutputWithPooling(last_hidden_state=hidden_states) class Ernie4_5_VL_MoeVisionMLP(nn.Module): @@ -1251,9 +1249,13 @@ def get_rope_index( return position_ids, mrope_position_deltas + @can_return_tuple def get_video_features( - self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None - ): + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1263,17 +1265,24 @@ def get_video_features( video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. """ - video_embeds = self.vision_tower(pixel_values_videos, video_grid_thw) - video_embeds = self.resampler_model(video_embeds, video_grid_thw) + video_outputs = self.vision_tower(pixel_values_videos, video_grid_thw, **kwargs) + video_embeds = self.resampler_model(video_outputs.last_hidden_state, video_grid_thw) split_sizes = ( video_grid_thw.prod(-1) // self.vision_tower.spatial_merge_size**2 // self.resampler_model.temporal_merge_size ).tolist() video_embeds = torch.split(video_embeds, split_sizes) - return video_embeds + video_outputs.pooler_output = video_embeds + return video_outputs - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None): + @can_return_tuple + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1283,11 +1292,12 @@ def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Op image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. """ - image_embeds = self.vision_tower(pixel_values, image_grid_thw) - image_embeds = self.resampler_model(image_embeds, image_grid_thw) + image_outputs = self.vision_tower(pixel_values, image_grid_thw, **kwargs) + image_embeds = self.resampler_model(image_outputs.last_hidden_state, image_grid_thw) split_sizes = (image_grid_thw.prod(-1) // self.vision_tower.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) - return image_embeds + image_outputs.pooler_output = image_embeds + return image_outputs @auto_docstring @can_return_tuple @@ -1325,7 +1335,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = self.get_image_features(pixel_values, image_grid_thw, return_dict=True).pooler_output image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _ = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds @@ -1333,7 +1343,7 @@ def forward( inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: - video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw, return_dict=True).pooler_output video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds From 2c677f9dba67cae73b5b8a04c85c49025b5a4128 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 17:20:37 +0100 Subject: [PATCH 055/131] Update get_image_features type hints to Union[tuple, BaseModelOutputWithPooling] With a handful of exceptions --- src/transformers/models/aimv2/modeling_aimv2.py | 2 +- src/transformers/models/align/modeling_align.py | 2 +- src/transformers/models/altclip/modeling_altclip.py | 2 +- src/transformers/models/aria/modeling_aria.py | 4 ++-- src/transformers/models/aria/modular_aria.py | 4 ++-- .../models/aya_vision/modeling_aya_vision.py | 6 +++--- .../models/aya_vision/modular_aya_vision.py | 3 ++- src/transformers/models/blip/modeling_blip.py | 2 +- src/transformers/models/blip_2/modeling_blip_2.py | 4 ++-- .../models/chameleon/modeling_chameleon.py | 6 ++++-- .../models/chinese_clip/modeling_chinese_clip.py | 2 +- src/transformers/models/clip/modeling_clip.py | 2 +- src/transformers/models/clipseg/modeling_clipseg.py | 2 +- .../models/cohere2_vision/modeling_cohere2_vision.py | 10 +++++++--- .../models/cohere2_vision/modular_cohere2_vision.py | 9 +++++++-- .../models/deepseek_vl/modeling_deepseek_vl.py | 6 ++++-- .../deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py | 2 +- .../deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py | 2 +- src/transformers/models/edgetam/modeling_edgetam.py | 7 +------ .../models/edgetam_video/modeling_edgetam_video.py | 7 +------ src/transformers/models/emu3/modeling_emu3.py | 2 +- src/transformers/models/emu3/modular_emu3.py | 2 +- src/transformers/models/fast_vlm/modeling_fast_vlm.py | 6 +++--- src/transformers/models/fast_vlm/modular_fast_vlm.py | 3 ++- src/transformers/models/flava/modeling_flava.py | 2 +- .../models/florence2/modeling_florence2.py | 8 ++++++-- src/transformers/models/florence2/modular_florence2.py | 8 ++++++-- src/transformers/models/fuyu/modeling_fuyu.py | 4 +++- src/transformers/models/gemma3/modeling_gemma3.py | 2 +- src/transformers/models/gemma3/modular_gemma3.py | 2 +- src/transformers/models/gemma3n/modeling_gemma3n.py | 2 +- src/transformers/models/gemma3n/modular_gemma3n.py | 2 +- src/transformers/models/glm46v/modeling_glm46v.py | 6 +++--- src/transformers/models/glm4v/modeling_glm4v.py | 4 ++-- .../models/glm4v_moe/modeling_glm4v_moe.py | 4 ++-- src/transformers/models/got_ocr2/modeling_got_ocr2.py | 4 ++-- src/transformers/models/got_ocr2/modular_got_ocr2.py | 2 +- src/transformers/models/groupvit/modeling_groupvit.py | 2 +- src/transformers/models/idefics2/modeling_idefics2.py | 4 ++-- src/transformers/models/idefics3/modeling_idefics3.py | 4 ++-- .../models/instructblip/modeling_instructblip.py | 2 +- src/transformers/models/internvl/modeling_internvl.py | 4 ++-- src/transformers/models/internvl/modular_internvl.py | 2 +- src/transformers/models/janus/modeling_janus.py | 4 +++- src/transformers/models/janus/modular_janus.py | 4 +++- src/transformers/models/kosmos2/modeling_kosmos2.py | 2 +- src/transformers/models/lfm2_vl/modeling_lfm2_vl.py | 6 +++--- src/transformers/models/lfm2_vl/modular_lfm2_vl.py | 5 +++-- src/transformers/models/llama4/modeling_llama4.py | 10 ++++++++-- src/transformers/models/llava/modeling_llava.py | 6 +++--- .../models/llava_next/modeling_llava_next.py | 8 +++----- .../llava_next_video/modeling_llava_next_video.py | 6 +++--- .../llava_next_video/modular_llava_next_video.py | 3 ++- .../models/llava_onevision/modeling_llava_onevision.py | 6 +++--- .../models/llava_onevision/modular_llava_onevision.py | 3 ++- .../models/metaclip_2/modeling_metaclip_2.py | 2 +- .../models/metaclip_2/modular_metaclip_2.py | 2 +- src/transformers/models/mistral3/modeling_mistral3.py | 6 +++--- src/transformers/models/mistral3/modular_mistral3.py | 5 +++-- src/transformers/models/ovis2/modeling_ovis2.py | 4 ++-- src/transformers/models/ovis2/modular_ovis2.py | 4 ++-- src/transformers/models/owlv2/modeling_owlv2.py | 2 +- src/transformers/models/owlvit/modeling_owlvit.py | 2 +- .../models/paddleocr_vl/modeling_paddleocr_vl.py | 4 ++-- .../models/paddleocr_vl/modular_paddleocr_vl.py | 2 +- .../models/paligemma/modeling_paligemma.py | 6 ++++-- .../models/perception_lm/modeling_perception_lm.py | 4 ++-- .../models/perception_lm/modular_perception_lm.py | 3 ++- .../models/qwen2_5_vl/modular_qwen2_5_vl.py | 4 ++-- src/transformers/models/qwen2_vl/modeling_qwen2_vl.py | 4 ++-- .../models/qwen3_omni_moe/modeling_qwen3_omni_moe.py | 2 +- src/transformers/models/qwen3_vl/modeling_qwen3_vl.py | 4 ++-- src/transformers/models/qwen3_vl/modular_qwen3_vl.py | 5 ++++- .../models/qwen3_vl_moe/modeling_qwen3_vl_moe.py | 4 ++-- src/transformers/models/sam2/modeling_sam2.py | 7 +------ src/transformers/models/sam2/modular_sam2.py | 7 +------ .../models/sam2_video/modeling_sam2_video.py | 7 +------ .../models/sam3_tracker/modeling_sam3_tracker.py | 7 +------ .../sam3_tracker_video/modeling_sam3_tracker_video.py | 7 +------ .../sam3_tracker_video/modular_sam3_tracker_video.py | 9 ++------- src/transformers/models/siglip/modeling_siglip.py | 2 +- src/transformers/models/siglip2/modeling_siglip2.py | 2 +- src/transformers/models/siglip2/modular_siglip2.py | 4 ++-- src/transformers/models/smolvlm/modeling_smolvlm.py | 4 ++-- src/transformers/models/smolvlm/modular_smolvlm.py | 3 ++- src/transformers/models/t5gemma2/modeling_t5gemma2.py | 9 +++++++-- src/transformers/models/t5gemma2/modular_t5gemma2.py | 9 +++++++-- .../models/video_llama_3/modeling_video_llama_3.py | 6 +++--- .../models/video_llama_3/modular_video_llama_3.py | 4 ++-- .../models/video_llava/modeling_video_llava.py | 6 +++--- src/transformers/models/vipllava/modeling_vipllava.py | 6 +++--- src/transformers/models/vipllava/modular_vipllava.py | 5 +++-- .../modeling_vision_text_dual_encoder.py | 2 +- 93 files changed, 209 insertions(+), 194 deletions(-) diff --git a/src/transformers/models/aimv2/modeling_aimv2.py b/src/transformers/models/aimv2/modeling_aimv2.py index 389d7b59f287..8098a9ebf688 100644 --- a/src/transformers/models/aimv2/modeling_aimv2.py +++ b/src/transformers/models/aimv2/modeling_aimv2.py @@ -650,7 +650,7 @@ def get_image_features( pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/align/modeling_align.py b/src/transformers/models/align/modeling_align.py index a4dc2d10b53b..bf2723fa25ba 100644 --- a/src/transformers/models/align/modeling_align.py +++ b/src/transformers/models/align/modeling_align.py @@ -1140,7 +1140,7 @@ def get_text_features( @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/altclip/modeling_altclip.py b/src/transformers/models/altclip/modeling_altclip.py index 510ba492efff..c03e6a03c854 100755 --- a/src/transformers/models/altclip/modeling_altclip.py +++ b/src/transformers/models/altclip/modeling_altclip.py @@ -1202,7 +1202,7 @@ def get_image_features( pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index 2646b38dbea4..753f1682c0b3 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -926,7 +926,7 @@ def get_image_features( pixel_mask: Optional[torch.FloatTensor] = None, vision_feature_layer: int = -1, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -1082,7 +1082,7 @@ def get_image_features( pixel_mask: Optional[torch.FloatTensor] = None, vision_feature_layer: int = -1, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, pixel_mask=pixel_mask, diff --git a/src/transformers/models/aria/modular_aria.py b/src/transformers/models/aria/modular_aria.py index cad8a0368541..68d19dffaafa 100644 --- a/src/transformers/models/aria/modular_aria.py +++ b/src/transformers/models/aria/modular_aria.py @@ -1282,7 +1282,7 @@ def get_image_features( pixel_mask: Optional[torch.FloatTensor] = None, vision_feature_layer: int = -1, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -1377,7 +1377,7 @@ def get_image_features( pixel_mask: Optional[torch.FloatTensor] = None, vision_feature_layer: int = -1, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, pixel_mask=pixel_mask, diff --git a/src/transformers/models/aya_vision/modeling_aya_vision.py b/src/transformers/models/aya_vision/modeling_aya_vision.py index 5b3a8ba45132..6fe425385c2e 100644 --- a/src/transformers/models/aya_vision/modeling_aya_vision.py +++ b/src/transformers/models/aya_vision/modeling_aya_vision.py @@ -28,7 +28,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -188,7 +188,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -362,7 +362,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, diff --git a/src/transformers/models/aya_vision/modular_aya_vision.py b/src/transformers/models/aya_vision/modular_aya_vision.py index bd6d5736ecc9..c907f6c5a127 100644 --- a/src/transformers/models/aya_vision/modular_aya_vision.py +++ b/src/transformers/models/aya_vision/modular_aya_vision.py @@ -29,6 +29,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from ...utils.generic import check_model_inputs @@ -113,7 +114,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index c6f18b239047..de68d80df17b 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -616,7 +616,7 @@ def get_image_features( pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index bdf75b3b0bcd..9ee0b3fe6fb5 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -1153,7 +1153,7 @@ def get_image_features( pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: vision_outputs (`torch.FloatTensor`): @@ -1628,7 +1628,7 @@ def get_image_features( pixel_values: torch.FloatTensor, interpolate_pos_encoding: Optional[bool] = False, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithVisionQformerOutputs]: """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 5764995fef2f..f4a57b2e12f0 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -884,7 +884,7 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor): @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] - ) -> BaseModelOutputWithPooling: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Tokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layer @@ -1053,7 +1053,9 @@ def __init__(self, config): def get_image_tokens(self, pixel_values): return self.model.get_image_tokens(pixel_values) - def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values, **kwargs) @can_return_tuple diff --git a/src/transformers/models/chinese_clip/modeling_chinese_clip.py b/src/transformers/models/chinese_clip/modeling_chinese_clip.py index 288cded83128..857989c6f734 100644 --- a/src/transformers/models/chinese_clip/modeling_chinese_clip.py +++ b/src/transformers/models/chinese_clip/modeling_chinese_clip.py @@ -1050,7 +1050,7 @@ def get_image_features( pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index 8ab75b0e64d0..70f1a2e4c106 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -828,7 +828,7 @@ def get_image_features( pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/clipseg/modeling_clipseg.py b/src/transformers/models/clipseg/modeling_clipseg.py index 643c180d4c6e..403d68e35c37 100644 --- a/src/transformers/models/clipseg/modeling_clipseg.py +++ b/src/transformers/models/clipseg/modeling_clipseg.py @@ -896,7 +896,7 @@ def get_image_features( pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = True, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py index b4a8bb94a748..db18836cc47b 100644 --- a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py @@ -27,7 +27,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -168,7 +168,9 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple - def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -280,7 +282,9 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head - def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, **kwargs) @check_model_inputs diff --git a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py index 7ae21954efea..2956527cf174 100644 --- a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py @@ -34,6 +34,7 @@ from ...image_processing_utils import BatchFeature from ...image_utils import ImageInput from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import ImagesKwargs, Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from ...utils.generic import check_model_inputs @@ -98,7 +99,9 @@ class Cohere2VisionModel(AyaVisionModel): _checkpoint_conversion_mapping = {} @can_return_tuple - def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -165,7 +168,9 @@ def forward( class Cohere2VisionForConditionalGeneration(AyaVisionForConditionalGeneration): _checkpoint_conversion_mapping = {} - def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, **kwargs) @check_model_inputs diff --git a/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py b/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py index cbc25c86c599..157320daba0e 100644 --- a/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py +++ b/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py @@ -26,7 +26,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import ModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -155,7 +155,9 @@ def set_input_embeddings(self, value): @can_return_tuple @auto_docstring - def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: vision_outputs = self.vision_model(pixel_values, **kwargs) vision_outputs.pooler_output = self.aligner(vision_outputs.last_hidden_state) diff --git a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py index e0400551dbd4..a6847413f282 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py @@ -299,7 +299,7 @@ def get_image_features( pixel_values: torch.FloatTensor, high_res_pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithHighResVisionEncodings]: low_res_outputs = self.get_low_res_image_features(pixel_values, **kwargs) high_res_outputs = self.get_high_res_image_features(high_res_pixel_values, **kwargs) image_features = self.aligner(low_res_outputs.last_hidden_state, high_res_outputs.last_hidden_state) diff --git a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py index 80b385e1e693..07e219b6c701 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py @@ -308,7 +308,7 @@ def get_image_features( pixel_values: torch.FloatTensor, high_res_pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithHighResVisionEncodings]: low_res_outputs = self.get_low_res_image_features(pixel_values, **kwargs) high_res_outputs = self.get_high_res_image_features(high_res_pixel_values, **kwargs) image_features = self.aligner(low_res_outputs.last_hidden_state, high_res_outputs.last_hidden_state) diff --git a/src/transformers/models/edgetam/modeling_edgetam.py b/src/transformers/models/edgetam/modeling_edgetam.py index 1bd378738445..dfd3546309c0 100644 --- a/src/transformers/models/edgetam/modeling_edgetam.py +++ b/src/transformers/models/edgetam/modeling_edgetam.py @@ -1198,12 +1198,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> tuple[ - list[torch.Tensor], - list[torch.Tensor], - Optional[tuple[torch.FloatTensor, ...]], - Optional[tuple[torch.FloatTensor, ...]], - ]: + ) -> Union[tuple, EdgeTamVisionEncoderOutput]: r""" Extract and preprocess image features using the vision encoder. diff --git a/src/transformers/models/edgetam_video/modeling_edgetam_video.py b/src/transformers/models/edgetam_video/modeling_edgetam_video.py index 4dea50a111c3..3a0f6eb22fd1 100644 --- a/src/transformers/models/edgetam_video/modeling_edgetam_video.py +++ b/src/transformers/models/edgetam_video/modeling_edgetam_video.py @@ -2226,12 +2226,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> tuple[ - list[torch.Tensor], - list[torch.Tensor], - Optional[tuple[torch.FloatTensor, ...]], - Optional[tuple[torch.FloatTensor, ...]], - ]: + ) -> Union[tuple, EdgeTamVideoVisionEncoderOutput]: r""" Extract and preprocess image features using the vision encoder. diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index 0fa2cab848a8..e26dcc811153 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -1399,7 +1399,7 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.L @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs] - ): + ) -> Union[tuple, Emu3VQVAEModelOutput]: """ Tokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layer diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index 5b6b6110267f..ca27739b2fc3 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -952,7 +952,7 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.L @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs] - ): + ) -> Union[tuple, Emu3VQVAEModelOutput]: """ Tokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layer diff --git a/src/transformers/models/fast_vlm/modeling_fast_vlm.py b/src/transformers/models/fast_vlm/modeling_fast_vlm.py index 68a0b2487a43..7ac7e17023cb 100644 --- a/src/transformers/models/fast_vlm/modeling_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modeling_fast_vlm.py @@ -27,7 +27,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -121,7 +121,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -308,7 +308,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, diff --git a/src/transformers/models/fast_vlm/modular_fast_vlm.py b/src/transformers/models/fast_vlm/modular_fast_vlm.py index 88d9606d42a6..edf2d7213cfd 100644 --- a/src/transformers/models/fast_vlm/modular_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modular_fast_vlm.py @@ -20,6 +20,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...configuration_utils import PreTrainedConfig +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ..auto import CONFIG_MAPPING @@ -177,7 +178,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. diff --git a/src/transformers/models/flava/modeling_flava.py b/src/transformers/models/flava/modeling_flava.py index 3d4869fdbf24..e3e7f1d81e8b 100644 --- a/src/transformers/models/flava/modeling_flava.py +++ b/src/transformers/models/flava/modeling_flava.py @@ -1060,7 +1060,7 @@ def get_image_features( interpolate_pos_encoding: Optional[bool] = None, attention_mask: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, image_num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). diff --git a/src/transformers/models/florence2/modeling_florence2.py b/src/transformers/models/florence2/modeling_florence2.py index 6702e288cd23..33bd0f8ef70c 100644 --- a/src/transformers/models/florence2/modeling_florence2.py +++ b/src/transformers/models/florence2/modeling_florence2.py @@ -672,7 +672,9 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple - def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -840,7 +842,9 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head - def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, **kwargs) @can_return_tuple diff --git a/src/transformers/models/florence2/modular_florence2.py b/src/transformers/models/florence2/modular_florence2.py index 5ff6a99005a0..2066d6188b01 100644 --- a/src/transformers/models/florence2/modular_florence2.py +++ b/src/transformers/models/florence2/modular_florence2.py @@ -1539,7 +1539,9 @@ def get_encoder(self, modality=None): return super().get_encoder(modality=modality) @can_return_tuple - def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -1646,7 +1648,9 @@ class Florence2ForConditionalGeneration(LlavaForConditionalGeneration): "lm_head.weight": "model.language_model.shared.weight", } - def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, **kwargs) @can_return_tuple diff --git a/src/transformers/models/fuyu/modeling_fuyu.py b/src/transformers/models/fuyu/modeling_fuyu.py index b49d4212ae2f..6ca33609dceb 100644 --- a/src/transformers/models/fuyu/modeling_fuyu.py +++ b/src/transformers/models/fuyu/modeling_fuyu.py @@ -118,7 +118,9 @@ def gather_continuous_embeddings( return output_embeddings @can_return_tuple - def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/gemma3/modeling_gemma3.py b/src/transformers/models/gemma3/modeling_gemma3.py index c650897d6d0d..04dd5360df28 100644 --- a/src/transformers/models/gemma3/modeling_gemma3.py +++ b/src/transformers/models/gemma3/modeling_gemma3.py @@ -847,7 +847,7 @@ def set_input_embeddings(self, value): @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. diff --git a/src/transformers/models/gemma3/modular_gemma3.py b/src/transformers/models/gemma3/modular_gemma3.py index ad8f2ee38580..a83935b8afa7 100644 --- a/src/transformers/models/gemma3/modular_gemma3.py +++ b/src/transformers/models/gemma3/modular_gemma3.py @@ -802,7 +802,7 @@ def __init__(self, config: Gemma3Config): @can_return_tuple def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index 8c157ee0134a..e540925d2bdc 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -1979,7 +1979,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index eee26e11c581..666a98ef1d29 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -2242,7 +2242,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Projects the last hidden state from the vision model into language model space. diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 87b65162009c..b8260e6d5d35 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -28,7 +28,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import ModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling @@ -327,7 +327,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -557,7 +557,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index 291b9d7bdabe..ca3b6e6d0366 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1196,7 +1196,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1426,7 +1426,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 28d55d8d41e4..37543599c577 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1412,7 +1412,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1695,7 +1695,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @auto_docstring diff --git a/src/transformers/models/got_ocr2/modeling_got_ocr2.py b/src/transformers/models/got_ocr2/modeling_got_ocr2.py index cd869b99ca12..fc1941fd303e 100644 --- a/src/transformers/models/got_ocr2/modeling_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modeling_got_ocr2.py @@ -557,7 +557,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -693,7 +693,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, diff --git a/src/transformers/models/got_ocr2/modular_got_ocr2.py b/src/transformers/models/got_ocr2/modular_got_ocr2.py index 3eb27472cc15..eec3752db82e 100644 --- a/src/transformers/models/got_ocr2/modular_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modular_got_ocr2.py @@ -313,7 +313,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. diff --git a/src/transformers/models/groupvit/modeling_groupvit.py b/src/transformers/models/groupvit/modeling_groupvit.py index 89515481a070..f1a24b2f7714 100644 --- a/src/transformers/models/groupvit/modeling_groupvit.py +++ b/src/transformers/models/groupvit/modeling_groupvit.py @@ -1272,7 +1272,7 @@ def get_image_features( self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index c1b4fbbde8c1..d4a116040500 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -831,7 +831,7 @@ def get_image_features( pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1003,7 +1003,7 @@ def get_image_features( pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask, **kwargs ) diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index c7829caecc9d..8e2057538704 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -574,7 +574,7 @@ def get_image_features( pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -757,7 +757,7 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask) @can_return_tuple diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index b7c138559a73..6a5b2886835d 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -1195,7 +1195,7 @@ def get_image_features( qformer_attention_mask: Optional[torch.LongTensor] = None, interpolate_pos_encoding: Optional[bool] = False, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithVisionQformerOutputs]: """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/internvl/modeling_internvl.py b/src/transformers/models/internvl/modeling_internvl.py index 937acc7f2b5a..4e6bdc939fce 100644 --- a/src/transformers/models/internvl/modeling_internvl.py +++ b/src/transformers/models/internvl/modeling_internvl.py @@ -553,7 +553,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -790,7 +790,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, diff --git a/src/transformers/models/internvl/modular_internvl.py b/src/transformers/models/internvl/modular_internvl.py index e485d4688c1e..126115ffac33 100644 --- a/src/transformers/models/internvl/modular_internvl.py +++ b/src/transformers/models/internvl/modular_internvl.py @@ -493,7 +493,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. diff --git a/src/transformers/models/janus/modeling_janus.py b/src/transformers/models/janus/modeling_janus.py index da2b39ad7474..252132f67f13 100644 --- a/src/transformers/models/janus/modeling_janus.py +++ b/src/transformers/models/janus/modeling_janus.py @@ -1112,7 +1112,9 @@ def set_input_embeddings(self, value): @can_return_tuple @auto_docstring - def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: vision_outputs = self.vision_model(pixel_values, **kwargs) vision_outputs.pooler_output = self.aligner(vision_outputs.last_hidden_state) diff --git a/src/transformers/models/janus/modular_janus.py b/src/transformers/models/janus/modular_janus.py index e91ce93e0744..bb0c01343193 100644 --- a/src/transformers/models/janus/modular_janus.py +++ b/src/transformers/models/janus/modular_janus.py @@ -933,7 +933,9 @@ def set_input_embeddings(self, value): @can_return_tuple @auto_docstring - def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: vision_outputs = self.vision_model(pixel_values, **kwargs) vision_outputs.pooler_output = self.aligner(vision_outputs.last_hidden_state) diff --git a/src/transformers/models/kosmos2/modeling_kosmos2.py b/src/transformers/models/kosmos2/modeling_kosmos2.py index 28fc9f1ef3bc..324693adbbbc 100644 --- a/src/transformers/models/kosmos2/modeling_kosmos2.py +++ b/src/transformers/models/kosmos2/modeling_kosmos2.py @@ -1514,7 +1514,7 @@ def get_image_features( pixel_values: torch.FloatTensor, interpolate_pos_encoding: Optional[bool] = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithProjectionAttentions]: + ) -> Union[tuple, BaseModelOutputWithProjectionAttentions]: if "return_attentions" in kwargs: warnings.warn( "`return_attentions` is deprecated and will be removed in a future version. Please use `return_dict`" diff --git a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py index f7e38f48c3ad..bad009b14153 100755 --- a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py @@ -28,7 +28,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -170,7 +170,7 @@ def get_image_features( spatial_shapes: torch.Tensor, pixel_attention_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], - ) -> list[torch.Tensor]: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -332,7 +332,7 @@ def get_image_features( spatial_shapes: torch.Tensor, pixel_attention_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, spatial_shapes=spatial_shapes, diff --git a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py index dba722cecd6d..38975e03711f 100644 --- a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py @@ -21,6 +21,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from ..llava.modeling_llava import ( @@ -101,7 +102,7 @@ def get_image_features( spatial_shapes: torch.Tensor, pixel_attention_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], - ) -> list[torch.Tensor]: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -242,7 +243,7 @@ def get_image_features( spatial_shapes: torch.Tensor, pixel_attention_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, spatial_shapes=spatial_shapes, diff --git a/src/transformers/models/llama4/modeling_llama4.py b/src/transformers/models/llama4/modeling_llama4.py index 88a8f26648a4..861d9e06e4b4 100644 --- a/src/transformers/models/llama4/modeling_llama4.py +++ b/src/transformers/models/llama4/modeling_llama4.py @@ -32,7 +32,13 @@ from ...masking_utils import create_causal_mask, create_chunked_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, CausalLMOutputWithPast, ModelOutput +from ...modeling_outputs import ( + BaseModelOutput, + BaseModelOutputWithPast, + BaseModelOutputWithPooling, + CausalLMOutputWithPast, + ModelOutput, +) from ...modeling_rope_utils import ( ROPE_INIT_FUNCTIONS, dynamic_rope_update, @@ -1204,7 +1210,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_select_strategy: str, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply al projection. diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 01d08d5fef1b..8eaf5079fb86 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -23,7 +23,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -154,7 +154,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -338,7 +338,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index fec6fe262a82..d3c57f0eaea7 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -28,7 +28,7 @@ from ...generation import GenerationMixin from ...image_processing_utils import select_best_resolution from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -354,7 +354,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -370,8 +370,6 @@ def get_image_features( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. Returns: image_features (list[`torch.Tensor`]): List of image feature tensor, each contains all the visual feature of all patches @@ -580,7 +578,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index bdcf9a0f85de..9b666e90b5e3 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -33,7 +33,7 @@ from ...generation import GenerationMixin from ...image_processing_utils import select_best_resolution from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -406,7 +406,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -723,7 +723,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index 11594cbae8fe..95df59755d65 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -33,6 +33,7 @@ from ...cache_utils import Cache from ...configuration_utils import PreTrainedConfig from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import can_return_tuple, logging from ..auto import CONFIG_MAPPING, AutoConfig @@ -280,7 +281,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index f3d551454de0..a6af4983e503 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -33,7 +33,7 @@ from ...generation import GenerationMixin from ...image_processing_utils import select_best_resolution from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -365,7 +365,7 @@ def get_image_features( vision_aspect_ratio: Optional[str] = None, batch_num_images: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -710,7 +710,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index 3ed341d39260..e1bcaa1c1bf2 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -46,6 +46,7 @@ get_image_size, ) from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import ( TensorType, @@ -323,7 +324,7 @@ def get_image_features( vision_aspect_ratio: Optional[str] = None, batch_num_images: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. diff --git a/src/transformers/models/metaclip_2/modeling_metaclip_2.py b/src/transformers/models/metaclip_2/modeling_metaclip_2.py index 84b9c646cbbb..9d36547ba210 100644 --- a/src/transformers/models/metaclip_2/modeling_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modeling_metaclip_2.py @@ -834,7 +834,7 @@ def get_image_features( pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/metaclip_2/modular_metaclip_2.py b/src/transformers/models/metaclip_2/modular_metaclip_2.py index 8db42d94b701..1f35c7a3c563 100644 --- a/src/transformers/models/metaclip_2/modular_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modular_metaclip_2.py @@ -596,7 +596,7 @@ def get_image_features( pixel_values: Optional[torch.FloatTensor] = None, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index 7aa2a65037b6..85ea5f25e83b 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -29,7 +29,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...integrations import use_kernel_forward_from_hub -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -220,7 +220,7 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -391,7 +391,7 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, diff --git a/src/transformers/models/mistral3/modular_mistral3.py b/src/transformers/models/mistral3/modular_mistral3.py index f5b99a373898..bf688a40eb1f 100644 --- a/src/transformers/models/mistral3/modular_mistral3.py +++ b/src/transformers/models/mistral3/modular_mistral3.py @@ -20,6 +20,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import logging from ..llava.modeling_llava import ( @@ -124,7 +125,7 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -241,7 +242,7 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index a32b744b49bf..01d9537fe1d3 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -553,7 +553,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithVisualIndicatorFeatures]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -720,7 +720,7 @@ def get_output_embeddings(self) -> nn.Module: def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithVisualIndicatorFeatures]: return self.model.get_image_features(pixel_values=pixel_values, **kwargs) @can_return_tuple diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index 5d289546b312..a89137653a7d 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -262,7 +262,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithVisualIndicatorFeatures]: image_outputs = self.vision_tower(pixel_values, **kwargs) image_features = image_outputs.pooler_output batch_size, img_seq_len, _ = image_features.shape @@ -377,7 +377,7 @@ def __init__(self, config: Ovis2Config): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithVisualIndicatorFeatures]: return self.model.get_image_features(pixel_values=pixel_values, **kwargs) @can_return_tuple diff --git a/src/transformers/models/owlv2/modeling_owlv2.py b/src/transformers/models/owlv2/modeling_owlv2.py index 0ac70bb97f2b..623bcb86f43c 100644 --- a/src/transformers/models/owlv2/modeling_owlv2.py +++ b/src/transformers/models/owlv2/modeling_owlv2.py @@ -1024,7 +1024,7 @@ def get_image_features( pixel_values: torch.Tensor, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/owlvit/modeling_owlvit.py b/src/transformers/models/owlvit/modeling_owlvit.py index 5aef274063f8..55e8a7354366 100644 --- a/src/transformers/models/owlvit/modeling_owlvit.py +++ b/src/transformers/models/owlvit/modeling_owlvit.py @@ -1005,7 +1005,7 @@ def get_image_features( pixel_values: torch.Tensor, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index d69f011e518d..89d759ac98c6 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -1226,7 +1226,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1389,7 +1389,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py index 1e1fa10006ba..e73a329a94b7 100644 --- a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py @@ -1119,7 +1119,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index 5be53ab06c4a..b7b9a5eb47c6 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -26,7 +26,7 @@ from ...generation import GenerationMixin from ...masking_utils import create_masks_for_generate from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import BaseModelOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import ( @@ -261,7 +261,9 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple - def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. diff --git a/src/transformers/models/perception_lm/modeling_perception_lm.py b/src/transformers/models/perception_lm/modeling_perception_lm.py index 716b5953dbd7..0d27496b53bf 100644 --- a/src/transformers/models/perception_lm/modeling_perception_lm.py +++ b/src/transformers/models/perception_lm/modeling_perception_lm.py @@ -28,7 +28,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -186,7 +186,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. diff --git a/src/transformers/models/perception_lm/modular_perception_lm.py b/src/transformers/models/perception_lm/modular_perception_lm.py index 11cb5502be3d..4f1c22f5eb57 100644 --- a/src/transformers/models/perception_lm/modular_perception_lm.py +++ b/src/transformers/models/perception_lm/modular_perception_lm.py @@ -21,6 +21,7 @@ from torch import nn from ...cache_utils import Cache +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import ( TransformersKwargs, @@ -152,7 +153,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. diff --git a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py index 4cbe1e34fe2d..ce1c7d95ca8f 100644 --- a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py @@ -45,12 +45,12 @@ PatchEmbed, PatchMerger, Qwen2RMSNorm, + Qwen2VLAttention, Qwen2VLCausalLMOutputWithPast, Qwen2VLForConditionalGeneration, Qwen2VLModel, Qwen2VLModelOutputWithPast, Qwen2VLPreTrainedModel, - Qwen2VLAttention, TransformersKwargs, VisionAttention, VisionRotaryEmbedding, @@ -179,10 +179,10 @@ def _init_weights(self, module): init.copy_(module.inv_freq, inv_freq) - class Qwen2_5_VLAttention(Qwen2VLAttention): pass + class Qwen2_5_VisionTransformerPretrainedModel(Qwen2_5_VLPreTrainedModel): config: Qwen2_5_VLVisionConfig _no_split_modules = ["Qwen2_5_VLVisionBlock"] diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 261a6ea34fb1..1ec54a793bb6 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -1140,7 +1140,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -1327,7 +1327,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index b855b7e66046..5abb2f194b58 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -1982,7 +1982,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index eb70f522e45f..631a9685187e 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -1112,7 +1112,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithDeepstackFeatures: + ) -> Union[tuple, BaseModelOutputWithDeepstackFeatures]: """ Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1356,7 +1356,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithDeepstackFeatures]: return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index 92ed6b3e5523..0673c0e64109 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -964,7 +964,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithDeepstackFeatures: + ) -> Union[tuple, BaseModelOutputWithDeepstackFeatures]: """ Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1130,6 +1130,9 @@ class Qwen3VLForConditionalGeneration(Qwen2_5_VLForConditionalGeneration): config: Qwen3VLConfig _checkpoint_conversion_mapping = {} + def get_image_features(self, **super_kwargs) -> Union[tuple, BaseModelOutputWithDeepstackFeatures]: + return super().get_image_features(**super_kwargs) + @can_return_tuple def forward( self, diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index 718a321d7e47..c84687727030 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -1238,7 +1238,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithDeepstackFeatures: + ) -> Union[tuple, BaseModelOutputWithDeepstackFeatures]: """ Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1535,7 +1535,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithDeepstackFeatures]: return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/sam2/modeling_sam2.py b/src/transformers/models/sam2/modeling_sam2.py index ee636655ece6..ba57a7b7ba80 100644 --- a/src/transformers/models/sam2/modeling_sam2.py +++ b/src/transformers/models/sam2/modeling_sam2.py @@ -1570,12 +1570,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> tuple[ - list[torch.Tensor], - list[torch.Tensor], - Optional[tuple[torch.FloatTensor, ...]], - Optional[tuple[torch.FloatTensor, ...]], - ]: + ) -> Union[tuple, Sam2VisionEncoderOutput]: r""" Extract and preprocess image features using the vision encoder. diff --git a/src/transformers/models/sam2/modular_sam2.py b/src/transformers/models/sam2/modular_sam2.py index 4934d284861d..dc429b4f24b3 100644 --- a/src/transformers/models/sam2/modular_sam2.py +++ b/src/transformers/models/sam2/modular_sam2.py @@ -1260,12 +1260,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> tuple[ - list[torch.Tensor], - list[torch.Tensor], - Optional[tuple[torch.FloatTensor, ...]], - Optional[tuple[torch.FloatTensor, ...]], - ]: + ) -> Union[tuple, Sam2VisionEncoderOutput]: r""" Extract and preprocess image features using the vision encoder. diff --git a/src/transformers/models/sam2_video/modeling_sam2_video.py b/src/transformers/models/sam2_video/modeling_sam2_video.py index 6ba8a5779796..1b1553a7017e 100644 --- a/src/transformers/models/sam2_video/modeling_sam2_video.py +++ b/src/transformers/models/sam2_video/modeling_sam2_video.py @@ -1830,12 +1830,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> tuple[ - list[torch.Tensor], - list[torch.Tensor], - Optional[tuple[torch.FloatTensor, ...]], - Optional[tuple[torch.FloatTensor, ...]], - ]: + ) -> Union[tuple, Sam2VideoVisionEncoderOutput]: r""" Extract and preprocess image features using the vision encoder. diff --git a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py index da4797b54c88..2906631174e7 100644 --- a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py +++ b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py @@ -1058,12 +1058,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> tuple[ - list[torch.Tensor], - list[torch.Tensor], - Optional[tuple[torch.FloatTensor, ...]], - Optional[tuple[torch.FloatTensor, ...]], - ]: + ) -> Union[tuple, Sam3TrackerVisionEncoderOutput]: r""" Extract and preprocess image features using the vision encoder. diff --git a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py index bf1235a2a0f1..44ee22502a91 100644 --- a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py @@ -1866,12 +1866,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> tuple[ - list[torch.Tensor], - list[torch.Tensor], - Optional[tuple[torch.FloatTensor, ...]], - Optional[tuple[torch.FloatTensor, ...]], - ]: + ) -> Union[tuple, Sam3TrackerVideoVisionEncoderOutput]: r""" Extract and preprocess image features using the vision encoder. diff --git a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py index a623190f95be..44ebcd4daf0d 100644 --- a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py @@ -14,7 +14,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Optional +from typing import Optional, Union import torch @@ -567,12 +567,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], - ) -> tuple[ - list[torch.Tensor], - list[torch.Tensor], - Optional[tuple[torch.FloatTensor, ...]], - Optional[tuple[torch.FloatTensor, ...]], - ]: + ) -> Union[tuple, Sam3TrackerVideoVisionEncoderOutput]: r""" Extract and preprocess image features using the vision encoder. diff --git a/src/transformers/models/siglip/modeling_siglip.py b/src/transformers/models/siglip/modeling_siglip.py index 7f0eceb02bf4..276093abc53c 100644 --- a/src/transformers/models/siglip/modeling_siglip.py +++ b/src/transformers/models/siglip/modeling_siglip.py @@ -831,7 +831,7 @@ def get_image_features( pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool = False, **kwargs: Unpack[TransformersKwargs], - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by diff --git a/src/transformers/models/siglip2/modeling_siglip2.py b/src/transformers/models/siglip2/modeling_siglip2.py index 6f250de981b6..158aca9cae29 100644 --- a/src/transformers/models/siglip2/modeling_siglip2.py +++ b/src/transformers/models/siglip2/modeling_siglip2.py @@ -889,7 +889,7 @@ def get_image_features( pixel_attention_mask: Optional[torch.Tensor] = None, spatial_shapes: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): Mask to avoid performing attention on padding pixel indices. diff --git a/src/transformers/models/siglip2/modular_siglip2.py b/src/transformers/models/siglip2/modular_siglip2.py index 8e3c9ddb6501..997ac4dbcb30 100644 --- a/src/transformers/models/siglip2/modular_siglip2.py +++ b/src/transformers/models/siglip2/modular_siglip2.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional +from typing import Optional, Union import torch import torch.nn as nn @@ -368,7 +368,7 @@ def get_image_features( pixel_attention_mask: Optional[torch.Tensor] = None, spatial_shapes: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): Mask to avoid performing attention on padding pixel indices. diff --git a/src/transformers/models/smolvlm/modeling_smolvlm.py b/src/transformers/models/smolvlm/modeling_smolvlm.py index c258d2453b06..a66d5657fe43 100644 --- a/src/transformers/models/smolvlm/modeling_smolvlm.py +++ b/src/transformers/models/smolvlm/modeling_smolvlm.py @@ -525,7 +525,7 @@ def get_image_features( pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> Union[torch.FloatTensor, BaseModelOutputWithPooling]: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -739,7 +739,7 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask) @can_return_tuple diff --git a/src/transformers/models/smolvlm/modular_smolvlm.py b/src/transformers/models/smolvlm/modular_smolvlm.py index 341dfd7bd1c4..3ecd68074929 100644 --- a/src/transformers/models/smolvlm/modular_smolvlm.py +++ b/src/transformers/models/smolvlm/modular_smolvlm.py @@ -21,6 +21,7 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationConfig from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from ..idefics3.configuration_idefics3 import Idefics3Config, Idefics3VisionConfig @@ -198,7 +199,7 @@ def get_image_features( pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/t5gemma2/modeling_t5gemma2.py b/src/transformers/models/t5gemma2/modeling_t5gemma2.py index 4829dc0ce676..d3cff1a9a915 100644 --- a/src/transformers/models/t5gemma2/modeling_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modeling_t5gemma2.py @@ -37,6 +37,7 @@ from ...modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPooling, Seq2SeqLMOutput, Seq2SeqModelOutput, SequenceClassifierOutput, @@ -810,7 +811,9 @@ def __init__( self.post_init() @can_return_tuple - def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> torch.Tensor: + def get_image_features( + self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Convert pixel image to image features via the encoder and projector. """ @@ -1225,7 +1228,9 @@ def get_encoder(self): def get_decoder(self): return self.model.get_decoder() - def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> torch.Tensor: + def get_image_features( + self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.get_encoder().get_image_features(pixel_values, **kwargs) @property diff --git a/src/transformers/models/t5gemma2/modular_t5gemma2.py b/src/transformers/models/t5gemma2/modular_t5gemma2.py index 2486094554a9..0848c5e447cb 100644 --- a/src/transformers/models/t5gemma2/modular_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modular_t5gemma2.py @@ -29,6 +29,7 @@ from ...modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPooling, Seq2SeqLMOutput, Seq2SeqModelOutput, SequenceClassifierOutput, @@ -865,7 +866,9 @@ def __init__( self.post_init() @can_return_tuple - def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> torch.Tensor: + def get_image_features( + self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Convert pixel image to image features via the encoder and projector. """ @@ -1267,7 +1270,9 @@ def get_encoder(self): def get_decoder(self): return self.model.get_decoder() - def get_image_features(self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> torch.Tensor: + def get_image_features( + self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.get_encoder().get_image_features(pixel_values, **kwargs) @property diff --git a/src/transformers/models/video_llama_3/modeling_video_llama_3.py b/src/transformers/models/video_llama_3/modeling_video_llama_3.py index e8847c70c4cc..d1f5b188bb22 100644 --- a/src/transformers/models/video_llama_3/modeling_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modeling_video_llama_3.py @@ -30,7 +30,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, ModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -579,7 +579,7 @@ def get_image_features( image_grid_thw: torch.LongTensor, image_merge_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. @@ -794,7 +794,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/video_llama_3/modular_video_llama_3.py b/src/transformers/models/video_llama_3/modular_video_llama_3.py index e6bbe6e6b736..de8fd50b9d6e 100644 --- a/src/transformers/models/video_llama_3/modular_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modular_video_llama_3.py @@ -37,7 +37,7 @@ valid_images, validate_preprocess_arguments, ) -from ...modeling_outputs import BaseModelOutput, ModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput @@ -623,7 +623,7 @@ def get_image_features( image_grid_thw: torch.LongTensor, image_merge_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index 15b0f0292996..56b4f9f50fec 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -25,7 +25,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import ModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -179,7 +179,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -442,7 +442,7 @@ def get_image_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values_images=pixel_values_images, vision_feature_layer=vision_feature_layer, diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index c0803b9fdfd1..b3cc60111dd6 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -28,7 +28,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -157,7 +157,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -318,7 +318,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, **kwargs ) diff --git a/src/transformers/models/vipllava/modular_vipllava.py b/src/transformers/models/vipllava/modular_vipllava.py index e99f74a2df0d..830938c6c6ae 100644 --- a/src/transformers/models/vipllava/modular_vipllava.py +++ b/src/transformers/models/vipllava/modular_vipllava.py @@ -28,6 +28,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache +from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from .configuration_vipllava import VipLlavaConfig @@ -79,7 +80,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains image last hidden states from the vision tower and apply multimodal projection. @@ -188,7 +189,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layers: Optional[Union[int, list[int]]] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_image_features( pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, **kwargs ) diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index f142788fa4ee..70f71b31e77c 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -149,7 +149,7 @@ def get_text_features( @auto_docstring def get_image_features( self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by From 1a8d14becdb5911c05be5e7210702c636f883620 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 17:30:59 +0100 Subject: [PATCH 056/131] Remove @auto_docstring from pe_video, it's seemingly not used on that arch (or well documented) --- src/transformers/models/pe_video/modeling_pe_video.py | 2 -- src/transformers/models/pe_video/modular_pe_video.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/transformers/models/pe_video/modeling_pe_video.py b/src/transformers/models/pe_video/modeling_pe_video.py index 4e3a941891c8..ad8f6f5671a1 100644 --- a/src/transformers/models/pe_video/modeling_pe_video.py +++ b/src/transformers/models/pe_video/modeling_pe_video.py @@ -571,7 +571,6 @@ def __init__(self, config: PeVideoConfig): self.post_init() @can_return_tuple - @auto_docstring def get_text_features( self, input_ids: torch.Tensor, @@ -587,7 +586,6 @@ def get_text_features( return text_outputs @can_return_tuple - @auto_docstring def get_video_features( self, pixel_values_videos: torch.Tensor, diff --git a/src/transformers/models/pe_video/modular_pe_video.py b/src/transformers/models/pe_video/modular_pe_video.py index c72c75f176ba..bc50803c52f2 100644 --- a/src/transformers/models/pe_video/modular_pe_video.py +++ b/src/transformers/models/pe_video/modular_pe_video.py @@ -152,7 +152,6 @@ def __init__(self, config: PeVideoConfig): self.post_init() @can_return_tuple - @auto_docstring def get_text_features( self, input_ids: torch.Tensor, @@ -168,7 +167,6 @@ def get_text_features( return text_outputs @can_return_tuple - @auto_docstring def get_video_features( self, pixel_values_videos: torch.Tensor, From 87d22d30255eeae8f2b2a492744f49b4beb309c3 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 17:31:36 +0100 Subject: [PATCH 057/131] Update get_video_features type hints to Union[tuple, BaseModelOutputWithPooling] Only exceptions for BaseModelOutputWithDeepstackFeatures --- .../models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py | 2 +- src/transformers/models/glm46v/modeling_glm46v.py | 4 ++-- src/transformers/models/glm4v/modeling_glm4v.py | 4 ++-- src/transformers/models/glm4v/modular_glm4v.py | 2 +- src/transformers/models/glm4v_moe/modeling_glm4v_moe.py | 4 ++-- .../models/instructblipvideo/modeling_instructblipvideo.py | 2 +- .../models/instructblipvideo/modular_instructblipvideo.py | 2 +- .../models/llava_next_video/modeling_llava_next_video.py | 4 ++-- .../models/llava_next_video/modular_llava_next_video.py | 4 ++-- .../models/llava_onevision/modeling_llava_onevision.py | 4 ++-- .../models/llava_onevision/modular_llava_onevision.py | 2 +- .../models/paddleocr_vl/modeling_paddleocr_vl.py | 4 ++-- .../models/qwen2_5_omni/modeling_qwen2_5_omni.py | 2 +- src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py | 2 +- src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py | 4 ++-- src/transformers/models/qwen2_vl/modeling_qwen2_vl.py | 4 ++-- .../models/qwen3_omni_moe/modeling_qwen3_omni_moe.py | 2 +- src/transformers/models/qwen3_vl/modeling_qwen3_vl.py | 4 ++-- src/transformers/models/qwen3_vl/modular_qwen3_vl.py | 5 ++++- .../models/qwen3_vl_moe/modeling_qwen3_vl_moe.py | 4 ++-- .../models/video_llama_3/modeling_video_llama_3.py | 4 ++-- .../models/video_llama_3/modular_video_llama_3.py | 2 +- src/transformers/models/video_llava/modeling_video_llava.py | 2 +- src/transformers/models/x_clip/modeling_x_clip.py | 2 +- 24 files changed, 39 insertions(+), 36 deletions(-) diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index a23d38ab12eb..13ee6891f4b7 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -1609,7 +1609,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index b8260e6d5d35..d0f4fca265b8 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -297,7 +297,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -547,7 +547,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index ca3b6e6d0366..b7ac3d83299c 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1166,7 +1166,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1416,7 +1416,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 239c581eca94..20640e2a7bbe 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -1158,7 +1158,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 37543599c577..bbaf61d1de90 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1382,7 +1382,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1685,7 +1685,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 4af413c969b9..b61326294b0e 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -1459,7 +1459,7 @@ def get_video_features( qformer_attention_mask: Optional[torch.LongTensor] = None, interpolate_pos_encoding: Optional[bool] = False, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithVisionQformerOutputs]: """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py index 3d07defaeb26..2d447e7f58cc 100644 --- a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py @@ -295,7 +295,7 @@ def get_video_features( qformer_attention_mask: Optional[torch.LongTensor] = None, interpolate_pos_encoding: Optional[bool] = False, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithVisionQformerOutputs]: """ Encodes images into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 9b666e90b5e3..47141c1817c0 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -625,7 +625,7 @@ def get_video_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains video last hidden states from the vision tower and apply multimodal projection. @@ -971,7 +971,7 @@ def get_video_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_video_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index 95df59755d65..8765312cc5a5 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -359,7 +359,7 @@ def get_video_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains video last hidden states from the vision tower and apply multimodal projection. @@ -552,7 +552,7 @@ def get_video_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_video_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index a6af4983e503..2acaa7cca1a1 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -608,7 +608,7 @@ def get_video_features( vision_feature_layer: Union[int, list[int]], vision_feature_select_strategy: str, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains video last hidden states from the vision tower, apply multimodal projection and pooling. @@ -949,7 +949,7 @@ def get_video_features( vision_feature_layer: Optional[Union[int, list[int]]] = None, vision_feature_select_strategy: Optional[str] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_video_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index e1bcaa1c1bf2..548f82005b29 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -414,7 +414,7 @@ def get_video_features( vision_feature_layer: Union[int, list[int]], vision_feature_select_strategy: str, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains video last hidden states from the vision tower, apply multimodal projection and pooling. diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index 89d759ac98c6..0ed947359a4d 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -1202,7 +1202,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1379,7 +1379,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index 2c323403826b..3461699cf099 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -1800,7 +1800,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index cf50a6c6eb8f..cd0e287685a9 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -2105,7 +2105,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index c9544d24d6aa..58eea5187663 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -1186,7 +1186,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1424,7 +1424,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 1ec54a793bb6..49ed475d32ac 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -1116,7 +1116,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -1317,7 +1317,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 5abb2f194b58..54b1be975f2a 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -1963,7 +1963,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index 631a9685187e..ad82d6f1b4ad 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -1093,7 +1093,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithDeepstackFeatures: + ) -> Union[tuple, BaseModelOutputWithDeepstackFeatures]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1346,7 +1346,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithDeepstackFeatures]: return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index 0673c0e64109..7dcb142142c1 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -990,7 +990,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithDeepstackFeatures: + ) -> Union[tuple, BaseModelOutputWithDeepstackFeatures]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1133,6 +1133,9 @@ class Qwen3VLForConditionalGeneration(Qwen2_5_VLForConditionalGeneration): def get_image_features(self, **super_kwargs) -> Union[tuple, BaseModelOutputWithDeepstackFeatures]: return super().get_image_features(**super_kwargs) + def get_video_features(self, **super_kwargs) -> Union[tuple, BaseModelOutputWithDeepstackFeatures]: + return super().get_video_features(**super_kwargs) + @can_return_tuple def forward( self, diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index c84687727030..9014251efdbe 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -1219,7 +1219,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithDeepstackFeatures: + ) -> Union[tuple, BaseModelOutputWithDeepstackFeatures]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. @@ -1525,7 +1525,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithDeepstackFeatures]: return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) diff --git a/src/transformers/models/video_llama_3/modeling_video_llama_3.py b/src/transformers/models/video_llama_3/modeling_video_llama_3.py index d1f5b188bb22..149d9515b200 100644 --- a/src/transformers/models/video_llama_3/modeling_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modeling_video_llama_3.py @@ -553,7 +553,7 @@ def get_video_features( video_grid_thw: torch.LongTensor, video_merge_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. @@ -784,7 +784,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) diff --git a/src/transformers/models/video_llama_3/modular_video_llama_3.py b/src/transformers/models/video_llama_3/modular_video_llama_3.py index de8fd50b9d6e..0bb7ebab1132 100644 --- a/src/transformers/models/video_llama_3/modular_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modular_video_llama_3.py @@ -597,7 +597,7 @@ def get_video_features( video_grid_thw: torch.LongTensor, video_merge_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes videos into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index 56b4f9f50fec..20038fe41b6e 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -236,7 +236,7 @@ def get_video_features( pixel_values_videos: torch.FloatTensor, vision_feature_layer: Optional[Union[int, list[int]]] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Obtains video last hidden states from the vision tower and apply multimodal projection. diff --git a/src/transformers/models/x_clip/modeling_x_clip.py b/src/transformers/models/x_clip/modeling_x_clip.py index e49192e5f76f..36854dda529b 100644 --- a/src/transformers/models/x_clip/modeling_x_clip.py +++ b/src/transformers/models/x_clip/modeling_x_clip.py @@ -1247,7 +1247,7 @@ def get_video_features( self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs], - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" Returns: video_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The video embeddings obtained by From 8d5802e2ae943d8975c03fe098de3718edf629ca Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 17:44:55 +0100 Subject: [PATCH 058/131] Fix pe_video import issue --- src/transformers/models/pe_video/modeling_pe_video.py | 3 ++- src/transformers/models/pe_video/modular_pe_video.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/pe_video/modeling_pe_video.py b/src/transformers/models/pe_video/modeling_pe_video.py index ad8f6f5671a1..936edf439a5d 100644 --- a/src/transformers/models/pe_video/modeling_pe_video.py +++ b/src/transformers/models/pe_video/modeling_pe_video.py @@ -35,7 +35,8 @@ from ...modeling_outputs import BaseModelOutputWithPooling, MaskedLMOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...utils import ModelOutput, TransformersKwargs, Unpack, auto_docstring, can_return_tuple +from ...processing_utils import Unpack +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple from ...utils.generic import check_model_inputs, maybe_autocast from ..auto import AutoModel, AutoModelForImageClassification from .configuration_pe_video import PeVideoConfig, PeVideoEncoderConfig diff --git a/src/transformers/models/pe_video/modular_pe_video.py b/src/transformers/models/pe_video/modular_pe_video.py index bc50803c52f2..4cb80b04fb2d 100644 --- a/src/transformers/models/pe_video/modular_pe_video.py +++ b/src/transformers/models/pe_video/modular_pe_video.py @@ -21,7 +21,8 @@ from ...modeling_attn_mask_utils import _prepare_4d_attention_mask from ...modeling_outputs import BaseModelOutputWithPooling, MaskedLMOutput -from ...utils import ModelOutput, TransformersKwargs, Unpack, auto_docstring, can_return_tuple +from ...processing_utils import Unpack +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple from ...utils.generic import check_model_inputs from ..auto import AutoModel, AutoModelForImageClassification from ..pe_audio_video.modeling_pe_audio_video import ( From a9ff924b5e5e2a289a3ccd67f742bc0d46f0463e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 17:45:06 +0100 Subject: [PATCH 059/131] Update forward, test, and docstring for sam3 --- src/transformers/models/sam3/modeling_sam3.py | 6 ++++-- tests/models/sam3/test_modeling_sam3.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/sam3/modeling_sam3.py b/src/transformers/models/sam3/modeling_sam3.py index c2790782339d..d17071acd0a0 100644 --- a/src/transformers/models/sam3/modeling_sam3.py +++ b/src/transformers/models/sam3/modeling_sam3.py @@ -2176,7 +2176,7 @@ def get_text_features( >>> # Pre-compute text embeddings >>> text_inputs = processor(text="cat", return_tensors="pt") - >>> text_embeds = model.get_text_features(**text_inputs) + >>> text_embeds = model.get_text_features(**text_inputs).pooler_output >>> # Reuse text embeddings for multiple images >>> img_url = "http://images.cocodataset.org/val2017/000000077595.jpg" @@ -2294,7 +2294,9 @@ def forward( fpn_position_encoding = vision_outputs.fpn_position_encoding[:-1] if text_embeds is None: - text_features = self.get_text_features(input_ids=input_ids, attention_mask=attention_mask, **kwargs) + text_features = self.get_text_features( + input_ids=input_ids, attention_mask=attention_mask, return_dict=True + ).pooler_output else: text_features = text_embeds diff --git a/tests/models/sam3/test_modeling_sam3.py b/tests/models/sam3/test_modeling_sam3.py index d99a4773f716..edef399fbe0a 100644 --- a/tests/models/sam3/test_modeling_sam3.py +++ b/tests/models/sam3/test_modeling_sam3.py @@ -753,8 +753,8 @@ def test_forward_with_text_embeds(self): # First get text embeddings with torch.no_grad(): text_embeds = model.get_text_features( - input_ids=inputs_dict["input_ids"], attention_mask=inputs_dict["attention_mask"] - ) + input_ids=inputs_dict["input_ids"], attention_mask=inputs_dict["attention_mask"], return_dict=True + ).pooler_output # Forward with text_embeds (remove input_ids) inputs_with_embeds = { From 8ad35e74c90fdd945cfa9ede902965d528463467 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 7 Jan 2026 17:48:54 +0100 Subject: [PATCH 060/131] Update get_audio_features type hints to Union[tuple, BaseModelOutputWithPooling] Also update BaseModelOutput to BaseModelOutputWithPooling in several places, leaving room for a potential pooled embedding to be computed by get_audio_features --- .../models/audioflamingo3/modeling_audioflamingo3.py | 6 +++--- .../models/audioflamingo3/modular_audioflamingo3.py | 6 +++--- src/transformers/models/clap/modeling_clap.py | 2 +- src/transformers/models/gemma3n/modeling_gemma3n.py | 2 +- src/transformers/models/gemma3n/modular_gemma3n.py | 2 +- src/transformers/models/glmasr/modeling_glmasr.py | 6 +++--- src/transformers/models/glmasr/modular_glmasr.py | 6 +++--- .../models/granite_speech/modeling_granite_speech.py | 4 +++- .../models/qwen2_5_omni/modeling_qwen2_5_omni.py | 6 +++--- .../models/qwen2_5_omni/modular_qwen2_5_omni.py | 6 +++--- .../models/qwen3_omni_moe/modeling_qwen3_omni_moe.py | 5 ++--- .../models/qwen3_omni_moe/modular_qwen3_omni_moe.py | 5 ++--- src/transformers/models/voxtral/modeling_voxtral.py | 8 +++++--- src/transformers/models/voxtral/modular_voxtral.py | 8 +++++--- 14 files changed, 38 insertions(+), 34 deletions(-) diff --git a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py index 4f18c849dc41..1bda97e2607d 100644 --- a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py @@ -33,7 +33,7 @@ from ...masking_utils import create_bidirectional_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPooling, CausalLMOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -368,7 +368,7 @@ def forward( hidden_states = self.avg_pooler(hidden_states).permute(0, 2, 1) hidden_states = self.layer_norm(hidden_states) - return BaseModelOutput( + return BaseModelOutputWithPooling( last_hidden_state=hidden_states, ) @@ -449,7 +449,7 @@ def get_audio_features( input_features: torch.FloatTensor, input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. Args: diff --git a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py index 81a545a4fbf4..120b75845ca8 100644 --- a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py @@ -22,7 +22,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache from ...masking_utils import create_bidirectional_mask -from ...modeling_outputs import BaseModelOutput, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPooling, CausalLMOutputWithPast from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging from ..qwen2_audio.modeling_qwen2_audio import ( @@ -105,7 +105,7 @@ def forward( hidden_states = self.avg_pooler(hidden_states).permute(0, 2, 1) hidden_states = self.layer_norm(hidden_states) - return BaseModelOutput( + return BaseModelOutputWithPooling( last_hidden_state=hidden_states, ) @@ -146,7 +146,7 @@ def get_audio_features( input_features: torch.FloatTensor, input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. Args: diff --git a/src/transformers/models/clap/modeling_clap.py b/src/transformers/models/clap/modeling_clap.py index cdcb72b293ce..07feff5f577b 100644 --- a/src/transformers/models/clap/modeling_clap.py +++ b/src/transformers/models/clap/modeling_clap.py @@ -1600,7 +1600,7 @@ def get_audio_features( is_longer: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, **kwargs: Unpack[TransformersKwargs], - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithPooling]: r""" is_longer (`torch.FloatTensor`, of shape `(batch_size, 1)`, *optional*): Whether the audio clip is longer than `max_length`. If `True`, a feature fusion will be enabled to enhance diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index e540925d2bdc..e6cc8f6f2423 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -2196,7 +2196,7 @@ def get_audio_features( input_features: torch.Tensor, input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], - ) -> tuple[torch.Tensor, torch.Tensor]: + ) -> Union[tuple, Gemma3nAudioEncoderModelOutput]: """ Projects the last hidden state from the audio encoder into language model space. diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index 666a98ef1d29..b4da9865ffae 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -2459,7 +2459,7 @@ def get_audio_features( input_features: torch.Tensor, input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], - ) -> tuple[torch.Tensor, torch.Tensor]: + ) -> Union[tuple, Gemma3nAudioEncoderModelOutput]: """ Projects the last hidden state from the audio encoder into language model space. diff --git a/src/transformers/models/glmasr/modeling_glmasr.py b/src/transformers/models/glmasr/modeling_glmasr.py index 8e883f28c32c..b0579e70ab1f 100644 --- a/src/transformers/models/glmasr/modeling_glmasr.py +++ b/src/transformers/models/glmasr/modeling_glmasr.py @@ -27,7 +27,7 @@ from ...generation import GenerationMixin from ...integrations import use_kernelized_func from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPooling, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -323,7 +323,7 @@ def forward(self, input_features, **kwargs: Unpack[TransformersKwargs]): hidden_states = encoder_layer(hidden_states, position_embeddings=position_embeddings, **kwargs) hidden_states = self.norm(hidden_states) - return BaseModelOutput(last_hidden_state=hidden_states) + return BaseModelOutputWithPooling(last_hidden_state=hidden_states) class GlmAsrMultiModalProjector(nn.Module): @@ -389,7 +389,7 @@ def get_audio_features( input_features: torch.FloatTensor, input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithPooling]: """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. Args: diff --git a/src/transformers/models/glmasr/modular_glmasr.py b/src/transformers/models/glmasr/modular_glmasr.py index d58538028721..612a841995ef 100644 --- a/src/transformers/models/glmasr/modular_glmasr.py +++ b/src/transformers/models/glmasr/modular_glmasr.py @@ -23,7 +23,7 @@ from ...cache_utils import Cache from ...feature_extraction_utils import BatchFeature from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPooling, CausalLMOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, is_torch_available, logging @@ -337,7 +337,7 @@ def forward(self, input_features, **kwargs: Unpack[TransformersKwargs]): hidden_states = encoder_layer(hidden_states, position_embeddings=position_embeddings, **kwargs) hidden_states = self.norm(hidden_states) - return BaseModelOutput(last_hidden_state=hidden_states) + return BaseModelOutputWithPooling(last_hidden_state=hidden_states) class GlmAsrMultiModalProjector(AudioFlamingo3MultiModalProjector): @@ -358,7 +358,7 @@ def get_audio_features( input_features: torch.FloatTensor, input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], - ) -> torch.FloatTensor: + ) -> Union[tuple, BaseModelOutputWithPooling]: audio_outputs = self.audio_tower(input_features, **kwargs) audio_hidden_states = audio_outputs.last_hidden_state audio_hidden_states = audio_hidden_states.reshape( diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index fbae189a11a0..0a7f1bc56a0a 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -349,7 +349,9 @@ def get_output_embeddings(self): return self.language_model.get_output_embeddings() @can_return_tuple - def get_audio_features(self, input_features: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> torch.Tensor: + def get_audio_features( + self, input_features: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: """Get the audio features to merged into the multimodal embeddings.""" audio_outputs = self.encoder(input_features, **kwargs) projected_embeds = self.projector(audio_outputs.last_hidden_state) diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index 3461699cf099..407717c04578 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -38,7 +38,7 @@ from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -888,7 +888,7 @@ def forward(self, input_features, feature_lens=None, aftercnn_lens=None, **kwarg each_audio_states = self.proj(each_audio_states) token_audio_list.append(each_audio_states) token_audio = torch.cat(token_audio_list, dim=0) - return BaseModelOutput(last_hidden_state=token_audio) + return BaseModelOutputWithPooling(last_hidden_state=token_audio) def padded_and_mask_function(self, tensor_list, tensor_len, padding_value=0, padding_side="right"): """ @@ -1839,7 +1839,7 @@ def get_audio_features( feature_attention_mask: Optional[torch.LongTensor] = None, audio_feature_lengths: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes audios into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index cd0e287685a9..d7d686a48201 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -30,7 +30,7 @@ from ...cache_utils import Cache from ...configuration_utils import PreTrainedConfig, layer_type_validation from ...generation import GenerationMixin -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput from ...modeling_rope_utils import RopeParameters from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -1768,7 +1768,7 @@ def forward(self, input_features, feature_lens=None, aftercnn_lens=None, **kwarg each_audio_states = self.proj(each_audio_states) token_audio_list.append(each_audio_states) token_audio = torch.cat(token_audio_list, dim=0) - return BaseModelOutput(last_hidden_state=token_audio) + return BaseModelOutputWithPooling(last_hidden_state=token_audio) def padded_and_mask_function(self, tensor_list, tensor_len, padding_value=0, padding_side="right"): """ @@ -2144,7 +2144,7 @@ def get_audio_features( feature_attention_mask: Optional[torch.LongTensor] = None, audio_feature_lengths: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes audios into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 54b1be975f2a..81febf172506 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -45,7 +45,6 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ( - BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling, CausalLMOutputWithPast, @@ -817,7 +816,7 @@ def forward( hidden_states = self.proj1(hidden_states) hidden_states = self.act(hidden_states) hidden_states = self.proj2(hidden_states) - return BaseModelOutput(last_hidden_state=hidden_states) + return BaseModelOutputWithPooling(last_hidden_state=hidden_states) def padded_and_mask_function(self, tensor_list, tensor_len, padding_value=0, padding_side="right"): """ @@ -2002,7 +2001,7 @@ def get_audio_features( feature_attention_mask: Optional[torch.LongTensor] = None, audio_feature_lengths: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes audios into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py index af86d6253b7d..7f64b5a6d0b4 100644 --- a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py @@ -36,7 +36,6 @@ from ...masking_utils import create_causal_mask from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ( - BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling, CausalLMOutputWithPast, @@ -1300,7 +1299,7 @@ def forward( hidden_states = self.proj1(hidden_states) hidden_states = self.act(hidden_states) hidden_states = self.proj2(hidden_states) - return BaseModelOutput(last_hidden_state=hidden_states) + return BaseModelOutputWithPooling(last_hidden_state=hidden_states) class Qwen3OmniMoeVisionAttention(Qwen3VLMoeVisionAttention): @@ -1443,7 +1442,7 @@ def get_audio_features( feature_attention_mask: Optional[torch.LongTensor] = None, audio_feature_lengths: Optional[torch.LongTensor] = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> Union[tuple, BaseModelOutputWithPooling]: """ Encodes audios into continuous embeddings that can be forwarded to the language model. diff --git a/src/transformers/models/voxtral/modeling_voxtral.py b/src/transformers/models/voxtral/modeling_voxtral.py index 3953a255c593..cf0cd1f62ecf 100644 --- a/src/transformers/models/voxtral/modeling_voxtral.py +++ b/src/transformers/models/voxtral/modeling_voxtral.py @@ -31,7 +31,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, CausalLMOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging @@ -336,7 +336,7 @@ def forward( hidden_states = self.layer_norm(hidden_states) - return BaseModelOutput( + return BaseModelOutputWithPooling( last_hidden_state=hidden_states, ) @@ -401,7 +401,9 @@ def get_decoder(self): return self.language_model.get_decoder() @can_return_tuple - def get_audio_features(self, input_features: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + def get_audio_features( + self, input_features: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. diff --git a/src/transformers/models/voxtral/modular_voxtral.py b/src/transformers/models/voxtral/modular_voxtral.py index ffacf9a51302..3b27abba14b2 100644 --- a/src/transformers/models/voxtral/modular_voxtral.py +++ b/src/transformers/models/voxtral/modular_voxtral.py @@ -23,8 +23,8 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_outputs import ( - BaseModelOutput, BaseModelOutputWithPast, + BaseModelOutputWithPooling, CausalLMOutputWithPast, ) from ...processing_utils import Unpack @@ -111,7 +111,7 @@ def forward( hidden_states = self.layer_norm(hidden_states) - return BaseModelOutput( + return BaseModelOutputWithPooling( last_hidden_state=hidden_states, ) @@ -167,7 +167,9 @@ def get_decoder(self): return self.language_model.get_decoder() @can_return_tuple - def get_audio_features(self, input_features: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): + def get_audio_features( + self, input_features: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> Union[tuple, BaseModelOutputWithPooling]: """ This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. From 7c99867a7e940f645320e66ebfc1c70f8bed6213 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 8 Jan 2026 16:44:59 +0100 Subject: [PATCH 061/131] Add simple test case for get_text_features Fails on CLIP, MetaCLIP, Siglip, Siglip2 as they use 'self.text_model = text_model.text_model', bypassing the TextModel that has `check_model_inputs` cc @zucchini-nlp related to #42564 --- .../models/groupvit/modeling_groupvit.py | 1 + tests/models/sam3/test_modeling_sam3.py | 2 +- tests/test_modeling_common.py | 113 ++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/groupvit/modeling_groupvit.py b/src/transformers/models/groupvit/modeling_groupvit.py index f1a24b2f7714..afae52ae5b45 100644 --- a/src/transformers/models/groupvit/modeling_groupvit.py +++ b/src/transformers/models/groupvit/modeling_groupvit.py @@ -1260,6 +1260,7 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + **kwargs, ) pooled_output = text_outputs.pooler_output text_outputs.pooler_output = self.text_projection(pooled_output) diff --git a/tests/models/sam3/test_modeling_sam3.py b/tests/models/sam3/test_modeling_sam3.py index edef399fbe0a..029b388ed615 100644 --- a/tests/models/sam3/test_modeling_sam3.py +++ b/tests/models/sam3/test_modeling_sam3.py @@ -341,7 +341,7 @@ def get_config(self): "hidden_size": 32, "intermediate_size": 64, "projection_dim": 32, - "num_hidden_layers": 1, + "num_hidden_layers": self.num_hidden_layers, "num_attention_heads": 4, "max_position_embeddings": 32, # Keep at 32 for stability "hidden_act": "gelu", diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 061638bc5273..c67359f38930 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -111,6 +111,7 @@ CONFIG_NAME, GENERATION_CONFIG_NAME, SAFE_WEIGHTS_NAME, + ModelOutput, is_torch_bf16_available_on_device, is_torch_fp16_available_on_device, ) @@ -4577,6 +4578,118 @@ def test_can_load_from_already_mapped_keys(self): # Make sure both saved state_dict are identical self.assertTrue(compare_state_dicts(model.state_dict(), model_reloaded.state_dict())) + def test_get_text_features(self): + def prepare_config_and_inputs_for_text_features(): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + if hasattr(self.model_tester, "text_model_tester"): + _, inputs_dict = self.model_tester.text_model_tester.prepare_config_and_inputs_for_common() + else: + inputs_dict = { + key: value + for key, value in inputs_dict.items() + if key in ["input_ids", "token_type_ids", "attention_mask"] + } + return config, inputs_dict + + def check_hidden_states_output(inputs_dict, config, model_class): + model = model_class(copy.deepcopy(config)) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model.get_text_features(**inputs_dict) + # hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states + hidden_states = outputs.hidden_states + if hasattr(self.model_tester, "text_model_tester"): + expected_num_layers = self.model_tester.text_model_tester.num_hidden_layers + 1 + else: + expected_num_layers = self.model_tester.num_hidden_layers + 1 + self.assertIsNotNone(hidden_states, "hidden_states should not be None") + self.assertEqual(len(hidden_states), expected_num_layers, "Number of hidden states layers mismatch") + + def check_attentions_output(inputs_dict, config, model_class): + model = model_class(copy.deepcopy(config)) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model.get_text_features(**inputs_dict) + attentions = outputs.attentions + # model.text_model(**inputs_dict) also no attentions for aimv2 + if hasattr(self.model_tester, "text_model_tester"): + expected_num_layers = self.model_tester.text_model_tester.num_hidden_layers + else: + expected_num_layers = self.model_tester.num_hidden_layers + self.assertIsNotNone(attentions, "attentions should not be None") + self.assertEqual(len(attentions), expected_num_layers, "Number of attention layers mismatch") + + for model_class in self.all_model_classes: + if not hasattr(model_class, "get_text_features"): + continue + + config, inputs_dict = prepare_config_and_inputs_for_text_features() + + model = model_class(config).eval() + model = model.to(torch_device) + + torch.manual_seed(0) + outputs = model.get_text_features(**inputs_dict) + self.assertTrue(isinstance(outputs, ModelOutput), "get_text_features() must return a BaseModelOutput") + self.assertTrue( + hasattr(outputs, "last_hidden_state"), + "get_text_features() must return a BaseModelOutput with last_hidden_state", + ) + self.assertTrue( + hasattr(outputs, "pooler_output"), + "get_text_features() must return a BaseModelOutput with pooler_output", + ) + self.assertTrue( + hasattr(outputs, "hidden_states"), + "get_text_features() must return a BaseModelOutput with hidden_states", + ) + self.assertTrue( + hasattr(outputs, "attentions"), "get_text_features() must return a BaseModelOutput with attentions" + ) + + last_hidden_state = outputs.last_hidden_state + if hasattr(self.model_tester, "text_model_tester"): + expected_hidden_size = self.model_tester.text_model_tester.hidden_size + else: + expected_hidden_size = self.model_tester.hidden_size + expected_shape = (*inputs_dict["input_ids"].shape, expected_hidden_size) + self.assertEqual(last_hidden_state.shape, expected_shape, "last_hidden_state shape mismatch") + + inputs_dict["output_hidden_states"] = True + check_hidden_states_output(inputs_dict, config, model_class) + + # check that output_hidden_states also work using config + del inputs_dict["output_hidden_states"] + config.output_hidden_states = True + for k in config.sub_configs: + if getattr(config, k) is not None: + getattr(config, k).output_hidden_states = True + + check_hidden_states_output(inputs_dict, config, model_class) + + if not self.has_attentions: + continue + + config, inputs_dict = prepare_config_and_inputs_for_text_features() + # force eager attention to support output attentions + config._attn_implementation = "eager" + inputs_dict["output_hidden_states"] = False + inputs_dict["output_attentions"] = True + check_attentions_output(inputs_dict, config, model_class) + + # check that output_attentions also work using config + del inputs_dict["output_attentions"] + config.output_attentions = True + for k in config.sub_configs: + if getattr(config, k) is not None: + getattr(config, k).output_attentions = True + + check_attentions_output(inputs_dict, config, model_class) + global_rng = random.Random() From 35feb85fe6de5a5dc015fa07beb08376ca112034 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 8 Jan 2026 18:20:05 +0100 Subject: [PATCH 062/131] First attempt to get get_image_features under test, still 26 failures --- .../models/idefics3/modeling_idefics3.py | 9 +- .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 64 ++++++--- .../models/qwen3_vl/modeling_qwen3_vl.py | 4 + .../models/qwen3_vl/modular_qwen3_vl.py | 4 + .../qwen3_vl_moe/modeling_qwen3_vl_moe.py | 125 +++++++++-------- .../qwen3_vl_moe/modular_qwen3_vl_moe.py | 17 ++- .../models/smolvlm/modeling_smolvlm.py | 9 +- tests/models/align/test_modeling_align.py | 7 + tests/test_modeling_common.py | 129 +++++++++++++++++- 9 files changed, 280 insertions(+), 88 deletions(-) diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index 8e2057538704..1dfbb13419a0 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -756,9 +756,14 @@ def set_input_embeddings(self, value): self.model.text_model.set_input_embeddings(value) def get_image_features( - self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None + self, + pixel_values: torch.FloatTensor, + pixel_attention_mask: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], ) -> Union[tuple, BaseModelOutputWithPooling]: - return self.model.get_image_features(pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask) + return self.model.get_image_features( + pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask, **kwargs + ) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 81febf172506..76d82ab1474d 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -1009,6 +1009,25 @@ def forward(self, seqlen: int) -> torch.Tensor: return freqs +class Qwen3OmniMoeTextTopKRouter(nn.Module): + def __init__(self, config): + super().__init__() + self.top_k = config.num_experts_per_tok + self.num_experts = config.num_experts + self.hidden_dim = config.hidden_size + self.weight = nn.Parameter(torch.zeros(self.num_experts, self.hidden_dim)) + + def forward(self, hidden_states): + hidden_states = hidden_states.reshape(-1, self.hidden_dim) + router_logits = F.linear(hidden_states, self.weight) # (seq_len, num_experts) + router_logits = torch.nn.functional.softmax(router_logits, dtype=torch.float, dim=-1) + router_top_value, router_indices = torch.topk(router_logits, self.top_k, dim=-1) # (seq_len, top_k) + router_top_value /= router_top_value.sum(dim=-1, keepdim=True) + router_top_value = router_top_value.to(router_logits.dtype) + router_scores = router_top_value + return router_logits, router_scores, router_indices + + class Qwen3OmniMoeVisionMLP(nn.Module): def __init__(self, config): super().__init__() @@ -1022,26 +1041,6 @@ def forward(self, hidden_state): return self.linear_fc2(self.act_fn(self.linear_fc1(hidden_state))) -class Qwen3OmniMoeVisionPatchEmbed(nn.Module): - def __init__(self, config) -> None: - super().__init__() - self.patch_size = config.patch_size - self.temporal_patch_size = config.temporal_patch_size - self.in_channels = config.in_channels - self.embed_dim = config.hidden_size - - kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size] - self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=True) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - target_dtype = self.proj.weight.dtype - hidden_states = hidden_states.view( - -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size - ) - hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) - return hidden_states - - class Qwen3OmniMoeVisionBlock(GradientCheckpointingLayer): def __init__(self, config, attn_implementation: str = "sdpa") -> None: super().__init__() @@ -1069,9 +1068,34 @@ def forward( return hidden_states +class Qwen3OmniMoeVisionPatchEmbed(nn.Module): + def __init__(self, config) -> None: + super().__init__() + self.patch_size = config.patch_size + self.temporal_patch_size = config.temporal_patch_size + self.in_channels = config.in_channels + self.embed_dim = config.hidden_size + + kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size] + self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=True) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + target_dtype = self.proj.weight.dtype + hidden_states = hidden_states.view( + -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size + ) + hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) + return hidden_states + + class Qwen3OmniMoeVisionEncoder(Qwen3OmniMoePreTrainedModel): config: Qwen3OmniMoeVisionEncoderConfig _no_split_modules = ["Qwen3OmniMoeVisionBlock"] + _can_record_outputs = { + "router_logits": OutputRecorder(Qwen3OmniMoeTextTopKRouter, layer_name="mlp.gate", index=0), + "hidden_states": Qwen3OmniMoeVisionBlock, + "attentions": Qwen3OmniMoeVisionAttention, + } def __init__(self, config, *inputs, **kwargs) -> None: super().__init__(config, *inputs, **kwargs) diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index ad82d6f1b4ad..ef724c9c78e3 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -616,6 +616,10 @@ def _init_weights(self, module): class Qwen3VLVisionModel(Qwen3VLPreTrainedModel): config: Qwen3VLVisionConfig _no_split_modules = ["Qwen3VLVisionBlock"] + _can_record_outputs = { + "hidden_states": Qwen3VLVisionBlock, + "attentions": Qwen3VLVisionAttention, + } def __init__(self, config, *inputs, **kwargs) -> None: super().__init__(config, *inputs, **kwargs) diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index 7dcb142142c1..79b5c97d1127 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -511,6 +511,10 @@ def _init_weights(self, module): class Qwen3VLVisionModel(Qwen3VLPreTrainedModel): config: Qwen3VLVisionConfig _no_split_modules = ["Qwen3VLVisionBlock"] + _can_record_outputs = { + "hidden_states": Qwen3VLVisionBlock, + "attentions": Qwen3VLVisionAttention, + } def __init__(self, config, *inputs, **kwargs) -> None: super().__init__(config, *inputs, **kwargs) diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index 9014251efdbe..5c3825bcf671 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -412,66 +412,6 @@ def forward(self, seqlen: int) -> torch.Tensor: return freqs -@dataclass -@auto_docstring -class BaseModelOutputWithDeepstackFeatures(BaseModelOutputWithPooling): - """ - deepstack_features (`List[torch.FloatTensor]`, *optional*): - List of hidden-states (feature maps) from deepstack layers. - """ - - deepstack_features: Optional[list[torch.FloatTensor]] = None - - -class Qwen3VLMoeVisionMLP(nn.Module): - def __init__(self, config): - super().__init__() - self.hidden_size = config.hidden_size - self.intermediate_size = config.intermediate_size - self.linear_fc1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=True) - self.linear_fc2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=True) - self.act_fn = ACT2FN[config.hidden_act] - - def forward(self, hidden_state): - return self.linear_fc2(self.act_fn(self.linear_fc1(hidden_state))) - - -class Qwen3VLMoeVisionPatchEmbed(nn.Module): - def __init__(self, config) -> None: - super().__init__() - self.patch_size = config.patch_size - self.temporal_patch_size = config.temporal_patch_size - self.in_channels = config.in_channels - self.embed_dim = config.hidden_size - - kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size] - self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=True) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - target_dtype = self.proj.weight.dtype - hidden_states = hidden_states.view( - -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size - ) - hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) - return hidden_states - - -class Qwen3VLMoeVisionPatchMerger(nn.Module): - def __init__(self, config: Qwen3VLMoeVisionConfig, use_postshuffle_norm=False) -> None: - super().__init__() - self.hidden_size = config.hidden_size * (config.spatial_merge_size**2) - self.use_postshuffle_norm = use_postshuffle_norm - self.norm = nn.LayerNorm(self.hidden_size if use_postshuffle_norm else config.hidden_size, eps=1e-6) - self.linear_fc1 = nn.Linear(self.hidden_size, self.hidden_size) - self.act_fn = nn.GELU() - self.linear_fc2 = nn.Linear(self.hidden_size, config.out_hidden_size) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = self.norm(x.view(-1, self.hidden_size) if self.use_postshuffle_norm else x).view(-1, self.hidden_size) - x = self.linear_fc2(self.act_fn(self.linear_fc1(x))) - return x - - def apply_rotary_pos_emb_vision( q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor]: @@ -569,6 +509,19 @@ def forward( return attn_output +class Qwen3VLMoeVisionMLP(nn.Module): + def __init__(self, config): + super().__init__() + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.linear_fc1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=True) + self.linear_fc2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=True) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_state): + return self.linear_fc2(self.act_fn(self.linear_fc1(hidden_state))) + + class Qwen3VLMoeVisionBlock(GradientCheckpointingLayer): def __init__(self, config, attn_implementation: str = "sdpa") -> None: super().__init__() @@ -596,9 +549,61 @@ def forward( return hidden_states +@dataclass +@auto_docstring +class BaseModelOutputWithDeepstackFeatures(BaseModelOutputWithPooling): + """ + deepstack_features (`List[torch.FloatTensor]`, *optional*): + List of hidden-states (feature maps) from deepstack layers. + """ + + deepstack_features: Optional[list[torch.FloatTensor]] = None + + +class Qwen3VLMoeVisionPatchEmbed(nn.Module): + def __init__(self, config) -> None: + super().__init__() + self.patch_size = config.patch_size + self.temporal_patch_size = config.temporal_patch_size + self.in_channels = config.in_channels + self.embed_dim = config.hidden_size + + kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size] + self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=True) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + target_dtype = self.proj.weight.dtype + hidden_states = hidden_states.view( + -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size + ) + hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) + return hidden_states + + +class Qwen3VLMoeVisionPatchMerger(nn.Module): + def __init__(self, config: Qwen3VLMoeVisionConfig, use_postshuffle_norm=False) -> None: + super().__init__() + self.hidden_size = config.hidden_size * (config.spatial_merge_size**2) + self.use_postshuffle_norm = use_postshuffle_norm + self.norm = nn.LayerNorm(self.hidden_size if use_postshuffle_norm else config.hidden_size, eps=1e-6) + self.linear_fc1 = nn.Linear(self.hidden_size, self.hidden_size) + self.act_fn = nn.GELU() + self.linear_fc2 = nn.Linear(self.hidden_size, config.out_hidden_size) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.norm(x.view(-1, self.hidden_size) if self.use_postshuffle_norm else x).view(-1, self.hidden_size) + x = self.linear_fc2(self.act_fn(self.linear_fc1(x))) + return x + + class Qwen3VLMoeVisionModel(Qwen3VLMoePreTrainedModel): config: Qwen3VLMoeVisionConfig _no_split_modules = ["Qwen3VLMoeVisionBlock"] + _can_record_outputs = { + "router_logits": OutputRecorder(Qwen3VLMoeTextTopKRouter, layer_name="mlp.gate", index=0), + "hidden_states": Qwen3VLMoeVisionBlock, + "attentions": Qwen3VLMoeVisionAttention, + } def __init__(self, config, *inputs, **kwargs) -> None: super().__init__(config, *inputs, **kwargs) diff --git a/src/transformers/models/qwen3_vl_moe/modular_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modular_qwen3_vl_moe.py index e8b032eaaf76..0dd619a03bb8 100644 --- a/src/transformers/models/qwen3_vl_moe/modular_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modular_qwen3_vl_moe.py @@ -27,6 +27,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, can_return_tuple, logging +from ...utils.generic import OutputRecorder from ..qwen3_moe.modeling_qwen3_moe import ( Qwen3MoeDecoderLayer, Qwen3MoeExperts, @@ -42,6 +43,8 @@ Qwen3VLModel, Qwen3VLTextAttention, Qwen3VLTextModel, + Qwen3VLVisionAttention, + Qwen3VLVisionBlock, Qwen3VLVisionModel, Qwen3VLVisionRotaryEmbedding, ) @@ -321,10 +324,22 @@ class Qwen3VLMoeVisionRotaryEmbedding(Qwen3VLVisionRotaryEmbedding): pass -class Qwen3VLMoeVisionModel(Qwen3VLVisionModel): +class Qwen3VLMoeVisionAttention(Qwen3VLVisionAttention): + pass + + +class Qwen3VLMoeVisionBlock(Qwen3VLVisionBlock): pass +class Qwen3VLMoeVisionModel(Qwen3VLVisionModel): + _can_record_outputs = { + "router_logits": OutputRecorder(Qwen3VLMoeTextTopKRouter, layer_name="mlp.gate", index=0), + "hidden_states": Qwen3VLMoeVisionBlock, + "attentions": Qwen3VLMoeVisionAttention, + } + + class Qwen3VLMoeTextModel(Qwen3VLTextModel): pass diff --git a/src/transformers/models/smolvlm/modeling_smolvlm.py b/src/transformers/models/smolvlm/modeling_smolvlm.py index a66d5657fe43..4b2f3e4a4752 100644 --- a/src/transformers/models/smolvlm/modeling_smolvlm.py +++ b/src/transformers/models/smolvlm/modeling_smolvlm.py @@ -738,9 +738,14 @@ def set_input_embeddings(self, value): self.model.text_model.set_input_embeddings(value) def get_image_features( - self, pixel_values: torch.FloatTensor, pixel_attention_mask: Optional[torch.LongTensor] = None + self, + pixel_values: torch.FloatTensor, + pixel_attention_mask: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], ) -> Union[tuple, BaseModelOutputWithPooling]: - return self.model.get_image_features(pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask) + return self.model.get_image_features( + pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask, **kwargs + ) @can_return_tuple @auto_docstring diff --git a/tests/models/align/test_modeling_align.py b/tests/models/align/test_modeling_align.py index 9b4e877c30f4..7042bf999452 100644 --- a/tests/models/align/test_modeling_align.py +++ b/tests/models/align/test_modeling_align.py @@ -14,6 +14,7 @@ """Testing suite for the PyTorch ALIGN model.""" import inspect +import math import tempfile import unittest @@ -59,6 +60,7 @@ def __init__( batch_size=12, image_size=32, num_channels=3, + depth_coefficient=3.1, kernel_sizes=[3, 3, 5], in_channels=[32, 16, 24], out_channels=[16, 24, 30], @@ -73,6 +75,7 @@ def __init__( self.batch_size = batch_size self.image_size = image_size self.num_channels = num_channels + self.depth_coefficient = depth_coefficient self.kernel_sizes = kernel_sizes self.in_channels = in_channels self.out_channels = out_channels @@ -83,6 +86,8 @@ def __init__( self.is_training = is_training self.hidden_act = hidden_act + self.num_hidden_layers = sum(math.ceil(self.depth_coefficient * repeat) for repeat in self.num_block_repeats) + def prepare_config_and_inputs(self): pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]) config = self.get_config() @@ -92,6 +97,7 @@ def prepare_config_and_inputs(self): def get_config(self): return AlignVisionConfig( num_channels=self.num_channels, + depth_coefficient=self.depth_coefficient, kernel_sizes=self.kernel_sizes, in_channels=self.in_channels, out_channels=self.out_channels, @@ -438,6 +444,7 @@ class AlignModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): test_resize_embeddings = False test_attention_outputs = False + has_attentions = False def setUp(self): self.model_tester = AlignModelTester(self) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index c67359f38930..707581f47d04 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4647,9 +4647,10 @@ def check_attentions_output(inputs_dict, config, model_class): hasattr(outputs, "hidden_states"), "get_text_features() must return a BaseModelOutput with hidden_states", ) - self.assertTrue( - hasattr(outputs, "attentions"), "get_text_features() must return a BaseModelOutput with attentions" - ) + if self.has_attentions: + self.assertTrue( + hasattr(outputs, "attentions"), "get_text_features() must return a BaseModelOutput with attentions" + ) last_hidden_state = outputs.last_hidden_state if hasattr(self.model_tester, "text_model_tester"): @@ -4690,6 +4691,128 @@ def check_attentions_output(inputs_dict, config, model_class): check_attentions_output(inputs_dict, config, model_class) + def test_get_image_features(self): + def prepare_config_and_inputs_for_image_features(): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + if hasattr(self.model_tester, "vision_model_tester"): + _, inputs_dict = self.model_tester.vision_model_tester.prepare_config_and_inputs_for_common() + else: + inputs_dict = { + key: value + for key, value in inputs_dict.items() + if ("pixel" in key or "image" in key) + and "video" not in key + or key in ["spatial_shapes", "qformer_input_ids"] + } + return config, inputs_dict + + def check_hidden_states_output(inputs_dict, config, model_class): + model = model_class(copy.deepcopy(config)) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model.get_image_features(**inputs_dict) + # hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states + hidden_states = outputs.hidden_states + # breakpoint() + if hasattr(self.model_tester, "vision_model_tester"): + expected_num_layers = self.model_tester.vision_model_tester.num_hidden_layers + 1 + else: + expected_num_layers = self.model_tester.num_hidden_layers + 1 + self.assertIsNotNone(hidden_states, "hidden_states should not be None") + self.assertEqual(len(hidden_states), expected_num_layers, "Number of hidden states layers mismatch") + + def check_attentions_output(inputs_dict, config, model_class): + model = model_class(copy.deepcopy(config)) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model.get_image_features(**inputs_dict) + attentions = outputs.attentions + # model.text_model(**inputs_dict) also no attentions for aimv2 + if hasattr(self.model_tester, "vision_model_tester"): + expected_num_layers = self.model_tester.vision_model_tester.num_hidden_layers + else: + expected_num_layers = self.model_tester.num_hidden_layers + self.assertIsNotNone(attentions, "attentions should not be None") + self.assertEqual(len(attentions), expected_num_layers, "Number of attention layers mismatch") + + for model_class in self.all_model_classes: + if not hasattr(model_class, "get_image_features"): + continue + + config, inputs_dict = prepare_config_and_inputs_for_image_features() + + model = model_class(config).eval() + model = model.to(torch_device) + + torch.manual_seed(0) + outputs = model.get_image_features(**inputs_dict) + self.assertTrue(isinstance(outputs, ModelOutput), "get_image_features() must return a BaseModelOutput") + self.assertTrue( + hasattr(outputs, "last_hidden_state"), + "get_image_features() must return a BaseModelOutput with last_hidden_state", + ) + self.assertTrue( + hasattr(outputs, "pooler_output"), + "get_image_features() must return a BaseModelOutput with pooler_output", + ) + self.assertTrue( + hasattr(outputs, "hidden_states"), + "get_image_features() must return a BaseModelOutput with hidden_states", + ) + if self.has_attentions: + self.assertTrue( + hasattr(outputs, "attentions"), + "get_image_features() must return a BaseModelOutput with attentions", + ) + + # TODO: @Tom let's re-enable shape tests supporting the common formats + """ + last_hidden_state = outputs.last_hidden_state + # We're expecting (batch_size, hidden_dim, height, width) + if hasattr(self.model_tester, "vision_model_tester"): + hidden_dim = self.model_tester.vision_model_tester.hidden_dim + else: + hidden_dim = self.model_tester.hidden_size + expected_shape = (inputs_dict["pixel_values"].shape[0], hidden_dim) + self.assertEqual(len(last_hidden_state.shape), 4, "last_hidden_state should be a 4D tensor") + self.assertEqual(last_hidden_state.shape[:2], expected_shape, "last_hidden_state shape mismatch") + """ + + inputs_dict["output_hidden_states"] = True + check_hidden_states_output(inputs_dict, config, model_class) + + # check that output_hidden_states also work using config + del inputs_dict["output_hidden_states"] + config.output_hidden_states = True + for k in config.sub_configs: + if getattr(config, k) is not None: + getattr(config, k).output_hidden_states = True + + check_hidden_states_output(inputs_dict, config, model_class) + + if not self.has_attentions: + continue + + config, inputs_dict = prepare_config_and_inputs_for_image_features() + # force eager attention to support output attentions + config._attn_implementation = "eager" + inputs_dict["output_hidden_states"] = False + inputs_dict["output_attentions"] = True + check_attentions_output(inputs_dict, config, model_class) + + # check that output_attentions also work using config + del inputs_dict["output_attentions"] + config.output_attentions = True + for k in config.sub_configs: + if getattr(config, k) is not None: + getattr(config, k).output_attentions = True + + check_attentions_output(inputs_dict, config, model_class) + global_rng = random.Random() From a64634bd465af887d14be5d29a7ba9fb292d5089 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 9 Jan 2026 16:20:57 +0100 Subject: [PATCH 063/131] Resolve several test failures, progress still slow and inconsistent --- .../models/chameleon/modeling_chameleon.py | 4 + src/transformers/models/emu3/modeling_emu3.py | 6 +- src/transformers/models/emu3/modular_emu3.py | 6 +- .../modeling_ernie4_5_vl_moe.py | 276 +++++++++--------- .../modular_ernie4_5_vl_moe.py | 35 ++- .../models/janus/modeling_janus.py | 48 +++ tests/models/flava/test_modeling_flava.py | 30 +- .../florence2/test_modeling_florence2.py | 2 +- tests/models/fuyu/test_modeling_fuyu.py | 1 + tests/test_modeling_common.py | 32 +- 10 files changed, 258 insertions(+), 182 deletions(-) diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index f4a57b2e12f0..2dc7e57e0e79 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -818,6 +818,10 @@ class ChameleonVQVAE(ChameleonPreTrainedModel): "ChameleonVQVAEEncoderAttnBlock", "ChameleonVQVAEEncoderResnetBlock", ] + _can_record_outputs = { + "hidden_states": ChameleonVQVAEEncoderResnetBlock, + "attentions": ChameleonAttention, + } def __init__(self, config: ChameleonVQVAEConfig): super().__init__(config) diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index e26dcc811153..68edf5695527 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -952,6 +952,10 @@ class Emu3VQVAE(PreTrainedModel): "Emu3VQVAEResnetBlock", "Emu3VQVAEVectorQuantizer", ] + _can_record_outputs = { + "hidden_states": Emu3VQVAETemporalResnetBlock, + "attentions": Emu3VQVAEAttentionBlock, + } @torch.no_grad() def _init_weights(self, module): @@ -1408,7 +1412,7 @@ def get_image_features( pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): The tensors corresponding to the input images. """ - vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, return_dict=True) + vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, **kwargs) split_sizes = [ (height // self.vqmodel.vision_spatial_factor) * (width // self.vqmodel.vision_spatial_factor + 1) for height, width in image_sizes diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index ca27739b2fc3..8a297f468101 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -701,6 +701,10 @@ class Emu3VQVAE(PreTrainedModel): "Emu3VQVAEResnetBlock", "Emu3VQVAEVectorQuantizer", ] + _can_record_outputs = { + "hidden_states": Emu3VQVAETemporalResnetBlock, + "attentions": Emu3VQVAEAttentionBlock, + } @torch.no_grad() def _init_weights(self, module): @@ -961,7 +965,7 @@ def get_image_features( pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): The tensors corresponding to the input images. """ - vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, return_dict=True) + vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, **kwargs) split_sizes = [ (height // self.vqmodel.vision_spatial_factor) * (width // self.vqmodel.vision_spatial_factor + 1) for height, width in image_sizes diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index 13ee6891f4b7..8c7e7e0101fc 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -276,6 +276,110 @@ def forward( return attn_output, attn_weights +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb_vision( + q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + orig_q_dtype = q.dtype + orig_k_dtype = k.dtype + q, k = q.float(), k.float() + cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float() + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + q_embed = q_embed.to(orig_q_dtype) + k_embed = k_embed.to(orig_k_dtype) + return q_embed, k_embed + + +class Ernie4_5_VL_MoeVisionAttention(nn.Module): + def __init__(self, config: Ernie4_5_VL_MoeVisionConfig) -> None: + super().__init__() + self.dim = config.hidden_size + self.num_heads = config.num_heads + self.head_dim = self.dim // self.num_heads + self.num_key_value_groups = 1 # needed for eager attention + self.qkv = nn.Linear(self.dim, self.dim * 3, bias=True) + self.proj = nn.Linear(self.dim, self.dim) + self.scaling = self.head_dim**-0.5 + self.config = config + self.attention_dropout = 0.0 + self.is_causal = False + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + rotary_pos_emb: Optional[torch.Tensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + **kwargs, + ) -> torch.Tensor: + seq_length = hidden_states.shape[0] + query_states, key_states, value_states = ( + self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) + ) + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin) + + query_states = query_states.transpose(0, 1).unsqueeze(0) + key_states = key_states.transpose(0, 1).unsqueeze(0) + value_states = value_states.transpose(0, 1).unsqueeze(0) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + if "flash" in self.config._attn_implementation: + # Flash Attention: Use cu_seqlens for variable length attention + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() + attn_output, _ = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask=None, + scaling=self.scaling, + dropout=0.0 if not self.training else self.attention_dropout, + cu_seq_lens_q=cu_seqlens, + cu_seq_lens_k=cu_seqlens, + max_length_q=max_seqlen, + max_length_k=max_seqlen, + is_causal=False, + **kwargs, + ) + else: + # Other implementations: Process each chunk separately + lengths = cu_seqlens[1:] - cu_seqlens[:-1] + splits = [ + torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states) + ] + + attn_outputs = [ + attention_interface( + self, + q, + k, + v, + attention_mask=None, + scaling=self.scaling, + dropout=0.0 if not self.training else self.attention_dropout, + is_causal=False, + **kwargs, + )[0] + for q, k, v in zip(*splits) + ] + attn_output = torch.cat(attn_outputs, dim=1) + + attn_output = attn_output.reshape(seq_length, -1).contiguous() + attn_output = self.proj(attn_output) + return attn_output + + @use_kernel_forward_from_hub("RMSNorm") class Ernie4_5_VL_MoeRMSNorm(nn.Module): def __init__(self, hidden_size, eps=1e-6): @@ -549,6 +653,38 @@ def forward( return hidden_states +class Ernie4_5_VL_MoeVisionBlock(GradientCheckpointingLayer): + def __init__(self, config) -> None: + super().__init__() + + self.norm1 = nn.LayerNorm(config.hidden_size, config.rms_norm_eps) + self.norm2 = nn.LayerNorm(config.hidden_size, config.rms_norm_eps) + self.attn = Ernie4_5_VL_MoeVisionAttention(config=config) + self.mlp = Ernie4_5VLVisionMLP( + dim=config.hidden_size, + hidden_dim=config.intermediate_size, + hidden_act=config.hidden_act, + ) + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + rotary_pos_emb: Optional[torch.Tensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + **kwargs, + ) -> torch.Tensor: + hidden_states = hidden_states + self.attn( + self.norm1(hidden_states), + cu_seqlens=cu_seqlens, + rotary_pos_emb=rotary_pos_emb, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) + return hidden_states + + @auto_docstring class Ernie4_5_VL_MoePreTrainedModel(PreTrainedModel): config: Ernie4_5_VL_MoeConfig @@ -564,8 +700,8 @@ class Ernie4_5_VL_MoePreTrainedModel(PreTrainedModel): _can_record_outputs = { "router_logits": OutputRecorder(Ernie4_5_VL_MoeMoeBlock, index=1), - "hidden_states": Ernie4_5_VL_MoeDecoderLayer, - "attentions": Ernie4_5_VL_MoeTextAttention, + "hidden_states": [Ernie4_5_VL_MoeDecoderLayer, Ernie4_5_VL_MoeVisionBlock], + "attentions": [Ernie4_5_VL_MoeTextAttention, Ernie4_5_VL_MoeVisionAttention], } _keep_in_fp32_modules_strict = ["gate.weight", "moe_statics"] @@ -733,142 +869,6 @@ def forward(self, seqlen: int) -> torch.Tensor: return freqs -def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - -def apply_rotary_pos_emb_vision( - q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor -) -> tuple[torch.Tensor, torch.Tensor]: - orig_q_dtype = q.dtype - orig_k_dtype = k.dtype - q, k = q.float(), k.float() - cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float() - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - q_embed = q_embed.to(orig_q_dtype) - k_embed = k_embed.to(orig_k_dtype) - return q_embed, k_embed - - -class Ernie4_5_VL_MoeVisionAttention(nn.Module): - def __init__(self, config: Ernie4_5_VL_MoeVisionConfig) -> None: - super().__init__() - self.dim = config.hidden_size - self.num_heads = config.num_heads - self.head_dim = self.dim // self.num_heads - self.num_key_value_groups = 1 # needed for eager attention - self.qkv = nn.Linear(self.dim, self.dim * 3, bias=True) - self.proj = nn.Linear(self.dim, self.dim) - self.scaling = self.head_dim**-0.5 - self.config = config - self.attention_dropout = 0.0 - self.is_causal = False - - def forward( - self, - hidden_states: torch.Tensor, - cu_seqlens: torch.Tensor, - rotary_pos_emb: Optional[torch.Tensor] = None, - position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, - **kwargs, - ) -> torch.Tensor: - seq_length = hidden_states.shape[0] - query_states, key_states, value_states = ( - self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) - ) - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin) - - query_states = query_states.transpose(0, 1).unsqueeze(0) - key_states = key_states.transpose(0, 1).unsqueeze(0) - value_states = value_states.transpose(0, 1).unsqueeze(0) - - attention_interface: Callable = eager_attention_forward - if self.config._attn_implementation != "eager": - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - - if "flash" in self.config._attn_implementation: - # Flash Attention: Use cu_seqlens for variable length attention - max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() - attn_output, _ = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask=None, - scaling=self.scaling, - dropout=0.0 if not self.training else self.attention_dropout, - cu_seq_lens_q=cu_seqlens, - cu_seq_lens_k=cu_seqlens, - max_length_q=max_seqlen, - max_length_k=max_seqlen, - is_causal=False, - **kwargs, - ) - else: - # Other implementations: Process each chunk separately - lengths = cu_seqlens[1:] - cu_seqlens[:-1] - splits = [ - torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states) - ] - - attn_outputs = [ - attention_interface( - self, - q, - k, - v, - attention_mask=None, - scaling=self.scaling, - dropout=0.0 if not self.training else self.attention_dropout, - is_causal=False, - **kwargs, - )[0] - for q, k, v in zip(*splits) - ] - attn_output = torch.cat(attn_outputs, dim=1) - - attn_output = attn_output.reshape(seq_length, -1).contiguous() - attn_output = self.proj(attn_output) - return attn_output - - -class Ernie4_5_VL_MoeVisionBlock(GradientCheckpointingLayer): - def __init__(self, config) -> None: - super().__init__() - - self.norm1 = nn.LayerNorm(config.hidden_size, config.rms_norm_eps) - self.norm2 = nn.LayerNorm(config.hidden_size, config.rms_norm_eps) - self.attn = Ernie4_5_VL_MoeVisionAttention(config=config) - self.mlp = Ernie4_5VLVisionMLP( - dim=config.hidden_size, - hidden_dim=config.intermediate_size, - hidden_act=config.hidden_act, - ) - - def forward( - self, - hidden_states: torch.Tensor, - cu_seqlens: torch.Tensor, - rotary_pos_emb: Optional[torch.Tensor] = None, - position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, - **kwargs, - ) -> torch.Tensor: - hidden_states = hidden_states + self.attn( - self.norm1(hidden_states), - cu_seqlens=cu_seqlens, - rotary_pos_emb=rotary_pos_emb, - position_embeddings=position_embeddings, - **kwargs, - ) - hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) - return hidden_states - - @auto_docstring class Ernie4_5_VL_MoeVisionTransformerPretrainedModel(Ernie4_5_VL_MoePreTrainedModel): config: Ernie4_5_VL_MoeVisionConfig diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index 78d1c0564f13..b56b40e8dcb9 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -81,6 +81,7 @@ Qwen2_5_VisionRotaryEmbedding, Qwen2_5_VLModel, Qwen2_5_VLPreTrainedModel, + Qwen2_5_VLVisionAttention, Qwen2_5_VLVisionBlock, ) from ..qwen2_vl.configuration_qwen2_vl import Qwen2VLVisionConfig @@ -526,6 +527,10 @@ class Ernie4_5_VL_MoeTextAttention(Ernie4_5_MoeAttention): pass +class Ernie4_5_VL_MoeVisionAttention(Qwen2_5_VLVisionAttention): + pass + + class Ernie4_5_VL_MoeRMSNorm(Ernie4_5_MoeRMSNorm): pass @@ -691,13 +696,26 @@ def forward( return hidden_states +class Ernie4_5_VL_MoeVisionBlock(Qwen2_5_VLVisionBlock): + def __init__(self, config) -> None: + super().__init__(config, None) + + self.norm1 = nn.LayerNorm(config.hidden_size, config.rms_norm_eps) + self.norm2 = nn.LayerNorm(config.hidden_size, config.rms_norm_eps) + self.mlp = Ernie4_5VLVisionMLP( + dim=config.hidden_size, + hidden_dim=config.intermediate_size, + hidden_act=config.hidden_act, + ) + + class Ernie4_5_VL_MoePreTrainedModel(Qwen2_5_VLPreTrainedModel): _can_compile_fullgraph = False _can_record_outputs = { "router_logits": OutputRecorder(Ernie4_5_VL_MoeMoeBlock, index=1), - "hidden_states": Ernie4_5_VL_MoeDecoderLayer, - "attentions": Ernie4_5_VL_MoeTextAttention, + "hidden_states": [Ernie4_5_VL_MoeDecoderLayer, Ernie4_5_VL_MoeVisionBlock], + "attentions": [Ernie4_5_VL_MoeTextAttention, Ernie4_5_VL_MoeVisionAttention], } _keep_in_fp32_modules_strict = ["gate.weight", "moe_statics"] @@ -833,19 +851,6 @@ class Ernie4_5_VL_MoeVisionRotaryEmbedding(Qwen2_5_VisionRotaryEmbedding): pass -class Ernie4_5_VL_MoeVisionBlock(Qwen2_5_VLVisionBlock): - def __init__(self, config) -> None: - super().__init__(config, None) - - self.norm1 = nn.LayerNorm(config.hidden_size, config.rms_norm_eps) - self.norm2 = nn.LayerNorm(config.hidden_size, config.rms_norm_eps) - self.mlp = Ernie4_5VLVisionMLP( - dim=config.hidden_size, - hidden_dim=config.intermediate_size, - hidden_act=config.hidden_act, - ) - - class Ernie4_5_VL_MoeVisionTransformerPretrainedModel(Qwen2VisionTransformerPretrainedModel): def __init__(self, config) -> None: super().__init__(config) diff --git a/src/transformers/models/janus/modeling_janus.py b/src/transformers/models/janus/modeling_janus.py index 252132f67f13..70d1a5e19316 100644 --- a/src/transformers/models/janus/modeling_janus.py +++ b/src/transformers/models/janus/modeling_janus.py @@ -969,6 +969,50 @@ class JanusVQVAEModelOutput(BaseModelOutputWithPooling): embedding_loss: Optional[torch.FloatTensor] = None +class JanusVQVAEEncoderResnetBlock(nn.Module): + def __init__( + self, + config, + in_channels, + out_channels=None, + conv_shortcut=False, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = in_channels if out_channels is None else out_channels + self.use_conv_shortcut = conv_shortcut + + self.norm1 = torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + self.conv1 = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + self.norm2 = torch.nn.GroupNorm(num_groups=32, num_channels=out_channels, eps=1e-6, affine=True) + self.dropout = torch.nn.Dropout(config.dropout) + self.conv2 = torch.nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1) + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + self.conv_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + else: + self.nin_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, hidden_states): + residual = hidden_states + hidden_states = self.norm1(hidden_states) + hidden_states *= torch.sigmoid(hidden_states) + hidden_states = self.conv1(hidden_states) + + hidden_states = self.norm2(hidden_states) + hidden_states *= torch.sigmoid(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.conv2(hidden_states) + + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + residual = self.conv_shortcut(residual) + else: + residual = self.nin_shortcut(residual) + + return residual + hidden_states + + @auto_docstring( custom_intro=""" The VQ-VAE model used in Janus for encoding/decoding images into discrete tokens. @@ -984,6 +1028,10 @@ class JanusVQVAE(JanusPreTrainedModel): "JanusVQVAEResnetBlock", "JanusVQVAEVectorQuantizer", ] + _can_record_outputs = { + "hidden_states": JanusVQVAEEncoderResnetBlock, + "attentions": JanusAttention, + } main_input_name = "pixel_values" def __init__(self, config: JanusVQVAEConfig): diff --git a/tests/models/flava/test_modeling_flava.py b/tests/models/flava/test_modeling_flava.py index 501208452474..d72fc890d4f2 100644 --- a/tests/models/flava/test_modeling_flava.py +++ b/tests/models/flava/test_modeling_flava.py @@ -777,7 +777,7 @@ def __init__( image_codebook_kwargs = {} self.parent = parent - self.image_model_tester = FlavaImageModelTester(parent, **image_kwargs) + self.vision_model_tester = FlavaImageModelTester(parent, **image_kwargs) self.text_model_tester = FlavaTextModelTester(parent, **text_kwargs) self.multimodal_model_tester = FlavaMultimodalModelTester(parent, **multimodal_kwargs) self.image_codebook_tester = FlavaImageCodebookTester(parent, **image_codebook_kwargs) @@ -793,7 +793,7 @@ def test_config(self): self.config_tester.run_common_tests() def prepare_config_and_inputs_for_common(self): - _, pixel_values, bool_masked_pos = self.image_model_tester.prepare_config_and_inputs() + _, pixel_values, bool_masked_pos = self.vision_model_tester.prepare_config_and_inputs() _, input_ids, token_type_ids, attention_mask = self.text_model_tester.prepare_config_and_inputs() config = self.get_config() @@ -808,7 +808,7 @@ def prepare_config_and_inputs_for_common(self): def get_config(self): return FlavaConfig( - image_config=self.image_model_tester.get_config(), + image_config=self.vision_model_tester.get_config(), text_config=self.text_model_tester.get_config(), multimodal_config=self.multimodal_model_tester.get_config(), image_codebook_config=self.image_codebook_tester.get_config(), @@ -833,14 +833,14 @@ def _test_model(self, config, inputs, test_image=False, test_text=False): pixel_values=inputs["pixel_values"] if test_image else None, bool_masked_pos=inputs["bool_masked_pos"] if test_image else None, ) - image_size = (self.image_model_tester.image_size, self.image_model_tester.image_size) - patch_size = (self.image_model_tester.patch_size, self.image_model_tester.patch_size) + image_size = (self.vision_model_tester.image_size, self.vision_model_tester.image_size) + patch_size = (self.vision_model_tester.patch_size, self.vision_model_tester.patch_size) num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0]) if test_image: self.parent.assertEqual( result.image_embeddings.shape, - (self.image_model_tester.batch_size, num_patches + 1, self.image_model_tester.hidden_size), + (self.vision_model_tester.batch_size, num_patches + 1, self.vision_model_tester.hidden_size), ) else: self.parent.assertIsNone(result.image_embeddings) @@ -942,7 +942,7 @@ class FlavaForPreTrainingTester(FlavaModelTester): model_class = FlavaForPreTraining def prepare_config_and_inputs_for_common(self): - _, pixel_values, bool_masked_pos = self.image_model_tester.prepare_config_and_inputs() + _, pixel_values, bool_masked_pos = self.vision_model_tester.prepare_config_and_inputs() _, input_ids, token_type_ids, attention_mask = self.text_model_tester.prepare_config_and_inputs() config = self.get_config() @@ -952,7 +952,7 @@ def prepare_config_and_inputs_for_common(self): mlm_labels[:, :] = config.ce_ignore_index mlm_labels[:, 1:3] = input_ids[:, 1:3] mim_labels = torch.randint( - 0, self.image_model_tester.vocab_size, bool_masked_pos.size(), device=bool_masked_pos.device + 0, self.vision_model_tester.vocab_size, bool_masked_pos.size(), device=bool_masked_pos.device ).long() mim_labels[bool_masked_pos.ne(True)] = config.ce_ignore_index itm_labels = torch.ones(mlm_labels.size(0), device=bool_masked_pos.device).long() @@ -985,14 +985,14 @@ def _test_model(self, config, inputs, test_image=False, test_text=False): itm_labels=inputs["itm_labels"], return_loss=inputs["return_loss"], ) - image_size = (self.image_model_tester.image_size, self.image_model_tester.image_size) - patch_size = (self.image_model_tester.patch_size, self.image_model_tester.patch_size) + image_size = (self.vision_model_tester.image_size, self.vision_model_tester.image_size) + patch_size = (self.vision_model_tester.patch_size, self.vision_model_tester.patch_size) num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0]) if test_image: self.parent.assertEqual( result.image_embeddings.shape, - (self.image_model_tester.batch_size, num_patches + 1, self.image_model_tester.hidden_size), + (self.vision_model_tester.batch_size, num_patches + 1, self.vision_model_tester.hidden_size), ) if not test_text: self.parent.assertEqual( @@ -1001,7 +1001,7 @@ def _test_model(self, config, inputs, test_image=False, test_text=False): ) self.parent.assertEqual( result.mim_logits.shape, - (inputs["bool_masked_pos"].sum().item(), self.image_model_tester.vocab_size), + (inputs["bool_masked_pos"].sum().item(), self.vision_model_tester.vocab_size), ) else: @@ -1050,15 +1050,15 @@ def _test_model(self, config, inputs, test_image=False, test_text=False): ) self.parent.assertEqual( result.mmm_image_logits.shape, - (inputs["bool_masked_pos"].sum().item(), self.image_model_tester.vocab_size), + (inputs["bool_masked_pos"].sum().item(), self.vision_model_tester.vocab_size), ) self.parent.assertEqual( result.contrastive_logits_per_image.shape, - (self.image_model_tester.batch_size, self.text_model_tester.batch_size), + (self.vision_model_tester.batch_size, self.text_model_tester.batch_size), ) self.parent.assertEqual( result.contrastive_logits_per_text.shape, - (self.text_model_tester.batch_size, self.image_model_tester.batch_size), + (self.text_model_tester.batch_size, self.vision_model_tester.batch_size), ) for item in [ diff --git a/tests/models/florence2/test_modeling_florence2.py b/tests/models/florence2/test_modeling_florence2.py index c9ee2e46f873..8fa57470e2d0 100644 --- a/tests/models/florence2/test_modeling_florence2.py +++ b/tests/models/florence2/test_modeling_florence2.py @@ -240,7 +240,7 @@ class Florence2ForConditionalGenerationModelTest( else {} ) - test_attention_outputs = False + has_attentions = False _is_composite = True def setUp(self): diff --git a/tests/models/fuyu/test_modeling_fuyu.py b/tests/models/fuyu/test_modeling_fuyu.py index 732014fb9f62..c5d5a075c459 100644 --- a/tests/models/fuyu/test_modeling_fuyu.py +++ b/tests/models/fuyu/test_modeling_fuyu.py @@ -171,6 +171,7 @@ class FuyuModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin test_cpu_offload = False test_disk_offload = False + test_get_image_features = False def setUp(self): self.model_tester = FuyuModelTester(self) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 707581f47d04..0b4ee31f4a2c 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4633,7 +4633,8 @@ def check_attentions_output(inputs_dict, config, model_class): model = model.to(torch_device) torch.manual_seed(0) - outputs = model.get_text_features(**inputs_dict) + with torch.no_grad(): + outputs = model.get_text_features(**inputs_dict) self.assertTrue(isinstance(outputs, ModelOutput), "get_text_features() must return a BaseModelOutput") self.assertTrue( hasattr(outputs, "last_hidden_state"), @@ -4706,6 +4707,21 @@ def prepare_config_and_inputs_for_image_features(): } return config, inputs_dict + def get_expected_num_layers(model_tester): + if hasattr(model_tester, "vision_model_tester"): + return get_expected_num_layers(model_tester.vision_model_tester) + elif ( + hasattr(model_tester, "vision_config") + and isinstance(model_tester.vision_config, dict) + and "num_hidden_layers" in model_tester.vision_config + ): + return model_tester.vision_config["num_hidden_layers"] + + if hasattr(model_tester, "expected_num_hidden_layers"): + return model_tester.expected_num_hidden_layers - 1 + elif hasattr(model_tester, "num_hidden_layers"): + return model_tester.num_hidden_layers + def check_hidden_states_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) model.to(torch_device) @@ -4715,11 +4731,7 @@ def check_hidden_states_output(inputs_dict, config, model_class): outputs = model.get_image_features(**inputs_dict) # hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states hidden_states = outputs.hidden_states - # breakpoint() - if hasattr(self.model_tester, "vision_model_tester"): - expected_num_layers = self.model_tester.vision_model_tester.num_hidden_layers + 1 - else: - expected_num_layers = self.model_tester.num_hidden_layers + 1 + expected_num_layers = get_expected_num_layers(self.model_tester) + 1 self.assertIsNotNone(hidden_states, "hidden_states should not be None") self.assertEqual(len(hidden_states), expected_num_layers, "Number of hidden states layers mismatch") @@ -4732,10 +4744,7 @@ def check_attentions_output(inputs_dict, config, model_class): outputs = model.get_image_features(**inputs_dict) attentions = outputs.attentions # model.text_model(**inputs_dict) also no attentions for aimv2 - if hasattr(self.model_tester, "vision_model_tester"): - expected_num_layers = self.model_tester.vision_model_tester.num_hidden_layers - else: - expected_num_layers = self.model_tester.num_hidden_layers + expected_num_layers = get_expected_num_layers(self.model_tester) self.assertIsNotNone(attentions, "attentions should not be None") self.assertEqual(len(attentions), expected_num_layers, "Number of attention layers mismatch") @@ -4749,7 +4758,8 @@ def check_attentions_output(inputs_dict, config, model_class): model = model.to(torch_device) torch.manual_seed(0) - outputs = model.get_image_features(**inputs_dict) + with torch.no_grad(): + outputs = model.get_image_features(**inputs_dict) self.assertTrue(isinstance(outputs, ModelOutput), "get_image_features() must return a BaseModelOutput") self.assertTrue( hasattr(outputs, "last_hidden_state"), From 5ad8ca520a98cae4ed735b3a4af827859e9416ff Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 12 Jan 2026 13:17:37 +0100 Subject: [PATCH 064/131] Split up get_..._features tests more, should be simpler to disable/customize specific parts per arch --- .../models/chameleon/modeling_chameleon.py | 2 +- .../cohere2_vision/modeling_cohere2_vision.py | 2 +- .../cohere2_vision/modular_cohere2_vision.py | 2 +- tests/models/blip_2/test_modeling_blip_2.py | 19 ++ .../chameleon/test_modeling_chameleon.py | 42 +++ tests/models/fuyu/test_modeling_fuyu.py | 4 +- tests/test_modeling_common.py | 252 ++++++++++-------- 7 files changed, 209 insertions(+), 114 deletions(-) diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 3e1debccd152..69b8bff13577 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -819,7 +819,7 @@ class ChameleonVQVAE(ChameleonPreTrainedModel): ] _can_record_outputs = { "hidden_states": ChameleonVQVAEEncoderResnetBlock, - "attentions": ChameleonAttention, + "attentions": [ChameleonAttention, ChameleonVQVAEEncoderAttnBlock], } def __init__(self, config: ChameleonVQVAEConfig): diff --git a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py index a0205be87128..980db148c0b5 100644 --- a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py @@ -179,7 +179,7 @@ def get_image_features( image_features (List[`torch.Tensor`]): List of image feature tensor, each contains all the visual feature of all patches and are of shape `(num_patches, image_length, embed_dim)`). """ - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, **kwargs) selected_image_feature = image_outputs.last_hidden_state image_outputs.pooler_output = self.multi_modal_projector(selected_image_feature) diff --git a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py index 9ee057ec952d..68f77c743e55 100644 --- a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py @@ -110,7 +110,7 @@ def get_image_features( image_features (List[`torch.Tensor`]): List of image feature tensor, each contains all the visual feature of all patches and are of shape `(num_patches, image_length, embed_dim)`). """ - image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) + image_outputs = self.vision_tower(pixel_values, **kwargs) selected_image_feature = image_outputs.last_hidden_state image_outputs.pooler_output = self.multi_modal_projector(selected_image_feature) diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index 55deab49b469..02c91307351a 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -686,6 +686,25 @@ def get_config(self): is_encoder_decoder=True, ) + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + ( + config, + input_ids, + decoder_input_ids, + attention_mask, + decoder_attention_mask, + labels, + ) = config_and_inputs + inputs_dict = { + "input_ids": input_ids, + "attention_mask": attention_mask, + "decoder_input_ids": decoder_input_ids, + "decoder_attention_mask": decoder_attention_mask, + "labels": labels, + } + return config, inputs_dict + # this model tester uses an encoder-decoder language model (T5) class Blip2ModelTester: diff --git a/tests/models/chameleon/test_modeling_chameleon.py b/tests/models/chameleon/test_modeling_chameleon.py index 50050dd3b469..f11cb57d5148 100644 --- a/tests/models/chameleon/test_modeling_chameleon.py +++ b/tests/models/chameleon/test_modeling_chameleon.py @@ -75,7 +75,11 @@ def __init__( pad_token_id=0, vq_num_embeds=5, vq_embed_dim=5, + vq_resolution=512, vq_channel_multiplier=[1, 2], + vq_num_res_blocks=2, + vq_attn_resolutions=None, + vq_attn_type="vanilla", vq_img_token_start_id=10, # has to be less than vocab size when added with vq_num_embeds scope=None, ): @@ -105,7 +109,11 @@ def __init__( self.scope = scope self.vq_num_embeds = vq_num_embeds self.vq_embed_dim = vq_embed_dim + self.vq_resolution = vq_resolution self.vq_channel_multiplier = vq_channel_multiplier + self.vq_num_res_blocks = vq_num_res_blocks + self.vq_attn_resolutions = vq_attn_resolutions + self.vq_attn_type = vq_attn_type self.vq_img_token_start_id = vq_img_token_start_id def prepare_config_and_inputs(self): @@ -166,9 +174,13 @@ def get_vq_config(self): "embed_dim": self.vq_embed_dim, "num_embeddings": self.vq_num_embeds, "latent_channels": self.vq_embed_dim, + "resolution": self.vq_resolution, "in_channels": 3, "base_channels": 32, # we have a GroupNorm of 32 groups, so can't do less "channel_multiplier": self.vq_channel_multiplier, + "num_res_blocks": self.vq_num_res_blocks, + "attn_resolutions": self.vq_attn_resolutions, + "attn_type": self.vq_attn_type, } def create_and_check_model(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): @@ -220,6 +232,18 @@ def test_model(self): def test_batching_equivalence(self): pass + @unittest.skip("Skip get_image_features tests as those are tested via ChameleonVision2SeqModelTest instead") + def test_get_image_features_output(self): + pass + + @unittest.skip("Skip get_image_features tests as those are tested via ChameleonVision2SeqModelTest instead") + def test_get_image_features_hidden_states(self): + pass + + @unittest.skip("Skip get_image_features tests as those are tested via ChameleonVision2SeqModelTest instead") + def test_get_image_features_attentions(self): + pass + class ChameleonVision2SeqModelTester(ChameleonModelTester): def __init__(self, parent, image_size=10, **kwargs): @@ -323,6 +347,24 @@ def test_mismatching_num_image_tokens(self): pixel_values = torch.cat([pixel_values, pixel_values], dim=0) _ = model(input_ids=input_ids, pixel_values=pixel_values) + def _image_features_get_expected_num_hidden_states(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + # The number of ChameleonVQVAEEncoderResnetBlock instances, plus 1 for before the block + return len(model_tester.vq_channel_multiplier) * model_tester.vq_num_res_blocks + 3 + + def _image_features_get_expected_num_attentions(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + # The number of ChameleonVQVAEEncoderAttnBlock instances + if ( + model_tester.vq_attn_resolutions + and model_tester.vq_resolution in model_tester.vq_attn_resolutions + and model_tester.vq_attn_type == "vanilla" + ): + return len(model_tester.vq_channel_multiplier) * model_tester.vq_num_res_blocks + 1 + return 1 + @require_torch class ChameleonIntegrationTest(unittest.TestCase): diff --git a/tests/models/fuyu/test_modeling_fuyu.py b/tests/models/fuyu/test_modeling_fuyu.py index c5d5a075c459..4b35f222ef10 100644 --- a/tests/models/fuyu/test_modeling_fuyu.py +++ b/tests/models/fuyu/test_modeling_fuyu.py @@ -171,7 +171,9 @@ class FuyuModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin test_cpu_offload = False test_disk_offload = False - test_get_image_features = False + test_get_image_features_output = False + test_get_image_features_hidden_states = False + test_get_image_features_attentions = False def setUp(self): self.model_tester = FuyuModelTester(self) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 4594b14a5081..69d097b0adbe 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4569,56 +4569,74 @@ def test_can_load_from_already_mapped_keys(self): # Make sure both saved state_dict are identical self.assertTrue(compare_state_dicts(model.state_dict(), model_reloaded.state_dict())) - def test_get_text_features(self): - def prepare_config_and_inputs_for_text_features(): - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - if hasattr(self.model_tester, "text_model_tester"): - _, inputs_dict = self.model_tester.text_model_tester.prepare_config_and_inputs_for_common() - else: - inputs_dict = { - key: value - for key, value in inputs_dict.items() - if key in ["input_ids", "token_type_ids", "attention_mask"] - } - return config, inputs_dict + def _text_features_prepare_config_and_inputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + if hasattr(self.model_tester, "text_model_tester"): + _, inputs_dict = self.model_tester.text_model_tester.prepare_config_and_inputs_for_common() + else: + inputs_dict = { + key: value + for key, value in inputs_dict.items() + if key in ["input_ids", "token_type_ids", "attention_mask"] + } + return config, inputs_dict - def check_hidden_states_output(inputs_dict, config, model_class): - model = model_class(copy.deepcopy(config)) - model.to(torch_device) - model.eval() + def _image_features_prepare_config_and_inputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + if hasattr(self.model_tester, "vision_model_tester"): + _, inputs_dict = self.model_tester.vision_model_tester.prepare_config_and_inputs_for_common() + else: + inputs_dict = { + key: value + for key, value in inputs_dict.items() + if ("pixel" in key or "image" in key) + and "video" not in key + or key in ["spatial_shapes", "qformer_input_ids"] + } + return config, inputs_dict + + def _text_features_get_expected_num_attentions(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + + if hasattr(model_tester, "text_model_tester"): + return self._text_features_get_expected_num_attentions(model_tester.text_model_tester) + if hasattr(model_tester, "expected_num_hidden_layers"): + return model_tester.expected_num_hidden_layers - 1 + if hasattr(model_tester, "num_hidden_layers"): + return model_tester.num_hidden_layers + raise ValueError("Cannot determine the expected number of layers for text features") + + def _text_features_get_expected_num_hidden_states(self, model_tester=None): + return self._text_features_get_expected_num_attentions(model_tester) + 1 + + def _image_features_get_expected_num_attentions(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + if hasattr(model_tester, "vision_model_tester"): + return self._image_features_get_expected_num_attentions(model_tester.vision_model_tester) + elif ( + hasattr(model_tester, "vision_config") + and isinstance(model_tester.vision_config, dict) + and "num_hidden_layers" in model_tester.vision_config + ): + return model_tester.vision_config["num_hidden_layers"] - with torch.no_grad(): - outputs = model.get_text_features(**inputs_dict) - # hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states - hidden_states = outputs.hidden_states - if hasattr(self.model_tester, "text_model_tester"): - expected_num_layers = self.model_tester.text_model_tester.num_hidden_layers + 1 - else: - expected_num_layers = self.model_tester.num_hidden_layers + 1 - self.assertIsNotNone(hidden_states, "hidden_states should not be None") - self.assertEqual(len(hidden_states), expected_num_layers, "Number of hidden states layers mismatch") + if hasattr(model_tester, "expected_num_hidden_layers"): + return model_tester.expected_num_hidden_layers - 1 + elif hasattr(model_tester, "num_hidden_layers"): + return model_tester.num_hidden_layers + raise ValueError("Cannot determine the expected number of layers for image features") - def check_attentions_output(inputs_dict, config, model_class): - model = model_class(copy.deepcopy(config)) - model.to(torch_device) - model.eval() - - with torch.no_grad(): - outputs = model.get_text_features(**inputs_dict) - attentions = outputs.attentions - # model.text_model(**inputs_dict) also no attentions for aimv2 - if hasattr(self.model_tester, "text_model_tester"): - expected_num_layers = self.model_tester.text_model_tester.num_hidden_layers - else: - expected_num_layers = self.model_tester.num_hidden_layers - self.assertIsNotNone(attentions, "attentions should not be None") - self.assertEqual(len(attentions), expected_num_layers, "Number of attention layers mismatch") + def _image_features_get_expected_num_hidden_states(self, model_tester=None): + return self._image_features_get_expected_num_attentions(model_tester) + 1 + def test_get_text_features_output(self): for model_class in self.all_model_classes: if not hasattr(model_class, "get_text_features"): continue - config, inputs_dict = prepare_config_and_inputs_for_text_features() + config, inputs_dict = self._text_features_prepare_config_and_inputs() model = model_class(config).eval() model = model.to(torch_device) @@ -4644,13 +4662,25 @@ def check_attentions_output(inputs_dict, config, model_class): hasattr(outputs, "attentions"), "get_text_features() must return a BaseModelOutput with attentions" ) - last_hidden_state = outputs.last_hidden_state - if hasattr(self.model_tester, "text_model_tester"): - expected_hidden_size = self.model_tester.text_model_tester.hidden_size - else: - expected_hidden_size = self.model_tester.hidden_size - expected_shape = (*inputs_dict["input_ids"].shape, expected_hidden_size) - self.assertEqual(last_hidden_state.shape, expected_shape, "last_hidden_state shape mismatch") + def test_get_text_features_hidden_states(self): + def check_hidden_states_output(inputs_dict, config, model_class): + model = model_class(copy.deepcopy(config)) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model.get_text_features(**inputs_dict) + # hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states + hidden_states = outputs.hidden_states + expected_num_hidden_states = self._text_features_get_expected_num_hidden_states() + self.assertIsNotNone(hidden_states, "hidden_states should not be None") + self.assertEqual(len(hidden_states), expected_num_hidden_states, "Number of hidden states layers mismatch") + + for model_class in self.all_model_classes: + if not hasattr(model_class, "get_text_features"): + continue + + config, inputs_dict = self._text_features_prepare_config_and_inputs() inputs_dict["output_hidden_states"] = True check_hidden_states_output(inputs_dict, config, model_class) @@ -4664,10 +4694,28 @@ def check_attentions_output(inputs_dict, config, model_class): check_hidden_states_output(inputs_dict, config, model_class) - if not self.has_attentions: + def test_get_text_features_attentions(self): + def check_attentions_output(inputs_dict, config, model_class): + model = model_class(copy.deepcopy(config)) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model.get_text_features(**inputs_dict) + attentions = outputs.attentions + # model.text_model(**inputs_dict) also no attentions for aimv2 + expected_num_attentions = self._text_features_get_expected_num_attentions() + self.assertIsNotNone(attentions, "attentions should not be None") + self.assertEqual(len(attentions), expected_num_attentions, "Number of attention layers mismatch") + + if not self.has_attentions: + return + + for model_class in self.all_model_classes: + if not hasattr(model_class, "get_text_features"): continue - config, inputs_dict = prepare_config_and_inputs_for_text_features() + config, inputs_dict = self._text_features_prepare_config_and_inputs() # force eager attention to support output attentions config._attn_implementation = "eager" inputs_dict["output_hidden_states"] = False @@ -4683,67 +4731,12 @@ def check_attentions_output(inputs_dict, config, model_class): check_attentions_output(inputs_dict, config, model_class) - def test_get_image_features(self): - def prepare_config_and_inputs_for_image_features(): - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - if hasattr(self.model_tester, "vision_model_tester"): - _, inputs_dict = self.model_tester.vision_model_tester.prepare_config_and_inputs_for_common() - else: - inputs_dict = { - key: value - for key, value in inputs_dict.items() - if ("pixel" in key or "image" in key) - and "video" not in key - or key in ["spatial_shapes", "qformer_input_ids"] - } - return config, inputs_dict - - def get_expected_num_layers(model_tester): - if hasattr(model_tester, "vision_model_tester"): - return get_expected_num_layers(model_tester.vision_model_tester) - elif ( - hasattr(model_tester, "vision_config") - and isinstance(model_tester.vision_config, dict) - and "num_hidden_layers" in model_tester.vision_config - ): - return model_tester.vision_config["num_hidden_layers"] - - if hasattr(model_tester, "expected_num_hidden_layers"): - return model_tester.expected_num_hidden_layers - 1 - elif hasattr(model_tester, "num_hidden_layers"): - return model_tester.num_hidden_layers - - def check_hidden_states_output(inputs_dict, config, model_class): - model = model_class(copy.deepcopy(config)) - model.to(torch_device) - model.eval() - - with torch.no_grad(): - outputs = model.get_image_features(**inputs_dict) - # hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states - hidden_states = outputs.hidden_states - expected_num_layers = get_expected_num_layers(self.model_tester) + 1 - self.assertIsNotNone(hidden_states, "hidden_states should not be None") - self.assertEqual(len(hidden_states), expected_num_layers, "Number of hidden states layers mismatch") - - def check_attentions_output(inputs_dict, config, model_class): - model = model_class(copy.deepcopy(config)) - model.to(torch_device) - model.eval() - - with torch.no_grad(): - outputs = model.get_image_features(**inputs_dict) - attentions = outputs.attentions - # model.text_model(**inputs_dict) also no attentions for aimv2 - expected_num_layers = get_expected_num_layers(self.model_tester) - self.assertIsNotNone(attentions, "attentions should not be None") - self.assertEqual(len(attentions), expected_num_layers, "Number of attention layers mismatch") - + def test_get_image_features_output(self): for model_class in self.all_model_classes: if not hasattr(model_class, "get_image_features"): continue - config, inputs_dict = prepare_config_and_inputs_for_image_features() + config, inputs_dict = self._image_features_prepare_config_and_inputs() model = model_class(config).eval() model = model.to(torch_device) @@ -4783,6 +4776,26 @@ def check_attentions_output(inputs_dict, config, model_class): self.assertEqual(last_hidden_state.shape[:2], expected_shape, "last_hidden_state shape mismatch") """ + def test_get_image_features_hidden_states(self): + def check_hidden_states_output(inputs_dict, config, model_class): + model = model_class(copy.deepcopy(config)) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model.get_image_features(**inputs_dict) + # hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states + hidden_states = outputs.hidden_states + expected_num_hidden_states = self._image_features_get_expected_num_hidden_states() + self.assertIsNotNone(hidden_states, "hidden_states should not be None") + self.assertEqual(len(hidden_states), expected_num_hidden_states, "Number of hidden states layers mismatch") + + for model_class in self.all_model_classes: + if not hasattr(model_class, "get_image_features"): + continue + + config, inputs_dict = self._image_features_prepare_config_and_inputs() + inputs_dict["output_hidden_states"] = True check_hidden_states_output(inputs_dict, config, model_class) @@ -4795,10 +4808,29 @@ def check_attentions_output(inputs_dict, config, model_class): check_hidden_states_output(inputs_dict, config, model_class) - if not self.has_attentions: + def test_get_image_features_attentions(self): + def check_attentions_output(inputs_dict, config, model_class): + model = model_class(copy.deepcopy(config)) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model.get_image_features(**inputs_dict) + attentions = outputs.attentions + # model.text_model(**inputs_dict) also no attentions for aimv2 + expected_num_attentions = self._image_features_get_expected_num_attentions() + self.assertIsNotNone(attentions, "attentions should not be None") + self.assertEqual(len(attentions), expected_num_attentions, "Number of attention layers mismatch") + + if not self.has_attentions: + return + + for model_class in self.all_model_classes: + if not hasattr(model_class, "get_image_features"): continue - config, inputs_dict = prepare_config_and_inputs_for_image_features() + config, inputs_dict = self._image_features_prepare_config_and_inputs() + # force eager attention to support output attentions config._attn_implementation = "eager" inputs_dict["output_hidden_states"] = False From 0284715e3f2e311d2aaf4f2992c5cea7c1db3216 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 12 Jan 2026 13:48:38 +0100 Subject: [PATCH 065/131] Fix emu3 tests, also track non-temporal ResNet in hidden_states --- src/transformers/models/emu3/modeling_emu3.py | 2 +- tests/models/emu3/test_modeling_emu3.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index 3ebccffe16ea..58d970593f5e 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -952,7 +952,7 @@ class Emu3VQVAE(PreTrainedModel): "Emu3VQVAEVectorQuantizer", ] _can_record_outputs = { - "hidden_states": Emu3VQVAETemporalResnetBlock, + "hidden_states": [Emu3VQVAEResnetBlock, Emu3VQVAETemporalResnetBlock], "attentions": Emu3VQVAEAttentionBlock, } diff --git a/tests/models/emu3/test_modeling_emu3.py b/tests/models/emu3/test_modeling_emu3.py index 5803b0be37b6..d71db167f340 100644 --- a/tests/models/emu3/test_modeling_emu3.py +++ b/tests/models/emu3/test_modeling_emu3.py @@ -167,6 +167,7 @@ def __init__( temporal_downsample_factor=1, base_channels=32, vq_channel_multiplier=[1, 2, 1], + vq_num_res_blocks=2, image_seq_length=12, vq_img_token_start_id=3, ): @@ -189,6 +190,7 @@ def __init__( self.codebook_size = codebook_size self.temporal_downsample_factor = temporal_downsample_factor self.vq_channel_multiplier = vq_channel_multiplier + self.vq_num_res_blocks = vq_num_res_blocks self.vq_img_token_start_id = vq_img_token_start_id self.base_channels = base_channels self.seq_length = seq_length + image_seq_length @@ -320,6 +322,20 @@ def test_cpu_offload(self): def test_generate_with_static_cache(self): pass + def _image_features_get_expected_num_attentions(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + # The number of Emu3VQVAEAttentionBlock instances in the encoder, assumes that attn_resolutions is empty (default) + # 0 via down due to attn_resolutions being empty, 1 via middle block, 0 via up due to attn_resolutions being empty + return 1 + + def _image_features_get_expected_num_hidden_states(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + # The number of Emu3VQVAEResnetBlock and Emu3VQVAETemporalResnetBlock instances in the encoder, plus 1 for before the block + # up_down_blocks for down, 2 for middle, vq_num_res_blocks for Emu3VQVAETemporalResnetBlock + up_down_blocks = len(model_tester.vq_channel_multiplier) * model_tester.vq_num_res_blocks + return up_down_blocks + 2 + model_tester.vq_num_res_blocks + 1 @require_torch class Emu3IntegrationTest(unittest.TestCase): From be41c044201c722de6789f36d8552fd804235250 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 12 Jan 2026 14:46:04 +0100 Subject: [PATCH 066/131] Patch chameleon, emu3, ernie4_5, janus --- .../models/chameleon/modeling_chameleon.py | 6 +- src/transformers/models/emu3/modeling_emu3.py | 9 +- src/transformers/models/emu3/modular_emu3.py | 7 +- .../modeling_ernie4_5_vl_moe.py | 9 +- .../modular_ernie4_5_vl_moe.py | 9 +- .../models/janus/modeling_janus.py | 174 +----------------- .../models/janus/modular_janus.py | 10 + 7 files changed, 40 insertions(+), 184 deletions(-) diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 69b8bff13577..3198ffa0588b 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -800,6 +800,10 @@ class ChameleonPreTrainedModel(PreTrainedModel): _can_compile_fullgraph = True _supports_flex_attn = True _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": [ChameleonDecoderLayer, ChameleonSwinDecoderLayer], + "attentions": ChameleonAttention, + } @auto_docstring( @@ -819,7 +823,7 @@ class ChameleonVQVAE(ChameleonPreTrainedModel): ] _can_record_outputs = { "hidden_states": ChameleonVQVAEEncoderResnetBlock, - "attentions": [ChameleonAttention, ChameleonVQVAEEncoderAttnBlock], + "attentions": ChameleonVQVAEEncoderAttnBlock, } def __init__(self, config: ChameleonVQVAEConfig): diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index 58d970593f5e..1b7039fa22bb 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -1133,6 +1133,10 @@ class Emu3PreTrainedModel(PreTrainedModel): _can_compile_fullgraph = True _supports_flex_attn = True _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": Emu3DecoderLayer, + "attentions": Emu3Attention, + } class Emu3RotaryEmbedding(nn.Module): @@ -1202,11 +1206,6 @@ def forward(self, x, position_ids): @auto_docstring class Emu3TextModel(Emu3PreTrainedModel): - _can_record_outputs = { - "hidden_states": Emu3DecoderLayer, - "attentions": Emu3Attention, - } - def __init__(self, config: Emu3Config): super().__init__(config) self.padding_idx = config.pad_token_id diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index 97fca19f8aff..d7f34c64b0d9 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -700,7 +700,7 @@ class Emu3VQVAE(PreTrainedModel): "Emu3VQVAEVectorQuantizer", ] _can_record_outputs = { - "hidden_states": Emu3VQVAETemporalResnetBlock, + "hidden_states": [Emu3VQVAEResnetBlock, Emu3VQVAETemporalResnetBlock], "attentions": Emu3VQVAEAttentionBlock, } @@ -871,14 +871,13 @@ class Emu3PreTrainedModel(ChameleonPreTrainedModel, Emu3VQVAE): ] _supports_flex_attn = True _supports_attention_backend = True - - -class Emu3TextModel(LlamaModel, Emu3PreTrainedModel): _can_record_outputs = { "hidden_states": Emu3DecoderLayer, "attentions": Emu3Attention, } + +class Emu3TextModel(LlamaModel, Emu3PreTrainedModel): def __init__(self, config: Emu3Config): super().__init__(config) self.layers = nn.ModuleList( diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index 630247057a68..d778eeb2f0f8 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -699,8 +699,8 @@ class Ernie4_5_VL_MoePreTrainedModel(PreTrainedModel): _can_record_outputs = { "router_logits": OutputRecorder(Ernie4_5_VL_MoeMoeBlock, index=1), - "hidden_states": [Ernie4_5_VL_MoeDecoderLayer, Ernie4_5_VL_MoeVisionBlock], - "attentions": [Ernie4_5_VL_MoeTextAttention, Ernie4_5_VL_MoeVisionAttention], + "hidden_states": Ernie4_5_VL_MoeDecoderLayer, + "attentions": Ernie4_5_VL_MoeTextAttention, } _keep_in_fp32_modules_strict = ["gate.weight", "moe_statics"] @@ -874,6 +874,11 @@ class Ernie4_5_VL_MoeVisionTransformerPretrainedModel(Ernie4_5_VL_MoePreTrainedM input_modalities = ("image", "video") _no_split_modules = ["Ernie4_5_VL_MoeVisionBlock"] _input_embed_layer = "patch_embed" + _can_record_outputs = { + "router_logits": OutputRecorder(Ernie4_5_VL_MoeMoeBlock, index=1), + "hidden_states": Ernie4_5_VL_MoeVisionBlock, + "attentions": Ernie4_5_VL_MoeVisionAttention, + } def __init__(self, config) -> None: super().__init__(config) diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index a8c7b204069a..6517e725e644 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -713,8 +713,8 @@ class Ernie4_5_VL_MoePreTrainedModel(Qwen2_5_VLPreTrainedModel): _can_record_outputs = { "router_logits": OutputRecorder(Ernie4_5_VL_MoeMoeBlock, index=1), - "hidden_states": [Ernie4_5_VL_MoeDecoderLayer, Ernie4_5_VL_MoeVisionBlock], - "attentions": [Ernie4_5_VL_MoeTextAttention, Ernie4_5_VL_MoeVisionAttention], + "hidden_states": Ernie4_5_VL_MoeDecoderLayer, + "attentions": Ernie4_5_VL_MoeTextAttention, } _keep_in_fp32_modules_strict = ["gate.weight", "moe_statics"] @@ -851,6 +851,11 @@ class Ernie4_5_VL_MoeVisionRotaryEmbedding(Qwen2_5_VisionRotaryEmbedding): class Ernie4_5_VL_MoeVisionTransformerPretrainedModel(Qwen2VisionTransformerPretrainedModel): + _can_record_outputs = { + "router_logits": OutputRecorder(Ernie4_5_VL_MoeMoeBlock, index=1), + "hidden_states": Ernie4_5_VL_MoeVisionBlock, + "attentions": Ernie4_5_VL_MoeVisionAttention, + } def __init__(self, config) -> None: super().__init__(config) diff --git a/src/transformers/models/janus/modeling_janus.py b/src/transformers/models/janus/modeling_janus.py index 36911ff6bda5..d74ee3b1bdf0 100644 --- a/src/transformers/models/janus/modeling_janus.py +++ b/src/transformers/models/janus/modeling_janus.py @@ -424,136 +424,14 @@ def forward( return BaseModelOutput(last_hidden_state=hidden_states) -class JanusAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__(self, config): - super().__init__() - self.config = config - self.embed_dim = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.embed_dim // self.num_heads - if self.head_dim * self.num_heads != self.embed_dim: - raise ValueError( - f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" - f" {self.num_heads})." - ) - self.scale = self.head_dim**-0.5 - self.is_causal = False - self.attention_dropout = config.attention_dropout - - # small tweak here compared to CLIP, no bias here - self.qkv = nn.Linear(self.embed_dim, 3 * self.embed_dim, bias=False) - - if config.qkv_bias: - q_bias = nn.Parameter(torch.zeros(self.embed_dim)) - v_bias = nn.Parameter(torch.zeros(self.embed_dim)) - else: - q_bias = None - v_bias = None - - if q_bias is not None: - qkv_bias = torch.cat((q_bias, torch.zeros_like(v_bias, requires_grad=False), v_bias)) - self.qkv.bias = nn.Parameter(qkv_bias) - - self.projection = nn.Linear(self.embed_dim, self.embed_dim) - - def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - - def forward( - self, - hidden_states: torch.Tensor, - **kwargs, - ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: - """Input shape: Batch x Time x Channel""" - - bsz, tgt_len, embed_dim = hidden_states.size() - - mixed_qkv = self.qkv(hidden_states) - - mixed_qkv = mixed_qkv.reshape(bsz, tgt_len, 3, self.num_heads, embed_dim // self.num_heads).permute( - 2, 0, 3, 1, 4 - ) - query_states, key_states, value_states = mixed_qkv[0], mixed_qkv[1], mixed_qkv[2] - - attention_interface: Callable = eager_attention_forward - - if self.config._attn_implementation != "eager": - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - - attn_output, attn_weights = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask=None, - dropout=0.0 if not self.training else self.attention_dropout, - scaling=self.scale, - **kwargs, - ) - - attn_output = attn_output.reshape(bsz, tgt_len, -1).contiguous() - attn_output = self.projection(attn_output) - - return attn_output, attn_weights - - -class JanusMLP(nn.Module): - def __init__(self, config): - super().__init__() - self.config = config - self.activation_fn = ACT2FN[config.hidden_act] - self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size) - self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - hidden_states = self.fc1(hidden_states) - hidden_states = self.activation_fn(hidden_states) - hidden_states = self.fc2(hidden_states) - return hidden_states - - -class JanusEncoderLayer(GradientCheckpointingLayer): - def __init__(self, config: JanusConfig): - super().__init__() - self.embed_dim = config.hidden_size - self.self_attn = JanusAttention(config) - self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps) - self.mlp = JanusMLP(config) - self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps) - - @auto_docstring - def forward( - self, - hidden_states: torch.Tensor, - **kwargs: Unpack[TransformersKwargs], - ) -> torch.FloatTensor: - residual = hidden_states - - hidden_states = self.layer_norm1(hidden_states) - hidden_states, _ = self.self_attn( - hidden_states=hidden_states, - **kwargs, - ) - hidden_states = hidden_states + residual - residual = hidden_states - hidden_states = self.layer_norm2(hidden_states) - hidden_states = self.mlp(hidden_states) - - hidden_states = hidden_states + residual - - return hidden_states - - @auto_docstring class JanusVisionModel(JanusPreTrainedModel): main_input_name = "pixel_values" input_modalities = ("image",) config: JanusVisionConfig _can_record_outputs = { - "hidden_states": JanusEncoderLayer, - "attentions": JanusAttention, + "hidden_states": JanusVisionEncoderLayer, + "attentions": JanusVisionAttention, } def __init__(self, config: JanusVisionConfig): @@ -967,50 +845,6 @@ class JanusVQVAEModelOutput(BaseModelOutputWithPooling): embedding_loss: torch.FloatTensor | None = None -class JanusVQVAEEncoderResnetBlock(nn.Module): - def __init__( - self, - config, - in_channels, - out_channels=None, - conv_shortcut=False, - ): - super().__init__() - self.in_channels = in_channels - self.out_channels = in_channels if out_channels is None else out_channels - self.use_conv_shortcut = conv_shortcut - - self.norm1 = torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) - self.conv1 = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) - self.norm2 = torch.nn.GroupNorm(num_groups=32, num_channels=out_channels, eps=1e-6, affine=True) - self.dropout = torch.nn.Dropout(config.dropout) - self.conv2 = torch.nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1) - if self.in_channels != self.out_channels: - if self.use_conv_shortcut: - self.conv_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) - else: - self.nin_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) - - def forward(self, hidden_states): - residual = hidden_states - hidden_states = self.norm1(hidden_states) - hidden_states *= torch.sigmoid(hidden_states) - hidden_states = self.conv1(hidden_states) - - hidden_states = self.norm2(hidden_states) - hidden_states *= torch.sigmoid(hidden_states) - hidden_states = self.dropout(hidden_states) - hidden_states = self.conv2(hidden_states) - - if self.in_channels != self.out_channels: - if self.use_conv_shortcut: - residual = self.conv_shortcut(residual) - else: - residual = self.nin_shortcut(residual) - - return residual + hidden_states - - @auto_docstring( custom_intro=""" The VQ-VAE model used in Janus for encoding/decoding images into discrete tokens. @@ -1027,8 +861,8 @@ class JanusVQVAE(JanusPreTrainedModel): "JanusVQVAEVectorQuantizer", ] _can_record_outputs = { - "hidden_states": JanusVQVAEEncoderResnetBlock, - "attentions": JanusAttention, + "hidden_states": JanusVQVAEResnetBlock, + "attentions": JanusVQVAEAttnBlock, } main_input_name = "pixel_values" diff --git a/src/transformers/models/janus/modular_janus.py b/src/transformers/models/janus/modular_janus.py index 68b5344a1b54..cdf3ed86a322 100644 --- a/src/transformers/models/janus/modular_janus.py +++ b/src/transformers/models/janus/modular_janus.py @@ -59,6 +59,7 @@ from ..blip_2.modeling_blip_2 import Blip2VisionModel from ..chameleon.configuration_chameleon import ChameleonVQVAEConfig from ..chameleon.modeling_chameleon import ( + ChameleonAttention, ChameleonVQVAE, ChameleonVQVAEEncoderAttnBlock, ChameleonVQVAEEncoderConvDownsample, @@ -552,6 +553,11 @@ def __init__(self, config: JanusVisionConfig): class JanusVisionModel(Blip2VisionModel): + _can_record_outputs = { + "hidden_states": JanusVisionEncoderLayer, + "attentions": JanusVisionAttention, + } + def __init__(self, config: JanusVisionConfig): super().__init__(config) self.encoder = JanusVisionEncoder(config) @@ -819,6 +825,10 @@ class JanusVQVAE(ChameleonVQVAE): "JanusVQVAEResnetBlock", "JanusVQVAEVectorQuantizer", ] + _can_record_outputs = { + "hidden_states": JanusVQVAEResnetBlock, + "attentions": JanusVQVAEAttnBlock, + } main_input_name = "pixel_values" def __init__(self, config: JanusVQVAEConfig): From 2743053859b6ec1a72ea88ca807838a259c4b38c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 12 Jan 2026 15:06:07 +0100 Subject: [PATCH 067/131] Skip output_attentions for FastVLM, timm doesn't accept it But I'm not sure how to handle the output_hidden_states case --- tests/models/fast_vlm/test_modeling_fast_vlm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/models/fast_vlm/test_modeling_fast_vlm.py b/tests/models/fast_vlm/test_modeling_fast_vlm.py index 3c9a85a08866..05b253559f28 100644 --- a/tests/models/fast_vlm/test_modeling_fast_vlm.py +++ b/tests/models/fast_vlm/test_modeling_fast_vlm.py @@ -227,6 +227,10 @@ def test_mismatching_num_image_tokens(self): def test_can_be_initialized_on_meta(self): pass + @unittest.skip("Cannot set output_attentions on timm models.") + def test_get_image_features_attentions(self): + pass + @require_torch @slow From 76371d8c5eb4afb18204108ce879b43c57cf3e52 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 12 Jan 2026 15:30:53 +0100 Subject: [PATCH 068/131] Patch groupvit, instructblip, ovis2 plus style --- .../modular_ernie4_5_vl_moe.py | 1 + .../models/janus/modular_janus.py | 1 - .../models/ovis2/modeling_ovis2.py | 67 +------------------ .../models/ovis2/modular_ovis2.py | 6 +- tests/models/emu3/test_modeling_emu3.py | 1 + .../models/groupvit/test_modeling_groupvit.py | 11 +++ .../test_modeling_instructblip.py | 6 ++ 7 files changed, 25 insertions(+), 68 deletions(-) diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index 6517e725e644..d71b61b31f50 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -856,6 +856,7 @@ class Ernie4_5_VL_MoeVisionTransformerPretrainedModel(Qwen2VisionTransformerPret "hidden_states": Ernie4_5_VL_MoeVisionBlock, "attentions": Ernie4_5_VL_MoeVisionAttention, } + def __init__(self, config) -> None: super().__init__(config) diff --git a/src/transformers/models/janus/modular_janus.py b/src/transformers/models/janus/modular_janus.py index cdf3ed86a322..54c7639a0da3 100644 --- a/src/transformers/models/janus/modular_janus.py +++ b/src/transformers/models/janus/modular_janus.py @@ -59,7 +59,6 @@ from ..blip_2.modeling_blip_2 import Blip2VisionModel from ..chameleon.configuration_chameleon import ChameleonVQVAEConfig from ..chameleon.modeling_chameleon import ( - ChameleonAttention, ChameleonVQVAE, ChameleonVQVAEEncoderAttnBlock, ChameleonVQVAEEncoderConvDownsample, diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index 50b9d412343d..e6e30233008f 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -195,7 +195,7 @@ def eager_attention_forward( return attn_output, attn_weights -class Ovis2VisionAttention(nn.Module): +class Ovis2Attention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper""" def __init__(self, config): @@ -272,67 +272,6 @@ def forward(self, x): return down_proj -class Ovis2Attention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__(self, config): - super().__init__() - self.config = config - self.embed_dim = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.embed_dim // self.num_heads - if self.head_dim * self.num_heads != self.embed_dim: - raise ValueError( - f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" - f" {self.num_heads})." - ) - self.scale = self.head_dim**-0.5 - self.dropout = config.attention_dropout - self.is_causal = False - self.k_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=config.qkv_bias) - self.v_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=config.qkv_bias) - self.q_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=config.qkv_bias) - self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=config.qkv_bias) - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: torch.Tensor | None = None, - **kwargs, - ) -> tuple[torch.Tensor, torch.Tensor | None]: - """Input shape: Batch x Time x Channel""" - - batch_size, seq_length, embed_dim = hidden_states.shape - - queries = self.q_proj(hidden_states) - keys = self.k_proj(hidden_states) - values = self.v_proj(hidden_states) - - queries = queries.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2) - keys = keys.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2) - values = values.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2) - - attention_interface: Callable = eager_attention_forward - if self.config._attn_implementation != "eager": - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - - attn_output, attn_weights = attention_interface( - self, - queries, - keys, - values, - attention_mask, - is_causal=self.is_causal, - scaling=self.scale, - dropout=0.0 if not self.training else self.dropout, - ) - - attn_output = attn_output.reshape(batch_size, seq_length, embed_dim).contiguous() - attn_output = self.out_proj(attn_output) - - return attn_output, attn_weights - - class Ovis2VisionEncoderLayer(GradientCheckpointingLayer): def __init__(self, config: Ovis2VisionConfig): super().__init__() @@ -431,7 +370,7 @@ class Ovis2PreTrainedModel(PreTrainedModel): base_model_prefix = "model" input_modalities = ("image", "text") supports_gradient_checkpointing = True - _no_split_modules = ["Ovis2VisionAttention"] + _no_split_modules = ["Ovis2Attention"] _skip_keys_device_placement = "past_key_values" _supports_cache_class = True _supports_flash_attn = True @@ -461,7 +400,7 @@ class Ovis2VisionModel(Ovis2PreTrainedModel): config: Ovis2VisionConfig _can_record_outputs = { "hidden_states": Ovis2VisionEncoderLayer, - "attentions": Ovis2VisionAttention, + "attentions": Ovis2Attention, } def __init__(self, config: Ovis2VisionConfig): diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index 469549d9d458..910ee99f88bf 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -91,7 +91,7 @@ def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: return embeddings -class Ovis2VisionAttention(Aimv2Attention): +class Ovis2Attention(Aimv2Attention): pass @@ -161,7 +161,7 @@ class Ovis2PreTrainedModel(PreTrainedModel): base_model_prefix = "model" input_modalities = ("image", "text") supports_gradient_checkpointing = True - _no_split_modules = ["Ovis2VisionAttention"] + _no_split_modules = ["Ovis2Attention"] _skip_keys_device_placement = "past_key_values" _supports_cache_class = True _supports_flash_attn = True @@ -181,7 +181,7 @@ class Ovis2VisionModel(Ovis2PreTrainedModel): config: Ovis2VisionConfig _can_record_outputs = { "hidden_states": Ovis2VisionEncoderLayer, - "attentions": Ovis2VisionAttention, + "attentions": Ovis2Attention, } def __init__(self, config: Ovis2VisionConfig): diff --git a/tests/models/emu3/test_modeling_emu3.py b/tests/models/emu3/test_modeling_emu3.py index d71db167f340..758d4c39b1a8 100644 --- a/tests/models/emu3/test_modeling_emu3.py +++ b/tests/models/emu3/test_modeling_emu3.py @@ -337,6 +337,7 @@ def _image_features_get_expected_num_hidden_states(self, model_tester=None): up_down_blocks = len(model_tester.vq_channel_multiplier) * model_tester.vq_num_res_blocks return up_down_blocks + 2 + model_tester.vq_num_res_blocks + 1 + @require_torch class Emu3IntegrationTest(unittest.TestCase): @slow diff --git a/tests/models/groupvit/test_modeling_groupvit.py b/tests/models/groupvit/test_modeling_groupvit.py index 7a656e404d13..526a8f949258 100644 --- a/tests/models/groupvit/test_modeling_groupvit.py +++ b/tests/models/groupvit/test_modeling_groupvit.py @@ -582,6 +582,17 @@ def test_model_from_pretrained(self): model = GroupViTModel.from_pretrained(model_name) self.assertIsNotNone(model) + def _image_features_get_expected_num_attentions(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester.vision_model_tester + # GroupViT returns attention grouping of each stage + return sum(g > 0 for g in self.model_tester.vision_model_tester.num_group_tokens) + + def _image_features_get_expected_num_hidden_states(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester.vision_model_tester + return model_tester.expected_num_hidden_layers + # We will verify our results on an image of cute cats def prepare_img(): diff --git a/tests/models/instructblip/test_modeling_instructblip.py b/tests/models/instructblip/test_modeling_instructblip.py index 6a6050abb146..2b1c153ba00c 100644 --- a/tests/models/instructblip/test_modeling_instructblip.py +++ b/tests/models/instructblip/test_modeling_instructblip.py @@ -613,6 +613,12 @@ def test_sdpa_can_dispatch_composite_models(self): ): raise ValueError("The eager model should not have SDPA attention layers") + def _image_features_prepare_config_and_inputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + del inputs_dict["input_ids"] + del inputs_dict["attention_mask"] + return config, inputs_dict + # We will verify our results on an image of cute cats def prepare_img(): From 88a5804f38dcf60dc55cfa3acc091d58bf2e0324 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 12 Jan 2026 16:05:57 +0100 Subject: [PATCH 069/131] Patch paddleocr_vl, qwen2_5_omni, qwen2_5_vl, qwen2_vl, and skip test for perception_lm perception_lm is still problematic with output_hidden_states, akin to fast_vlm --- .../paddleocr_vl/modeling_paddleocr_vl.py | 77 ++--- .../paddleocr_vl/modular_paddleocr_vl.py | 77 ++--- .../qwen2_5_omni/modeling_qwen2_5_omni.py | 2 +- .../qwen2_5_omni/modular_qwen2_5_omni.py | 2 +- .../models/qwen2_5_vl/modeling_qwen2_5_vl.py | 278 +++++++++--------- .../models/qwen2_5_vl/modular_qwen2_5_vl.py | 9 +- .../models/qwen2_vl/modeling_qwen2_vl.py | 4 + .../test_modeling_perception_lm.py | 4 + 8 files changed, 233 insertions(+), 220 deletions(-) diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index 0b595953faa6..e17c28df5b5b 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -544,42 +544,6 @@ def forward( ) -class PaddleOCRVisionModel(PaddleOCRVLPreTrainedModel): - config: PaddleOCRVisionConfig - main_input_name = "pixel_values" - input_modalities = "image" - - def __init__(self, config: PaddleOCRVisionConfig): - super().__init__(config) - - self.vision_model = PaddleOCRVisionTransformer(config) - - # Initialize weights and apply final processing - self.post_init() - - def forward( - self, - pixel_values: torch.FloatTensor, - cu_seqlens: torch.Tensor, - image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]] | None = None, - **kwargs, - ) -> BaseModelOutputWithPooling: - """ - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, sequence_length, image_channels, patch_size, patch_size)`): - The tensors corresponding to the input images. - cu_seqlens (`torch.Tensor` of shape `(num_images + 1,)`): - The cumulative sequence lengths of each image or video feature. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - """ - return self.vision_model( - pixel_values=pixel_values, - cu_seqlens=cu_seqlens, - image_grid_thw=image_grid_thw, - ) - - class PaddleOCRVisionEmbeddings(nn.Module): def __init__(self, config: PaddleOCRVisionConfig): super().__init__() @@ -970,6 +934,47 @@ def forward( ) +class PaddleOCRVisionModel(PaddleOCRVLPreTrainedModel): + config: PaddleOCRVisionConfig + main_input_name = "pixel_values" + input_modalities = "image" + _can_record_outputs = { + "hidden_states": PaddleOCRVisionEncoderLayer, + "attentions": PaddleOCRVisionAttention, + } + + def __init__(self, config: PaddleOCRVisionConfig): + super().__init__(config) + + self.vision_model = PaddleOCRVisionTransformer(config) + + # Initialize weights and apply final processing + self.post_init() + + @check_model_inputs(tie_last_hidden_states=False) + def forward( + self, + pixel_values: torch.FloatTensor, + cu_seqlens: torch.Tensor, + image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]] | None = None, + **kwargs, + ) -> BaseModelOutputWithPooling: + """ + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, sequence_length, image_channels, patch_size, patch_size)`): + The tensors corresponding to the input images. + cu_seqlens (`torch.Tensor` of shape `(num_images + 1,)`): + The cumulative sequence lengths of each image or video feature. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ + return self.vision_model( + pixel_values=pixel_values, + cu_seqlens=cu_seqlens, + image_grid_thw=image_grid_thw, + ) + + @dataclass @auto_docstring( custom_intro=""" diff --git a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py index d4c76efb677a..7625bb041a36 100644 --- a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py @@ -860,42 +860,6 @@ def forward( ) -class PaddleOCRVisionModel(PaddleOCRVLPreTrainedModel): - config: PaddleOCRVisionConfig - main_input_name = "pixel_values" - input_modalities = "image" - - def __init__(self, config: PaddleOCRVisionConfig): - super().__init__(config) - - self.vision_model = PaddleOCRVisionTransformer(config) - - # Initialize weights and apply final processing - self.post_init() - - def forward( - self, - pixel_values: torch.FloatTensor, - cu_seqlens: torch.Tensor, - image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]] | None = None, - **kwargs, - ) -> BaseModelOutputWithPooling: - """ - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, sequence_length, image_channels, patch_size, patch_size)`): - The tensors corresponding to the input images. - cu_seqlens (`torch.Tensor` of shape `(num_images + 1,)`): - The cumulative sequence lengths of each image or video feature. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - """ - return self.vision_model( - pixel_values=pixel_values, - cu_seqlens=cu_seqlens, - image_grid_thw=image_grid_thw, - ) - - class PaddleOCRVisionEmbeddings(SiglipVisionEmbeddings): def __init__(self, config: PaddleOCRVisionConfig): super().__init__() @@ -1086,6 +1050,47 @@ def forward( ) +class PaddleOCRVisionModel(PaddleOCRVLPreTrainedModel): + config: PaddleOCRVisionConfig + main_input_name = "pixel_values" + input_modalities = "image" + _can_record_outputs = { + "hidden_states": PaddleOCRVisionEncoderLayer, + "attentions": PaddleOCRVisionAttention, + } + + def __init__(self, config: PaddleOCRVisionConfig): + super().__init__(config) + + self.vision_model = PaddleOCRVisionTransformer(config) + + # Initialize weights and apply final processing + self.post_init() + + @check_model_inputs(tie_last_hidden_states=False) + def forward( + self, + pixel_values: torch.FloatTensor, + cu_seqlens: torch.Tensor, + image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]] | None = None, + **kwargs, + ) -> BaseModelOutputWithPooling: + """ + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, sequence_length, image_channels, patch_size, patch_size)`): + The tensors corresponding to the input images. + cu_seqlens (`torch.Tensor` of shape `(num_images + 1,)`): + The cumulative sequence lengths of each image or video feature. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ + return self.vision_model( + pixel_values=pixel_values, + cu_seqlens=cu_seqlens, + image_grid_thw=image_grid_thw, + ) + + class PaddleOCRVLModelOutputWithPast(Qwen2VLModelOutputWithPast): pass diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index 6f4c81b6f054..883e2c199808 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -1141,7 +1141,7 @@ class Qwen2_5OmniVisionEncoder(Qwen2_5OmniPreTrainedModel): config: Qwen2_5OmniVisionEncoderConfig _no_split_modules = ["Qwen2_5OmniVisionBlock"] _input_embed_layer = "patch_embed" - _can_record_inputs = { + _can_record_outputs = { "hidden_states": Qwen2_5OmniVisionBlock, "attentions": Qwen2_5OmniVisionAttention, } diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index 0bf81d9f9441..cae986aa6dfb 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -1944,7 +1944,7 @@ class Qwen2_5OmniVisionEncoder(Qwen2_5_VisionTransformerPretrainedModel): input_modalities = ("image", "video") _no_split_modules = ["Qwen2_5OmniVisionBlock"] _input_embed_layer = "patch_embed" - _can_record_inputs = { + _can_record_outputs = { "hidden_states": Qwen2_5OmniVisionBlock, "attentions": Qwen2_5OmniVisionAttention, } diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index c1867372eb3e..9c480bd69bf9 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -312,150 +312,13 @@ def _init_weights(self, module): init.copy_(module.inv_freq, inv_freq) -def apply_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section, unsqueeze_dim=1): - """Applies Rotary Position Embedding with Multimodal Sections to the query and key tensors (https://qwenlm.github.io/blog/qwen2-vl/). - - Explanation: - Multimodal 3D rotary position embedding is an extension to 1D rotary position embedding. The input embedding - sequence contains vision (images / videos) embedding and text embedding or just contains text embedding. For - vision embedding part, we apply rotary position embedding on temporal, height and width dimension separately. - Here we split the channel dimension to 3 chunks for the temporal, height and width rotary position embedding. - For text embedding part, we just apply 1D rotary position embedding. The three rotary position index (temporal, - height and width) of text embedding is always the same, so the text embedding rotary position embedding has no - difference with modern LLMs. - - Args: - q (`torch.Tensor`): The query tensor. - k (`torch.Tensor`): The key tensor. - cos (`torch.Tensor`): The cosine part of the rotary embedding. - sin (`torch.Tensor`): The sine part of the rotary embedding. - position_ids (`torch.Tensor`): - The position indices of the tokens corresponding to the query and key tensors. For example, this can be - used to pass offsetted position ids when working with a KV-cache. - mrope_section(`List(int)`): - Multimodal rope section is for channel dimension of temporal, height and width in rope calculation. - unsqueeze_dim (`int`, *optional*, defaults to 1): - The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and - sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note - that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and - k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes - cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have - the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. - Returns: - `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. - """ - mrope_section = mrope_section * 2 - cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section, dim=-1))], dim=-1).unsqueeze( - unsqueeze_dim - ) - sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section, dim=-1))], dim=-1).unsqueeze( - unsqueeze_dim - ) - - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - return q_embed, k_embed - - -class Qwen2_5_VLAttention(nn.Module): - """ - Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer - and "Generating Long Sequences with Sparse Transformers". - """ - - def __init__(self, config: Qwen2_5_VLTextConfig, layer_idx: int | None = None): - super().__init__() - self.config = config - self.layer_idx = layer_idx - if layer_idx is None: - logger.warning_once( - f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will " - "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` " - "when creating this class." - ) - - self.hidden_size = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.hidden_size // self.num_heads - self.num_key_value_heads = config.num_key_value_heads - self.num_key_value_groups = self.num_heads // self.num_key_value_heads - self.is_causal = True - self.attention_dropout = config.attention_dropout - self.rope_parameters = config.rope_parameters - self.scaling = self.head_dim**-0.5 - - if (self.head_dim * self.num_heads) != self.hidden_size: - raise ValueError( - f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" - f" and `num_heads`: {self.num_heads})." - ) - self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True) - self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) - self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) - self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) - self.layer_type = config.layer_types[layer_idx] if hasattr(config, "layer_types") else None - self.sliding_window = config.sliding_window if self.layer_type == "sliding_attention" else None - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: torch.Tensor | None = None, - position_ids: torch.LongTensor | None = None, - past_key_values: Cache | None = None, - output_attentions: bool = False, - use_cache: bool = False, - cache_position: torch.LongTensor | None = None, - position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, - **kwargs: Unpack[FlashAttentionKwargs], - ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - - cos, sin = position_embeddings - query_states, key_states = apply_multimodal_rotary_pos_emb( - query_states, key_states, cos, sin, self.config.rope_parameters["mrope_section"] - ) - - if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) - - attention_interface: Callable = eager_attention_forward - if self.config._attn_implementation != "eager": - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - - attn_output, attn_weights = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask, - dropout=0.0 if not self.training else self.attention_dropout, - scaling=self.scaling, - sliding_window=self.sliding_window, - position_ids=position_ids, # pass positions for FA2 - **kwargs, - ) - - attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() - attn_output = self.o_proj(attn_output) - return attn_output, attn_weights - - class Qwen2_5_VisionTransformerPretrainedModel(Qwen2_5_VLPreTrainedModel): config: Qwen2_5_VLVisionConfig _no_split_modules = ["Qwen2_5_VLVisionBlock"] _input_embed_layer = "patch_embed" - _can_record_inputs = { + _can_record_outputs = { "hidden_states": Qwen2_5_VLVisionBlock, - "attentions": Qwen2_5_VLAttention, + "attentions": Qwen2_5_VLVisionAttention, } def __init__(self, config, *inputs, **kwargs) -> None: @@ -729,6 +592,143 @@ def forward(self, x): return down_proj +def apply_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section, unsqueeze_dim=1): + """Applies Rotary Position Embedding with Multimodal Sections to the query and key tensors (https://qwenlm.github.io/blog/qwen2-vl/). + + Explanation: + Multimodal 3D rotary position embedding is an extension to 1D rotary position embedding. The input embedding + sequence contains vision (images / videos) embedding and text embedding or just contains text embedding. For + vision embedding part, we apply rotary position embedding on temporal, height and width dimension separately. + Here we split the channel dimension to 3 chunks for the temporal, height and width rotary position embedding. + For text embedding part, we just apply 1D rotary position embedding. The three rotary position index (temporal, + height and width) of text embedding is always the same, so the text embedding rotary position embedding has no + difference with modern LLMs. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`): + The position indices of the tokens corresponding to the query and key tensors. For example, this can be + used to pass offsetted position ids when working with a KV-cache. + mrope_section(`List(int)`): + Multimodal rope section is for channel dimension of temporal, height and width in rope calculation. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + mrope_section = mrope_section * 2 + cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section, dim=-1))], dim=-1).unsqueeze( + unsqueeze_dim + ) + sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section, dim=-1))], dim=-1).unsqueeze( + unsqueeze_dim + ) + + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +class Qwen2_5_VLAttention(nn.Module): + """ + Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer + and "Generating Long Sequences with Sparse Transformers". + """ + + def __init__(self, config: Qwen2_5_VLTextConfig, layer_idx: int | None = None): + super().__init__() + self.config = config + self.layer_idx = layer_idx + if layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will " + "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.is_causal = True + self.attention_dropout = config.attention_dropout + self.rope_parameters = config.rope_parameters + self.scaling = self.head_dim**-0.5 + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + self.layer_type = config.layer_types[layer_idx] if hasattr(config, "layer_types") else None + self.sliding_window = config.sliding_window if self.layer_type == "sliding_attention" else None + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: torch.LongTensor | None = None, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + + cos, sin = position_embeddings + query_states, key_states = apply_multimodal_rotary_pos_emb( + query_states, key_states, cos, sin, self.config.rope_parameters["mrope_section"] + ) + + if past_key_values is not None: + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + sliding_window=self.sliding_window, + position_ids=position_ids, # pass positions for FA2 + **kwargs, + ) + + attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + class Qwen2_5_VLDecoderLayer(GradientCheckpointingLayer): def __init__(self, config: Qwen2_5_VLTextConfig, layer_idx: int): super().__init__() diff --git a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py index ac11e0aad10b..e2e84e7d568c 100644 --- a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py @@ -42,7 +42,6 @@ PatchEmbed, PatchMerger, Qwen2RMSNorm, - Qwen2VLAttention, Qwen2VLCausalLMOutputWithPast, Qwen2VLForConditionalGeneration, Qwen2VLModel, @@ -176,17 +175,13 @@ def _init_weights(self, module): init.copy_(module.inv_freq, inv_freq) -class Qwen2_5_VLAttention(Qwen2VLAttention): - pass - - class Qwen2_5_VisionTransformerPretrainedModel(Qwen2_5_VLPreTrainedModel): config: Qwen2_5_VLVisionConfig _no_split_modules = ["Qwen2_5_VLVisionBlock"] _input_embed_layer = "patch_embed" - _can_record_inputs = { + _can_record_outputs = { "hidden_states": Qwen2_5_VLVisionBlock, - "attentions": Qwen2_5_VLAttention, + "attentions": Qwen2_5_VLVisionAttention, } def __init__(self, config, *inputs, **kwargs) -> None: diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index e17cd676a7df..7385363359b0 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -680,6 +680,10 @@ class Qwen2VisionTransformerPretrainedModel(Qwen2VLPreTrainedModel): input_modalities = ("image", "video") _no_split_modules = ["Qwen2VLVisionBlock"] _input_embed_layer = "patch_embed" + _can_record_outputs = { + "hidden_states": Qwen2VLVisionBlock, + "attentions": VisionAttention, + } def __init__(self, config) -> None: super().__init__(config) diff --git a/tests/models/perception_lm/test_modeling_perception_lm.py b/tests/models/perception_lm/test_modeling_perception_lm.py index b0ecdba86f58..c402b1a8cef2 100644 --- a/tests/models/perception_lm/test_modeling_perception_lm.py +++ b/tests/models/perception_lm/test_modeling_perception_lm.py @@ -367,6 +367,10 @@ def test_retain_grad_hidden_states_attentions(self): def test_generate_compilation_all_outputs(self): pass + @unittest.skip("Cannot set output_attentions on timm models.") + def test_get_image_features_attentions(self): + pass + TEST_MODEL_PATH = "facebook/Perception-LM-1B" From 13875af6f1f5a3ff1b4fea072bf5680b28d6a3f0 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 12 Jan 2026 16:58:42 +0100 Subject: [PATCH 070/131] Patch qwen3_omni_moe, sam family, edgetam P.s. edgetam had incorrect _can_record_outputs Now, all issues that remain with get_image_features are due to 1) CLIP family issue and 2) unclarity with expected output_hidden_states for timm-based models --- .../models/edgetam/modeling_edgetam.py | 16 +++------------- .../models/edgetam/modular_edgetam.py | 7 +------ .../edgetam_video/modeling_edgetam_video.py | 9 ++------- src/transformers/models/sam2/modeling_sam2.py | 7 +------ src/transformers/models/sam2/modular_sam2.py | 7 +------ .../models/sam2_video/modeling_sam2_video.py | 9 ++------- src/transformers/models/sam3/modeling_sam3.py | 9 +-------- .../models/sam3_tracker/modeling_sam3_tracker.py | 9 ++------- .../modeling_sam3_tracker_video.py | 7 +------ .../modular_sam3_tracker_video.py | 2 -- tests/models/edgetam/test_modeling_edgetam.py | 4 ++++ .../test_modeling_qwen3_omni_moe.py | 10 ++++++++++ tests/models/sam2/test_modeling_sam2.py | 10 ++++++++++ 13 files changed, 38 insertions(+), 68 deletions(-) diff --git a/src/transformers/models/edgetam/modeling_edgetam.py b/src/transformers/models/edgetam/modeling_edgetam.py index c7f551c30856..b02e95952bc9 100644 --- a/src/transformers/models/edgetam/modeling_edgetam.py +++ b/src/transformers/models/edgetam/modeling_edgetam.py @@ -32,7 +32,7 @@ from ... import initialization as init from ...activations import ACT2FN -from ...modeling_outputs import BaseModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import compile_compatible_method_lru_cache @@ -47,11 +47,6 @@ ) -# fix this in modular -if True: - from ..timm_wrapper.modeling_timm_wrapper import TimmWrapperModel - - class EdgeTamLayerNorm(nn.LayerNorm): r"""LayerNorm that supports two data formats: channels_last (default) or channels_first. The ordering of the dimensions in the inputs. channels_last corresponds to inputs with shape (batch_size, height, @@ -80,7 +75,7 @@ def forward(self, features: torch.Tensor) -> torch.Tensor: @dataclass @auto_docstring(custom_intro="Base class for the vision encoder's outputs.") -class EdgeTamVisionEncoderOutput(ModelOutput): +class EdgeTamVisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. @@ -100,11 +95,8 @@ class EdgeTamVisionEncoderOutput(ModelOutput): the self-attention heads. """ - last_hidden_state: torch.FloatTensor | None = None fpn_hidden_states: torch.FloatTensor | None = None fpn_position_encoding: torch.FloatTensor | None = None - hidden_states: tuple[torch.FloatTensor, ...] | None = None - attentions: tuple[torch.FloatTensor, ...] | None = None def eager_attention_forward( @@ -425,7 +417,7 @@ def forward(self, hidden_states: torch.Tensor) -> tuple[tuple[torch.Tensor, ...] class EdgeTamVisionModel(EdgeTamPreTrainedModel): config_class = EdgeTamVisionConfig main_input_name = "pixel_values" - _can_record_outputs = {"hidden_states": TimmWrapperModel, "attentions": TimmWrapperModel} + _can_record_outputs = {} def __init__(self, config: EdgeTamVisionConfig): super().__init__(config) @@ -1231,8 +1223,6 @@ def get_image_features( vision_outputs.fpn_hidden_states = feature_maps vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the - # fpn hidden states/position encoding order, still have to double-check return vision_outputs diff --git a/src/transformers/models/edgetam/modular_edgetam.py b/src/transformers/models/edgetam/modular_edgetam.py index de0ef2960956..1cce16fcfe0a 100644 --- a/src/transformers/models/edgetam/modular_edgetam.py +++ b/src/transformers/models/edgetam/modular_edgetam.py @@ -38,11 +38,6 @@ ) -# fix this in modular -if True: - from ..timm_wrapper.modeling_timm_wrapper import TimmWrapperModel - - class EdgeTamVisionConfig(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`EdgeTamVisionModel`]. It is used to instantiate a SAM @@ -189,7 +184,7 @@ def _init_weights(self, module): class EdgeTamVisionModel(Sam2VisionModel): config_class = EdgeTamVisionConfig main_input_name = "pixel_values" - _can_record_outputs = {"hidden_states": TimmWrapperModel, "attentions": TimmWrapperModel} + _can_record_outputs = {} def get_input_embeddings(self): raise NotImplementedError("Can't get input embeddings from timm wrapper model") diff --git a/src/transformers/models/edgetam_video/modeling_edgetam_video.py b/src/transformers/models/edgetam_video/modeling_edgetam_video.py index 08c57ff38cc2..b54c79d610f1 100644 --- a/src/transformers/models/edgetam_video/modeling_edgetam_video.py +++ b/src/transformers/models/edgetam_video/modeling_edgetam_video.py @@ -37,7 +37,7 @@ from ...activations import ACT2FN from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import compile_compatible_method_lru_cache @@ -116,7 +116,7 @@ def forward(self, hidden_states): @dataclass @auto_docstring(custom_intro="Base class for the vision encoder's outputs.") -class EdgeTamVideoVisionEncoderOutput(ModelOutput): +class EdgeTamVideoVisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. @@ -136,11 +136,8 @@ class EdgeTamVideoVisionEncoderOutput(ModelOutput): the self-attention heads. """ - last_hidden_state: torch.FloatTensor | None = None fpn_hidden_states: torch.FloatTensor | None = None fpn_position_encoding: torch.FloatTensor | None = None - hidden_states: tuple[torch.FloatTensor, ...] | None = None - attentions: tuple[torch.FloatTensor, ...] | None = None class EdgeTamVideoVisionRotaryEmbedding(nn.Module): @@ -2260,8 +2257,6 @@ def get_image_features( vision_outputs.fpn_hidden_states = feature_maps vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the - # fpn hidden states/position encoding order, still have to double-check return vision_outputs def _prepare_vision_features( diff --git a/src/transformers/models/sam2/modeling_sam2.py b/src/transformers/models/sam2/modeling_sam2.py index 724b7012087d..ee877ba81774 100644 --- a/src/transformers/models/sam2/modeling_sam2.py +++ b/src/transformers/models/sam2/modeling_sam2.py @@ -65,7 +65,7 @@ class BaseModelOutputWithFeatureMaps(BaseModelOutputWithPooling): @dataclass @auto_docstring(custom_intro="Base class for the vision encoder's outputs.") -class Sam2VisionEncoderOutput(ModelOutput): +class Sam2VisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. @@ -85,11 +85,8 @@ class Sam2VisionEncoderOutput(ModelOutput): the self-attention heads. """ - last_hidden_state: torch.FloatTensor | None = None fpn_hidden_states: torch.FloatTensor | None = None fpn_position_encoding: torch.FloatTensor | None = None - hidden_states: tuple[torch.FloatTensor, ...] | None = None - attentions: tuple[torch.FloatTensor, ...] | None = None @dataclass @@ -1603,8 +1600,6 @@ def get_image_features( vision_outputs.fpn_hidden_states = feature_maps vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the - # fpn hidden states/position encoding order, still have to double-check return vision_outputs diff --git a/src/transformers/models/sam2/modular_sam2.py b/src/transformers/models/sam2/modular_sam2.py index ad8a74a2e24a..3b0acd7a4b83 100644 --- a/src/transformers/models/sam2/modular_sam2.py +++ b/src/transformers/models/sam2/modular_sam2.py @@ -312,7 +312,7 @@ def resize(self): @dataclass @auto_docstring(custom_intro="Base class for the vision encoder's outputs.") -class Sam2VisionEncoderOutput(ModelOutput): +class Sam2VisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. @@ -332,11 +332,8 @@ class Sam2VisionEncoderOutput(ModelOutput): the self-attention heads. """ - last_hidden_state: torch.FloatTensor | None = None fpn_hidden_states: torch.FloatTensor | None = None fpn_position_encoding: torch.FloatTensor | None = None - hidden_states: tuple[torch.FloatTensor, ...] | None = None - attentions: tuple[torch.FloatTensor, ...] | None = None @dataclass @@ -1294,8 +1291,6 @@ def get_image_features( vision_outputs.fpn_hidden_states = feature_maps vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the - # fpn hidden states/position encoding order, still have to double-check return vision_outputs @check_model_inputs diff --git a/src/transformers/models/sam2_video/modeling_sam2_video.py b/src/transformers/models/sam2_video/modeling_sam2_video.py index 4821d1a8022c..4b65f8d30460 100644 --- a/src/transformers/models/sam2_video/modeling_sam2_video.py +++ b/src/transformers/models/sam2_video/modeling_sam2_video.py @@ -35,7 +35,7 @@ from ...activations import ACT2FN from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import compile_compatible_method_lru_cache @@ -1140,7 +1140,7 @@ def forward(self, input_coords, input_shape=None): @dataclass @auto_docstring(custom_intro="Base class for the vision encoder's outputs.") -class Sam2VideoVisionEncoderOutput(ModelOutput): +class Sam2VideoVisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. @@ -1160,11 +1160,8 @@ class Sam2VideoVisionEncoderOutput(ModelOutput): the self-attention heads. """ - last_hidden_state: torch.FloatTensor | None = None fpn_hidden_states: torch.FloatTensor | None = None fpn_position_encoding: torch.FloatTensor | None = None - hidden_states: tuple[torch.FloatTensor, ...] | None = None - attentions: tuple[torch.FloatTensor, ...] | None = None class Sam2VideoMaskEmbedding(nn.Module): @@ -1864,8 +1861,6 @@ def get_image_features( vision_outputs.fpn_hidden_states = feature_maps vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the - # fpn hidden states/position encoding order, still have to double-check return vision_outputs def _prepare_vision_features( diff --git a/src/transformers/models/sam3/modeling_sam3.py b/src/transformers/models/sam3/modeling_sam3.py index 50ff52dda6ff..bba5e3123f20 100644 --- a/src/transformers/models/sam3/modeling_sam3.py +++ b/src/transformers/models/sam3/modeling_sam3.py @@ -56,23 +56,16 @@ @dataclass @auto_docstring -class Sam3VisionEncoderOutput(ModelOutput): +class Sam3VisionEncoderOutput(BaseModelOutputWithPooling): r""" fpn_hidden_states (`tuple[torch.FloatTensor]`): Tuple of multi-level FPN feature maps. fpn_position_encoding (`tuple[torch.FloatTensor]`): Tuple of position encodings for each FPN level. - hidden_states (`tuple[torch.FloatTensor]`, *optional*): - Tuple of hidden states from all ViT layers. - attentions (`tuple[torch.FloatTensor]`, *optional*): - Tuple of attention weights from all ViT layers. """ - last_hidden_state: torch.FloatTensor = None fpn_hidden_states: tuple[torch.FloatTensor, ...] = None fpn_position_encoding: tuple[torch.FloatTensor, ...] = None - hidden_states: tuple[torch.FloatTensor] | None = None - attentions: tuple[torch.FloatTensor] | None = None @dataclass diff --git a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py index 383f7d15623b..070e8b3159b0 100644 --- a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py +++ b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py @@ -31,7 +31,7 @@ from ... import initialization as init from ...activations import ACT2FN -from ...modeling_outputs import BaseModelOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import ModelOutput, auto_docstring, can_return_tuple @@ -725,7 +725,7 @@ def _dynamic_multimask_via_stability(self, all_mask_logits, all_iou_scores): @dataclass @auto_docstring(custom_intro="Base class for the vision encoder's outputs.") -class Sam3TrackerVisionEncoderOutput(ModelOutput): +class Sam3TrackerVisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. @@ -745,11 +745,8 @@ class Sam3TrackerVisionEncoderOutput(ModelOutput): the self-attention heads. """ - last_hidden_state: torch.FloatTensor | None = None fpn_hidden_states: torch.FloatTensor | None = None fpn_position_encoding: torch.FloatTensor | None = None - hidden_states: tuple[torch.FloatTensor, ...] | None = None - attentions: tuple[torch.FloatTensor, ...] | None = None @auto_docstring( @@ -1090,8 +1087,6 @@ def get_image_features( vision_outputs.fpn_hidden_states = feature_maps vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the - # fpn hidden states/position encoding order, still have to double-check return vision_outputs diff --git a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py index ba4db7ae538c..2c67e02b51e6 100644 --- a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py @@ -1137,7 +1137,7 @@ def forward( @dataclass @auto_docstring(custom_intro="Base class for the vision encoder's outputs.") -class Sam3TrackerVideoVisionEncoderOutput(ModelOutput): +class Sam3TrackerVideoVisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. @@ -1157,11 +1157,8 @@ class Sam3TrackerVideoVisionEncoderOutput(ModelOutput): the self-attention heads. """ - last_hidden_state: torch.FloatTensor | None = None fpn_hidden_states: torch.FloatTensor | None = None fpn_position_encoding: torch.FloatTensor | None = None - hidden_states: tuple[torch.FloatTensor, ...] | None = None - attentions: tuple[torch.FloatTensor, ...] | None = None class Sam3TrackerVideoPositionalEmbedding(nn.Module): @@ -1900,8 +1897,6 @@ def get_image_features( vision_outputs.fpn_hidden_states = feature_maps vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the - # fpn hidden states/position encoding order, still have to double-check return vision_outputs def _prepare_vision_features( diff --git a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py index bb28049312e5..a9f6500339d6 100644 --- a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py @@ -600,8 +600,6 @@ def get_image_features( vision_outputs.fpn_hidden_states = feature_maps vision_outputs.fpn_position_encoding = feature_maps_position_embeddings - # NOTE: @Tom I'm not 100% sure that the feature_maps/feature_maps_position_embeddings match the - # fpn hidden states/position encoding order, still have to double-check return vision_outputs diff --git a/tests/models/edgetam/test_modeling_edgetam.py b/tests/models/edgetam/test_modeling_edgetam.py index 06a400a854b5..c7a2e23117b1 100644 --- a/tests/models/edgetam/test_modeling_edgetam.py +++ b/tests/models/edgetam/test_modeling_edgetam.py @@ -430,6 +430,10 @@ def test_model_from_pretrained(self): def test_sdpa_can_compile_dynamic(self): self.skipTest(reason="EDGETAM model can't be compiled dynamic yet") + @unittest.skip("Cannot set `output_attentions` for timm models.") + def test_get_image_features_attentions(self): + pass + def prepare_image(): img_url = "https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/truck.jpg" diff --git a/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py b/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py index 56d9b02177be..23a2090dd40e 100644 --- a/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py +++ b/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py @@ -618,6 +618,16 @@ def test_get_rope_index_video_with_audio(self): self.assertTrue(torch.equal(position_ids, expected_position_ids)) + def _image_features_get_expected_num_attentions(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + return model_tester.vision_config["depth"] + + def _image_features_get_expected_num_hidden_states(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + return model_tester.vision_config["depth"] + 1 + @require_torch class Qwen3OmniModelIntegrationTest(unittest.TestCase): diff --git a/tests/models/sam2/test_modeling_sam2.py b/tests/models/sam2/test_modeling_sam2.py index 05ee70da4355..256dbccefe5b 100644 --- a/tests/models/sam2/test_modeling_sam2.py +++ b/tests/models/sam2/test_modeling_sam2.py @@ -711,6 +711,16 @@ def test_model_from_pretrained(self): def test_sdpa_can_compile_dynamic(self): self.skipTest(reason="SAM2 model can't be compiled dynamic yet") + def _image_features_get_expected_num_attentions(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + return sum(model_tester.blocks_per_stage) + + def _image_features_get_expected_num_hidden_states(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + return sum(model_tester.blocks_per_stage) + 1 + def prepare_image(): img_url = "https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/truck.jpg" From e480bc0e8fde90b578c3d1c2ff0a87698caf1cc8 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 12 Jan 2026 17:04:11 +0100 Subject: [PATCH 071/131] Kill now unused BaseModelOutputWithFeatureMaps --- src/transformers/models/sam2/modeling_sam2.py | 14 -------------- src/transformers/models/sam2/modular_sam2.py | 14 -------------- .../modeling_sam3_tracker_video.py | 15 +-------------- .../modular_sam3_tracker_video.py | 18 +----------------- 4 files changed, 2 insertions(+), 59 deletions(-) diff --git a/src/transformers/models/sam2/modeling_sam2.py b/src/transformers/models/sam2/modeling_sam2.py index ee877ba81774..2c2d2987865e 100644 --- a/src/transformers/models/sam2/modeling_sam2.py +++ b/src/transformers/models/sam2/modeling_sam2.py @@ -49,20 +49,6 @@ ) -@dataclass -@auto_docstring -class BaseModelOutputWithFeatureMaps(BaseModelOutputWithPooling): - """ - feature_maps (`list[torch.Tensor]`): - List of feature maps from different layers of the model. - feature_maps_position_embeddings (`list[torch.Tensor]`): - List of position embeddings corresponding to the feature maps. - """ - - feature_maps: list[torch.Tensor] | None = None - feature_maps_position_embeddings: list[torch.Tensor] | None = None - - @dataclass @auto_docstring(custom_intro="Base class for the vision encoder's outputs.") class Sam2VisionEncoderOutput(BaseModelOutputWithPooling): diff --git a/src/transformers/models/sam2/modular_sam2.py b/src/transformers/models/sam2/modular_sam2.py index 3b0acd7a4b83..8761be78ea8b 100644 --- a/src/transformers/models/sam2/modular_sam2.py +++ b/src/transformers/models/sam2/modular_sam2.py @@ -73,20 +73,6 @@ logger = logging.get_logger(__name__) -@dataclass -@auto_docstring -class BaseModelOutputWithFeatureMaps(BaseModelOutputWithPooling): - """ - feature_maps (`list[torch.Tensor]`): - List of feature maps from different layers of the model. - feature_maps_position_embeddings (`list[torch.Tensor]`): - List of position embeddings corresponding to the feature maps. - """ - - feature_maps: list[torch.Tensor] | None = None - feature_maps_position_embeddings: list[torch.Tensor] | None = None - - class Sam2FastImageProcessorKwargs(ImagesKwargs, total=False): r""" mask_size (`dict[str, int]`, *optional*): diff --git a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py index 2c67e02b51e6..fa5d6bb1ea52 100644 --- a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py @@ -18,6 +18,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + import math from collections import OrderedDict from collections.abc import Callable, Iterator @@ -49,20 +50,6 @@ ) -@dataclass -@auto_docstring -class BaseModelOutputWithFeatureMaps(BaseModelOutputWithPooling): - """ - feature_maps (`list[torch.Tensor]`): - List of feature maps from different layers of the model. - feature_maps_position_embeddings (`list[torch.Tensor]`): - List of position embeddings corresponding to the feature maps. - """ - - feature_maps: list[torch.Tensor] | None = None - feature_maps_position_embeddings: list[torch.Tensor] | None = None - - class Sam3TrackerVideoInferenceCache: """Cache for vision features and model constants.""" diff --git a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py index a9f6500339d6..09f24be9a84b 100644 --- a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py @@ -12,14 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dataclasses import dataclass import torch from ...configuration_utils import PreTrainedConfig -from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils import TransformersKwargs, can_return_tuple from ..auto import CONFIG_MAPPING, AutoConfig, AutoModel from ..sam2_video.configuration_sam2_video import Sam2VideoMaskDecoderConfig, Sam2VideoPromptEncoderConfig from ..sam2_video.modeling_sam2_video import ( @@ -53,20 +51,6 @@ from ..sam2_video.processing_sam2_video import Sam2VideoProcessor -@dataclass -@auto_docstring -class BaseModelOutputWithFeatureMaps(BaseModelOutputWithPooling): - """ - feature_maps (`list[torch.Tensor]`): - List of feature maps from different layers of the model. - feature_maps_position_embeddings (`list[torch.Tensor]`): - List of position embeddings corresponding to the feature maps. - """ - - feature_maps: list[torch.Tensor] | None = None - feature_maps_position_embeddings: list[torch.Tensor] | None = None - - class Sam3TrackerVideoPromptEncoderConfig(Sam2VideoPromptEncoderConfig): r""" This is the configuration class to store the configuration of a [`Sam3TrackerVideoPromptEncoder`]. The [`Sam3TrackerVideoPromptEncoder`] From 2bd9a49a12136f08f2494a35a265eaf967a0aec5 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 12 Jan 2026 17:17:20 +0100 Subject: [PATCH 072/131] Remove left-over return_dict from prior attempt --- src/transformers/models/sam3/modeling_sam3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/models/sam3/modeling_sam3.py b/src/transformers/models/sam3/modeling_sam3.py index bba5e3123f20..f5f57bf6e681 100644 --- a/src/transformers/models/sam3/modeling_sam3.py +++ b/src/transformers/models/sam3/modeling_sam3.py @@ -2146,7 +2146,6 @@ def get_text_features( self, input_ids: torch.LongTensor, attention_mask: torch.Tensor | None = None, - return_dict: bool = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" From 54550383527f49a8f40979e10b2921a7bce545db Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 12 Jan 2026 19:30:35 +0100 Subject: [PATCH 073/131] Allow for output_hidden_states in theory, but skip impossible tests The tests are failing as edgetam doesn't output hidden_states. It used to, because of a broken TimmWrapper in _can_return_outputs. --- .../models/edgetam/modeling_edgetam.py | 3 +- .../models/edgetam/modular_edgetam.py | 3 +- tests/models/edgetam/test_modeling_edgetam.py | 68 ++++++++++++++++++- 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/edgetam/modeling_edgetam.py b/src/transformers/models/edgetam/modeling_edgetam.py index b02e95952bc9..2a4291829244 100644 --- a/src/transformers/models/edgetam/modeling_edgetam.py +++ b/src/transformers/models/edgetam/modeling_edgetam.py @@ -440,7 +440,7 @@ def forward( raise ValueError("You have to specify pixel_values") # Forward through backbone - backbone_output = self.backbone(pixel_values) + backbone_output = self.backbone(pixel_values, **kwargs) intermediate_hidden_states = backbone_output.last_hidden_state intermediate_hidden_states = [hidden_state.permute(0, 2, 3, 1) for hidden_state in intermediate_hidden_states] @@ -453,6 +453,7 @@ def forward( last_hidden_state=intermediate_hidden_states[-1], fpn_hidden_states=fpn_hidden_states, fpn_position_encoding=fpn_position_encoding, + hidden_states=backbone_output.hidden_states, ) diff --git a/src/transformers/models/edgetam/modular_edgetam.py b/src/transformers/models/edgetam/modular_edgetam.py index 1cce16fcfe0a..07f11cbe661c 100644 --- a/src/transformers/models/edgetam/modular_edgetam.py +++ b/src/transformers/models/edgetam/modular_edgetam.py @@ -199,7 +199,7 @@ def forward( raise ValueError("You have to specify pixel_values") # Forward through backbone - backbone_output = self.backbone(pixel_values) + backbone_output = self.backbone(pixel_values, **kwargs) intermediate_hidden_states = backbone_output.last_hidden_state intermediate_hidden_states = [hidden_state.permute(0, 2, 3, 1) for hidden_state in intermediate_hidden_states] @@ -212,6 +212,7 @@ def forward( last_hidden_state=intermediate_hidden_states[-1], fpn_hidden_states=fpn_hidden_states, fpn_position_encoding=fpn_position_encoding, + hidden_states=backbone_output.hidden_states, ) diff --git a/tests/models/edgetam/test_modeling_edgetam.py b/tests/models/edgetam/test_modeling_edgetam.py index c7a2e23117b1..f384d4e15001 100644 --- a/tests/models/edgetam/test_modeling_edgetam.py +++ b/tests/models/edgetam/test_modeling_edgetam.py @@ -13,11 +13,13 @@ # limitations under the License. """Testing suite for the PyTorch EDGETAM model.""" +import copy import gc import tempfile import unittest import requests +from parameterized import parameterized from transformers import ( EdgeTamConfig, @@ -37,7 +39,7 @@ from transformers.video_utils import load_video from ...test_configuration_common import ConfigTester -from ...test_modeling_common import ModelTesterMixin, floats_tensor +from ...test_modeling_common import TEST_EAGER_MATCHES_SDPA_INFERENCE_PARAMETERIZATION, ModelTesterMixin, floats_tensor from ...test_pipeline_mixin import PipelineTesterMixin @@ -401,6 +403,13 @@ def test_flash_attn_2_inference_equivalence_right_padding(self): def test_eager_matches_sdpa_generate(self): pass + @parameterized.expand(TEST_EAGER_MATCHES_SDPA_INFERENCE_PARAMETERIZATION) + @unittest.skip("Test requires hidden_states in outputs, which is not available in EdgeTamModel.forward's output") + def test_eager_matches_sdpa_inference( + self, name, dtype, padding_side, use_attention_mask, output_attentions, enable_kernels + ): + pass + @unittest.skip("Flash attn test is not configured correctly as we need to configure vision/timm model to 'eager'.") def test_flash_attn_2_inference_equivalence(self): pass @@ -430,10 +439,67 @@ def test_model_from_pretrained(self): def test_sdpa_can_compile_dynamic(self): self.skipTest(reason="EDGETAM model can't be compiled dynamic yet") + def test_model_outputs_equivalence(self): + # Modified from upstream to remove output_hidden_states as the timm model doesn't support it + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + def set_nan_tensor_to_zero(t): + t[t != t] = 0 + return t + + def check_equivalence(model, tuple_inputs, dict_inputs, additional_kwargs={}): + with torch.no_grad(): + tuple_output = model(**tuple_inputs, return_dict=False, **additional_kwargs) + dict_output = model(**dict_inputs, return_dict=True, **additional_kwargs).to_tuple() + + def recursive_check(tuple_object, dict_object): + if isinstance(tuple_object, (list, tuple)): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object, dict_object): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif isinstance(tuple_object, dict): + for tuple_iterable_value, dict_iterable_value in zip( + tuple_object.values(), dict_object.values() + ): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif tuple_object is None: + return + # model might return non-tensors objects (e.g. Cache class) + elif isinstance(tuple_object, torch.Tensor): + self.assertTrue( + torch.allclose( + set_nan_tensor_to_zero(tuple_object), set_nan_tensor_to_zero(dict_object), atol=1e-5 + ), + msg=( + "Tuple and dict output are not equal. Difference:" + f" {torch.max(torch.abs(tuple_object - dict_object))}. Tuple has `nan`:" + f" {torch.isnan(tuple_object).any()} and `inf`: {torch.isinf(tuple_object)}. Dict has" + f" `nan`: {torch.isnan(dict_object).any()} and `inf`: {torch.isinf(dict_object)}." + ), + ) + + recursive_check(tuple_output, dict_output) + + for model_class in self.all_model_classes: + model = model_class(copy.deepcopy(config)) + model.to(torch_device) + model.eval() + + tuple_inputs = self._prepare_for_class(inputs_dict, model_class) + dict_inputs = self._prepare_for_class(inputs_dict, model_class) + check_equivalence(model, tuple_inputs, dict_inputs) + + tuple_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True) + dict_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True) + check_equivalence(model, tuple_inputs, dict_inputs) + @unittest.skip("Cannot set `output_attentions` for timm models.") def test_get_image_features_attentions(self): pass + @unittest.skip("Cannot set `output_hidden_states` for this timm model.") + def test_get_image_features_hidden_states(self): + pass + def prepare_image(): img_url = "https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/truck.jpg" From 3f75c03e0971ee25f8a9f0a13359393234a9370b Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 12 Jan 2026 20:04:09 +0100 Subject: [PATCH 074/131] Introduce tests for get_audio_features, fixed all architectures --- .../audioflamingo3/modeling_audioflamingo3.py | 8 +- .../audioflamingo3/modular_audioflamingo3.py | 14 +- .../models/glmasr/modeling_glmasr.py | 4 + .../models/glmasr/modular_glmasr.py | 4 + .../granite_speech/modeling_granite_speech.py | 58 ++++---- .../qwen2_5_omni/modeling_qwen2_5_omni.py | 4 + .../qwen2_5_omni/modular_qwen2_5_omni.py | 4 + .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 4 + .../test_modeling_qwen3_omni_moe.py | 10 ++ .../test_modeling_seamless_m4t.py | 6 +- .../test_modeling_seamless_m4t_v2.py | 6 +- tests/test_modeling_common.py | 137 ++++++++++++++++++ 12 files changed, 225 insertions(+), 34 deletions(-) diff --git a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py index 4219a75733ac..ea56bc997dbd 100644 --- a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py @@ -35,6 +35,7 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging +from ...utils.generic import check_model_inputs from ..auto import AutoModel, AutoModelForCausalLM from .configuration_audioflamingo3 import AudioFlamingo3Config, AudioFlamingo3EncoderConfig @@ -279,6 +280,11 @@ class AudioFlamingo3Encoder(AudioFlamingo3PreTrainedModel): input_modalities = "audio" _no_split_modules = ["AudioFlamingo3EncoderLayer"] + _can_record_outputs = { + "hidden_states": AudioFlamingo3EncoderLayer, + "attentions": AudioFlamingo3Attention, + } + def __init__(self, config: AudioFlamingo3EncoderConfig): super().__init__(config) self.dropout = config.dropout @@ -316,7 +322,7 @@ def get_input_embeddings(self) -> nn.Module: def set_input_embeddings(self, value: nn.Module): self.conv1 = value - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) def forward( self, input_features: torch.Tensor, diff --git a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py index a2747382dfce..e6086d68a6a8 100644 --- a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py @@ -22,18 +22,23 @@ from ...modeling_outputs import BaseModelOutputWithPooling, CausalLMOutputWithPast from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging +from ...utils.generic import check_model_inputs from ..qwen2_audio.modeling_qwen2_audio import ( Qwen2AudioEncoder, Qwen2AudioPreTrainedModel, ) from ..voxtral.modeling_voxtral import VoxtralForConditionalGeneration, VoxtralMultiModalProjector -from ..whisper.modeling_whisper import WhisperEncoderLayer +from ..whisper.modeling_whisper import WhisperAttention, WhisperEncoderLayer from .configuration_audioflamingo3 import AudioFlamingo3Config logger = logging.get_logger(__name__) +class AudioFlamingo3Attention(WhisperAttention): + pass + + class AudioFlamingo3EncoderLayer(WhisperEncoderLayer): pass @@ -52,7 +57,12 @@ class AudioFlamingo3Encoder(Qwen2AudioEncoder): AudioFlamingo3 encoder: Whisper encoder, average pool (time/2), then LayerNorm. """ - @can_return_tuple + _can_record_outputs = { + "hidden_states": AudioFlamingo3EncoderLayer, + "attentions": AudioFlamingo3Attention, + } + + @check_model_inputs(tie_last_hidden_states=False) def forward( self, input_features: torch.Tensor, diff --git a/src/transformers/models/glmasr/modeling_glmasr.py b/src/transformers/models/glmasr/modeling_glmasr.py index 819f72befe75..412dd4143dd0 100644 --- a/src/transformers/models/glmasr/modeling_glmasr.py +++ b/src/transformers/models/glmasr/modeling_glmasr.py @@ -292,6 +292,10 @@ class GlmAsrEncoder(GlmAsrPreTrainedModel): main_input_name = "input_features" input_modalities = "audio" _no_split_modules = ["GlmAsrEncoderLayer"] + _can_record_outputs = { + "hidden_states": GlmAsrEncoderLayer, + "attentions": GlmAsrAttention, + } def __init__(self, config: GlmAsrEncoderConfig): super().__init__(config) diff --git a/src/transformers/models/glmasr/modular_glmasr.py b/src/transformers/models/glmasr/modular_glmasr.py index 0de89a41b266..0d6ba7ab78ca 100644 --- a/src/transformers/models/glmasr/modular_glmasr.py +++ b/src/transformers/models/glmasr/modular_glmasr.py @@ -305,6 +305,10 @@ class GlmAsrEncoder(GlmAsrPreTrainedModel): main_input_name = "input_features" input_modalities = "audio" _no_split_modules = ["GlmAsrEncoderLayer"] + _can_record_outputs = { + "hidden_states": GlmAsrEncoderLayer, + "attentions": GlmAsrAttention, + } def __init__(self, config: GlmAsrEncoderConfig): super().__init__(config) diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index d3ed7e7ca267..da349ad65288 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -26,6 +26,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_peft_available, logging +from ...utils.generic import check_model_inputs from ..auto import AutoModel, AutoModelForCausalLM from .configuration_granite_speech import GraniteSpeechConfig, GraniteSpeechEncoderConfig @@ -249,10 +250,38 @@ def forward(self, hidden_states: torch.Tensor, attention_dists: torch.Tensor) -> return hidden_states -class GraniteSpeechCTCEncoder(nn.Module): +@auto_docstring +class GraniteSpeechPreTrainedModel(PreTrainedModel): + config: GraniteSpeechConfig + input_modalities = ("audio", "text") + + _supports_flash_attn = False # `blip_2_qformer` dependency does not allow for this + _supports_sdpa = True + + @torch.no_grad() + def _init_weights(self, module: nn.Module): + """Initialize the weights.""" + super()._init_weights(module) + if isinstance(module, GraniteSpeechEncoderProjector): + init.normal_(module.query) + elif isinstance(module, GraniteSpeechCTCEncoder): + context_size = module.config.context_size + seq = torch.arange(context_size) + relpos_dist = seq.view(-1, 1) - seq.view(1, -1) + attention_dists = torch.clamp(relpos_dist, -context_size, context_size) + module.config.max_pos_emb + init.copy_(module.attention_dists, attention_dists) + + +class GraniteSpeechCTCEncoder(GraniteSpeechPreTrainedModel): + config: GraniteSpeechEncoderConfig + input_modalities = "audio" + _can_record_outputs = { + "hidden_states": GraniteSpeechConformerBlock, + "attentions": GraniteSpeechConformerAttention, + } + def __init__(self, config: GraniteSpeechEncoderConfig): - super().__init__() - self.config = config + super().__init__(config) # Precompute clamped relative positional encoding distances seq = torch.arange(config.context_size) @@ -266,6 +295,7 @@ def __init__(self, config: GraniteSpeechEncoderConfig): self.out_mid = nn.Linear(config.output_dim, config.hidden_dim, bias=True) self.num_layers = config.num_layers + @check_model_inputs(tie_last_hidden_states=False) def forward(self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> BaseModelOutputWithPooling: hidden_states = self.input_linear(hidden_states) for idx, layer in enumerate(self.layers, start=1): @@ -279,28 +309,6 @@ def forward(self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwar return BaseModelOutputWithPooling(last_hidden_state=hidden_states) -@auto_docstring -class GraniteSpeechPreTrainedModel(PreTrainedModel): - config: GraniteSpeechConfig - input_modalities = ("audio", "text") - - _supports_flash_attn = False # `blip_2_qformer` dependency does not allow for this - _supports_sdpa = True - - @torch.no_grad() - def _init_weights(self, module: nn.Module): - """Initialize the weights.""" - super()._init_weights(module) - if isinstance(module, GraniteSpeechEncoderProjector): - init.normal_(module.query) - elif isinstance(module, GraniteSpeechCTCEncoder): - context_size = module.config.context_size - seq = torch.arange(context_size) - relpos_dist = seq.view(-1, 1) - seq.view(1, -1) - attention_dists = torch.clamp(relpos_dist, -context_size, context_size) + module.config.max_pos_emb - init.copy_(module.attention_dists, attention_dists) - - @auto_docstring( custom_intro=""" The Granite Speech model, which consists of an audio encoder, projector, and language model. diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index 883e2c199808..ec613f1b76c0 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -779,6 +779,10 @@ class Qwen2_5OmniAudioEncoder(Qwen2_5OmniPreTrainedModel): input_modalities = "audio" _no_split_modules = ["Qwen2_5OmniAudioEncoderLayer"] _supports_sdpa = True + _can_record_outputs = { + "hidden_states": Qwen2_5OmniAudioEncoderLayer, + "attentions": Qwen2_5OmniAudioAttention, + } def __init__(self, config: Qwen2_5OmniAudioEncoderConfig): super().__init__(config) diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index cae986aa6dfb..32da27277c2a 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -1659,6 +1659,10 @@ class Qwen2_5OmniAudioEncoder(Qwen2_5OmniPreTrainedModel): input_modalities = "audio" _no_split_modules = ["Qwen2_5OmniAudioEncoderLayer"] _supports_sdpa = True + _can_record_outputs = { + "hidden_states": Qwen2_5OmniAudioEncoderLayer, + "attentions": Qwen2_5OmniAudioAttention, + } def __init__(self, config: Qwen2_5OmniAudioEncoderConfig): super().__init__(config) diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 5e11d5eb2ee4..c823b4f6bd9c 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -681,6 +681,10 @@ class Qwen3OmniMoeAudioEncoder(Qwen3OmniMoePreTrainedModel): input_modalities = "audio" _no_split_modules = ["Qwen3OmniMoeAudioEncoderLayer"] _supports_sdpa = True + _can_record_outputs = { + "hidden_states": Qwen3OmniMoeAudioEncoderLayer, + "attentions": Qwen3OmniMoeAudioAttention, + } def __init__(self, config: Qwen3OmniMoeAudioEncoderConfig): super().__init__(config) diff --git a/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py b/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py index 23a2090dd40e..25248153fff6 100644 --- a/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py +++ b/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py @@ -628,6 +628,16 @@ def _image_features_get_expected_num_hidden_states(self, model_tester=None): model_tester = self.model_tester return model_tester.vision_config["depth"] + 1 + def _audio_features_get_expected_num_attentions(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + return model_tester.audio_config["encoder_layers"] + + def _audio_features_get_expected_num_hidden_states(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + return model_tester.audio_config["encoder_layers"] + 1 + @require_torch class Qwen3OmniModelIntegrationTest(unittest.TestCase): diff --git a/tests/models/seamless_m4t/test_modeling_seamless_m4t.py b/tests/models/seamless_m4t/test_modeling_seamless_m4t.py index b2ac9930a4f9..be6119d9e549 100644 --- a/tests/models/seamless_m4t/test_modeling_seamless_m4t.py +++ b/tests/models/seamless_m4t/test_modeling_seamless_m4t.py @@ -624,7 +624,7 @@ class SeamlessM4TGenerationTest(unittest.TestCase): # test generation of: SeamlessM4TModel, SeamlessM4TForSpeechToSpeech, SeamlessM4TForSpeechToText, SeamlessM4TForTextToSpeech def setUp(self): - self.speech_model_tester = SeamlessM4TModelTester(self, input_modality="speech") + self.audio_model_tester = SeamlessM4TModelTester(self, input_modality="speech") self.text_model_tester = SeamlessM4TModelTester(self, input_modality="text") self.tmpdirname = tempfile.mkdtemp() @@ -658,7 +658,7 @@ def prepare_text_input(self): return config, input_dict def prepare_speech_input(self): - config, inputs, decoder_input_ids, input_mask, lm_labels = self.speech_model_tester.prepare_config_and_inputs() + config, inputs, decoder_input_ids, input_mask, lm_labels = self.audio_model_tester.prepare_config_and_inputs() input_dict = { "input_features": inputs, @@ -671,7 +671,7 @@ def prepare_speech_input(self): return config, input_dict def prepare_speech_and_text_input(self): - config, inputs, decoder_input_ids, input_mask, lm_labels = self.speech_model_tester.prepare_config_and_inputs() + config, inputs, decoder_input_ids, input_mask, lm_labels = self.audio_model_tester.prepare_config_and_inputs() input_speech = { "input_features": inputs, diff --git a/tests/models/seamless_m4t_v2/test_modeling_seamless_m4t_v2.py b/tests/models/seamless_m4t_v2/test_modeling_seamless_m4t_v2.py index 1a07f8b7d3c7..66af78bd4bc3 100644 --- a/tests/models/seamless_m4t_v2/test_modeling_seamless_m4t_v2.py +++ b/tests/models/seamless_m4t_v2/test_modeling_seamless_m4t_v2.py @@ -631,7 +631,7 @@ class SeamlessM4Tv2GenerationTest(unittest.TestCase): # test generation of: SeamlessM4Tv2Model, SeamlessM4Tv2ForSpeechToSpeech, SeamlessM4Tv2ForSpeechToText, SeamlessM4Tv2ForTextToSpeech def setUp(self): - self.speech_model_tester = SeamlessM4Tv2ModelTester(self, input_modality="speech") + self.audio_model_tester = SeamlessM4Tv2ModelTester(self, input_modality="speech") self.text_model_tester = SeamlessM4Tv2ModelTester(self, input_modality="text") self.tmpdirname = tempfile.mkdtemp() @@ -682,7 +682,7 @@ def prepare_text_input(self, tgt_lang): return config, input_dict def prepare_speech_input(self): - config, inputs, decoder_input_ids, input_mask, lm_labels = self.speech_model_tester.prepare_config_and_inputs() + config, inputs, decoder_input_ids, input_mask, lm_labels = self.audio_model_tester.prepare_config_and_inputs() input_dict = { "input_features": inputs, @@ -695,7 +695,7 @@ def prepare_speech_input(self): return config, input_dict def prepare_speech_and_text_input(self): - config, inputs, decoder_input_ids, input_mask, lm_labels = self.speech_model_tester.prepare_config_and_inputs() + config, inputs, decoder_input_ids, input_mask, lm_labels = self.audio_model_tester.prepare_config_and_inputs() input_speech = { "input_features": inputs, diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 69d097b0adbe..213669423a88 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4595,6 +4595,21 @@ def _image_features_prepare_config_and_inputs(self): } return config, inputs_dict + def _audio_features_prepare_config_and_inputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + if hasattr(self.model_tester, "audio_model_tester"): + _, inputs_dict = self.model_tester.audio_model_tester.prepare_config_and_inputs_for_common() + else: + inputs_dict = { + key: value + for key, value in inputs_dict.items() + if "audio" in key + or "input_values" in key + or "input_features" in key + or key in ["padding_mask", "is_longer", "feature_attention_mask"] + } + return config, inputs_dict + def _text_features_get_expected_num_attentions(self, model_tester=None): if model_tester is None: model_tester = self.model_tester @@ -4631,6 +4646,28 @@ def _image_features_get_expected_num_attentions(self, model_tester=None): def _image_features_get_expected_num_hidden_states(self, model_tester=None): return self._image_features_get_expected_num_attentions(model_tester) + 1 + def _audio_features_get_expected_num_attentions(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + + if hasattr(model_tester, "audio_model_tester"): + return self._audio_features_get_expected_num_attentions(model_tester.audio_model_tester) + elif ( + hasattr(model_tester, "audio_config") + and isinstance(model_tester.audio_config, dict) + and "num_hidden_layers" in model_tester.audio_config + ): + return model_tester.audio_config["num_hidden_layers"] + + if hasattr(model_tester, "expected_num_hidden_layers"): + return model_tester.expected_num_hidden_layers - 1 + elif hasattr(model_tester, "num_hidden_layers"): + return model_tester.num_hidden_layers + raise ValueError("Cannot determine the expected number of layers for audio features") + + def _audio_features_get_expected_num_hidden_states(self, model_tester=None): + return self._audio_features_get_expected_num_attentions(model_tester) + 1 + def test_get_text_features_output(self): for model_class in self.all_model_classes: if not hasattr(model_class, "get_text_features"): @@ -4846,6 +4883,106 @@ def check_attentions_output(inputs_dict, config, model_class): check_attentions_output(inputs_dict, config, model_class) + def test_get_audio_features_output(self): + for model_class in self.all_model_classes: + if not hasattr(model_class, "get_audio_features"): + continue + + config, inputs_dict = self._audio_features_prepare_config_and_inputs() + + model = model_class(config).eval() + model = model.to(torch_device) + + torch.manual_seed(0) + with torch.no_grad(): + outputs = model.get_audio_features(**inputs_dict) + self.assertTrue(isinstance(outputs, ModelOutput), "get_audio_features() must return a BaseModelOutput") + self.assertTrue( + hasattr(outputs, "last_hidden_state"), + "get_audio_features() must return a BaseModelOutput with last_hidden_state", + ) + self.assertTrue( + hasattr(outputs, "pooler_output"), + "get_audio_features() must return a BaseModelOutput with pooler_output", + ) + self.assertTrue( + hasattr(outputs, "hidden_states"), + "get_audio_features() must return a BaseModelOutput with hidden_states", + ) + if self.has_attentions: + self.assertTrue( + hasattr(outputs, "attentions"), + "get_audio_features() must return a BaseModelOutput with attentions", + ) + + def test_get_audio_features_hidden_states(self): + def check_hidden_states_output(inputs_dict, config, model_class): + model = model_class(copy.deepcopy(config)) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model.get_audio_features(**inputs_dict) + hidden_states = outputs.hidden_states + expected_num_hidden_states = self._audio_features_get_expected_num_hidden_states() + self.assertIsNotNone(hidden_states, "hidden_states should not be None") + self.assertEqual(len(hidden_states), expected_num_hidden_states, "Number of hidden states layers mismatch") + + for model_class in self.all_model_classes: + if not hasattr(model_class, "get_audio_features"): + continue + + config, inputs_dict = self._audio_features_prepare_config_and_inputs() + + inputs_dict["output_hidden_states"] = True + check_hidden_states_output(inputs_dict, config, model_class) + + # check that output_hidden_states also work using config + del inputs_dict["output_hidden_states"] + config.output_hidden_states = True + for k in config.sub_configs: + if getattr(config, k) is not None: + getattr(config, k).output_hidden_states = True + + check_hidden_states_output(inputs_dict, config, model_class) + + def test_get_audio_features_attentions(self): + def check_attentions_output(inputs_dict, config, model_class): + model = model_class(copy.deepcopy(config)) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model.get_audio_features(**inputs_dict) + attentions = outputs.attentions + expected_num_attentions = self._audio_features_get_expected_num_attentions() + self.assertIsNotNone(attentions, "attentions should not be None") + self.assertEqual(len(attentions), expected_num_attentions, "Number of attention layers mismatch") + + if not self.has_attentions: + return + + for model_class in self.all_model_classes: + if not hasattr(model_class, "get_audio_features"): + continue + + config, inputs_dict = self._audio_features_prepare_config_and_inputs() + + # force eager attention to support output attentions + config._attn_implementation = "eager" + inputs_dict["output_hidden_states"] = False + inputs_dict["output_attentions"] = True + check_attentions_output(inputs_dict, config, model_class) + + # check that output_attentions also work using config + del inputs_dict["output_attentions"] + config.output_attentions = True + for k in config.sub_configs: + if getattr(config, k) is not None: + getattr(config, k).output_attentions = True + + check_attentions_output(inputs_dict, config, model_class) + global_rng = random.Random() From 5e7d821f70102e3138e3e89501119197af3db6e8 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 12 Jan 2026 22:28:03 +0100 Subject: [PATCH 075/131] Introduce tests for get_video_features, only ernie4_5_vl_moe is failing It's failing as the split_sizes gets made too small, such that the video_embeds doesn't sum to the split_sizes anymore. I'm not sure how to best tackle it. I also removed the get_video_features from PaddleOCR_vl, as I don't think it's meant to be used with video --- .../modeling_llava_onevision.py | 13 +- .../modular_llava_onevision.py | 13 +- .../paddleocr_vl/modeling_paddleocr_vl.py | 34 ----- .../paddleocr_vl/modular_paddleocr_vl.py | 6 + .../video_llava/modeling_video_llava.py | 3 +- .../models/x_clip/modeling_x_clip.py | 9 +- .../test_modeling_instructblipvideo.py | 6 + .../test_modeling_llava_next_video.py | 5 + .../test_modeling_llava_onevision.py | 19 ++- .../test_modeling_qwen3_omni_moe.py | 10 ++ tests/models/x_clip/test_modeling_x_clip.py | 7 + tests/test_modeling_common.py | 142 ++++++++++++++++++ 12 files changed, 222 insertions(+), 45 deletions(-) diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index dc3d70987422..ef73b2ea9ddb 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -603,8 +603,8 @@ def forward( def get_video_features( self, pixel_values: torch.FloatTensor, - vision_feature_layer: int | list[int], - vision_feature_select_strategy: str, + vision_feature_layer: int | list[int] | None = None, + vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: """ @@ -625,6 +625,15 @@ def get_video_features( video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches and are of shape `(num_videos, video_length, embed_dim)`). """ + vision_feature_layer = ( + vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer + ) + vision_feature_select_strategy = ( + vision_feature_select_strategy + if vision_feature_select_strategy is not None + else self.config.vision_feature_select_strategy + ) + batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) kwargs["output_hidden_states"] = True diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index 2d1efab52b9b..b240977db11c 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -410,8 +410,8 @@ def get_image_features( def get_video_features( self, pixel_values: torch.FloatTensor, - vision_feature_layer: int | list[int], - vision_feature_select_strategy: str, + vision_feature_layer: int | list[int] | None = None, + vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: """ @@ -432,6 +432,15 @@ def get_video_features( video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches and are of shape `(num_videos, video_length, embed_dim)`). """ + vision_feature_layer = ( + vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer + ) + vision_feature_select_strategy = ( + vision_feature_select_strategy + if vision_feature_select_strategy is not None + else self.config.vision_feature_select_strategy + ) + batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) kwargs["output_hidden_states"] = True diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index e17c28df5b5b..9ce9c653c77e 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -1201,30 +1201,6 @@ def get_rope_index( return position_ids, mrope_position_deltas - @can_return_tuple - def get_video_features( - self, - pixel_values_videos: torch.FloatTensor, - video_grid_thw: torch.LongTensor | None = None, - **kwargs: Unpack[TransformersKwargs], - ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. - """ - pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) - split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) - vision_outputs.pooler_output = video_embeds - - return vision_outputs - @can_return_tuple def get_image_features( self, @@ -1379,16 +1355,6 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) - def get_video_features( - self, - pixel_values_videos: torch.FloatTensor, - video_grid_thw: torch.LongTensor | None = None, - **kwargs: Unpack[TransformersKwargs], - ) -> tuple | BaseModelOutputWithPooling: - return self.model.get_video_features( - pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs - ) - def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py index 7625bb041a36..85f183920259 100644 --- a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py @@ -1118,6 +1118,9 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.embed_tokens = value + def get_video_features(self): + raise AttributeError("PaddleOCRVLModel does not support video.") + @can_return_tuple def get_image_features( self, @@ -1258,6 +1261,9 @@ class PaddleOCRVLForConditionalGeneration(Qwen2VLForConditionalGeneration): } _keys_to_ignore_on_load_unexpected = ["packing_position_embedding", "vision_model.head"] + def get_video_features(self): + raise AttributeError("PaddleOCRVLForConditionalGeneration does not support video.") + @can_return_tuple @auto_docstring def forward( diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index a76578103bd5..ddbb638624e8 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -257,7 +257,8 @@ def get_video_features( batch_size_vid, num_frames, channels, height, width = pixel_values_videos.shape pixel_values = pixel_values_videos.reshape(batch_size_vid * num_frames, channels, height, width) - video_outputs = self.video_tower(pixel_values, output_hidden_states=True, return_dict=True) + kwargs["output_hidden_states"] = True + video_outputs = self.video_tower(pixel_values, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/x_clip/modeling_x_clip.py b/src/transformers/models/x_clip/modeling_x_clip.py index e047653a0771..ef88cda08e63 100644 --- a/src/transformers/models/x_clip/modeling_x_clip.py +++ b/src/transformers/models/x_clip/modeling_x_clip.py @@ -1325,14 +1325,15 @@ def get_video_features( batch_size, num_frames, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.reshape(-1, num_channels, height, width) - vision_outputs: BaseModelOutputWithPooling = self.vision_model(pixel_values=pixel_values, **kwargs) - video_embeds = vision_outputs.pooler_output + video_outputs: BaseModelOutputWithPooling = self.vision_model(pixel_values=pixel_values, **kwargs) + video_embeds = video_outputs.pooler_output video_embeds = self.visual_projection(video_embeds) cls_features = video_embeds.view(batch_size, num_frames, -1) - mit_outputs: BaseModelOutputWithPooling = self.mit(cls_features, **kwargs) + mit_outputs: BaseModelOutputWithPooling = self.mit(cls_features, return_dict=True, **kwargs) + video_outputs.pooler_output = mit_outputs.pooler_output - return mit_outputs + return video_outputs @auto_docstring def forward( diff --git a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py index 0046ccb0bf58..d0db595c720c 100644 --- a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py +++ b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py @@ -625,6 +625,12 @@ def test_sdpa_can_dispatch_composite_models(self): ): raise ValueError("The eager model should not have SDPA attention layers") + def _video_features_prepare_config_and_inputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + del inputs_dict["input_ids"] + del inputs_dict["attention_mask"] + return config, inputs_dict + # We will verify our results on an image of cute cats def prepare_video(): diff --git a/tests/models/llava_next_video/test_modeling_llava_next_video.py b/tests/models/llava_next_video/test_modeling_llava_next_video.py index aba7b644f733..5e23baa53db7 100644 --- a/tests/models/llava_next_video/test_modeling_llava_next_video.py +++ b/tests/models/llava_next_video/test_modeling_llava_next_video.py @@ -329,6 +329,11 @@ def test_flash_attn_2_fp32_ln(self): def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): pass + def _video_features_prepare_config_and_inputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + inputs_dict = {"pixel_values": inputs_dict["pixel_values_videos"]} + return config, inputs_dict + @require_torch class LlavaNextVideoForConditionalGenerationIntegrationTest(unittest.TestCase): diff --git a/tests/models/llava_onevision/test_modeling_llava_onevision.py b/tests/models/llava_onevision/test_modeling_llava_onevision.py index 603bd260ad75..865f8a076d08 100644 --- a/tests/models/llava_onevision/test_modeling_llava_onevision.py +++ b/tests/models/llava_onevision/test_modeling_llava_onevision.py @@ -152,13 +152,22 @@ def prepare_config_and_inputs(self): self.vision_config["image_size"], ] ) + pixel_values_videos = floats_tensor( + [ + self.batch_size, + 8, + self.vision_config["num_channels"], + self.vision_config["image_size"], + self.vision_config["image_size"], + ] + ) config = self.get_config() - return config, pixel_values + return config, pixel_values, pixel_values_videos def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() - config, pixel_values = config_and_inputs + config, pixel_values, pixel_values_videos = config_and_inputs input_ids = ids_tensor([self.batch_size, self.seq_length], config.text_config.vocab_size - 2) + 2 attention_mask = torch.ones(input_ids.shape, dtype=torch.long).to(torch_device) @@ -170,6 +179,7 @@ def prepare_config_and_inputs_for_common(self): inputs_dict = { "pixel_values": pixel_values, + "pixel_values_videos": pixel_values_videos, "image_sizes": torch.tensor([[45, 45]] * self.batch_size), "input_ids": input_ids, "attention_mask": attention_mask, @@ -292,6 +302,11 @@ def test_training_gradient_checkpointing_use_reentrant_false(self): def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): pass + def _video_features_prepare_config_and_inputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + inputs_dict = {"pixel_values": inputs_dict["pixel_values_videos"]} + return config, inputs_dict + @require_torch class LlavaOnevisionForConditionalGenerationIntegrationTest(unittest.TestCase): diff --git a/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py b/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py index 25248153fff6..8f18b7e8ba29 100644 --- a/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py +++ b/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py @@ -638,6 +638,16 @@ def _audio_features_get_expected_num_hidden_states(self, model_tester=None): model_tester = self.model_tester return model_tester.audio_config["encoder_layers"] + 1 + def _video_features_get_expected_num_attentions(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + return model_tester.vision_config["depth"] + + def _video_features_get_expected_num_hidden_states(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + return model_tester.vision_config["depth"] + 1 + @require_torch class Qwen3OmniModelIntegrationTest(unittest.TestCase): diff --git a/tests/models/x_clip/test_modeling_x_clip.py b/tests/models/x_clip/test_modeling_x_clip.py index dfa7084403f8..71afd6ddccfe 100644 --- a/tests/models/x_clip/test_modeling_x_clip.py +++ b/tests/models/x_clip/test_modeling_x_clip.py @@ -575,6 +575,13 @@ def test_model_from_pretrained(self): model = XCLIPModel.from_pretrained(model_name) self.assertIsNotNone(model) + def _video_features_prepare_config_and_inputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + del inputs_dict["input_ids"] + del inputs_dict["attention_mask"] + del inputs_dict["return_loss"] + return config, inputs_dict + # We will verify our results on a spaghetti video def prepare_video(): diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 213669423a88..546f36933bf8 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4610,6 +4610,24 @@ def _audio_features_prepare_config_and_inputs(self): } return config, inputs_dict + def _video_features_prepare_config_and_inputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + if hasattr(self.model_tester, "video_model_tester"): + _, inputs_dict = self.model_tester.video_model_tester.prepare_config_and_inputs_for_common() + else: + if "pixel_values" in inputs_dict and "pixel_values_videos" not in inputs_dict: + inputs_dict["pixel_values_videos"] = inputs_dict.pop("pixel_values") + if "image_grid_thw" in inputs_dict and "video_grid_thw" not in inputs_dict: + inputs_dict["video_grid_thw"] = inputs_dict.pop("image_grid_thw") + if "image_merge_sizes" in inputs_dict and "video_merge_sizes" not in inputs_dict: + inputs_dict["video_merge_sizes"] = inputs_dict.pop("image_merge_sizes") + inputs_dict = { + key: value + for key, value in inputs_dict.items() + if "video" in key or key in ["vision_feature_layer", "vision_feature_select_strategy", "cu_seqlens"] + } + return config, inputs_dict + def _text_features_get_expected_num_attentions(self, model_tester=None): if model_tester is None: model_tester = self.model_tester @@ -4668,6 +4686,30 @@ def _audio_features_get_expected_num_attentions(self, model_tester=None): def _audio_features_get_expected_num_hidden_states(self, model_tester=None): return self._audio_features_get_expected_num_attentions(model_tester) + 1 + def _video_features_get_expected_num_attentions(self, model_tester=None): + if model_tester is None: + model_tester = self.model_tester + + if hasattr(model_tester, "video_model_tester"): + return self._video_features_get_expected_num_attentions(model_tester.video_model_tester) + if hasattr(model_tester, "vision_model_tester"): + return self._video_features_get_expected_num_attentions(model_tester.vision_model_tester) + elif ( + hasattr(model_tester, "video_config") + and isinstance(model_tester.video_config, dict) + and "num_hidden_layers" in model_tester.video_config + ): + return model_tester.video_config["num_hidden_layers"] + + if hasattr(model_tester, "expected_num_hidden_layers"): + return model_tester.expected_num_hidden_layers - 1 + elif hasattr(model_tester, "num_hidden_layers"): + return model_tester.num_hidden_layers + raise ValueError("Cannot determine the expected number of layers for video features") + + def _video_features_get_expected_num_hidden_states(self, model_tester=None): + return self._video_features_get_expected_num_attentions(model_tester) + 1 + def test_get_text_features_output(self): for model_class in self.all_model_classes: if not hasattr(model_class, "get_text_features"): @@ -4983,6 +5025,106 @@ def check_attentions_output(inputs_dict, config, model_class): check_attentions_output(inputs_dict, config, model_class) + def test_get_video_features_output(self): + for model_class in self.all_model_classes: + if not hasattr(model_class, "get_video_features"): + continue + + config, inputs_dict = self._video_features_prepare_config_and_inputs() + + model = model_class(config).eval() + model = model.to(torch_device) + + torch.manual_seed(0) + with torch.no_grad(): + outputs = model.get_video_features(**inputs_dict) + self.assertTrue(isinstance(outputs, ModelOutput), "get_video_features() must return a BaseModelOutput") + self.assertTrue( + hasattr(outputs, "last_hidden_state"), + "get_video_features() must return a BaseModelOutput with last_hidden_state", + ) + self.assertTrue( + hasattr(outputs, "pooler_output"), + "get_video_features() must return a BaseModelOutput with pooler_output", + ) + self.assertTrue( + hasattr(outputs, "hidden_states"), + "get_video_features() must return a BaseModelOutput with hidden_states", + ) + if self.has_attentions: + self.assertTrue( + hasattr(outputs, "attentions"), + "get_video_features() must return a BaseModelOutput with attentions", + ) + + def test_get_video_features_hidden_states(self): + def check_hidden_states_output(inputs_dict, config, model_class): + model = model_class(copy.deepcopy(config)) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model.get_video_features(**inputs_dict) + hidden_states = outputs.hidden_states + expected_num_hidden_states = self._video_features_get_expected_num_hidden_states() + self.assertIsNotNone(hidden_states, "hidden_states should not be None") + self.assertEqual(len(hidden_states), expected_num_hidden_states, "Number of hidden states layers mismatch") + + for model_class in self.all_model_classes: + if not hasattr(model_class, "get_video_features"): + continue + + config, inputs_dict = self._video_features_prepare_config_and_inputs() + + inputs_dict["output_hidden_states"] = True + check_hidden_states_output(inputs_dict, config, model_class) + + # check that output_hidden_states also work using config + del inputs_dict["output_hidden_states"] + config.output_hidden_states = True + for k in config.sub_configs: + if getattr(config, k) is not None: + getattr(config, k).output_hidden_states = True + + check_hidden_states_output(inputs_dict, config, model_class) + + def test_get_video_features_attentions(self): + def check_attentions_output(inputs_dict, config, model_class): + model = model_class(copy.deepcopy(config)) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model.get_video_features(**inputs_dict) + attentions = outputs.attentions + expected_num_attentions = self._video_features_get_expected_num_attentions() + self.assertIsNotNone(attentions, "attentions should not be None") + self.assertEqual(len(attentions), expected_num_attentions, "Number of attention layers mismatch") + + if not self.has_attentions: + return + + for model_class in self.all_model_classes: + if not hasattr(model_class, "get_video_features"): + continue + + config, inputs_dict = self._video_features_prepare_config_and_inputs() + + # force eager attention to support output attentions + config._attn_implementation = "eager" + inputs_dict["output_hidden_states"] = False + inputs_dict["output_attentions"] = True + check_attentions_output(inputs_dict, config, model_class) + + # check that output_attentions also work using config + del inputs_dict["output_attentions"] + config.output_attentions = True + for k in config.sub_configs: + if getattr(config, k) is not None: + getattr(config, k).output_attentions = True + + check_attentions_output(inputs_dict, config, model_class) + global_rng = random.Random() From 1b8ab38b91afe0cc1bb2e466aa3535ad2a32c3c5 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 12 Jan 2026 22:29:22 +0100 Subject: [PATCH 076/131] Call post_init on GraniteSpeechCTCEncoder, which was given a PreTrainedModel subclass --- .../models/granite_speech/modeling_granite_speech.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index da349ad65288..4e2dc907d907 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -294,6 +294,7 @@ def __init__(self, config: GraniteSpeechEncoderConfig): self.out = nn.Linear(config.hidden_dim, config.output_dim, bias=True) self.out_mid = nn.Linear(config.output_dim, config.hidden_dim, bias=True) self.num_layers = config.num_layers + self.post_init() @check_model_inputs(tie_last_hidden_states=False) def forward(self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> BaseModelOutputWithPooling: From 34677988822b61d026a7b3fa7c8ccfc0703a0453 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 13 Jan 2026 08:51:38 +0100 Subject: [PATCH 077/131] Update llava_onevision test suite, only create video pixel_values in new method Instead of in the common one, as that negatively affects other tests (as there's no video tokens in the inputs_ids then) --- .../test_modeling_llava_onevision.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/models/llava_onevision/test_modeling_llava_onevision.py b/tests/models/llava_onevision/test_modeling_llava_onevision.py index 865f8a076d08..77d72ff110fd 100644 --- a/tests/models/llava_onevision/test_modeling_llava_onevision.py +++ b/tests/models/llava_onevision/test_modeling_llava_onevision.py @@ -152,22 +152,13 @@ def prepare_config_and_inputs(self): self.vision_config["image_size"], ] ) - pixel_values_videos = floats_tensor( - [ - self.batch_size, - 8, - self.vision_config["num_channels"], - self.vision_config["image_size"], - self.vision_config["image_size"], - ] - ) config = self.get_config() - return config, pixel_values, pixel_values_videos + return config, pixel_values def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() - config, pixel_values, pixel_values_videos = config_and_inputs + config, pixel_values = config_and_inputs input_ids = ids_tensor([self.batch_size, self.seq_length], config.text_config.vocab_size - 2) + 2 attention_mask = torch.ones(input_ids.shape, dtype=torch.long).to(torch_device) @@ -179,7 +170,6 @@ def prepare_config_and_inputs_for_common(self): inputs_dict = { "pixel_values": pixel_values, - "pixel_values_videos": pixel_values_videos, "image_sizes": torch.tensor([[45, 45]] * self.batch_size), "input_ids": input_ids, "attention_mask": attention_mask, @@ -303,8 +293,17 @@ def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): pass def _video_features_prepare_config_and_inputs(self): - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - inputs_dict = {"pixel_values": inputs_dict["pixel_values_videos"]} + pixel_values_videos = floats_tensor( + [ + self.model_tester.batch_size, + 8, + self.model_tester.vision_config["num_channels"], + self.model_tester.vision_config["image_size"], + self.model_tester.vision_config["image_size"], + ] + ) + config = self.model_tester.get_config() + inputs_dict = {"pixel_values": pixel_values_videos} return config, inputs_dict From 6f23bf5ac29bd4d457423150a3dfc0be36330e77 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 13 Jan 2026 10:04:24 +0100 Subject: [PATCH 078/131] Create custom video input for ernie4_5_vl_moe --- .../test_modeling_ernie4_5_vl_moe.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/models/ernie4_5_vl_moe/test_modeling_ernie4_5_vl_moe.py b/tests/models/ernie4_5_vl_moe/test_modeling_ernie4_5_vl_moe.py index dbc41f22064b..1d14b6ca414a 100644 --- a/tests/models/ernie4_5_vl_moe/test_modeling_ernie4_5_vl_moe.py +++ b/tests/models/ernie4_5_vl_moe/test_modeling_ernie4_5_vl_moe.py @@ -266,6 +266,27 @@ def test_inputs_embeds_matches_input_ids(self): def test_multi_gpu_data_parallel_forward(self): pass + def _video_features_prepare_config_and_inputs(self): + config = self.model_tester.get_config() + patch_size = config.vision_config.patch_size + batch_size = self.model_tester.batch_size + image_size = self.model_tester.image_size + num_channels = self.model_tester.num_channels + num_frames = 12 + pixel_values_videos = floats_tensor( + [num_frames * batch_size * (image_size**2) // (patch_size**2), num_channels * (patch_size**2)] + ) + + patches_per_side = image_size // patch_size + video_grid_thw = torch.tensor( + [[num_frames, patches_per_side, patches_per_side]] * batch_size, device=torch_device + ) + inputs_dict = { + "pixel_values_videos": pixel_values_videos, + "video_grid_thw": video_grid_thw, + } + return config, inputs_dict + @slow @require_torch_large_accelerator(memory=64) # Tested on A100 From a8e5f920d04afa9a8e7c3139acbeb2b64716f098 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 13 Jan 2026 10:29:18 +0100 Subject: [PATCH 079/131] Skip CLIP family tests; they don't support output_hidden_states/output_attentions due to bug --- tests/models/clip/test_modeling_clip.py | 20 +++++++++++++++++++ .../models/fast_vlm/test_modeling_fast_vlm.py | 5 +++++ .../metaclip_2/test_modeling_metaclip_2.py | 20 +++++++++++++++++++ .../test_modeling_perception_lm.py | 5 +++++ tests/models/siglip/test_modeling_siglip.py | 10 ++++++++++ tests/models/siglip2/test_modeling_siglip2.py | 10 ++++++++++ 6 files changed, 70 insertions(+) diff --git a/tests/models/clip/test_modeling_clip.py b/tests/models/clip/test_modeling_clip.py index 4ce1b9faa6aa..c0a393efed44 100644 --- a/tests/models/clip/test_modeling_clip.py +++ b/tests/models/clip/test_modeling_clip.py @@ -595,6 +595,26 @@ def test_sdpa_can_dispatch_on_flash(self): def test_sdpa_can_compile_dynamic(self): self.skipTest(reason="CLIP model can't be compiled dynamic, error in clip_loss`") + @unittest.skip(reason="The CLIP family currently does not work with output_attentions.") + def test_get_text_features_attentions(self): + # This test should no longer be skipped once this architecture is refactored to work with output_attentions. + pass + + @unittest.skip(reason="The CLIP family currently does not work with output_hidden_states.") + def test_get_text_features_hidden_states(self): + # This test should no longer be skipped once this architecture is refactored to work with output_hidden_states. + pass + + @unittest.skip(reason="The CLIP family currently does not work with output_attentions.") + def test_get_image_features_attentions(self): + # This test should no longer be skipped once this architecture is refactored to work with output_attentions. + pass + + @unittest.skip(reason="The CLIP family currently does not work with output_hidden_states.") + def test_get_image_features_hidden_states(self): + # This test should no longer be skipped once this architecture is refactored to work with output_hidden_states. + pass + class CLIPForImageClassificationModelTester(CLIPModelTester): def __init__(self, parent): diff --git a/tests/models/fast_vlm/test_modeling_fast_vlm.py b/tests/models/fast_vlm/test_modeling_fast_vlm.py index 05b253559f28..dd9540b6aab4 100644 --- a/tests/models/fast_vlm/test_modeling_fast_vlm.py +++ b/tests/models/fast_vlm/test_modeling_fast_vlm.py @@ -231,6 +231,11 @@ def test_can_be_initialized_on_meta(self): def test_get_image_features_attentions(self): pass + def _image_features_get_expected_num_hidden_states(self, model_tester=None): + # For models that rely on timm for their vision backend, it's hard to infer how many layers the model has + # from the timm config alone. So, we're just hardcoding the expected number of hidden states here. + return 2 + @require_torch @slow diff --git a/tests/models/metaclip_2/test_modeling_metaclip_2.py b/tests/models/metaclip_2/test_modeling_metaclip_2.py index 40e17e652098..29621585c476 100644 --- a/tests/models/metaclip_2/test_modeling_metaclip_2.py +++ b/tests/models/metaclip_2/test_modeling_metaclip_2.py @@ -606,6 +606,26 @@ def test_sdpa_can_dispatch_on_flash(self): def test_sdpa_can_compile_dynamic(self): self.skipTest(reason="MetaClip2 model can't be compiled dynamic, error in metaclip_2_loss`") + @unittest.skip(reason="The MetaCLIP2 family currently does not work with output_attentions.") + def test_get_text_features_attentions(self): + # This test should no longer be skipped once this architecture is refactored to work with output_attentions. + pass + + @unittest.skip(reason="The MetaCLIP2 family currently does not work with output_hidden_states.") + def test_get_text_features_hidden_states(self): + # This test should no longer be skipped once this architecture is refactored to work with output_hidden_states. + pass + + @unittest.skip(reason="The MetaCLIP2 family currently does not work with output_attentions.") + def test_get_image_features_attentions(self): + # This test should no longer be skipped once this architecture is refactored to work with output_attentions. + pass + + @unittest.skip(reason="The MetaCLIP2 family currently does not work with output_hidden_states.") + def test_get_image_features_hidden_states(self): + # This test should no longer be skipped once this architecture is refactored to work with output_hidden_states. + pass + class MetaClip2ForImageClassificationModelTester(MetaClip2ModelTester): def __init__(self, parent): diff --git a/tests/models/perception_lm/test_modeling_perception_lm.py b/tests/models/perception_lm/test_modeling_perception_lm.py index c402b1a8cef2..b492f29d75e3 100644 --- a/tests/models/perception_lm/test_modeling_perception_lm.py +++ b/tests/models/perception_lm/test_modeling_perception_lm.py @@ -371,6 +371,11 @@ def test_generate_compilation_all_outputs(self): def test_get_image_features_attentions(self): pass + def _image_features_get_expected_num_hidden_states(self, model_tester=None): + # For models that rely on timm for their vision backend, it's hard to infer how many layers the model has + # from the timm config alone. So, we're just hardcoding the expected number of hidden states here. + return 2 + TEST_MODEL_PATH = "facebook/Perception-LM-1B" diff --git a/tests/models/siglip/test_modeling_siglip.py b/tests/models/siglip/test_modeling_siglip.py index 4cc51ebaaba4..eb5a1e45dc7e 100644 --- a/tests/models/siglip/test_modeling_siglip.py +++ b/tests/models/siglip/test_modeling_siglip.py @@ -515,6 +515,16 @@ def test_load_vision_text_config(self): text_config = SiglipTextConfig.from_pretrained(tmp_dir_name) self.assertDictEqual(config.text_config.to_dict(), text_config.to_dict()) + @unittest.skip(reason="The SigLIP family currently does not work with output_attentions on the text side.") + def test_get_text_features_attentions(self): + # This test should no longer be skipped once this architecture is refactored to work with output_attentions. + pass + + @unittest.skip(reason="The SigLIP family currently does not work with output_hidden_states on the text side.") + def test_get_text_features_hidden_states(self): + # This test should no longer be skipped once this architecture is refactored to work with output_hidden_states. + pass + @slow def test_model_from_pretrained(self): model_name = "google/siglip-base-patch16-224" diff --git a/tests/models/siglip2/test_modeling_siglip2.py b/tests/models/siglip2/test_modeling_siglip2.py index f45165cbf200..ddf815b988d8 100644 --- a/tests/models/siglip2/test_modeling_siglip2.py +++ b/tests/models/siglip2/test_modeling_siglip2.py @@ -592,6 +592,16 @@ def test_load_vision_text_config(self): text_config = Siglip2TextConfig.from_pretrained(tmp_dir_name) self.assertDictEqual(config.text_config.to_dict(), text_config.to_dict()) + @unittest.skip(reason="The SigLIP2 family currently does not work with output_attentions on the text side.") + def test_get_text_features_attentions(self): + # This test should no longer be skipped once this architecture is refactored to work with output_attentions. + pass + + @unittest.skip(reason="The SigLIP2 family currently does not work with output_hidden_states on the text side.") + def test_get_text_features_hidden_states(self): + # This test should no longer be skipped once this architecture is refactored to work with output_hidden_states. + pass + @slow def test_model_from_pretrained(self): model_name = "google/siglip2-base-patch16-naflex" From 508955e415bd93faf0794f16018aa0e8343f0cfa Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 13 Jan 2026 12:40:30 +0100 Subject: [PATCH 080/131] Breaking: update Blip2Model.get_text_features to no longer output logits --- .../models/blip_2/modeling_blip_2.py | 22 +++++++++---------- tests/models/blip_2/test_modeling_blip_2.py | 3 +-- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 0b73ec062d2c..429b2ec589c2 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -28,11 +28,10 @@ from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ( BaseModelOutput, + BaseModelOutputWithPast, BaseModelOutputWithPastAndCrossAttentions, BaseModelOutputWithPooling, BaseModelOutputWithPoolingAndCrossAttentions, - CausalLMOutputWithPast, - Seq2SeqLMOutput, ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -1090,7 +1089,7 @@ def get_text_features( decoder_attention_mask: torch.Tensor | None = None, labels: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> tuple | BaseModelOutput: + ) -> tuple | BaseModelOutputWithPooling: r""" decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. @@ -1127,23 +1126,22 @@ def get_text_features( ```""" if self.config.use_decoder_only_language_model: - text_outputs: CausalLMOutputWithPast = self.language_model( + text_outputs: BaseModelOutputWithPast = self.language_model.base_model( input_ids=input_ids, attention_mask=attention_mask, **kwargs, ) else: - inputs_embeds = self.language_model.get_input_embeddings()(input_ids) - text_outputs: Seq2SeqLMOutput = self.language_model( - inputs_embeds=inputs_embeds, + text_outputs: BaseModelOutputWithPastAndCrossAttentions = self.language_model.get_encoder()( + input_ids=input_ids, attention_mask=attention_mask, - decoder_input_ids=decoder_input_ids, - decoder_attention_mask=decoder_attention_mask, - labels=labels, **kwargs, ) - - return text_outputs + return BaseModelOutputWithPooling( + last_hidden_state=text_outputs.last_hidden_state, + hidden_states=text_outputs.hidden_states, + attentions=text_outputs.attentions, + ) @can_return_tuple @auto_docstring diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index 02c91307351a..627eed2fe423 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -964,13 +964,12 @@ def test_get_text_features(self): inputs_dict = { "input_ids": torch.LongTensor([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]).to(torch_device), "attention_mask": torch.LongTensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(torch_device), - "decoder_input_ids": torch.LongTensor([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]).to(torch_device), } model = Blip2Model(config).to(torch_device) model.eval() text_features = model.get_text_features(**inputs_dict) - self.assertEqual(text_features.logits[0].shape, (10, config.text_config.vocab_size)) + self.assertEqual(text_features[0].shape, (1, 10, config.text_config.hidden_size)) def test_get_image_features(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() From df4d75122b812199a9ffaea3f27aac6d7937a8c6 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 13 Jan 2026 12:52:42 +0100 Subject: [PATCH 081/131] Satisfy test_num_layers_is_small test for align --- tests/models/align/test_modeling_align.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/models/align/test_modeling_align.py b/tests/models/align/test_modeling_align.py index 7042bf999452..4dbf8f899abb 100644 --- a/tests/models/align/test_modeling_align.py +++ b/tests/models/align/test_modeling_align.py @@ -86,8 +86,6 @@ def __init__( self.is_training = is_training self.hidden_act = hidden_act - self.num_hidden_layers = sum(math.ceil(self.depth_coefficient * repeat) for repeat in self.num_block_repeats) - def prepare_config_and_inputs(self): pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]) config = self.get_config() @@ -504,6 +502,12 @@ def test_load_vision_text_config(self): text_config = AlignTextConfig.from_pretrained(tmp_dir_name) self.assertDictEqual(config.text_config.to_dict(), text_config.to_dict()) + def _image_features_get_expected_num_attentions(self, model_tester=None): + return sum( + math.ceil(self.model_tester.vision_model_tester.depth_coefficient * repeat) + for repeat in self.model_tester.vision_model_tester.num_block_repeats + ) + @slow def test_model_from_pretrained(self): model_name = "kakaobrain/align-base" From 1254b2953f652e283c427dc8ea3db8915a1859aa Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 13 Jan 2026 17:58:35 +0100 Subject: [PATCH 082/131] Test against last_hidden_state against batch_size and hidden_size 19 failures, mostly if architectures merge the first dimension with e.g. num_frames for videos, or swap dimensions from the norm with the hidden_state at index 1 in a 4d-tensor I don't think it's reasonable to expect these to be 'fixed', they would require drastic changes in the architectures or somewhat arbitrary changes in the post-processing of the hidden states. --- tests/test_modeling_common.py | 71 +++++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 546f36933bf8..f16947dc2f6c 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4741,6 +4741,16 @@ def test_get_text_features_output(self): hasattr(outputs, "attentions"), "get_text_features() must return a BaseModelOutput with attentions" ) + # Test against (batch_size, seq_len, hidden_size) + last_hidden_state = outputs.last_hidden_state + expected_hidden_size = config.text_config.hidden_size + expected_shape = ( + inputs_dict["input_ids"].shape[0], + inputs_dict["input_ids"].shape[1], + expected_hidden_size, + ) + self.assertEqual(last_hidden_state.shape, expected_shape, "last_hidden_state shape mismatch") + def test_get_text_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) @@ -4842,18 +4852,26 @@ def test_get_image_features_output(self): "get_image_features() must return a BaseModelOutput with attentions", ) - # TODO: @Tom let's re-enable shape tests supporting the common formats - """ - last_hidden_state = outputs.last_hidden_state - # We're expecting (batch_size, hidden_dim, height, width) - if hasattr(self.model_tester, "vision_model_tester"): - hidden_dim = self.model_tester.vision_model_tester.hidden_dim - else: - hidden_dim = self.model_tester.hidden_size - expected_shape = (inputs_dict["pixel_values"].shape[0], hidden_dim) - self.assertEqual(len(last_hidden_state.shape), 4, "last_hidden_state should be a 4D tensor") - self.assertEqual(last_hidden_state.shape[:2], expected_shape, "last_hidden_state shape mismatch") - """ + last_hidden_state_shape = outputs.last_hidden_state.shape + vision_config = config.vision_config if hasattr(config, "vision_config") else config + if hasattr(vision_config, "backbone_config"): + if "embed_dim_per_stage" in vision_config.backbone_config: + hidden_size = vision_config.backbone_config.embed_dim_per_stage[-1] + elif "hidden_size" in vision_config.backbone_config: + hidden_size = vision_config.backbone_config.hidden_size + else: + hidden_size = vision_config.backbone_config.model_args["embed_dim"][-1] + elif hasattr(vision_config, "hidden_size"): + hidden_size = vision_config.hidden_size + elif hasattr(vision_config, "hidden_dim"): + hidden_size = vision_config.hidden_dim + elif hasattr(vision_config, "embed_dim"): + hidden_size = vision_config.embed_dim[-1] + elif hasattr(vision_config, "model_args"): + hidden_size = vision_config.model_args["embed_dim"] + elif hasattr(vision_config, "vq_config"): + hidden_size = vision_config.vq_config.hidden_size + self.assertEqual(last_hidden_state_shape[-1], hidden_size, "hidden_size mismatch") def test_get_image_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): @@ -4957,6 +4975,21 @@ def test_get_audio_features_output(self): "get_audio_features() must return a BaseModelOutput with attentions", ) + last_hidden_state_shape = outputs.last_hidden_state.shape + batch_size = inputs_dict["input_features"].shape[0] + self.assertEqual(last_hidden_state_shape[0], batch_size, "batch_size mismatch") + + audio_config = config.audio_config if hasattr(config, "audio_config") else config + if hasattr(audio_config, "projection_dim"): + hidden_size = audio_config.projection_dim + elif hasattr(audio_config, "hidden_size"): + hidden_size = audio_config.hidden_size + elif hasattr(audio_config, "encoder_config"): + hidden_size = audio_config.encoder_config.hidden_dim + elif hasattr(audio_config, "encoder_ffn_dim"): + hidden_size = audio_config.encoder_ffn_dim + self.assertEqual(last_hidden_state_shape[-1], hidden_size, "hidden_size mismatch") + def test_get_audio_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) @@ -5057,6 +5090,20 @@ def test_get_video_features_output(self): "get_video_features() must return a BaseModelOutput with attentions", ) + last_hidden_state_shape = outputs.last_hidden_state.shape + if "pixel_values_videos" in inputs_dict: + batch_size = inputs_dict["pixel_values_videos"].shape[0] + elif "pixel_values" in inputs_dict: + batch_size = inputs_dict["pixel_values"].shape[0] + self.assertEqual(last_hidden_state_shape[0], batch_size, "batch_size mismatch") + video_config = config + if hasattr(config, "video_config"): + video_config = config.video_config + elif hasattr(config, "vision_config"): + video_config = config.vision_config + hidden_size = video_config.hidden_size + self.assertEqual(last_hidden_state_shape[-1], hidden_size, "hidden_size mismatch") + def test_get_video_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) From c8b712f582825757d5f64fc0d8429240a2171850 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 14 Jan 2026 11:56:51 +0100 Subject: [PATCH 083/131] Skip last_hidden_state shape tests for unusual cases E.g. when batch_size is merged with num_frames or num_patches, or hidden_size is in index -3 instead of index -1 --- tests/models/align/test_modeling_align.py | 1 + .../chameleon/test_modeling_chameleon.py | 1 + tests/models/emu3/test_modeling_emu3.py | 1 + .../models/fast_vlm/test_modeling_fast_vlm.py | 1 + .../florence2/test_modeling_florence2.py | 1 + .../models/idefics2/test_modeling_idefics2.py | 5 + .../models/idefics3/test_modeling_idefics3.py | 5 + .../test_modeling_instructblipvideo.py | 2 + .../llava_next/test_modeling_llava_next.py | 2 + .../test_modeling_llava_next_video.py | 4 + .../test_modeling_llava_onevision.py | 4 + .../models/mistral3/test_modeling_mistral3.py | 2 + tests/models/ovis2/test_modeling_ovis2.py | 2 + .../test_modeling_qwen2_5_omni.py | 1 + .../test_modeling_qwen3_omni_moe.py | 2 +- tests/models/smolvlm/test_modeling_smolvlm.py | 3 +- .../video_llava/test_modeling_video_llava.py | 3 +- tests/models/x_clip/test_modeling_x_clip.py | 3 +- tests/test_modeling_common.py | 91 ++++++++++++++----- 19 files changed, 107 insertions(+), 27 deletions(-) diff --git a/tests/models/align/test_modeling_align.py b/tests/models/align/test_modeling_align.py index 4dbf8f899abb..9d0e5a49ad52 100644 --- a/tests/models/align/test_modeling_align.py +++ b/tests/models/align/test_modeling_align.py @@ -443,6 +443,7 @@ class AlignModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): test_resize_embeddings = False test_attention_outputs = False has_attentions = False + skip_test_image_features_output_shape = True # Align uses index -3 for hidden_size instead of -1 def setUp(self): self.model_tester = AlignModelTester(self) diff --git a/tests/models/chameleon/test_modeling_chameleon.py b/tests/models/chameleon/test_modeling_chameleon.py index f11cb57d5148..b6120647fae5 100644 --- a/tests/models/chameleon/test_modeling_chameleon.py +++ b/tests/models/chameleon/test_modeling_chameleon.py @@ -280,6 +280,7 @@ class ChameleonVision2SeqModelTest(ModelTesterMixin, GenerationTesterMixin, Pipe if is_torch_available() else {} ) + # skip_test_image_features_output_shape = True # Align uses index 1 for hidden_size instead of last index def setUp(self): self.model_tester = ChameleonVision2SeqModelTester(self) diff --git a/tests/models/emu3/test_modeling_emu3.py b/tests/models/emu3/test_modeling_emu3.py index 758d4c39b1a8..91edf5ecf0cf 100644 --- a/tests/models/emu3/test_modeling_emu3.py +++ b/tests/models/emu3/test_modeling_emu3.py @@ -291,6 +291,7 @@ class Emu3Vision2TextModelTest(ModelTesterMixin, GenerationTesterMixin, Pipeline if is_torch_available() else {} ) + skip_test_image_features_output_shape = True # Emu3 uses index -3 for hidden_size instead of -1 def setUp(self): self.model_tester = Emu3Vision2TextModelTester(self) diff --git a/tests/models/fast_vlm/test_modeling_fast_vlm.py b/tests/models/fast_vlm/test_modeling_fast_vlm.py index dd9540b6aab4..ac288d2c0519 100644 --- a/tests/models/fast_vlm/test_modeling_fast_vlm.py +++ b/tests/models/fast_vlm/test_modeling_fast_vlm.py @@ -176,6 +176,7 @@ class FastVlmForConditionalGenerationModelTest(ModelTesterMixin, GenerationTeste if is_torch_available() else {} ) + skip_test_image_features_output_shape = True # FastVLM uses index -3 for hidden_size instead of -1 _is_composite = True diff --git a/tests/models/florence2/test_modeling_florence2.py b/tests/models/florence2/test_modeling_florence2.py index 29ea711472bd..d6caa9b36b1a 100644 --- a/tests/models/florence2/test_modeling_florence2.py +++ b/tests/models/florence2/test_modeling_florence2.py @@ -238,6 +238,7 @@ class Florence2ForConditionalGenerationModelTest( if is_torch_available() else {} ) + skip_test_image_features_output_shape = True # Florence2 uses index -3 for hidden_size instead of -1 has_attentions = False _is_composite = True diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index 8e681b91718e..18db4501c185 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -177,6 +177,8 @@ class Idefics2ModelTest(ModelTesterMixin, unittest.TestCase): """ all_model_classes = (Idefics2Model,) if is_torch_available() else () + # Idefics2 merges batch_size and num_frames in the first output dimension + skip_test_image_features_output_shape = True test_resize_embeddings = True _is_composite = True @@ -367,6 +369,9 @@ class Idefics2ForConditionalGenerationModelTest(GenerationTesterMixin, ModelTest all_model_classes = (Idefics2ForConditionalGeneration,) if is_torch_available() else () pipeline_model_mapping = {"image-text-to-text": Idefics2ForConditionalGeneration} if is_torch_available() else () + skip_test_image_features_output_shape = ( + True # Idefics2 merges batch_size and num_frames in the first output dimension + ) test_resize_embeddings = True diff --git a/tests/models/idefics3/test_modeling_idefics3.py b/tests/models/idefics3/test_modeling_idefics3.py index 4642444be237..3a9b402e96c6 100644 --- a/tests/models/idefics3/test_modeling_idefics3.py +++ b/tests/models/idefics3/test_modeling_idefics3.py @@ -167,6 +167,8 @@ class Idefics3ModelTest(ModelTesterMixin, unittest.TestCase): """ all_model_classes = (Idefics3Model,) if is_torch_available() else () + # Idefics3 merges batch_size and num_frames in the first output dimension + skip_test_image_features_output_shape = True test_resize_embeddings = True @@ -332,6 +334,9 @@ class Idefics3ForConditionalGenerationModelTest(GenerationTesterMixin, ModelTest all_model_classes = (Idefics3ForConditionalGeneration,) if is_torch_available() else () pipeline_model_mapping = {"image-text-to-text": Idefics3ForConditionalGeneration} if is_torch_available() else () + skip_test_image_features_output_shape = ( + True # Idefics3 merges batch_size and num_frames in the first output dimension + ) test_resize_embeddings = True diff --git a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py index d0db595c720c..6c8d8a9eb73c 100644 --- a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py +++ b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py @@ -489,6 +489,8 @@ class InstructBlipVideoForConditionalGenerationDecoderOnlyTest( (InstructBlipVideoForConditionalGeneration, InstructBlipVideoModel) if is_torch_available() else () ) additional_model_inputs = ["qformer_input_ids", "input_ids"] + # InstructBlipVideo merges batch_size and num_frames in the first output dimension + skip_test_video_features_output_shape = True test_resize_embeddings = True test_attention_outputs = False diff --git a/tests/models/llava_next/test_modeling_llava_next.py b/tests/models/llava_next/test_modeling_llava_next.py index cb6a6c934803..96e93fdcb172 100644 --- a/tests/models/llava_next/test_modeling_llava_next.py +++ b/tests/models/llava_next/test_modeling_llava_next.py @@ -199,6 +199,8 @@ class LlavaNextForConditionalGenerationModelTest(ModelTesterMixin, GenerationTes if is_torch_available() else {} ) + # Llava-NeXT merges batch_size and num_patches in the first output dimension + skip_test_image_features_output_shape = True _is_composite = True def setUp(self): diff --git a/tests/models/llava_next_video/test_modeling_llava_next_video.py b/tests/models/llava_next_video/test_modeling_llava_next_video.py index 5e23baa53db7..86a8030fa682 100644 --- a/tests/models/llava_next_video/test_modeling_llava_next_video.py +++ b/tests/models/llava_next_video/test_modeling_llava_next_video.py @@ -205,6 +205,10 @@ class LlavaNextVideoForConditionalGenerationModelTest(ModelTesterMixin, Generati if is_torch_available() else () ) + # LlavaNextVideo merges batch_size and num_patches in the first output dimension + skip_test_image_features_output_shape = True + # LlavaNextVideo merges batch_size and num_frames in the first output dimension + skip_test_video_features_output_shape = True _is_composite = True diff --git a/tests/models/llava_onevision/test_modeling_llava_onevision.py b/tests/models/llava_onevision/test_modeling_llava_onevision.py index 77d72ff110fd..71890bfd8690 100644 --- a/tests/models/llava_onevision/test_modeling_llava_onevision.py +++ b/tests/models/llava_onevision/test_modeling_llava_onevision.py @@ -200,6 +200,10 @@ class LlavaOnevisionForConditionalGenerationModelTest(ModelTesterMixin, Generati if is_torch_available() else {} ) + # LlavaOnevision merges batch_size and num_patches in the first output dimension + skip_test_image_features_output_shape = True + # LlavaOnevision merges batch_size and num_frames in the first output dimension + skip_test_video_features_output_shape = True # MP works but offload doesn't work when the MultiheadAttention is offloaded # TODO: One potential solution would be to add to set preload_module_classes = ["Siglip2MultiheadAttentionPoolingHead"] diff --git a/tests/models/mistral3/test_modeling_mistral3.py b/tests/models/mistral3/test_modeling_mistral3.py index 15109b2aec8c..63024c25e469 100644 --- a/tests/models/mistral3/test_modeling_mistral3.py +++ b/tests/models/mistral3/test_modeling_mistral3.py @@ -175,6 +175,8 @@ class Mistral3ModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterM if is_torch_available() else {} ) + # Mistral3 merges batch_size and num_patches in index 1, with index 0 hardcoded to 1 + skip_test_image_features_output_shape = True _is_composite = True def setUp(self): diff --git a/tests/models/ovis2/test_modeling_ovis2.py b/tests/models/ovis2/test_modeling_ovis2.py index 9b4717947fc9..a72ce04d0cfd 100644 --- a/tests/models/ovis2/test_modeling_ovis2.py +++ b/tests/models/ovis2/test_modeling_ovis2.py @@ -178,6 +178,8 @@ class Ovis2ModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixi if is_torch_available() else {} ) + # Ovis2 post-processes the last_hidden_state to hidden_size * hidden_stride**2 + skip_test_image_features_output_shape = True _is_composite = True def setUp(self): diff --git a/tests/models/qwen2_5_omni/test_modeling_qwen2_5_omni.py b/tests/models/qwen2_5_omni/test_modeling_qwen2_5_omni.py index e3f910e8168a..be7a2ba36421 100644 --- a/tests/models/qwen2_5_omni/test_modeling_qwen2_5_omni.py +++ b/tests/models/qwen2_5_omni/test_modeling_qwen2_5_omni.py @@ -268,6 +268,7 @@ class Qwen2_5OmniThinkerForConditionalGenerationModelTest( # ) # FIXME @raushan Omni tests take ages because the model is big. Try to make it even smaller pipeline_model_mapping = {} + skip_test_audio_features_output_shape = True # Qwen2_5Omni merges batch_size and audio_output_lengths in index 0 _is_composite = True model_split_percents = [0.5, 0.9] diff --git a/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py b/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py index 8f18b7e8ba29..34a2e7f5bd7b 100644 --- a/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py +++ b/tests/models/qwen3_omni_moe/test_modeling_qwen3_omni_moe.py @@ -262,7 +262,7 @@ class Qwen3OmniMoeThinkerForConditionalGenerationModelTest(ModelTesterMixin, Gen all_model_classes = (Qwen3OmniMoeThinkerForConditionalGeneration,) if is_torch_available() else () all_generative_model_classes = (Qwen3OmniMoeThinkerForConditionalGeneration,) if is_torch_available() else () - + skip_test_audio_features_output_shape = True # Qwen3OmniMoe merges batch_size and audio_output_lengths in index 0 _is_composite = True model_split_percents = [0.5, 0.9] diff --git a/tests/models/smolvlm/test_modeling_smolvlm.py b/tests/models/smolvlm/test_modeling_smolvlm.py index 8005b7f88a87..0a75bfd63d90 100644 --- a/tests/models/smolvlm/test_modeling_smolvlm.py +++ b/tests/models/smolvlm/test_modeling_smolvlm.py @@ -168,7 +168,7 @@ class SmolVLMModelTest(ModelTesterMixin, unittest.TestCase): """ all_model_classes = (SmolVLMModel,) if is_torch_available() else () - + skip_test_image_features_output_shape = True # SmolVLM merges batch_size with num_images in index 0 test_resize_embeddings = True def setUp(self): @@ -339,6 +339,7 @@ class SmolVLMForConditionalGenerationModelTest( if is_torch_available() else () ) + skip_test_image_features_output_shape = True # SmolVLM merges batch_size with num_images in index 0 test_resize_embeddings = True def setUp(self): diff --git a/tests/models/video_llava/test_modeling_video_llava.py b/tests/models/video_llava/test_modeling_video_llava.py index 3bcea272dd51..fa1514750bbc 100644 --- a/tests/models/video_llava/test_modeling_video_llava.py +++ b/tests/models/video_llava/test_modeling_video_llava.py @@ -201,7 +201,8 @@ class VideoLlavaForConditionalGenerationModelTest(ModelTesterMixin, GenerationTe if is_torch_available() else () ) - + # VideoLlava merges batch_size and num_frames in the first output dimension + skip_test_video_features_output_shape = True test_resize_embeddings = True _is_composite = True diff --git a/tests/models/x_clip/test_modeling_x_clip.py b/tests/models/x_clip/test_modeling_x_clip.py index 71afd6ddccfe..6bc3bae48fac 100644 --- a/tests/models/x_clip/test_modeling_x_clip.py +++ b/tests/models/x_clip/test_modeling_x_clip.py @@ -515,7 +515,8 @@ def prepare_config_and_inputs_for_common(self): class XCLIPModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): all_model_classes = (XCLIPModel,) if is_torch_available() else () pipeline_model_mapping = {"feature-extraction": XCLIPModel} if is_torch_available() else {} - + # XCLIP merges batch_size and num_frames in the first output dimension + skip_test_video_features_output_shape = True test_resize_embeddings = False test_attention_outputs = False maxdiff = None diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index f16947dc2f6c..c592a8cc0eda 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4852,26 +4852,50 @@ def test_get_image_features_output(self): "get_image_features() must return a BaseModelOutput with attentions", ) + if getattr(self, "skip_test_image_features_output_shape", False): + return + last_hidden_state_shape = outputs.last_hidden_state.shape + batch_size = ( + inputs_dict["pixel_values"].shape[0] + if "pixel_values" in inputs_dict + else inputs_dict["pixel_values_images"].shape[0] + ) + self.assertEqual( + last_hidden_state_shape[0], batch_size, f"batch_size mismatch, full shape: {last_hidden_state_shape}" + ) + vision_config = config.vision_config if hasattr(config, "vision_config") else config - if hasattr(vision_config, "backbone_config"): - if "embed_dim_per_stage" in vision_config.backbone_config: - hidden_size = vision_config.backbone_config.embed_dim_per_stage[-1] - elif "hidden_size" in vision_config.backbone_config: - hidden_size = vision_config.backbone_config.hidden_size - else: - hidden_size = vision_config.backbone_config.model_args["embed_dim"][-1] - elif hasattr(vision_config, "hidden_size"): - hidden_size = vision_config.hidden_size - elif hasattr(vision_config, "hidden_dim"): - hidden_size = vision_config.hidden_dim - elif hasattr(vision_config, "embed_dim"): - hidden_size = vision_config.embed_dim[-1] - elif hasattr(vision_config, "model_args"): - hidden_size = vision_config.model_args["embed_dim"] - elif hasattr(vision_config, "vq_config"): - hidden_size = vision_config.vq_config.hidden_size - self.assertEqual(last_hidden_state_shape[-1], hidden_size, "hidden_size mismatch") + vision_config = ( + vision_config.backbone_config if hasattr(vision_config, "backbone_config") else vision_config + ) + vision_config = vision_config.vq_config if hasattr(vision_config, "vq_config") else vision_config + vision_config = vision_config.model_args if hasattr(vision_config, "model_args") else vision_config + attribute_candidates = [ + "embed_dim_per_stage", + "embed_dim", + "embed_dims", + "out_hidden_size", + "hidden_size", + "hidden_dim", + ] + hidden_size = None + for attr in attribute_candidates: + if hasattr(vision_config, attr): + hidden_size = getattr(vision_config, attr) + break + elif isinstance(vision_config, dict) and attr in vision_config: + hidden_size = vision_config[attr] + break + else: + raise ValueError("Cannot find the hidden size attribute in vision_config") + if isinstance(hidden_size, (list, tuple)): + hidden_size = hidden_size[-1] + self.assertEqual( + last_hidden_state_shape[-1], + hidden_size, + f"hidden_size mismatch, full shape: {last_hidden_state_shape}", + ) def test_get_image_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): @@ -4975,9 +4999,14 @@ def test_get_audio_features_output(self): "get_audio_features() must return a BaseModelOutput with attentions", ) + if getattr(self, "skip_test_audio_features_output_shape", False): + return + last_hidden_state_shape = outputs.last_hidden_state.shape batch_size = inputs_dict["input_features"].shape[0] - self.assertEqual(last_hidden_state_shape[0], batch_size, "batch_size mismatch") + self.assertEqual( + last_hidden_state_shape[0], batch_size, f"batch_size mismatch, full shape: {last_hidden_state_shape}" + ) audio_config = config.audio_config if hasattr(config, "audio_config") else config if hasattr(audio_config, "projection_dim"): @@ -4988,7 +5017,11 @@ def test_get_audio_features_output(self): hidden_size = audio_config.encoder_config.hidden_dim elif hasattr(audio_config, "encoder_ffn_dim"): hidden_size = audio_config.encoder_ffn_dim - self.assertEqual(last_hidden_state_shape[-1], hidden_size, "hidden_size mismatch") + self.assertEqual( + last_hidden_state_shape[-1], + hidden_size, + f"hidden_size mismatch, full shape: {last_hidden_state_shape}", + ) def test_get_audio_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): @@ -5090,19 +5123,31 @@ def test_get_video_features_output(self): "get_video_features() must return a BaseModelOutput with attentions", ) + if getattr(self, "skip_test_video_features_output_shape", False): + return + last_hidden_state_shape = outputs.last_hidden_state.shape if "pixel_values_videos" in inputs_dict: batch_size = inputs_dict["pixel_values_videos"].shape[0] elif "pixel_values" in inputs_dict: batch_size = inputs_dict["pixel_values"].shape[0] - self.assertEqual(last_hidden_state_shape[0], batch_size, "batch_size mismatch") + self.assertEqual( + last_hidden_state_shape[0], batch_size, f"batch_size mismatch, full shape: {last_hidden_state_shape}" + ) video_config = config if hasattr(config, "video_config"): video_config = config.video_config elif hasattr(config, "vision_config"): video_config = config.vision_config - hidden_size = video_config.hidden_size - self.assertEqual(last_hidden_state_shape[-1], hidden_size, "hidden_size mismatch") + if hasattr(video_config, "out_hidden_size"): + hidden_size = video_config.out_hidden_size + else: + hidden_size = video_config.hidden_size + self.assertEqual( + last_hidden_state_shape[-1], + hidden_size, + f"hidden_size mismatch, full shape: {last_hidden_state_shape}", + ) def test_get_video_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): From d6f0fb91439add3093e8ea55df7f08cda833b16c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 14 Jan 2026 15:05:52 +0100 Subject: [PATCH 084/131] Update docstrings via auto_docstring for all get_..._features methods Also add to e.g. aria.md to ensure that get_..._features methods are documented --- docs/source/en/model_doc/aimv2.md | 2 + docs/source/en/model_doc/aria.md | 1 + docs/source/en/model_doc/audioflamingo3.md | 1 + docs/source/en/model_doc/aya_vision.md | 1 + docs/source/en/model_doc/blip-2.md | 1 + docs/source/en/model_doc/chameleon.md | 2 + docs/source/en/model_doc/cohere2_vision.md | 2 + docs/source/en/model_doc/deepseek_vl.md | 1 + .../source/en/model_doc/deepseek_vl_hybrid.md | 1 + docs/source/en/model_doc/edgetam.md | 1 + docs/source/en/model_doc/edgetam_video.md | 1 + docs/source/en/model_doc/ernie4_5_vl_moe.md | 4 ++ docs/source/en/model_doc/fast_vlm.md | 1 + docs/source/en/model_doc/florence2.md | 2 + docs/source/en/model_doc/gemma3.md | 1 + docs/source/en/model_doc/gemma3n.md | 3 + docs/source/en/model_doc/glm46v.md | 4 ++ docs/source/en/model_doc/glm4v.md | 4 ++ docs/source/en/model_doc/glm4v_moe.md | 4 ++ docs/source/en/model_doc/got_ocr2.md | 1 + docs/source/en/model_doc/granite_speech.md | 1 + docs/source/en/model_doc/idefics2.md | 2 + docs/source/en/model_doc/idefics3.md | 2 + docs/source/en/model_doc/instructblip.md | 1 + docs/source/en/model_doc/instructblipvideo.md | 1 + docs/source/en/model_doc/internvl.md | 2 + docs/source/en/model_doc/janus.md | 1 + docs/source/en/model_doc/kosmos-2.md | 1 + docs/source/en/model_doc/lfm2_vl.md | 2 + docs/source/en/model_doc/llama4.md | 1 + docs/source/en/model_doc/llava.md | 1 + docs/source/en/model_doc/llava_next.md | 1 + docs/source/en/model_doc/llava_next_video.md | 2 + docs/source/en/model_doc/llava_onevision.md | 2 + docs/source/en/model_doc/mistral3.md | 1 + docs/source/en/model_doc/ovis2.md | 1 + docs/source/en/model_doc/paddleocr_vl.md | 1 + docs/source/en/model_doc/paligemma.md | 1 + docs/source/en/model_doc/qwen2_5_vl.md | 4 ++ docs/source/en/model_doc/qwen2_vl.md | 4 ++ docs/source/en/model_doc/qwen3_vl.md | 4 ++ docs/source/en/model_doc/qwen3_vl_moe.md | 4 ++ docs/source/en/model_doc/sam2.md | 1 + docs/source/en/model_doc/sam2_video.md | 1 + docs/source/en/model_doc/sam3.md | 1 + docs/source/en/model_doc/sam3_tracker.md | 1 + .../source/en/model_doc/sam3_tracker_video.md | 1 + docs/source/en/model_doc/smolvlm.md | 2 + docs/source/en/model_doc/t5gemma2.md | 1 + docs/source/en/model_doc/video_llama_3.md | 4 ++ docs/source/en/model_doc/video_llava.md | 1 + docs/source/en/model_doc/vipllava.md | 1 + .../en/model_doc/vision-text-dual-encoder.md | 2 + docs/source/en/model_doc/voxtral.md | 1 + .../models/aimv2/modeling_aimv2.py | 8 --- .../models/align/modeling_align.py | 8 --- .../models/altclip/modeling_altclip.py | 8 --- src/transformers/models/aria/modeling_aria.py | 18 +++-- src/transformers/models/aria/modular_aria.py | 11 ++- .../audioflamingo3/modeling_audioflamingo3.py | 27 ++++---- .../audioflamingo3/modular_audioflamingo3.py | 27 ++++---- .../models/aya_vision/modeling_aya_vision.py | 30 ++++---- .../models/aya_vision/modular_aya_vision.py | 29 ++++---- src/transformers/models/blip/modeling_blip.py | 8 --- .../models/blip_2/modeling_blip_2.py | 18 ++--- .../models/chameleon/modeling_chameleon.py | 14 ++-- .../chinese_clip/modeling_chinese_clip.py | 8 --- src/transformers/models/clap/modeling_clap.py | 8 --- src/transformers/models/clip/modeling_clip.py | 8 --- .../models/clipseg/modeling_clipseg.py | 8 --- src/transformers/models/clvp/modeling_clvp.py | 31 +++------ .../cohere2_vision/modeling_cohere2_vision.py | 16 ++--- .../cohere2_vision/modular_cohere2_vision.py | 16 ++--- .../modeling_deepseek_vl_hybrid.py | 1 + .../modular_deepseek_vl_hybrid.py | 1 + .../models/edgetam/modeling_edgetam.py | 15 +--- .../edgetam_video/modeling_edgetam_video.py | 15 +--- src/transformers/models/emu3/modeling_emu3.py | 11 ++- src/transformers/models/emu3/modular_emu3.py | 11 ++- .../modeling_ernie4_5_vl_moe.py | 30 ++++---- .../modular_ernie4_5_vl_moe.py | 28 ++++---- .../models/fast_vlm/modeling_fast_vlm.py | 25 ++++--- .../models/fast_vlm/modular_fast_vlm.py | 24 +++---- .../models/flava/modeling_flava.py | 8 --- .../models/florence2/modeling_florence2.py | 15 ++-- .../models/florence2/modular_florence2.py | 15 ++-- src/transformers/models/fuyu/modeling_fuyu.py | 10 ++- .../models/gemma3/modeling_gemma3.py | 14 ++-- .../models/gemma3/modular_gemma3.py | 13 ++-- .../models/gemma3n/modeling_gemma3n.py | 30 +++----- .../models/gemma3n/modular_gemma3n.py | 29 +++----- .../models/glm46v/modeling_glm46v.py | 29 ++++---- .../models/glm4v/modeling_glm4v.py | 29 ++++---- .../models/glm4v/modular_glm4v.py | 13 ++-- .../models/glm4v_moe/modeling_glm4v_moe.py | 29 ++++---- .../models/glmasr/modeling_glmasr.py | 31 ++++----- .../models/glmasr/modular_glmasr.py | 6 +- .../models/got_ocr2/modeling_got_ocr2.py | 12 ++-- .../models/got_ocr2/modular_got_ocr2.py | 11 ++- .../granite_speech/modeling_granite_speech.py | 2 +- .../models/groupvit/modeling_groupvit.py | 8 --- .../models/idefics2/modeling_idefics2.py | 15 ++-- .../models/idefics3/modeling_idefics3.py | 19 +++-- .../instructblip/modeling_instructblip.py | 10 ++- .../modeling_instructblipvideo.py | 10 ++- .../modular_instructblipvideo.py | 12 ++-- .../models/internvl/modeling_internvl.py | 20 +++--- .../models/internvl/modular_internvl.py | 19 +++-- .../models/lfm2_vl/modeling_lfm2_vl.py | 24 +++---- .../models/lfm2_vl/modular_lfm2_vl.py | 24 +++---- .../models/llama4/modeling_llama4.py | 19 +++-- .../models/llava/modeling_llava.py | 29 ++++---- .../models/llava_next/modeling_llava_next.py | 35 +++++----- .../modeling_llava_next_video.py | 65 ++++++++--------- .../modular_llava_next_video.py | 66 ++++++++---------- .../modeling_llava_onevision.py | 69 +++++++++---------- .../modular_llava_onevision.py | 67 ++++++++---------- .../models/metaclip_2/modeling_metaclip_2.py | 8 --- .../models/metaclip_2/modular_metaclip_2.py | 8 --- .../models/mistral3/modeling_mistral3.py | 28 ++++---- .../models/mistral3/modular_mistral3.py | 31 ++++----- .../models/ovis2/modeling_ovis2.py | 29 ++++---- .../models/ovis2/modular_ovis2.py | 4 ++ .../models/owlv2/modeling_owlv2.py | 8 --- .../models/owlvit/modeling_owlvit.py | 8 --- .../paddleocr_vl/modeling_paddleocr_vl.py | 15 ++-- .../paddleocr_vl/modular_paddleocr_vl.py | 14 ++-- .../models/paligemma/modeling_paligemma.py | 16 ++--- .../models/pe_video/modeling_pe_video.py | 2 + .../models/pe_video/modular_pe_video.py | 2 + .../perception_lm/modeling_perception_lm.py | 15 ++-- .../perception_lm/modular_perception_lm.py | 15 ++-- .../qwen2_5_omni/modeling_qwen2_5_omni.py | 46 ++++++------- .../qwen2_5_omni/modular_qwen2_5_omni.py | 46 ++++++------- .../models/qwen2_5_vl/modeling_qwen2_5_vl.py | 30 ++++---- .../models/qwen2_vl/modeling_qwen2_vl.py | 30 ++++---- .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 46 ++++++------- .../qwen3_omni_moe/modular_qwen3_omni_moe.py | 18 +++-- .../models/qwen3_vl/modeling_qwen3_vl.py | 30 ++++---- .../models/qwen3_vl/modular_qwen3_vl.py | 31 ++++----- .../qwen3_vl_moe/modeling_qwen3_vl_moe.py | 30 ++++---- src/transformers/models/sam2/modeling_sam2.py | 15 +--- src/transformers/models/sam2/modular_sam2.py | 15 +--- .../models/sam2_video/modeling_sam2_video.py | 15 +--- src/transformers/models/sam3/modeling_sam3.py | 8 --- .../sam3_tracker/modeling_sam3_tracker.py | 15 +--- .../modeling_sam3_tracker_video.py | 15 +--- .../modular_sam3_tracker_video.py | 17 ++--- .../models/siglip/modeling_siglip.py | 8 --- .../models/siglip2/modeling_siglip2.py | 8 --- .../models/siglip2/modular_siglip2.py | 4 -- .../models/smolvlm/modeling_smolvlm.py | 17 ++--- .../models/smolvlm/modular_smolvlm.py | 16 ++--- .../models/t5gemma2/modeling_t5gemma2.py | 5 +- .../models/t5gemma2/modular_t5gemma2.py | 5 +- .../video_llama_3/modeling_video_llama_3.py | 38 +++++----- .../video_llama_3/modular_video_llama_3.py | 36 +++++----- .../video_llava/modeling_video_llava.py | 55 +++++++-------- .../models/vipllava/modeling_vipllava.py | 22 +++--- .../models/vipllava/modular_vipllava.py | 22 +++--- .../modeling_vision_text_dual_encoder.py | 8 --- .../models/voxtral/modeling_voxtral.py | 24 +++---- .../models/voxtral/modular_voxtral.py | 24 +++---- .../models/x_clip/modeling_x_clip.py | 9 --- 164 files changed, 971 insertions(+), 1304 deletions(-) diff --git a/docs/source/en/model_doc/aimv2.md b/docs/source/en/model_doc/aimv2.md index acf9c4de12fe..eb0fdd3b47cd 100644 --- a/docs/source/en/model_doc/aimv2.md +++ b/docs/source/en/model_doc/aimv2.md @@ -89,6 +89,8 @@ probs = outputs.logits_per_image.softmax(dim=-1) [[autodoc]] Aimv2Model - forward + - get_text_features + - get_image_features ## Aimv2VisionModel diff --git a/docs/source/en/model_doc/aria.md b/docs/source/en/model_doc/aria.md index 1d33981b334d..44875b48315d 100644 --- a/docs/source/en/model_doc/aria.md +++ b/docs/source/en/model_doc/aria.md @@ -175,3 +175,4 @@ print(response) [[autodoc]] AriaForConditionalGeneration - forward + - get_image_features diff --git a/docs/source/en/model_doc/audioflamingo3.md b/docs/source/en/model_doc/audioflamingo3.md index c9e670274063..6c585e8adb51 100644 --- a/docs/source/en/model_doc/audioflamingo3.md +++ b/docs/source/en/model_doc/audioflamingo3.md @@ -401,3 +401,4 @@ are forwarded, so you can tweak padding or tensor formats just like when calling [[autodoc]] AudioFlamingo3ForConditionalGeneration - forward + - get_audio_features diff --git a/docs/source/en/model_doc/aya_vision.md b/docs/source/en/model_doc/aya_vision.md index 4f3a77007d6a..99ed5d28cf4d 100644 --- a/docs/source/en/model_doc/aya_vision.md +++ b/docs/source/en/model_doc/aya_vision.md @@ -274,3 +274,4 @@ print(processor.tokenizer.decode(generated[0], skip_special_tokens=True)) [[autodoc]] AyaVisionForConditionalGeneration - forward + - get_image_features diff --git a/docs/source/en/model_doc/blip-2.md b/docs/source/en/model_doc/blip-2.md index e2a260f8def7..43abbb6d31a2 100644 --- a/docs/source/en/model_doc/blip-2.md +++ b/docs/source/en/model_doc/blip-2.md @@ -97,6 +97,7 @@ If you're interested in submitting a resource to be included here, please feel f [[autodoc]] Blip2ForConditionalGeneration - forward - generate + - get_image_features ## Blip2ForImageTextRetrieval diff --git a/docs/source/en/model_doc/chameleon.md b/docs/source/en/model_doc/chameleon.md index 3a9f1a707a46..9faeade71bde 100644 --- a/docs/source/en/model_doc/chameleon.md +++ b/docs/source/en/model_doc/chameleon.md @@ -203,8 +203,10 @@ model = ChameleonForConditionalGeneration.from_pretrained( [[autodoc]] ChameleonModel - forward + - get_image_features ## ChameleonForConditionalGeneration [[autodoc]] ChameleonForConditionalGeneration - forward + - get_image_features diff --git a/docs/source/en/model_doc/cohere2_vision.md b/docs/source/en/model_doc/cohere2_vision.md index 49771d1feca4..b8a1ceae0e55 100644 --- a/docs/source/en/model_doc/cohere2_vision.md +++ b/docs/source/en/model_doc/cohere2_vision.md @@ -125,11 +125,13 @@ print(outputs) [[autodoc]] Cohere2VisionForConditionalGeneration - forward + - get_image_features ## Cohere2VisionModel [[autodoc]] Cohere2VisionModel - forward + - get_image_features ## Cohere2VisionImageProcessorFast diff --git a/docs/source/en/model_doc/deepseek_vl.md b/docs/source/en/model_doc/deepseek_vl.md index 7c5502849b7e..e88ad717561e 100644 --- a/docs/source/en/model_doc/deepseek_vl.md +++ b/docs/source/en/model_doc/deepseek_vl.md @@ -223,6 +223,7 @@ model = DeepseekVLForConditionalGeneration.from_pretrained( [[autodoc]] DeepseekVLModel - forward + - get_image_features ## DeepseekVLForConditionalGeneration diff --git a/docs/source/en/model_doc/deepseek_vl_hybrid.md b/docs/source/en/model_doc/deepseek_vl_hybrid.md index 35cf380f95ba..879f3afa51fc 100644 --- a/docs/source/en/model_doc/deepseek_vl_hybrid.md +++ b/docs/source/en/model_doc/deepseek_vl_hybrid.md @@ -222,6 +222,7 @@ model = DeepseekVLHybridForConditionalGeneration.from_pretrained( [[autodoc]] DeepseekVLHybridModel - forward + - get_image_features ## DeepseekVLHybridForConditionalGeneration diff --git a/docs/source/en/model_doc/edgetam.md b/docs/source/en/model_doc/edgetam.md index 7e52f74da982..173b89533c83 100644 --- a/docs/source/en/model_doc/edgetam.md +++ b/docs/source/en/model_doc/edgetam.md @@ -330,3 +330,4 @@ EdgeTAM can use masks from previous predictions as input to refine segmentation: [[autodoc]] EdgeTamModel - forward + - get_image_features diff --git a/docs/source/en/model_doc/edgetam_video.md b/docs/source/en/model_doc/edgetam_video.md index 733591d6565b..2274d9eb9ae2 100644 --- a/docs/source/en/model_doc/edgetam_video.md +++ b/docs/source/en/model_doc/edgetam_video.md @@ -294,3 +294,4 @@ Tracked 2 objects through 200 frames [[autodoc]] EdgeTamVideoModel - forward + - get_image_features diff --git a/docs/source/en/model_doc/ernie4_5_vl_moe.md b/docs/source/en/model_doc/ernie4_5_vl_moe.md index 71ffc1ba0a97..715bc55184bc 100644 --- a/docs/source/en/model_doc/ernie4_5_vl_moe.md +++ b/docs/source/en/model_doc/ernie4_5_vl_moe.md @@ -222,8 +222,12 @@ print(output_text) [[autodoc]] Ernie4_5_VL_MoeModel - forward + - get_video_features + - get_image_features ## Ernie4_5_VL_MoeForConditionalGeneration [[autodoc]] Ernie4_5_VL_MoeForConditionalGeneration - forward + - get_video_features + - get_image_features diff --git a/docs/source/en/model_doc/fast_vlm.md b/docs/source/en/model_doc/fast_vlm.md index acb1aeb7c0c6..fcd28e1130fc 100644 --- a/docs/source/en/model_doc/fast_vlm.md +++ b/docs/source/en/model_doc/fast_vlm.md @@ -171,3 +171,4 @@ Flash Attention 2 is an even faster, optimized version of the previous optimizat [[autodoc]] FastVlmForConditionalGeneration - forward + - get_image_features diff --git a/docs/source/en/model_doc/florence2.md b/docs/source/en/model_doc/florence2.md index bc7dd1368e3d..2ef085502824 100644 --- a/docs/source/en/model_doc/florence2.md +++ b/docs/source/en/model_doc/florence2.md @@ -177,11 +177,13 @@ print(parsed_answer) [[autodoc]] Florence2Model - forward + - get_image_features ## Florence2ForConditionalGeneration [[autodoc]] Florence2ForConditionalGeneration - forward + - get_image_features ## Florence2VisionBackbone diff --git a/docs/source/en/model_doc/gemma3.md b/docs/source/en/model_doc/gemma3.md index 381a6a4ff0f6..948b0ed829c2 100644 --- a/docs/source/en/model_doc/gemma3.md +++ b/docs/source/en/model_doc/gemma3.md @@ -271,6 +271,7 @@ visualizer("What is shown in this image?") [[autodoc]] Gemma3ForConditionalGeneration - forward + - get_image_features ## Gemma3ForSequenceClassification diff --git a/docs/source/en/model_doc/gemma3n.md b/docs/source/en/model_doc/gemma3n.md index edfde4041395..a3531262c36b 100644 --- a/docs/source/en/model_doc/gemma3n.md +++ b/docs/source/en/model_doc/gemma3n.md @@ -188,6 +188,8 @@ echo -e "Plants create energy through a process known as" | transformers run --t [[autodoc]] Gemma3nModel - forward + - get_image_features + - get_audio_features ## Gemma3nForCausalLM @@ -198,6 +200,7 @@ echo -e "Plants create energy through a process known as" | transformers run --t [[autodoc]] Gemma3nForConditionalGeneration - forward + - get_image_features [altup]: https://proceedings.neurips.cc/paper_files/paper/2023/hash/f2059277ac6ce66e7e5543001afa8bb5-Abstract-Conference.html [attention-mask-viz]: https://github.com/huggingface/transformers/blob/beb9b5b02246b9b7ee81ddf938f93f44cfeaad19/src/transformers/utils/attention_visualizer.py#L139 diff --git a/docs/source/en/model_doc/glm46v.md b/docs/source/en/model_doc/glm46v.md index ab62530a438a..694485683bb1 100644 --- a/docs/source/en/model_doc/glm46v.md +++ b/docs/source/en/model_doc/glm46v.md @@ -45,8 +45,12 @@ rendered properly in your Markdown viewer. [[autodoc]] Glm46VModel - forward + - get_video_features + - get_image_features ## Glm46VForConditionalGeneration [[autodoc]] Glm46VForConditionalGeneration - forward + - get_video_features + - get_image_features diff --git a/docs/source/en/model_doc/glm4v.md b/docs/source/en/model_doc/glm4v.md index 38e43ab5c5c8..05a26c0396c2 100644 --- a/docs/source/en/model_doc/glm4v.md +++ b/docs/source/en/model_doc/glm4v.md @@ -212,8 +212,12 @@ print(output_text) [[autodoc]] Glm4vModel - forward + - get_video_features + - get_image_features ## Glm4vForConditionalGeneration [[autodoc]] Glm4vForConditionalGeneration - forward + - get_video_features + - get_image_features diff --git a/docs/source/en/model_doc/glm4v_moe.md b/docs/source/en/model_doc/glm4v_moe.md index ffb6a3d85cb2..638fa8dc877a 100644 --- a/docs/source/en/model_doc/glm4v_moe.md +++ b/docs/source/en/model_doc/glm4v_moe.md @@ -70,8 +70,12 @@ The model also introduces a **Thinking Mode** switch, allowing users to balance [[autodoc]] Glm4vMoeModel - forward + - get_video_features + - get_image_features ## Glm4vMoeForConditionalGeneration [[autodoc]] Glm4vMoeForConditionalGeneration - forward + - get_video_features + - get_image_features diff --git a/docs/source/en/model_doc/got_ocr2.md b/docs/source/en/model_doc/got_ocr2.md index 8deb85fb144c..3611917dcdf6 100644 --- a/docs/source/en/model_doc/got_ocr2.md +++ b/docs/source/en/model_doc/got_ocr2.md @@ -291,3 +291,4 @@ alt="drawing" width="600"/> [[autodoc]] GotOcr2ForConditionalGeneration - forward + - get_image_features diff --git a/docs/source/en/model_doc/granite_speech.md b/docs/source/en/model_doc/granite_speech.md index 286d62bd9341..72699bf442cb 100644 --- a/docs/source/en/model_doc/granite_speech.md +++ b/docs/source/en/model_doc/granite_speech.md @@ -170,3 +170,4 @@ for i, transcription in enumerate(transcriptions): [[autodoc]] GraniteSpeechForConditionalGeneration - forward + - get_audio_features diff --git a/docs/source/en/model_doc/idefics2.md b/docs/source/en/model_doc/idefics2.md index 858f5de9b5b8..38ba46e7a0f6 100644 --- a/docs/source/en/model_doc/idefics2.md +++ b/docs/source/en/model_doc/idefics2.md @@ -208,11 +208,13 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h [[autodoc]] Idefics2Model - forward + - get_image_features ## Idefics2ForConditionalGeneration [[autodoc]] Idefics2ForConditionalGeneration - forward + - get_image_features ## Idefics2ImageProcessor diff --git a/docs/source/en/model_doc/idefics3.md b/docs/source/en/model_doc/idefics3.md index 2a4d9eb242f8..c8b59da5f492 100644 --- a/docs/source/en/model_doc/idefics3.md +++ b/docs/source/en/model_doc/idefics3.md @@ -70,11 +70,13 @@ This model was contributed by [amyeroberts](https://huggingface.co/amyeroberts) [[autodoc]] Idefics3Model - forward + - get_image_features ## Idefics3ForConditionalGeneration [[autodoc]] Idefics3ForConditionalGeneration - forward + - get_image_features ## Idefics3ImageProcessor diff --git a/docs/source/en/model_doc/instructblip.md b/docs/source/en/model_doc/instructblip.md index e2fef6c1e35c..ef2d96d97213 100644 --- a/docs/source/en/model_doc/instructblip.md +++ b/docs/source/en/model_doc/instructblip.md @@ -78,3 +78,4 @@ The attributes can be obtained from model config, as `model.config.num_query_tok [[autodoc]] InstructBlipForConditionalGeneration - forward - generate + - get_image_features diff --git a/docs/source/en/model_doc/instructblipvideo.md b/docs/source/en/model_doc/instructblipvideo.md index 3a36d5366b66..27852b2f2039 100644 --- a/docs/source/en/model_doc/instructblipvideo.md +++ b/docs/source/en/model_doc/instructblipvideo.md @@ -83,3 +83,4 @@ The attributes can be obtained from model config, as `model.config.num_query_tok [[autodoc]] InstructBlipVideoForConditionalGeneration - forward - generate + - get_video_features diff --git a/docs/source/en/model_doc/internvl.md b/docs/source/en/model_doc/internvl.md index e2a712697d1d..a838d9f14531 100644 --- a/docs/source/en/model_doc/internvl.md +++ b/docs/source/en/model_doc/internvl.md @@ -339,11 +339,13 @@ This example showcases how to handle a batch of chat conversations with interlea [[autodoc]] InternVLModel - forward + - get_image_features ## InternVLForConditionalGeneration [[autodoc]] InternVLForConditionalGeneration - forward + - get_image_features ## InternVLProcessor diff --git a/docs/source/en/model_doc/janus.md b/docs/source/en/model_doc/janus.md index 916592489b5f..56b9ef5d0ff5 100644 --- a/docs/source/en/model_doc/janus.md +++ b/docs/source/en/model_doc/janus.md @@ -229,6 +229,7 @@ for i, image in enumerate(images['pixel_values']): [[autodoc]] JanusModel - forward + - get_image_features ## JanusForConditionalGeneration diff --git a/docs/source/en/model_doc/kosmos-2.md b/docs/source/en/model_doc/kosmos-2.md index c449dfd71ad1..c51498418c16 100644 --- a/docs/source/en/model_doc/kosmos-2.md +++ b/docs/source/en/model_doc/kosmos-2.md @@ -96,6 +96,7 @@ This model was contributed by [Yih-Dar SHIEH](https://huggingface.co/ydshieh). T [[autodoc]] Kosmos2Model - forward + - get_image_features ## Kosmos2ForConditionalGeneration diff --git a/docs/source/en/model_doc/lfm2_vl.md b/docs/source/en/model_doc/lfm2_vl.md index de77b984369d..d9c101439e34 100644 --- a/docs/source/en/model_doc/lfm2_vl.md +++ b/docs/source/en/model_doc/lfm2_vl.md @@ -92,8 +92,10 @@ processor.batch_decode(outputs, skip_special_tokens=True)[0] [[autodoc]] Lfm2VlModel - forward + - get_image_features ## Lfm2VlForConditionalGeneration [[autodoc]] Lfm2VlForConditionalGeneration - forward + - get_image_features diff --git a/docs/source/en/model_doc/llama4.md b/docs/source/en/model_doc/llama4.md index 6ccb3190e485..f430366f49b1 100644 --- a/docs/source/en/model_doc/llama4.md +++ b/docs/source/en/model_doc/llama4.md @@ -426,6 +426,7 @@ model = Llama4ForConditionalGeneration.from_pretrained( [[autodoc]] Llama4ForConditionalGeneration - forward + - get_image_features ## Llama4ForCausalLM diff --git a/docs/source/en/model_doc/llava.md b/docs/source/en/model_doc/llava.md index 2adaa15db8f3..bb24a909d42b 100644 --- a/docs/source/en/model_doc/llava.md +++ b/docs/source/en/model_doc/llava.md @@ -260,3 +260,4 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h [[autodoc]] LlavaForConditionalGeneration - forward + - get_image_features diff --git a/docs/source/en/model_doc/llava_next.md b/docs/source/en/model_doc/llava_next.md index 11a7a8dfe76c..7e78219bc633 100644 --- a/docs/source/en/model_doc/llava_next.md +++ b/docs/source/en/model_doc/llava_next.md @@ -216,3 +216,4 @@ print(processor.decode(output[0], skip_special_tokens=True)) [[autodoc]] LlavaNextForConditionalGeneration - forward + - get_image_features diff --git a/docs/source/en/model_doc/llava_next_video.md b/docs/source/en/model_doc/llava_next_video.md index d7c6b82104d5..06c0686f5cfd 100644 --- a/docs/source/en/model_doc/llava_next_video.md +++ b/docs/source/en/model_doc/llava_next_video.md @@ -259,3 +259,5 @@ model = LlavaNextVideoForConditionalGeneration.from_pretrained( [[autodoc]] LlavaNextVideoForConditionalGeneration - forward + - get_image_features + - get_video_features diff --git a/docs/source/en/model_doc/llava_onevision.md b/docs/source/en/model_doc/llava_onevision.md index c271421862fe..0a292bfd8b10 100644 --- a/docs/source/en/model_doc/llava_onevision.md +++ b/docs/source/en/model_doc/llava_onevision.md @@ -322,3 +322,5 @@ model = LlavaOnevisionForConditionalGeneration.from_pretrained( [[autodoc]] LlavaOnevisionForConditionalGeneration - forward + - get_image_features + - get_video_features diff --git a/docs/source/en/model_doc/mistral3.md b/docs/source/en/model_doc/mistral3.md index 7911200ad744..60338fb99682 100644 --- a/docs/source/en/model_doc/mistral3.md +++ b/docs/source/en/model_doc/mistral3.md @@ -254,3 +254,4 @@ messages = [ [[autodoc]] Mistral3ForConditionalGeneration - forward + - get_image_features diff --git a/docs/source/en/model_doc/ovis2.md b/docs/source/en/model_doc/ovis2.md index eacce1b30b38..0476c19339c4 100644 --- a/docs/source/en/model_doc/ovis2.md +++ b/docs/source/en/model_doc/ovis2.md @@ -95,6 +95,7 @@ with torch.inference_mode(): [[autodoc]] Ovis2ForConditionalGeneration - forward + - get_image_features ## Ovis2ImageProcessor diff --git a/docs/source/en/model_doc/paddleocr_vl.md b/docs/source/en/model_doc/paddleocr_vl.md index cc4df0774316..07e68e488cb5 100644 --- a/docs/source/en/model_doc/paddleocr_vl.md +++ b/docs/source/en/model_doc/paddleocr_vl.md @@ -206,6 +206,7 @@ model = AutoModelForImageTextToText.from_pretrained("PaddlePaddle/PaddleOCR-VL", [[autodoc]] PaddleOCRVLForConditionalGeneration - forward + - get_image_features ## PaddleOCRVLConfig diff --git a/docs/source/en/model_doc/paligemma.md b/docs/source/en/model_doc/paligemma.md index 638d5f47ebc2..0d984a5e220e 100644 --- a/docs/source/en/model_doc/paligemma.md +++ b/docs/source/en/model_doc/paligemma.md @@ -185,3 +185,4 @@ visualizer(" What is in this image?") [[autodoc]] PaliGemmaForConditionalGeneration - forward + - get_image_features diff --git a/docs/source/en/model_doc/qwen2_5_vl.md b/docs/source/en/model_doc/qwen2_5_vl.md index 693172157317..f579dc85f1b0 100644 --- a/docs/source/en/model_doc/qwen2_5_vl.md +++ b/docs/source/en/model_doc/qwen2_5_vl.md @@ -257,8 +257,12 @@ model = Qwen2_5_VLForConditionalGeneration.from_pretrained( [[autodoc]] Qwen2_5_VLModel - forward + - get_video_features + - get_image_features ## Qwen2_5_VLForConditionalGeneration [[autodoc]] Qwen2_5_VLForConditionalGeneration - forward + - get_video_features + - get_image_features diff --git a/docs/source/en/model_doc/qwen2_vl.md b/docs/source/en/model_doc/qwen2_vl.md index 6960f7f07b03..84017f0028ee 100644 --- a/docs/source/en/model_doc/qwen2_vl.md +++ b/docs/source/en/model_doc/qwen2_vl.md @@ -313,8 +313,12 @@ model = Qwen2VLForConditionalGeneration.from_pretrained( [[autodoc]] Qwen2VLModel - forward + - get_video_features + - get_image_features ## Qwen2VLForConditionalGeneration [[autodoc]] Qwen2VLForConditionalGeneration - forward + - get_video_features + - get_image_features diff --git a/docs/source/en/model_doc/qwen3_vl.md b/docs/source/en/model_doc/qwen3_vl.md index 856c92cf9897..f5542da032e6 100644 --- a/docs/source/en/model_doc/qwen3_vl.md +++ b/docs/source/en/model_doc/qwen3_vl.md @@ -112,8 +112,12 @@ print(output_text) [[autodoc]] Qwen3VLModel - forward + - get_video_features + - get_image_features ## Qwen3VLForConditionalGeneration [[autodoc]] Qwen3VLForConditionalGeneration - forward + - get_video_features + - get_image_features diff --git a/docs/source/en/model_doc/qwen3_vl_moe.md b/docs/source/en/model_doc/qwen3_vl_moe.md index 771f6d411cf2..00923c7a245e 100644 --- a/docs/source/en/model_doc/qwen3_vl_moe.md +++ b/docs/source/en/model_doc/qwen3_vl_moe.md @@ -103,8 +103,12 @@ print(output_text) [[autodoc]] Qwen3VLMoeModel - forward + - get_video_features + - get_image_features ## Qwen3VLMoeForConditionalGeneration [[autodoc]] Qwen3VLMoeForConditionalGeneration - forward + - get_video_features + - get_image_features diff --git a/docs/source/en/model_doc/sam2.md b/docs/source/en/model_doc/sam2.md index f0363616867a..3d0514de57cb 100644 --- a/docs/source/en/model_doc/sam2.md +++ b/docs/source/en/model_doc/sam2.md @@ -351,3 +351,4 @@ SAM2 can use masks from previous predictions as input to refine segmentation: [[autodoc]] Sam2Model - forward + - get_image_features diff --git a/docs/source/en/model_doc/sam2_video.md b/docs/source/en/model_doc/sam2_video.md index b55daf141250..efc2b4c43207 100644 --- a/docs/source/en/model_doc/sam2_video.md +++ b/docs/source/en/model_doc/sam2_video.md @@ -316,3 +316,4 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h [[autodoc]] Sam2VideoModel - forward - propagate_in_video_iterator + - get_image_features diff --git a/docs/source/en/model_doc/sam3.md b/docs/source/en/model_doc/sam3.md index 3ec8b7d7b69f..bae0fe97bde3 100644 --- a/docs/source/en/model_doc/sam3.md +++ b/docs/source/en/model_doc/sam3.md @@ -437,3 +437,4 @@ SAM3 uses the following label conventions: [[autodoc]] Sam3Model - forward + - get_text_features diff --git a/docs/source/en/model_doc/sam3_tracker.md b/docs/source/en/model_doc/sam3_tracker.md index 8aa2342e4e7a..c64c8b711c45 100644 --- a/docs/source/en/model_doc/sam3_tracker.md +++ b/docs/source/en/model_doc/sam3_tracker.md @@ -319,6 +319,7 @@ Sam3Tracker can use masks from previous predictions as input to refine segmentat [[autodoc]] Sam3TrackerModel - forward + - get_image_features ## Sam3TrackerPreTrainedModel diff --git a/docs/source/en/model_doc/sam3_tracker_video.md b/docs/source/en/model_doc/sam3_tracker_video.md index 97c13d05a8f1..5c7a7679600e 100644 --- a/docs/source/en/model_doc/sam3_tracker_video.md +++ b/docs/source/en/model_doc/sam3_tracker_video.md @@ -303,3 +303,4 @@ Tracked 2 objects through 180 frames [[autodoc]] Sam3TrackerVideoModel - forward - propagate_in_video_iterator + - get_image_features diff --git a/docs/source/en/model_doc/smolvlm.md b/docs/source/en/model_doc/smolvlm.md index 63fd34731038..762ff70408c0 100644 --- a/docs/source/en/model_doc/smolvlm.md +++ b/docs/source/en/model_doc/smolvlm.md @@ -187,11 +187,13 @@ print(generated_texts[0]) [[autodoc]] SmolVLMModel - forward + - get_image_features ## SmolVLMForConditionalGeneration [[autodoc]] SmolVLMForConditionalGeneration - forward + - get_image_features ## SmolVLMImageProcessor diff --git a/docs/source/en/model_doc/t5gemma2.md b/docs/source/en/model_doc/t5gemma2.md index bcadd6022b45..346f7ac0ae6b 100644 --- a/docs/source/en/model_doc/t5gemma2.md +++ b/docs/source/en/model_doc/t5gemma2.md @@ -107,6 +107,7 @@ print(processor.decode(generation[0])) [[autodoc]] T5Gemma2ForConditionalGeneration - forward + - get_image_features ## T5Gemma2ForSequenceClassification diff --git a/docs/source/en/model_doc/video_llama_3.md b/docs/source/en/model_doc/video_llama_3.md index dc5c8f1c2a22..7cb0aea5a2d2 100644 --- a/docs/source/en/model_doc/video_llama_3.md +++ b/docs/source/en/model_doc/video_llama_3.md @@ -218,6 +218,8 @@ model = VideoLlama3ForConditionalGeneration.from_pretrained( [[autodoc]] VideoLlama3Model - forward + - get_video_features + - get_image_features ## VideoLlama3VisionModel @@ -228,3 +230,5 @@ model = VideoLlama3ForConditionalGeneration.from_pretrained( [[autodoc]] VideoLlama3ForConditionalGeneration - forward + - get_video_features + - get_image_features diff --git a/docs/source/en/model_doc/video_llava.md b/docs/source/en/model_doc/video_llava.md index 24437684716f..ccc77575d451 100644 --- a/docs/source/en/model_doc/video_llava.md +++ b/docs/source/en/model_doc/video_llava.md @@ -223,3 +223,4 @@ model = VideoLlavaForConditionalGeneration.from_pretrained( [[autodoc]] VideoLlavaForConditionalGeneration - forward + - get_image_features diff --git a/docs/source/en/model_doc/vipllava.md b/docs/source/en/model_doc/vipllava.md index 336d0c100a52..3a687cc271ca 100644 --- a/docs/source/en/model_doc/vipllava.md +++ b/docs/source/en/model_doc/vipllava.md @@ -109,3 +109,4 @@ A chat between a curious human and an artificial intelligence assistant. The ass [[autodoc]] VipLlavaForConditionalGeneration - forward + - get_image_features diff --git a/docs/source/en/model_doc/vision-text-dual-encoder.md b/docs/source/en/model_doc/vision-text-dual-encoder.md index d4ba9878bdbd..39ba554a0455 100644 --- a/docs/source/en/model_doc/vision-text-dual-encoder.md +++ b/docs/source/en/model_doc/vision-text-dual-encoder.md @@ -48,3 +48,5 @@ new zero-shot vision tasks such as image classification or retrieval. [[autodoc]] VisionTextDualEncoderModel - forward + - get_text_features + - get_image_features diff --git a/docs/source/en/model_doc/voxtral.md b/docs/source/en/model_doc/voxtral.md index b22bb18efab1..605d04a5f15f 100644 --- a/docs/source/en/model_doc/voxtral.md +++ b/docs/source/en/model_doc/voxtral.md @@ -370,3 +370,4 @@ This model was contributed by [Eustache Le Bihan](https://huggingface.co/eustlb) [[autodoc]] VoxtralForConditionalGeneration - forward + - get_audio_features diff --git a/src/transformers/models/aimv2/modeling_aimv2.py b/src/transformers/models/aimv2/modeling_aimv2.py index 21b3658cc8d0..77fd3b9f08ee 100644 --- a/src/transformers/models/aimv2/modeling_aimv2.py +++ b/src/transformers/models/aimv2/modeling_aimv2.py @@ -613,10 +613,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`Aimv2TextModel`]. - Examples: ```python @@ -651,10 +647,6 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`Aimv2VisionModel`]. - Examples: ```python diff --git a/src/transformers/models/align/modeling_align.py b/src/transformers/models/align/modeling_align.py index df5ee0578abc..37281fc2e146 100644 --- a/src/transformers/models/align/modeling_align.py +++ b/src/transformers/models/align/modeling_align.py @@ -1105,10 +1105,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`AlignTextModel`]. - Examples: ```python @@ -1141,10 +1137,6 @@ def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`AlignVisionModel`]. - Examples: ```python diff --git a/src/transformers/models/altclip/modeling_altclip.py b/src/transformers/models/altclip/modeling_altclip.py index 485338dcc728..2d02c91f4f8c 100755 --- a/src/transformers/models/altclip/modeling_altclip.py +++ b/src/transformers/models/altclip/modeling_altclip.py @@ -1165,10 +1165,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`AltCLIPTextModel`]. - Examples: ```python @@ -1203,10 +1199,6 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`AltCLIPVisionModel`]. - Examples: ```python diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index 3895b99a064e..c7e752230110 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -918,7 +918,9 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple - @auto_docstring + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -927,10 +929,15 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -1075,6 +1082,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/aria/modular_aria.py b/src/transformers/models/aria/modular_aria.py index 9a09f64ffa5a..f9a1f8d0ace4 100644 --- a/src/transformers/models/aria/modular_aria.py +++ b/src/transformers/models/aria/modular_aria.py @@ -1268,7 +1268,9 @@ def _create_patch_attention_mask(self, pixel_mask): return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool() @can_return_tuple - @auto_docstring + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -1276,12 +1278,6 @@ def get_image_features( vision_feature_layer: int = -1, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - r""" - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). - """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) @@ -1364,6 +1360,7 @@ def forward( class AriaForConditionalGeneration(LlavaForConditionalGeneration): _tied_weights_keys = {"lm_head.weight": "model.language_model.embed_tokens.weight"} + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py index ea56bc997dbd..d8a42560b249 100644 --- a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py @@ -448,27 +448,24 @@ def get_decoder(self): return self.language_model.get_decoder() @can_return_tuple + @auto_docstring( + custom_intro="This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector." + ) def get_audio_features( self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. - Args: - input_features (`torch.FloatTensor`): - Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be - obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a - `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into - `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding - and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] - input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`): - Mask to avoid performing attention on padded feature indices. - - Returns: - `torch.FloatTensor`: - The audio embeddings. + r""" + input_features (`torch.FloatTensor`): + Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be + obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a + `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into + `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding + and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] + input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`): + Mask to avoid performing attention on padded feature indices. """ # Encode audio diff --git a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py index e6086d68a6a8..dc2a4ebeabbe 100644 --- a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py @@ -148,27 +148,24 @@ def __init__(self, config): super().__init__(config) @can_return_tuple + @auto_docstring( + custom_intro="This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector." + ) def get_audio_features( self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. - Args: - input_features (`torch.FloatTensor`): - Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be - obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a - `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into - `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding - and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] - input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`): - Mask to avoid performing attention on padded feature indices. - - Returns: - `torch.FloatTensor`: - The audio embeddings. + r""" + input_features (`torch.FloatTensor`): + Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be + obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a + `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into + `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding + and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] + input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`): + Mask to avoid performing attention on padded feature indices. """ # Encode audio diff --git a/src/transformers/models/aya_vision/modeling_aya_vision.py b/src/transformers/models/aya_vision/modeling_aya_vision.py index 0e3c5c8514fc..05001249b42c 100644 --- a/src/transformers/models/aya_vision/modeling_aya_vision.py +++ b/src/transformers/models/aya_vision/modeling_aya_vision.py @@ -180,6 +180,9 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -187,22 +190,16 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -354,6 +351,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/aya_vision/modular_aya_vision.py b/src/transformers/models/aya_vision/modular_aya_vision.py index 89018eccd56b..c6dfdada6b4f 100644 --- a/src/transformers/models/aya_vision/modular_aya_vision.py +++ b/src/transformers/models/aya_vision/modular_aya_vision.py @@ -105,6 +105,9 @@ class AyaVisionModelOutputWithPast(LlavaModelOutputWithPast): class AyaVisionModel(LlavaModel): # Unlike LLaVA, the model doesn't have to deal with Pixtral-style image states @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -112,22 +115,16 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 9ad1aef8b8ab..bbc4c3e81960 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -572,10 +572,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`BlipTextModel`]. - Examples: ```python @@ -607,10 +603,6 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`BlipVisionModel`]. - Examples: ```python diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 429b2ec589c2..1c48341d7d97 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -1108,10 +1108,6 @@ def get_text_features( Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. - Returns: - text_outputs (``torch.FloatTensor`): - The language model's last hidden states. - Examples: ```python >>> import torch @@ -1152,10 +1148,6 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - vision_outputs (`torch.FloatTensor`): - The vision model's last layer pooled logits. - Examples: ```python >>> import torch @@ -1620,18 +1612,16 @@ def _preprocess_accelerate(self): self.language_model._hf_hook.io_same_device = True # For `generate` compatibility @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, interpolate_pos_encoding: bool | None = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithVisionQformerOutputs: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. """ # step 1: forward the images through the vision encoder, # to get image embeddings of shape (batch_size, seq_len, hidden_size) diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 3198ffa0588b..c85757d1da5c 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -889,16 +889,15 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor): return bpe_toks @can_return_tuple + @auto_docstring( + custom_intro="Tokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layer." + ) def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ - Tokenizes images into discrete tokens with VQGAN module and embeds - them with text embeddings layer - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. """ vqmodel_outputs: ChameleonVQVAEModelOutput = self.vqmodel.encode(pixel_values, **kwargs) vqmodel_outputs.pooler_output = self.get_input_embeddings()(vqmodel_outputs.image_tokens) @@ -1060,6 +1059,7 @@ def __init__(self, config): def get_image_tokens(self, pixel_values): return self.model.get_image_tokens(pixel_values) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/chinese_clip/modeling_chinese_clip.py b/src/transformers/models/chinese_clip/modeling_chinese_clip.py index 4d214358fa0e..19702c869af1 100644 --- a/src/transformers/models/chinese_clip/modeling_chinese_clip.py +++ b/src/transformers/models/chinese_clip/modeling_chinese_clip.py @@ -1012,10 +1012,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the final [CLS] hidden state of Text-Transformer. - Examples: ```python @@ -1051,10 +1047,6 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the final [CLS] hidden state of Vision-Transformer. - Examples: ```python diff --git a/src/transformers/models/clap/modeling_clap.py b/src/transformers/models/clap/modeling_clap.py index 5ffc94cbc7a9..c117fc07241d 100644 --- a/src/transformers/models/clap/modeling_clap.py +++ b/src/transformers/models/clap/modeling_clap.py @@ -1563,10 +1563,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`ClapTextModel`]. - Examples: ```python @@ -1605,10 +1601,6 @@ def get_audio_features( Whether the audio clip is longer than `max_length`. If `True`, a feature fusion will be enabled to enhance the features. - Returns: - audio_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The audio embeddings obtained by - applying the projection layer to the pooled output of [`ClapAudioModel`]. - Examples: ```python diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index 1ed689ed3bab..be61e34c1493 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -791,10 +791,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`CLIPTextModel`]. - Examples: ```python @@ -829,10 +825,6 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`CLIPVisionModel`]. - Examples: ```python diff --git a/src/transformers/models/clipseg/modeling_clipseg.py b/src/transformers/models/clipseg/modeling_clipseg.py index 49b0a7e7228c..47be86bd7b7f 100644 --- a/src/transformers/models/clipseg/modeling_clipseg.py +++ b/src/transformers/models/clipseg/modeling_clipseg.py @@ -860,10 +860,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`CLIPSegTextModel`]. - Examples: ```python @@ -897,10 +893,6 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`CLIPSegVisionModel`]. - Examples: ```python diff --git a/src/transformers/models/clvp/modeling_clvp.py b/src/transformers/models/clvp/modeling_clvp.py index db372cb75bef..374c18a2f862 100644 --- a/src/transformers/models/clvp/modeling_clvp.py +++ b/src/transformers/models/clvp/modeling_clvp.py @@ -1493,6 +1493,12 @@ def fix_speech_decoder_output(self, speech_ids: torch.LongTensor) -> torch.LongT return speech_ids @can_return_tuple + @auto_docstring( + custom_intro=""" + This method can be used to extract text_embeds from a text. The text embeddings obtained by applying the + projection layer to the pooled output of the CLVP text encoder model. + """ + ) def get_text_features( self, input_ids: torch.LongTensor | None = None, @@ -1501,29 +1507,8 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | ClvpEncoderOutput: r""" - This method can be used to extract text_embeds from a text. The text embeddings obtained by applying the - projection layer to the pooled output of the CLVP text encoder model. - - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. - - [What are input IDs?](../glossary#input-ids) - text_encoder_inputs_embeds (`torch.FloatTensor`, *optional*): - inputs_embeds for the text encoder model passed in place of `input_ids`. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Returns: - `torch.FloatTensor` of shape `(batch_size, output_dim)`: - The text embeddings obtained by applying the projection layer to the pooled output of the CLVP Text - Model. + text_encoder_inputs_embeds (`torch.FloatTensor`, *optional*): + inputs_embeds for the text encoder model passed in place of `input_ids`. Examples: diff --git a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py index 980db148c0b5..68d3eb066a53 100644 --- a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py @@ -166,18 +166,15 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) - The tensors corresponding to the input images. - Returns: - image_features (List[`torch.Tensor`]): List of image feature tensor, each contains all the visual feature of all patches - and are of shape `(num_patches, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) + The tensors corresponding to the input images. """ image_outputs = self.vision_tower(pixel_values, **kwargs) selected_image_feature = image_outputs.last_hidden_state @@ -280,6 +277,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py index 68f77c743e55..7e6bc1527645 100644 --- a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py @@ -97,18 +97,15 @@ class Cohere2VisionModel(AyaVisionModel): _checkpoint_conversion_mapping = {} @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) - The tensors corresponding to the input images. - Returns: - image_features (List[`torch.Tensor`]): List of image feature tensor, each contains all the visual feature of all patches - and are of shape `(num_patches, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) + The tensors corresponding to the input images. """ image_outputs = self.vision_tower(pixel_values, **kwargs) selected_image_feature = image_outputs.last_hidden_state @@ -166,6 +163,7 @@ def forward( class Cohere2VisionForConditionalGeneration(AyaVisionForConditionalGeneration): _checkpoint_conversion_mapping = {} + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py index d666fec6bd59..72137d85b176 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py @@ -293,6 +293,7 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py index 6caaa3cf4015..017bdec68dc8 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py @@ -303,6 +303,7 @@ def get_high_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: return high_res_outputs @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/edgetam/modeling_edgetam.py b/src/transformers/models/edgetam/modeling_edgetam.py index 2a4291829244..723ad8cbeb78 100644 --- a/src/transformers/models/edgetam/modeling_edgetam.py +++ b/src/transformers/models/edgetam/modeling_edgetam.py @@ -1185,24 +1185,15 @@ def forward( ) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | EdgeTamVisionEncoderOutput: r""" - Extract and preprocess image features using the vision encoder. - - Args: - pixel_values (`torch.FloatTensor`): - Input pixel values of shape `(batch_size, num_channels, height, width)`. - - Returns: - `tuple`: A tuple containing: - - feature_maps (`list[torch.Tensor]`): List of feature maps from different levels. - - feature_maps_position_embeddings (`list[torch.Tensor]`): List of positional embeddings for each feature level. - - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. + pixel_values (`torch.FloatTensor`): + Input pixel values of shape `(batch_size, num_channels, height, width)`. """ vision_outputs: EdgeTamVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) diff --git a/src/transformers/models/edgetam_video/modeling_edgetam_video.py b/src/transformers/models/edgetam_video/modeling_edgetam_video.py index b54c79d610f1..6c323093bdbd 100644 --- a/src/transformers/models/edgetam_video/modeling_edgetam_video.py +++ b/src/transformers/models/edgetam_video/modeling_edgetam_video.py @@ -2218,24 +2218,15 @@ def forward( ) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | EdgeTamVideoVisionEncoderOutput: r""" - Extract and preprocess image features using the vision encoder. - - Args: - pixel_values (`torch.FloatTensor`): - Input pixel values of shape `(batch_size, num_channels, height, width)`. - - Returns: - `tuple`: A tuple containing: - - feature_maps (`list[torch.Tensor]`): List of feature maps from different levels. - - feature_maps_position_embeddings (`list[torch.Tensor]`): List of positional embeddings for each feature level. - - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. + pixel_values (`torch.FloatTensor`): + Input pixel values of shape `(batch_size, num_channels, height, width)`. """ vision_outputs: EdgeTamVideoVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index 1b7039fa22bb..da69e7fccd1a 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -1399,16 +1399,15 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.L return bpe_tokens @can_return_tuple + @auto_docstring( + custom_intro="Tokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layer" + ) def get_image_features( self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | Emu3VQVAEModelOutput: """ - Tokenizes images into discrete tokens with VQGAN module and embeds - them with text embeddings layer - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): + The tensors corresponding to the input images. """ vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, **kwargs) split_sizes = [ diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index d7f34c64b0d9..fe4cec5aae2b 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -951,16 +951,15 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.L return bpe_tokens @can_return_tuple + @auto_docstring( + custom_intro="Tokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layer" + ) def get_image_features( self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | Emu3VQVAEModelOutput: """ - Tokenizes images into discrete tokens with VQGAN module and embeds - them with text embeddings layer - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): + The tensors corresponding to the input images. """ vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, **kwargs) split_sizes = [ diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index d778eeb2f0f8..f949d63ee8b5 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -1263,20 +1263,18 @@ def get_rope_index( return position_ids, mrope_position_deltas @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ video_outputs = self.vision_tower(pixel_values_videos, video_grid_thw, **kwargs) video_embeds = self.resampler_model(video_outputs.last_hidden_state, video_grid_thw) @@ -1290,20 +1288,18 @@ def get_video_features( return video_outputs @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ image_outputs = self.vision_tower(pixel_values, image_grid_thw, **kwargs) image_embeds = self.resampler_model(image_outputs.last_hidden_state, image_grid_thw) @@ -1608,6 +1604,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -1618,6 +1615,7 @@ def get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index d71b61b31f50..7af37c5fecc7 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -1260,20 +1260,18 @@ def get_rope_index( return position_ids, mrope_position_deltas @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ video_outputs = self.vision_tower(pixel_values_videos, video_grid_thw, **kwargs) video_embeds = self.resampler_model(video_outputs.last_hidden_state, video_grid_thw) @@ -1287,20 +1285,18 @@ def get_video_features( return video_outputs @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ image_outputs = self.vision_tower(pixel_values, image_grid_thw, **kwargs) image_embeds = self.resampler_model(image_outputs.last_hidden_state, image_grid_thw) diff --git a/src/transformers/models/fast_vlm/modeling_fast_vlm.py b/src/transformers/models/fast_vlm/modeling_fast_vlm.py index f773bc94615f..1762a853d4d4 100644 --- a/src/transformers/models/fast_vlm/modeling_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modeling_fast_vlm.py @@ -114,6 +114,9 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -121,19 +124,14 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index/indices of the layer to select the vision feature. Only -1 supported. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Only "full" supported. - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index/indices of the layer to select the vision feature. Only -1 supported. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Only "full" supported. """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -301,6 +299,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/fast_vlm/modular_fast_vlm.py b/src/transformers/models/fast_vlm/modular_fast_vlm.py index cf3eecdfddfb..8364e2eac5be 100644 --- a/src/transformers/models/fast_vlm/modular_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modular_fast_vlm.py @@ -170,6 +170,9 @@ def __init__(self, config: FastVlmConfig): super().__init__(config) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -177,19 +180,14 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index/indices of the layer to select the vision feature. Only -1 supported. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Only "full" supported. - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index/indices of the layer to select the vision feature. Only -1 supported. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Only "full" supported. """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer diff --git a/src/transformers/models/flava/modeling_flava.py b/src/transformers/models/flava/modeling_flava.py index 6ab36dc22365..e249d177f9bd 100644 --- a/src/transformers/models/flava/modeling_flava.py +++ b/src/transformers/models/flava/modeling_flava.py @@ -1018,10 +1018,6 @@ def get_text_features( - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`FlavaTextModel`]. - Examples: ```python @@ -1064,10 +1060,6 @@ def get_image_features( bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, image_num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`FlavaImageModel`]. - Examples: ```python diff --git a/src/transformers/models/florence2/modeling_florence2.py b/src/transformers/models/florence2/modeling_florence2.py index 6c3d1d60ba53..08fa8e40459a 100644 --- a/src/transformers/models/florence2/modeling_florence2.py +++ b/src/transformers/models/florence2/modeling_florence2.py @@ -671,17 +671,15 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. """ image_outputs = self.vision_tower(pixel_values, **kwargs) image_outputs.pooler_output = self.multi_modal_projector(image_outputs.last_hidden_state) @@ -841,6 +839,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head + @auto_docstring def get_image_features( self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/florence2/modular_florence2.py b/src/transformers/models/florence2/modular_florence2.py index 25e1c15a1247..870dc220312f 100644 --- a/src/transformers/models/florence2/modular_florence2.py +++ b/src/transformers/models/florence2/modular_florence2.py @@ -1510,17 +1510,15 @@ def get_encoder(self, modality=None): return super().get_encoder(modality=modality) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. """ image_outputs = self.vision_tower(pixel_values, **kwargs) image_outputs.pooler_output = self.multi_modal_projector(image_outputs.last_hidden_state) @@ -1619,6 +1617,7 @@ class Florence2ForConditionalGeneration(LlavaForConditionalGeneration): "lm_head.weight": "model.language_model.shared.weight", } + @auto_docstring def get_image_features( self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/fuyu/modeling_fuyu.py b/src/transformers/models/fuyu/modeling_fuyu.py index a49918b78398..e1437795c072 100644 --- a/src/transformers/models/fuyu/modeling_fuyu.py +++ b/src/transformers/models/fuyu/modeling_fuyu.py @@ -115,15 +115,13 @@ def gather_continuous_embeddings( return output_embeddings @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. """ patch_embeddings = [ self.vision_embed_tokens(patch.to(self.vision_embed_tokens.weight.dtype)).squeeze(0) diff --git a/src/transformers/models/gemma3/modeling_gemma3.py b/src/transformers/models/gemma3/modeling_gemma3.py index 11ae36d475f7..c7d8d9169b66 100644 --- a/src/transformers/models/gemma3/modeling_gemma3.py +++ b/src/transformers/models/gemma3/modeling_gemma3.py @@ -844,18 +844,13 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring(custom_intro="Projects the last hidden state from the vision model into language model space.") def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ - Projects the last hidden state from the vision model into language model space. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. """ vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) last_hidden_state = vision_outputs.last_hidden_state @@ -1022,6 +1017,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @auto_docstring def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): return self.model.get_image_features(pixel_values, **kwargs) diff --git a/src/transformers/models/gemma3/modular_gemma3.py b/src/transformers/models/gemma3/modular_gemma3.py index 3e2fa26dc05c..267d642896ab 100644 --- a/src/transformers/models/gemma3/modular_gemma3.py +++ b/src/transformers/models/gemma3/modular_gemma3.py @@ -799,18 +799,13 @@ def __init__(self, config: Gemma3Config): del self.text_config_dtype @can_return_tuple + @auto_docstring(custom_intro="Projects the last hidden state from the vision model into language model space.") def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ - Projects the last hidden state from the vision model into language model space. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. """ vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) last_hidden_state = vision_outputs.last_hidden_state diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index a78174ff09bc..11b9b9cc0537 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -1974,20 +1974,15 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring(custom_intro="Projects the last hidden state from the vision model into language model space.") def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: """ - Projects the last hidden state from the vision model into language model space. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. """ vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, **kwargs) last_hidden_state = vision_outputs.last_hidden_state @@ -2190,23 +2185,19 @@ def forward( audio_hidden_states=audio_features if input_features is not None else None, ) + @can_return_tuple + @auto_docstring(custom_intro="Projects the last hidden state from the audio encoder into language model space.") def get_audio_features( self, input_features: torch.Tensor, input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Gemma3nAudioEncoderModelOutput: - """ - Projects the last hidden state from the audio encoder into language model space. - - Args: - input_features (`torch.FloatTensor]` of shape `(num_images, seq_length, num_features)`): - The tensors corresponding to the input audio. - input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`): - The attention mask for the input audio. - - Returns: - audio_features (`torch.Tensor`): Audio feature tensor of shape `(num_images, audio_length, embed_dim)`). + r""" + input_features (`torch.FloatTensor]` of shape `(num_images, seq_length, num_features)`): + The tensors corresponding to the input audio. + input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`): + The attention mask for the input audio. """ audio_outputs: Gemma3nAudioEncoderModelOutput = self.audio_tower(input_features, input_features_mask) audio_embeds = self.embed_audio(inputs_embeds=audio_outputs.last_hidden_state) @@ -2237,6 +2228,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @auto_docstring def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): return self.model.get_image_features(pixel_values, **kwargs) diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index ed7b597c954c..c2fc796cdbc4 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -2237,20 +2237,15 @@ def __init__(self, config: Gemma3nConfig): self.embed_audio = Gemma3nMultimodalEmbedder(config.audio_config, config.text_config) @can_return_tuple + @auto_docstring(custom_intro="Projects the last hidden state from the vision model into language model space.") def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: """ - Projects the last hidden state from the vision model into language model space. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. """ vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, **kwargs) last_hidden_state = vision_outputs.last_hidden_state @@ -2453,23 +2448,19 @@ def forward( audio_hidden_states=audio_features if input_features is not None else None, ) + @can_return_tuple + @auto_docstring(custom_intro="Projects the last hidden state from the audio encoder into language model space.") def get_audio_features( self, input_features: torch.Tensor, input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Gemma3nAudioEncoderModelOutput: - """ - Projects the last hidden state from the audio encoder into language model space. - - Args: - input_features (`torch.FloatTensor]` of shape `(num_images, seq_length, num_features)`): - The tensors corresponding to the input audio. - input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`): - The attention mask for the input audio. - - Returns: - audio_features (`torch.Tensor`): Audio feature tensor of shape `(num_images, audio_length, embed_dim)`). + r""" + input_features (`torch.FloatTensor]` of shape `(num_images, seq_length, num_features)`): + The tensors corresponding to the input audio. + input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`): + The attention mask for the input audio. """ audio_outputs: Gemma3nAudioEncoderModelOutput = self.audio_tower(input_features, input_features_mask) audio_embeds = self.embed_audio(inputs_embeds=audio_outputs.last_hidden_state) diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 65d292f1bd9e..8cfbab594efb 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -298,14 +298,11 @@ def get_video_features( video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames @@ -322,20 +319,18 @@ def get_video_features( return vision_outputs @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) @@ -542,6 +537,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -552,6 +548,7 @@ def get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index f34333a7490f..da495a1fa295 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1166,14 +1166,11 @@ def get_video_features( video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames @@ -1190,20 +1187,18 @@ def get_video_features( return vision_outputs @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) @@ -1410,6 +1405,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -1420,6 +1416,7 @@ def get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 413c1fd4f422..fc5f3dddfe05 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -1157,14 +1157,11 @@ def get_video_features( video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 6a575e6516e5..41d7187a772e 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1382,14 +1382,11 @@ def get_video_features( video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) # reshape video_grid_thw -> [b, 3] -> [1, h, w] * frames @@ -1406,20 +1403,18 @@ def get_video_features( return vision_outputs @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) @@ -1679,6 +1674,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -1689,6 +1685,7 @@ def get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/glmasr/modeling_glmasr.py b/src/transformers/models/glmasr/modeling_glmasr.py index 412dd4143dd0..cac3231c30e2 100644 --- a/src/transformers/models/glmasr/modeling_glmasr.py +++ b/src/transformers/models/glmasr/modeling_glmasr.py @@ -30,8 +30,8 @@ from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torch_available -from ...utils.generic import check_model_inputs, maybe_autocast +from ...utils import TransformersKwargs, auto_docstring, is_torch_available +from ...utils.generic import can_return_tuple, check_model_inputs, maybe_autocast from ..auto import AutoModel, AutoModelForCausalLM from .configuration_glmasr import GlmAsrConfig, GlmAsrEncoderConfig @@ -387,27 +387,24 @@ def get_decoder(self): return self.language_model.get_decoder() @can_return_tuple + @auto_docstring( + custom_intro="Compute audio embeddings from log-mel input features using the audio encoder and multi-modal projector." + ) def get_audio_features( self, input_features: torch.FloatTensor, input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. - Args: - input_features (`torch.FloatTensor`): - Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be - obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a - `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into - `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding - and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] - input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`): - Mask to avoid performing attention on padded feature indices. - - Returns: - `torch.FloatTensor`: - The audio embeddings. + r""" + input_features (`torch.FloatTensor`): + Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be + obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a + `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into + `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding + and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] + input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`): + Mask to avoid performing attention on padded feature indices. """ audio_outputs = self.audio_tower(input_features, **kwargs) audio_hidden_states = audio_outputs.last_hidden_state diff --git a/src/transformers/models/glmasr/modular_glmasr.py b/src/transformers/models/glmasr/modular_glmasr.py index 0d6ba7ab78ca..86353932a1a1 100644 --- a/src/transformers/models/glmasr/modular_glmasr.py +++ b/src/transformers/models/glmasr/modular_glmasr.py @@ -25,7 +25,7 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, is_torch_available, logging -from ...utils.generic import check_model_inputs +from ...utils.generic import can_return_tuple, check_model_inputs from ..audioflamingo3.modeling_audioflamingo3 import ( AudioFlamingo3ForConditionalGeneration, AudioFlamingo3MultiModalProjector, @@ -355,6 +355,10 @@ def __init__(self, config: GlmAsrConfig): """ ) class GlmAsrForConditionalGeneration(AudioFlamingo3ForConditionalGeneration): + @can_return_tuple + @auto_docstring( + custom_intro="Compute audio embeddings from log-mel input features using the audio encoder and multi-modal projector." + ) def get_audio_features( self, input_features: torch.FloatTensor, diff --git a/src/transformers/models/got_ocr2/modeling_got_ocr2.py b/src/transformers/models/got_ocr2/modeling_got_ocr2.py index 53a4c441ae2c..0be2846372cc 100644 --- a/src/transformers/models/got_ocr2/modeling_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modeling_got_ocr2.py @@ -550,19 +550,16 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`) - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + pixel_values (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`) """ image_outputs = self.vision_tower(pixel_values, **kwargs) last_hidden_state = image_outputs.last_hidden_state @@ -684,6 +681,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/got_ocr2/modular_got_ocr2.py b/src/transformers/models/got_ocr2/modular_got_ocr2.py index 16dc027905d9..759672a62afe 100644 --- a/src/transformers/models/got_ocr2/modular_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modular_got_ocr2.py @@ -305,19 +305,16 @@ def __init__(self, config: GotOcr2Config): self.vision_tower = GotOcr2VisionEncoder(config.vision_config) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`) - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + pixel_values (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`) """ image_outputs = self.vision_tower(pixel_values, **kwargs) last_hidden_state = image_outputs.last_hidden_state diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index 4e2dc907d907..887b3a1ee3ad 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -356,10 +356,10 @@ def get_output_embeddings(self): return self.language_model.get_output_embeddings() @can_return_tuple + @auto_docstring def get_audio_features( self, input_features: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """Get the audio features to merged into the multimodal embeddings.""" audio_outputs = self.encoder(input_features, **kwargs) projected_embeds = self.projector(audio_outputs.last_hidden_state) audio_outputs.pooler_output = projected_embeds diff --git a/src/transformers/models/groupvit/modeling_groupvit.py b/src/transformers/models/groupvit/modeling_groupvit.py index 0e3f2971bc16..d92c8126981b 100644 --- a/src/transformers/models/groupvit/modeling_groupvit.py +++ b/src/transformers/models/groupvit/modeling_groupvit.py @@ -1238,10 +1238,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`GroupViTTextModel`]. - Examples: ```python @@ -1274,10 +1270,6 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`GroupViTVisionModel`]. - Examples: ```python diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index 01fa61f6a602..a00b453e8d25 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -824,20 +824,18 @@ def inputs_merger( return inputs_embeds @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, pixel_attention_mask: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - pixel_attention_mask (`torch.LongTensor`, *optional*): - The attention mask indicating padded regions in the image. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + pixel_attention_mask (`torch.LongTensor`, *optional*): + The attention mask indicating padded regions in the image. """ batch_size, num_images, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility @@ -996,6 +994,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.text_model.set_input_embeddings(value) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index 69b1698d2b79..3a6f88d81758 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -567,22 +567,20 @@ def inputs_merger( return inputs_embeds @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, pixel_attention_mask: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - pixel_attention_mask (`torch.LongTensor`, *optional*): - The attention mask indicating padded regions in the image. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + pixel_attention_mask (`torch.LongTensor`, *optional*): + The attention mask indicating padded regions in the image. + return_dict (`bool`, *optional*, default to `False`): + Whether to return a `ModelOutput` instead of a pooled embedding. """ batch_size, num_images, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility @@ -753,6 +751,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.text_model.set_input_embeddings(value) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index 805e1610cf58..6f660a5d50a6 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -1187,6 +1187,7 @@ def _preprocess_accelerate(self): self.language_model._hf_hook.io_same_device = True # For `generate` compatibility @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, @@ -1195,12 +1196,9 @@ def get_image_features( interpolate_pos_encoding: bool | None = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithVisionQformerOutputs: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. """ # step 1: forward the images through the vision encoder, # to get image embeddings of shape (batch_size, seq_len, hidden_size) diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 3070e7faf688..e259ea45453a 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -1452,6 +1452,7 @@ def generate( return outputs @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values: torch.FloatTensor, @@ -1460,12 +1461,9 @@ def get_video_features( interpolate_pos_encoding: bool | None = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithVisionQformerOutputs: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. """ # step 1: forward the images through the vision encoder, # we process in a batched way, later unbatch it back (video has frames=4 always) diff --git a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py index 786a0d2dce10..6f4bc70b9977 100644 --- a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py @@ -35,7 +35,7 @@ from ...modeling_outputs import BaseModelOutputWithPooling from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES from ...processing_utils import Unpack -from ...utils import can_return_tuple, logging +from ...utils import auto_docstring, can_return_tuple, logging from ..auto import CONFIG_MAPPING, AutoConfig @@ -286,6 +286,7 @@ def forward( class InstructBlipVideoForConditionalGeneration(InstructBlipForConditionalGeneration): @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values: torch.FloatTensor, @@ -294,12 +295,9 @@ def get_video_features( interpolate_pos_encoding: bool | None = False, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithVisionQformerOutputs: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. """ # step 1: forward the images through the vision encoder, # we process in a batched way, later unbatch it back (video has frames=4 always) diff --git a/src/transformers/models/internvl/modeling_internvl.py b/src/transformers/models/internvl/modeling_internvl.py index a301628439af..e37888e33661 100644 --- a/src/transformers/models/internvl/modeling_internvl.py +++ b/src/transformers/models/internvl/modeling_internvl.py @@ -545,6 +545,9 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -552,17 +555,11 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - vision_feature_layer (`int` or `list[int]`): - Layer index or list of layer indices to extract features from. - - Returns: - vision_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`. + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. + vision_feature_layer (`int` or `list[int]`): + Layer index or list of layer indices to extract features from. """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -782,6 +779,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/internvl/modular_internvl.py b/src/transformers/models/internvl/modular_internvl.py index 700d035ea683..dbeba84fa94f 100644 --- a/src/transformers/models/internvl/modular_internvl.py +++ b/src/transformers/models/internvl/modular_internvl.py @@ -485,6 +485,9 @@ def pixel_shuffle(self, vision_features: torch.Tensor, scale_factor: float = 0.5 return vision_features @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -492,17 +495,11 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - vision_feature_layer (`int` or `list[int]`): - Layer index or list of layer indices to extract features from. - - Returns: - vision_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`. + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. + vision_feature_layer (`int` or `list[int]`): + Layer index or list of layer indices to extract features from. """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer diff --git a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py index 56c1e2606c13..a479c5466d5f 100755 --- a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py @@ -162,6 +162,9 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -169,19 +172,13 @@ def get_image_features( pixel_attention_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - spatial_shapes (`torch.Tensor` of shape `(batch_size, 2)`): - The spatial shapes of the input images. - pixel_attention_mask (`torch.Tensor` of shape `(batch_size, height, width)`): - The pixel attention mask of the input images. - - Returns: - image_features (`list[torch.Tensor]`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + spatial_shapes (`torch.Tensor` of shape `(batch_size, 2)`): + The spatial shapes of the input images. + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, height, width)`): + The pixel attention mask of the input images. """ image_outputs = self.vision_tower( pixel_values=pixel_values, @@ -324,6 +321,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py index 69d17db3b12f..e35bd9d6763b 100644 --- a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py @@ -93,6 +93,9 @@ def __init__(self, config: Lfm2VlConfig): super().__init__(config) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -100,19 +103,13 @@ def get_image_features( pixel_attention_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - spatial_shapes (`torch.Tensor` of shape `(batch_size, 2)`): - The spatial shapes of the input images. - pixel_attention_mask (`torch.Tensor` of shape `(batch_size, height, width)`): - The pixel attention mask of the input images. - - Returns: - image_features (`list[torch.Tensor]`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + spatial_shapes (`torch.Tensor` of shape `(batch_size, 2)`): + The spatial shapes of the input images. + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, height, width)`): + The pixel attention mask of the input images. """ image_outputs = self.vision_tower( pixel_values=pixel_values, @@ -234,6 +231,7 @@ def forward( class Lfm2VlForConditionalGeneration(LlavaForConditionalGeneration): _checkpoint_conversion_mapping = {} + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/llama4/modeling_llama4.py b/src/transformers/models/llama4/modeling_llama4.py index 8dc55ce1ddea..6c149d4a6dd2 100644 --- a/src/transformers/models/llama4/modeling_llama4.py +++ b/src/transformers/models/llama4/modeling_llama4.py @@ -1204,23 +1204,20 @@ def set_decoder(self, decoder): def get_decoder(self): return self.language_model.get_decoder() + @can_return_tuple + @auto_docstring(custom_intro="Obtains image last hidden states from the vision tower and apply al projection.") def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_select_strategy: str, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply al projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - vision_feature_select_strategy (`str`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. + vision_feature_select_strategy (`str`): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ if vision_feature_select_strategy not in ["default", "full"]: raise ValueError(f"Unexpected select feature strategy: {vision_feature_select_strategy}") diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 5b48fa55e9fd..c2e0f1094246 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -146,6 +146,9 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -153,21 +156,16 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -330,6 +328,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index d6bf1291d66e..5b6c3a4276ee 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -345,6 +345,9 @@ def pack_image_features(self, image_features, image_sizes, vision_feature_select return new_image_features, feature_lens @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -353,25 +356,18 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) - The tensors corresponding to the input images. - image_sizes (`torch.Tensor` of shape `(num_images, 2)`) - Actual image size of each images (H, W). - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - - Returns: - image_features (list[`torch.Tensor`]): List of image feature tensor, each contains all the visual feature of all patches - and are of shape `(num_patches, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) + The tensors corresponding to the input images. + image_sizes (`torch.Tensor` of shape `(num_images, 2)`) + Actual image size of each images (H, W). + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -569,6 +565,7 @@ def pack_image_features(self, image_features, image_sizes, vision_feature_select image_newline=image_newline, ) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 1d5773a08631..ddf9b5f6b6a6 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -397,6 +397,9 @@ def pack_image_features(self, image_features, image_sizes, vision_feature_select return new_image_features, feature_lens @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -405,24 +408,18 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) - The tensors corresponding to the input images. - image_sizes (`torch.Tensor` of shape `(num_images, 2)`) - Actual image size of each images (H, W). - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - Returns: - image_features (list[`torch.Tensor`]): List of image feature tensor, each contains all the visual feature of all patches - and are of shape `(num_patches, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) + The tensors corresponding to the input images. + image_sizes (`torch.Tensor` of shape `(num_images, 2)`) + Actual image size of each images (H, W). + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -617,6 +614,9 @@ def forward( ) @can_return_tuple + @auto_docstring( + custom_intro="Obtains video last hidden states from the vision tower and apply multimodal projection." + ) def get_video_features( self, pixel_values: torch.FloatTensor, @@ -624,23 +624,16 @@ def get_video_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains video last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_frames, channels, height, width)`) - The tensors corresponding to the input video. - vision_feature_layer (`Union[int, list[int]]`, *optional;*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - - Returns: - video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches - and are of shape `(num_videos, video_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_frames, channels, height, width)`) + The tensors corresponding to the input video. + vision_feature_layer (`Union[int, list[int]]`, *optional;*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -714,6 +707,7 @@ def pack_image_features(self, image_features, image_sizes, vision_feature_select image_newline=image_newline, ) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, @@ -963,6 +957,7 @@ def _prepare_4d_causal_attention_mask_with_cache_position( return causal_mask + @auto_docstring def get_video_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index e0b5aa776ac0..02f5461c9d41 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -33,7 +33,7 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack -from ...utils import can_return_tuple, logging +from ...utils import auto_docstring, can_return_tuple, logging from ..auto import CONFIG_MAPPING, AutoConfig @@ -272,6 +272,9 @@ def __init__(self, config: LlavaNextVideoConfig, **super_kwargs): self.post_init() @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -280,24 +283,18 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) - The tensors corresponding to the input images. - image_sizes (`torch.Tensor` of shape `(num_images, 2)`) - Actual image size of each images (H, W). - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - Returns: - image_features (list[`torch.Tensor`]): List of image feature tensor, each contains all the visual feature of all patches - and are of shape `(num_patches, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) + The tensors corresponding to the input images. + image_sizes (`torch.Tensor` of shape `(num_images, 2)`) + Actual image size of each images (H, W). + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -351,6 +348,9 @@ def get_image_features( return image_outputs @can_return_tuple + @auto_docstring( + custom_intro="Obtains video last hidden states from the vision tower and apply multimodal projection." + ) def get_video_features( self, pixel_values: torch.FloatTensor, @@ -358,23 +358,16 @@ def get_video_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains video last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_frames, channels, height, width)`) - The tensors corresponding to the input video. - vision_feature_layer (`Union[int, list[int]]`, *optional;*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - - Returns: - video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches - and are of shape `(num_videos, video_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_frames, channels, height, width)`) + The tensors corresponding to the input video. + vision_feature_layer (`Union[int, list[int]]`, *optional;*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -544,6 +537,7 @@ def forward( class LlavaNextVideoForConditionalGeneration(LlavaNextForConditionalGeneration): + @auto_docstring def get_video_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index ef73b2ea9ddb..13fa007b1321 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -354,6 +354,9 @@ def pack_image_features(self, image_features, image_sizes, image_newline=None, v return new_image_features, feature_lens @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -364,26 +367,20 @@ def get_image_features( batch_num_images: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) - The tensors corresponding to the input images. - image_sizes (`torch.Tensor` of shape `(num_images, 2)`) - Actual image size of each images (H, W). - vision_feature_layer (`Union[int, list[int]]`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - batch_num_images (`torch.LongTensor`, *optional*): - Number of images in each sample. - Returns: - image_features (list[`torch.Tensor`]): List of image feature tensor, each contains all the visual feature of all patches - and are of shape `(num_patches, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) + The tensors corresponding to the input images. + image_sizes (`torch.Tensor` of shape `(num_images, 2)`) + Actual image size of each images (H, W). + vision_feature_layer (`Union[int, list[int]]`): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` + batch_num_images (`torch.LongTensor`, *optional*): + Number of images in each sample. """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -600,6 +597,9 @@ def forward( ) @can_return_tuple + @auto_docstring( + custom_intro="Obtains video last hidden states from the vision tower, apply multimodal projection and pooling." + ) def get_video_features( self, pixel_values: torch.FloatTensor, @@ -607,23 +607,16 @@ def get_video_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains video last hidden states from the vision tower, apply multimodal projection and pooling. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_frames, channels, height, width)`) - The tensors corresponding to the input video. - vision_feature_layer (`Union[int, list[int]], *optional*, defaults to -2`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - - Returns: - video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches - and are of shape `(num_videos, video_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_frames, channels, height, width)`) + The tensors corresponding to the input video. + vision_feature_layer (`Union[int, list[int]], *optional*, defaults to -2`): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -710,6 +703,7 @@ def pack_image_features(self, image_features, image_sizes, vision_feature_select image_newline=image_newline, ) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, @@ -950,6 +944,7 @@ def _prepare_4d_causal_attention_mask_with_cache_position( return causal_mask + @auto_docstring def get_video_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index b240977db11c..cb9282cb79da 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -314,6 +314,9 @@ def apply_pooling(self, image_features): return image_features @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -324,26 +327,20 @@ def get_image_features( batch_num_images: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) - The tensors corresponding to the input images. - image_sizes (`torch.Tensor` of shape `(num_images, 2)`) - Actual image size of each images (H, W). - vision_feature_layer (`Union[int, list[int]]`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - batch_num_images (`torch.LongTensor`, *optional*): - Number of images in each sample. - Returns: - image_features (list[`torch.Tensor`]): List of image feature tensor, each contains all the visual feature of all patches - and are of shape `(num_patches, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) + The tensors corresponding to the input images. + image_sizes (`torch.Tensor` of shape `(num_images, 2)`) + Actual image size of each images (H, W). + vision_feature_layer (`Union[int, list[int]]`): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` + batch_num_images (`torch.LongTensor`, *optional*): + Number of images in each sample. """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -407,6 +404,9 @@ def get_image_features( return image_outputs @can_return_tuple + @auto_docstring( + custom_intro="Obtains video last hidden states from the vision tower, apply multimodal projection and pooling." + ) def get_video_features( self, pixel_values: torch.FloatTensor, @@ -414,23 +414,16 @@ def get_video_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains video last hidden states from the vision tower, apply multimodal projection and pooling. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_frames, channels, height, width)`) - The tensors corresponding to the input video. - vision_feature_layer (`Union[int, list[int]], *optional*, defaults to -2`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - - Returns: - video_features (list[`torch.Tensor`]): List of video feature tensor, each contains all the visual feature of all patches - and are of shape `(num_videos, video_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_frames, channels, height, width)`) + The tensors corresponding to the input video. + vision_feature_layer (`Union[int, list[int]], *optional*, defaults to -2`): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer diff --git a/src/transformers/models/metaclip_2/modeling_metaclip_2.py b/src/transformers/models/metaclip_2/modeling_metaclip_2.py index 9b4de364f19b..0191841bc539 100644 --- a/src/transformers/models/metaclip_2/modeling_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modeling_metaclip_2.py @@ -801,10 +801,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`MetaClip2TextModel`]. - Examples: ```python @@ -836,10 +832,6 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`MetaClip2VisionModel`]. - Examples: ```python diff --git a/src/transformers/models/metaclip_2/modular_metaclip_2.py b/src/transformers/models/metaclip_2/modular_metaclip_2.py index 17caa1bd8c0d..279bd9ce8572 100644 --- a/src/transformers/models/metaclip_2/modular_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modular_metaclip_2.py @@ -565,10 +565,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`MetaClip2TextModel`]. - Examples: ```python @@ -596,10 +592,6 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`MetaClip2VisionModel`]. - Examples: ```python diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index d63f137d3e1a..a86c7a92a9c8 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -213,6 +213,9 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -220,21 +223,15 @@ def get_image_features( vision_feature_layer: int | list[int] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - image_sizes (`torch.Tensor`, *optional*): - Tensor containing the image sizes as returned by the processor. - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + image_sizes (`torch.Tensor`, *optional*): + Tensor containing the image sizes as returned by the processor. """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -384,6 +381,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/mistral3/modular_mistral3.py b/src/transformers/models/mistral3/modular_mistral3.py index d401853a8bea..9635a9692b23 100644 --- a/src/transformers/models/mistral3/modular_mistral3.py +++ b/src/transformers/models/mistral3/modular_mistral3.py @@ -20,7 +20,7 @@ from ...cache_utils import Cache from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack -from ...utils import logging +from ...utils import auto_docstring, can_return_tuple, logging from ..llava.modeling_llava import ( LlavaCausalLMOutputWithPast, LlavaForConditionalGeneration, @@ -117,6 +117,10 @@ class Mistral3PreTrainedModel(LlavaPreTrainedModel): class Mistral3Model(LlavaModel): + @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -124,21 +128,15 @@ def get_image_features( vision_feature_layer: int | list[int] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - image_sizes (`torch.Tensor`, *optional*): - Tensor containing the image sizes as returned by the processor. - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + image_sizes (`torch.Tensor`, *optional*): + Tensor containing the image sizes as returned by the processor. """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -234,6 +232,7 @@ def forward( class Mistral3ForConditionalGeneration(LlavaForConditionalGeneration): + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index e6e30233008f..1a8334fd9503 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -486,26 +486,24 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ image_outputs = self.vision_tower(pixel_values, **kwargs) image_features = image_outputs.pooler_output @@ -655,6 +653,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures: diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index 910ee99f88bf..9e424228a4aa 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -256,6 +256,9 @@ def __init__(self, config: Ovis2Config): del self.multi_modal_projector @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -373,6 +376,7 @@ def __init__(self, config: Ovis2Config): super().__init__(config) self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures: diff --git a/src/transformers/models/owlv2/modeling_owlv2.py b/src/transformers/models/owlv2/modeling_owlv2.py index 916d5c8b6e8a..b615fcaa4d1b 100644 --- a/src/transformers/models/owlv2/modeling_owlv2.py +++ b/src/transformers/models/owlv2/modeling_owlv2.py @@ -988,10 +988,6 @@ def get_text_features( [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`Owlv2TextModel`]. - Examples: ```python >>> import torch @@ -1025,10 +1021,6 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`Owlv2VisionModel`]. - Examples: ```python >>> import torch diff --git a/src/transformers/models/owlvit/modeling_owlvit.py b/src/transformers/models/owlvit/modeling_owlvit.py index c82553f5bc65..0eb7663d4af7 100644 --- a/src/transformers/models/owlvit/modeling_owlvit.py +++ b/src/transformers/models/owlvit/modeling_owlvit.py @@ -969,10 +969,6 @@ def get_text_features( [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`OwlViTTextModel`]. - Examples: ```python >>> import torch @@ -1006,10 +1002,6 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`OwlViTVisionModel`]. - Examples: ```python >>> import torch diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index 9ce9c653c77e..be6dfc27a0a3 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -1202,20 +1202,18 @@ def get_rope_index( return position_ids, mrope_position_deltas @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype).unsqueeze(0) cu_seqlens = torch.repeat_interleave(image_grid_thw[:, 1] * image_grid_thw[:, 2], image_grid_thw[:, 0]).cumsum( @@ -1355,6 +1353,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py index 85f183920259..628461d1a71f 100644 --- a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py @@ -1122,20 +1122,18 @@ def get_video_features(self): raise AttributeError("PaddleOCRVLModel does not support video.") @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype).unsqueeze(0) cu_seqlens = torch.repeat_interleave(image_grid_thw[:, 1] * image_grid_thw[:, 2], image_grid_thw[:, 0]).cumsum( diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index 74c9fba7702c..0ecfd472a13f 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -259,18 +259,15 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. """ image_outputs = self.vision_tower(pixel_values, **kwargs) selected_image_feature = image_outputs.last_hidden_state @@ -451,6 +448,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @auto_docstring def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): return self.model.get_image_features(pixel_values, **kwargs) diff --git a/src/transformers/models/pe_video/modeling_pe_video.py b/src/transformers/models/pe_video/modeling_pe_video.py index 60ce3a9f17fb..ad9165e6f939 100644 --- a/src/transformers/models/pe_video/modeling_pe_video.py +++ b/src/transformers/models/pe_video/modeling_pe_video.py @@ -571,6 +571,7 @@ def __init__(self, config: PeVideoConfig): self.post_init() @can_return_tuple + @auto_docstring def get_text_features( self, input_ids: torch.Tensor, @@ -586,6 +587,7 @@ def get_text_features( return text_outputs @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.Tensor, diff --git a/src/transformers/models/pe_video/modular_pe_video.py b/src/transformers/models/pe_video/modular_pe_video.py index 568b556d64ad..12430d1ed382 100644 --- a/src/transformers/models/pe_video/modular_pe_video.py +++ b/src/transformers/models/pe_video/modular_pe_video.py @@ -152,6 +152,7 @@ def __init__(self, config: PeVideoConfig): self.post_init() @can_return_tuple + @auto_docstring def get_text_features( self, input_ids: torch.Tensor, @@ -167,6 +168,7 @@ def get_text_features( return text_outputs @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.Tensor, diff --git a/src/transformers/models/perception_lm/modeling_perception_lm.py b/src/transformers/models/perception_lm/modeling_perception_lm.py index d2efb5516bae..3b290c721340 100644 --- a/src/transformers/models/perception_lm/modeling_perception_lm.py +++ b/src/transformers/models/perception_lm/modeling_perception_lm.py @@ -180,20 +180,17 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_tiles, channels, height, width)`) - The tensors corresponding to the input images. - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_tiles, num_patches, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_tiles, channels, height, width)`) + The tensors corresponding to the input images. """ image_outputs = self.vision_tower(pixel_values.flatten(0, 1), **kwargs) last_hidden_state = image_outputs.last_hidden_state diff --git a/src/transformers/models/perception_lm/modular_perception_lm.py b/src/transformers/models/perception_lm/modular_perception_lm.py index 9e742b073032..49d1d3629812 100644 --- a/src/transformers/models/perception_lm/modular_perception_lm.py +++ b/src/transformers/models/perception_lm/modular_perception_lm.py @@ -147,20 +147,17 @@ def __init__(self, config: PerceptionLMConfig): self.language_model = AutoModel.from_config(config.text_config) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_tiles, channels, height, width)`) - The tensors corresponding to the input images. - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_tiles, num_patches, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_tiles, channels, height, width)`) + The tensors corresponding to the input images. """ image_outputs = self.vision_tower(pixel_values.flatten(0, 1), **kwargs) last_hidden_state = image_outputs.last_hidden_state diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index ec613f1b76c0..81958ad40e8d 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -1798,44 +1798,41 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) return self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) return self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) @can_return_tuple + @auto_docstring def get_audio_features( self, input_features: torch.FloatTensor, @@ -1843,16 +1840,13 @@ def get_audio_features( audio_feature_lengths: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes audios into continuous embeddings that can be forwarded to the language model. - - Args: - input_features (`torch.FloatTensor`): - The tensors corresponding to the input audios. - feature_attention_mask (`torch.LongTensor`, *optional*): - Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: - audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): - The length of feature shape of each audio in LLM. + r""" + input_features (`torch.FloatTensor`): + The tensors corresponding to the input audios. + feature_attention_mask (`torch.LongTensor`, *optional*): + Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: + audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): + The length of feature shape of each audio in LLM. """ if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index 32da27277c2a..4fa719c364e2 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -2103,44 +2103,41 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) return self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) return self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) @can_return_tuple + @auto_docstring def get_audio_features( self, input_features: torch.FloatTensor, @@ -2148,16 +2145,13 @@ def get_audio_features( audio_feature_lengths: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes audios into continuous embeddings that can be forwarded to the language model. - - Args: - input_features (`torch.FloatTensor`): - The tensors corresponding to the input audios. - feature_attention_mask (`torch.LongTensor`, *optional*): - Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: - audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): - The length of feature shape of each audio in LLM. + r""" + input_features (`torch.FloatTensor`): + The tensors corresponding to the input audios. + feature_attention_mask (`torch.LongTensor`, *optional*): + Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: + audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): + The length of feature shape of each audio in LLM. """ if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index 9c480bd69bf9..c32e6935e63d 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -1179,20 +1179,18 @@ def get_rope_index( return position_ids, mrope_position_deltas @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) @@ -1203,20 +1201,18 @@ def get_video_features( return vision_outputs @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) @@ -1417,6 +1413,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -1427,6 +1424,7 @@ def get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 7385363359b0..5184ecff8d78 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -1114,20 +1114,18 @@ def get_rope_index( return position_ids, mrope_position_deltas @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) @@ -1138,20 +1136,18 @@ def get_video_features( return vision_outputs @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) @@ -1315,6 +1311,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -1325,6 +1322,7 @@ def get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index c823b4f6bd9c..7e6d504864ab 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -1984,44 +1984,41 @@ def set_input_embeddings(self, value): self.model.set_input_embeddings(value) @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) return self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) return self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) @can_return_tuple + @auto_docstring def get_audio_features( self, input_features: torch.FloatTensor, @@ -2029,16 +2026,13 @@ def get_audio_features( audio_feature_lengths: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes audios into continuous embeddings that can be forwarded to the language model. - - Args: - input_features (`torch.FloatTensor`): - The tensors corresponding to the input audios. - feature_attention_mask (`torch.LongTensor`, *optional*): - Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: - audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): - The length of feature shape of each audio in LLM. + r""" + input_features (`torch.FloatTensor`): + The tensors corresponding to the input audios. + feature_attention_mask (`torch.LongTensor`, *optional*): + Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: + audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): + The length of feature shape of each audio in LLM. """ if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) diff --git a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py index 90d0c88e75e2..40dea0e45385 100644 --- a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py @@ -1434,6 +1434,7 @@ def __init__(self, config): self.router_aux_loss_coef = config.text_config.router_aux_loss_coef @can_return_tuple + @auto_docstring def get_audio_features( self, input_features: torch.FloatTensor, @@ -1441,16 +1442,13 @@ def get_audio_features( audio_feature_lengths: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes audios into continuous embeddings that can be forwarded to the language model. - - Args: - input_features (`torch.FloatTensor`): - The tensors corresponding to the input audios. - feature_attention_mask (`torch.LongTensor`, *optional*): - Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: - audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): - The length of feature shape of each audio in LLM. + r""" + input_features (`torch.FloatTensor`): + The tensors corresponding to the input audios. + feature_attention_mask (`torch.LongTensor`, *optional*): + Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: + audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): + The length of feature shape of each audio in LLM. """ if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index d2807c1ce6f0..8b66e8e38f23 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -1091,39 +1091,35 @@ def get_rope_index( return position_ids, mrope_position_deltas @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithDeepstackFeatures: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ # Same implementation as for images return self.get_image_features(pixel_values_videos, video_grid_thw, **kwargs) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithDeepstackFeatures: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) vision_output: BaseModelOutputWithDeepstackFeatures = self.visual( @@ -1344,6 +1340,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -1354,6 +1351,7 @@ def get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index 7db367f2fef6..07e1f2cfd43a 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -962,20 +962,18 @@ def get_rope_index( return position_ids, mrope_position_deltas @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithDeepstackFeatures: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) vision_output: BaseModelOutputWithDeepstackFeatures = self.visual( @@ -988,20 +986,19 @@ def get_image_features( return vision_output + @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithDeepstackFeatures: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ # Same implementation as for images return self.get_image_features(pixel_values_videos, video_grid_thw, **kwargs) @@ -1133,9 +1130,11 @@ class Qwen3VLForConditionalGeneration(Qwen2_5_VLForConditionalGeneration): config: Qwen3VLConfig _checkpoint_conversion_mapping = {} + @auto_docstring def get_image_features(self, **super_kwargs) -> tuple | BaseModelOutputWithDeepstackFeatures: return super().get_image_features(**super_kwargs) + @auto_docstring def get_video_features(self, **super_kwargs) -> tuple | BaseModelOutputWithDeepstackFeatures: return super().get_video_features(**super_kwargs) diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index b9bdd722a403..002381630bdc 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -1218,39 +1218,35 @@ def get_rope_index( return position_ids, mrope_position_deltas @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithDeepstackFeatures: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. """ # Same implementation as for images return self.get_image_features(pixel_values_videos, video_grid_thw, **kwargs) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithDeepstackFeatures: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) vision_output: BaseModelOutputWithDeepstackFeatures = self.visual( @@ -1524,6 +1520,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -1534,6 +1531,7 @@ def get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/sam2/modeling_sam2.py b/src/transformers/models/sam2/modeling_sam2.py index 2c2d2987865e..b024b5c90cf4 100644 --- a/src/transformers/models/sam2/modeling_sam2.py +++ b/src/transformers/models/sam2/modeling_sam2.py @@ -1547,24 +1547,15 @@ def forward( ) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Sam2VisionEncoderOutput: r""" - Extract and preprocess image features using the vision encoder. - - Args: - pixel_values (`torch.FloatTensor`): - Input pixel values of shape `(batch_size, num_channels, height, width)`. - - Returns: - `tuple`: A tuple containing: - - feature_maps (`list[torch.Tensor]`): List of feature maps from different levels. - - feature_maps_position_embeddings (`list[torch.Tensor]`): List of positional embeddings for each feature level. - - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. + pixel_values (`torch.FloatTensor`): + Input pixel values of shape `(batch_size, num_channels, height, width)`. """ vision_outputs: Sam2VisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) diff --git a/src/transformers/models/sam2/modular_sam2.py b/src/transformers/models/sam2/modular_sam2.py index 8761be78ea8b..55eb4478eb70 100644 --- a/src/transformers/models/sam2/modular_sam2.py +++ b/src/transformers/models/sam2/modular_sam2.py @@ -1238,24 +1238,15 @@ def get_image_embeddings( return image_embeddings @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Sam2VisionEncoderOutput: r""" - Extract and preprocess image features using the vision encoder. - - Args: - pixel_values (`torch.FloatTensor`): - Input pixel values of shape `(batch_size, num_channels, height, width)`. - - Returns: - `tuple`: A tuple containing: - - feature_maps (`list[torch.Tensor]`): List of feature maps from different levels. - - feature_maps_position_embeddings (`list[torch.Tensor]`): List of positional embeddings for each feature level. - - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. + pixel_values (`torch.FloatTensor`): + Input pixel values of shape `(batch_size, num_channels, height, width)`. """ vision_outputs: Sam2VisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) diff --git a/src/transformers/models/sam2_video/modeling_sam2_video.py b/src/transformers/models/sam2_video/modeling_sam2_video.py index 4b65f8d30460..83af1c6cf433 100644 --- a/src/transformers/models/sam2_video/modeling_sam2_video.py +++ b/src/transformers/models/sam2_video/modeling_sam2_video.py @@ -1822,24 +1822,15 @@ def forward( ) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Sam2VideoVisionEncoderOutput: r""" - Extract and preprocess image features using the vision encoder. - - Args: - pixel_values (`torch.FloatTensor`): - Input pixel values of shape `(batch_size, num_channels, height, width)`. - - Returns: - `tuple`: A tuple containing: - - feature_maps (`list[torch.Tensor]`): List of feature maps from different levels. - - feature_maps_position_embeddings (`list[torch.Tensor]`): List of positional embeddings for each feature level. - - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. + pixel_values (`torch.FloatTensor`): + Input pixel values of shape `(batch_size, num_channels, height, width)`. """ vision_outputs: Sam2VideoVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) diff --git a/src/transformers/models/sam3/modeling_sam3.py b/src/transformers/models/sam3/modeling_sam3.py index f5f57bf6e681..5695ab332f69 100644 --- a/src/transformers/models/sam3/modeling_sam3.py +++ b/src/transformers/models/sam3/modeling_sam3.py @@ -2149,10 +2149,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Text embeddings that can be passed as `text_embeds` to the forward method. - Example: ```python @@ -2187,10 +2183,6 @@ def get_vision_features( **kwargs: Unpack[TransformersKwargs], ) -> Sam3VisionEncoderOutput: r""" - Returns: - vision_embeds (`Sam3VisionEncoderOutput`): - Vision embeddings that can be passed as `vision_embeds` to the forward method. - Example: ```python diff --git a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py index 070e8b3159b0..7fb854a8ef40 100644 --- a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py +++ b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py @@ -1048,24 +1048,15 @@ def forward( ) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Sam3TrackerVisionEncoderOutput: r""" - Extract and preprocess image features using the vision encoder. - - Args: - pixel_values (`torch.FloatTensor`): - Input pixel values of shape `(batch_size, num_channels, height, width)`. - - Returns: - `tuple`: A tuple containing: - - feature_maps (`list[torch.Tensor]`): List of feature maps from different levels. - - feature_maps_position_embeddings (`list[torch.Tensor]`): List of positional embeddings for each feature level. - - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. + pixel_values (`torch.FloatTensor`): + Input pixel values of shape `(batch_size, num_channels, height, width)`. """ vision_outputs: Sam3TrackerVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) diff --git a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py index fa5d6bb1ea52..81b9bdfd44a8 100644 --- a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py @@ -1845,24 +1845,15 @@ def forward( ) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Sam3TrackerVideoVisionEncoderOutput: r""" - Extract and preprocess image features using the vision encoder. - - Args: - pixel_values (`torch.FloatTensor`): - Input pixel values of shape `(batch_size, num_channels, height, width)`. - - Returns: - `tuple`: A tuple containing: - - feature_maps (`list[torch.Tensor]`): List of feature maps from different levels. - - feature_maps_position_embeddings (`list[torch.Tensor]`): List of positional embeddings for each feature level. - - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. + pixel_values (`torch.FloatTensor`): + Input pixel values of shape `(batch_size, num_channels, height, width)`. """ vision_outputs: Sam3TrackerVideoVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) diff --git a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py index 09f24be9a84b..ba2e1bc54d3f 100644 --- a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py @@ -17,7 +17,7 @@ from ...configuration_utils import PreTrainedConfig from ...processing_utils import Unpack -from ...utils import TransformersKwargs, can_return_tuple +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple from ..auto import CONFIG_MAPPING, AutoConfig, AutoModel from ..sam2_video.configuration_sam2_video import Sam2VideoMaskDecoderConfig, Sam2VideoPromptEncoderConfig from ..sam2_video.modeling_sam2_video import ( @@ -545,24 +545,15 @@ def __init__(self, config: Sam3TrackerVideoConfig, remove_vision_encoder: bool = self.post_init() @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Sam3TrackerVideoVisionEncoderOutput: r""" - Extract and preprocess image features using the vision encoder. - - Args: - pixel_values (`torch.FloatTensor`): - Input pixel values of shape `(batch_size, num_channels, height, width)`. - - Returns: - `tuple`: A tuple containing: - - feature_maps (`list[torch.Tensor]`): List of feature maps from different levels. - - feature_maps_position_embeddings (`list[torch.Tensor]`): List of positional embeddings for each feature level. - - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder. - - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder. + pixel_values (`torch.FloatTensor`): + Input pixel values of shape `(batch_size, num_channels, height, width)`. """ vision_outputs: Sam3TrackerVideoVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) diff --git a/src/transformers/models/siglip/modeling_siglip.py b/src/transformers/models/siglip/modeling_siglip.py index 612f844c1e81..9c38b8beb7bb 100644 --- a/src/transformers/models/siglip/modeling_siglip.py +++ b/src/transformers/models/siglip/modeling_siglip.py @@ -798,10 +798,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`SiglipTextModel`]. - Examples: ```python @@ -832,10 +828,6 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`SiglipVisionModel`]. - Examples: ```python diff --git a/src/transformers/models/siglip2/modeling_siglip2.py b/src/transformers/models/siglip2/modeling_siglip2.py index b125d81cf2b8..1305fedada09 100644 --- a/src/transformers/models/siglip2/modeling_siglip2.py +++ b/src/transformers/models/siglip2/modeling_siglip2.py @@ -856,10 +856,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`Siglip2TextModel`]. - Examples: ```python @@ -896,10 +892,6 @@ def get_image_features( spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): Tensor containing the spatial dimensions (height, width) of the input images. - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`Siglip2VisionModel`]. - Examples: ```python diff --git a/src/transformers/models/siglip2/modular_siglip2.py b/src/transformers/models/siglip2/modular_siglip2.py index 572fc98db7ad..fa31fbecaa0c 100644 --- a/src/transformers/models/siglip2/modular_siglip2.py +++ b/src/transformers/models/siglip2/modular_siglip2.py @@ -373,10 +373,6 @@ def get_image_features( spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): Tensor containing the spatial dimensions (height, width) of the input images. - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`Siglip2VisionModel`]. - Examples: ```python diff --git a/src/transformers/models/smolvlm/modeling_smolvlm.py b/src/transformers/models/smolvlm/modeling_smolvlm.py index 1532a768dd26..4abd07b1d67b 100644 --- a/src/transformers/models/smolvlm/modeling_smolvlm.py +++ b/src/transformers/models/smolvlm/modeling_smolvlm.py @@ -519,20 +519,20 @@ def inputs_merger( return merged_embeds @can_return_tuple + @auto_docstring( + custom_intro="Encodes images into continuous embeddings that can be forwarded to the language model." + ) def get_image_features( self, pixel_values: torch.FloatTensor, pixel_attention_mask: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - pixel_attention_mask (`torch.LongTensor`, *optional*): - The attention mask indicating padded regions in the image. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + pixel_attention_mask (`torch.LongTensor`, *optional*): + The attention mask indicating padded regions in the image. """ batch_size, num_images, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility @@ -736,6 +736,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.text_model.set_input_embeddings(value) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/smolvlm/modular_smolvlm.py b/src/transformers/models/smolvlm/modular_smolvlm.py index 3eae9314a843..6524fc5c5c6e 100644 --- a/src/transformers/models/smolvlm/modular_smolvlm.py +++ b/src/transformers/models/smolvlm/modular_smolvlm.py @@ -192,20 +192,20 @@ def inputs_merger( return merged_embeds @can_return_tuple + @auto_docstring( + custom_intro="Encodes images into continuous embeddings that can be forwarded to the language model." + ) def get_image_features( self, pixel_values: torch.FloatTensor, pixel_attention_mask: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - pixel_attention_mask (`torch.LongTensor`, *optional*): - The attention mask indicating padded regions in the image. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + pixel_attention_mask (`torch.LongTensor`, *optional*): + The attention mask indicating padded regions in the image. """ batch_size, num_images, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility diff --git a/src/transformers/models/t5gemma2/modeling_t5gemma2.py b/src/transformers/models/t5gemma2/modeling_t5gemma2.py index 4c4e36c3935e..d04523b85bb4 100644 --- a/src/transformers/models/t5gemma2/modeling_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modeling_t5gemma2.py @@ -810,12 +810,10 @@ def __init__( self.post_init() @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ - Convert pixel image to image features via the encoder and projector. - """ # pixel_values: (batch_size, channels, height, width) # image_features: Image feature tensor of shape (num_images, image_length, embed_dim). vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) @@ -1227,6 +1225,7 @@ def get_encoder(self): def get_decoder(self): return self.model.get_decoder() + @auto_docstring def get_image_features( self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/t5gemma2/modular_t5gemma2.py b/src/transformers/models/t5gemma2/modular_t5gemma2.py index a61230cfc60e..9d8c13024d58 100644 --- a/src/transformers/models/t5gemma2/modular_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modular_t5gemma2.py @@ -865,12 +865,10 @@ def __init__( self.post_init() @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ - Convert pixel image to image features via the encoder and projector. - """ # pixel_values: (batch_size, channels, height, width) # image_features: Image feature tensor of shape (num_images, image_length, embed_dim). vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) @@ -1269,6 +1267,7 @@ def get_encoder(self): def get_decoder(self): return self.model.get_decoder() + @auto_docstring def get_image_features( self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/video_llama_3/modeling_video_llama_3.py b/src/transformers/models/video_llama_3/modeling_video_llama_3.py index 1c63e5fa4771..22917c32b07e 100644 --- a/src/transformers/models/video_llama_3/modeling_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modeling_video_llama_3.py @@ -547,6 +547,7 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -554,16 +555,13 @@ def get_video_features( video_merge_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. - video_merge_sizes (`torch.Tensor` of shape `(num_videos,)`): - The spatial downsampling ratio of each video feature. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + video_merge_sizes (`torch.Tensor` of shape `(num_videos,)`): + The spatial downsampling ratio of each video feature. """ return self.get_image_features( pixel_values=pixel_values_videos, @@ -573,6 +571,7 @@ def get_video_features( ) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, @@ -580,16 +579,13 @@ def get_image_features( image_merge_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - image_merge_sizes (`torch.Tensor` of shape `(num_images,)`): - The spatial downsampling ratio of each image feature. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + image_merge_sizes (`torch.Tensor` of shape `(num_images,)`): + The spatial downsampling ratio of each image feature. """ vision_outputs = self.vision_model( pixel_values=pixel_values, @@ -779,6 +775,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -789,6 +786,7 @@ def get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/video_llama_3/modular_video_llama_3.py b/src/transformers/models/video_llama_3/modular_video_llama_3.py index a62e6c2d0443..bb3ef3ed32fd 100644 --- a/src/transformers/models/video_llama_3/modular_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modular_video_llama_3.py @@ -591,6 +591,7 @@ def get_rope_index(self): raise AttributeError("Not needed for VideoLLaMA3") @can_return_tuple + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -598,16 +599,13 @@ def get_video_features( video_merge_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes videos into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. - video_merge_sizes (`torch.Tensor` of shape `(num_videos,)`): - The spatial downsampling ratio of each video feature. + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + video_merge_sizes (`torch.Tensor` of shape `(num_videos,)`): + The spatial downsampling ratio of each video feature. """ return self.get_image_features( pixel_values=pixel_values_videos, @@ -617,6 +615,7 @@ def get_video_features( ) @can_return_tuple + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, @@ -624,16 +623,13 @@ def get_image_features( image_merge_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - image_merge_sizes (`torch.Tensor` of shape `(num_images,)`): - The spatial downsampling ratio of each image feature. + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + image_merge_sizes (`torch.Tensor` of shape `(num_images,)`): + The spatial downsampling ratio of each image feature. """ vision_outputs = self.vision_model( pixel_values=pixel_values, diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index ddbb638624e8..711f022a652f 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -171,6 +171,9 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values_images: torch.FloatTensor, @@ -178,22 +181,16 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values_images (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values_images (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -229,26 +226,22 @@ def get_image_features( return image_outputs @can_return_tuple + @auto_docstring( + custom_intro="Obtains video last hidden states from the vision tower and apply multimodal projection." + ) def get_video_features( self, pixel_values_videos: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains video last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values_videos (`torch.FloatTensor]` of shape `(batch_size, num_frames, channels, height, width)`) - The tensors corresponding to the input videos. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - - Returns: - video_features (`torch.Tensor`): Video feature tensor of shape `(num_videos * num_frames, image_length, embed_dim)`). - frames (`int`): Number of frames the videos have. + r""" + pixel_values_videos (`torch.FloatTensor]` of shape `(batch_size, num_frames, channels, height, width)`) + The tensors corresponding to the input videos. + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer @@ -435,6 +428,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head + @auto_docstring def get_image_features( self, pixel_values_images: torch.FloatTensor, @@ -491,7 +485,6 @@ def forward( >>> from huggingface_hub import hf_hub_download >>> from transformers import VideoLlavaProcessor, VideoLlavaForConditionalGeneration - >>> def read_video_pyav(container, indices): ... ''' ... Decode the video with PyAV decoder. diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index f854b8129922..bd83f95d2073 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -150,24 +150,21 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layers: int | list[int] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - vision_feature_layers (`Union[int, list[int]]`): - The vision feature layer, or the list of indexes of the layers to select - the vision feature. - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. + vision_feature_layers (`Union[int, list[int]]`): + The vision feature layer, or the list of indexes of the layers to select + the vision feature. """ vision_feature_layers = ( vision_feature_layers if vision_feature_layers is not None else self.config.vision_feature_layers @@ -311,6 +308,7 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/vipllava/modular_vipllava.py b/src/transformers/models/vipllava/modular_vipllava.py index 9943c518f1d4..8d0e265cd2ef 100644 --- a/src/transformers/models/vipllava/modular_vipllava.py +++ b/src/transformers/models/vipllava/modular_vipllava.py @@ -72,24 +72,21 @@ class VipLlavaPreTrainedModel(LlavaPreTrainedModel): class VipLlavaModel(LlavaModel): @can_return_tuple + @auto_docstring( + custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." + ) def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layers: int | list[int] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - vision_feature_layers (`Union[int, list[int]]`): - The vision feature layer, or the list of indexes of the layers to select - the vision feature. - - Returns: - image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. + vision_feature_layers (`Union[int, list[int]]`): + The vision feature layer, or the list of indexes of the layers to select + the vision feature. """ vision_feature_layers = ( vision_feature_layers if vision_feature_layers is not None else self.config.vision_feature_layers @@ -181,6 +178,7 @@ def forward( class VipLlavaForConditionalGeneration(LlavaForConditionalGeneration): + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index 252c87852220..6cc1fb58d705 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -113,10 +113,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`CLIPTextModel`]. - Examples: ```python @@ -148,10 +144,6 @@ def get_image_features( self, pixel_values: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`CLIPVisionModel`]. - Examples: ```python diff --git a/src/transformers/models/voxtral/modeling_voxtral.py b/src/transformers/models/voxtral/modeling_voxtral.py index 0507206697e3..483f60b2893f 100644 --- a/src/transformers/models/voxtral/modeling_voxtral.py +++ b/src/transformers/models/voxtral/modeling_voxtral.py @@ -399,23 +399,19 @@ def get_decoder(self): return self.language_model.get_decoder() @can_return_tuple + @auto_docstring( + custom_intro="This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector." + ) def get_audio_features( self, input_features: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ - This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. - - Args: - input_features (`torch.FloatTensor`): - Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be - obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a - `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into - `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding - and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] - - Returns: - `torch.FloatTensor`: - The audio embeddings. + r""" + input_features (`torch.FloatTensor`): + Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be + obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a + `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into + `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding + and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] """ audio_outputs = self.audio_tower(input_features, **kwargs) audio_hidden_states = audio_outputs.last_hidden_state diff --git a/src/transformers/models/voxtral/modular_voxtral.py b/src/transformers/models/voxtral/modular_voxtral.py index a22075920b78..1771c8f33d36 100644 --- a/src/transformers/models/voxtral/modular_voxtral.py +++ b/src/transformers/models/voxtral/modular_voxtral.py @@ -165,23 +165,19 @@ def get_decoder(self): return self.language_model.get_decoder() @can_return_tuple + @auto_docstring( + custom_intro="This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector." + ) def get_audio_features( self, input_features: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ - This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. - - Args: - input_features (`torch.FloatTensor`): - Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be - obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a - `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into - `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding - and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] - - Returns: - `torch.FloatTensor`: - The audio embeddings. + r""" + input_features (`torch.FloatTensor`): + Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be + obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a + `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into + `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding + and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] """ audio_outputs = self.audio_tower(input_features, **kwargs) audio_hidden_states = audio_outputs.last_hidden_state diff --git a/src/transformers/models/x_clip/modeling_x_clip.py b/src/transformers/models/x_clip/modeling_x_clip.py index ef88cda08e63..c26b43279d67 100644 --- a/src/transformers/models/x_clip/modeling_x_clip.py +++ b/src/transformers/models/x_clip/modeling_x_clip.py @@ -1212,10 +1212,6 @@ def get_text_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`XCLIPTextModel`]. - Examples: ```python @@ -1248,11 +1244,6 @@ def get_video_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - Returns: - video_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The video embeddings obtained by - applying the projection layer to the pooled output of [`XCLIPVisionModel`] and - [`XCLIPMultiframeIntegrationTransformer`]. - Examples: ```python From 51638d6cfa516a457974ad25808b3948398424b2 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 14 Jan 2026 15:51:35 +0100 Subject: [PATCH 085/131] Ensure all auto_doc arguments are documented --- .../modeling_deepseek_vl_hybrid.py | 2 +- .../modular_deepseek_vl_hybrid.py | 2 +- .../modeling_ernie4_5_vl_moe.py | 35 +++++++++++-- .../modular_ernie4_5_vl_moe.py | 38 ++++++++------ .../models/glm46v/modeling_glm46v.py | 12 +++++ .../models/glm4v/modeling_glm4v.py | 12 +++++ .../models/glm4v_moe/modeling_glm4v_moe.py | 12 +++++ .../models/idefics2/modeling_idefics2.py | 6 +++ .../models/idefics3/modeling_idefics3.py | 8 ++- .../instructblip/modeling_instructblip.py | 15 ++++++ .../modeling_instructblipvideo.py | 4 ++ .../modular_instructblipvideo.py | 4 ++ .../models/lfm2_vl/modeling_lfm2_vl.py | 8 +++ .../models/lfm2_vl/modular_lfm2_vl.py | 8 +++ .../models/llava_next/modeling_llava_next.py | 13 +++++ .../modeling_llava_next_video.py | 13 +++++ .../modeling_llava_onevision.py | 33 ++++++++++-- .../modular_llava_onevision.py | 50 +++++++++++++++++-- .../paddleocr_vl/modeling_paddleocr_vl.py | 6 +++ .../models/qwen2_5_vl/modeling_qwen2_5_vl.py | 12 +++++ .../models/qwen2_vl/modeling_qwen2_vl.py | 12 +++++ .../models/qwen3_vl/modeling_qwen3_vl.py | 12 +++++ .../qwen3_vl_moe/modeling_qwen3_vl_moe.py | 12 +++++ .../models/smolvlm/modeling_smolvlm.py | 6 +++ .../video_llama_3/modeling_video_llama_3.py | 12 +++++ .../video_llava/modeling_video_llava.py | 11 ++++ .../models/vipllava/modeling_vipllava.py | 11 +++- .../models/vipllava/modular_vipllava.py | 11 +++- 28 files changed, 344 insertions(+), 36 deletions(-) diff --git a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py index 72137d85b176..cbc6e69843db 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py @@ -293,7 +293,7 @@ def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) @can_return_tuple - @auto_docstring + @auto_docstring(custom_args=DEEPSEEK_VL_COMMON_CUSTOM_ARGS) def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py index 017bdec68dc8..a4bbca85b829 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py @@ -303,7 +303,7 @@ def get_high_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: return high_res_outputs @can_return_tuple - @auto_docstring + @auto_docstring(custom_args=DEEPSEEK_VL_COMMON_CUSTOM_ARGS) def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index f949d63ee8b5..69d57c6e8b69 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -1081,6 +1081,21 @@ def forward(self, hidden_states, grid_thw): return hidden_states +ERNIE4_5_VL_MOE_VIDEO_COMMON_CUSTOM_ARGS = r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. +""" + +ERNIE4_5_VL_MOE_IMAGE_COMMON_CUSTOM_ARGS = r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. +""" + + @auto_docstring class Ernie4_5_VL_MoeModel(Ernie4_5_VL_MoePreTrainedModel): base_model_prefix = "model" @@ -1263,7 +1278,7 @@ def get_rope_index( return position_ids, mrope_position_deltas @can_return_tuple - @auto_docstring + @auto_docstring(custom_args=ERNIE4_5_VL_MOE_VIDEO_COMMON_CUSTOM_ARGS) def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -1288,7 +1303,7 @@ def get_video_features( return video_outputs @can_return_tuple - @auto_docstring + @auto_docstring(custom_args=ERNIE4_5_VL_MOE_IMAGE_COMMON_CUSTOM_ARGS) def get_image_features( self, pixel_values: torch.FloatTensor, @@ -1604,24 +1619,36 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) - @auto_docstring + @auto_docstring(custom_args=ERNIE4_5_VL_MOE_VIDEO_COMMON_CUSTOM_ARGS) def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + """ return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) - @auto_docstring + @auto_docstring(custom_args=ERNIE4_5_VL_MOE_IMAGE_COMMON_CUSTOM_ARGS) def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @auto_docstring diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index 7af37c5fecc7..11eb4186afd5 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -90,6 +90,20 @@ logger = logging.get_logger(__name__) +ERNIE4_5_VL_MOE_VIDEO_COMMON_CUSTOM_ARGS = r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. +""" + +ERNIE4_5_VL_MOE_IMAGE_COMMON_CUSTOM_ARGS = r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. +""" + class Ernie4_5_VL_MoeVisionConfig(Qwen2VLVisionConfig): r""" @@ -1260,19 +1274,13 @@ def get_rope_index( return position_ids, mrope_position_deltas @can_return_tuple - @auto_docstring + @auto_docstring(custom_args=ERNIE4_5_VL_MOE_VIDEO_COMMON_CUSTOM_ARGS) def get_video_features( self, pixel_values_videos: torch.FloatTensor, video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. - """ video_outputs = self.vision_tower(pixel_values_videos, video_grid_thw, **kwargs) video_embeds = self.resampler_model(video_outputs.last_hidden_state, video_grid_thw) split_sizes = ( @@ -1285,19 +1293,13 @@ def get_video_features( return video_outputs @can_return_tuple - @auto_docstring + @auto_docstring(custom_args=ERNIE4_5_VL_MOE_IMAGE_COMMON_CUSTOM_ARGS) def get_image_features( self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - """ image_outputs = self.vision_tower(pixel_values, image_grid_thw, **kwargs) image_embeds = self.resampler_model(image_outputs.last_hidden_state, image_grid_thw) split_sizes = (image_grid_thw.prod(-1) // self.vision_tower.spatial_merge_size**2).tolist() @@ -1400,6 +1402,14 @@ def __init__(self, config): self.num_experts = config.text_config.moe_num_experts self.num_experts_per_tok = config.text_config.moe_k + @auto_docstring(custom_args=ERNIE4_5_VL_MOE_VIDEO_COMMON_CUSTOM_ARGS) + def get_video_features(self, **super_kwargs): + return super().get_video_features(**super_kwargs) + + @auto_docstring(custom_args=ERNIE4_5_VL_MOE_IMAGE_COMMON_CUSTOM_ARGS) + def get_image_features(self, **super_kwargs): + return super().get_image_features(**super_kwargs) + def prepare_inputs_for_generation( self, input_ids, diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 8cfbab594efb..8d87fd38669e 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -544,6 +544,12 @@ def get_video_features( video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + """ return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) @@ -555,6 +561,12 @@ def get_image_features( image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index da495a1fa295..cf8e24cb955c 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1412,6 +1412,12 @@ def get_video_features( video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + """ return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) @@ -1423,6 +1429,12 @@ def get_image_features( image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 41d7187a772e..56707bded1a5 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1681,6 +1681,12 @@ def get_video_features( video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + """ return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) @@ -1692,6 +1698,12 @@ def get_image_features( image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @auto_docstring diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index a00b453e8d25..f305660fc9c1 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -1001,6 +1001,12 @@ def get_image_features( pixel_attention_mask: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + pixel_attention_mask (`torch.LongTensor`, *optional*): + The attention mask indicating padded regions in the image. + """ return self.model.get_image_features( pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask, **kwargs ) diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index 3a6f88d81758..6fd4c91176e5 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -579,8 +579,6 @@ def get_image_features( The tensors corresponding to the input images. pixel_attention_mask (`torch.LongTensor`, *optional*): The attention mask indicating padded regions in the image. - return_dict (`bool`, *optional*, default to `False`): - Whether to return a `ModelOutput` instead of a pooled embedding. """ batch_size, num_images, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility @@ -758,6 +756,12 @@ def get_image_features( pixel_attention_mask: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + pixel_attention_mask (`torch.LongTensor`, *optional*): + The attention mask indicating padded regions in the image. + """ return self.model.get_image_features( pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask, **kwargs ) diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index 6f660a5d50a6..b105e20e73b6 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -1199,6 +1199,21 @@ def get_image_features( r""" pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): The tensors corresponding to the input images. + qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided + to serve as text prompt, which the Q-Former model will encode. + + Indices can be obtained using [`InstructBlipProcessor`]. See [`InstructBlipProcessor.__call__`] for + details. + + [What are input IDs?](../glossary#input-ids) + qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) """ # step 1: forward the images through the vision encoder, # to get image embeddings of shape (batch_size, seq_len, hidden_size) diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index e259ea45453a..28680f2040a7 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -1464,6 +1464,10 @@ def get_video_features( r""" pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): The tensors corresponding to the input images. + qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)): + The sequence used as a prompt to be fed to the Q-Former module. + qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): + Mask to avoid performing attention on padding token indices. """ # step 1: forward the images through the vision encoder, # we process in a batched way, later unbatch it back (video has frames=4 always) diff --git a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py index 6f4bc70b9977..8b76bfc6fba1 100644 --- a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py @@ -298,6 +298,10 @@ def get_video_features( r""" pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): The tensors corresponding to the input images. + qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)): + The sequence used as a prompt to be fed to the Q-Former module. + qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): + Mask to avoid performing attention on padding token indices. """ # step 1: forward the images through the vision encoder, # we process in a batched way, later unbatch it back (video has frames=4 always) diff --git a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py index a479c5466d5f..19dccd8046c6 100755 --- a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py @@ -329,6 +329,14 @@ def get_image_features( pixel_attention_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + spatial_shapes (`torch.Tensor` of shape `(batch_size, 2)`): + The spatial shapes of the input images. + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, height, width)`): + The pixel attention mask of the input images. + """ return self.model.get_image_features( pixel_values=pixel_values, spatial_shapes=spatial_shapes, diff --git a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py index e35bd9d6763b..81222a4b704f 100644 --- a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py @@ -239,6 +239,14 @@ def get_image_features( pixel_attention_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + spatial_shapes (`torch.Tensor` of shape `(batch_size, 2)`): + The spatial shapes of the input images. + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, height, width)`): + The pixel attention mask of the input images. + """ return self.model.get_image_features( pixel_values=pixel_values, spatial_shapes=spatial_shapes, diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index 5b6c3a4276ee..311f9ae9e15c 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -574,6 +574,19 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) + The tensors corresponding to the input images. + image_sizes (`torch.Tensor` of shape `(num_images, 2)`) + Actual image size of each images (H, W). + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` + """ return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index ddf9b5f6b6a6..a85364aaa1f6 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -716,6 +716,19 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) + The tensors corresponding to the input images. + image_sizes (`torch.Tensor` of shape `(num_images, 2)`) + Actual image size of each images (H, W). + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` + """ return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index 13fa007b1321..819567991fc8 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -300,7 +300,7 @@ def pack_image_features(self, image_features, image_sizes, image_newline=None, v image_newline (`torch.Tensor` of shape `(embed_dim)`) New line embedding vector. vision_aspect_ratio (`str`, *optional*, "anyres_max_9"): - Aspect ratio used when processong image features. The default value is "anyres_max_9". + Aspect ratio used when processing image features. The default value is "anyres_max_9". Returns: image_features (`torch.Tensor` of shape `(all_feat_len, embed_dim)`) feature_lens (`list[int]`) @@ -368,9 +368,9 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`): The tensors corresponding to the input images. - image_sizes (`torch.Tensor` of shape `(num_images, 2)`) + image_sizes (`torch.Tensor` of shape `(num_images, 2)`): Actual image size of each images (H, W). vision_feature_layer (`Union[int, list[int]]`): The index of the layer to select the vision feature. If multiple indices are provided, @@ -379,6 +379,8 @@ def get_image_features( vision_feature_select_strategy (`str`): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` + vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): + Aspect ratio used when processing image features. The default value is "anyres_max_9". batch_num_images (`torch.LongTensor`, *optional*): Number of images in each sample. """ @@ -511,7 +513,7 @@ def forward( image_sizes_videos (`torch.LongTensor` of shape `(batch_size, frames, 2)`, *optional*): The sizes of the videos in the batch, being (height, width) for each frame in the video. vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): - Aspect ratio used when processong image features. The default value is "anyres_max_9". + Aspect ratio used when processing image features. The default value is "anyres_max_9". batch_num_images (`torch.LongTensor`, *optional*): Number of images in each sample. """ @@ -710,13 +712,34 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + vision_aspect_ratio: str | None = None, + batch_num_images: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) + The tensors corresponding to the input images. + image_sizes (`torch.Tensor` of shape `(num_images, 2)`) + Actual image size of each images (H, W). + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` + vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): + Aspect ratio used when processing image features. The default value is "anyres_max_9". + batch_num_images (`torch.LongTensor`, *optional*): + Number of images in each sample. + """ return self.model.get_image_features( pixel_values=pixel_values, image_sizes=image_sizes, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, + vision_aspect_ratio=vision_aspect_ratio, + batch_num_images=batch_num_images, **kwargs, ) @@ -750,7 +773,7 @@ def forward( image_sizes_videos (`torch.LongTensor` of shape `(batch_size, frames, 2)`, *optional*): The sizes of the videos in the batch, being (height, width) for each frame in the video. vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): - Aspect ratio used when processong image features. The default value is "anyres_max_9". + Aspect ratio used when processing image features. The default value is "anyres_max_9". batch_num_images (`torch.LongTensor`, *optional*): Number of images in each sample. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index cb9282cb79da..122bc899d728 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -246,7 +246,7 @@ def pack_image_features(self, image_features, image_sizes, image_newline=None, v image_newline (`torch.Tensor` of shape `(embed_dim)`) New line embedding vector. vision_aspect_ratio (`str`, *optional*, "anyres_max_9"): - Aspect ratio used when processong image features. The default value is "anyres_max_9". + Aspect ratio used when processing image features. The default value is "anyres_max_9". Returns: image_features (`torch.Tensor` of shape `(all_feat_len, embed_dim)`) feature_lens (`list[int]`) @@ -328,9 +328,9 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`): The tensors corresponding to the input images. - image_sizes (`torch.Tensor` of shape `(num_images, 2)`) + image_sizes (`torch.Tensor` of shape `(num_images, 2)`): Actual image size of each images (H, W). vision_feature_layer (`Union[int, list[int]]`): The index of the layer to select the vision feature. If multiple indices are provided, @@ -339,6 +339,8 @@ def get_image_features( vision_feature_select_strategy (`str`): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` + vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): + Aspect ratio used when processing image features. The default value is "anyres_max_9". batch_num_images (`torch.LongTensor`, *optional*): Number of images in each sample. """ @@ -483,7 +485,7 @@ def forward( image_sizes_videos (`torch.LongTensor` of shape `(batch_size, frames, 2)`, *optional*): The sizes of the videos in the batch, being (height, width) for each frame in the video. vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): - Aspect ratio used when processong image features. The default value is "anyres_max_9". + Aspect ratio used when processing image features. The default value is "anyres_max_9". batch_num_images (`torch.LongTensor`, *optional*): Number of images in each sample. """ @@ -600,7 +602,7 @@ def forward( image_sizes_videos (`torch.LongTensor` of shape `(batch_size, frames, 2)`, *optional*): The sizes of the videos in the batch, being (height, width) for each frame in the video. vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): - Aspect ratio used when processong image features. The default value is "anyres_max_9". + Aspect ratio used when processing image features. The default value is "anyres_max_9". batch_num_images (`torch.LongTensor`, *optional*): Number of images in each sample. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): @@ -739,6 +741,44 @@ def prepare_inputs_for_generation( return model_inputs + @auto_docstring + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_sizes: torch.Tensor, + vision_feature_layer: int | list[int] | None = None, + vision_feature_select_strategy: str | None = None, + vision_aspect_ratio: str | None = None, + batch_num_images: torch.LongTensor | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) + The tensors corresponding to the input images. + image_sizes (`torch.Tensor` of shape `(num_images, 2)`) + Actual image size of each images (H, W). + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` + vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): + Aspect ratio used when processing image features. The default value is "anyres_max_9". + batch_num_images (`torch.LongTensor`, *optional*): + Number of images in each sample. + """ + return self.model.get_image_features( + pixel_values=pixel_values, + image_sizes=image_sizes, + vision_feature_layer=vision_feature_layer, + vision_feature_select_strategy=vision_feature_select_strategy, + vision_aspect_ratio=vision_aspect_ratio, + batch_num_images=batch_num_images, + **kwargs, + ) + __all__ = [ "LlavaOnevisionImageProcessorFast", diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index be6dfc27a0a3..6fe1c5355b5b 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -1360,6 +1360,12 @@ def get_image_features( image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index c32e6935e63d..632d9e2129ba 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -1420,6 +1420,12 @@ def get_video_features( video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + """ return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) @@ -1431,6 +1437,12 @@ def get_image_features( image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 5184ecff8d78..54766a2b26db 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -1318,6 +1318,12 @@ def get_video_features( video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + """ return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) @@ -1329,6 +1335,12 @@ def get_image_features( image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index 8b66e8e38f23..5a838cd3d7f8 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -1347,6 +1347,12 @@ def get_video_features( video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithDeepstackFeatures: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + """ return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) @@ -1358,6 +1364,12 @@ def get_image_features( image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithDeepstackFeatures: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index 002381630bdc..3ce9c2feb37e 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -1527,6 +1527,12 @@ def get_video_features( video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithDeepstackFeatures: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + """ return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) @@ -1538,6 +1544,12 @@ def get_image_features( image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithDeepstackFeatures: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/smolvlm/modeling_smolvlm.py b/src/transformers/models/smolvlm/modeling_smolvlm.py index 4abd07b1d67b..300635317263 100644 --- a/src/transformers/models/smolvlm/modeling_smolvlm.py +++ b/src/transformers/models/smolvlm/modeling_smolvlm.py @@ -743,6 +743,12 @@ def get_image_features( pixel_attention_mask: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + pixel_attention_mask (`torch.LongTensor`, *optional*): + The attention mask indicating padded regions in the image. + """ return self.model.get_image_features( pixel_values=pixel_values, pixel_attention_mask=pixel_attention_mask, **kwargs ) diff --git a/src/transformers/models/video_llama_3/modeling_video_llama_3.py b/src/transformers/models/video_llama_3/modeling_video_llama_3.py index 22917c32b07e..b5a7a1b370ec 100644 --- a/src/transformers/models/video_llama_3/modeling_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modeling_video_llama_3.py @@ -782,6 +782,12 @@ def get_video_features( video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + """ return self.model.get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) @@ -793,6 +799,12 @@ def get_image_features( image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs) @can_return_tuple diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index 711f022a652f..ed6802f97c09 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -436,6 +436,17 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values_images (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. + vision_feature_layer (`Union[int, list[int]]`, *optional*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` + """ return self.model.get_image_features( pixel_values_images=pixel_values_images, vision_feature_layer=vision_feature_layer, diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index bd83f95d2073..5aa4b750d956 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -160,9 +160,9 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): The tensors corresponding to the input images. - vision_feature_layers (`Union[int, list[int]]`): + vision_feature_layers (`Union[int, list[int]]`, *optional*): The vision feature layer, or the list of indexes of the layers to select the vision feature. """ @@ -315,6 +315,13 @@ def get_image_features( vision_feature_layers: int | list[int] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + vision_feature_layers (`Union[int, list[int]]`, *optional*): + The vision feature layer, or the list of indexes of the layers to select + the vision feature. + """ return self.model.get_image_features( pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, **kwargs ) diff --git a/src/transformers/models/vipllava/modular_vipllava.py b/src/transformers/models/vipllava/modular_vipllava.py index 8d0e265cd2ef..f20f1c18dc89 100644 --- a/src/transformers/models/vipllava/modular_vipllava.py +++ b/src/transformers/models/vipllava/modular_vipllava.py @@ -82,9 +82,9 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): The tensors corresponding to the input images. - vision_feature_layers (`Union[int, list[int]]`): + vision_feature_layers (`Union[int, list[int]]`, *optional*): The vision feature layer, or the list of indexes of the layers to select the vision feature. """ @@ -185,6 +185,13 @@ def get_image_features( vision_feature_layers: int | list[int] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. + vision_feature_layers (`Union[int, list[int]]`, *optional*): + The vision feature layer, or the list of indexes of the layers to select + the vision feature. + """ return self.model.get_image_features( pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, **kwargs ) From af3b70fca8c5d77a8817836a4e29abd0bd4a0e67 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 14 Jan 2026 16:14:17 +0100 Subject: [PATCH 086/131] Remove redundant docstrings --- src/transformers/models/aria/modeling_aria.py | 8 ++------ .../models/aya_vision/modeling_aya_vision.py | 4 ---- src/transformers/models/blip_2/modeling_blip_2.py | 2 +- .../models/chameleon/modeling_chameleon.py | 2 +- .../models/cohere2_vision/modeling_cohere2_vision.py | 4 ---- src/transformers/models/edgetam/modeling_edgetam.py | 12 ++++++------ .../models/edgetam_video/modeling_edgetam_video.py | 12 ++++++------ src/transformers/models/emu3/modeling_emu3.py | 4 ++-- src/transformers/models/emu3/modular_emu3.py | 2 +- src/transformers/models/gemma3/modeling_gemma3.py | 4 ---- src/transformers/models/gemma3n/modeling_gemma3n.py | 4 ---- src/transformers/models/glm4v/modeling_glm4v.py | 11 +++++------ .../models/glm4v_moe/modeling_glm4v_moe.py | 11 +++++------ .../models/got_ocr2/modeling_got_ocr2.py | 3 --- .../models/instructblip/modeling_instructblip.py | 2 +- .../instructblipvideo/modeling_instructblipvideo.py | 2 +- src/transformers/models/janus/modeling_janus.py | 2 +- src/transformers/models/kosmos2/modeling_kosmos2.py | 2 +- src/transformers/models/llava/modeling_llava.py | 4 ---- .../llava_onevision/modeling_llava_onevision.py | 4 ---- .../models/mistral3/modeling_mistral3.py | 4 ---- src/transformers/models/ovis2/modeling_ovis2.py | 10 +++------- .../models/paligemma/modeling_paligemma.py | 4 ---- .../models/perception_lm/modeling_perception_lm.py | 4 ---- .../models/qwen3_omni_moe/modeling_qwen3_omni_moe.py | 2 +- .../models/qwen3_vl/modeling_qwen3_vl.py | 2 +- .../models/qwen3_vl_moe/modeling_qwen3_vl_moe.py | 2 +- src/transformers/models/sam2/modeling_sam2.py | 12 ++++++------ .../models/sam2_video/modeling_sam2_video.py | 12 ++++++------ .../models/sam3_tracker/modeling_sam3_tracker.py | 12 ++++++------ .../modeling_sam3_tracker_video.py | 12 ++++++------ 31 files changed, 63 insertions(+), 112 deletions(-) diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index c7e752230110..c92a4b155eef 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -929,15 +929,11 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer diff --git a/src/transformers/models/aya_vision/modeling_aya_vision.py b/src/transformers/models/aya_vision/modeling_aya_vision.py index 05001249b42c..715446dfedbd 100644 --- a/src/transformers/models/aya_vision/modeling_aya_vision.py +++ b/src/transformers/models/aya_vision/modeling_aya_vision.py @@ -193,10 +193,6 @@ def get_image_features( r""" pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 1c48341d7d97..33892bfbde48 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -56,7 +56,7 @@ @dataclass @auto_docstring class BaseModelOutputWithVisionQformerOutputs(BaseModelOutputWithPooling): - """ + r""" vision_outputs (`BaseModelOutputWithPooling`): Outputs of the vision encoder. qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index c85757d1da5c..ef04330e305c 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -51,7 +51,7 @@ class ChameleonVQVAEModelOutput(BaseModelOutputWithPooling): r""" quantized_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): Quantized last hidden state from the VQ-VAE model. - image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):` + image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`): Indices of the image tokens predicted by the VQ-VAE model. embedding_loss (`torch.FloatTensor`): The embedding loss computed during quantization. diff --git a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py index 68d3eb066a53..a17bd276da59 100644 --- a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py @@ -172,10 +172,6 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) - The tensors corresponding to the input images. - """ image_outputs = self.vision_tower(pixel_values, **kwargs) selected_image_feature = image_outputs.last_hidden_state image_outputs.pooler_output = self.multi_modal_projector(selected_image_feature) diff --git a/src/transformers/models/edgetam/modeling_edgetam.py b/src/transformers/models/edgetam/modeling_edgetam.py index 723ad8cbeb78..b7d5c9e20a83 100644 --- a/src/transformers/models/edgetam/modeling_edgetam.py +++ b/src/transformers/models/edgetam/modeling_edgetam.py @@ -79,12 +79,6 @@ class EdgeTamVisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. - fpn_hidden_states (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. - fpn_position_encoding (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each stage) of shape `(batch_size, height, width, hidden_size)`. Hidden-states of the @@ -93,6 +87,12 @@ class EdgeTamVisionEncoderOutput(BaseModelOutputWithPooling): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + fpn_hidden_states (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. + fpn_position_encoding (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. """ fpn_hidden_states: torch.FloatTensor | None = None diff --git a/src/transformers/models/edgetam_video/modeling_edgetam_video.py b/src/transformers/models/edgetam_video/modeling_edgetam_video.py index 6c323093bdbd..7c58d47ccfbd 100644 --- a/src/transformers/models/edgetam_video/modeling_edgetam_video.py +++ b/src/transformers/models/edgetam_video/modeling_edgetam_video.py @@ -120,12 +120,6 @@ class EdgeTamVideoVisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. - fpn_hidden_states (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. - fpn_position_encoding (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each stage) of shape `(batch_size, height, width, hidden_size)`. Hidden-states of the @@ -134,6 +128,12 @@ class EdgeTamVideoVisionEncoderOutput(BaseModelOutputWithPooling): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + fpn_hidden_states (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. + fpn_position_encoding (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. """ fpn_hidden_states: torch.FloatTensor | None = None diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index da69e7fccd1a..984266fe8f30 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -49,7 +49,7 @@ @auto_docstring class Emu3VQVAEModelOutput(BaseModelOutputWithPooling): r""" - image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):` + image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`): Indices of the image tokens predicted by the VQ-VAE model. """ @@ -1405,7 +1405,7 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.L def get_image_features( self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | Emu3VQVAEModelOutput: - """ + r""" pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): The tensors corresponding to the input images. """ diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index fe4cec5aae2b..6b2614d50b80 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -45,7 +45,7 @@ @auto_docstring class Emu3VQVAEModelOutput(BaseModelOutputWithPooling): r""" - image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):` + image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`): Indices of the image tokens predicted by the VQ-VAE model. """ diff --git a/src/transformers/models/gemma3/modeling_gemma3.py b/src/transformers/models/gemma3/modeling_gemma3.py index c7d8d9169b66..cad5d2817791 100644 --- a/src/transformers/models/gemma3/modeling_gemma3.py +++ b/src/transformers/models/gemma3/modeling_gemma3.py @@ -848,10 +848,6 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - """ vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) last_hidden_state = vision_outputs.last_hidden_state vision_outputs.pooler_output = self.multi_modal_projector(last_hidden_state) diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index 11b9b9cc0537..d60474931255 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -1980,10 +1980,6 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - """ vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, **kwargs) last_hidden_state = vision_outputs.last_hidden_state # Convert from (batch, channels, height, width) to (batch, height * width, channels) where: diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index cf8e24cb955c..c01cf1efe638 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -784,12 +784,11 @@ def rot_pos_emb(self, grid_thw): def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> torch.Tensor: - """ - Args: - hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): - The final hidden states of the model. - grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): - The temporal, height and width of feature shape of each image in LLM. + r""" + hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): + The final hidden states of the model. + grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): + The temporal, height and width of feature shape of each image in LLM. Returns: `torch.Tensor`: hidden_states. diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 56707bded1a5..662ea94aa968 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -998,12 +998,11 @@ def rot_pos_emb(self, grid_thw): def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> torch.Tensor: - """ - Args: - hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): - The final hidden states of the model. - grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): - The temporal, height and width of feature shape of each image in LLM. + r""" + hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): + The final hidden states of the model. + grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): + The temporal, height and width of feature shape of each image in LLM. Returns: `torch.Tensor`: hidden_states. diff --git a/src/transformers/models/got_ocr2/modeling_got_ocr2.py b/src/transformers/models/got_ocr2/modeling_got_ocr2.py index 0be2846372cc..8f2103fa82cb 100644 --- a/src/transformers/models/got_ocr2/modeling_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modeling_got_ocr2.py @@ -558,9 +558,6 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - pixel_values (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`) - """ image_outputs = self.vision_tower(pixel_values, **kwargs) last_hidden_state = image_outputs.last_hidden_state image_outputs.pooler_output = self.multi_modal_projector(last_hidden_state) diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index b105e20e73b6..b51730e5c9a3 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -47,7 +47,7 @@ @dataclass @auto_docstring class BaseModelOutputWithVisionQformerOutputs(BaseModelOutputWithPooling): - """ + r""" vision_outputs (`BaseModelOutputWithPooling`): Outputs of the vision encoder. qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 28680f2040a7..730a86299715 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -1123,7 +1123,7 @@ def forward( @dataclass @auto_docstring class BaseModelOutputWithVisionQformerOutputs(BaseModelOutputWithPooling): - """ + r""" vision_outputs (`BaseModelOutputWithPooling`): Outputs of the vision encoder. qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): diff --git a/src/transformers/models/janus/modeling_janus.py b/src/transformers/models/janus/modeling_janus.py index d74ee3b1bdf0..cd3ad9a5b2fd 100644 --- a/src/transformers/models/janus/modeling_janus.py +++ b/src/transformers/models/janus/modeling_janus.py @@ -834,7 +834,7 @@ class JanusVQVAEModelOutput(BaseModelOutputWithPooling): r""" quantized_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): Quantized last hidden state from the VQ-VAE model. - image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):` + image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`): Indices of the image tokens predicted by the VQ-VAE model. embedding_loss (`torch.FloatTensor`): The embedding loss computed during quantization. diff --git a/src/transformers/models/kosmos2/modeling_kosmos2.py b/src/transformers/models/kosmos2/modeling_kosmos2.py index 31998df0cf74..d69847db3229 100644 --- a/src/transformers/models/kosmos2/modeling_kosmos2.py +++ b/src/transformers/models/kosmos2/modeling_kosmos2.py @@ -77,7 +77,7 @@ def _make_causal_mask( @dataclass @auto_docstring class BaseModelOutputWithProjectionAttentions(BaseModelOutputWithPooling): - """ + r""" projection_attentions (`tuple(torch.FloatTensor)`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index c2e0f1094246..53e2c2d19b28 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -159,10 +159,6 @@ def get_image_features( r""" pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index 819567991fc8..b2ac6e374172 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -372,10 +372,6 @@ def get_image_features( The tensors corresponding to the input images. image_sizes (`torch.Tensor` of shape `(num_images, 2)`): Actual image size of each images (H, W). - vision_feature_layer (`Union[int, list[int]]`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. vision_feature_select_strategy (`str`): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index a86c7a92a9c8..edd5cbaee7e5 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -226,10 +226,6 @@ def get_image_features( r""" pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. image_sizes (`torch.Tensor`, *optional*): Tensor containing the image sizes as returned by the processor. """ diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index 1a8334fd9503..63f7f40afed0 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -43,7 +43,7 @@ @dataclass @auto_docstring class BaseModelOutputWithVisualIndicatorFeatures(BaseModelOutputWithPooling): - """ + r""" visual_indicator_features (`torch.FloatTensor` of shape `(batch_size, visual_indicator_size)`): Visual indicator features extracted from the model, which can be used for auxiliary tasks or further processing. """ @@ -495,15 +495,11 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures: r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): + The tensors corresponding to the input images. """ image_outputs = self.vision_tower(pixel_values, **kwargs) image_features = image_outputs.pooler_output diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index 0ecfd472a13f..8e82cb23a3d4 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -265,10 +265,6 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - """ image_outputs = self.vision_tower(pixel_values, **kwargs) selected_image_feature = image_outputs.last_hidden_state image_features = self.multi_modal_projector(selected_image_feature) diff --git a/src/transformers/models/perception_lm/modeling_perception_lm.py b/src/transformers/models/perception_lm/modeling_perception_lm.py index 3b290c721340..1c5436fd4367 100644 --- a/src/transformers/models/perception_lm/modeling_perception_lm.py +++ b/src/transformers/models/perception_lm/modeling_perception_lm.py @@ -188,10 +188,6 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_tiles, channels, height, width)`) - The tensors corresponding to the input images. - """ image_outputs = self.vision_tower(pixel_values.flatten(0, 1), **kwargs) last_hidden_state = image_outputs.last_hidden_state if self.config.vision_use_cls_token: diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 7e6d504864ab..501d70776766 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -71,7 +71,7 @@ @dataclass @auto_docstring class BaseModelOutputWithDeepstackFeatures(BaseModelOutputWithPooling): - """ + r""" deepstack_features (`List[torch.FloatTensor]`, *optional*): List of hidden-states (feature maps) from deepstack layers. """ diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index 5a838cd3d7f8..6ab56f5a3b62 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -46,7 +46,7 @@ @dataclass @auto_docstring class BaseModelOutputWithDeepstackFeatures(BaseModelOutputWithPooling): - """ + r""" deepstack_features (`List[torch.FloatTensor]`, *optional*): List of hidden-states (feature maps) from deepstack layers. """ diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index 3ce9c2feb37e..449a8910c22c 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -551,7 +551,7 @@ def forward( @dataclass @auto_docstring class BaseModelOutputWithDeepstackFeatures(BaseModelOutputWithPooling): - """ + r""" deepstack_features (`List[torch.FloatTensor]`, *optional*): List of hidden-states (feature maps) from deepstack layers. """ diff --git a/src/transformers/models/sam2/modeling_sam2.py b/src/transformers/models/sam2/modeling_sam2.py index b024b5c90cf4..98a4d0070508 100644 --- a/src/transformers/models/sam2/modeling_sam2.py +++ b/src/transformers/models/sam2/modeling_sam2.py @@ -55,12 +55,6 @@ class Sam2VisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. - fpn_hidden_states (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. - fpn_position_encoding (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each stage) of shape `(batch_size, height, width, hidden_size)`. Hidden-states of the @@ -69,6 +63,12 @@ class Sam2VisionEncoderOutput(BaseModelOutputWithPooling): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + fpn_hidden_states (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. + fpn_position_encoding (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. """ fpn_hidden_states: torch.FloatTensor | None = None diff --git a/src/transformers/models/sam2_video/modeling_sam2_video.py b/src/transformers/models/sam2_video/modeling_sam2_video.py index 83af1c6cf433..7cab5f69bad7 100644 --- a/src/transformers/models/sam2_video/modeling_sam2_video.py +++ b/src/transformers/models/sam2_video/modeling_sam2_video.py @@ -1144,12 +1144,6 @@ class Sam2VideoVisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. - fpn_hidden_states (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. - fpn_position_encoding (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each stage) of shape `(batch_size, height, width, hidden_size)`. Hidden-states of the @@ -1158,6 +1152,12 @@ class Sam2VideoVisionEncoderOutput(BaseModelOutputWithPooling): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + fpn_hidden_states (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. + fpn_position_encoding (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. """ fpn_hidden_states: torch.FloatTensor | None = None diff --git a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py index 7fb854a8ef40..cffbb2be4616 100644 --- a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py +++ b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py @@ -729,12 +729,6 @@ class Sam3TrackerVisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. - fpn_hidden_states (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. - fpn_position_encoding (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each stage) of shape `(batch_size, height, width, hidden_size)`. Hidden-states of the @@ -743,6 +737,12 @@ class Sam3TrackerVisionEncoderOutput(BaseModelOutputWithPooling): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + fpn_hidden_states (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. + fpn_position_encoding (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. """ fpn_hidden_states: torch.FloatTensor | None = None diff --git a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py index 81b9bdfd44a8..1f248c7f6e49 100644 --- a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py @@ -1128,12 +1128,6 @@ class Sam3TrackerVideoVisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. - fpn_hidden_states (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. - fpn_position_encoding (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each stage) of shape `(batch_size, height, width, hidden_size)`. Hidden-states of the @@ -1142,6 +1136,12 @@ class Sam3TrackerVideoVisionEncoderOutput(BaseModelOutputWithPooling): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + fpn_hidden_states (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. + fpn_position_encoding (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. """ fpn_hidden_states: torch.FloatTensor | None = None From 356404522980dc10a3af7a63490f4e95fc39a3ba Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 14 Jan 2026 17:12:26 +0100 Subject: [PATCH 087/131] Also patch the new glm_image for get_image_features/output_hidden_states --- docs/source/en/model_doc/glm_image.md | 2 + .../models/glm4v/modeling_glm4v.py | 2 +- .../models/glm_image/modeling_glm_image.py | 108 +++++++++++++----- .../models/glm_image/modular_glm_image.py | 84 +++++++++++--- .../glm_image/test_modeling_glm_image.py | 8 +- 5 files changed, 156 insertions(+), 48 deletions(-) diff --git a/docs/source/en/model_doc/glm_image.md b/docs/source/en/model_doc/glm_image.md index 4b4ff609d2dd..0e16b245c25b 100644 --- a/docs/source/en/model_doc/glm_image.md +++ b/docs/source/en/model_doc/glm_image.md @@ -199,8 +199,10 @@ print(f"Output tokens: {output_tokens}") [[autodoc]] GlmImageModel - forward + - get_image_features ## GlmImageForConditionalGeneration [[autodoc]] GlmImageForConditionalGeneration - forward + - get_image_features diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index e166e1b8d745..3d65ce741852 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -761,7 +761,7 @@ def rot_pos_emb(self, grid_thw): @auto_docstring def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] - ) -> torch.Tensor: + ) -> tuple | BaseModelOutputWithPooling: r""" hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. diff --git a/src/transformers/models/glm_image/modeling_glm_image.py b/src/transformers/models/glm_image/modeling_glm_image.py index fb6fb23db4d7..40e0370ec926 100644 --- a/src/transformers/models/glm_image/modeling_glm_image.py +++ b/src/transformers/models/glm_image/modeling_glm_image.py @@ -32,7 +32,7 @@ from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -612,6 +612,23 @@ def forward(self, hidden_state: torch.Tensor): return hidden_state_quant, loss, min_encoding_indices +@dataclass +@auto_docstring +class GlmImageVQVAEModelOutput(BaseModelOutputWithPooling): + r""" + quantized_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + Quantized last hidden state from the VQ-VAE model. + image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`): + Indices of the image tokens predicted by the VQ-VAE model. + embedding_loss (`torch.FloatTensor`): + The embedding loss computed during quantization. + """ + + quantized_last_hidden_state: torch.FloatTensor | None = None + image_tokens: torch.FloatTensor | None = None + embedding_loss: torch.FloatTensor | None = None + + @auto_docstring( custom_intro=""" The VQ-VAE model used in GlmImage for encoding/decoding images into discrete tokens. @@ -625,6 +642,7 @@ class GlmImageVQVAE(GlmImagePreTrainedModel): _no_split_modules = [ "GlmImageVQVAEVectorQuantizer", ] + _can_record_outputs = {} def __init__(self, config: GlmImageVQVAEConfig): super().__init__(config) @@ -634,16 +652,26 @@ def __init__(self, config: GlmImageVQVAEConfig): self.eval() # GlmImage's VQ model is frozen self.post_init() - def encode(self, hidden_states): - hidden_states = self.quant_conv(hidden_states) - quant, emb_loss, indices = self.quantize(hidden_states) - return quant, emb_loss, indices + @check_model_inputs(tie_last_hidden_states=False) + def encode(self, hidden_states) -> GlmImageVQVAEModelOutput: + conv_hidden_states = self.quant_conv(hidden_states) + quantized_last_hidden_state, emb_loss, indices = self.quantize(conv_hidden_states) + return GlmImageVQVAEModelOutput( + last_hidden_state=hidden_states, + quantized_last_hidden_state=quantized_last_hidden_state, + image_tokens=indices, + embedding_loss=emb_loss, + ) class GlmImageVisionModel(GlmImagePreTrainedModel): config: GlmImageVisionConfig input_modalities = ("image",) _no_split_modules = ["GlmImageVisionBlock"] + _can_record_outputs = { + "hidden_states": GlmImageVisionBlock, + "attentions": GlmImageVisionAttention, + } main_input_name = "pixel_values" def __init__(self, config: GlmImageVisionConfig) -> None: @@ -688,13 +716,16 @@ def rot_pos_emb(self, grid_thw): pos_ids = torch.cat(pos_ids, dim=0) return pos_ids - def forward(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor: + @check_model_inputs(tie_last_hidden_states=False) + @auto_docstring + def forward( + self, pixel_values: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> tuple | BaseModelOutputWithPooling: """ - Args: - pixel_values (`torch.Tensor` of shape `(total_patches, num_channels * patch_size * patch_size)`): - Packed pixel values. - grid_thw (`torch.Tensor` of shape `(num_images, 3)`): - The temporal, height and width of feature shape of each image. + pixel_values (`torch.Tensor` of shape `(total_patches, num_channels * patch_size * patch_size)`): + Packed pixel values. + grid_thw (`torch.Tensor` of shape `(num_images, 3)`): + The temporal, height and width of feature shape of each image. Returns: `torch.Tensor` of shape `(total_patches, hidden_size)`: Hidden states. @@ -723,7 +754,8 @@ def forward(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor, **kwargs) hidden_states, cu_seqlens=cu_seqlens, ) - return hidden_states + + return BaseModelOutputWithPooling(last_hidden_state=hidden_states) class GlmImageTextRotaryEmbedding(nn.Module): @@ -1161,21 +1193,27 @@ def get_rope_index( return position_ids, mrope_position_deltas - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None): - """ - Encodes images into continuous embeddings that can be forwarded to the language model. - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. + @can_return_tuple + @auto_docstring + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: torch.LongTensor | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) - image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() - image_embeds = torch.split(image_embeds, split_sizes) - return image_embeds + image_embeds = torch.split(vision_outputs.last_hidden_state, split_sizes) + vision_outputs.pooler_output = image_embeds + + return vision_outputs def get_placeholder_mask( self, @@ -1233,7 +1271,7 @@ def forward( raise ValueError("You must specify exactly one of input_ids or inputs_embeds") if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw[:-1]) + image_embeds = self.get_image_features(pixel_values, image_grid_thw[:-1], return_dict=True).pooler_output image_embeds = torch.cat(image_embeds, dim=0) image_ids = self.get_image_tokens(image_embeds, image_grid_thw[:-1]) image_ids = image_ids.view(-1).to(input_ids.device) @@ -1319,8 +1357,8 @@ def get_image_tokens( grid_t, grid_h, grid_w = image_grid_thw[i].tolist() hs = hs.view(grid_t, grid_h, grid_w, hidden_size) hs = hs.permute(0, 3, 1, 2).contiguous() - _, _, image_toks = self.vqmodel.encode(hs) - all_image_toks.append(image_toks) + vqmodel_outputs: GlmImageVQVAEModelOutput = self.vqmodel.encode(hs) + all_image_toks.append(vqmodel_outputs.image_tokens) return torch.cat(all_image_toks, dim=0) @@ -1369,8 +1407,20 @@ def __init__(self, config): # Initialize weights and apply final processing self.post_init() - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None): - return self.model.get_image_features(pixel_values, image_grid_thw) + @auto_docstring + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: torch.LongTensor | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ + return self.model.get_image_features(pixel_values, image_grid_thw, **kwargs) def get_image_tokens(self, hidden_states: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None): return self.model.get_image_tokens(hidden_states, image_grid_thw) diff --git a/src/transformers/models/glm_image/modular_glm_image.py b/src/transformers/models/glm_image/modular_glm_image.py index ac7d1c33b92d..cbd41bc29d20 100644 --- a/src/transformers/models/glm_image/modular_glm_image.py +++ b/src/transformers/models/glm_image/modular_glm_image.py @@ -25,11 +25,13 @@ from ...feature_extraction_utils import BatchFeature from ...generation import GenerationMixin from ...image_utils import ImageInput +from ...modeling_outputs import BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import ImagesKwargs, ProcessorMixin, Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput -from ...utils import TransformersKwargs, is_torch_available, logging -from ..chameleon.modeling_chameleon import ChameleonVQVAE, ChameleonVQVAEVectorQuantizer +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torch_available, logging +from ...utils.generic import check_model_inputs +from ..chameleon.modeling_chameleon import ChameleonVQVAE, ChameleonVQVAEModelOutput, ChameleonVQVAEVectorQuantizer from ..glm4v.configuration_glm4v import Glm4vTextConfig, Glm4vVisionConfig from ..glm4v.modeling_glm4v import ( Glm4vCausalLMOutputWithPast, @@ -516,19 +518,29 @@ def forward(self, hidden_state: torch.Tensor): return hidden_state_quant, loss, min_encoding_indices +class GlmImageVQVAEModelOutput(ChameleonVQVAEModelOutput): + pass + + class GlmImageVQVAE(ChameleonVQVAE): _no_split_modules = [ "GlmImageVQVAEVectorQuantizer", ] + _can_record_outputs = {} def __init__(self, config: GlmImageVQVAEConfig): super().__init__(config) del self.encoder def encode(self, hidden_states): - hidden_states = self.quant_conv(hidden_states) - quant, emb_loss, indices = self.quantize(hidden_states) - return quant, emb_loss, indices + conv_hidden_states = self.quant_conv(hidden_states) + quantized_last_hidden_state, emb_loss, indices = self.quantize(conv_hidden_states) + return GlmImageVQVAEModelOutput( + last_hidden_state=hidden_states, + quantized_last_hidden_state=quantized_last_hidden_state, + image_tokens=indices, + embedding_loss=emb_loss, + ) class GlmImageVisionModel(Glm4vVisionModel): @@ -574,13 +586,16 @@ def rot_pos_emb(self, grid_thw): pos_ids = torch.cat(pos_ids, dim=0) return pos_ids - def forward(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor: + @check_model_inputs(tie_last_hidden_states=False) + @auto_docstring + def forward( + self, pixel_values: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> tuple | BaseModelOutputWithPooling: """ - Args: - pixel_values (`torch.Tensor` of shape `(total_patches, num_channels * patch_size * patch_size)`): - Packed pixel values. - grid_thw (`torch.Tensor` of shape `(num_images, 3)`): - The temporal, height and width of feature shape of each image. + pixel_values (`torch.Tensor` of shape `(total_patches, num_channels * patch_size * patch_size)`): + Packed pixel values. + grid_thw (`torch.Tensor` of shape `(num_images, 3)`): + The temporal, height and width of feature shape of each image. Returns: `torch.Tensor` of shape `(total_patches, hidden_size)`: Hidden states. @@ -609,7 +624,8 @@ def forward(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor, **kwargs) hidden_states, cu_seqlens=cu_seqlens, ) - return hidden_states + + return BaseModelOutputWithPooling(last_hidden_state=hidden_states) class GlmImageTextModel(Glm4vTextModel): @@ -880,13 +896,35 @@ def get_image_tokens( grid_t, grid_h, grid_w = image_grid_thw[i].tolist() hs = hs.view(grid_t, grid_h, grid_w, hidden_size) hs = hs.permute(0, 3, 1, 2).contiguous() - _, _, image_toks = self.vqmodel.encode(hs) - all_image_toks.append(image_toks) + vqmodel_outputs: GlmImageVQVAEModelOutput = self.vqmodel.encode(hs) + all_image_toks.append(vqmodel_outputs.image_tokens) return torch.cat(all_image_toks, dim=0) def get_video_features(self): raise AttributeError("Not needed for GlmImage") + @can_return_tuple + @auto_docstring + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: torch.LongTensor | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ + pixel_values = pixel_values.type(self.visual.dtype) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) + split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() + image_embeds = torch.split(vision_outputs.last_hidden_state, split_sizes) + vision_outputs.pooler_output = image_embeds + + return vision_outputs + def get_placeholder_mask( self, input_ids: torch.LongTensor, @@ -941,7 +979,7 @@ def forward( raise ValueError("You must specify exactly one of input_ids or inputs_embeds") if pixel_values is not None: - image_embeds = self.get_image_features(pixel_values, image_grid_thw[:-1]) + image_embeds = self.get_image_features(pixel_values, image_grid_thw[:-1], return_dict=True).pooler_output image_embeds = torch.cat(image_embeds, dim=0) image_ids = self.get_image_tokens(image_embeds, image_grid_thw[:-1]) image_ids = image_ids.view(-1).to(input_ids.device) @@ -1021,8 +1059,20 @@ def __init__(self, config): # Initialize weights and apply final processing self.post_init() - def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None): - return self.model.get_image_features(pixel_values, image_grid_thw) + @auto_docstring + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: torch.LongTensor | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ + return self.model.get_image_features(pixel_values, image_grid_thw, **kwargs) def get_image_tokens(self, hidden_states: torch.FloatTensor, image_grid_thw: torch.LongTensor | None = None): return self.model.get_image_tokens(hidden_states, image_grid_thw) diff --git a/tests/models/glm_image/test_modeling_glm_image.py b/tests/models/glm_image/test_modeling_glm_image.py index 5aeb255d25fd..986010507ca2 100644 --- a/tests/models/glm_image/test_modeling_glm_image.py +++ b/tests/models/glm_image/test_modeling_glm_image.py @@ -90,7 +90,6 @@ def __init__( "depth": 2, "hidden_act": "gelu", "hidden_size": 32, - "out_hidden_size": 16, "intermediate_size": 22, "patch_size": 16, "spatial_merge_size": 1, @@ -341,6 +340,13 @@ def test_retain_grad_hidden_states_attentions(self): def test_generate_compile_model_forward_fullgraph(self): pass + def _image_features_prepare_config_and_inputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + inputs_dict["image_grid_thw"] = inputs_dict["image_grid_thw"][:-1] + del inputs_dict["input_ids"] + del inputs_dict["attention_mask"] + return config, inputs_dict + @require_torch @slow From f7100d3a6a9a89d7d1bc7280ff7621be47f9200d Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 14 Jan 2026 18:57:32 +0100 Subject: [PATCH 088/131] Update modular files as per check_docstring rules ... ... to avoid modular/check_docstring conflicts. Modular would propargate changes from modular to modeling files, and then check_docstring would complain and update the modeling files only. This created an unstable state where one of the two scripts was unhappy. I resolved this by manually tracking down the check_docstring issues in the modular files. --- src/transformers/models/aria/modeling_aria.py | 7 ------- .../models/aya_vision/modeling_aya_vision.py | 7 ------- .../models/aya_vision/modular_aya_vision.py | 11 ---------- .../cohere2_vision/modular_cohere2_vision.py | 4 ---- src/transformers/models/emu3/modular_emu3.py | 2 +- .../models/gemma3/modular_gemma3.py | 4 ---- .../models/gemma3n/modular_gemma3n.py | 4 ---- .../models/glm4v/modular_glm4v.py | 13 ++++++------ .../models/glm4v_moe/modeling_glm4v_moe.py | 2 +- .../models/glm_image/modeling_glm_image.py | 2 +- .../models/glm_image/modular_glm_image.py | 2 +- .../models/got_ocr2/modular_got_ocr2.py | 3 --- .../models/llava/modeling_llava.py | 7 ------- .../modeling_llava_onevision.py | 16 +-------------- .../modular_llava_onevision.py | 20 +------------------ .../models/mistral3/modeling_mistral3.py | 6 ------ .../models/mistral3/modular_mistral3.py | 10 ---------- .../models/ovis2/modeling_ovis2.py | 7 ------- .../models/ovis2/modular_ovis2.py | 2 +- .../perception_lm/modular_perception_lm.py | 4 ---- .../qwen3_omni_moe/modular_qwen3_omni_moe.py | 2 +- .../models/qwen3_vl/modular_qwen3_vl.py | 2 +- src/transformers/models/sam2/modular_sam2.py | 12 +++++------ 23 files changed, 21 insertions(+), 128 deletions(-) diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index 070440e5f9cd..ccf2998ba2d5 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -926,13 +926,6 @@ def get_image_features( vision_feature_layer: int = -1, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - r""" - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) diff --git a/src/transformers/models/aya_vision/modeling_aya_vision.py b/src/transformers/models/aya_vision/modeling_aya_vision.py index 715446dfedbd..7a044003503b 100644 --- a/src/transformers/models/aya_vision/modeling_aya_vision.py +++ b/src/transformers/models/aya_vision/modeling_aya_vision.py @@ -190,13 +190,6 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) diff --git a/src/transformers/models/aya_vision/modular_aya_vision.py b/src/transformers/models/aya_vision/modular_aya_vision.py index c6dfdada6b4f..7b8b386132cd 100644 --- a/src/transformers/models/aya_vision/modular_aya_vision.py +++ b/src/transformers/models/aya_vision/modular_aya_vision.py @@ -115,17 +115,6 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) diff --git a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py index 7e6bc1527645..558d4fe39d7f 100644 --- a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py @@ -103,10 +103,6 @@ class Cohere2VisionModel(AyaVisionModel): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) - The tensors corresponding to the input images. - """ image_outputs = self.vision_tower(pixel_values, **kwargs) selected_image_feature = image_outputs.last_hidden_state image_outputs.pooler_output = self.multi_modal_projector(selected_image_feature) diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index 6b2614d50b80..2b8da342ae7d 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -957,7 +957,7 @@ def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.L def get_image_features( self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | Emu3VQVAEModelOutput: - """ + r""" pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): The tensors corresponding to the input images. """ diff --git a/src/transformers/models/gemma3/modular_gemma3.py b/src/transformers/models/gemma3/modular_gemma3.py index 267d642896ab..705d5917052f 100644 --- a/src/transformers/models/gemma3/modular_gemma3.py +++ b/src/transformers/models/gemma3/modular_gemma3.py @@ -803,10 +803,6 @@ def __init__(self, config: Gemma3Config): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - """ vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) last_hidden_state = vision_outputs.last_hidden_state vision_outputs.pooler_output = self.multi_modal_projector(last_hidden_state) diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index 6e9165536bac..ccc9938f8bff 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -2235,10 +2235,6 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - """ vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, **kwargs) last_hidden_state = vision_outputs.last_hidden_state # Convert from (batch, channels, height, width) to (batch, height * width, channels) where: diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 2aa692f784d1..06282673cd77 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -780,13 +780,12 @@ def rot_pos_emb(self, grid_thw): @auto_docstring def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] - ) -> torch.Tensor: - """ - Args: - hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): - The final hidden states of the model. - grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): - The temporal, height and width of feature shape of each image in LLM. + ) -> tuple | BaseModelOutputWithPooling: + r""" + hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): + The final hidden states of the model. + grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`): + The temporal, height and width of feature shape of each image in LLM. Returns: `torch.Tensor`: hidden_states. diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index d020b8bb3212..b8c0109cb494 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -832,7 +832,7 @@ def rot_pos_emb(self, grid_thw): @auto_docstring def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] - ) -> torch.Tensor: + ) -> tuple | BaseModelOutputWithPooling: r""" hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): The final hidden states of the model. diff --git a/src/transformers/models/glm_image/modeling_glm_image.py b/src/transformers/models/glm_image/modeling_glm_image.py index 40e0370ec926..bbfa90c5d9e8 100644 --- a/src/transformers/models/glm_image/modeling_glm_image.py +++ b/src/transformers/models/glm_image/modeling_glm_image.py @@ -721,7 +721,7 @@ def rot_pos_emb(self, grid_thw): def forward( self, pixel_values: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ + r""" pixel_values (`torch.Tensor` of shape `(total_patches, num_channels * patch_size * patch_size)`): Packed pixel values. grid_thw (`torch.Tensor` of shape `(num_images, 3)`): diff --git a/src/transformers/models/glm_image/modular_glm_image.py b/src/transformers/models/glm_image/modular_glm_image.py index cbd41bc29d20..1787a1745a6d 100644 --- a/src/transformers/models/glm_image/modular_glm_image.py +++ b/src/transformers/models/glm_image/modular_glm_image.py @@ -591,7 +591,7 @@ def rot_pos_emb(self, grid_thw): def forward( self, pixel_values: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - """ + r""" pixel_values (`torch.Tensor` of shape `(total_patches, num_channels * patch_size * patch_size)`): Packed pixel values. grid_thw (`torch.Tensor` of shape `(num_images, 3)`): diff --git a/src/transformers/models/got_ocr2/modular_got_ocr2.py b/src/transformers/models/got_ocr2/modular_got_ocr2.py index 759672a62afe..3e78f75e6776 100644 --- a/src/transformers/models/got_ocr2/modular_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modular_got_ocr2.py @@ -313,9 +313,6 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - """ - pixel_values (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`) - """ image_outputs = self.vision_tower(pixel_values, **kwargs) last_hidden_state = image_outputs.last_hidden_state image_outputs.pooler_output = self.multi_modal_projector(last_hidden_state) diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 53e2c2d19b28..afbb060b0853 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -156,13 +156,6 @@ def get_image_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index b2ac6e374172..c26ed543c092 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -368,13 +368,8 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`): - The tensors corresponding to the input images. image_sizes (`torch.Tensor` of shape `(num_images, 2)`): Actual image size of each images (H, W). - vision_feature_select_strategy (`str`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): Aspect ratio used when processing image features. The default value is "anyres_max_9". batch_num_images (`torch.LongTensor`, *optional*): @@ -713,17 +708,8 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) - The tensors corresponding to the input images. - image_sizes (`torch.Tensor` of shape `(num_images, 2)`) + image_sizes (`torch.Tensor` of shape `(num_images, 2)`): Actual image size of each images (H, W). - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): Aspect ratio used when processing image features. The default value is "anyres_max_9". batch_num_images (`torch.LongTensor`, *optional*): diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index 122bc899d728..64acc242c052 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -328,17 +328,8 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`): - The tensors corresponding to the input images. image_sizes (`torch.Tensor` of shape `(num_images, 2)`): Actual image size of each images (H, W). - vision_feature_layer (`Union[int, list[int]]`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): Aspect ratio used when processing image features. The default value is "anyres_max_9". batch_num_images (`torch.LongTensor`, *optional*): @@ -753,17 +744,8 @@ def get_image_features( **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_patches, channels, height, width)`) - The tensors corresponding to the input images. - image_sizes (`torch.Tensor` of shape `(num_images, 2)`) + image_sizes (`torch.Tensor` of shape `(num_images, 2)`): Actual image size of each images (H, W). - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): Aspect ratio used when processing image features. The default value is "anyres_max_9". batch_num_images (`torch.LongTensor`, *optional*): diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index edd5cbaee7e5..3ddab22cc2bd 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -223,12 +223,6 @@ def get_image_features( vision_feature_layer: int | list[int] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - image_sizes (`torch.Tensor`, *optional*): - Tensor containing the image sizes as returned by the processor. - """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) diff --git a/src/transformers/models/mistral3/modular_mistral3.py b/src/transformers/models/mistral3/modular_mistral3.py index 9635a9692b23..2ad3c8ccc9a5 100644 --- a/src/transformers/models/mistral3/modular_mistral3.py +++ b/src/transformers/models/mistral3/modular_mistral3.py @@ -128,16 +128,6 @@ def get_image_features( vision_feature_layer: int | list[int] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - image_sizes (`torch.Tensor`, *optional*): - Tensor containing the image sizes as returned by the processor. - """ vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index 63f7f40afed0..20814d7e9534 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -494,13 +494,6 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures: - r""" - vision_feature_select_strategy (`str`, *optional*): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): - The tensors corresponding to the input images. - """ image_outputs = self.vision_tower(pixel_values, **kwargs) image_features = image_outputs.pooler_output batch_size, img_seq_len, _ = image_features.shape diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index 9e424228a4aa..883d15960439 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -48,7 +48,7 @@ def hard_softmax(logits: torch.Tensor, dim: int): @dataclass @auto_docstring class BaseModelOutputWithVisualIndicatorFeatures(BaseModelOutputWithPooling): - """ + r""" visual_indicator_features (`torch.FloatTensor` of shape `(batch_size, visual_indicator_size)`): Visual indicator features extracted from the model, which can be used for auxiliary tasks or further processing. """ diff --git a/src/transformers/models/perception_lm/modular_perception_lm.py b/src/transformers/models/perception_lm/modular_perception_lm.py index 49d1d3629812..9c23970e93e9 100644 --- a/src/transformers/models/perception_lm/modular_perception_lm.py +++ b/src/transformers/models/perception_lm/modular_perception_lm.py @@ -155,10 +155,6 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - r""" - pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_tiles, channels, height, width)`) - The tensors corresponding to the input images. - """ image_outputs = self.vision_tower(pixel_values.flatten(0, 1), **kwargs) last_hidden_state = image_outputs.last_hidden_state if self.config.vision_use_cls_token: diff --git a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py index 40dea0e45385..fe95e08a4e7c 100644 --- a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py @@ -103,7 +103,7 @@ @dataclass @auto_docstring class BaseModelOutputWithDeepstackFeatures(BaseModelOutputWithPooling): - """ + r""" deepstack_features (`List[torch.FloatTensor]`, *optional*): List of hidden-states (feature maps) from deepstack layers. """ diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index 07e1f2cfd43a..2802667389de 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -69,7 +69,7 @@ @dataclass @auto_docstring class BaseModelOutputWithDeepstackFeatures(BaseModelOutputWithPooling): - """ + r""" deepstack_features (`List[torch.FloatTensor]`, *optional*): List of hidden-states (feature maps) from deepstack layers. """ diff --git a/src/transformers/models/sam2/modular_sam2.py b/src/transformers/models/sam2/modular_sam2.py index 55eb4478eb70..3676f824eda8 100644 --- a/src/transformers/models/sam2/modular_sam2.py +++ b/src/transformers/models/sam2/modular_sam2.py @@ -302,12 +302,6 @@ class Sam2VisionEncoderOutput(BaseModelOutputWithPooling): r""" last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. - fpn_hidden_states (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. - fpn_position_encoding (`tuple(torch.FloatTensor)`): - Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape - `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each stage) of shape `(batch_size, height, width, hidden_size)`. Hidden-states of the @@ -316,6 +310,12 @@ class Sam2VisionEncoderOutput(BaseModelOutputWithPooling): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + fpn_hidden_states (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck. + fpn_position_encoding (`tuple(torch.FloatTensor)`): + Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape + `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`. """ fpn_hidden_states: torch.FloatTensor | None = None From a41491fce14b476d44bdc76acaf5b8bd1dadba4a Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 14 Jan 2026 19:35:54 +0100 Subject: [PATCH 089/131] Update glm-image dates via fix-repo --- docs/source/en/model_doc/glm4_moe_lite.md | 2 +- docs/source/en/model_doc/glm_image.md | 2 +- docs/source/en/model_doc/lw_detr.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/en/model_doc/glm4_moe_lite.md b/docs/source/en/model_doc/glm4_moe_lite.md index 0a343d9a8ac1..472970475b3c 100644 --- a/docs/source/en/model_doc/glm4_moe_lite.md +++ b/docs/source/en/model_doc/glm4_moe_lite.md @@ -16,7 +16,7 @@ limitations under the License. ⚠️ Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be rendered properly in your Markdown viewer. --> -*This model was released on {release_date} and added to Hugging Face Transformers on 2025-12-24.* +*This model was released on {release_date} and added to Hugging Face Transformers on 2026-01-13.* # y diff --git a/docs/source/en/model_doc/glm_image.md b/docs/source/en/model_doc/glm_image.md index 0e16b245c25b..2d08b6b7db1b 100644 --- a/docs/source/en/model_doc/glm_image.md +++ b/docs/source/en/model_doc/glm_image.md @@ -16,7 +16,7 @@ limitations under the License. ⚠️ Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be rendered properly in your Markdown viewer. --> -*This model was released on 2026-01-10 and added to Hugging Face Transformers on 2026-01-10.* +*This model was released on 2026-01-10 and added to Hugging Face Transformers on 2026-01-13.* # GlmImage diff --git a/docs/source/en/model_doc/lw_detr.md b/docs/source/en/model_doc/lw_detr.md index 7d31bd43c647..ee0a20051c74 100644 --- a/docs/source/en/model_doc/lw_detr.md +++ b/docs/source/en/model_doc/lw_detr.md @@ -13,7 +13,7 @@ specific language governing permissions and limitations under the License. rendered properly in your Markdown viewer. --> -*This model was released on 2024-04-05 and added to Hugging Face Transformers on 2026-01-10.* +*This model was released on 2024-04-05 and added to Hugging Face Transformers on 2026-01-12.*
From de56122695e8157fb52953aa3c4538e97eedc425 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 15 Jan 2026 13:10:31 +0100 Subject: [PATCH 090/131] FloatTensor -> LongTensor for image_tokens --- src/transformers/models/emu3/modeling_emu3.py | 6 +++--- src/transformers/models/emu3/modular_emu3.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index 86beb47c7e5b..a3b8fb1fe2f1 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -49,11 +49,11 @@ @auto_docstring class Emu3VQVAEModelOutput(BaseModelOutputWithPooling): r""" - image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`): + image_tokens (`torch.LongTensor` of shape `(batch_size, config.vocab_size`): Indices of the image tokens predicted by the VQ-VAE model. """ - image_tokens: torch.FloatTensor | None = None + image_tokens: torch.LongTensor | None = None def rotate_half(x): @@ -1377,7 +1377,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.text_model.set_input_embeddings(value) - def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor): + def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor) -> torch.LongTensor: """ Tokenizes images into discrete tokens with VQGAN module. Converts obtained image tokens into BPE tokens and wraps with "boi" and "eoi" diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index 2b8da342ae7d..420f4419dab6 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -45,11 +45,11 @@ @auto_docstring class Emu3VQVAEModelOutput(BaseModelOutputWithPooling): r""" - image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`): + image_tokens (`torch.LongTensor` of shape `(batch_size, config.vocab_size`): Indices of the image tokens predicted by the VQ-VAE model. """ - image_tokens: torch.FloatTensor | None = None + image_tokens: torch.LongTensor | None = None class Emu3Attention(LlamaAttention): @@ -931,7 +931,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.text_model.set_input_embeddings(value) - def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor): + def get_image_tokens(self, pixel_values: torch.FloatTensor, image_sizes: torch.LongTensor) -> torch.LongTensor: """ Tokenizes images into discrete tokens with VQGAN module. Converts obtained image tokens into BPE tokens and wraps with "boi" and "eoi" From d6fd917402dab7788af234f73ca81d282451bb42 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 15 Jan 2026 13:18:43 +0100 Subject: [PATCH 091/131] Add simple last_hidden_state description, fix output typing of Gemma3nAudioEncoder.forward --- src/transformers/models/gemma3n/modeling_gemma3n.py | 3 ++- src/transformers/models/gemma3n/modular_gemma3n.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index defde272cb39..c6cf465d0e85 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -51,6 +51,7 @@ class Gemma3nAudioEncoderModelOutput(BaseModelOutputWithPooling): Args: last_hidden_state (`torch.FloatTensor` of shape `(batch_size, audio_soft_tokens_per_image, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the audio encoder. pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): Last layer hidden-state of the first token of the sequence (classification token) after further processing through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns @@ -961,7 +962,7 @@ def __init__(self, config: Gemma3nAudioConfig): @check_model_inputs(tie_last_hidden_states=False) def forward( self, audio_mel: torch.Tensor, audio_mel_mask: torch.BoolTensor, **kwargs: Unpack[TransformersKwargs] - ) -> tuple[torch.Tensor, torch.BoolTensor]: + ) -> tuple | Gemma3nAudioEncoderModelOutput: """Encodes a batch of MELs. Args: diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index ccc9938f8bff..47393bf5f2a2 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -630,6 +630,7 @@ class Gemma3nAudioEncoderModelOutput(BaseModelOutputWithPooling): Args: last_hidden_state (`torch.FloatTensor` of shape `(batch_size, audio_soft_tokens_per_image, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the audio encoder. pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): Last layer hidden-state of the first token of the sequence (classification token) after further processing through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns @@ -1516,7 +1517,7 @@ def __init__(self, config: Gemma3nAudioConfig): @check_model_inputs(tie_last_hidden_states=False) def forward( self, audio_mel: torch.Tensor, audio_mel_mask: torch.BoolTensor, **kwargs: Unpack[TransformersKwargs] - ) -> tuple[torch.Tensor, torch.BoolTensor]: + ) -> tuple | Gemma3nAudioEncoderModelOutput: """Encodes a batch of MELs. Args: From 7329ebc4d77f57bfc7b81b710e690559fd174a43 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 15 Jan 2026 13:31:17 +0100 Subject: [PATCH 092/131] Add missing `-> tuple | BaseModel...` on check_model_inputs Using ``check_model_inputs[^\n]*\n\s*def forward\([^\)]*\):`` --- .../models/audioflamingo3/modeling_audioflamingo3.py | 2 +- .../models/audioflamingo3/modular_audioflamingo3.py | 2 +- src/transformers/models/sam3/modeling_sam3.py | 2 +- .../models/switch_transformers/modeling_switch_transformers.py | 2 +- .../models/switch_transformers/modular_switch_transformers.py | 2 +- src/transformers/models/voxtral/modeling_voxtral.py | 2 +- src/transformers/models/voxtral/modular_voxtral.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py index d8a42560b249..645f38f1e3eb 100644 --- a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py @@ -328,7 +328,7 @@ def forward( input_features: torch.Tensor, input_features_mask: torch.Tensor | None = None, **kwargs, - ): + ) -> tuple | BaseModelOutputWithPooling: r""" Args: input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`): diff --git a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py index 173034a3c6e8..2ba8b754dd0c 100644 --- a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py @@ -68,7 +68,7 @@ def forward( input_features: torch.Tensor, input_features_mask: torch.Tensor | None = None, **kwargs, - ): + ) -> tuple | BaseModelOutputWithPooling: r""" Args: input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`): diff --git a/src/transformers/models/sam3/modeling_sam3.py b/src/transformers/models/sam3/modeling_sam3.py index 5695ab332f69..90cf4996829f 100644 --- a/src/transformers/models/sam3/modeling_sam3.py +++ b/src/transformers/models/sam3/modeling_sam3.py @@ -1399,7 +1399,7 @@ def forward( text_mask: torch.Tensor | None = None, spatial_sizes: list[tuple[int, int]] | None = None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> tuple | Sam3DETREncoderOutput: """ Forward pass for the DETR encoder. diff --git a/src/transformers/models/switch_transformers/modeling_switch_transformers.py b/src/transformers/models/switch_transformers/modeling_switch_transformers.py index 96a059091bca..0a80458410ad 100644 --- a/src/transformers/models/switch_transformers/modeling_switch_transformers.py +++ b/src/transformers/models/switch_transformers/modeling_switch_transformers.py @@ -691,7 +691,7 @@ def forward( use_cache=None, cache_position=None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> tuple | MoEModelOutputWithPastAndCrossAttentions: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/switch_transformers/modular_switch_transformers.py b/src/transformers/models/switch_transformers/modular_switch_transformers.py index fac7e1820c0b..03efb6b35eeb 100644 --- a/src/transformers/models/switch_transformers/modular_switch_transformers.py +++ b/src/transformers/models/switch_transformers/modular_switch_transformers.py @@ -447,7 +447,7 @@ def forward( use_cache=None, cache_position=None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> tuple | MoEModelOutputWithPastAndCrossAttentions: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/voxtral/modeling_voxtral.py b/src/transformers/models/voxtral/modeling_voxtral.py index 486cef4ca5c9..d7a69fc83cdb 100644 --- a/src/transformers/models/voxtral/modeling_voxtral.py +++ b/src/transformers/models/voxtral/modeling_voxtral.py @@ -297,7 +297,7 @@ def forward( input_features, attention_mask=None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> tuple | BaseModelOutputWithPooling: r""" Args: input_features (`torch.LongTensor` of shape `(batch_size, feature_size, sequence_length)`): diff --git a/src/transformers/models/voxtral/modular_voxtral.py b/src/transformers/models/voxtral/modular_voxtral.py index 1698e993b28a..50a687b0ecca 100644 --- a/src/transformers/models/voxtral/modular_voxtral.py +++ b/src/transformers/models/voxtral/modular_voxtral.py @@ -71,7 +71,7 @@ def forward( input_features, attention_mask=None, **kwargs: Unpack[TransformersKwargs], - ): + ) -> tuple | BaseModelOutputWithPooling: r""" Args: input_features (`torch.LongTensor` of shape `(batch_size, feature_size, sequence_length)`): From 72a9ac95e1fc0813f7b6c1ca789db9f56295272b Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 15 Jan 2026 13:41:04 +0100 Subject: [PATCH 093/131] Ensure forward typing with check_model_inputs is `-> tuple | BaseModel...` Using ``check_model_inputs[^\n]*\n\s*def forward\([^\)]+\) -> (?!tuple | )`` --- src/transformers/models/afmoe/modeling_afmoe.py | 2 +- src/transformers/models/afmoe/modular_afmoe.py | 2 +- src/transformers/models/blt/modeling_blt.py | 2 +- src/transformers/models/blt/modular_blt.py | 2 +- src/transformers/models/florence2/modeling_florence2.py | 4 +++- src/transformers/models/florence2/modular_florence2.py | 4 +++- src/transformers/models/glm4v_moe/modeling_glm4v_moe.py | 2 +- src/transformers/models/glm4v_moe/modular_glm4v_moe.py | 2 +- src/transformers/models/got_ocr2/modeling_got_ocr2.py | 2 +- .../models/granite_speech/modeling_granite_speech.py | 4 +++- src/transformers/models/minimax/modeling_minimax.py | 2 +- src/transformers/models/minimax/modular_minimax.py | 2 +- src/transformers/models/moonshine/modeling_moonshine.py | 2 +- src/transformers/models/moonshine/modular_moonshine.py | 2 +- src/transformers/models/ovis2/modeling_ovis2.py | 2 +- src/transformers/models/ovis2/modular_ovis2.py | 2 +- src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py | 2 +- src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py | 2 +- src/transformers/models/pe_audio/modeling_pe_audio.py | 2 +- src/transformers/models/pe_audio/modular_pe_audio.py | 2 +- .../models/pe_audio_video/modeling_pe_audio_video.py | 2 +- .../models/pe_audio_video/modular_pe_audio_video.py | 2 +- src/transformers/models/pe_video/modeling_pe_video.py | 2 +- src/transformers/models/pe_video/modular_pe_video.py | 2 +- .../models/phi4_multimodal/modeling_phi4_multimodal.py | 4 ++-- .../models/phi4_multimodal/modular_phi4_multimodal.py | 4 ++-- src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py | 2 +- src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py | 2 +- src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py | 2 +- src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py | 2 +- .../models/qwen3_omni_moe/modeling_qwen3_omni_moe.py | 2 +- src/transformers/models/qwen3_vl/modeling_qwen3_vl.py | 2 +- src/transformers/models/qwen3_vl/modular_qwen3_vl.py | 2 +- src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py | 2 +- src/transformers/models/sam/modeling_sam.py | 2 +- src/transformers/models/sam3/modeling_sam3.py | 4 ++-- src/transformers/models/starcoder2/modeling_starcoder2.py | 2 +- src/transformers/models/starcoder2/modular_starcoder2.py | 2 +- src/transformers/models/t5gemma/modeling_t5gemma.py | 4 ++-- src/transformers/models/t5gemma/modular_t5gemma.py | 4 ++-- 40 files changed, 51 insertions(+), 45 deletions(-) diff --git a/src/transformers/models/afmoe/modeling_afmoe.py b/src/transformers/models/afmoe/modeling_afmoe.py index 9330e00c996b..8c061ada1eff 100644 --- a/src/transformers/models/afmoe/modeling_afmoe.py +++ b/src/transformers/models/afmoe/modeling_afmoe.py @@ -571,7 +571,7 @@ def forward( cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> MoeModelOutputWithPast: + ) -> tuple | MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/afmoe/modular_afmoe.py b/src/transformers/models/afmoe/modular_afmoe.py index 904f7dc69b2d..edd44f1df87d 100644 --- a/src/transformers/models/afmoe/modular_afmoe.py +++ b/src/transformers/models/afmoe/modular_afmoe.py @@ -392,7 +392,7 @@ def forward( cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> MoeModelOutputWithPast: + ) -> tuple | MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/blt/modeling_blt.py b/src/transformers/models/blt/modeling_blt.py index d1812a825538..0dea1ef44a67 100644 --- a/src/transformers/models/blt/modeling_blt.py +++ b/src/transformers/models/blt/modeling_blt.py @@ -1226,7 +1226,7 @@ def forward( use_cache: bool | None = None, cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPast: + ) -> tuple | BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/blt/modular_blt.py b/src/transformers/models/blt/modular_blt.py index 421c2e286c40..e27a24ee59d7 100644 --- a/src/transformers/models/blt/modular_blt.py +++ b/src/transformers/models/blt/modular_blt.py @@ -950,7 +950,7 @@ def forward( use_cache: bool | None = None, cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPast: + ) -> tuple | BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/florence2/modeling_florence2.py b/src/transformers/models/florence2/modeling_florence2.py index 08fa8e40459a..88818f106fb2 100644 --- a/src/transformers/models/florence2/modeling_florence2.py +++ b/src/transformers/models/florence2/modeling_florence2.py @@ -544,7 +544,9 @@ def __init__(self, config: Florence2VisionConfig): self.post_init() @check_model_inputs(tie_last_hidden_states=False) - def forward(self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> BaseModelOutputWithPooling: + def forward( + self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> tuple | BaseModelOutputWithPooling: for conv, block in zip(self.convs, self.blocks): hidden_states = conv(hidden_states) for layer in block: diff --git a/src/transformers/models/florence2/modular_florence2.py b/src/transformers/models/florence2/modular_florence2.py index 870dc220312f..195618f9cb76 100644 --- a/src/transformers/models/florence2/modular_florence2.py +++ b/src/transformers/models/florence2/modular_florence2.py @@ -1397,7 +1397,9 @@ def __init__(self, config: Florence2VisionConfig): self.post_init() @check_model_inputs(tie_last_hidden_states=False) - def forward(self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> BaseModelOutputWithPooling: + def forward( + self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> tuple | BaseModelOutputWithPooling: for conv, block in zip(self.convs, self.blocks): hidden_states = conv(hidden_states) for layer in block: diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index b8c0109cb494..b769eac5896d 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1000,7 +1000,7 @@ def forward( use_cache: bool | None = None, cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], - ) -> MoeModelOutputWithPast: + ) -> tuple | MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/glm4v_moe/modular_glm4v_moe.py b/src/transformers/models/glm4v_moe/modular_glm4v_moe.py index 5e81dc84c6c9..64a6957f20ee 100644 --- a/src/transformers/models/glm4v_moe/modular_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modular_glm4v_moe.py @@ -404,7 +404,7 @@ def forward( use_cache: bool | None = None, cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], - ) -> MoeModelOutputWithPast: + ) -> tuple | MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/got_ocr2/modeling_got_ocr2.py b/src/transformers/models/got_ocr2/modeling_got_ocr2.py index 8f2103fa82cb..2d8cc7c06046 100644 --- a/src/transformers/models/got_ocr2/modeling_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modeling_got_ocr2.py @@ -438,7 +438,7 @@ def get_input_embeddings(self): @check_model_inputs(tie_last_hidden_states=False) def forward( self, pixel_values: torch.FloatTensor | None = None, **kwargs: Unpack[TransformersKwargs] - ) -> GotOcr2VisionEncoderOutput: + ) -> tuple | GotOcr2VisionEncoderOutput: if pixel_values is None: raise ValueError("You have to specify pixel_values") diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index 887b3a1ee3ad..42da9f03e2ef 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -297,7 +297,9 @@ def __init__(self, config: GraniteSpeechEncoderConfig): self.post_init() @check_model_inputs(tie_last_hidden_states=False) - def forward(self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs]) -> BaseModelOutputWithPooling: + def forward( + self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> tuple | BaseModelOutputWithPooling: hidden_states = self.input_linear(hidden_states) for idx, layer in enumerate(self.layers, start=1): hidden_states = layer(hidden_states, attention_dists=self.attention_dists) diff --git a/src/transformers/models/minimax/modeling_minimax.py b/src/transformers/models/minimax/modeling_minimax.py index 841c81694c8c..a2dff7e9401b 100644 --- a/src/transformers/models/minimax/modeling_minimax.py +++ b/src/transformers/models/minimax/modeling_minimax.py @@ -654,7 +654,7 @@ def forward( use_cache: bool | None = None, cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> MoeModelOutputWithPast: + ) -> tuple | MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/minimax/modular_minimax.py b/src/transformers/models/minimax/modular_minimax.py index 46125e91d902..5dd9a59c182e 100644 --- a/src/transformers/models/minimax/modular_minimax.py +++ b/src/transformers/models/minimax/modular_minimax.py @@ -548,7 +548,7 @@ def forward( use_cache: bool | None = None, cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> MoeModelOutputWithPast: + ) -> tuple | MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/moonshine/modeling_moonshine.py b/src/transformers/models/moonshine/modeling_moonshine.py index cc5f7a9b6c9d..4058e0547ac3 100644 --- a/src/transformers/models/moonshine/modeling_moonshine.py +++ b/src/transformers/models/moonshine/modeling_moonshine.py @@ -556,7 +556,7 @@ def forward( input_values: torch.FloatTensor, attention_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPast: + ) -> tuple | BaseModelOutputWithPast: r""" Args: input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`): diff --git a/src/transformers/models/moonshine/modular_moonshine.py b/src/transformers/models/moonshine/modular_moonshine.py index 14c82bb24ffe..7d7b84245c71 100644 --- a/src/transformers/models/moonshine/modular_moonshine.py +++ b/src/transformers/models/moonshine/modular_moonshine.py @@ -513,7 +513,7 @@ def forward( input_values: torch.FloatTensor, attention_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPast: + ) -> tuple | BaseModelOutputWithPast: r""" Args: input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`): diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index 20814d7e9534..97ee55603ce8 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -421,7 +421,7 @@ def __init__(self, config: Ovis2VisionConfig): @check_model_inputs(tie_last_hidden_states=False) def forward( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] - ) -> tuple[torch.Tensor, torch.Tensor]: + ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures: outputs = self.transformer(pixel_values, **kwargs) last_hidden_state = outputs[0] if self.config.hidden_stride > 1: diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index 883d15960439..83f5b596b328 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -202,7 +202,7 @@ def __init__(self, config: Ovis2VisionConfig): @check_model_inputs(tie_last_hidden_states=False) def forward( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] - ) -> tuple[torch.Tensor, torch.Tensor]: + ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures: outputs = self.transformer(pixel_values, **kwargs) last_hidden_state = outputs[0] if self.config.hidden_stride > 1: diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index 6fe1c5355b5b..15702772930f 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -958,7 +958,7 @@ def forward( cu_seqlens: torch.Tensor, image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]] | None = None, **kwargs, - ) -> BaseModelOutputWithPooling: + ) -> tuple | BaseModelOutputWithPooling: """ Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, sequence_length, image_channels, patch_size, patch_size)`): diff --git a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py index 628461d1a71f..e9c5c1fe85f0 100644 --- a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py @@ -1074,7 +1074,7 @@ def forward( cu_seqlens: torch.Tensor, image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]] | None = None, **kwargs, - ) -> BaseModelOutputWithPooling: + ) -> tuple | BaseModelOutputWithPooling: """ Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, sequence_length, image_channels, patch_size, patch_size)`): diff --git a/src/transformers/models/pe_audio/modeling_pe_audio.py b/src/transformers/models/pe_audio/modeling_pe_audio.py index 0daf26ea1649..948cd6e1fd16 100644 --- a/src/transformers/models/pe_audio/modeling_pe_audio.py +++ b/src/transformers/models/pe_audio/modeling_pe_audio.py @@ -642,7 +642,7 @@ def forward( input_values: torch.Tensor, padding_mask: torch.Tensor | None = None, **kwargs, - ) -> BaseModelOutputWithPooling: + ) -> tuple | BaseModelOutputWithPooling: inputs_embeds, padding_mask = self.embedder(input_values, padding_mask=padding_mask) inputs_embeds, attention_mask = self.patch_embedder(inputs_embeds, padding_mask=padding_mask) diff --git a/src/transformers/models/pe_audio/modular_pe_audio.py b/src/transformers/models/pe_audio/modular_pe_audio.py index b5e594393344..84a1ad80df85 100644 --- a/src/transformers/models/pe_audio/modular_pe_audio.py +++ b/src/transformers/models/pe_audio/modular_pe_audio.py @@ -117,7 +117,7 @@ def forward( input_values: torch.Tensor, padding_mask: torch.Tensor | None = None, **kwargs, - ) -> BaseModelOutputWithPooling: + ) -> tuple | BaseModelOutputWithPooling: inputs_embeds, padding_mask = self.embedder(input_values, padding_mask=padding_mask) inputs_embeds, attention_mask = self.patch_embedder(inputs_embeds, padding_mask=padding_mask) diff --git a/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py b/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py index bf47a4ea519d..0fb693d67941 100644 --- a/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py +++ b/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py @@ -589,7 +589,7 @@ def forward( padding_mask: torch.Tensor | None = None, padding_mask_videos: torch.Tensor | None = None, **kwargs, - ) -> PeAudioVideoEncoderOutput: + ) -> tuple | PeAudioVideoEncoderOutput: inputs_embeds, padding_mask, audio_output, video_output = self.embedder( input_values, pixel_values_videos, diff --git a/src/transformers/models/pe_audio_video/modular_pe_audio_video.py b/src/transformers/models/pe_audio_video/modular_pe_audio_video.py index 7127de028c6a..78bd0a044259 100644 --- a/src/transformers/models/pe_audio_video/modular_pe_audio_video.py +++ b/src/transformers/models/pe_audio_video/modular_pe_audio_video.py @@ -378,7 +378,7 @@ def forward( padding_mask: torch.Tensor | None = None, padding_mask_videos: torch.Tensor | None = None, **kwargs, - ) -> PeAudioVideoEncoderOutput: + ) -> tuple | PeAudioVideoEncoderOutput: inputs_embeds, padding_mask, audio_output, video_output = self.embedder( input_values, pixel_values_videos, diff --git a/src/transformers/models/pe_video/modeling_pe_video.py b/src/transformers/models/pe_video/modeling_pe_video.py index ad9165e6f939..55c7fa5d932d 100644 --- a/src/transformers/models/pe_video/modeling_pe_video.py +++ b/src/transformers/models/pe_video/modeling_pe_video.py @@ -526,7 +526,7 @@ def forward( pixel_values_videos: torch.Tensor, padding_mask_videos: torch.Tensor | None = None, **kwargs, - ) -> BaseModelOutputWithPooling: + ) -> tuple | BaseModelOutputWithPooling: inputs_embeds, padding_mask = self.embedder(pixel_values_videos, padding_mask=padding_mask_videos) inputs_embeds, attention_mask = self.patch_embedder(inputs_embeds, padding_mask=padding_mask) diff --git a/src/transformers/models/pe_video/modular_pe_video.py b/src/transformers/models/pe_video/modular_pe_video.py index 12430d1ed382..026e5b858715 100644 --- a/src/transformers/models/pe_video/modular_pe_video.py +++ b/src/transformers/models/pe_video/modular_pe_video.py @@ -107,7 +107,7 @@ def forward( pixel_values_videos: torch.Tensor, padding_mask_videos: torch.Tensor | None = None, **kwargs, - ) -> BaseModelOutputWithPooling: + ) -> tuple | BaseModelOutputWithPooling: inputs_embeds, padding_mask = self.embedder(pixel_values_videos, padding_mask=padding_mask_videos) inputs_embeds, attention_mask = self.patch_embedder(inputs_embeds, padding_mask=padding_mask) diff --git a/src/transformers/models/phi4_multimodal/modeling_phi4_multimodal.py b/src/transformers/models/phi4_multimodal/modeling_phi4_multimodal.py index 081693f811b9..b98b5c3d80ca 100644 --- a/src/transformers/models/phi4_multimodal/modeling_phi4_multimodal.py +++ b/src/transformers/models/phi4_multimodal/modeling_phi4_multimodal.py @@ -431,7 +431,7 @@ def forward( pixel_values, patch_attention_mask: torch.BoolTensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPooling: + ) -> tuple | BaseModelOutputWithPooling: batch_size = pixel_values.size(0) if patch_attention_mask is None: patch_attention_mask = torch.ones( @@ -1552,7 +1552,7 @@ def forward( output_hidden_states: bool | None = None, cache_position: torch.LongTensor | None = None, **kwargs, - ) -> BaseModelOutputWithPast: + ) -> tuple | BaseModelOutputWithPast: r""" image_pixel_values (`torch.FloatTensor`, *optional*): If the input contains images, these correspond to the pixel values after transformations (as returned by diff --git a/src/transformers/models/phi4_multimodal/modular_phi4_multimodal.py b/src/transformers/models/phi4_multimodal/modular_phi4_multimodal.py index cf6d529ae047..c42033e2fcab 100644 --- a/src/transformers/models/phi4_multimodal/modular_phi4_multimodal.py +++ b/src/transformers/models/phi4_multimodal/modular_phi4_multimodal.py @@ -672,7 +672,7 @@ def forward( pixel_values, patch_attention_mask: torch.BoolTensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPooling: + ) -> tuple | BaseModelOutputWithPooling: batch_size = pixel_values.size(0) if patch_attention_mask is None: patch_attention_mask = torch.ones( @@ -1489,7 +1489,7 @@ def forward( output_hidden_states: bool | None = None, cache_position: torch.LongTensor | None = None, **kwargs, - ) -> BaseModelOutputWithPast: + ) -> tuple | BaseModelOutputWithPast: r""" image_pixel_values (`torch.FloatTensor`, *optional*): If the input contains images, these correspond to the pixel values after transformations (as returned by diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index bf03583049aa..168cf292f1e2 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -1251,7 +1251,7 @@ def get_window_index(self, grid_thw): @check_model_inputs(tie_last_hidden_states=False) def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] - ) -> torch.Tensor: + ) -> tuple | BaseModelOutputWithPooling: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index 38f8d7b78a72..074ebad7e1e7 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -1960,7 +1960,7 @@ def __init__(self, config: Qwen2_5OmniVisionEncoderConfig, *inputs, **kwargs) -> @check_model_inputs(tie_last_hidden_states=False) def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] - ) -> torch.Tensor: + ) -> tuple | BaseModelOutputWithPooling: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index 632d9e2129ba..f311ccbb033e 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -422,7 +422,7 @@ def get_window_index(self, grid_thw): @check_model_inputs(tie_last_hidden_states=False) def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] - ) -> torch.Tensor: + ) -> tuple | BaseModelOutputWithPooling: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): diff --git a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py index e2e84e7d568c..98e43945c2b6 100644 --- a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py @@ -285,7 +285,7 @@ def get_window_index(self, grid_thw): @check_model_inputs(tie_last_hidden_states=False) def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] - ) -> torch.Tensor: + ) -> tuple | BaseModelOutputWithPooling: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index ffd23f1c460e..ce4cfab1f739 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -1239,7 +1239,7 @@ def fast_pos_embed_interpolate(self, grid_thw): @check_model_inputs(tie_last_hidden_states=False) def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] - ) -> BaseModelOutputWithDeepstackFeatures: + ) -> tuple | BaseModelOutputWithDeepstackFeatures: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index 32d0d2a05d5c..33902df8bf46 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -757,7 +757,7 @@ def fast_pos_embed_interpolate(self, grid_thw): @check_model_inputs(tie_last_hidden_states=False) def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] - ) -> BaseModelOutputWithDeepstackFeatures: + ) -> tuple | BaseModelOutputWithDeepstackFeatures: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index 2802667389de..d556bb6e9560 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -654,7 +654,7 @@ def fast_pos_embed_interpolate(self, grid_thw): @check_model_inputs(tie_last_hidden_states=False) def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] - ) -> BaseModelOutputWithDeepstackFeatures: + ) -> tuple | BaseModelOutputWithDeepstackFeatures: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index b57ba61a9b0e..c574b6f0614d 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -741,7 +741,7 @@ def fast_pos_embed_interpolate(self, grid_thw): @check_model_inputs(tie_last_hidden_states=False) def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] - ) -> BaseModelOutputWithDeepstackFeatures: + ) -> tuple | BaseModelOutputWithDeepstackFeatures: """ Args: hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`): diff --git a/src/transformers/models/sam/modeling_sam.py b/src/transformers/models/sam/modeling_sam.py index 18b5a86eb8c9..ef1181ba641a 100644 --- a/src/transformers/models/sam/modeling_sam.py +++ b/src/transformers/models/sam/modeling_sam.py @@ -1056,7 +1056,7 @@ def get_input_embeddings(self): @check_model_inputs(tie_last_hidden_states=False) def forward( self, pixel_values: torch.FloatTensor | None = None, **kwargs: Unpack[TransformersKwargs] - ) -> SamVisionEncoderOutput: + ) -> tuple | SamVisionEncoderOutput: if pixel_values is None: raise ValueError("You have to specify pixel_values") diff --git a/src/transformers/models/sam3/modeling_sam3.py b/src/transformers/models/sam3/modeling_sam3.py index 90cf4996829f..a592aab34657 100644 --- a/src/transformers/models/sam3/modeling_sam3.py +++ b/src/transformers/models/sam3/modeling_sam3.py @@ -1694,7 +1694,7 @@ def forward( text_mask: torch.Tensor | None = None, spatial_shapes: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]: + ) -> tuple | Sam3DETRDecoderOutput: """ Forward pass for the DETR decoder. @@ -2010,7 +2010,7 @@ def forward( prompt_features: torch.Tensor | None = None, prompt_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> dict[str, torch.Tensor]: + ) -> tuple | Sam3MaskDecoderOutput: """ Args: decoder_queries: Decoder output queries [batch_size, num_queries, hidden_size] diff --git a/src/transformers/models/starcoder2/modeling_starcoder2.py b/src/transformers/models/starcoder2/modeling_starcoder2.py index 100da24a3c1d..8613fef45695 100644 --- a/src/transformers/models/starcoder2/modeling_starcoder2.py +++ b/src/transformers/models/starcoder2/modeling_starcoder2.py @@ -364,7 +364,7 @@ def forward( use_cache: bool | None = None, cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPast: + ) -> tuple | BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/starcoder2/modular_starcoder2.py b/src/transformers/models/starcoder2/modular_starcoder2.py index 478593ecb60d..f6fd0841b217 100644 --- a/src/transformers/models/starcoder2/modular_starcoder2.py +++ b/src/transformers/models/starcoder2/modular_starcoder2.py @@ -153,7 +153,7 @@ def forward( use_cache: bool | None = None, cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPast: + ) -> tuple | BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/t5gemma/modeling_t5gemma.py b/src/transformers/models/t5gemma/modeling_t5gemma.py index 2fe975e71ba9..f6bc2ca10eae 100644 --- a/src/transformers/models/t5gemma/modeling_t5gemma.py +++ b/src/transformers/models/t5gemma/modeling_t5gemma.py @@ -691,7 +691,7 @@ def forward( position_ids: torch.LongTensor | None = None, inputs_embeds: torch.FloatTensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutput: + ) -> tuple | BaseModelOutput: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -790,7 +790,7 @@ def forward( encoder_hidden_states: torch.Tensor | None = None, encoder_attention_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPastAndCrossAttentions: + ) -> tuple | BaseModelOutputWithPastAndCrossAttentions: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") if encoder_hidden_states is None: diff --git a/src/transformers/models/t5gemma/modular_t5gemma.py b/src/transformers/models/t5gemma/modular_t5gemma.py index 7ebdc99e286e..419fa12bca4c 100644 --- a/src/transformers/models/t5gemma/modular_t5gemma.py +++ b/src/transformers/models/t5gemma/modular_t5gemma.py @@ -696,7 +696,7 @@ def forward( position_ids: torch.LongTensor | None = None, inputs_embeds: torch.FloatTensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutput: + ) -> tuple | BaseModelOutput: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -795,7 +795,7 @@ def forward( encoder_hidden_states: torch.Tensor | None = None, encoder_attention_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPastAndCrossAttentions: + ) -> tuple | BaseModelOutputWithPastAndCrossAttentions: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") if encoder_hidden_states is None: From 9b670147593a4b049694a94f84cfb4d2d7b5788a Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 15 Jan 2026 13:56:25 +0100 Subject: [PATCH 094/131] Undo accidental rename of Ovis2VisionAttention --- .../models/ovis2/modeling_ovis2.py | 67 ++++++++++++++++++- .../models/ovis2/modular_ovis2.py | 6 +- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index 97ee55603ce8..011989b71820 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -195,7 +195,7 @@ def eager_attention_forward( return attn_output, attn_weights -class Ovis2Attention(nn.Module): +class Ovis2VisionAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper""" def __init__(self, config): @@ -272,6 +272,67 @@ def forward(self, x): return down_proj +class Ovis2Attention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" + f" {self.num_heads})." + ) + self.scale = self.head_dim**-0.5 + self.dropout = config.attention_dropout + self.is_causal = False + self.k_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=config.qkv_bias) + self.v_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=config.qkv_bias) + self.q_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=config.qkv_bias) + self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=config.qkv_bias) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + **kwargs, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + """Input shape: Batch x Time x Channel""" + + batch_size, seq_length, embed_dim = hidden_states.shape + + queries = self.q_proj(hidden_states) + keys = self.k_proj(hidden_states) + values = self.v_proj(hidden_states) + + queries = queries.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2) + keys = keys.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2) + values = values.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + attn_output, attn_weights = attention_interface( + self, + queries, + keys, + values, + attention_mask, + is_causal=self.is_causal, + scaling=self.scale, + dropout=0.0 if not self.training else self.dropout, + ) + + attn_output = attn_output.reshape(batch_size, seq_length, embed_dim).contiguous() + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights + + class Ovis2VisionEncoderLayer(GradientCheckpointingLayer): def __init__(self, config: Ovis2VisionConfig): super().__init__() @@ -370,7 +431,7 @@ class Ovis2PreTrainedModel(PreTrainedModel): base_model_prefix = "model" input_modalities = ("image", "text") supports_gradient_checkpointing = True - _no_split_modules = ["Ovis2Attention"] + _no_split_modules = ["Ovis2VisionAttention"] _skip_keys_device_placement = "past_key_values" _supports_cache_class = True _supports_flash_attn = True @@ -400,7 +461,7 @@ class Ovis2VisionModel(Ovis2PreTrainedModel): config: Ovis2VisionConfig _can_record_outputs = { "hidden_states": Ovis2VisionEncoderLayer, - "attentions": Ovis2Attention, + "attentions": Ovis2VisionAttention, } def __init__(self, config: Ovis2VisionConfig): diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index 83f5b596b328..430dd62be44d 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -91,7 +91,7 @@ def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: return embeddings -class Ovis2Attention(Aimv2Attention): +class Ovis2VisionAttention(Aimv2Attention): pass @@ -161,7 +161,7 @@ class Ovis2PreTrainedModel(PreTrainedModel): base_model_prefix = "model" input_modalities = ("image", "text") supports_gradient_checkpointing = True - _no_split_modules = ["Ovis2Attention"] + _no_split_modules = ["Ovis2VisionAttention"] _skip_keys_device_placement = "past_key_values" _supports_cache_class = True _supports_flash_attn = True @@ -181,7 +181,7 @@ class Ovis2VisionModel(Ovis2PreTrainedModel): config: Ovis2VisionConfig _can_record_outputs = { "hidden_states": Ovis2VisionEncoderLayer, - "attentions": Ovis2Attention, + "attentions": Ovis2VisionAttention, } def __init__(self, config: Ovis2VisionConfig): From cd881792ea2e3a4206aad30351c2e294c46f7084 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 15 Jan 2026 14:04:17 +0100 Subject: [PATCH 095/131] Fix incorrect type hints for blip family --- src/transformers/models/blip_2/modeling_blip_2.py | 12 +++++++----- .../models/instructblip/modeling_instructblip.py | 12 +++++++----- .../instructblipvideo/modeling_instructblipvideo.py | 12 +++++++----- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 33892bfbde48..e26d880bb98e 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -32,6 +32,8 @@ BaseModelOutputWithPastAndCrossAttentions, BaseModelOutputWithPooling, BaseModelOutputWithPoolingAndCrossAttentions, + CausalLMOutputWithPast, + Seq2SeqLMOutput, ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -63,8 +65,8 @@ class BaseModelOutputWithVisionQformerOutputs(BaseModelOutputWithPooling): Outputs of the Q-Former (Querying Transformer). """ - vision_outputs: torch.FloatTensor | None = None - qformer_outputs: tuple[torch.FloatTensor] | None = None + vision_outputs: BaseModelOutputWithPooling | None = None + qformer_outputs: BaseModelOutputWithPoolingAndCrossAttentions | None = None @dataclass @@ -89,9 +91,9 @@ class Blip2ForConditionalGenerationModelOutput(ModelOutput): loss: tuple[torch.FloatTensor] | None = None logits: tuple[torch.FloatTensor] | None = None - vision_outputs: torch.FloatTensor | None = None - qformer_outputs: tuple[torch.FloatTensor] | None = None - language_model_outputs: tuple[torch.FloatTensor] | None = None + vision_outputs: BaseModelOutputWithPooling | None = None + qformer_outputs: BaseModelOutputWithPoolingAndCrossAttentions | None = None + language_model_outputs: CausalLMOutputWithPast | Seq2SeqLMOutput | None = None def to_tuple(self) -> tuple[Any]: return tuple( diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index b51730e5c9a3..4efd7091ec40 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -31,6 +31,8 @@ BaseModelOutputWithPastAndCrossAttentions, BaseModelOutputWithPooling, BaseModelOutputWithPoolingAndCrossAttentions, + CausalLMOutputWithPast, + Seq2SeqLMOutput, ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -54,8 +56,8 @@ class BaseModelOutputWithVisionQformerOutputs(BaseModelOutputWithPooling): Outputs of the Q-Former (Querying Transformer). """ - vision_outputs: torch.FloatTensor | None = None - qformer_outputs: tuple[torch.FloatTensor] | None = None + vision_outputs: BaseModelOutputWithPooling | None = None + qformer_outputs: BaseModelOutputWithPoolingAndCrossAttentions | None = None @dataclass @@ -81,9 +83,9 @@ class InstructBlipForConditionalGenerationModelOutput(ModelOutput): loss: tuple[torch.FloatTensor] | None = None logits: tuple[torch.FloatTensor] | None = None - vision_outputs: torch.FloatTensor | None = None - qformer_outputs: tuple[torch.FloatTensor] | None = None - language_model_outputs: tuple[torch.FloatTensor] | None = None + vision_outputs: BaseModelOutputWithPooling | None = None + qformer_outputs: BaseModelOutputWithPoolingAndCrossAttentions | None = None + language_model_outputs: CausalLMOutputWithPast | Seq2SeqLMOutput | None = None def to_tuple(self) -> tuple[Any]: return tuple( diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 730a86299715..121db617af5c 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -37,6 +37,8 @@ BaseModelOutputWithPastAndCrossAttentions, BaseModelOutputWithPooling, BaseModelOutputWithPoolingAndCrossAttentions, + CausalLMOutputWithPast, + Seq2SeqLMOutput, ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack @@ -906,9 +908,9 @@ class InstructBlipVideoForConditionalGenerationModelOutput(ModelOutput): loss: tuple[torch.FloatTensor] | None = None logits: tuple[torch.FloatTensor] | None = None - vision_outputs: torch.FloatTensor | None = None - qformer_outputs: tuple[torch.FloatTensor] | None = None - language_model_outputs: tuple[torch.FloatTensor] | None = None + vision_outputs: BaseModelOutputWithPooling | None = None + qformer_outputs: BaseModelOutputWithPoolingAndCrossAttentions | None = None + language_model_outputs: CausalLMOutputWithPast | Seq2SeqLMOutput | None = None def to_tuple(self) -> tuple[Any]: return tuple( @@ -1130,8 +1132,8 @@ class BaseModelOutputWithVisionQformerOutputs(BaseModelOutputWithPooling): Outputs of the Q-Former (Querying Transformer). """ - vision_outputs: torch.FloatTensor | None = None - qformer_outputs: tuple[torch.FloatTensor] | None = None + vision_outputs: BaseModelOutputWithPooling | None = None + qformer_outputs: BaseModelOutputWithPoolingAndCrossAttentions | None = None @auto_docstring( From e7476694c6691ead664d61cbbaf92f638ef9d432 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 15 Jan 2026 14:57:32 +0100 Subject: [PATCH 096/131] Patch get_image_features for lighton_ocr --- docs/source/en/model_doc/lighton_ocr.md | 4 ++- .../lighton_ocr/modeling_lighton_ocr.py | 34 +++++++++---------- .../models/lighton_ocr/modular_lighton_ocr.py | 33 +++++++++--------- 3 files changed, 35 insertions(+), 36 deletions(-) diff --git a/docs/source/en/model_doc/lighton_ocr.md b/docs/source/en/model_doc/lighton_ocr.md index 274a2c473848..bff704e4bd71 100644 --- a/docs/source/en/model_doc/lighton_ocr.md +++ b/docs/source/en/model_doc/lighton_ocr.md @@ -12,7 +12,7 @@ Unless required by applicable law or agreed to in writing, software distributed rendered properly in your Markdown viewer. specific language governing permissions and limitations under the License. --> -*This model was released on {release_date} and added to Hugging Face Transformers on 2026-01-14.* +*This model was released on {release_date} and added to Hugging Face Transformers on .* # LightOnOcr @@ -73,8 +73,10 @@ print(output_text) [[autodoc]] LightOnOcrModel - forward + - get_image_features ## LightOnOcrForConditionalGeneration [[autodoc]] LightOnOcrForConditionalGeneration - forward + - get_image_features diff --git a/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py b/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py index ada6f400fa15..be78e2f86b2b 100644 --- a/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py +++ b/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py @@ -25,7 +25,7 @@ from ...cache_utils import Cache from ...generation import GenerationMixin from ...integrations import use_kernel_forward_from_hub -from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple @@ -170,27 +170,22 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - def get_image_features(self, pixel_values: torch.Tensor, image_sizes: torch.Tensor | list): - """ - Obtains image features from the vision encoder and projection. - - Args: - pixel_values: Image tensors - image_sizes: Tensor or list of (height, width) pairs for each image - - Returns: - List of image feature tensors, one per image - """ - visual_features = self.vision_encoder(pixel_values, image_sizes=image_sizes).last_hidden_state - - image_features = self.vision_projection(visual_features.squeeze(0), image_sizes) + @can_return_tuple + @auto_docstring + def get_image_features( + self, pixel_values: torch.Tensor, image_sizes: torch.Tensor | list, **kwargs: Unpack[TransformersKwargs] + ) -> tuple | BaseModelOutputWithPooling: + image_outputs = self.vision_encoder(pixel_values, image_sizes=image_sizes, **kwargs) + image_features = image_outputs.last_hidden_state + image_features = self.vision_projection(image_features.squeeze(0), image_sizes) # Split features per image based on the effective patch size downsample_ratio = self.config.vision_config.patch_size * self.config.spatial_merge_size split_sizes = [(height // downsample_ratio) * (width // downsample_ratio) for height, width in image_sizes] image_features = torch.split(image_features, split_sizes) + image_outputs.pooler_output = image_features - return image_features + return image_outputs def get_placeholder_mask( self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor @@ -330,8 +325,11 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> nn.Module: return self.lm_head - def get_image_features(self, pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, **kwargs): - return self.model.get_image_features(pixel_values=pixel_values, image_sizes=image_sizes) + @auto_docstring + def get_image_features( + self, pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> tuple | BaseModelOutputWithPooling: + return self.model.get_image_features(pixel_values=pixel_values, image_sizes=image_sizes, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/lighton_ocr/modular_lighton_ocr.py b/src/transformers/models/lighton_ocr/modular_lighton_ocr.py index d7527f29d752..d53864d18cfa 100644 --- a/src/transformers/models/lighton_ocr/modular_lighton_ocr.py +++ b/src/transformers/models/lighton_ocr/modular_lighton_ocr.py @@ -21,6 +21,7 @@ from ...configuration_utils import PretrainedConfig from ...feature_extraction_utils import BatchFeature from ...image_utils import ImageInput +from ...modeling_outputs import BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel from ...processing_utils import ( MultiModalData, @@ -305,27 +306,22 @@ def __init__(self, config: LightOnOcrConfig): self.language_model = AutoModel.from_config(config.text_config) self.post_init() - def get_image_features(self, pixel_values: torch.Tensor, image_sizes: torch.Tensor | list): - """ - Obtains image features from the vision encoder and projection. - - Args: - pixel_values: Image tensors - image_sizes: Tensor or list of (height, width) pairs for each image - - Returns: - List of image feature tensors, one per image - """ - visual_features = self.vision_encoder(pixel_values, image_sizes=image_sizes).last_hidden_state - - image_features = self.vision_projection(visual_features.squeeze(0), image_sizes) + @can_return_tuple + @auto_docstring + def get_image_features( + self, pixel_values: torch.Tensor, image_sizes: torch.Tensor | list, **kwargs: Unpack[TransformersKwargs] + ) -> tuple | BaseModelOutputWithPooling: + image_outputs = self.vision_encoder(pixel_values, image_sizes=image_sizes, **kwargs) + image_features = image_outputs.last_hidden_state + image_features = self.vision_projection(image_features.squeeze(0), image_sizes) # Split features per image based on the effective patch size downsample_ratio = self.config.vision_config.patch_size * self.config.spatial_merge_size split_sizes = [(height // downsample_ratio) * (width // downsample_ratio) for height, width in image_sizes] image_features = torch.split(image_features, split_sizes) + image_outputs.pooler_output = image_features - return image_features + return image_outputs @can_return_tuple @auto_docstring @@ -390,8 +386,11 @@ def forward( class LightOnOcrForConditionalGeneration(Mistral3ForConditionalGeneration): _checkpoint_conversion_mapping = {} - def get_image_features(self, pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, **kwargs): - return self.model.get_image_features(pixel_values=pixel_values, image_sizes=image_sizes) + @auto_docstring + def get_image_features( + self, pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, **kwargs: Unpack[TransformersKwargs] + ) -> tuple | BaseModelOutputWithPooling: + return self.model.get_image_features(pixel_values=pixel_values, image_sizes=image_sizes, **kwargs) __all__ = [ From 95a55ad7f7abe88a233f393f4aa95271e8a61032 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 15 Jan 2026 14:58:04 +0100 Subject: [PATCH 097/131] Explicitly use Ovis2VisionAttention in Ovis2VisionEncoderLayer in modular --- .../models/ovis2/modeling_ovis2.py | 63 +------------------ .../models/ovis2/modular_ovis2.py | 4 +- 2 files changed, 4 insertions(+), 63 deletions(-) diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index 011989b71820..51887ed693a4 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -272,71 +272,10 @@ def forward(self, x): return down_proj -class Ovis2Attention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__(self, config): - super().__init__() - self.config = config - self.embed_dim = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.embed_dim // self.num_heads - if self.head_dim * self.num_heads != self.embed_dim: - raise ValueError( - f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" - f" {self.num_heads})." - ) - self.scale = self.head_dim**-0.5 - self.dropout = config.attention_dropout - self.is_causal = False - self.k_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=config.qkv_bias) - self.v_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=config.qkv_bias) - self.q_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=config.qkv_bias) - self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=config.qkv_bias) - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: torch.Tensor | None = None, - **kwargs, - ) -> tuple[torch.Tensor, torch.Tensor | None]: - """Input shape: Batch x Time x Channel""" - - batch_size, seq_length, embed_dim = hidden_states.shape - - queries = self.q_proj(hidden_states) - keys = self.k_proj(hidden_states) - values = self.v_proj(hidden_states) - - queries = queries.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2) - keys = keys.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2) - values = values.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2) - - attention_interface: Callable = eager_attention_forward - if self.config._attn_implementation != "eager": - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - - attn_output, attn_weights = attention_interface( - self, - queries, - keys, - values, - attention_mask, - is_causal=self.is_causal, - scaling=self.scale, - dropout=0.0 if not self.training else self.dropout, - ) - - attn_output = attn_output.reshape(batch_size, seq_length, embed_dim).contiguous() - attn_output = self.out_proj(attn_output) - - return attn_output, attn_weights - - class Ovis2VisionEncoderLayer(GradientCheckpointingLayer): def __init__(self, config: Ovis2VisionConfig): super().__init__() - self.attention = Ovis2Attention(config) + self.attention = Ovis2VisionAttention(config) self.ffn = Ovis2MLP(config) self.rms_norm1 = Ovis2RMSNorm(config.hidden_size, config.rms_norm_eps) self.rms_norm2 = Ovis2RMSNorm(config.hidden_size, config.rms_norm_eps) diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index 430dd62be44d..ac408d7212d9 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -96,7 +96,9 @@ class Ovis2VisionAttention(Aimv2Attention): class Ovis2VisionEncoderLayer(Aimv2EncoderLayer): - pass + def __init__(self, config: Ovis2VisionConfig): + super().__init__() + self.attention = Ovis2VisionAttention(config) class Ovis2VisionEncoder(SiglipEncoder): From ef7783243e32f63cc3924cad46b53754fb2f0462 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 15 Jan 2026 15:05:48 +0100 Subject: [PATCH 098/131] Update use of get_image_features for lighton_ocr Forgot to run tests to verify that it worked, oops --- src/transformers/models/lighton_ocr/modeling_lighton_ocr.py | 4 +++- src/transformers/models/lighton_ocr/modular_lighton_ocr.py | 4 +++- tests/models/lighton_ocr/test_modeling_lighton_ocr.py | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py b/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py index be78e2f86b2b..f014078462e0 100644 --- a/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py +++ b/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py @@ -242,7 +242,9 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_features = self.get_image_features(pixel_values=pixel_values, image_sizes=image_sizes) + image_features = self.get_image_features( + pixel_values=pixel_values, image_sizes=image_sizes, return_dict=True + ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/src/transformers/models/lighton_ocr/modular_lighton_ocr.py b/src/transformers/models/lighton_ocr/modular_lighton_ocr.py index d53864d18cfa..fbc2072e18bc 100644 --- a/src/transformers/models/lighton_ocr/modular_lighton_ocr.py +++ b/src/transformers/models/lighton_ocr/modular_lighton_ocr.py @@ -354,7 +354,9 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) if pixel_values is not None: - image_features = self.get_image_features(pixel_values=pixel_values, image_sizes=image_sizes) + image_features = self.get_image_features( + pixel_values=pixel_values, image_sizes=image_sizes, return_dict=True + ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) special_image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_features diff --git a/tests/models/lighton_ocr/test_modeling_lighton_ocr.py b/tests/models/lighton_ocr/test_modeling_lighton_ocr.py index 6dd0fc8be378..528e9fdd9530 100644 --- a/tests/models/lighton_ocr/test_modeling_lighton_ocr.py +++ b/tests/models/lighton_ocr/test_modeling_lighton_ocr.py @@ -225,6 +225,8 @@ class LightOnOcrForConditionalGenerationModelTest(ModelTesterMixin, GenerationTe else () ) pipeline_model_mapping = {"image-text-to-text": LightOnOcrForConditionalGeneration} if is_torch_available() else {} + # LightOnOcr uses a PixtralVisionModel, which merges batch_size and num_patches in index 1, with index 0 hardcoded to 1 + skip_test_image_features_output_shape = True _is_composite = True @@ -421,7 +423,7 @@ def test_get_image_features(self): image_features_list = model.get_image_features( pixel_values=input_dict["pixel_values"].to(torch_device), image_sizes=input_dict["image_sizes"], - ) + ).pooler_output # Check that features are returned as a list self.assertIsNotNone(image_features_list) From 194a1bd76a3636a0bc6b44e02c23977066712687 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 15 Jan 2026 15:23:07 +0100 Subject: [PATCH 099/131] Rerun python utils/add_dates.py Not sure which script removed the date... :/ --- docs/source/en/model_doc/lighton_ocr.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/model_doc/lighton_ocr.md b/docs/source/en/model_doc/lighton_ocr.md index bff704e4bd71..77671206c2ae 100644 --- a/docs/source/en/model_doc/lighton_ocr.md +++ b/docs/source/en/model_doc/lighton_ocr.md @@ -12,7 +12,7 @@ Unless required by applicable law or agreed to in writing, software distributed rendered properly in your Markdown viewer. specific language governing permissions and limitations under the License. --> -*This model was released on {release_date} and added to Hugging Face Transformers on .* +*This model was released on {release_date} and added to Hugging Face Transformers on 2026-01-14.* # LightOnOcr From 0ce7baccbad77bfd8ae26bfd554068a07163a3ec Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 15 Jan 2026 18:09:47 +0100 Subject: [PATCH 100/131] Remove tie_last_hidden_states=False from check_model_inputs from ... forward methods that previously did not return a BaseModelOutput --- .../models/audioflamingo3/modeling_audioflamingo3.py | 2 +- .../models/audioflamingo3/modular_audioflamingo3.py | 2 +- src/transformers/models/chameleon/modeling_chameleon.py | 2 +- src/transformers/models/emu3/modeling_emu3.py | 2 +- src/transformers/models/emu3/modular_emu3.py | 2 +- .../models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py | 2 +- .../models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py | 2 +- src/transformers/models/florence2/modeling_florence2.py | 2 +- src/transformers/models/florence2/modular_florence2.py | 2 +- src/transformers/models/gemma3n/modeling_gemma3n.py | 2 +- src/transformers/models/gemma3n/modular_gemma3n.py | 2 +- src/transformers/models/glm4v/modeling_glm4v.py | 2 +- src/transformers/models/glm4v/modular_glm4v.py | 2 +- src/transformers/models/glm4v_moe/modeling_glm4v_moe.py | 2 +- src/transformers/models/glm_image/modeling_glm_image.py | 4 ++-- src/transformers/models/glm_image/modular_glm_image.py | 2 +- .../models/granite_speech/modeling_granite_speech.py | 2 +- src/transformers/models/janus/modeling_janus.py | 2 +- .../models/metaclip_2/convert_metaclip_2_to_hf.py | 1 - src/transformers/models/ovis2/modeling_ovis2.py | 2 +- src/transformers/models/ovis2/modular_ovis2.py | 2 +- src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py | 2 +- src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py | 2 +- src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py | 2 +- src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py | 2 +- src/transformers/models/qwen2_vl/modeling_qwen2_vl.py | 2 +- .../models/qwen3_omni_moe/modeling_qwen3_omni_moe.py | 2 +- src/transformers/models/qwen3_vl/modeling_qwen3_vl.py | 2 +- src/transformers/models/qwen3_vl/modular_qwen3_vl.py | 2 +- src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py | 2 +- 30 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py index 645f38f1e3eb..a0e1dd88399d 100644 --- a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py @@ -322,7 +322,7 @@ def get_input_embeddings(self) -> nn.Module: def set_input_embeddings(self, value: nn.Module): self.conv1 = value - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, input_features: torch.Tensor, diff --git a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py index 2ba8b754dd0c..8a00e44194aa 100644 --- a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py @@ -62,7 +62,7 @@ class AudioFlamingo3Encoder(Qwen2AudioEncoder): "attentions": AudioFlamingo3Attention, } - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, input_features: torch.Tensor, diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 7cb31ccf3e65..59be6a14e3df 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -834,7 +834,7 @@ def __init__(self, config: ChameleonVQVAEConfig): self.eval() # Chameleon's VQ model is frozen self.post_init() - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def encode( self, pixel_values: torch.LongTensor, **kwargs: Unpack[TransformersKwargs] ) -> ChameleonVQVAEModelOutput: diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index a3b8fb1fe2f1..a3129ab42ef5 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -1002,7 +1002,7 @@ def __init__(self, config: Emu3VQVAEConfig): self.post_init() - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def encode( self, pixel_values: torch.Tensor, image_sizes: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> Emu3VQVAEModelOutput: diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index 420f4419dab6..31e4d9465b16 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -752,7 +752,7 @@ def __init__(self, config: Emu3VQVAEConfig): self.post_init() - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def encode( self, pixel_values: torch.Tensor, image_sizes: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> Emu3VQVAEModelOutput: diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index 18581c0ed845..a9c359257027 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -927,7 +927,7 @@ def rot_pos_emb(self, grid_thw): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index d2b204ccdf6e..6f50cdfa5c94 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -891,7 +891,7 @@ def get_dtype(self): def get_device(self): raise AttributeError("Ernie 4.5 VL Moe does not need this!") - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/florence2/modeling_florence2.py b/src/transformers/models/florence2/modeling_florence2.py index 88818f106fb2..2a312506d86b 100644 --- a/src/transformers/models/florence2/modeling_florence2.py +++ b/src/transformers/models/florence2/modeling_florence2.py @@ -543,7 +543,7 @@ def __init__(self, config: Florence2VisionConfig): # Initialize weights and apply final processing self.post_init() - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/florence2/modular_florence2.py b/src/transformers/models/florence2/modular_florence2.py index 195618f9cb76..27033ab34935 100644 --- a/src/transformers/models/florence2/modular_florence2.py +++ b/src/transformers/models/florence2/modular_florence2.py @@ -1396,7 +1396,7 @@ def __init__(self, config: Florence2VisionConfig): # Initialize weights and apply final processing self.post_init() - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index c6cf465d0e85..0f322b7072d2 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -959,7 +959,7 @@ def __init__(self, config: Gemma3nAudioConfig): ) self.post_init() - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, audio_mel: torch.Tensor, audio_mel_mask: torch.BoolTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | Gemma3nAudioEncoderModelOutput: diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index 47393bf5f2a2..5d80d2512328 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -1514,7 +1514,7 @@ def __init__(self, config: Gemma3nAudioConfig): ) self.post_init() - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, audio_mel: torch.Tensor, audio_mel_mask: torch.BoolTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | Gemma3nAudioEncoderModelOutput: diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index 3d65ce741852..b39fe5b10bad 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -757,7 +757,7 @@ def rot_pos_emb(self, grid_thw): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb, pos_ids - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs @auto_docstring def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 06282673cd77..e0f657fa811a 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -776,7 +776,7 @@ def rot_pos_emb(self, grid_thw): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb, pos_ids - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs @auto_docstring def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index b769eac5896d..2040f17a352c 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -828,7 +828,7 @@ def rot_pos_emb(self, grid_thw): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb, pos_ids - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs @auto_docstring def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] diff --git a/src/transformers/models/glm_image/modeling_glm_image.py b/src/transformers/models/glm_image/modeling_glm_image.py index bbfa90c5d9e8..51918f905ee0 100644 --- a/src/transformers/models/glm_image/modeling_glm_image.py +++ b/src/transformers/models/glm_image/modeling_glm_image.py @@ -652,7 +652,7 @@ def __init__(self, config: GlmImageVQVAEConfig): self.eval() # GlmImage's VQ model is frozen self.post_init() - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def encode(self, hidden_states) -> GlmImageVQVAEModelOutput: conv_hidden_states = self.quant_conv(hidden_states) quantized_last_hidden_state, emb_loss, indices = self.quantize(conv_hidden_states) @@ -716,7 +716,7 @@ def rot_pos_emb(self, grid_thw): pos_ids = torch.cat(pos_ids, dim=0) return pos_ids - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs @auto_docstring def forward( self, pixel_values: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] diff --git a/src/transformers/models/glm_image/modular_glm_image.py b/src/transformers/models/glm_image/modular_glm_image.py index 1787a1745a6d..6ddcde4c8b35 100644 --- a/src/transformers/models/glm_image/modular_glm_image.py +++ b/src/transformers/models/glm_image/modular_glm_image.py @@ -586,7 +586,7 @@ def rot_pos_emb(self, grid_thw): pos_ids = torch.cat(pos_ids, dim=0) return pos_ids - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs @auto_docstring def forward( self, pixel_values: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index 42da9f03e2ef..3e653597f643 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -296,7 +296,7 @@ def __init__(self, config: GraniteSpeechEncoderConfig): self.num_layers = config.num_layers self.post_init() - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, hidden_states: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/janus/modeling_janus.py b/src/transformers/models/janus/modeling_janus.py index cd3ad9a5b2fd..2aa3188dc851 100644 --- a/src/transformers/models/janus/modeling_janus.py +++ b/src/transformers/models/janus/modeling_janus.py @@ -878,7 +878,7 @@ def __init__(self, config: JanusVQVAEConfig): self.gradient_checkpointing = False self.post_init() - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def encode(self, pixel_values: torch.LongTensor, **kwargs: Unpack[TransformersKwargs]) -> JanusVQVAEModelOutput: hidden_states = self.encoder(pixel_values) conv_hidden_states = self.quant_conv(hidden_states) diff --git a/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py b/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py index ae3a682fdb58..6db36199dca3 100644 --- a/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py +++ b/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py @@ -25,7 +25,6 @@ # Import MetaCLIP modules from src.mini_clip.factory import create_model_and_transforms - from transformers import ( AutoTokenizer, CLIPImageProcessor, diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index 51887ed693a4..52a7442320f1 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -418,7 +418,7 @@ def __init__(self, config: Ovis2VisionConfig): self.post_init() - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures: diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index ac408d7212d9..cac63f839963 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -201,7 +201,7 @@ def __init__(self, config: Ovis2VisionConfig): self.post_init() - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures: diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index 168cf292f1e2..087d6ac13457 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -1248,7 +1248,7 @@ def get_window_index(self, grid_thw): return window_index, cu_window_seqlens - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index 074ebad7e1e7..53e0e4ccef74 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -1957,7 +1957,7 @@ def __init__(self, config: Qwen2_5OmniVisionEncoderConfig, *inputs, **kwargs) -> super().__init__(config, *inputs, **kwargs) self.blocks = nn.ModuleList([Qwen2_5OmniVisionBlock(config) for _ in range(config.depth)]) - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index f311ccbb033e..2e43782c42d0 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -419,7 +419,7 @@ def get_window_index(self, grid_thw): return window_index, cu_window_seqlens - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py index 98e43945c2b6..6e266e062f16 100644 --- a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py @@ -282,7 +282,7 @@ def get_window_index(self, grid_thw): return window_index, cu_window_seqlens - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 54766a2b26db..6c5153c10029 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -742,7 +742,7 @@ def rot_pos_emb(self, grid_thw): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs @auto_docstring def forward( self, diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index ce4cfab1f739..d34e0aed6802 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -1236,7 +1236,7 @@ def fast_pos_embed_interpolate(self, grid_thw): patch_pos_embeds = torch.cat(patch_pos_embeds_permute) return patch_pos_embeds - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithDeepstackFeatures: diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index 33902df8bf46..07f5a1de2de5 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -754,7 +754,7 @@ def fast_pos_embed_interpolate(self, grid_thw): patch_pos_embeds = torch.cat(patch_pos_embeds_permute) return patch_pos_embeds - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithDeepstackFeatures: diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index d556bb6e9560..692807684f90 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -651,7 +651,7 @@ def fast_pos_embed_interpolate(self, grid_thw): patch_pos_embeds = torch.cat(patch_pos_embeds_permute) return patch_pos_embeds - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithDeepstackFeatures: diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index c574b6f0614d..9500341af7c4 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -738,7 +738,7 @@ def fast_pos_embed_interpolate(self, grid_thw): patch_pos_embeds = torch.cat(patch_pos_embeds_permute) return patch_pos_embeds - @check_model_inputs(tie_last_hidden_states=False) + @check_model_inputs def forward( self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithDeepstackFeatures: From 6604784b40da4af4aeb1a0dc6d67d3a4cc8727ed Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 19 Jan 2026 09:26:41 +0100 Subject: [PATCH 101/131] Revert accidental metaclip import change --- src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py b/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py index 6db36199dca3..ae3a682fdb58 100644 --- a/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py +++ b/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py @@ -25,6 +25,7 @@ # Import MetaCLIP modules from src.mini_clip.factory import create_model_and_transforms + from transformers import ( AutoTokenizer, CLIPImageProcessor, From ed5c1364805ac23624ba29731dc44314f5a9126e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 19 Jan 2026 11:45:34 +0100 Subject: [PATCH 102/131] Add missing return_dict=True in get_..._features methods --- src/transformers/models/aimv2/modeling_aimv2.py | 2 ++ src/transformers/models/align/modeling_align.py | 1 + src/transformers/models/altclip/modeling_altclip.py | 2 ++ src/transformers/models/aria/modeling_aria.py | 1 + src/transformers/models/aria/modular_aria.py | 1 + .../models/audioflamingo3/modeling_audioflamingo3.py | 4 +++- .../models/audioflamingo3/modular_audioflamingo3.py | 4 +++- src/transformers/models/aya_vision/modeling_aya_vision.py | 2 +- src/transformers/models/aya_vision/modular_aya_vision.py | 2 +- src/transformers/models/blip/modeling_blip.py | 2 ++ src/transformers/models/blip_2/modeling_blip_2.py | 2 ++ src/transformers/models/chameleon/modeling_chameleon.py | 2 +- .../models/chinese_clip/modeling_chinese_clip.py | 2 ++ src/transformers/models/clap/modeling_clap.py | 3 ++- src/transformers/models/clip/modeling_clip.py | 2 ++ src/transformers/models/clipseg/modeling_clipseg.py | 2 ++ src/transformers/models/clvp/modeling_clvp.py | 1 + .../models/cohere2_vision/modeling_cohere2_vision.py | 2 +- .../models/cohere2_vision/modular_cohere2_vision.py | 2 +- src/transformers/models/deepseek_vl/modeling_deepseek_vl.py | 2 +- .../deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py | 4 ++-- .../models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py | 4 ++-- src/transformers/models/edgetam/modeling_edgetam.py | 2 +- .../models/edgetam_video/modeling_edgetam_video.py | 2 +- src/transformers/models/emu3/modeling_emu3.py | 4 +++- src/transformers/models/emu3/modular_emu3.py | 4 +++- .../models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py | 4 ++-- .../models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py | 4 ++-- src/transformers/models/fast_vlm/modeling_fast_vlm.py | 2 +- src/transformers/models/fast_vlm/modular_fast_vlm.py | 2 +- src/transformers/models/flava/modeling_flava.py | 2 ++ src/transformers/models/florence2/modeling_florence2.py | 2 +- src/transformers/models/florence2/modular_florence2.py | 2 +- src/transformers/models/gemma3/modeling_gemma3.py | 2 +- src/transformers/models/gemma3/modular_gemma3.py | 2 +- src/transformers/models/gemma3n/modeling_gemma3n.py | 6 ++++-- src/transformers/models/gemma3n/modular_gemma3n.py | 6 ++++-- src/transformers/models/glm46v/modeling_glm46v.py | 6 ++++-- src/transformers/models/glm4v/modeling_glm4v.py | 6 ++++-- src/transformers/models/glm4v/modular_glm4v.py | 4 +++- src/transformers/models/glm4v_moe/modeling_glm4v_moe.py | 6 ++++-- src/transformers/models/glm_image/modeling_glm_image.py | 2 +- src/transformers/models/glm_image/modular_glm_image.py | 2 +- src/transformers/models/glmasr/modeling_glmasr.py | 2 +- src/transformers/models/glmasr/modular_glmasr.py | 2 +- src/transformers/models/got_ocr2/modeling_got_ocr2.py | 2 +- src/transformers/models/got_ocr2/modular_got_ocr2.py | 2 +- .../models/granite_speech/modeling_granite_speech.py | 2 +- src/transformers/models/groupvit/modeling_groupvit.py | 3 ++- src/transformers/models/idefics2/modeling_idefics2.py | 2 +- src/transformers/models/idefics3/modeling_idefics3.py | 2 +- src/transformers/models/internvl/modeling_internvl.py | 2 +- src/transformers/models/internvl/modular_internvl.py | 2 +- src/transformers/models/janus/modeling_janus.py | 2 +- src/transformers/models/janus/modular_janus.py | 2 +- src/transformers/models/kosmos2/modeling_kosmos2.py | 1 + src/transformers/models/lfm2_vl/modeling_lfm2_vl.py | 1 + src/transformers/models/lfm2_vl/modular_lfm2_vl.py | 1 + src/transformers/models/lighton_ocr/modeling_lighton_ocr.py | 2 +- src/transformers/models/lighton_ocr/modular_lighton_ocr.py | 2 +- src/transformers/models/llava/modeling_llava.py | 2 +- src/transformers/models/llava_next/modeling_llava_next.py | 2 +- .../models/llava_next_video/modeling_llava_next_video.py | 4 ++-- .../models/llava_next_video/modular_llava_next_video.py | 4 ++-- .../models/llava_onevision/modeling_llava_onevision.py | 4 ++-- .../models/llava_onevision/modular_llava_onevision.py | 4 ++-- .../models/metaclip_2/convert_metaclip_2_to_hf.py | 1 - src/transformers/models/metaclip_2/modeling_metaclip_2.py | 2 ++ src/transformers/models/metaclip_2/modular_metaclip_2.py | 2 ++ src/transformers/models/mistral3/modeling_mistral3.py | 2 +- src/transformers/models/mistral3/modular_mistral3.py | 2 +- src/transformers/models/ovis2/modeling_ovis2.py | 2 +- src/transformers/models/ovis2/modular_ovis2.py | 2 +- src/transformers/models/owlv2/modeling_owlv2.py | 2 ++ src/transformers/models/owlvit/modeling_owlvit.py | 2 ++ .../models/paddleocr_vl/modeling_paddleocr_vl.py | 1 + .../models/paddleocr_vl/modular_paddleocr_vl.py | 1 + src/transformers/models/paligemma/modeling_paligemma.py | 2 +- src/transformers/models/pe_video/modeling_pe_video.py | 2 ++ src/transformers/models/pe_video/modular_pe_video.py | 2 ++ .../models/perception_lm/modeling_perception_lm.py | 2 +- .../models/perception_lm/modular_perception_lm.py | 2 +- .../models/qwen2_5_omni/modeling_qwen2_5_omni.py | 1 + .../models/qwen2_5_omni/modular_qwen2_5_omni.py | 1 + src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py | 4 ++-- src/transformers/models/qwen2_vl/modeling_qwen2_vl.py | 4 ++-- .../models/qwen3_omni_moe/modeling_qwen3_omni_moe.py | 1 + .../models/qwen3_omni_moe/modular_qwen3_omni_moe.py | 1 + src/transformers/models/qwen3_vl/modeling_qwen3_vl.py | 2 +- src/transformers/models/qwen3_vl/modular_qwen3_vl.py | 2 +- .../models/qwen3_vl_moe/modeling_qwen3_vl_moe.py | 2 +- src/transformers/models/sam2/modeling_sam2.py | 2 +- src/transformers/models/sam2/modular_sam2.py | 2 +- src/transformers/models/sam2_video/modeling_sam2_video.py | 2 +- src/transformers/models/sam3/modeling_sam3.py | 4 +++- .../models/sam3_tracker/modeling_sam3_tracker.py | 2 +- .../sam3_tracker_video/modeling_sam3_tracker_video.py | 4 +++- .../models/sam3_tracker_video/modular_sam3_tracker_video.py | 4 +++- src/transformers/models/smolvlm/modeling_smolvlm.py | 2 +- src/transformers/models/smolvlm/modular_smolvlm.py | 2 +- src/transformers/models/t5gemma2/modeling_t5gemma2.py | 2 +- src/transformers/models/t5gemma2/modular_t5gemma2.py | 2 +- .../models/video_llama_3/modeling_video_llama_3.py | 1 + .../models/video_llama_3/modular_video_llama_3.py | 1 + src/transformers/models/video_llava/modeling_video_llava.py | 4 ++-- src/transformers/models/vipllava/modeling_vipllava.py | 2 +- src/transformers/models/vipllava/modular_vipllava.py | 2 +- .../modeling_vision_text_dual_encoder.py | 3 ++- src/transformers/models/voxtral/modeling_voxtral.py | 2 +- src/transformers/models/voxtral/modular_voxtral.py | 2 +- src/transformers/models/x_clip/modeling_x_clip.py | 5 ++++- 111 files changed, 172 insertions(+), 98 deletions(-) diff --git a/src/transformers/models/aimv2/modeling_aimv2.py b/src/transformers/models/aimv2/modeling_aimv2.py index 77fd3b9f08ee..1000765e53a8 100644 --- a/src/transformers/models/aimv2/modeling_aimv2.py +++ b/src/transformers/models/aimv2/modeling_aimv2.py @@ -631,6 +631,7 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + return_dict=True, **kwargs, ) pooled_output = text_outputs.pooler_output @@ -668,6 +669,7 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=True, **kwargs, ) pooled_output = vision_outputs.pooler_output diff --git a/src/transformers/models/align/modeling_align.py b/src/transformers/models/align/modeling_align.py index 37281fc2e146..98d14fbb307c 100644 --- a/src/transformers/models/align/modeling_align.py +++ b/src/transformers/models/align/modeling_align.py @@ -1124,6 +1124,7 @@ def get_text_features( token_type_ids=token_type_ids, position_ids=position_ids, inputs_embeds=inputs_embeds, + return_dict=True, **kwargs, ) last_hidden_state = text_outputs[0][:, 0, :] diff --git a/src/transformers/models/altclip/modeling_altclip.py b/src/transformers/models/altclip/modeling_altclip.py index b7f7660f41c9..13c94fa88362 100755 --- a/src/transformers/models/altclip/modeling_altclip.py +++ b/src/transformers/models/altclip/modeling_altclip.py @@ -1184,6 +1184,7 @@ def get_text_features( attention_mask=attention_mask, position_ids=position_ids, token_type_ids=token_type_ids, + return_dict=True, **kwargs, ) pooled_output = text_outputs.pooler_output @@ -1220,6 +1221,7 @@ def get_image_features( vision_outputs = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=True, **kwargs, ) pooled_output = vision_outputs.pooler_output diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index ccf2998ba2d5..b346706be0c6 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -934,6 +934,7 @@ def get_image_features( image_outputs = self.vision_tower( pixel_values, patch_attention_mask=patch_attention_mask, + return_dict=True, **kwargs, ) image_attn_mask = None diff --git a/src/transformers/models/aria/modular_aria.py b/src/transformers/models/aria/modular_aria.py index 05f4eabdbd56..90701f9ac6a1 100644 --- a/src/transformers/models/aria/modular_aria.py +++ b/src/transformers/models/aria/modular_aria.py @@ -1278,6 +1278,7 @@ def get_image_features( image_outputs = self.vision_tower( pixel_values, patch_attention_mask=patch_attention_mask, + return_dict=True, **kwargs, ) image_attn_mask = None diff --git a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py index 334488da557f..f88a19796f34 100644 --- a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py @@ -468,7 +468,9 @@ def get_audio_features( """ # Encode audio - audio_output = self.audio_tower(input_features, input_features_mask=input_features_mask, **kwargs) + audio_output = self.audio_tower( + input_features, input_features_mask=input_features_mask, return_dict=True, **kwargs + ) audio_embeds = self.multi_modal_projector(audio_output.last_hidden_state) # Mask according to avg pooling (which is after attention blocks) diff --git a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py index 8a00e44194aa..b846957940cc 100644 --- a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py @@ -169,7 +169,9 @@ def get_audio_features( """ # Encode audio - audio_output = self.audio_tower(input_features, input_features_mask=input_features_mask, **kwargs) + audio_output = self.audio_tower( + input_features, input_features_mask=input_features_mask, return_dict=True, **kwargs + ) audio_embeds = self.multi_modal_projector(audio_output.last_hidden_state) # Mask according to avg pooling (which is after attention blocks) diff --git a/src/transformers/models/aya_vision/modeling_aya_vision.py b/src/transformers/models/aya_vision/modeling_aya_vision.py index 7a044003503b..9e21a46770b5 100644 --- a/src/transformers/models/aya_vision/modeling_aya_vision.py +++ b/src/transformers/models/aya_vision/modeling_aya_vision.py @@ -205,7 +205,7 @@ def get_image_features( kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/aya_vision/modular_aya_vision.py b/src/transformers/models/aya_vision/modular_aya_vision.py index 7b8b386132cd..d30acc180e1e 100644 --- a/src/transformers/models/aya_vision/modular_aya_vision.py +++ b/src/transformers/models/aya_vision/modular_aya_vision.py @@ -130,7 +130,7 @@ def get_image_features( kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index bbc4c3e81960..e79ccdfa7d5e 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -587,6 +587,7 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + return_dict=True, **kwargs, ) pooled_output = text_outputs.pooler_output @@ -624,6 +625,7 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=True, **kwargs, ) pooled_output = vision_outputs.pooler_output diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index e26d880bb98e..0ec86489224e 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -1127,12 +1127,14 @@ def get_text_features( text_outputs: BaseModelOutputWithPast = self.language_model.base_model( input_ids=input_ids, attention_mask=attention_mask, + return_dict=True, **kwargs, ) else: text_outputs: BaseModelOutputWithPastAndCrossAttentions = self.language_model.get_encoder()( input_ids=input_ids, attention_mask=attention_mask, + return_dict=True, **kwargs, ) return BaseModelOutputWithPooling( diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 59be6a14e3df..1baea2dbf77d 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -897,7 +897,7 @@ def get_image_features( pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): The tensors corresponding to the input images. """ - vqmodel_outputs: ChameleonVQVAEModelOutput = self.vqmodel.encode(pixel_values, **kwargs) + vqmodel_outputs: ChameleonVQVAEModelOutput = self.vqmodel.encode(pixel_values, return_dict=True, **kwargs) vqmodel_outputs.pooler_output = self.get_input_embeddings()(vqmodel_outputs.image_tokens) return vqmodel_outputs diff --git a/src/transformers/models/chinese_clip/modeling_chinese_clip.py b/src/transformers/models/chinese_clip/modeling_chinese_clip.py index 19702c869af1..e3e45c96d5d5 100644 --- a/src/transformers/models/chinese_clip/modeling_chinese_clip.py +++ b/src/transformers/models/chinese_clip/modeling_chinese_clip.py @@ -1031,6 +1031,7 @@ def get_text_features( attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, + return_dict=True, **kwargs, ) pooled_output = text_outputs.last_hidden_state[:, 0, :] @@ -1069,6 +1070,7 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=True, **kwargs, ) pooled_output = vision_outputs.pooler_output diff --git a/src/transformers/models/clap/modeling_clap.py b/src/transformers/models/clap/modeling_clap.py index c117fc07241d..64223c23e8c1 100644 --- a/src/transformers/models/clap/modeling_clap.py +++ b/src/transformers/models/clap/modeling_clap.py @@ -1580,6 +1580,7 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + return_dict=True, **kwargs, ) text_features = self.text_projection(text_outputs.pooler_output) @@ -1616,7 +1617,7 @@ def get_audio_features( ... audio_features = model.get_audio_features(**inputs) ```""" audio_outputs: BaseModelOutputWithPooling = self.audio_model( - input_features=input_features, is_longer=is_longer, **kwargs + input_features=input_features, is_longer=is_longer, return_dict=True, **kwargs ) audio_features = self.audio_projection(audio_outputs.pooler_output) audio_outputs.pooler_output = F.normalize(audio_features, dim=-1) diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index be61e34c1493..54e02843c2de 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -809,6 +809,7 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + return_dict=True, **kwargs, ) pooled_output = text_outputs.pooler_output @@ -846,6 +847,7 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=True, **kwargs, ) pooled_output = vision_outputs.pooler_output diff --git a/src/transformers/models/clipseg/modeling_clipseg.py b/src/transformers/models/clipseg/modeling_clipseg.py index 865aa20641f0..818ebab02f88 100644 --- a/src/transformers/models/clipseg/modeling_clipseg.py +++ b/src/transformers/models/clipseg/modeling_clipseg.py @@ -878,6 +878,7 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + return_dict=True, **kwargs, ) pooled_output = text_outputs.pooler_output @@ -915,6 +916,7 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=True, **kwargs, ) pooled_output = vision_outputs.pooler_output diff --git a/src/transformers/models/clvp/modeling_clvp.py b/src/transformers/models/clvp/modeling_clvp.py index 374c18a2f862..1a9fcccea86d 100644 --- a/src/transformers/models/clvp/modeling_clvp.py +++ b/src/transformers/models/clvp/modeling_clvp.py @@ -1531,6 +1531,7 @@ def get_text_features( input_ids=input_ids, inputs_embeds=text_encoder_inputs_embeds, attention_mask=attention_mask, + return_dict=True, **kwargs, ) diff --git a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py index a17bd276da59..904c19d717ae 100644 --- a/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modeling_cohere2_vision.py @@ -172,7 +172,7 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) selected_image_feature = image_outputs.last_hidden_state image_outputs.pooler_output = self.multi_modal_projector(selected_image_feature) diff --git a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py index 558d4fe39d7f..ffe43efe3b80 100644 --- a/src/transformers/models/cohere2_vision/modular_cohere2_vision.py +++ b/src/transformers/models/cohere2_vision/modular_cohere2_vision.py @@ -103,7 +103,7 @@ class Cohere2VisionModel(AyaVisionModel): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) selected_image_feature = image_outputs.last_hidden_state image_outputs.pooler_output = self.multi_modal_projector(selected_image_feature) diff --git a/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py b/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py index 52a93b2be8e2..a2f326890bdd 100644 --- a/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py +++ b/src/transformers/models/deepseek_vl/modeling_deepseek_vl.py @@ -158,7 +158,7 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - vision_outputs = self.vision_model(pixel_values, **kwargs) + vision_outputs = self.vision_model(pixel_values, return_dict=True, **kwargs) vision_outputs.pooler_output = self.aligner(vision_outputs.last_hidden_state) return vision_outputs diff --git a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py index cbc6e69843db..b53c958a011c 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py @@ -300,8 +300,8 @@ def get_image_features( high_res_pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithHighResVisionEncodings: - low_res_outputs = self.get_low_res_image_features(pixel_values, **kwargs) - high_res_outputs = self.get_high_res_image_features(high_res_pixel_values, **kwargs) + low_res_outputs = self.get_low_res_image_features(pixel_values, return_dict=True, **kwargs) + high_res_outputs = self.get_high_res_image_features(high_res_pixel_values, return_dict=True, **kwargs) image_features = self.aligner(low_res_outputs.last_hidden_state, high_res_outputs.last_hidden_state) return BaseModelOutputWithHighResVisionEncodings( diff --git a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py index a4bbca85b829..13bcddf41a1f 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py @@ -310,8 +310,8 @@ def get_image_features( high_res_pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithHighResVisionEncodings: - low_res_outputs = self.get_low_res_image_features(pixel_values, **kwargs) - high_res_outputs = self.get_high_res_image_features(high_res_pixel_values, **kwargs) + low_res_outputs = self.get_low_res_image_features(pixel_values, return_dict=True, **kwargs) + high_res_outputs = self.get_high_res_image_features(high_res_pixel_values, return_dict=True, **kwargs) image_features = self.aligner(low_res_outputs.last_hidden_state, high_res_outputs.last_hidden_state) return BaseModelOutputWithHighResVisionEncodings( diff --git a/src/transformers/models/edgetam/modeling_edgetam.py b/src/transformers/models/edgetam/modeling_edgetam.py index b05b89c8b81d..75712a7fbee4 100644 --- a/src/transformers/models/edgetam/modeling_edgetam.py +++ b/src/transformers/models/edgetam/modeling_edgetam.py @@ -1207,7 +1207,7 @@ def get_image_features( pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. """ - vision_outputs: EdgeTamVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) + vision_outputs: EdgeTamVisionEncoderOutput = self.vision_encoder(pixel_values, return_dict=True, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding diff --git a/src/transformers/models/edgetam_video/modeling_edgetam_video.py b/src/transformers/models/edgetam_video/modeling_edgetam_video.py index ba5e1c68058c..ef8623eb3bfe 100644 --- a/src/transformers/models/edgetam_video/modeling_edgetam_video.py +++ b/src/transformers/models/edgetam_video/modeling_edgetam_video.py @@ -2240,7 +2240,7 @@ def get_image_features( pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. """ - vision_outputs: EdgeTamVideoVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) + vision_outputs: EdgeTamVideoVisionEncoderOutput = self.vision_encoder(pixel_values, return_dict=True, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index a3129ab42ef5..b6636885bb7b 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -1407,7 +1407,9 @@ def get_image_features( pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): The tensors corresponding to the input images. """ - vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, **kwargs) + vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode( + pixel_values, image_sizes, return_dict=True, **kwargs + ) split_sizes = [ (height // self.vqmodel.vision_spatial_factor) * (width // self.vqmodel.vision_spatial_factor + 1) for height, width in image_sizes diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index 31e4d9465b16..2cd3ad8f4836 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -961,7 +961,9 @@ def get_image_features( pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): The tensors corresponding to the input images. """ - vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode(pixel_values, image_sizes, **kwargs) + vqmodel_outputs: Emu3VQVAEModelOutput = self.vqmodel.encode( + pixel_values, image_sizes, return_dict=True, **kwargs + ) split_sizes = [ (height // self.vqmodel.vision_spatial_factor) * (width // self.vqmodel.vision_spatial_factor + 1) for height, width in image_sizes diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index 92942f92379c..ce551f1a7dc7 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -1289,7 +1289,7 @@ def get_video_features( video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. """ - video_outputs = self.vision_tower(pixel_values_videos, video_grid_thw, **kwargs) + video_outputs = self.vision_tower(pixel_values_videos, video_grid_thw, return_dict=True, **kwargs) video_embeds = self.resampler_model(video_outputs.last_hidden_state, video_grid_thw) split_sizes = ( video_grid_thw.prod(-1) @@ -1314,7 +1314,7 @@ def get_image_features( image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. """ - image_outputs = self.vision_tower(pixel_values, image_grid_thw, **kwargs) + image_outputs = self.vision_tower(pixel_values, image_grid_thw, return_dict=True, **kwargs) image_embeds = self.resampler_model(image_outputs.last_hidden_state, image_grid_thw) split_sizes = (image_grid_thw.prod(-1) // self.vision_tower.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index cf99079cb940..60037437e9fe 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -1290,7 +1290,7 @@ def get_video_features( video_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - video_outputs = self.vision_tower(pixel_values_videos, video_grid_thw, **kwargs) + video_outputs = self.vision_tower(pixel_values_videos, video_grid_thw, return_dict=True, **kwargs) video_embeds = self.resampler_model(video_outputs.last_hidden_state, video_grid_thw) split_sizes = ( video_grid_thw.prod(-1) @@ -1309,7 +1309,7 @@ def get_image_features( image_grid_thw: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - image_outputs = self.vision_tower(pixel_values, image_grid_thw, **kwargs) + image_outputs = self.vision_tower(pixel_values, image_grid_thw, return_dict=True, **kwargs) image_embeds = self.resampler_model(image_outputs.last_hidden_state, image_grid_thw) split_sizes = (image_grid_thw.prod(-1) // self.vision_tower.spatial_merge_size**2).tolist() image_embeds = torch.split(image_embeds, split_sizes) diff --git a/src/transformers/models/fast_vlm/modeling_fast_vlm.py b/src/transformers/models/fast_vlm/modeling_fast_vlm.py index 1762a853d4d4..25865665b86f 100644 --- a/src/transformers/models/fast_vlm/modeling_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modeling_fast_vlm.py @@ -143,7 +143,7 @@ def get_image_features( ) kwargs = {k: v for k, v in kwargs.items() if v is not None} - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # since the vision tower is hybrid in FastVLM, its output needs to be handled differently from Llava selected_image_feature = image_outputs.last_hidden_state diff --git a/src/transformers/models/fast_vlm/modular_fast_vlm.py b/src/transformers/models/fast_vlm/modular_fast_vlm.py index 8364e2eac5be..4d2b18d4127f 100644 --- a/src/transformers/models/fast_vlm/modular_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modular_fast_vlm.py @@ -199,7 +199,7 @@ def get_image_features( ) kwargs = {k: v for k, v in kwargs.items() if v is not None} - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # since the vision tower is hybrid in FastVLM, its output needs to be handled differently from Llava selected_image_feature = image_outputs.last_hidden_state diff --git a/src/transformers/models/flava/modeling_flava.py b/src/transformers/models/flava/modeling_flava.py index 38ce28e8140c..aa3117eb4981 100644 --- a/src/transformers/models/flava/modeling_flava.py +++ b/src/transformers/models/flava/modeling_flava.py @@ -1041,6 +1041,7 @@ def get_text_features( attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, + return_dict=True, **kwargs, ) last_hidden_state = text_outputs.last_hidden_state @@ -1086,6 +1087,7 @@ def get_image_features( bool_masked_pos=bool_masked_pos, attention_mask=attention_mask, interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=True, **kwargs, ) last_hidden_state = image_outputs.last_hidden_state diff --git a/src/transformers/models/florence2/modeling_florence2.py b/src/transformers/models/florence2/modeling_florence2.py index 2a312506d86b..63e70b93d52b 100644 --- a/src/transformers/models/florence2/modeling_florence2.py +++ b/src/transformers/models/florence2/modeling_florence2.py @@ -683,7 +683,7 @@ def get_image_features( pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): The tensors corresponding to the input images. """ - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) image_outputs.pooler_output = self.multi_modal_projector(image_outputs.last_hidden_state) return image_outputs diff --git a/src/transformers/models/florence2/modular_florence2.py b/src/transformers/models/florence2/modular_florence2.py index 27033ab34935..7501381e17af 100644 --- a/src/transformers/models/florence2/modular_florence2.py +++ b/src/transformers/models/florence2/modular_florence2.py @@ -1522,7 +1522,7 @@ def get_image_features( pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): The tensors corresponding to the input images. """ - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) image_outputs.pooler_output = self.multi_modal_projector(image_outputs.last_hidden_state) return image_outputs diff --git a/src/transformers/models/gemma3/modeling_gemma3.py b/src/transformers/models/gemma3/modeling_gemma3.py index c0df429eee83..3f23165765eb 100644 --- a/src/transformers/models/gemma3/modeling_gemma3.py +++ b/src/transformers/models/gemma3/modeling_gemma3.py @@ -844,7 +844,7 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) + vision_outputs = self.vision_tower(pixel_values=pixel_values, return_dict=True, **kwargs) last_hidden_state = vision_outputs.last_hidden_state vision_outputs.pooler_output = self.multi_modal_projector(last_hidden_state) diff --git a/src/transformers/models/gemma3/modular_gemma3.py b/src/transformers/models/gemma3/modular_gemma3.py index ef9925dc16e2..31424e010f76 100644 --- a/src/transformers/models/gemma3/modular_gemma3.py +++ b/src/transformers/models/gemma3/modular_gemma3.py @@ -804,7 +804,7 @@ def __init__(self, config: Gemma3Config): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) + vision_outputs = self.vision_tower(pixel_values=pixel_values, return_dict=True, **kwargs) last_hidden_state = vision_outputs.last_hidden_state vision_outputs.pooler_output = self.multi_modal_projector(last_hidden_state) diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index 25d6fe0ad65c..647385bb3d50 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -1971,7 +1971,7 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, **kwargs) + vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, return_dict=True, **kwargs) last_hidden_state = vision_outputs.last_hidden_state # Convert from (batch, channels, height, width) to (batch, height * width, channels) where: # height == width and height * width == Gemma3nConfig.vision_soft_tokens_per_image. @@ -2186,7 +2186,9 @@ def get_audio_features( input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`): The attention mask for the input audio. """ - audio_outputs: Gemma3nAudioEncoderModelOutput = self.audio_tower(input_features, input_features_mask) + audio_outputs: Gemma3nAudioEncoderModelOutput = self.audio_tower( + input_features, input_features_mask, return_dict=True, **kwargs + ) audio_embeds = self.embed_audio(inputs_embeds=audio_outputs.last_hidden_state) audio_outputs.pooler_output = audio_embeds diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index c93f91d5d973..155c5e68e91c 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -2244,7 +2244,7 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, **kwargs) + vision_outputs = self.vision_tower(pixel_values=pixel_values, do_pooling=False, return_dict=True, **kwargs) last_hidden_state = vision_outputs.last_hidden_state # Convert from (batch, channels, height, width) to (batch, height * width, channels) where: # height == width and height * width == Gemma3nConfig.vision_soft_tokens_per_image. @@ -2459,7 +2459,9 @@ def get_audio_features( input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`): The attention mask for the input audio. """ - audio_outputs: Gemma3nAudioEncoderModelOutput = self.audio_tower(input_features, input_features_mask) + audio_outputs: Gemma3nAudioEncoderModelOutput = self.audio_tower( + input_features, input_features_mask, return_dict=True, **kwargs + ) audio_embeds = self.embed_audio(inputs_embeds=audio_outputs.last_hidden_state) audio_outputs.pooler_output = audio_embeds diff --git a/src/transformers/models/glm46v/modeling_glm46v.py b/src/transformers/models/glm46v/modeling_glm46v.py index 8d87fd38669e..b975a87d4e90 100644 --- a/src/transformers/models/glm46v/modeling_glm46v.py +++ b/src/transformers/models/glm46v/modeling_glm46v.py @@ -311,7 +311,9 @@ def get_video_features( repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) temp_frames_hw.append(repeated_row) flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) - vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, **kwargs) + vision_outputs = self.visual( + pixel_values_videos, grid_thw=flattened_video_grid_thw, return_dict=True, **kwargs + ) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) vision_outputs.pooler_output = video_embeds @@ -333,7 +335,7 @@ def get_image_features( The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) vision_outputs.pooler_output = image_embeds diff --git a/src/transformers/models/glm4v/modeling_glm4v.py b/src/transformers/models/glm4v/modeling_glm4v.py index 4c5cd337c732..8c9a5c254b51 100644 --- a/src/transformers/models/glm4v/modeling_glm4v.py +++ b/src/transformers/models/glm4v/modeling_glm4v.py @@ -1161,7 +1161,9 @@ def get_video_features( repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) temp_frames_hw.append(repeated_row) flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) - vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, **kwargs) + vision_outputs = self.visual( + pixel_values_videos, grid_thw=flattened_video_grid_thw, return_dict=True, **kwargs + ) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) vision_outputs.pooler_output = video_embeds @@ -1183,7 +1185,7 @@ def get_image_features( The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) vision_outputs.pooler_output = image_embeds diff --git a/src/transformers/models/glm4v/modular_glm4v.py b/src/transformers/models/glm4v/modular_glm4v.py index 4e72f9487305..50d7b6bd4e68 100644 --- a/src/transformers/models/glm4v/modular_glm4v.py +++ b/src/transformers/models/glm4v/modular_glm4v.py @@ -1153,7 +1153,9 @@ def get_video_features( repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) temp_frames_hw.append(repeated_row) flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) - vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, **kwargs) + vision_outputs = self.visual( + pixel_values_videos, grid_thw=flattened_video_grid_thw, return_dict=True, **kwargs + ) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) vision_outputs.pooler_output = video_embeds diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index ba2a446c119f..6c88d0456dbc 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -1335,7 +1335,9 @@ def get_video_features( repeated_row = torch.tensor([1, h.item(), w.item()]).unsqueeze(0).repeat(t, 1) temp_frames_hw.append(repeated_row) flattened_video_grid_thw = torch.cat(temp_frames_hw, dim=0) - vision_outputs = self.visual(pixel_values_videos, grid_thw=flattened_video_grid_thw, **kwargs) + vision_outputs = self.visual( + pixel_values_videos, grid_thw=flattened_video_grid_thw, return_dict=True, **kwargs + ) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) vision_outputs.pooler_output = video_embeds @@ -1357,7 +1359,7 @@ def get_image_features( The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) vision_outputs.pooler_output = image_embeds diff --git a/src/transformers/models/glm_image/modeling_glm_image.py b/src/transformers/models/glm_image/modeling_glm_image.py index 51918f905ee0..b694814e82df 100644 --- a/src/transformers/models/glm_image/modeling_glm_image.py +++ b/src/transformers/models/glm_image/modeling_glm_image.py @@ -1208,7 +1208,7 @@ def get_image_features( The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(vision_outputs.last_hidden_state, split_sizes) vision_outputs.pooler_output = image_embeds diff --git a/src/transformers/models/glm_image/modular_glm_image.py b/src/transformers/models/glm_image/modular_glm_image.py index 47a7306b95d2..d2f4604a4e72 100644 --- a/src/transformers/models/glm_image/modular_glm_image.py +++ b/src/transformers/models/glm_image/modular_glm_image.py @@ -917,7 +917,7 @@ def get_image_features( The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(vision_outputs.last_hidden_state, split_sizes) vision_outputs.pooler_output = image_embeds diff --git a/src/transformers/models/glmasr/modeling_glmasr.py b/src/transformers/models/glmasr/modeling_glmasr.py index cac3231c30e2..9a08c0ec1adb 100644 --- a/src/transformers/models/glmasr/modeling_glmasr.py +++ b/src/transformers/models/glmasr/modeling_glmasr.py @@ -406,7 +406,7 @@ def get_audio_features( input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`): Mask to avoid performing attention on padded feature indices. """ - audio_outputs = self.audio_tower(input_features, **kwargs) + audio_outputs = self.audio_tower(input_features, return_dict=True, **kwargs) audio_hidden_states = audio_outputs.last_hidden_state audio_hidden_states = audio_hidden_states.reshape( input_features.shape[0], -1, self.config.audio_config.intermediate_size diff --git a/src/transformers/models/glmasr/modular_glmasr.py b/src/transformers/models/glmasr/modular_glmasr.py index 86353932a1a1..a81a6ed7748d 100644 --- a/src/transformers/models/glmasr/modular_glmasr.py +++ b/src/transformers/models/glmasr/modular_glmasr.py @@ -365,7 +365,7 @@ def get_audio_features( input_features_mask: torch.Tensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - audio_outputs = self.audio_tower(input_features, **kwargs) + audio_outputs = self.audio_tower(input_features, return_dict=True, **kwargs) audio_hidden_states = audio_outputs.last_hidden_state audio_hidden_states = audio_hidden_states.reshape( input_features.shape[0], -1, self.config.audio_config.intermediate_size diff --git a/src/transformers/models/got_ocr2/modeling_got_ocr2.py b/src/transformers/models/got_ocr2/modeling_got_ocr2.py index 2d8cc7c06046..f3b2bc20fdba 100644 --- a/src/transformers/models/got_ocr2/modeling_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modeling_got_ocr2.py @@ -558,7 +558,7 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) last_hidden_state = image_outputs.last_hidden_state image_outputs.pooler_output = self.multi_modal_projector(last_hidden_state) diff --git a/src/transformers/models/got_ocr2/modular_got_ocr2.py b/src/transformers/models/got_ocr2/modular_got_ocr2.py index 63781b30b071..8432c3f4d19d 100644 --- a/src/transformers/models/got_ocr2/modular_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modular_got_ocr2.py @@ -309,7 +309,7 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) last_hidden_state = image_outputs.last_hidden_state image_outputs.pooler_output = self.multi_modal_projector(last_hidden_state) diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index 3e653597f643..5f6bf6461141 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -362,7 +362,7 @@ def get_output_embeddings(self): def get_audio_features( self, input_features: torch.Tensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - audio_outputs = self.encoder(input_features, **kwargs) + audio_outputs = self.encoder(input_features, return_dict=True, **kwargs) projected_embeds = self.projector(audio_outputs.last_hidden_state) audio_outputs.pooler_output = projected_embeds diff --git a/src/transformers/models/groupvit/modeling_groupvit.py b/src/transformers/models/groupvit/modeling_groupvit.py index d92c8126981b..c7b9870cd606 100644 --- a/src/transformers/models/groupvit/modeling_groupvit.py +++ b/src/transformers/models/groupvit/modeling_groupvit.py @@ -1255,6 +1255,7 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + return_dict=True, **kwargs, ) pooled_output = text_outputs.pooler_output @@ -1288,7 +1289,7 @@ def get_image_features( >>> with torch.inference_mode(): ... image_features = model.get_image_features(**inputs) ```""" - vision_outputs: BaseModelOutputWithPooling = self.vision_model(pixel_values, **kwargs) + vision_outputs: BaseModelOutputWithPooling = self.vision_model(pixel_values, return_dict=True, **kwargs) vision_outputs.pooler_output = self.visual_projection(vision_outputs.pooler_output) return vision_outputs diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index fabaf8fd61e2..a0acb0859678 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -864,7 +864,7 @@ def get_image_features( patch_attention_mask = (patches_subgrid.sum(dim=(-1, -2)) == patch_size * patch_size).bool() # Get sequence from the vision encoder image_outputs = self.vision_model( - pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, **kwargs + pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, return_dict=True, **kwargs ) image_hidden_states = image_outputs.last_hidden_state diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index 6fd4c91176e5..d85d97b5daca 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -608,7 +608,7 @@ def get_image_features( # Get sequence from the vision encoder image_outputs = self.vision_model( - pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, **kwargs + pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, return_dict=True, **kwargs ) image_hidden_states = image_outputs.last_hidden_state diff --git a/src/transformers/models/internvl/modeling_internvl.py b/src/transformers/models/internvl/modeling_internvl.py index e37888e33661..abd53c3ea740 100644 --- a/src/transformers/models/internvl/modeling_internvl.py +++ b/src/transformers/models/internvl/modeling_internvl.py @@ -572,7 +572,7 @@ def get_image_features( pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio - vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) + vision_outputs = self.vision_tower(pixel_values=pixel_values, return_dict=True, **kwargs) if vision_feature_layer == -1: vision_features = vision_outputs.last_hidden_state else: diff --git a/src/transformers/models/internvl/modular_internvl.py b/src/transformers/models/internvl/modular_internvl.py index dbeba84fa94f..276685ce2a23 100644 --- a/src/transformers/models/internvl/modular_internvl.py +++ b/src/transformers/models/internvl/modular_internvl.py @@ -512,7 +512,7 @@ def get_image_features( pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio - vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) + vision_outputs = self.vision_tower(pixel_values=pixel_values, return_dict=True, **kwargs) if vision_feature_layer == -1: vision_features = vision_outputs.last_hidden_state else: diff --git a/src/transformers/models/janus/modeling_janus.py b/src/transformers/models/janus/modeling_janus.py index 2aa3188dc851..5b7dfd7b007a 100644 --- a/src/transformers/models/janus/modeling_janus.py +++ b/src/transformers/models/janus/modeling_janus.py @@ -995,7 +995,7 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - vision_outputs = self.vision_model(pixel_values, **kwargs) + vision_outputs = self.vision_model(pixel_values, return_dict=True, **kwargs) vision_outputs.pooler_output = self.aligner(vision_outputs.last_hidden_state) return vision_outputs diff --git a/src/transformers/models/janus/modular_janus.py b/src/transformers/models/janus/modular_janus.py index 59b53a7ad8ea..254059b8ac32 100644 --- a/src/transformers/models/janus/modular_janus.py +++ b/src/transformers/models/janus/modular_janus.py @@ -942,7 +942,7 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - vision_outputs = self.vision_model(pixel_values, **kwargs) + vision_outputs = self.vision_model(pixel_values, return_dict=True, **kwargs) vision_outputs.pooler_output = self.aligner(vision_outputs.last_hidden_state) return vision_outputs diff --git a/src/transformers/models/kosmos2/modeling_kosmos2.py b/src/transformers/models/kosmos2/modeling_kosmos2.py index 8e0bfe28917c..bcfff60f0b02 100644 --- a/src/transformers/models/kosmos2/modeling_kosmos2.py +++ b/src/transformers/models/kosmos2/modeling_kosmos2.py @@ -1526,6 +1526,7 @@ def get_image_features( vision_output: BaseModelOutputWithProjectionAttentions = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=True, **kwargs, ) # The whole `last_hidden_state` through `post_layernorm` instead of just `pooled_output`. diff --git a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py index 19dccd8046c6..295fe2fd63f3 100755 --- a/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modeling_lfm2_vl.py @@ -184,6 +184,7 @@ def get_image_features( pixel_values=pixel_values, spatial_shapes=spatial_shapes, pixel_attention_mask=pixel_attention_mask, + return_dict=True, **kwargs, ) last_hidden_state = image_outputs.last_hidden_state diff --git a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py index 81222a4b704f..6972045df382 100644 --- a/src/transformers/models/lfm2_vl/modular_lfm2_vl.py +++ b/src/transformers/models/lfm2_vl/modular_lfm2_vl.py @@ -115,6 +115,7 @@ def get_image_features( pixel_values=pixel_values, spatial_shapes=spatial_shapes, pixel_attention_mask=pixel_attention_mask, + return_dict=True, **kwargs, ) last_hidden_state = image_outputs.last_hidden_state diff --git a/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py b/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py index f014078462e0..74e1ef893dbf 100644 --- a/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py +++ b/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py @@ -175,7 +175,7 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.Tensor, image_sizes: torch.Tensor | list, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - image_outputs = self.vision_encoder(pixel_values, image_sizes=image_sizes, **kwargs) + image_outputs = self.vision_encoder(pixel_values, image_sizes=image_sizes, return_dict=True, **kwargs) image_features = image_outputs.last_hidden_state image_features = self.vision_projection(image_features.squeeze(0), image_sizes) diff --git a/src/transformers/models/lighton_ocr/modular_lighton_ocr.py b/src/transformers/models/lighton_ocr/modular_lighton_ocr.py index fbc2072e18bc..310118ac1a4b 100644 --- a/src/transformers/models/lighton_ocr/modular_lighton_ocr.py +++ b/src/transformers/models/lighton_ocr/modular_lighton_ocr.py @@ -311,7 +311,7 @@ def __init__(self, config: LightOnOcrConfig): def get_image_features( self, pixel_values: torch.Tensor, image_sizes: torch.Tensor | list, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - image_outputs = self.vision_encoder(pixel_values, image_sizes=image_sizes, **kwargs) + image_outputs = self.vision_encoder(pixel_values, image_sizes=image_sizes, return_dict=True, **kwargs) image_features = image_outputs.last_hidden_state image_features = self.vision_projection(image_features.squeeze(0), image_sizes) diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index afbb060b0853..f913fe569c6b 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -171,7 +171,7 @@ def get_image_features( kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index 5aeb47801fe1..5518cd1a2af5 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -395,7 +395,7 @@ def get_image_features( raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index c611407b8fcc..7ac0690ee148 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -447,7 +447,7 @@ def get_image_features( raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -646,7 +646,7 @@ def get_video_features( batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channels, height, width) kwargs["output_hidden_states"] = True - video_outputs = self.vision_tower(pixel_values, **kwargs) + video_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index 06a47eae574c..86f461d107d0 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -321,7 +321,7 @@ def get_image_features( raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -379,7 +379,7 @@ def get_video_features( batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channels, height, width) kwargs["output_hidden_states"] = True - video_outputs = self.vision_tower(pixel_values, **kwargs) + video_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index 88cef517fe56..28b33bac1178 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -411,7 +411,7 @@ def get_image_features( raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -622,7 +622,7 @@ def get_video_features( batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) kwargs["output_hidden_states"] = True - vision_outputs = self.vision_tower(pixel_values, **kwargs) + vision_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index 64acc242c052..8780efed0e6f 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -372,7 +372,7 @@ def get_image_features( raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -430,7 +430,7 @@ def get_video_features( batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) kwargs["output_hidden_states"] = True - vision_outputs = self.vision_tower(pixel_values, **kwargs) + vision_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py b/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py index ae3a682fdb58..6db36199dca3 100644 --- a/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py +++ b/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py @@ -25,7 +25,6 @@ # Import MetaCLIP modules from src.mini_clip.factory import create_model_and_transforms - from transformers import ( AutoTokenizer, CLIPImageProcessor, diff --git a/src/transformers/models/metaclip_2/modeling_metaclip_2.py b/src/transformers/models/metaclip_2/modeling_metaclip_2.py index 0191841bc539..2440e5fcb58f 100644 --- a/src/transformers/models/metaclip_2/modeling_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modeling_metaclip_2.py @@ -816,6 +816,7 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + return_dict=True, **kwargs, ) pooled_output = text_outputs.pooler_output @@ -852,6 +853,7 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=True, **kwargs, ) pooled_output = vision_outputs.pooler_output diff --git a/src/transformers/models/metaclip_2/modular_metaclip_2.py b/src/transformers/models/metaclip_2/modular_metaclip_2.py index 279bd9ce8572..4fc1aa6e3d6a 100644 --- a/src/transformers/models/metaclip_2/modular_metaclip_2.py +++ b/src/transformers/models/metaclip_2/modular_metaclip_2.py @@ -580,6 +580,7 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + return_dict=True, **kwargs, ) @@ -612,6 +613,7 @@ def get_image_features( return super().get_image_features( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=True, **kwargs, ) diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index 13620876320a..e4d95236f9f4 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -232,7 +232,7 @@ def get_image_features( kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, **kwargs) + image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): diff --git a/src/transformers/models/mistral3/modular_mistral3.py b/src/transformers/models/mistral3/modular_mistral3.py index b27c6398735d..e31626ff93df 100644 --- a/src/transformers/models/mistral3/modular_mistral3.py +++ b/src/transformers/models/mistral3/modular_mistral3.py @@ -137,7 +137,7 @@ def get_image_features( kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, **kwargs) + image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index 52a7442320f1..cbcc7e82f4c5 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -494,7 +494,7 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures: - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) image_features = image_outputs.pooler_output batch_size, img_seq_len, _ = image_features.shape padding_tensor = torch.zeros( diff --git a/src/transformers/models/ovis2/modular_ovis2.py b/src/transformers/models/ovis2/modular_ovis2.py index cac63f839963..aa659655fb49 100644 --- a/src/transformers/models/ovis2/modular_ovis2.py +++ b/src/transformers/models/ovis2/modular_ovis2.py @@ -266,7 +266,7 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures: - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) image_features = image_outputs.pooler_output batch_size, img_seq_len, _ = image_features.shape padding_tensor = torch.zeros( diff --git a/src/transformers/models/owlv2/modeling_owlv2.py b/src/transformers/models/owlv2/modeling_owlv2.py index 3ccdecdd09d9..ded895ebb814 100644 --- a/src/transformers/models/owlv2/modeling_owlv2.py +++ b/src/transformers/models/owlv2/modeling_owlv2.py @@ -1005,6 +1005,7 @@ def get_text_features( text_outputs: BaseModelOutputWithPooling = self.text_model( input_ids=input_ids, attention_mask=attention_mask, + return_dict=True, **kwargs, ) pooled_output = text_outputs.pooler_output @@ -1040,6 +1041,7 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=True, **kwargs, ) vision_outputs.pooler_output = self.visual_projection(vision_outputs.pooler_output) diff --git a/src/transformers/models/owlvit/modeling_owlvit.py b/src/transformers/models/owlvit/modeling_owlvit.py index 3d4b3aa96f35..bbda1109fd3e 100644 --- a/src/transformers/models/owlvit/modeling_owlvit.py +++ b/src/transformers/models/owlvit/modeling_owlvit.py @@ -986,6 +986,7 @@ def get_text_features( text_outputs: BaseModelOutputWithPooling = self.text_model( input_ids=input_ids, attention_mask=attention_mask, + return_dict=True, **kwargs, ) pooled_output = text_outputs.pooler_output @@ -1021,6 +1022,7 @@ def get_image_features( vision_outputs: BaseModelOutputWithPooling = self.vision_model( pixel_values=pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=True, **kwargs, ) vision_outputs.pooler_output = self.visual_projection(vision_outputs.pooler_output) diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index 7babef74cf38..f813ac0c10ea 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -1229,6 +1229,7 @@ def get_image_features( pixel_values=pixel_values, image_grid_thw=image_grid_thw, cu_seqlens=cu_seqlens, + return_dict=True, **kwargs, ) image_embeds = vision_outputs.last_hidden_state diff --git a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py index edd2284041c5..6a320b66440d 100644 --- a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py @@ -1152,6 +1152,7 @@ def get_image_features( pixel_values=pixel_values, image_grid_thw=image_grid_thw, cu_seqlens=cu_seqlens, + return_dict=True, **kwargs, ) image_embeds = vision_outputs.last_hidden_state diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index b2988232dd2d..33d59b99a294 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -264,7 +264,7 @@ def set_input_embeddings(self, value): def get_image_features( self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) selected_image_feature = image_outputs.last_hidden_state image_features = self.multi_modal_projector(selected_image_feature) image_features = image_features / (self.config.text_config.hidden_size**0.5) diff --git a/src/transformers/models/pe_video/modeling_pe_video.py b/src/transformers/models/pe_video/modeling_pe_video.py index 55c7fa5d932d..a94e53b77dc4 100644 --- a/src/transformers/models/pe_video/modeling_pe_video.py +++ b/src/transformers/models/pe_video/modeling_pe_video.py @@ -581,6 +581,7 @@ def get_text_features( text_outputs: BaseModelOutputWithPooling = self.text_model( input_ids=input_ids, attention_mask=attention_mask, + return_dict=True, **kwargs, ) text_outputs.pooler_output = self.text_video_head(text_outputs.last_hidden_state) @@ -597,6 +598,7 @@ def get_video_features( video_outputs: BaseModelOutputWithPooling = self.video_encoder( pixel_values_videos=pixel_values_videos, padding_mask_videos=padding_mask_videos, + return_dict=True, **kwargs, ) video_outputs.pooler_output = self.video_head(video_outputs.pooler_output) diff --git a/src/transformers/models/pe_video/modular_pe_video.py b/src/transformers/models/pe_video/modular_pe_video.py index 026e5b858715..38576598cf44 100644 --- a/src/transformers/models/pe_video/modular_pe_video.py +++ b/src/transformers/models/pe_video/modular_pe_video.py @@ -162,6 +162,7 @@ def get_text_features( text_outputs: BaseModelOutputWithPooling = self.text_model( input_ids=input_ids, attention_mask=attention_mask, + return_dict=True, **kwargs, ) text_outputs.pooler_output = self.text_video_head(text_outputs.last_hidden_state) @@ -178,6 +179,7 @@ def get_video_features( video_outputs: BaseModelOutputWithPooling = self.video_encoder( pixel_values_videos=pixel_values_videos, padding_mask_videos=padding_mask_videos, + return_dict=True, **kwargs, ) video_outputs.pooler_output = self.video_head(video_outputs.pooler_output) diff --git a/src/transformers/models/perception_lm/modeling_perception_lm.py b/src/transformers/models/perception_lm/modeling_perception_lm.py index 1c5436fd4367..be1a7da6f7f2 100644 --- a/src/transformers/models/perception_lm/modeling_perception_lm.py +++ b/src/transformers/models/perception_lm/modeling_perception_lm.py @@ -188,7 +188,7 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - image_outputs = self.vision_tower(pixel_values.flatten(0, 1), **kwargs) + image_outputs = self.vision_tower(pixel_values.flatten(0, 1), return_dict=True, **kwargs) last_hidden_state = image_outputs.last_hidden_state if self.config.vision_use_cls_token: last_hidden_state = last_hidden_state[:, 1:, :] diff --git a/src/transformers/models/perception_lm/modular_perception_lm.py b/src/transformers/models/perception_lm/modular_perception_lm.py index 9c23970e93e9..207ad0081197 100644 --- a/src/transformers/models/perception_lm/modular_perception_lm.py +++ b/src/transformers/models/perception_lm/modular_perception_lm.py @@ -155,7 +155,7 @@ def get_image_features( pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - image_outputs = self.vision_tower(pixel_values.flatten(0, 1), **kwargs) + image_outputs = self.vision_tower(pixel_values.flatten(0, 1), return_dict=True, **kwargs) last_hidden_state = image_outputs.last_hidden_state if self.config.vision_use_cls_token: last_hidden_state = last_hidden_state[:, 1:, :] diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index 78e4fde756a9..a7efd9edaee1 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -1861,6 +1861,7 @@ def get_audio_features( input_features, feature_lens=feature_lens, aftercnn_lens=audio_feat_lengths, + return_dict=True, **kwargs, ) if audio_outputs.last_hidden_state.shape[0] != sum(audio_output_lengths.tolist()): diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index e64ee3bb7fbf..19267732d160 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -2173,6 +2173,7 @@ def get_audio_features( input_features, feature_lens=feature_lens, aftercnn_lens=audio_feat_lengths, + return_dict=True, **kwargs, ) if audio_outputs.last_hidden_state.shape[0] != sum(audio_output_lengths.tolist()): diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index 7222e3a84816..b0fc44ce4b05 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -1193,7 +1193,7 @@ def get_video_features( The temporal, height and width of feature shape of each video in LLM. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) + vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=True, **kwargs) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) vision_outputs.pooler_output = video_embeds @@ -1215,7 +1215,7 @@ def get_image_features( The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) vision_outputs.pooler_output = image_embeds diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 8736cf2c4c5c..ef4650b703ff 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -1126,7 +1126,7 @@ def get_video_features( The temporal, height and width of feature shape of each video in LLM. """ pixel_values_videos = pixel_values_videos.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, **kwargs) + vision_outputs = self.visual(pixel_values_videos, grid_thw=video_grid_thw, return_dict=True, **kwargs) split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() video_embeds = torch.split(vision_outputs.pooler_output, split_sizes) vision_outputs.pooler_output = video_embeds @@ -1148,7 +1148,7 @@ def get_image_features( The temporal, height and width of feature shape of each image in LLM. """ pixel_values = pixel_values.type(self.visual.dtype) - vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, **kwargs) + vision_outputs = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs) split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() image_embeds = torch.split(vision_outputs.pooler_output, split_sizes) vision_outputs.pooler_output = image_embeds diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index af5c92a2f12e..b3d0a261eacb 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -2047,6 +2047,7 @@ def get_audio_features( audio_outputs = self.audio_tower( input_features, feature_lens=feature_lens, + return_dict=True, **kwargs, ) diff --git a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py index a60b931c9696..7c4cf6d70b23 100644 --- a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py @@ -1483,6 +1483,7 @@ def get_audio_features( audio_outputs = self.audio_tower( input_features, feature_lens=feature_lens, + return_dict=True, **kwargs, ) diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index cebccd8f48bc..ccee6a45c7d8 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -1121,7 +1121,7 @@ def get_image_features( """ pixel_values = pixel_values.type(self.visual.dtype) vision_output: BaseModelOutputWithDeepstackFeatures = self.visual( - pixel_values, grid_thw=image_grid_thw, **kwargs + pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs ) image_embeds = vision_output.pooler_output split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index 790f9dbb3385..2988ea9528b8 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -974,7 +974,7 @@ def get_image_features( """ pixel_values = pixel_values.type(self.visual.dtype) vision_output: BaseModelOutputWithDeepstackFeatures = self.visual( - pixel_values, grid_thw=image_grid_thw, **kwargs + pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs ) image_embeds = vision_output.pooler_output split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index ab1646e36397..8de4b5e69b5f 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -1248,7 +1248,7 @@ def get_image_features( """ pixel_values = pixel_values.type(self.visual.dtype) vision_output: BaseModelOutputWithDeepstackFeatures = self.visual( - pixel_values, grid_thw=image_grid_thw, **kwargs + pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs ) image_embeds = vision_output.pooler_output split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() diff --git a/src/transformers/models/sam2/modeling_sam2.py b/src/transformers/models/sam2/modeling_sam2.py index 4bdca76419eb..7d93142aff65 100644 --- a/src/transformers/models/sam2/modeling_sam2.py +++ b/src/transformers/models/sam2/modeling_sam2.py @@ -1574,7 +1574,7 @@ def get_image_features( pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. """ - vision_outputs: Sam2VisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) + vision_outputs: Sam2VisionEncoderOutput = self.vision_encoder(pixel_values, return_dict=True, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding diff --git a/src/transformers/models/sam2/modular_sam2.py b/src/transformers/models/sam2/modular_sam2.py index 13d2864e231b..42f30cafd05a 100644 --- a/src/transformers/models/sam2/modular_sam2.py +++ b/src/transformers/models/sam2/modular_sam2.py @@ -1251,7 +1251,7 @@ def get_image_features( pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. """ - vision_outputs: Sam2VisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) + vision_outputs: Sam2VisionEncoderOutput = self.vision_encoder(pixel_values, return_dict=True, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding diff --git a/src/transformers/models/sam2_video/modeling_sam2_video.py b/src/transformers/models/sam2_video/modeling_sam2_video.py index fd6b46a3ef5e..232ed9a1aa40 100644 --- a/src/transformers/models/sam2_video/modeling_sam2_video.py +++ b/src/transformers/models/sam2_video/modeling_sam2_video.py @@ -1844,7 +1844,7 @@ def get_image_features( pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. """ - vision_outputs: Sam2VideoVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) + vision_outputs: Sam2VideoVisionEncoderOutput = self.vision_encoder(pixel_values, return_dict=True, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding diff --git a/src/transformers/models/sam3/modeling_sam3.py b/src/transformers/models/sam3/modeling_sam3.py index 562c37a008f2..f068c9f61a95 100644 --- a/src/transformers/models/sam3/modeling_sam3.py +++ b/src/transformers/models/sam3/modeling_sam3.py @@ -2170,7 +2170,9 @@ def get_text_features( >>> outputs = model(pixel_values=img_inputs.pixel_values, text_embeds=text_embeds) ``` """ - text_outputs = self.text_encoder(input_ids=input_ids, attention_mask=attention_mask, **kwargs) + text_outputs = self.text_encoder( + input_ids=input_ids, attention_mask=attention_mask, return_dict=True, **kwargs + ) last_hidden_state = text_outputs.last_hidden_state text_outputs.pooler_output = self.text_projection(last_hidden_state) diff --git a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py index 1e624ac94483..78674ca80427 100644 --- a/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py +++ b/src/transformers/models/sam3_tracker/modeling_sam3_tracker.py @@ -1070,7 +1070,7 @@ def get_image_features( pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. """ - vision_outputs: Sam3TrackerVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) + vision_outputs: Sam3TrackerVisionEncoderOutput = self.vision_encoder(pixel_values, return_dict=True, **kwargs) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding diff --git a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py index aab4ce095cd2..e1804efaccef 100644 --- a/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py @@ -1867,7 +1867,9 @@ def get_image_features( pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. """ - vision_outputs: Sam3TrackerVideoVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) + vision_outputs: Sam3TrackerVideoVisionEncoderOutput = self.vision_encoder( + pixel_values, return_dict=True, **kwargs + ) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding diff --git a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py index ba2e1bc54d3f..1d357fe923b0 100644 --- a/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py +++ b/src/transformers/models/sam3_tracker_video/modular_sam3_tracker_video.py @@ -555,7 +555,9 @@ def get_image_features( pixel_values (`torch.FloatTensor`): Input pixel values of shape `(batch_size, num_channels, height, width)`. """ - vision_outputs: Sam3TrackerVideoVisionEncoderOutput = self.vision_encoder(pixel_values, **kwargs) + vision_outputs: Sam3TrackerVideoVisionEncoderOutput = self.vision_encoder( + pixel_values, return_dict=True, **kwargs + ) feature_maps = vision_outputs.fpn_hidden_states feature_maps_position_embeddings = vision_outputs.fpn_position_encoding diff --git a/src/transformers/models/smolvlm/modeling_smolvlm.py b/src/transformers/models/smolvlm/modeling_smolvlm.py index 300635317263..90b1dd3feaab 100644 --- a/src/transformers/models/smolvlm/modeling_smolvlm.py +++ b/src/transformers/models/smolvlm/modeling_smolvlm.py @@ -565,7 +565,7 @@ def get_image_features( # Get sequence from the vision encoder image_outputs = self.vision_model( - pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, **kwargs + pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, return_dict=True, **kwargs ) image_hidden_states = image_outputs.last_hidden_state diff --git a/src/transformers/models/smolvlm/modular_smolvlm.py b/src/transformers/models/smolvlm/modular_smolvlm.py index 6524fc5c5c6e..3d95ac27d890 100644 --- a/src/transformers/models/smolvlm/modular_smolvlm.py +++ b/src/transformers/models/smolvlm/modular_smolvlm.py @@ -238,7 +238,7 @@ def get_image_features( # Get sequence from the vision encoder image_outputs = self.vision_model( - pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, **kwargs + pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, return_dict=True, **kwargs ) image_hidden_states = image_outputs.last_hidden_state diff --git a/src/transformers/models/t5gemma2/modeling_t5gemma2.py b/src/transformers/models/t5gemma2/modeling_t5gemma2.py index 23e8e03df5b4..7eee065d88c8 100644 --- a/src/transformers/models/t5gemma2/modeling_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modeling_t5gemma2.py @@ -814,7 +814,7 @@ def get_image_features( ) -> tuple | BaseModelOutputWithPooling: # pixel_values: (batch_size, channels, height, width) # image_features: Image feature tensor of shape (num_images, image_length, embed_dim). - vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) + vision_outputs = self.vision_tower(pixel_values=pixel_values, return_dict=True, **kwargs) last_hidden_state = vision_outputs.last_hidden_state image_features = self.multi_modal_projector(last_hidden_state) vision_outputs.pooler_output = image_features diff --git a/src/transformers/models/t5gemma2/modular_t5gemma2.py b/src/transformers/models/t5gemma2/modular_t5gemma2.py index e8a1b9189e3d..eb36b260ab28 100644 --- a/src/transformers/models/t5gemma2/modular_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modular_t5gemma2.py @@ -864,7 +864,7 @@ def get_image_features( ) -> tuple | BaseModelOutputWithPooling: # pixel_values: (batch_size, channels, height, width) # image_features: Image feature tensor of shape (num_images, image_length, embed_dim). - vision_outputs = self.vision_tower(pixel_values=pixel_values, **kwargs) + vision_outputs = self.vision_tower(pixel_values=pixel_values, return_dict=True, **kwargs) last_hidden_state = vision_outputs.last_hidden_state image_features = self.multi_modal_projector(last_hidden_state) vision_outputs.pooler_output = image_features diff --git a/src/transformers/models/video_llama_3/modeling_video_llama_3.py b/src/transformers/models/video_llama_3/modeling_video_llama_3.py index c7a2c08b10d4..67a7da9ac0d1 100644 --- a/src/transformers/models/video_llama_3/modeling_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modeling_video_llama_3.py @@ -591,6 +591,7 @@ def get_image_features( pixel_values=pixel_values, grid_thw=image_grid_thw, merge_sizes=image_merge_sizes, + return_dict=True, **kwargs, ) last_hidden_state = vision_outputs.last_hidden_state diff --git a/src/transformers/models/video_llama_3/modular_video_llama_3.py b/src/transformers/models/video_llama_3/modular_video_llama_3.py index 861b7ce77a9c..caeb8483afdd 100644 --- a/src/transformers/models/video_llama_3/modular_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modular_video_llama_3.py @@ -639,6 +639,7 @@ def get_image_features( pixel_values=pixel_values, grid_thw=image_grid_thw, merge_sizes=image_merge_sizes, + return_dict=True, **kwargs, ) last_hidden_state = vision_outputs.last_hidden_state diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index 98aa8e74349a..fc336cb41ff2 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -204,7 +204,7 @@ def get_image_features( raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}") kwargs["output_hidden_states"] = True - image_outputs = self.image_tower(pixel_values_images, **kwargs) + image_outputs = self.image_tower(pixel_values_images, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them @@ -250,7 +250,7 @@ def get_video_features( pixel_values = pixel_values_videos.reshape(batch_size_vid * num_frames, channels, height, width) kwargs["output_hidden_states"] = True - video_outputs = self.video_tower(pixel_values, **kwargs) + video_outputs = self.video_tower(pixel_values, return_dict=True, **kwargs) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index 5aa4b750d956..d3ceecff262f 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -170,7 +170,7 @@ def get_image_features( vision_feature_layers if vision_feature_layers is not None else self.config.vision_feature_layers ) kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If multiple feature layers are provided (which is usually the case) # then the image features are concatenated after the CLS is removed. diff --git a/src/transformers/models/vipllava/modular_vipllava.py b/src/transformers/models/vipllava/modular_vipllava.py index f20f1c18dc89..e8baefda48a6 100644 --- a/src/transformers/models/vipllava/modular_vipllava.py +++ b/src/transformers/models/vipllava/modular_vipllava.py @@ -92,7 +92,7 @@ def get_image_features( vision_feature_layers if vision_feature_layers is not None else self.config.vision_feature_layers ) kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, **kwargs) + image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) # If multiple feature layers are provided (which is usually the case) # then the image features are concatenated after the CLS is removed. diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index 6cc1fb58d705..77aa6feb384a 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -131,6 +131,7 @@ def get_text_features( attention_mask=attention_mask, position_ids=position_ids, token_type_ids=token_type_ids, + return_dict=True, **kwargs, ) pooled_output = text_outputs.pooler_output @@ -162,7 +163,7 @@ def get_image_features( >>> with torch.inference_mode(): ... image_features = model.get_image_features(**inputs) ```""" - vision_outputs = self.vision_model(pixel_values=pixel_values, **kwargs) + vision_outputs = self.vision_model(pixel_values=pixel_values, return_dict=True, **kwargs) vision_outputs.pooler_output = self.visual_projection(vision_outputs.pooler_output) return vision_outputs diff --git a/src/transformers/models/voxtral/modeling_voxtral.py b/src/transformers/models/voxtral/modeling_voxtral.py index f49a5f03ed67..594ca09efe55 100644 --- a/src/transformers/models/voxtral/modeling_voxtral.py +++ b/src/transformers/models/voxtral/modeling_voxtral.py @@ -412,7 +412,7 @@ def get_audio_features( `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] """ - audio_outputs = self.audio_tower(input_features, **kwargs) + audio_outputs = self.audio_tower(input_features, return_dict=True, **kwargs) audio_hidden_states = audio_outputs.last_hidden_state audio_hidden_states = audio_hidden_states.reshape(-1, self.config.audio_config.intermediate_size) audio_embeds = self.multi_modal_projector(audio_hidden_states) diff --git a/src/transformers/models/voxtral/modular_voxtral.py b/src/transformers/models/voxtral/modular_voxtral.py index 50a687b0ecca..7fddbcd29648 100644 --- a/src/transformers/models/voxtral/modular_voxtral.py +++ b/src/transformers/models/voxtral/modular_voxtral.py @@ -178,7 +178,7 @@ def get_audio_features( `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] """ - audio_outputs = self.audio_tower(input_features, **kwargs) + audio_outputs = self.audio_tower(input_features, return_dict=True, **kwargs) audio_hidden_states = audio_outputs.last_hidden_state audio_hidden_states = audio_hidden_states.reshape(-1, self.config.audio_config.intermediate_size) audio_embeds = self.multi_modal_projector(audio_hidden_states) diff --git a/src/transformers/models/x_clip/modeling_x_clip.py b/src/transformers/models/x_clip/modeling_x_clip.py index a594d3f21e5b..f08c4a695e2b 100644 --- a/src/transformers/models/x_clip/modeling_x_clip.py +++ b/src/transformers/models/x_clip/modeling_x_clip.py @@ -1230,6 +1230,7 @@ def get_text_features( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, + return_dict=True, **kwargs, ) pooled_output = text_outputs.pooler_output @@ -1317,7 +1318,9 @@ def get_video_features( batch_size, num_frames, num_channels, height, width = pixel_values.shape pixel_values = pixel_values.reshape(-1, num_channels, height, width) - video_outputs: BaseModelOutputWithPooling = self.vision_model(pixel_values=pixel_values, **kwargs) + video_outputs: BaseModelOutputWithPooling = self.vision_model( + pixel_values=pixel_values, return_dict=True, **kwargs + ) video_embeds = video_outputs.pooler_output video_embeds = self.visual_projection(video_embeds) From 3f0c7545c9bd19c84a2bbd3f0161a603d7be10df Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 19 Jan 2026 11:46:08 +0100 Subject: [PATCH 103/131] Add `output_hidden_states=True` in InternVL get_image_features Only if needed --- src/transformers/models/internvl/modeling_internvl.py | 2 ++ src/transformers/models/internvl/modular_internvl.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/transformers/models/internvl/modeling_internvl.py b/src/transformers/models/internvl/modeling_internvl.py index abd53c3ea740..46c32cd8a86e 100644 --- a/src/transformers/models/internvl/modeling_internvl.py +++ b/src/transformers/models/internvl/modeling_internvl.py @@ -572,6 +572,8 @@ def get_image_features( pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio + if vision_feature_layer != -1: + kwargs["output_hidden_states"] = True vision_outputs = self.vision_tower(pixel_values=pixel_values, return_dict=True, **kwargs) if vision_feature_layer == -1: vision_features = vision_outputs.last_hidden_state diff --git a/src/transformers/models/internvl/modular_internvl.py b/src/transformers/models/internvl/modular_internvl.py index 276685ce2a23..705790705150 100644 --- a/src/transformers/models/internvl/modular_internvl.py +++ b/src/transformers/models/internvl/modular_internvl.py @@ -512,6 +512,8 @@ def get_image_features( pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio + if vision_feature_layer != -1: + kwargs["output_hidden_states"] = True vision_outputs = self.vision_tower(pixel_values=pixel_values, return_dict=True, **kwargs) if vision_feature_layer == -1: vision_features = vision_outputs.last_hidden_state From 061527def49bf1d3a1868d8bea4aed4ee3c706db Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 19 Jan 2026 12:18:16 +0100 Subject: [PATCH 104/131] Add missing docstring for llava_next_video get_video_features --- .../llava_next_video/modeling_llava_next_video.py | 11 +++++++++++ .../llava_next_video/modular_llava_next_video.py | 11 +++++++++++ .../llava_onevision/modeling_llava_onevision.py | 11 +++++++++++ 3 files changed, 33 insertions(+) diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 7ac0690ee148..d0af89d56921 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -977,6 +977,17 @@ def get_video_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_frames, channels, height, width)`) + The tensors corresponding to the input video. + vision_feature_layer (`Union[int, list[int]]`, *optional;*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` + """ return self.model.get_video_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index 86f461d107d0..efcc694a8831 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -543,6 +543,17 @@ def get_video_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_frames, channels, height, width)`) + The tensors corresponding to the input video. + vision_feature_layer (`Union[int, list[int]]`, *optional;*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` + """ return self.model.get_video_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index 28b33bac1178..ede15e7a52db 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -956,6 +956,17 @@ def get_video_features( vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, num_frames, channels, height, width)`) + The tensors corresponding to the input video. + vision_feature_layer (`Union[int, list[int]]`, *optional;*): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + vision_feature_select_strategy (`str`, *optional*): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` + """ return self.model.get_video_features( pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, From af776e91bf96d396d196a57854e8b98880191e1d Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 19 Jan 2026 12:18:44 +0100 Subject: [PATCH 105/131] Quick clean-up in _video_features_prepare_config_and_inputs test helper --- tests/test_modeling_common.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index b08d3352ad40..6012d6fe6890 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4608,16 +4608,19 @@ def _video_features_prepare_config_and_inputs(self): if hasattr(self.model_tester, "video_model_tester"): _, inputs_dict = self.model_tester.video_model_tester.prepare_config_and_inputs_for_common() else: - if "pixel_values" in inputs_dict and "pixel_values_videos" not in inputs_dict: - inputs_dict["pixel_values_videos"] = inputs_dict.pop("pixel_values") - if "image_grid_thw" in inputs_dict and "video_grid_thw" not in inputs_dict: - inputs_dict["video_grid_thw"] = inputs_dict.pop("image_grid_thw") - if "image_merge_sizes" in inputs_dict and "video_merge_sizes" not in inputs_dict: - inputs_dict["video_merge_sizes"] = inputs_dict.pop("image_merge_sizes") + key_mappings = { + "pixel_values": "pixel_values_videos", + "image_grid_thw": "video_grid_thw", + "image_merge_sizes": "video_merge_sizes", + } + + for src_key, dst_key in key_mappings.items(): + if src_key in inputs_dict and dst_key not in inputs_dict: + inputs_dict[dst_key] = inputs_dict.pop(src_key) + + allowed_non_video_keys = {"vision_feature_layer", "vision_feature_select_strategy", "cu_seqlens"} inputs_dict = { - key: value - for key, value in inputs_dict.items() - if "video" in key or key in ["vision_feature_layer", "vision_feature_select_strategy", "cu_seqlens"] + key: value for key, value in inputs_dict.items() if "video" in key or key in allowed_non_video_keys } return config, inputs_dict From 125a49d786d28956de05083e093dcb90cb3bc3a2 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 19 Jan 2026 12:22:03 +0100 Subject: [PATCH 106/131] model.set_attn_implementation instead of config._attn_implementation Note: There's about ~10 other places that use config._attn_implementation in this test file alone --- tests/test_modeling_common.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 6012d6fe6890..779bcd0248d8 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4782,6 +4782,7 @@ def check_hidden_states_output(inputs_dict, config, model_class): def test_get_text_features_attentions(self): def check_attentions_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) + model.set_attn_implementation("eager") model.to(torch_device) model.eval() @@ -4801,8 +4802,6 @@ def check_attentions_output(inputs_dict, config, model_class): continue config, inputs_dict = self._text_features_prepare_config_and_inputs() - # force eager attention to support output attentions - config._attn_implementation = "eager" inputs_dict["output_hidden_states"] = False inputs_dict["output_attentions"] = True check_attentions_output(inputs_dict, config, model_class) @@ -4928,6 +4927,7 @@ def check_hidden_states_output(inputs_dict, config, model_class): def test_get_image_features_attentions(self): def check_attentions_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) + model.set_attn_implementation("eager") model.to(torch_device) model.eval() @@ -4947,9 +4947,6 @@ def check_attentions_output(inputs_dict, config, model_class): continue config, inputs_dict = self._image_features_prepare_config_and_inputs() - - # force eager attention to support output attentions - config._attn_implementation = "eager" inputs_dict["output_hidden_states"] = False inputs_dict["output_attentions"] = True check_attentions_output(inputs_dict, config, model_class) @@ -5053,6 +5050,7 @@ def check_hidden_states_output(inputs_dict, config, model_class): def test_get_audio_features_attentions(self): def check_attentions_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) + model.set_attn_implementation("eager") model.to(torch_device) model.eval() @@ -5071,9 +5069,6 @@ def check_attentions_output(inputs_dict, config, model_class): continue config, inputs_dict = self._audio_features_prepare_config_and_inputs() - - # force eager attention to support output attentions - config._attn_implementation = "eager" inputs_dict["output_hidden_states"] = False inputs_dict["output_attentions"] = True check_attentions_output(inputs_dict, config, model_class) @@ -5179,6 +5174,7 @@ def check_hidden_states_output(inputs_dict, config, model_class): def test_get_video_features_attentions(self): def check_attentions_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) + model.set_attn_implementation("eager") model.to(torch_device) model.eval() @@ -5197,9 +5193,6 @@ def check_attentions_output(inputs_dict, config, model_class): continue config, inputs_dict = self._video_features_prepare_config_and_inputs() - - # force eager attention to support output attentions - config._attn_implementation = "eager" inputs_dict["output_hidden_states"] = False inputs_dict["output_attentions"] = True check_attentions_output(inputs_dict, config, model_class) From 71f9f76891e06488dbe7d5c8813f944955ffba4c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 19 Jan 2026 12:32:51 +0100 Subject: [PATCH 107/131] Add simple docstring to some helper methods re. inputs. It's not extremely useful I think, as it has to be somewhat generic due to the large differences in the architectures --- tests/test_modeling_common.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 779bcd0248d8..75e2390ed755 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4563,6 +4563,12 @@ def test_can_load_from_already_mapped_keys(self): self.assertTrue(compare_state_dicts(model.state_dict(), model_reloaded.state_dict())) def _text_features_prepare_config_and_inputs(self): + """ + Helper method to extract only text-related inputs from the full set of inputs, for testing `get_text_features`. + + Specifically, it tests both the model_tester and its text_model_tester (if any), + and filters for "input_ids", "token_type_ids", and "attention_mask" keys. + """ config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() if hasattr(self.model_tester, "text_model_tester"): _, inputs_dict = self.model_tester.text_model_tester.prepare_config_and_inputs_for_common() @@ -4575,6 +4581,13 @@ def _text_features_prepare_config_and_inputs(self): return config, inputs_dict def _image_features_prepare_config_and_inputs(self): + """ + Helper method to extract only image-related inputs from the full set of inputs, for testing `get_image_features`. + + Specifically, it tests both the model_tester and its vision_model_tester (if any), + and filters for keys related to images. It excludes video-related keys, but allows + "spatial_shapes" and "qformer_input_ids" keys as required by some architectures. + """ config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() if hasattr(self.model_tester, "vision_model_tester"): _, inputs_dict = self.model_tester.vision_model_tester.prepare_config_and_inputs_for_common() @@ -4589,6 +4602,12 @@ def _image_features_prepare_config_and_inputs(self): return config, inputs_dict def _audio_features_prepare_config_and_inputs(self): + """ + Helper method to extract only audio-related inputs from the full set of inputs, for testing `get_audio_features`. + + Specifically, it tests both the model_tester and its audio_model_tester (if any), + and filters for keys related to audio. + """ config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() if hasattr(self.model_tester, "audio_model_tester"): _, inputs_dict = self.model_tester.audio_model_tester.prepare_config_and_inputs_for_common() @@ -4604,6 +4623,13 @@ def _audio_features_prepare_config_and_inputs(self): return config, inputs_dict def _video_features_prepare_config_and_inputs(self): + """ + Helper method to extract only video-related inputs from the full set of inputs, for testing `get_video_features`. + + Specifically, it tests both the model_tester and its video_model_tester (if any), + and filters for keys related to videos. It also handles key renaming for video inputs + if there is no dedicated video_model_tester. + """ config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() if hasattr(self.model_tester, "video_model_tester"): _, inputs_dict = self.model_tester.video_model_tester.prepare_config_and_inputs_for_common() From c69c4c541112275183d7a5d71f10d62f8ff32d46 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 19 Jan 2026 13:16:27 +0100 Subject: [PATCH 108/131] Explain why get_..._features test inputs are overridden --- .../ernie4_5_vl_moe/test_modeling_ernie4_5_vl_moe.py | 7 +++++++ tests/models/glm_image/test_modeling_glm_image.py | 6 ++++++ tests/models/instructblip/test_modeling_instructblip.py | 6 ++++++ .../instructblipvideo/test_modeling_instructblipvideo.py | 6 ++++++ .../llava_next_video/test_modeling_llava_next_video.py | 6 ++++++ .../llava_onevision/test_modeling_llava_onevision.py | 6 ++++++ tests/models/x_clip/test_modeling_x_clip.py | 7 +++++++ 7 files changed, 44 insertions(+) diff --git a/tests/models/ernie4_5_vl_moe/test_modeling_ernie4_5_vl_moe.py b/tests/models/ernie4_5_vl_moe/test_modeling_ernie4_5_vl_moe.py index 1d14b6ca414a..d734e73be849 100644 --- a/tests/models/ernie4_5_vl_moe/test_modeling_ernie4_5_vl_moe.py +++ b/tests/models/ernie4_5_vl_moe/test_modeling_ernie4_5_vl_moe.py @@ -267,6 +267,13 @@ def test_multi_gpu_data_parallel_forward(self): pass def _video_features_prepare_config_and_inputs(self): + """ + Helper method to extract only video-related inputs from the full set of inputs, for testing `get_video_features`. + + The superclass method simply calls the model_tester.prepare_config_and_inputs_for_common(), + but that method only prepared image inputs, i.e. where the temporal dimension in grid_thw is 1. + This override prepares proper video inputs with 12 frames. + """ config = self.model_tester.get_config() patch_size = config.vision_config.patch_size batch_size = self.model_tester.batch_size diff --git a/tests/models/glm_image/test_modeling_glm_image.py b/tests/models/glm_image/test_modeling_glm_image.py index 3a9dc81625c0..d21a9fb35ff2 100644 --- a/tests/models/glm_image/test_modeling_glm_image.py +++ b/tests/models/glm_image/test_modeling_glm_image.py @@ -345,6 +345,12 @@ def test_generate_compile_model_forward_fullgraph(self): pass def _image_features_prepare_config_and_inputs(self): + """ + Helper method to extract only image-related inputs from the full set of inputs, for testing `get_image_features`. + + GlmImage internally preprocesses the image_grid_thw input by slicing off the last entry, + so we need to prepare inputs accordingly for testing get_image_features. We also discard text-related inputs. + """ config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() inputs_dict["image_grid_thw"] = inputs_dict["image_grid_thw"][:-1] del inputs_dict["input_ids"] diff --git a/tests/models/instructblip/test_modeling_instructblip.py b/tests/models/instructblip/test_modeling_instructblip.py index 80fb22cd38b3..eb7abffcc195 100644 --- a/tests/models/instructblip/test_modeling_instructblip.py +++ b/tests/models/instructblip/test_modeling_instructblip.py @@ -616,6 +616,12 @@ def test_sdpa_can_dispatch_composite_models(self): raise ValueError("The eager model should not have SDPA attention layers") def _image_features_prepare_config_and_inputs(self): + """ + Helper method to extract only image-related inputs from the full set of inputs, for testing `get_image_features`. + + InstructBlip's `get_image_features` uses `qformer_input_ids` and `qformer_attention_mask` along with `pixel_values`, + so we override this method to keep those, and only discard `input_ids` and `attention_mask`. + """ config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() del inputs_dict["input_ids"] del inputs_dict["attention_mask"] diff --git a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py index e5b0ae059211..3abf88e79239 100644 --- a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py +++ b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py @@ -626,6 +626,12 @@ def test_sdpa_can_dispatch_composite_models(self): raise ValueError("The eager model should not have SDPA attention layers") def _video_features_prepare_config_and_inputs(self): + """ + Helper method to extract only video-related inputs from the full set of inputs, for testing `get_video_features`. + + InstructBlip's `get_video_features` uses `qformer_input_ids` and `qformer_attention_mask` along with `pixel_values`, + so we override this method to keep those, and only discard `input_ids` and `attention_mask`. + """ config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() del inputs_dict["input_ids"] del inputs_dict["attention_mask"] diff --git a/tests/models/llava_next_video/test_modeling_llava_next_video.py b/tests/models/llava_next_video/test_modeling_llava_next_video.py index 8a4ea8614f8f..862544a856d6 100644 --- a/tests/models/llava_next_video/test_modeling_llava_next_video.py +++ b/tests/models/llava_next_video/test_modeling_llava_next_video.py @@ -329,6 +329,12 @@ def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): pass def _video_features_prepare_config_and_inputs(self): + """ + Helper method to extract only image-related inputs from the full set of inputs, for testing `get_image_features`. + + Despite using `pixel_values_videos` in forward, LlavaNextVideo's `get_video_features` method + instead uses `pixel_values` as input, so we need to override the inputs accordingly. + """ config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() inputs_dict = {"pixel_values": inputs_dict["pixel_values_videos"]} return config, inputs_dict diff --git a/tests/models/llava_onevision/test_modeling_llava_onevision.py b/tests/models/llava_onevision/test_modeling_llava_onevision.py index 9b0ab653791f..99a33b8fc1dd 100644 --- a/tests/models/llava_onevision/test_modeling_llava_onevision.py +++ b/tests/models/llava_onevision/test_modeling_llava_onevision.py @@ -292,6 +292,12 @@ def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): pass def _video_features_prepare_config_and_inputs(self): + """ + Helper method to extract only video-related inputs from the full set of inputs, for testing `get_video_features`. + + The superclass method will rename "pixel_values" to "pixel_values_videos" automatically, but LlavaOnevision's + `get_video_features` uses "pixel_values" as input, so we need to override the inputs accordingly. + """ pixel_values_videos = floats_tensor( [ self.model_tester.batch_size, diff --git a/tests/models/x_clip/test_modeling_x_clip.py b/tests/models/x_clip/test_modeling_x_clip.py index 2da551a88802..5b4aac347e79 100644 --- a/tests/models/x_clip/test_modeling_x_clip.py +++ b/tests/models/x_clip/test_modeling_x_clip.py @@ -569,6 +569,13 @@ def test_model_from_pretrained(self): self.assertIsNotNone(model) def _video_features_prepare_config_and_inputs(self): + """ + Helper method to extract only video-related inputs from the full set of inputs, for testing `get_video_features`. + + The model_tester.vision_model_tester.prepare_config_and_inputs() method prepares image inputs + where the batch size * time dimension is flattened. So, instead we use the model_tester.prepare_config_and_inputs() + which prepares video inputs with shape (batch_size, num_frames, num_channels, height, width) instead. + """ config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() del inputs_dict["input_ids"] del inputs_dict["attention_mask"] From 72891b921117cf752ce11eb1c0512a14268a43c1 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 19 Jan 2026 13:21:35 +0100 Subject: [PATCH 109/131] Undo incorrect return_dict=True change in deepseek_vl_hybrid I added return_dict to get_low_res_image_features and get_high_res_image_features calls, but these methods already set return_dict automatically --- .../models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py | 4 ++-- .../models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py index b53c958a011c..cbc6e69843db 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py @@ -300,8 +300,8 @@ def get_image_features( high_res_pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithHighResVisionEncodings: - low_res_outputs = self.get_low_res_image_features(pixel_values, return_dict=True, **kwargs) - high_res_outputs = self.get_high_res_image_features(high_res_pixel_values, return_dict=True, **kwargs) + low_res_outputs = self.get_low_res_image_features(pixel_values, **kwargs) + high_res_outputs = self.get_high_res_image_features(high_res_pixel_values, **kwargs) image_features = self.aligner(low_res_outputs.last_hidden_state, high_res_outputs.last_hidden_state) return BaseModelOutputWithHighResVisionEncodings( diff --git a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py index 13bcddf41a1f..a4bbca85b829 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py @@ -310,8 +310,8 @@ def get_image_features( high_res_pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithHighResVisionEncodings: - low_res_outputs = self.get_low_res_image_features(pixel_values, return_dict=True, **kwargs) - high_res_outputs = self.get_high_res_image_features(high_res_pixel_values, return_dict=True, **kwargs) + low_res_outputs = self.get_low_res_image_features(pixel_values, **kwargs) + high_res_outputs = self.get_high_res_image_features(high_res_pixel_values, **kwargs) image_features = self.aligner(low_res_outputs.last_hidden_state, high_res_outputs.last_hidden_state) return BaseModelOutputWithHighResVisionEncodings( From 0d61f66479ddb48f3ec8256cc966cfdf97ea7d2a Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 19 Jan 2026 13:37:04 +0100 Subject: [PATCH 110/131] Revert accidental metaclip import change --- src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py b/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py index 6db36199dca3..ae3a682fdb58 100644 --- a/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py +++ b/src/transformers/models/metaclip_2/convert_metaclip_2_to_hf.py @@ -25,6 +25,7 @@ # Import MetaCLIP modules from src.mini_clip.factory import create_model_and_transforms + from transformers import ( AutoTokenizer, CLIPImageProcessor, From fa32eff46f5f8b27422b33022381c359e891fc3e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 19 Jan 2026 13:51:12 +0100 Subject: [PATCH 111/131] Adopt **vision_outputs in instructblip, but mess remains --- .../models/instructblip/modeling_instructblip.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index 4efd7091ec40..78ab67d2c09a 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -1225,14 +1225,7 @@ def get_image_features( return_dict=True, **kwargs, ) - vision_outputs = BaseModelOutputWithVisionQformerOutputs( - last_hidden_state=vision_outputs.last_hidden_state, - pooler_output=vision_outputs.pooler_output, - hidden_states=vision_outputs.hidden_states, - attentions=vision_outputs.attentions, - vision_outputs=vision_outputs, - qformer_outputs=None, - ) + vision_outputs = BaseModelOutputWithVisionQformerOutputs(**vision_outputs, vision_outputs=vision_outputs) image_embeds = vision_outputs[0] # step 2: forward the query tokens through the QFormer, using the image embeddings for cross-attention From a1e67675a3dcf338bbe9649edbf44f8b27fba829 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 11:03:47 +0100 Subject: [PATCH 112/131] Avoid kwargs["output_hidden_states"] = True in get_..._features methods --- src/transformers/models/aria/modeling_aria.py | 3 ++- src/transformers/models/aria/modular_aria.py | 3 ++- .../models/aya_vision/modeling_aya_vision.py | 9 +++++++-- .../models/aya_vision/modular_aya_vision.py | 9 +++++++-- .../modeling_deepseek_vl_hybrid.py | 9 +++++++-- .../modular_deepseek_vl_hybrid.py | 9 +++++++-- .../models/internvl/modeling_internvl.py | 10 +++++++--- .../models/internvl/modular_internvl.py | 10 +++++++--- .../models/llava/modeling_llava.py | 9 +++++++-- .../models/llava_next/modeling_llava_next.py | 9 +++++++-- .../modeling_llava_next_video.py | 18 ++++++++++++++---- .../modular_llava_next_video.py | 18 ++++++++++++++---- .../modeling_llava_onevision.py | 18 ++++++++++++++---- .../llava_onevision/modular_llava_onevision.py | 18 ++++++++++++++---- .../models/mistral3/modeling_mistral3.py | 10 ++++++++-- .../models/mistral3/modular_mistral3.py | 10 ++++++++-- .../models/video_llava/modeling_video_llava.py | 18 ++++++++++++++---- .../models/vipllava/modeling_vipllava.py | 9 +++++++-- .../models/vipllava/modular_vipllava.py | 9 +++++++-- 19 files changed, 160 insertions(+), 48 deletions(-) diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index b346706be0c6..d9ebe86dbda2 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -924,16 +924,17 @@ def get_image_features( pixel_values: torch.FloatTensor, pixel_mask: torch.FloatTensor | None = None, vision_feature_layer: int = -1, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) patch_attention_mask = self._create_patch_attention_mask(pixel_mask) - kwargs["output_hidden_states"] = True image_outputs = self.vision_tower( pixel_values, patch_attention_mask=patch_attention_mask, + output_hidden_states=True, # Ignore arg on purpose return_dict=True, **kwargs, ) diff --git a/src/transformers/models/aria/modular_aria.py b/src/transformers/models/aria/modular_aria.py index 90701f9ac6a1..bac4508364ca 100644 --- a/src/transformers/models/aria/modular_aria.py +++ b/src/transformers/models/aria/modular_aria.py @@ -1268,16 +1268,17 @@ def get_image_features( pixel_values: torch.FloatTensor, pixel_mask: torch.FloatTensor | None = None, vision_feature_layer: int = -1, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: vision_feature_layer = ( vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer ) patch_attention_mask = self._create_patch_attention_mask(pixel_mask) - kwargs["output_hidden_states"] = True image_outputs = self.vision_tower( pixel_values, patch_attention_mask=patch_attention_mask, + output_hidden_states=True, # Ignore arg on purpose return_dict=True, **kwargs, ) diff --git a/src/transformers/models/aya_vision/modeling_aya_vision.py b/src/transformers/models/aya_vision/modeling_aya_vision.py index 9e21a46770b5..5f32c87c0abd 100644 --- a/src/transformers/models/aya_vision/modeling_aya_vision.py +++ b/src/transformers/models/aya_vision/modeling_aya_vision.py @@ -188,6 +188,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: vision_feature_layer = ( @@ -204,8 +205,12 @@ def get_image_features( kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. - kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + image_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/aya_vision/modular_aya_vision.py b/src/transformers/models/aya_vision/modular_aya_vision.py index d30acc180e1e..0c08c24564f8 100644 --- a/src/transformers/models/aya_vision/modular_aya_vision.py +++ b/src/transformers/models/aya_vision/modular_aya_vision.py @@ -113,6 +113,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: vision_feature_layer = ( @@ -129,8 +130,12 @@ def get_image_features( kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. - kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + image_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py index cbc6e69843db..7cc8b629a465 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modeling_deepseek_vl_hybrid.py @@ -402,10 +402,15 @@ def forward( def get_low_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): return self.vision_model(pixel_values, return_dict=True, **kwargs) - def get_high_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): - kwargs["output_hidden_states"] = True + def get_high_res_image_features( + self, + pixel_values: torch.FloatTensor, + output_hidden_states: bool | None = None, + **kwargs: Unpack[TransformersKwargs], + ): high_res_outputs = self.high_res_vision_model( pixel_values=pixel_values, + output_hidden_states=True, # Ignore arg on purpose return_dict=True, **kwargs, ) diff --git a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py index 43c754423c70..8488833e6aaf 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py @@ -278,10 +278,15 @@ def __init__(self, config): def get_low_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): return self.vision_model(pixel_values, return_dict=True, **kwargs) - def get_high_res_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): - kwargs["output_hidden_states"] = True + def get_high_res_image_features( + self, + pixel_values: torch.FloatTensor, + output_hidden_states: bool | None = None, + **kwargs: Unpack[TransformersKwargs], + ): high_res_outputs = self.high_res_vision_model( pixel_values=pixel_values, + output_hidden_states=True, # Ignore arg on purpose return_dict=True, **kwargs, ) diff --git a/src/transformers/models/internvl/modeling_internvl.py b/src/transformers/models/internvl/modeling_internvl.py index 46c32cd8a86e..9f94e5dfa425 100644 --- a/src/transformers/models/internvl/modeling_internvl.py +++ b/src/transformers/models/internvl/modeling_internvl.py @@ -553,6 +553,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -572,9 +573,12 @@ def get_image_features( pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio - if vision_feature_layer != -1: - kwargs["output_hidden_states"] = True - vision_outputs = self.vision_tower(pixel_values=pixel_values, return_dict=True, **kwargs) + vision_outputs = self.vision_tower( + pixel_values=pixel_values, + output_hidden_states=vision_feature_layer != -1, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) if vision_feature_layer == -1: vision_features = vision_outputs.last_hidden_state else: diff --git a/src/transformers/models/internvl/modular_internvl.py b/src/transformers/models/internvl/modular_internvl.py index 705790705150..7c0ef9a695a9 100644 --- a/src/transformers/models/internvl/modular_internvl.py +++ b/src/transformers/models/internvl/modular_internvl.py @@ -493,6 +493,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -512,9 +513,12 @@ def get_image_features( pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio - if vision_feature_layer != -1: - kwargs["output_hidden_states"] = True - vision_outputs = self.vision_tower(pixel_values=pixel_values, return_dict=True, **kwargs) + vision_outputs = self.vision_tower( + pixel_values=pixel_values, + output_hidden_states=vision_feature_layer != -1, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) if vision_feature_layer == -1: vision_features = vision_outputs.last_hidden_state else: diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index f913fe569c6b..5f538dcf7a0e 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -154,6 +154,7 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: vision_feature_layer = ( @@ -170,8 +171,12 @@ def get_image_features( kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. - kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + image_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index 5518cd1a2af5..5ad8a188b76e 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -353,6 +353,7 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -394,8 +395,12 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + image_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index d0af89d56921..3ffd368be5bd 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -405,6 +405,7 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -446,8 +447,12 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + image_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -621,6 +626,7 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -645,8 +651,12 @@ def get_video_features( batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channels, height, width) - kwargs["output_hidden_states"] = True - video_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + video_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index efcc694a8831..be50698d1498 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -279,6 +279,7 @@ def get_image_features( image_sizes: torch.Tensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -320,8 +321,12 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + image_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -354,6 +359,7 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -378,8 +384,12 @@ def get_video_features( batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channels, height, width) - kwargs["output_hidden_states"] = True - video_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + video_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index ede15e7a52db..47219154d517 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -364,6 +364,7 @@ def get_image_features( vision_feature_select_strategy: str | None = None, vision_aspect_ratio: str | None = None, batch_num_images: torch.LongTensor | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -410,8 +411,12 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + image_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -597,6 +602,7 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -621,8 +627,12 @@ def get_video_features( batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) - kwargs["output_hidden_states"] = True - vision_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + vision_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index 35edc9eea81e..5b2d2c04d0da 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -326,6 +326,7 @@ def get_image_features( vision_feature_select_strategy: str | None = None, vision_aspect_ratio: str | None = None, batch_num_images: torch.LongTensor | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -372,8 +373,12 @@ def get_image_features( # otherwise has to be stacked from list of (num_patches, num_channels, height, width) raise ValueError(f"pixel_values of shape {pixel_values.shape}, expect to be of 4 or 5 dimensions") - kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + image_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): @@ -406,6 +411,7 @@ def get_video_features( pixel_values: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -430,8 +436,12 @@ def get_video_features( batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) - kwargs["output_hidden_states"] = True - vision_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + vision_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index e4d95236f9f4..42f5b1999664 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -223,6 +223,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, vision_feature_layer: int | list[int] | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: vision_feature_layer = ( @@ -231,8 +232,13 @@ def get_image_features( kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. - kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, return_dict=True, **kwargs) + image_outputs = self.vision_tower( + pixel_values, + image_sizes=image_sizes, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): diff --git a/src/transformers/models/mistral3/modular_mistral3.py b/src/transformers/models/mistral3/modular_mistral3.py index e31626ff93df..58d5b3847027 100644 --- a/src/transformers/models/mistral3/modular_mistral3.py +++ b/src/transformers/models/mistral3/modular_mistral3.py @@ -128,6 +128,7 @@ def get_image_features( pixel_values: torch.FloatTensor, image_sizes: torch.Tensor, vision_feature_layer: int | list[int] | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: vision_feature_layer = ( @@ -136,8 +137,13 @@ def get_image_features( kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. - kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, image_sizes=image_sizes, return_dict=True, **kwargs) + image_outputs = self.vision_tower( + pixel_values, + image_sizes=image_sizes, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them if isinstance(vision_feature_layer, int): diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index fc336cb41ff2..f5e13fe89137 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -178,6 +178,7 @@ def get_image_features( pixel_values_images: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -203,8 +204,12 @@ def get_image_features( if vision_feature_select_strategy not in ["default", "full"]: raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}") - kwargs["output_hidden_states"] = True - image_outputs = self.image_tower(pixel_values_images, return_dict=True, **kwargs) + image_outputs = self.image_tower( + pixel_values_images, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them @@ -232,6 +237,7 @@ def get_video_features( self, pixel_values_videos: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -249,8 +255,12 @@ def get_video_features( batch_size_vid, num_frames, channels, height, width = pixel_values_videos.shape pixel_values = pixel_values_videos.reshape(batch_size_vid * num_frames, channels, height, width) - kwargs["output_hidden_states"] = True - video_outputs = self.video_tower(pixel_values, return_dict=True, **kwargs) + video_outputs = self.video_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If we have one vision feature layer, return the corresponding hidden states, # otherwise, select the hidden states of each feature layer and concatenate them diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index d3ceecff262f..ab46ae5814cf 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -157,6 +157,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layers: int | list[int] | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -169,8 +170,12 @@ def get_image_features( vision_feature_layers = ( vision_feature_layers if vision_feature_layers is not None else self.config.vision_feature_layers ) - kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + image_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If multiple feature layers are provided (which is usually the case) # then the image features are concatenated after the CLS is removed. diff --git a/src/transformers/models/vipllava/modular_vipllava.py b/src/transformers/models/vipllava/modular_vipllava.py index e8baefda48a6..3b9a2dfdf284 100644 --- a/src/transformers/models/vipllava/modular_vipllava.py +++ b/src/transformers/models/vipllava/modular_vipllava.py @@ -79,6 +79,7 @@ def get_image_features( self, pixel_values: torch.FloatTensor, vision_feature_layers: int | list[int] | None = None, + output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -91,8 +92,12 @@ def get_image_features( vision_feature_layers = ( vision_feature_layers if vision_feature_layers is not None else self.config.vision_feature_layers ) - kwargs["output_hidden_states"] = True - image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) + image_outputs = self.vision_tower( + pixel_values, + output_hidden_states=True, # Ignore arg on purpose + return_dict=True, + **kwargs, + ) # If multiple feature layers are provided (which is usually the case) # then the image features are concatenated after the CLS is removed. From d9001cc845095821fcc58cc9bc2bfd6dddd4cc1a Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 12:08:19 +0100 Subject: [PATCH 113/131] Update check_model_inputs to default vision args based on config --- src/transformers/models/aria/modeling_aria.py | 5 +- src/transformers/models/aria/modular_aria.py | 6 +- .../models/aya_vision/modeling_aya_vision.py | 36 +--------- .../models/aya_vision/modular_aya_vision.py | 25 +------ .../models/fast_vlm/modeling_fast_vlm.py | 36 ++-------- .../models/fast_vlm/modular_fast_vlm.py | 36 ++-------- .../models/got_ocr2/modeling_got_ocr2.py | 18 ++--- .../models/got_ocr2/modular_got_ocr2.py | 6 ++ .../models/internvl/modeling_internvl.py | 34 ++-------- .../models/internvl/modular_internvl.py | 23 +------ .../lighton_ocr/modeling_lighton_ocr.py | 3 +- .../models/llama4/modeling_llama4.py | 10 +-- .../models/llava/modeling_llava.py | 39 ++--------- .../models/llava_next/modeling_llava_next.py | 30 +-------- .../modeling_llava_next_video.py | 45 ++----------- .../modular_llava_next_video.py | 45 +++---------- .../modeling_llava_onevision.py | 54 ++------------- .../modular_llava_onevision.py | 59 ++--------------- .../models/mistral3/modeling_mistral3.py | 5 +- .../video_llava/modeling_video_llava.py | 35 ++-------- .../models/vipllava/modeling_vipllava.py | 3 +- src/transformers/utils/generic.py | 66 +++++++++++-------- 22 files changed, 122 insertions(+), 497 deletions(-) diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index d9ebe86dbda2..73646c709d38 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -915,7 +915,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -927,9 +927,6 @@ def get_image_features( output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) patch_attention_mask = self._create_patch_attention_mask(pixel_mask) image_outputs = self.vision_tower( pixel_values, diff --git a/src/transformers/models/aria/modular_aria.py b/src/transformers/models/aria/modular_aria.py index bac4508364ca..cd84941a9c34 100644 --- a/src/transformers/models/aria/modular_aria.py +++ b/src/transformers/models/aria/modular_aria.py @@ -41,6 +41,7 @@ from ...processing_utils import ImagesKwargs, MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack from ...tokenization_python import PreTokenizedInput, TextInput from ...utils import TensorType, TransformersKwargs, auto_docstring, can_return_tuple, logging +from ...utils.generic import check_model_inputs from ..auto import CONFIG_MAPPING, AutoConfig, AutoTokenizer from ..llama.configuration_llama import LlamaConfig from ..llama.modeling_llama import ( @@ -1259,7 +1260,7 @@ def _create_patch_attention_mask(self, pixel_mask): ) return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool() - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -1271,9 +1272,6 @@ def get_image_features( output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) patch_attention_mask = self._create_patch_attention_mask(pixel_mask) image_outputs = self.vision_tower( pixel_values, diff --git a/src/transformers/models/aya_vision/modeling_aya_vision.py b/src/transformers/models/aya_vision/modeling_aya_vision.py index 5f32c87c0abd..5263e625bf97 100644 --- a/src/transformers/models/aya_vision/modeling_aya_vision.py +++ b/src/transformers/models/aya_vision/modeling_aya_vision.py @@ -29,7 +29,7 @@ from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils import TransformersKwargs, auto_docstring from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_aya_vision import AyaVisionConfig @@ -179,7 +179,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -191,18 +191,6 @@ def get_image_features( output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - - if vision_feature_select_strategy not in ["default", "full"]: - raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}") - kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. image_outputs = self.vision_tower( @@ -269,15 +257,6 @@ def forward( cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | AyaVisionModelOutputWithPast: - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -360,7 +339,7 @@ def get_image_features( **kwargs, ) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -414,15 +393,6 @@ def forward( >>> gen_tokens = model.generate(**inputs, max_new_tokens=300, do_sample=True, temperature=0.3) >>> processor.tokenizer.decode(gen_tokens[0][inputs.input_ids.shape[1]:], skip_special_tokens=True) ```""" - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - outputs = self.model( input_ids=input_ids, pixel_values=pixel_values, diff --git a/src/transformers/models/aya_vision/modular_aya_vision.py b/src/transformers/models/aya_vision/modular_aya_vision.py index 0c08c24564f8..378a826e1f2e 100644 --- a/src/transformers/models/aya_vision/modular_aya_vision.py +++ b/src/transformers/models/aya_vision/modular_aya_vision.py @@ -28,7 +28,7 @@ from ...cache_utils import Cache from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging +from ...utils import TransformersKwargs, auto_docstring, logging from ...utils.generic import check_model_inputs from .configuration_aya_vision import AyaVisionConfig @@ -104,7 +104,7 @@ class AyaVisionModelOutputWithPast(LlavaModelOutputWithPast): class AyaVisionModel(LlavaModel): # Unlike LLaVA, the model doesn't have to deal with Pixtral-style image states - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -116,18 +116,6 @@ def get_image_features( output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - - if vision_feature_select_strategy not in ["default", "full"]: - raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}") - kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. image_outputs = self.vision_tower( @@ -170,15 +158,6 @@ def forward( cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | AyaVisionModelOutputWithPast: - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/fast_vlm/modeling_fast_vlm.py b/src/transformers/models/fast_vlm/modeling_fast_vlm.py index 25865665b86f..15377dc20f1f 100644 --- a/src/transformers/models/fast_vlm/modeling_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modeling_fast_vlm.py @@ -29,7 +29,8 @@ from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils import TransformersKwargs, auto_docstring +from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_fast_vlm import FastVlmConfig @@ -113,7 +114,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -133,15 +134,6 @@ def get_image_features( The feature selection strategy used to select the vision feature from the vision backbone. Only "full" supported. """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - kwargs = {k: v for k, v in kwargs.items() if v is not None} image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) @@ -177,7 +169,7 @@ def get_placeholder_mask( ) return special_image_mask - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -199,15 +191,6 @@ def forward( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Only "full" supported. """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -314,7 +297,7 @@ def get_image_features( **kwargs, ) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -376,15 +359,6 @@ def forward( >>> print(processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]) system\n You are a helpful assistant.\n user\n What are these?\n assistant\n The image depicts a traditional Chinese street... ```""" - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - outputs = self.model( input_ids=input_ids, pixel_values=pixel_values, diff --git a/src/transformers/models/fast_vlm/modular_fast_vlm.py b/src/transformers/models/fast_vlm/modular_fast_vlm.py index 4d2b18d4127f..fbe891b334fd 100644 --- a/src/transformers/models/fast_vlm/modular_fast_vlm.py +++ b/src/transformers/models/fast_vlm/modular_fast_vlm.py @@ -20,7 +20,8 @@ from ...configuration_utils import PreTrainedConfig from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils import TransformersKwargs, auto_docstring +from ...utils.generic import check_model_inputs from ..auto import CONFIG_MAPPING from ..llava.configuration_llava import LlavaConfig from ..llava.modeling_llava import ( @@ -169,7 +170,7 @@ class FastVlmModel(LlavaModel): def __init__(self, config: FastVlmConfig): super().__init__(config) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -189,15 +190,6 @@ def get_image_features( The feature selection strategy used to select the vision feature from the vision backbone. Only "full" supported. """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - kwargs = {k: v for k, v in kwargs.items() if v is not None} image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs) @@ -209,7 +201,7 @@ def get_image_features( return image_outputs - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -231,15 +223,6 @@ def forward( vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Only "full" supported. """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -289,7 +272,7 @@ class FastVlmCausalLMOutputWithPast(LlavaCausalLMOutputWithPast): class FastVlmForConditionalGeneration(LlavaForConditionalGeneration): _checkpoint_conversion_mapping = {} - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -351,15 +334,6 @@ def forward( >>> print(processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]) system\n You are a helpful assistant.\n user\n What are these?\n assistant\n The image depicts a traditional Chinese street... ```""" - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - outputs = self.model( input_ids=input_ids, pixel_values=pixel_values, diff --git a/src/transformers/models/got_ocr2/modeling_got_ocr2.py b/src/transformers/models/got_ocr2/modeling_got_ocr2.py index f3b2bc20fdba..b96d315b1d0d 100644 --- a/src/transformers/models/got_ocr2/modeling_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modeling_got_ocr2.py @@ -25,8 +25,6 @@ import torch.nn as nn import torch.nn.functional as F -from transformers.utils.generic import check_model_inputs - from ... import initialization as init from ...activations import ACT2FN from ...cache_utils import Cache @@ -36,6 +34,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_got_ocr2 import GotOcr2Config, GotOcr2VisionConfig @@ -588,7 +587,7 @@ def get_placeholder_mask( ) return special_image_mask - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -680,18 +679,9 @@ def get_output_embeddings(self) -> nn.Module: @auto_docstring def get_image_features( - self, - pixel_values: torch.FloatTensor, - vision_feature_layer: int | list[int] | None = None, - vision_feature_select_strategy: str | None = None, - **kwargs: Unpack[TransformersKwargs], + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] ) -> tuple | BaseModelOutputWithPooling: - return self.model.get_image_features( - pixel_values=pixel_values, - vision_feature_layer=vision_feature_layer, - vision_feature_select_strategy=vision_feature_select_strategy, - **kwargs, - ) + return self.model.get_image_features(pixel_values=pixel_values, **kwargs) @can_return_tuple @auto_docstring diff --git a/src/transformers/models/got_ocr2/modular_got_ocr2.py b/src/transformers/models/got_ocr2/modular_got_ocr2.py index 8432c3f4d19d..cf7a3e584549 100644 --- a/src/transformers/models/got_ocr2/modular_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modular_got_ocr2.py @@ -470,6 +470,12 @@ def forward( image_hidden_states=outputs.image_hidden_states, ) + @auto_docstring + def get_image_features( + self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs] + ) -> tuple | BaseModelOutputWithPooling: + return self.model.get_image_features(pixel_values=pixel_values, **kwargs) + __all__ = [ "GotOcr2VisionConfig", diff --git a/src/transformers/models/internvl/modeling_internvl.py b/src/transformers/models/internvl/modeling_internvl.py index 9f94e5dfa425..c6dba799b1e8 100644 --- a/src/transformers/models/internvl/modeling_internvl.py +++ b/src/transformers/models/internvl/modeling_internvl.py @@ -35,7 +35,7 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, torch_int +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, torch_int from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_internvl import InternVLConfig, InternVLVisionConfig @@ -544,7 +544,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -562,14 +562,6 @@ def get_image_features( vision_feature_layer (`int` or `list[int]`): Layer index or list of layer indices to extract features from. """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio @@ -630,7 +622,7 @@ def get_placeholder_mask( ) return special_image_mask - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -645,15 +637,6 @@ def forward( cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | InternVLModelOutputWithPast: - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -800,7 +783,7 @@ def get_image_features( **kwargs, ) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -853,15 +836,6 @@ def forward( >>> print(processor.decode(generate_ids[0, inputs["input_ids"].shape[1] :], skip_special_tokens=True)) The images depict the Statue of Liberty and the Golden Gate Bridge. ```""" - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - outputs = self.model( input_ids=input_ids, pixel_values=pixel_values, diff --git a/src/transformers/models/internvl/modular_internvl.py b/src/transformers/models/internvl/modular_internvl.py index 7c0ef9a695a9..e0477953d98f 100644 --- a/src/transformers/models/internvl/modular_internvl.py +++ b/src/transformers/models/internvl/modular_internvl.py @@ -27,7 +27,7 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, torch_int +from ...utils import TransformersKwargs, auto_docstring, torch_int from ...utils.generic import check_model_inputs from ..clip.modeling_clip import CLIPMLP from ..janus.modeling_janus import JanusVisionAttention @@ -484,7 +484,7 @@ def pixel_shuffle(self, vision_features: torch.Tensor, scale_factor: float = 0.5 return vision_features - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -502,14 +502,6 @@ def get_image_features( vision_feature_layer (`int` or `list[int]`): Layer index or list of layer indices to extract features from. """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio @@ -546,7 +538,7 @@ def get_image_features( return vision_outputs - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -561,15 +553,6 @@ def forward( cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | InternVLModelOutputWithPast: - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py b/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py index 74e1ef893dbf..30aac94030c4 100644 --- a/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py +++ b/src/transformers/models/lighton_ocr/modeling_lighton_ocr.py @@ -29,6 +29,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_lighton_ocr import LightOnOcrConfig @@ -333,7 +334,7 @@ def get_image_features( ) -> tuple | BaseModelOutputWithPooling: return self.model.get_image_features(pixel_values=pixel_values, image_sizes=image_sizes, **kwargs) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, diff --git a/src/transformers/models/llama4/modeling_llama4.py b/src/transformers/models/llama4/modeling_llama4.py index 6c149d4a6dd2..83a21d700568 100644 --- a/src/transformers/models/llama4/modeling_llama4.py +++ b/src/transformers/models/llama4/modeling_llama4.py @@ -1204,7 +1204,7 @@ def set_decoder(self, decoder): def get_decoder(self): return self.language_model.get_decoder() - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring(custom_intro="Obtains image last hidden states from the vision tower and apply al projection.") def get_image_features( self, @@ -1219,8 +1219,6 @@ def get_image_features( The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` """ - if vision_feature_select_strategy not in ["default", "full"]: - raise ValueError(f"Unexpected select feature strategy: {vision_feature_select_strategy}") kwargs = {k: v for k, v in kwargs.items() if v is not None} return self.vision_model(pixel_values, **kwargs) @@ -1247,6 +1245,7 @@ def get_placeholder_mask( ) return special_image_mask + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -1299,11 +1298,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_config.vision_feature_select_strategy - ) if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 5f538dcf7a0e..2588b1cd0093 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -24,7 +24,8 @@ from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging +from ...utils import TransformersKwargs, auto_docstring, logging +from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_llava import LlavaConfig @@ -145,7 +146,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -157,18 +158,6 @@ def get_image_features( output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - - if vision_feature_select_strategy not in ["default", "full"]: - raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}") - kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. image_outputs = self.vision_tower( @@ -230,7 +219,7 @@ def get_placeholder_mask( ) return special_image_mask - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -246,15 +235,6 @@ def forward( image_sizes: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | LlavaModelOutputWithPast: - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -337,7 +317,7 @@ def get_image_features( **kwargs, ) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -382,15 +362,6 @@ def forward( >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "USER: \nWhat's the content of the image? ASSISTANT: The image features a busy city street with a stop sign prominently displayed" ```""" - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - outputs = self.model( input_ids=input_ids, pixel_values=pixel_values, diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index 5ad8a188b76e..9cc6b1aee4e0 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -30,6 +30,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging +from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_llava_next import LlavaNextConfig @@ -343,7 +344,7 @@ def pack_image_features(self, image_features, image_sizes, vision_feature_select feature_lens = torch.tensor(feature_lens, dtype=torch.long, device=image_features[0].device) return new_image_features, feature_lens - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -369,15 +370,6 @@ def get_image_features( The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - # ! infer image_num_patches from image_sizes image_num_patches = [ image_size_to_num_patches( @@ -480,14 +472,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -599,7 +583,7 @@ def get_image_features( **kwargs, ) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -656,14 +640,6 @@ def forward( output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) outputs = self.model( input_ids, diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 3ffd368be5bd..91d44b6255bb 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -34,7 +34,8 @@ from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging +from ...utils import TransformersKwargs, auto_docstring, logging +from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_llava_next_video import LlavaNextVideoConfig @@ -395,7 +396,7 @@ def pack_image_features(self, image_features, image_sizes, vision_feature_select feature_lens = torch.tensor(feature_lens, dtype=torch.long, device=image_features[0].device) return new_image_features, feature_lens - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -421,15 +422,6 @@ def get_image_features( The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - # ! infer image_num_patches from image_sizes image_num_patches = [ image_size_to_num_patches( @@ -516,7 +508,7 @@ def get_placeholder_mask( return special_image_mask, special_video_mask - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -548,14 +540,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - self.vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - self.vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -617,7 +601,7 @@ def forward( video_hidden_states=video_features if pixel_values_videos is not None else None, ) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains video last hidden states from the vision tower and apply multimodal projection." ) @@ -640,15 +624,6 @@ def get_video_features( The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channels, height, width) video_outputs = self.vision_tower( @@ -746,7 +721,7 @@ def get_image_features( **kwargs, ) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -837,14 +812,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) outputs = self.model( input_ids=input_ids, diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index be50698d1498..aad09e7df8a5 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -33,7 +33,8 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack -from ...utils import auto_docstring, can_return_tuple, logging +from ...utils import auto_docstring, logging +from ...utils.generic import check_model_inputs from ..auto import CONFIG_MAPPING, AutoConfig @@ -269,7 +270,7 @@ def __init__(self, config: LlavaNextVideoConfig, **super_kwargs): self.vision_resampler = LlavaNextVideoPooler(config) self.post_init() - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -295,15 +296,6 @@ def get_image_features( The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - # ! infer image_num_patches from image_sizes image_num_patches = [ image_size_to_num_patches( @@ -350,7 +342,7 @@ def get_image_features( return image_outputs - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains video last hidden states from the vision tower and apply multimodal projection." ) @@ -373,15 +365,6 @@ def get_video_features( The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.reshape(batch_size * frames, channels, height, width) video_outputs = self.vision_tower( @@ -450,6 +433,8 @@ def get_placeholder_mask( return special_image_mask, special_video_mask + @check_model_inputs(tie_last_hidden_states=False) + @auto_docstring def forward( self, input_ids: torch.LongTensor | None = None, @@ -474,14 +459,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - self.vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - self.vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -571,6 +548,8 @@ def get_video_features( **kwargs, ) + @check_model_inputs(tie_last_hidden_states=False) + @auto_docstring def forward( self, input_ids: torch.LongTensor | None = None, @@ -660,14 +639,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) outputs = self.model( input_ids=input_ids, diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index 47219154d517..46c6e109fc6a 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -34,7 +34,8 @@ from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils import TransformersKwargs, auto_docstring +from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_llava_onevision import LlavaOnevisionConfig @@ -352,7 +353,7 @@ def pack_image_features(self, image_features, image_sizes, image_newline=None, v feature_lens = torch.tensor(feature_lens, dtype=torch.long, device=image_features[0].device) return new_image_features, feature_lens - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -375,18 +376,6 @@ def get_image_features( batch_num_images (`torch.LongTensor`, *optional*): Number of images in each sample. """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - vision_aspect_ratio = ( - vision_aspect_ratio if vision_aspect_ratio is not None else self.config.vision_aspect_ratio - ) - # ! infer image_num_patches from image_sizes if batch_num_images is None: # treat this as a single-image case for backward compatibility @@ -480,7 +469,7 @@ def get_placeholder_mask( return special_image_mask, special_video_mask - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -518,17 +507,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - vision_aspect_ratio = ( - vision_aspect_ratio if vision_aspect_ratio is not None else self.config.vision_aspect_ratio - ) if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -593,7 +571,7 @@ def forward( video_hidden_states=video_features if pixel_values_videos is not None else None, ) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains video last hidden states from the vision tower, apply multimodal projection and pooling." ) @@ -616,15 +594,6 @@ def get_video_features( The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) vision_outputs = self.vision_tower( @@ -734,7 +703,7 @@ def get_image_features( **kwargs, ) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -807,17 +776,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - vision_aspect_ratio = ( - vision_aspect_ratio if vision_aspect_ratio is not None else self.config.vision_aspect_ratio - ) outputs = self.model( input_ids=input_ids, diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index 5b2d2c04d0da..6c49d02662f2 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -47,12 +47,8 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack -from ...utils import ( - TensorType, - auto_docstring, - can_return_tuple, - logging, -) +from ...utils import TensorType, auto_docstring, logging +from ...utils.generic import check_model_inputs from .image_processing_llava_onevision import LlavaOnevisionImageProcessorKwargs @@ -314,7 +310,7 @@ def apply_pooling(self, image_features): image_features = image_features.view(batch_frames, -1, dim) return image_features - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -337,18 +333,6 @@ def get_image_features( batch_num_images (`torch.LongTensor`, *optional*): Number of images in each sample. """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - vision_aspect_ratio = ( - vision_aspect_ratio if vision_aspect_ratio is not None else self.config.vision_aspect_ratio - ) - # ! infer image_num_patches from image_sizes if batch_num_images is None: # treat this as a single-image case for backward compatibility @@ -402,7 +386,7 @@ def get_image_features( return image_outputs - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains video last hidden states from the vision tower, apply multimodal projection and pooling." ) @@ -425,15 +409,6 @@ def get_video_features( The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - batch_size, frames, channels, height, width = pixel_values.shape pixel_values = pixel_values.view(batch_size * frames, channels, height, width) vision_outputs = self.vision_tower( @@ -461,6 +436,8 @@ def get_video_features( return vision_outputs + @check_model_inputs(tie_last_hidden_states=False) + @auto_docstring def forward( self, input_ids: torch.LongTensor | None = None, @@ -497,17 +474,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - vision_aspect_ratio = ( - vision_aspect_ratio if vision_aspect_ratio is not None else self.config.vision_aspect_ratio - ) if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -574,7 +540,7 @@ def forward( class LlavaOnevisionForConditionalGeneration(LlavaNextVideoForConditionalGeneration): - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -647,17 +613,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - vision_aspect_ratio = ( - vision_aspect_ratio if vision_aspect_ratio is not None else self.config.vision_aspect_ratio - ) outputs = self.model( input_ids=input_ids, diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index 42f5b1999664..a25295aea468 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -32,6 +32,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_mistral3 import Mistral3Config @@ -281,7 +282,7 @@ def get_placeholder_mask( ) return special_image_mask - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -394,7 +395,7 @@ def get_image_features( **kwargs, ) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index f5e13fe89137..cedf092af1b0 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -27,6 +27,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging +from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_video_llava import VideoLlavaConfig @@ -169,7 +170,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -192,18 +193,6 @@ def get_image_features( The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - - if vision_feature_select_strategy not in ["default", "full"]: - raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}") - image_outputs = self.image_tower( pixel_values_images, output_hidden_states=True, # Ignore arg on purpose @@ -315,7 +304,7 @@ def get_placeholder_mask( return special_image_mask, special_video_mask - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -346,14 +335,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -463,7 +444,7 @@ def get_image_features( **kwargs, ) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -565,14 +546,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) outputs = self.model( input_ids=input_ids, diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index ab46ae5814cf..73c168585c90 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -30,6 +30,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_vipllava import VipLlavaConfig @@ -331,7 +332,7 @@ def get_image_features( pixel_values=pixel_values, vision_feature_layers=vision_feature_layers, **kwargs ) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, diff --git a/src/transformers/utils/generic.py b/src/transformers/utils/generic.py index 53aee7450972..e530b5f9acc8 100644 --- a/src/transformers/utils/generic.py +++ b/src/transformers/utils/generic.py @@ -875,34 +875,46 @@ def check_model_inputs(func=None, *, tie_last_hidden_states=True): def wrapped_fn(func): @wraps(func) def wrapper(self, *args, **kwargs): - use_cache_arg_index = None - if "use_cache" in func.__code__.co_varnames: - use_cache_arg_index = func.__code__.co_varnames.index("use_cache") - 1 # -1 for self - - if ( - use_cache_arg_index is not None - and len(args) > use_cache_arg_index - and args[use_cache_arg_index] is not None - ): - use_cache = args[use_cache_arg_index] - elif kwargs.get("use_cache") is not None: - use_cache = kwargs["use_cache"] - else: - use_cache = getattr(self.config, "use_cache", None) - - if use_cache is not None: - if getattr(self, "gradient_checkpointing", False) and self.training and use_cache: - logger.warning_once( - "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`." - ) - use_cache = False - - if use_cache_arg_index is not None and len(args) > use_cache_arg_index: - args = list(args) - args[use_cache_arg_index] = use_cache - args = tuple(args) + args_with_config_defaults = [ + "use_cache", + "vision_feature_layer", + "vision_feature_select_strategy", + "vision_aspect_ratio", + ] + for arg_name in args_with_config_defaults: + arg_index = None + if arg_name in func.__code__.co_varnames: + arg_index = func.__code__.co_varnames.index(arg_name) - 1 # -1 for self + + if arg_index is not None and len(args) > arg_index and args[arg_index] is not None: + arg_value = args[arg_index] + elif kwargs.get(arg_name) is not None: + arg_value = kwargs[arg_name] else: - kwargs["use_cache"] = use_cache + arg_value = getattr(self.config, arg_name, None) + + if arg_value is not None: + # Arg-specific handling + if arg_name == "use_cache": + if getattr(self, "gradient_checkpointing", False) and self.training and arg_value: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`." + ) + arg_value = False + elif arg_name == "vision_feature_select_strategy": + valid_strategies = ["default", "full"] + if arg_value not in valid_strategies: + raise ValueError( + f"`Unexpected select feature strategy: {arg_value}. " + f"Please select from {valid_strategies}." + ) + + if arg_index is not None and len(args) > arg_index: + args = list(args) + args[arg_index] = arg_value + args = tuple(args) + else: + kwargs[arg_name] = arg_value return_dict = kwargs.pop("return_dict", None) if return_dict is None: From 092321671197d878e48b1c89edd154f47bb43a30 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 12:11:50 +0100 Subject: [PATCH 114/131] Unrelated but important: patch set_attn_implementation for Windows idem with set_experts_implementation --- src/transformers/modeling_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 623f20f4c8cb..0072a4d9dbd4 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1909,7 +1909,7 @@ def _can_set_attn_implementation(cls) -> bool: opening the file, but avoids maintaining yet another property flag. """ class_file = sys.modules[cls.__module__].__file__ - with open(class_file, "r") as f: + with open(class_file, "r", encoding="utf8") as f: code = f.read() # heuristic -> if we find those patterns, the model uses the correct interface if re.search(r"class \w+Attention\(nn.Module\)", code): @@ -1927,7 +1927,7 @@ def _can_set_experts_implementation(cls) -> bool: opening the file, but avoids maintaining yet another property flag. """ class_file = sys.modules[cls.__module__].__file__ - with open(class_file, "r") as f: + with open(class_file, "r", encoding="utf8") as f: code = f.read() # heuristic -> if we the use_experts_implementation decorator is used, then we can set it return "@use_experts_implementation" in code From e3b774e36601e4770a505f7de88b8e1e55bb6772 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 12:20:06 +0100 Subject: [PATCH 115/131] Revert output_hidden_states changes on InternVL On this architecture, it seems cleaner to go the `kwargs["output_hidden_states"] = True` route, as a simple `output_hidden_states=vision_feature_layer != -1` prevents setting the `output_hidden_states` to True if requested for downstream use. --- src/transformers/models/internvl/modeling_internvl.py | 10 +++------- src/transformers/models/internvl/modular_internvl.py | 10 +++------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/transformers/models/internvl/modeling_internvl.py b/src/transformers/models/internvl/modeling_internvl.py index c6dba799b1e8..5ca54b55d4e4 100644 --- a/src/transformers/models/internvl/modeling_internvl.py +++ b/src/transformers/models/internvl/modeling_internvl.py @@ -553,7 +553,6 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, - output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -565,12 +564,9 @@ def get_image_features( pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio - vision_outputs = self.vision_tower( - pixel_values=pixel_values, - output_hidden_states=vision_feature_layer != -1, # Ignore arg on purpose - return_dict=True, - **kwargs, - ) + if vision_feature_layer != -1: + kwargs["output_hidden_states"] = True + vision_outputs = self.vision_tower(pixel_values=pixel_values, return_dict=True, **kwargs) if vision_feature_layer == -1: vision_features = vision_outputs.last_hidden_state else: diff --git a/src/transformers/models/internvl/modular_internvl.py b/src/transformers/models/internvl/modular_internvl.py index e0477953d98f..d5ec73e498da 100644 --- a/src/transformers/models/internvl/modular_internvl.py +++ b/src/transformers/models/internvl/modular_internvl.py @@ -493,7 +493,6 @@ def get_image_features( pixel_values: torch.FloatTensor, vision_feature_layer: int | list[int] | None = None, vision_feature_select_strategy: str | None = None, - output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" @@ -505,12 +504,9 @@ def get_image_features( pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility downsample_ratio = self.config.downsample_ratio - vision_outputs = self.vision_tower( - pixel_values=pixel_values, - output_hidden_states=vision_feature_layer != -1, # Ignore arg on purpose - return_dict=True, - **kwargs, - ) + if vision_feature_layer != -1: + kwargs["output_hidden_states"] = True + vision_outputs = self.vision_tower(pixel_values=pixel_values, return_dict=True, **kwargs) if vision_feature_layer == -1: vision_features = vision_outputs.last_hidden_state else: From 37a495c1d8f2ce57d6b850b35c70ebc9e9ad50c3 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 12:27:12 +0100 Subject: [PATCH 116/131] Extend d9001cc (check_model_inputs); remove more vision_feature_layer defaulting --- .../models/mistral3/modeling_mistral3.py | 11 ++--------- .../models/mistral3/modular_mistral3.py | 14 +++++--------- .../models/video_llava/modeling_video_llava.py | 8 ++------ 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index a25295aea468..33af7535f27d 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -31,7 +31,7 @@ from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils import TransformersKwargs, auto_docstring from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_mistral3 import Mistral3Config @@ -215,7 +215,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.language_model.set_input_embeddings(value) - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -227,10 +227,6 @@ def get_image_features( output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. image_outputs = self.vision_tower( @@ -306,9 +302,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/mistral3/modular_mistral3.py b/src/transformers/models/mistral3/modular_mistral3.py index 58d5b3847027..7ebf180b3e3e 100644 --- a/src/transformers/models/mistral3/modular_mistral3.py +++ b/src/transformers/models/mistral3/modular_mistral3.py @@ -20,7 +20,8 @@ from ...cache_utils import Cache from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import Unpack -from ...utils import auto_docstring, can_return_tuple, logging +from ...utils import auto_docstring, logging +from ...utils.generic import check_model_inputs from ..llava.modeling_llava import ( LlavaCausalLMOutputWithPast, LlavaForConditionalGeneration, @@ -119,7 +120,7 @@ class Mistral3PreTrainedModel(LlavaPreTrainedModel): class Mistral3Model(LlavaModel): - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) @@ -131,10 +132,6 @@ def get_image_features( output_hidden_states: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - kwargs = {k: v for k, v in kwargs.items() if v is not None} # this is not memory efficient at all (output_hidden_states=True) will save all the hidden states. image_outputs = self.vision_tower( @@ -162,6 +159,8 @@ def get_image_features( return image_outputs + @check_model_inputs(tie_last_hidden_states=False) + @auto_docstring def forward( self, input_ids: torch.LongTensor | None = None, @@ -184,9 +183,6 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index cedf092af1b0..5decbf2c9ffe 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -26,7 +26,7 @@ from ...modeling_outputs import BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging +from ...utils import TransformersKwargs, auto_docstring, logging from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_video_llava import VideoLlavaConfig @@ -218,7 +218,7 @@ def get_image_features( return image_outputs - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring( custom_intro="Obtains video last hidden states from the vision tower and apply multimodal projection." ) @@ -237,10 +237,6 @@ def get_video_features( the vision feature of the corresponding indices will be concatenated to form the vision features. """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - batch_size_vid, num_frames, channels, height, width = pixel_values_videos.shape pixel_values = pixel_values_videos.reshape(batch_size_vid * num_frames, channels, height, width) From bf9182da98d57cf3031549b28931d142ddf239f5 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 12:34:45 +0100 Subject: [PATCH 117/131] Patch unusual bug: llava_next_video used self.vision_feature_layer Doesn't seem like this was being used elsewhere, so I can just update it to use the local variant like elsewhere --- .../models/llava_next_video/modeling_llava_next_video.py | 8 ++++---- .../models/llava_next_video/modular_llava_next_video.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 91d44b6255bb..ceee509ab2ed 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -551,8 +551,8 @@ def forward( image_features = self.get_image_features( pixel_values, image_sizes, - vision_feature_layer=self.vision_feature_layer, - vision_feature_select_strategy=self.vision_feature_select_strategy, + vision_feature_layer=vision_feature_layer, + vision_feature_select_strategy=vision_feature_select_strategy, return_dict=True, ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) @@ -564,8 +564,8 @@ def forward( if pixel_values_videos is not None: video_features = self.get_video_features( pixel_values_videos, - vision_feature_layer=self.vision_feature_layer, - vision_feature_select_strategy=self.vision_feature_select_strategy, + vision_feature_layer=vision_feature_layer, + vision_feature_select_strategy=vision_feature_select_strategy, return_dict=True, ).pooler_output video_features = [feature.flatten(0, 1) for feature in video_features] diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index aad09e7df8a5..92ae68420b62 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -470,8 +470,8 @@ def forward( image_features = self.get_image_features( pixel_values, image_sizes, - vision_feature_layer=self.vision_feature_layer, - vision_feature_select_strategy=self.vision_feature_select_strategy, + vision_feature_layer=vision_feature_layer, + vision_feature_select_strategy=vision_feature_select_strategy, return_dict=True, ).pooler_output image_features = torch.cat(image_features, dim=0).to(inputs_embeds.device, inputs_embeds.dtype) @@ -483,8 +483,8 @@ def forward( if pixel_values_videos is not None: video_features = self.get_video_features( pixel_values_videos, - vision_feature_layer=self.vision_feature_layer, - vision_feature_select_strategy=self.vision_feature_select_strategy, + vision_feature_layer=vision_feature_layer, + vision_feature_select_strategy=vision_feature_select_strategy, return_dict=True, ).pooler_output video_features = [feature.flatten(0, 1) for feature in video_features] From 15c2a597ab3d743d6b6976155349b2540965566c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 12:43:25 +0100 Subject: [PATCH 118/131] Add unused use_cache to TimmWrapperModel to patch FastVLM FastVLM now forwards this argument due to the check_model_inputs, and TimmWrapper can't use it --- src/transformers/models/timm_wrapper/modeling_timm_wrapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/timm_wrapper/modeling_timm_wrapper.py b/src/transformers/models/timm_wrapper/modeling_timm_wrapper.py index 461ce2ab30ba..0d944fc02421 100644 --- a/src/transformers/models/timm_wrapper/modeling_timm_wrapper.py +++ b/src/transformers/models/timm_wrapper/modeling_timm_wrapper.py @@ -168,6 +168,7 @@ def forward( output_hidden_states: bool | list[int] | None = None, return_dict: bool | None = None, do_pooling: bool | None = None, + use_cache: bool | None = None, **kwargs, ) -> TimmWrapperModelOutput | tuple[Tensor, ...]: r""" From d8604707391c39f823e537931dc21b9022511494 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 13:06:16 +0100 Subject: [PATCH 119/131] Update check_config_attributes to allow for vision attributes And rerun fix-repo --- src/transformers/models/internvl/modeling_internvl.py | 8 +------- utils/check_config_attributes.py | 4 ++++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/transformers/models/internvl/modeling_internvl.py b/src/transformers/models/internvl/modeling_internvl.py index 69fc22a62726..498caae1044e 100644 --- a/src/transformers/models/internvl/modeling_internvl.py +++ b/src/transformers/models/internvl/modeling_internvl.py @@ -35,13 +35,7 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - ModelOutput, - TransformersKwargs, - auto_docstring, - torch_compilable_check, - torch_int, -) +from ...utils import ModelOutput, TransformersKwargs, auto_docstring, torch_compilable_check, torch_int from ...utils.generic import check_model_inputs from ..auto import AutoModel from .configuration_internvl import InternVLConfig, InternVLVisionConfig diff --git a/utils/check_config_attributes.py b/utils/check_config_attributes.py index 996e14c5e7f5..d2e0d83e1719 100644 --- a/utils/check_config_attributes.py +++ b/utils/check_config_attributes.py @@ -164,6 +164,10 @@ "pretraining_tp", "use_sliding_window", "max_window_layers", + # vision attributes that may be used indirectly via check_model_inputs + "vision_feature_layer", + "vision_feature_select_strategy", + "vision_aspect_ratio", ) From 45d2c3374c55121fd4178c499fc57ef76fea910f Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 14:11:51 +0100 Subject: [PATCH 120/131] Add tests for config.return_dict=False Also; siglip had "nested" check_model_inputs: the VisionModel and VisionTransformer (below it) both used `check_model_inputs`. This means that the VisionModel.forward eats the 'return_dict=True', and the lower VisionTransformer.forward its `check_model_inputs` uses the config.return_dict=False to turn the output to a tuple. The siglip/clip/metaclip family is still broken due to the `text_model = text_model.text_model` bypassing the class with the `check_model_inputs`. --- .../models/siglip/modeling_siglip.py | 5 -- .../models/siglip2/modeling_siglip2.py | 7 +-- .../models/siglip2/modular_siglip2.py | 4 +- .../models/t5gemma2/configuration_t5gemma2.py | 1 + .../models/t5gemma2/modular_t5gemma2.py | 1 + tests/models/siglip/test_modeling_siglip.py | 14 +++++- tests/models/siglip2/test_modeling_siglip2.py | 14 +++++- tests/test_modeling_common.py | 46 +++++++++++++++++++ 8 files changed, 75 insertions(+), 17 deletions(-) diff --git a/src/transformers/models/siglip/modeling_siglip.py b/src/transformers/models/siglip/modeling_siglip.py index 541b5ffabeae..657d5e3e4554 100644 --- a/src/transformers/models/siglip/modeling_siglip.py +++ b/src/transformers/models/siglip/modeling_siglip.py @@ -620,10 +620,6 @@ def forward( class SiglipVisionTransformer(SiglipPreTrainedModel): _input_embed_layer = "patch_embedding" - _can_record_outputs = { - "hidden_states": SiglipEncoderLayer, - "attentions": SiglipAttention, - } def __init__(self, config: SiglipVisionConfig): super().__init__(config) @@ -639,7 +635,6 @@ def __init__(self, config: SiglipVisionConfig): self.post_init() - @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, diff --git a/src/transformers/models/siglip2/modeling_siglip2.py b/src/transformers/models/siglip2/modeling_siglip2.py index 82deb25b397e..3182fac5ca8c 100644 --- a/src/transformers/models/siglip2/modeling_siglip2.py +++ b/src/transformers/models/siglip2/modeling_siglip2.py @@ -529,10 +529,6 @@ def forward( class Siglip2VisionTransformer(Siglip2PreTrainedModel): _input_embed_layer = "patch_embedding" - _can_record_outputs = { - "hidden_states": Siglip2EncoderLayer, - "attentions": Siglip2Attention, - } def __init__(self, config: Siglip2VisionConfig): super().__init__(config) @@ -548,7 +544,6 @@ def __init__(self, config: Siglip2VisionConfig): self.post_init() - @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, @@ -762,7 +757,7 @@ def __init__(self, config: Siglip2VisionConfig): def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, diff --git a/src/transformers/models/siglip2/modular_siglip2.py b/src/transformers/models/siglip2/modular_siglip2.py index eb2c7642e9d7..9c022de838e9 100644 --- a/src/transformers/models/siglip2/modular_siglip2.py +++ b/src/transformers/models/siglip2/modular_siglip2.py @@ -36,7 +36,7 @@ from ...modeling_attn_mask_utils import _prepare_4d_attention_mask from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple -from ...utils.generic import is_flash_attention_requested +from ...utils.generic import check_model_inputs, is_flash_attention_requested class Siglip2TextConfig(SiglipTextConfig): @@ -315,7 +315,7 @@ def forward(self, hidden_state: torch.Tensor, attention_mask: torch.Tensor | Non class Siglip2VisionModel(SiglipVisionModel): # Update: add `spatial_shapes` and `pixel_attention_mask` - @can_return_tuple + @check_model_inputs(tie_last_hidden_states=False) @auto_docstring def forward( self, diff --git a/src/transformers/models/t5gemma2/configuration_t5gemma2.py b/src/transformers/models/t5gemma2/configuration_t5gemma2.py index 87a83649346b..88e6599ab4a3 100644 --- a/src/transformers/models/t5gemma2/configuration_t5gemma2.py +++ b/src/transformers/models/t5gemma2/configuration_t5gemma2.py @@ -609,6 +609,7 @@ def __setattr__(self, key, value): "attention_dropout", "vocab_size", "dtype", + "return_dict", ] if key in shared_attr_with_submodules: diff --git a/src/transformers/models/t5gemma2/modular_t5gemma2.py b/src/transformers/models/t5gemma2/modular_t5gemma2.py index 1dbd79dc557c..3245a041d20b 100644 --- a/src/transformers/models/t5gemma2/modular_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modular_t5gemma2.py @@ -483,6 +483,7 @@ def __setattr__(self, key, value): "attention_dropout", "vocab_size", "dtype", + "return_dict", ] if key in shared_attr_with_submodules: diff --git a/tests/models/siglip/test_modeling_siglip.py b/tests/models/siglip/test_modeling_siglip.py index b30dc6c58237..ef5d69b95dd5 100644 --- a/tests/models/siglip/test_modeling_siglip.py +++ b/tests/models/siglip/test_modeling_siglip.py @@ -504,16 +504,26 @@ def test_load_vision_text_config(self): text_config = SiglipTextConfig.from_pretrained(tmp_dir_name) self.assertDictEqual(config.text_config.to_dict(), text_config.to_dict()) - @unittest.skip(reason="The SigLIP family currently does not work with output_attentions on the text side.") + @unittest.skip(reason="The SigLIP family currently does not work with output_attentions.") def test_get_text_features_attentions(self): # This test should no longer be skipped once this architecture is refactored to work with output_attentions. pass - @unittest.skip(reason="The SigLIP family currently does not work with output_hidden_states on the text side.") + @unittest.skip(reason="The SigLIP family currently does not work with output_hidden_states.") def test_get_text_features_hidden_states(self): # This test should no longer be skipped once this architecture is refactored to work with output_hidden_states. pass + @unittest.skip(reason="The SigLIP family currently does not work with output_attentions.") + def test_get_image_features_attentions(self): + # This test should no longer be skipped once this architecture is refactored to work with output_attentions. + pass + + @unittest.skip(reason="The SigLIP family currently does not work with output_hidden_states.") + def test_get_image_features_hidden_states(self): + # This test should no longer be skipped once this architecture is refactored to work with output_hidden_states. + pass + @slow def test_model_from_pretrained(self): model_name = "google/siglip-base-patch16-224" diff --git a/tests/models/siglip2/test_modeling_siglip2.py b/tests/models/siglip2/test_modeling_siglip2.py index e03a4dbbcf35..e2049a89fbbf 100644 --- a/tests/models/siglip2/test_modeling_siglip2.py +++ b/tests/models/siglip2/test_modeling_siglip2.py @@ -577,16 +577,26 @@ def test_load_vision_text_config(self): text_config = Siglip2TextConfig.from_pretrained(tmp_dir_name) self.assertDictEqual(config.text_config.to_dict(), text_config.to_dict()) - @unittest.skip(reason="The SigLIP2 family currently does not work with output_attentions on the text side.") + @unittest.skip(reason="The SigLIP2 family currently does not work with output_attentions.") def test_get_text_features_attentions(self): # This test should no longer be skipped once this architecture is refactored to work with output_attentions. pass - @unittest.skip(reason="The SigLIP2 family currently does not work with output_hidden_states on the text side.") + @unittest.skip(reason="The SigLIP2 family currently does not work with output_hidden_states.") def test_get_text_features_hidden_states(self): # This test should no longer be skipped once this architecture is refactored to work with output_hidden_states. pass + @unittest.skip(reason="The SigLIP2 family currently does not work with output_attentions.") + def test_get_image_features_attentions(self): + # This test should no longer be skipped once this architecture is refactored to work with output_attentions. + pass + + @unittest.skip(reason="The SigLIP2 family currently does not work with output_hidden_states.") + def test_get_image_features_hidden_states(self): + # This test should no longer be skipped once this architecture is refactored to work with output_hidden_states. + pass + @slow def test_model_from_pretrained(self): model_name = "google/siglip2-base-patch16-naflex" diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 62dd08a87f44..6660c60f5cc8 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4846,6 +4846,16 @@ def test_get_text_features_output(self): ) self.assertEqual(last_hidden_state.shape, expected_shape, "last_hidden_state shape mismatch") + config, inputs_dict = self._text_features_prepare_config_and_inputs() + config.return_dict = False + + model = model_class(config).eval() + model = model.to(torch_device) + + with torch.no_grad(): + outputs = model.get_text_features(**inputs_dict) + self.assertTrue(isinstance(outputs, tuple), "get_text_features() must return a tuple if return_dict=False") + def test_get_text_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) @@ -4991,6 +5001,18 @@ def test_get_image_features_output(self): f"hidden_size mismatch, full shape: {last_hidden_state_shape}", ) + config, inputs_dict = self._image_features_prepare_config_and_inputs() + config.return_dict = False + + model = model_class(config).eval() + model = model.to(torch_device) + + with torch.no_grad(): + outputs = model.get_image_features(**inputs_dict) + self.assertTrue( + isinstance(outputs, tuple), "get_image_features() must return a tuple if return_dict=False" + ) + def test_get_image_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) @@ -5115,6 +5137,18 @@ def test_get_audio_features_output(self): f"hidden_size mismatch, full shape: {last_hidden_state_shape}", ) + config, inputs_dict = self._audio_features_prepare_config_and_inputs() + config.return_dict = False + + model = model_class(config).eval() + model = model.to(torch_device) + + with torch.no_grad(): + outputs = model.get_audio_features(**inputs_dict) + self.assertTrue( + isinstance(outputs, tuple), "get_audio_features() must return a tuple if return_dict=False" + ) + def test_get_audio_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) @@ -5239,6 +5273,18 @@ def test_get_video_features_output(self): f"hidden_size mismatch, full shape: {last_hidden_state_shape}", ) + config, inputs_dict = self._video_features_prepare_config_and_inputs() + config.return_dict = False + + model = model_class(config).eval() + model = model.to(torch_device) + + with torch.no_grad(): + outputs = model.get_video_features(**inputs_dict) + self.assertTrue( + isinstance(outputs, tuple), "get_video_features() must return a tuple if return_dict=False" + ) + def test_get_video_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) From 5199c47225deabcb8ec3bc1bb97cfcc3c120b783 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 14:25:46 +0100 Subject: [PATCH 121/131] permute and quantize separately for the comment --- src/transformers/models/emu3/modeling_emu3.py | 3 ++- src/transformers/models/emu3/modular_emu3.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index 60e017b1aae4..2d41f4aead43 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -1017,7 +1017,8 @@ def encode( hidden_states = self.encoder(pixel_values) # b t c h w -> b c t h w - conv_hidden_states = self.quant_conv(hidden_states.permute(0, 2, 1, 3, 4)) + conv_hidden_states = hidden_states.permute(0, 2, 1, 3, 4) + conv_hidden_states = self.quant_conv(conv_hidden_states) # b c t h w -> b t c h w conv_hidden_states = conv_hidden_states.permute(0, 2, 1, 3, 4) diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index c7747958b2f7..ac4c79ae3cf2 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -767,7 +767,8 @@ def encode( hidden_states = self.encoder(pixel_values) # b t c h w -> b c t h w - conv_hidden_states = self.quant_conv(hidden_states.permute(0, 2, 1, 3, 4)) + conv_hidden_states = hidden_states.permute(0, 2, 1, 3, 4) + conv_hidden_states = self.quant_conv(conv_hidden_states) # b c t h w -> b t c h w conv_hidden_states = conv_hidden_states.permute(0, 2, 1, 3, 4) From 9865895bdee47b102888edeb4780a97b6238c195 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 14:31:55 +0100 Subject: [PATCH 122/131] Ditch shared custom_args for ernie4_5_vl_moe --- .../modeling_ernie4_5_vl_moe.py | 23 ++++--------------- .../modular_ernie4_5_vl_moe.py | 22 ++++-------------- 2 files changed, 8 insertions(+), 37 deletions(-) diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index dc81249036ba..2a3f3add9432 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -1085,21 +1085,6 @@ def forward(self, hidden_states, grid_thw): return hidden_states -ERNIE4_5_VL_MOE_VIDEO_COMMON_CUSTOM_ARGS = r""" - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. -""" - -ERNIE4_5_VL_MOE_IMAGE_COMMON_CUSTOM_ARGS = r""" - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. -""" - - @auto_docstring class Ernie4_5_VL_MoeModel(Ernie4_5_VL_MoePreTrainedModel): base_model_prefix = "model" @@ -1282,7 +1267,7 @@ def get_rope_index( return position_ids, mrope_position_deltas @can_return_tuple - @auto_docstring(custom_args=ERNIE4_5_VL_MOE_VIDEO_COMMON_CUSTOM_ARGS) + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -1307,7 +1292,7 @@ def get_video_features( return video_outputs @can_return_tuple - @auto_docstring(custom_args=ERNIE4_5_VL_MOE_IMAGE_COMMON_CUSTOM_ARGS) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, @@ -1624,7 +1609,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.set_input_embeddings(value) - @auto_docstring(custom_args=ERNIE4_5_VL_MOE_VIDEO_COMMON_CUSTOM_ARGS) + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -1641,7 +1626,7 @@ def get_video_features( pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs ) - @auto_docstring(custom_args=ERNIE4_5_VL_MOE_IMAGE_COMMON_CUSTOM_ARGS) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index 60037437e9fe..97f30ddc9420 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -90,20 +90,6 @@ logger = logging.get_logger(__name__) -ERNIE4_5_VL_MOE_VIDEO_COMMON_CUSTOM_ARGS = r""" - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input videos. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. -""" - -ERNIE4_5_VL_MOE_IMAGE_COMMON_CUSTOM_ARGS = r""" - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The tensors corresponding to the input images. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. -""" - class Ernie4_5_VL_MoeVisionConfig(Qwen2VLVisionConfig): r""" @@ -1283,7 +1269,7 @@ def get_rope_index( return position_ids, mrope_position_deltas @can_return_tuple - @auto_docstring(custom_args=ERNIE4_5_VL_MOE_VIDEO_COMMON_CUSTOM_ARGS) + @auto_docstring def get_video_features( self, pixel_values_videos: torch.FloatTensor, @@ -1302,7 +1288,7 @@ def get_video_features( return video_outputs @can_return_tuple - @auto_docstring(custom_args=ERNIE4_5_VL_MOE_IMAGE_COMMON_CUSTOM_ARGS) + @auto_docstring def get_image_features( self, pixel_values: torch.FloatTensor, @@ -1411,11 +1397,11 @@ def __init__(self, config): self.num_experts = config.text_config.moe_num_experts self.num_experts_per_tok = config.text_config.moe_k - @auto_docstring(custom_args=ERNIE4_5_VL_MOE_VIDEO_COMMON_CUSTOM_ARGS) + @auto_docstring def get_video_features(self, **super_kwargs): return super().get_video_features(**super_kwargs) - @auto_docstring(custom_args=ERNIE4_5_VL_MOE_IMAGE_COMMON_CUSTOM_ARGS) + @auto_docstring def get_image_features(self, **super_kwargs): return super().get_image_features(**super_kwargs) From 276dcaaf7fbdc999a9eb670ec6b4574106aba8cc Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 14:40:36 +0100 Subject: [PATCH 123/131] Move Ernie4_5_VL_MoeVisionAttention next to VisionBlock --- .../modeling_ernie4_5_vl_moe.py | 208 +++++++++--------- .../modular_ernie4_5_vl_moe.py | 8 +- 2 files changed, 108 insertions(+), 108 deletions(-) diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index 2a3f3add9432..3b02f84c8d84 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -279,110 +279,6 @@ def forward( return attn_output, attn_weights -def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - -def apply_rotary_pos_emb_vision( - q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor -) -> tuple[torch.Tensor, torch.Tensor]: - orig_q_dtype = q.dtype - orig_k_dtype = k.dtype - q, k = q.float(), k.float() - cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float() - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - q_embed = q_embed.to(orig_q_dtype) - k_embed = k_embed.to(orig_k_dtype) - return q_embed, k_embed - - -class Ernie4_5_VL_MoeVisionAttention(nn.Module): - def __init__(self, config: Ernie4_5_VL_MoeVisionConfig) -> None: - super().__init__() - self.dim = config.hidden_size - self.num_heads = config.num_heads - self.head_dim = self.dim // self.num_heads - self.num_key_value_groups = 1 # needed for eager attention - self.qkv = nn.Linear(self.dim, self.dim * 3, bias=True) - self.proj = nn.Linear(self.dim, self.dim) - self.scaling = self.head_dim**-0.5 - self.config = config - self.attention_dropout = 0.0 - self.is_causal = False - - def forward( - self, - hidden_states: torch.Tensor, - cu_seqlens: torch.Tensor, - rotary_pos_emb: torch.Tensor | None = None, - position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, - **kwargs, - ) -> torch.Tensor: - seq_length = hidden_states.shape[0] - query_states, key_states, value_states = ( - self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) - ) - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin) - - query_states = query_states.transpose(0, 1).unsqueeze(0) - key_states = key_states.transpose(0, 1).unsqueeze(0) - value_states = value_states.transpose(0, 1).unsqueeze(0) - - attention_interface: Callable = eager_attention_forward - if self.config._attn_implementation != "eager": - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - - if is_flash_attention_requested(self.config): - # Flash Attention: Use cu_seqlens for variable length attention - max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() - attn_output, _ = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask=None, - scaling=self.scaling, - dropout=0.0 if not self.training else self.attention_dropout, - cu_seq_lens_q=cu_seqlens, - cu_seq_lens_k=cu_seqlens, - max_length_q=max_seqlen, - max_length_k=max_seqlen, - is_causal=False, - **kwargs, - ) - else: - # Other implementations: Process each chunk separately - lengths = cu_seqlens[1:] - cu_seqlens[:-1] - splits = [ - torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states) - ] - - attn_outputs = [ - attention_interface( - self, - q, - k, - v, - attention_mask=None, - scaling=self.scaling, - dropout=0.0 if not self.training else self.attention_dropout, - is_causal=False, - **kwargs, - )[0] - for q, k, v in zip(*splits) - ] - attn_output = torch.cat(attn_outputs, dim=1) - - attn_output = attn_output.reshape(seq_length, -1).contiguous() - attn_output = self.proj(attn_output) - return attn_output - - @use_kernel_forward_from_hub("RMSNorm") class Ernie4_5_VL_MoeRMSNorm(nn.Module): def __init__(self, hidden_size, eps=1e-6): @@ -656,6 +552,110 @@ def forward( return hidden_states +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb_vision( + q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + orig_q_dtype = q.dtype + orig_k_dtype = k.dtype + q, k = q.float(), k.float() + cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float() + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + q_embed = q_embed.to(orig_q_dtype) + k_embed = k_embed.to(orig_k_dtype) + return q_embed, k_embed + + +class Ernie4_5_VL_MoeVisionAttention(nn.Module): + def __init__(self, config: Ernie4_5_VL_MoeVisionConfig) -> None: + super().__init__() + self.dim = config.hidden_size + self.num_heads = config.num_heads + self.head_dim = self.dim // self.num_heads + self.num_key_value_groups = 1 # needed for eager attention + self.qkv = nn.Linear(self.dim, self.dim * 3, bias=True) + self.proj = nn.Linear(self.dim, self.dim) + self.scaling = self.head_dim**-0.5 + self.config = config + self.attention_dropout = 0.0 + self.is_causal = False + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + rotary_pos_emb: torch.Tensor | None = None, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + **kwargs, + ) -> torch.Tensor: + seq_length = hidden_states.shape[0] + query_states, key_states, value_states = ( + self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) + ) + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin) + + query_states = query_states.transpose(0, 1).unsqueeze(0) + key_states = key_states.transpose(0, 1).unsqueeze(0) + value_states = value_states.transpose(0, 1).unsqueeze(0) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + if is_flash_attention_requested(self.config): + # Flash Attention: Use cu_seqlens for variable length attention + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() + attn_output, _ = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask=None, + scaling=self.scaling, + dropout=0.0 if not self.training else self.attention_dropout, + cu_seq_lens_q=cu_seqlens, + cu_seq_lens_k=cu_seqlens, + max_length_q=max_seqlen, + max_length_k=max_seqlen, + is_causal=False, + **kwargs, + ) + else: + # Other implementations: Process each chunk separately + lengths = cu_seqlens[1:] - cu_seqlens[:-1] + splits = [ + torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states) + ] + + attn_outputs = [ + attention_interface( + self, + q, + k, + v, + attention_mask=None, + scaling=self.scaling, + dropout=0.0 if not self.training else self.attention_dropout, + is_causal=False, + **kwargs, + )[0] + for q, k, v in zip(*splits) + ] + attn_output = torch.cat(attn_outputs, dim=1) + + attn_output = attn_output.reshape(seq_length, -1).contiguous() + attn_output = self.proj(attn_output) + return attn_output + + class Ernie4_5_VL_MoeVisionBlock(GradientCheckpointingLayer): def __init__(self, config) -> None: super().__init__() diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index 97f30ddc9420..1093d0d9edc9 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -535,10 +535,6 @@ class Ernie4_5_VL_MoeTextAttention(Ernie4_5_MoeAttention): pass -class Ernie4_5_VL_MoeVisionAttention(Qwen2_5_VLVisionAttention): - pass - - class Ernie4_5_VL_MoeRMSNorm(Ernie4_5_MoeRMSNorm): pass @@ -704,6 +700,10 @@ def forward( return hidden_states +class Ernie4_5_VL_MoeVisionAttention(Qwen2_5_VLVisionAttention): + pass + + class Ernie4_5_VL_MoeVisionBlock(Qwen2_5_VLVisionBlock): def __init__(self, config) -> None: super().__init__(config, None) From c804de4e8e7c8f938fe3704f3241b359c6d333f8 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 14:56:30 +0100 Subject: [PATCH 124/131] Add missing "attentions" from Florence2 _can_record_outputs --- src/transformers/models/florence2/modeling_florence2.py | 5 ++++- src/transformers/models/florence2/modular_florence2.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/florence2/modeling_florence2.py b/src/transformers/models/florence2/modeling_florence2.py index 28595289e0f3..f4fe735ce4f5 100644 --- a/src/transformers/models/florence2/modeling_florence2.py +++ b/src/transformers/models/florence2/modeling_florence2.py @@ -499,7 +499,10 @@ class Florence2VisionPreTrainedModel(PreTrainedModel): _supports_flex_attn = True _can_compile_fullgraph = True - _can_record_outputs = {"hidden_states": Florence2VisionBlock} + _can_record_outputs = { + "hidden_states": Florence2VisionBlock, + "attentions": [Florence2VisionChannelAttention, Florence2VisionWindowAttention], + } @auto_docstring diff --git a/src/transformers/models/florence2/modular_florence2.py b/src/transformers/models/florence2/modular_florence2.py index 7501381e17af..f94974769f7a 100644 --- a/src/transformers/models/florence2/modular_florence2.py +++ b/src/transformers/models/florence2/modular_florence2.py @@ -1345,7 +1345,10 @@ class Florence2VisionPreTrainedModel(PreTrainedModel): _supports_flex_attn = True _can_compile_fullgraph = True - _can_record_outputs = {"hidden_states": Florence2VisionBlock} + _can_record_outputs = { + "hidden_states": Florence2VisionBlock, + "attentions": [Florence2VisionChannelAttention, Florence2VisionWindowAttention], + } @auto_docstring From 72a1a0938441620b6f2c502c7d7c56caf5369f14 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 17:14:16 +0100 Subject: [PATCH 125/131] Clarify kwargs.get("image_sizes") in modeling_llava --- src/transformers/models/llava/modeling_llava.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 0aa6964a5de9..9fbd58c786df 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -182,6 +182,8 @@ def get_image_features( image_features = self.multi_modal_projector(selected_image_feature) + # If image_sizes is provided, we need to split the image features accordingly, + # but only if the image_sizes is not None (the default in this and related architectures) if kwargs.get("image_sizes") is not None: split_sizes = ( (torch.as_tensor(kwargs["image_sizes"], device=image_features.device) // self.vision_tower.patch_size) From 43ec4b38ec8d86874635d52e7da5255a0d89c185 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 17:15:58 +0100 Subject: [PATCH 126/131] Remove commented skip_test_image_features_output_shape in chameleon tests --- tests/models/chameleon/test_modeling_chameleon.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/models/chameleon/test_modeling_chameleon.py b/tests/models/chameleon/test_modeling_chameleon.py index a85bf9a25e9f..33df14c76739 100644 --- a/tests/models/chameleon/test_modeling_chameleon.py +++ b/tests/models/chameleon/test_modeling_chameleon.py @@ -279,7 +279,6 @@ class ChameleonVision2SeqModelTest(ModelTesterMixin, GenerationTesterMixin, Pipe if is_torch_available() else {} ) - # skip_test_image_features_output_shape = True # Align uses index 1 for hidden_size instead of last index def setUp(self): self.model_tester = ChameleonVision2SeqModelTester(self) From 4515b29b4dfdf87c674772006f14d62837152588 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 22 Jan 2026 17:35:03 +0100 Subject: [PATCH 127/131] Add a migration guide under 'Library-wide changes with lesser impact' --- MIGRATION_GUIDE_V5.md | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/MIGRATION_GUIDE_V5.md b/MIGRATION_GUIDE_V5.md index 7c755fa0c3b2..f42a7bb2c039 100644 --- a/MIGRATION_GUIDE_V5.md +++ b/MIGRATION_GUIDE_V5.md @@ -453,6 +453,37 @@ We dropped support for two torch APIs: Those APIs were deprecated by the PyTorch team, and we're instead focusing on the supported APIs `dynamo` and `export`. +### Feature extraction helpers: `get_*_features` + +Many multi-modal models expose convenience methods such as `get_text_features`, `get_image_features`, `get_audio_features`, and `get_video_features` to run inference on a single modality without calling `model(**inputs)` directly. + +Starting with v5, these 4 helper methods now return a `BaseModelOutputWithPooling` (or a subclass) instead of only a pooled embedding tensor: + +- `last_hidden_state`: unpooled token/patch/frame embeddings for the requested modality. +- `pooler_output`: pooled representation (what most models previously returned from `get_*_features`). +- `hidden_states`: full hidden states for all layers when `output_hidden_states=True` is passed. +- `attentions`: attention maps when `output_attentions=True` is passed. + +> [!IMPORTANT] +> There is **no single universal shape** for `last_hidden_state` or `pooler_output`. It's recommended to inspect a small forward pass before making assumptions about shapes or semantics. + +If your code previously did something like this: + +```python +text_embeddings = model.get_text_features(**inputs) +``` + +and you used `text_embeddings` as a tensor, you should now explicitly use `return_dict=True` take the `pooler_output` field from the returned `BaseModelOutputWithPooling`: + +```python +outputs = model.get_text_features(**inputs, return_dict=True) +text_embeddings = outputs.pooler_output +``` + +This will match the previous behavior in the large majority of cases. If your model-specific implementation returned a tuple of results before, those values should now be accessible as fields on the corresponding `BaseModelOutputWithPooling` subclass. + +Linked PR: https://github.com/huggingface/transformers/pull/42564 + ## Quantization changes We clean up the quantization API in transformers, and significantly refactor the weight loading as highlighted @@ -558,7 +589,7 @@ Linked PRs: - `use_mps_device` -> mps will be used by default if detected - `fp16_backend` and `half_precision_backend` -> we will only rely on torch.amp as everything has been upstream to torch - `no_cuda` -> `use_cpu` -- ` include_tokens_per_second` -> `include_num_input_tokens_seen` +- `include_tokens_per_second` -> `include_num_input_tokens_seen` - `use_legacy_prediction_loop` -> we only use `evaluation_loop` function from now on ### Removing deprecated arguments in `Trainer` @@ -574,7 +605,7 @@ Linked PRs: ### New defaults for `Trainer` -- `use_cache` in the model config will be set to `False`. You can still change the cache value through `TrainingArguments` `usel_cache` argument if needed. +- `use_cache` in the model config will be set to `False`. You can still change the cache value through `TrainingArguments` `use_cache` argument if needed. ## Pipelines From cd4c0cb07543a39bf1709ea28b490b7ea685f1cb Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 23 Jan 2026 09:55:57 +0100 Subject: [PATCH 128/131] Parameterize get_..._features tests with return_dict (True, False, None) --- .../chameleon/test_modeling_chameleon.py | 4 +- tests/models/fuyu/test_modeling_fuyu.py | 21 +- tests/test_modeling_common.py | 403 +++++++++--------- 3 files changed, 219 insertions(+), 209 deletions(-) diff --git a/tests/models/chameleon/test_modeling_chameleon.py b/tests/models/chameleon/test_modeling_chameleon.py index 33df14c76739..ccf84abaf353 100644 --- a/tests/models/chameleon/test_modeling_chameleon.py +++ b/tests/models/chameleon/test_modeling_chameleon.py @@ -17,6 +17,7 @@ import unittest import requests +from parameterized import parameterized from transformers import BitsAndBytesConfig, ChameleonConfig, is_torch_available, is_vision_available from transformers.testing_utils import ( @@ -232,7 +233,8 @@ def test_batching_equivalence(self): pass @unittest.skip("Skip get_image_features tests as those are tested via ChameleonVision2SeqModelTest instead") - def test_get_image_features_output(self): + @parameterized.expand([True, False, None]) + def test_get_image_features_output(self, return_dict: bool | None): pass @unittest.skip("Skip get_image_features tests as those are tested via ChameleonVision2SeqModelTest instead") diff --git a/tests/models/fuyu/test_modeling_fuyu.py b/tests/models/fuyu/test_modeling_fuyu.py index 58c00cdda621..6f06c8d5f68d 100644 --- a/tests/models/fuyu/test_modeling_fuyu.py +++ b/tests/models/fuyu/test_modeling_fuyu.py @@ -171,9 +171,6 @@ class FuyuModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin test_cpu_offload = False test_disk_offload = False - test_get_image_features_output = False - test_get_image_features_hidden_states = False - test_get_image_features_attentions = False def setUp(self): self.model_tester = FuyuModelTester(self) @@ -249,6 +246,24 @@ def test_sdpa_padding_matches_padding_free_with_position_ids(self): def test_model_base_model_prefix(self): pass + def _image_features_prepare_config_and_inputs(self): + """ + Helper method to extract only image-related inputs from the full set of inputs, for testing `get_image_features`. + + The Fuyu model uses image_patches, except for get_image_features, where they're called pixel_values. + """ + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + inputs_dict = {"pixel_values": inputs_dict["image_patches"]} + return config, inputs_dict + + @unittest.skip("Skip get_image_features tests as Fuyu's image features originate from a simple Linear") + def test_get_image_features_hidden_states(self): + pass + + @unittest.skip("Skip get_image_features tests as Fuyu's image features originate from a simple Linear") + def test_get_image_features_attentions(self): + pass + @slow @require_torch_accelerator diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 6660c60f5cc8..0b1177fe7f92 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4805,12 +4805,15 @@ def _video_features_get_expected_num_attentions(self, model_tester=None): def _video_features_get_expected_num_hidden_states(self, model_tester=None): return self._video_features_get_expected_num_attentions(model_tester) + 1 - def test_get_text_features_output(self): + @parameterized.expand([True, False, None]) + def test_get_text_features_output(self, return_dict: bool | None): for model_class in self.all_model_classes: if not hasattr(model_class, "get_text_features"): continue config, inputs_dict = self._text_features_prepare_config_and_inputs() + if return_dict is not None: + config.return_dict = return_dict model = model_class(config).eval() model = model.to(torch_device) @@ -4818,43 +4821,39 @@ def test_get_text_features_output(self): torch.manual_seed(0) with torch.no_grad(): outputs = model.get_text_features(**inputs_dict) - self.assertTrue(isinstance(outputs, ModelOutput), "get_text_features() must return a BaseModelOutput") - self.assertTrue( - hasattr(outputs, "last_hidden_state"), - "get_text_features() must return a BaseModelOutput with last_hidden_state", - ) - self.assertTrue( - hasattr(outputs, "pooler_output"), - "get_text_features() must return a BaseModelOutput with pooler_output", - ) - self.assertTrue( - hasattr(outputs, "hidden_states"), - "get_text_features() must return a BaseModelOutput with hidden_states", - ) - if self.has_attentions: + + if return_dict in (True, None): + self.assertTrue(isinstance(outputs, ModelOutput), "get_text_features() must return a BaseModelOutput") self.assertTrue( - hasattr(outputs, "attentions"), "get_text_features() must return a BaseModelOutput with attentions" + hasattr(outputs, "last_hidden_state"), + "get_text_features() must return a BaseModelOutput with last_hidden_state", ) + self.assertTrue( + hasattr(outputs, "pooler_output"), + "get_text_features() must return a BaseModelOutput with pooler_output", + ) + self.assertTrue( + hasattr(outputs, "hidden_states"), + "get_text_features() must return a BaseModelOutput with hidden_states", + ) + if self.has_attentions: + self.assertTrue( + hasattr(outputs, "attentions"), + "get_text_features() must return a BaseModelOutput with attentions", + ) - # Test against (batch_size, seq_len, hidden_size) - last_hidden_state = outputs.last_hidden_state - expected_hidden_size = config.text_config.hidden_size - expected_shape = ( - inputs_dict["input_ids"].shape[0], - inputs_dict["input_ids"].shape[1], - expected_hidden_size, - ) - self.assertEqual(last_hidden_state.shape, expected_shape, "last_hidden_state shape mismatch") - - config, inputs_dict = self._text_features_prepare_config_and_inputs() - config.return_dict = False - - model = model_class(config).eval() - model = model.to(torch_device) + # Test against (batch_size, seq_len, hidden_size) + last_hidden_state = outputs.last_hidden_state + expected_hidden_size = config.text_config.hidden_size + expected_shape = ( + inputs_dict["input_ids"].shape[0], + inputs_dict["input_ids"].shape[1], + expected_hidden_size, + ) + self.assertEqual(last_hidden_state.shape, expected_shape, "last_hidden_state shape mismatch") - with torch.no_grad(): - outputs = model.get_text_features(**inputs_dict) - self.assertTrue(isinstance(outputs, tuple), "get_text_features() must return a tuple if return_dict=False") + else: + self.assertIsInstance(outputs, tuple, "get_text_features() must return a tuple if return_dict=False") def test_get_text_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): @@ -4924,12 +4923,15 @@ def check_attentions_output(inputs_dict, config, model_class): check_attentions_output(inputs_dict, config, model_class) - def test_get_image_features_output(self): + @parameterized.expand([True, False, None]) + def test_get_image_features_output(self, return_dict: bool | None): for model_class in self.all_model_classes: if not hasattr(model_class, "get_image_features"): continue config, inputs_dict = self._image_features_prepare_config_and_inputs() + if return_dict is not None: + config.return_dict = return_dict model = model_class(config).eval() model = model.to(torch_device) @@ -4937,81 +4939,76 @@ def test_get_image_features_output(self): torch.manual_seed(0) with torch.no_grad(): outputs = model.get_image_features(**inputs_dict) - self.assertTrue(isinstance(outputs, ModelOutput), "get_image_features() must return a BaseModelOutput") - self.assertTrue( - hasattr(outputs, "last_hidden_state"), - "get_image_features() must return a BaseModelOutput with last_hidden_state", - ) - self.assertTrue( - hasattr(outputs, "pooler_output"), - "get_image_features() must return a BaseModelOutput with pooler_output", - ) - self.assertTrue( - hasattr(outputs, "hidden_states"), - "get_image_features() must return a BaseModelOutput with hidden_states", - ) - if self.has_attentions: + + if return_dict in (True, None): + self.assertTrue(isinstance(outputs, ModelOutput), "get_image_features() must return a BaseModelOutput") self.assertTrue( - hasattr(outputs, "attentions"), - "get_image_features() must return a BaseModelOutput with attentions", + hasattr(outputs, "last_hidden_state"), + "get_image_features() must return a BaseModelOutput with last_hidden_state", ) + self.assertTrue( + hasattr(outputs, "pooler_output"), + "get_image_features() must return a BaseModelOutput with pooler_output", + ) + self.assertTrue( + hasattr(outputs, "hidden_states"), + "get_image_features() must return a BaseModelOutput with hidden_states", + ) + if self.has_attentions: + self.assertTrue( + hasattr(outputs, "attentions"), + "get_image_features() must return a BaseModelOutput with attentions", + ) - if getattr(self, "skip_test_image_features_output_shape", False): - return - - last_hidden_state_shape = outputs.last_hidden_state.shape - batch_size = ( - inputs_dict["pixel_values"].shape[0] - if "pixel_values" in inputs_dict - else inputs_dict["pixel_values_images"].shape[0] - ) - self.assertEqual( - last_hidden_state_shape[0], batch_size, f"batch_size mismatch, full shape: {last_hidden_state_shape}" - ) - - vision_config = config.vision_config if hasattr(config, "vision_config") else config - vision_config = ( - vision_config.backbone_config if hasattr(vision_config, "backbone_config") else vision_config - ) - vision_config = vision_config.vq_config if hasattr(vision_config, "vq_config") else vision_config - vision_config = vision_config.model_args if hasattr(vision_config, "model_args") else vision_config - attribute_candidates = [ - "embed_dim_per_stage", - "embed_dim", - "embed_dims", - "out_hidden_size", - "hidden_size", - "hidden_dim", - ] - hidden_size = None - for attr in attribute_candidates: - if hasattr(vision_config, attr): - hidden_size = getattr(vision_config, attr) - break - elif isinstance(vision_config, dict) and attr in vision_config: - hidden_size = vision_config[attr] - break - else: - raise ValueError("Cannot find the hidden size attribute in vision_config") - if isinstance(hidden_size, (list, tuple)): - hidden_size = hidden_size[-1] - self.assertEqual( - last_hidden_state_shape[-1], - hidden_size, - f"hidden_size mismatch, full shape: {last_hidden_state_shape}", - ) + if getattr(self, "skip_test_image_features_output_shape", False): + return - config, inputs_dict = self._image_features_prepare_config_and_inputs() - config.return_dict = False + last_hidden_state_shape = outputs.last_hidden_state.shape + batch_size = ( + inputs_dict["pixel_values"].shape[0] + if "pixel_values" in inputs_dict + else inputs_dict["pixel_values_images"].shape[0] + ) + self.assertEqual( + last_hidden_state_shape[0], + batch_size, + f"batch_size mismatch, full shape: {last_hidden_state_shape}", + ) - model = model_class(config).eval() - model = model.to(torch_device) + vision_config = config.vision_config if hasattr(config, "vision_config") else config + vision_config = ( + vision_config.backbone_config if hasattr(vision_config, "backbone_config") else vision_config + ) + vision_config = vision_config.vq_config if hasattr(vision_config, "vq_config") else vision_config + vision_config = vision_config.model_args if hasattr(vision_config, "model_args") else vision_config + attribute_candidates = [ + "embed_dim_per_stage", + "embed_dim", + "embed_dims", + "out_hidden_size", + "hidden_size", + "hidden_dim", + ] + hidden_size = None + for attr in attribute_candidates: + if hasattr(vision_config, attr): + hidden_size = getattr(vision_config, attr) + break + elif isinstance(vision_config, dict) and attr in vision_config: + hidden_size = vision_config[attr] + break + else: + raise ValueError("Cannot find the hidden size attribute in vision_config") + if isinstance(hidden_size, (list, tuple)): + hidden_size = hidden_size[-1] + self.assertEqual( + last_hidden_state_shape[-1], + hidden_size, + f"hidden_size mismatch, full shape: {last_hidden_state_shape}", + ) - with torch.no_grad(): - outputs = model.get_image_features(**inputs_dict) - self.assertTrue( - isinstance(outputs, tuple), "get_image_features() must return a tuple if return_dict=False" - ) + else: + self.assertIsInstance(outputs, tuple, "get_image_features() must return a tuple if return_dict=False") def test_get_image_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): @@ -5081,12 +5078,15 @@ def check_attentions_output(inputs_dict, config, model_class): check_attentions_output(inputs_dict, config, model_class) - def test_get_audio_features_output(self): + @parameterized.expand([True, False, None]) + def test_get_audio_features_output(self, return_dict: bool | None): for model_class in self.all_model_classes: if not hasattr(model_class, "get_audio_features"): continue config, inputs_dict = self._audio_features_prepare_config_and_inputs() + if return_dict is not None: + config.return_dict = return_dict model = model_class(config).eval() model = model.to(torch_device) @@ -5094,60 +5094,55 @@ def test_get_audio_features_output(self): torch.manual_seed(0) with torch.no_grad(): outputs = model.get_audio_features(**inputs_dict) - self.assertTrue(isinstance(outputs, ModelOutput), "get_audio_features() must return a BaseModelOutput") - self.assertTrue( - hasattr(outputs, "last_hidden_state"), - "get_audio_features() must return a BaseModelOutput with last_hidden_state", - ) - self.assertTrue( - hasattr(outputs, "pooler_output"), - "get_audio_features() must return a BaseModelOutput with pooler_output", - ) - self.assertTrue( - hasattr(outputs, "hidden_states"), - "get_audio_features() must return a BaseModelOutput with hidden_states", - ) - if self.has_attentions: + + if return_dict in (True, None): + self.assertTrue(isinstance(outputs, ModelOutput), "get_audio_features() must return a BaseModelOutput") self.assertTrue( - hasattr(outputs, "attentions"), - "get_audio_features() must return a BaseModelOutput with attentions", + hasattr(outputs, "last_hidden_state"), + "get_audio_features() must return a BaseModelOutput with last_hidden_state", ) + self.assertTrue( + hasattr(outputs, "pooler_output"), + "get_audio_features() must return a BaseModelOutput with pooler_output", + ) + self.assertTrue( + hasattr(outputs, "hidden_states"), + "get_audio_features() must return a BaseModelOutput with hidden_states", + ) + if self.has_attentions: + self.assertTrue( + hasattr(outputs, "attentions"), + "get_audio_features() must return a BaseModelOutput with attentions", + ) - if getattr(self, "skip_test_audio_features_output_shape", False): - return - - last_hidden_state_shape = outputs.last_hidden_state.shape - batch_size = inputs_dict["input_features"].shape[0] - self.assertEqual( - last_hidden_state_shape[0], batch_size, f"batch_size mismatch, full shape: {last_hidden_state_shape}" - ) - - audio_config = config.audio_config if hasattr(config, "audio_config") else config - if hasattr(audio_config, "projection_dim"): - hidden_size = audio_config.projection_dim - elif hasattr(audio_config, "hidden_size"): - hidden_size = audio_config.hidden_size - elif hasattr(audio_config, "encoder_config"): - hidden_size = audio_config.encoder_config.hidden_dim - elif hasattr(audio_config, "encoder_ffn_dim"): - hidden_size = audio_config.encoder_ffn_dim - self.assertEqual( - last_hidden_state_shape[-1], - hidden_size, - f"hidden_size mismatch, full shape: {last_hidden_state_shape}", - ) + if getattr(self, "skip_test_audio_features_output_shape", False): + return - config, inputs_dict = self._audio_features_prepare_config_and_inputs() - config.return_dict = False + last_hidden_state_shape = outputs.last_hidden_state.shape + batch_size = inputs_dict["input_features"].shape[0] + self.assertEqual( + last_hidden_state_shape[0], + batch_size, + f"batch_size mismatch, full shape: {last_hidden_state_shape}", + ) - model = model_class(config).eval() - model = model.to(torch_device) + audio_config = config.audio_config if hasattr(config, "audio_config") else config + if hasattr(audio_config, "projection_dim"): + hidden_size = audio_config.projection_dim + elif hasattr(audio_config, "hidden_size"): + hidden_size = audio_config.hidden_size + elif hasattr(audio_config, "encoder_config"): + hidden_size = audio_config.encoder_config.hidden_dim + elif hasattr(audio_config, "encoder_ffn_dim"): + hidden_size = audio_config.encoder_ffn_dim + self.assertEqual( + last_hidden_state_shape[-1], + hidden_size, + f"hidden_size mismatch, full shape: {last_hidden_state_shape}", + ) - with torch.no_grad(): - outputs = model.get_audio_features(**inputs_dict) - self.assertTrue( - isinstance(outputs, tuple), "get_audio_features() must return a tuple if return_dict=False" - ) + else: + self.assertIsInstance(outputs, tuple, "get_audio_features() must return a tuple if return_dict=False") def test_get_audio_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): @@ -5215,12 +5210,15 @@ def check_attentions_output(inputs_dict, config, model_class): check_attentions_output(inputs_dict, config, model_class) - def test_get_video_features_output(self): + @parameterized.expand([True, False, None]) + def test_get_video_features_output(self, return_dict: bool | None): for model_class in self.all_model_classes: if not hasattr(model_class, "get_video_features"): continue config, inputs_dict = self._video_features_prepare_config_and_inputs() + if return_dict is not None: + config.return_dict = return_dict model = model_class(config).eval() model = model.to(torch_device) @@ -5228,62 +5226,57 @@ def test_get_video_features_output(self): torch.manual_seed(0) with torch.no_grad(): outputs = model.get_video_features(**inputs_dict) - self.assertTrue(isinstance(outputs, ModelOutput), "get_video_features() must return a BaseModelOutput") - self.assertTrue( - hasattr(outputs, "last_hidden_state"), - "get_video_features() must return a BaseModelOutput with last_hidden_state", - ) - self.assertTrue( - hasattr(outputs, "pooler_output"), - "get_video_features() must return a BaseModelOutput with pooler_output", - ) - self.assertTrue( - hasattr(outputs, "hidden_states"), - "get_video_features() must return a BaseModelOutput with hidden_states", - ) - if self.has_attentions: + + if return_dict in (True, None): + self.assertTrue(isinstance(outputs, ModelOutput), "get_video_features() must return a BaseModelOutput") self.assertTrue( - hasattr(outputs, "attentions"), - "get_video_features() must return a BaseModelOutput with attentions", + hasattr(outputs, "last_hidden_state"), + "get_video_features() must return a BaseModelOutput with last_hidden_state", ) + self.assertTrue( + hasattr(outputs, "pooler_output"), + "get_video_features() must return a BaseModelOutput with pooler_output", + ) + self.assertTrue( + hasattr(outputs, "hidden_states"), + "get_video_features() must return a BaseModelOutput with hidden_states", + ) + if self.has_attentions: + self.assertTrue( + hasattr(outputs, "attentions"), + "get_video_features() must return a BaseModelOutput with attentions", + ) - if getattr(self, "skip_test_video_features_output_shape", False): - return - - last_hidden_state_shape = outputs.last_hidden_state.shape - if "pixel_values_videos" in inputs_dict: - batch_size = inputs_dict["pixel_values_videos"].shape[0] - elif "pixel_values" in inputs_dict: - batch_size = inputs_dict["pixel_values"].shape[0] - self.assertEqual( - last_hidden_state_shape[0], batch_size, f"batch_size mismatch, full shape: {last_hidden_state_shape}" - ) - video_config = config - if hasattr(config, "video_config"): - video_config = config.video_config - elif hasattr(config, "vision_config"): - video_config = config.vision_config - if hasattr(video_config, "out_hidden_size"): - hidden_size = video_config.out_hidden_size - else: - hidden_size = video_config.hidden_size - self.assertEqual( - last_hidden_state_shape[-1], - hidden_size, - f"hidden_size mismatch, full shape: {last_hidden_state_shape}", - ) - - config, inputs_dict = self._video_features_prepare_config_and_inputs() - config.return_dict = False + if getattr(self, "skip_test_video_features_output_shape", False): + return - model = model_class(config).eval() - model = model.to(torch_device) + last_hidden_state_shape = outputs.last_hidden_state.shape + if "pixel_values_videos" in inputs_dict: + batch_size = inputs_dict["pixel_values_videos"].shape[0] + elif "pixel_values" in inputs_dict: + batch_size = inputs_dict["pixel_values"].shape[0] + self.assertEqual( + last_hidden_state_shape[0], + batch_size, + f"batch_size mismatch, full shape: {last_hidden_state_shape}", + ) + video_config = config + if hasattr(config, "video_config"): + video_config = config.video_config + elif hasattr(config, "vision_config"): + video_config = config.vision_config + if hasattr(video_config, "out_hidden_size"): + hidden_size = video_config.out_hidden_size + else: + hidden_size = video_config.hidden_size + self.assertEqual( + last_hidden_state_shape[-1], + hidden_size, + f"hidden_size mismatch, full shape: {last_hidden_state_shape}", + ) - with torch.no_grad(): - outputs = model.get_video_features(**inputs_dict) - self.assertTrue( - isinstance(outputs, tuple), "get_video_features() must return a tuple if return_dict=False" - ) + else: + self.assertIsInstance(outputs, tuple, "get_video_features() must return a tuple if return_dict=False") def test_get_video_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): From 292ef3ab0655607843025e654becd23816eb2ef6 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 23 Jan 2026 10:07:59 +0100 Subject: [PATCH 129/131] Add comment re. TimmWrapper _can_record_outputs --- src/transformers/models/edgetam/modeling_edgetam.py | 2 ++ src/transformers/models/edgetam/modular_edgetam.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/transformers/models/edgetam/modeling_edgetam.py b/src/transformers/models/edgetam/modeling_edgetam.py index 75712a7fbee4..34c59b50a8d4 100644 --- a/src/transformers/models/edgetam/modeling_edgetam.py +++ b/src/transformers/models/edgetam/modeling_edgetam.py @@ -429,6 +429,8 @@ def forward(self, hidden_states: torch.Tensor) -> tuple[tuple[torch.Tensor, ...] class EdgeTamVisionModel(EdgeTamPreTrainedModel): config_class = EdgeTamVisionConfig main_input_name = "pixel_values" + # TODO: TimmWrapper models aren't compatible with _can_record_outputs yet. We specifically set this to + # an empty dict to avoid the _can_record_outputs from Sam2VisionModel being inherited here. _can_record_outputs = {} def __init__(self, config: EdgeTamVisionConfig): diff --git a/src/transformers/models/edgetam/modular_edgetam.py b/src/transformers/models/edgetam/modular_edgetam.py index 0744ef81f9f0..7113d1a8dadc 100644 --- a/src/transformers/models/edgetam/modular_edgetam.py +++ b/src/transformers/models/edgetam/modular_edgetam.py @@ -183,6 +183,8 @@ def _init_weights(self, module): class EdgeTamVisionModel(Sam2VisionModel): config_class = EdgeTamVisionConfig main_input_name = "pixel_values" + # TODO: TimmWrapper models aren't compatible with _can_record_outputs yet. We specifically set this to + # an empty dict to avoid the _can_record_outputs from Sam2VisionModel being inherited here. _can_record_outputs = {} def get_input_embeddings(self): From 355bcb411dfc3bfddf1e16755136b1046e8e965b Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 23 Jan 2026 10:15:28 +0100 Subject: [PATCH 130/131] Shrink Gemma3nAudioEncoderModelOutput with auto_docstring & superclass --- .../models/gemma3n/modeling_gemma3n.py | 30 ++----------------- .../models/gemma3n/modular_gemma3n.py | 30 ++----------------- 2 files changed, 6 insertions(+), 54 deletions(-) diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index ffc418030032..500e770c3df1 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -51,37 +51,13 @@ @dataclass +@auto_docstring class Gemma3nAudioEncoderModelOutput(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, audio_soft_tokens_per_image, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the audio encoder. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + - one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - - Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. - attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, - sequence_length)`. - - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention - heads. - audio_mel_mask (`torch.FloatTensor`, *optional*): - A torch.BoolTensor of shape `(batch_size, num_frames)` + audio_mel_mask (`torch.FloatTensor`, *optional*): + A torch.BoolTensor of shape `(batch_size, num_frames)` """ - last_hidden_state: torch.FloatTensor | None = None - pooler_output: torch.FloatTensor | None = None - hidden_states: tuple[torch.FloatTensor, ...] | None = None - attentions: tuple[torch.FloatTensor, ...] | None = None audio_mel_mask: torch.BoolTensor | None = None diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index 37a0471261a0..fd95a8b37366 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -632,37 +632,13 @@ def __init__( @dataclass +@auto_docstring class Gemma3nAudioEncoderModelOutput(BaseModelOutputWithPooling): """ - Base class for model's outputs that also contains a pooling of the last hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, audio_soft_tokens_per_image, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the audio encoder. - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Last layer hidden-state of the first token of the sequence (classification token) after further processing - through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns - the classification token after processing through a linear layer and a tanh activation function. The linear - layer weights are trained from the next sentence prediction (classification) objective during pretraining. - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + - one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - - Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. - attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, - sequence_length)`. - - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention - heads. - audio_mel_mask (`torch.FloatTensor`, *optional*): - A torch.BoolTensor of shape `(batch_size, num_frames)` + audio_mel_mask (`torch.FloatTensor`, *optional*): + A torch.BoolTensor of shape `(batch_size, num_frames)` """ - last_hidden_state: torch.FloatTensor | None = None - pooler_output: torch.FloatTensor | None = None - hidden_states: tuple[torch.FloatTensor, ...] | None = None - attentions: tuple[torch.FloatTensor, ...] | None = None audio_mel_mask: torch.BoolTensor | None = None From bf0ae70295c9460ce26e67f81a489484b19a8b40 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 23 Jan 2026 10:21:33 +0100 Subject: [PATCH 131/131] Revert "Unrelated but important: patch set_attn_implementation for Windows" This reverts commit 092321671197d878e48b1c89edd154f47bb43a30. --- src/transformers/modeling_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index ae2b0fc3778b..bab3a2a09220 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1909,7 +1909,7 @@ def _can_set_attn_implementation(cls) -> bool: opening the file, but avoids maintaining yet another property flag. """ class_file = sys.modules[cls.__module__].__file__ - with open(class_file, "r", encoding="utf8") as f: + with open(class_file, "r") as f: code = f.read() # heuristic -> if we find those patterns, the model uses the correct interface if re.search(r"class \w+Attention\(nn.Module\)", code): @@ -1927,7 +1927,7 @@ def _can_set_experts_implementation(cls) -> bool: opening the file, but avoids maintaining yet another property flag. """ class_file = sys.modules[cls.__module__].__file__ - with open(class_file, "r", encoding="utf8") as f: + with open(class_file, "r") as f: code = f.read() # heuristic -> if we the use_experts_implementation decorator is used, then we can set it return "@use_experts_implementation" in code