diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index f9c9143f164f..1638c75e5f5d 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -1509,6 +1509,8 @@ title: OneFormer - local: model_doc/ovis2 title: Ovis2 + - local: model_doc/ovis2_5 + title: Ovis2.5 - local: model_doc/owlvit title: OWL-ViT - local: model_doc/owlv2 diff --git a/docs/source/en/model_doc/ovis2_5.md b/docs/source/en/model_doc/ovis2_5.md new file mode 100644 index 000000000000..e80c77400fad --- /dev/null +++ b/docs/source/en/model_doc/ovis2_5.md @@ -0,0 +1,194 @@ + +*This model was published in HF papers on 2025-08-16 and contributed to Hugging Face Transformers on 2026-11-14.* + +# Ovis2.5 + +## Overview + +[Ovis2.5](https://huggingface.co/papers/2508.11737) is a multimodal model from +[AIDC-AI](https://github.com/AIDC-AI/Ovis) for image, video, and text understanding. It combines a native-resolution +vision encoder with a visual tokenizer and a Qwen3 language backbone. Unlike a fixed-resolution tiled encoder, the +vision encoder keeps variable-resolution visual inputs as a sequence of patches. Ovis2.5 also provides an optional +thinking prompt format for tasks that benefit from a longer reasoning response. + +Transformers-compatible checkpoints are available from the Ovis community: + +- [ovis-community/Ovis2.5-2B](https://huggingface.co/ovis-community/Ovis2.5-2B) +- [ovis-community/ovis-2.5-9B](https://huggingface.co/ovis-community/ovis-2.5-9B) + +These checkpoints retain the original weights and include native model, tokenizer, and processor metadata. The +original repositories still use legacy processor metadata. Weight names are mapped during loading. + +## Usage + + + +Ovis2.5 accepts exactly one video per request and does not support mixing images and video in the same request. + +The two-stage `thinking_budget` behavior implemented by the original custom remote code is not part of the native +Transformers generation API. + + + +### Image inference + +Use [`Ovis2_5Processor.apply_chat_template`] to load the image, format the conversation, and tokenize it in one call. +For the 9B checkpoint, replace the model ID. + +```python +from transformers import AutoModelForImageTextToText, AutoProcessor + + +model_id = "ovis-community/Ovis2.5-2B" +processor = AutoProcessor.from_pretrained(model_id) +model = AutoModelForImageTextToText.from_pretrained( + model_id, + device_map="auto", +) + +messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg", + }, + {"type": "text", "text": "Describe this image."}, + ], + } +] + +inputs = processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + enable_thinking=False, +).to(model.device, dtype=model.dtype) + +generated_ids = model.generate(**inputs, max_new_tokens=128) +generated_ids = generated_ids[:, inputs["input_ids"].shape[1] :] +response = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] +print(response) +``` + +Multiple images can be supplied by adding more `{"type": "image", ...}` items to the message in the same order in which +they should appear in the prompt. + +### Video inference + +The official example uniformly samples eight frames. Pass the video path directly in the conversation and let the +processor load and sample it. The processor prepares the inputs needed for generation. + +```python +messages = [ + { + "role": "user", + "content": [ + {"type": "video", "path": "path/to/video.mp4"}, + {"type": "text", "text": "Describe what happens in this video."}, + ], + } +] + +inputs = processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + enable_thinking=False, + processor_kwargs={ + "num_frames": 8, + "size": {"shortest_edge": 448 * 448, "longest_edge": 896 * 896}, + }, +).to(model.device, dtype=model.dtype) + +generated_ids = model.generate(**inputs, max_new_tokens=256) +generated_ids = generated_ids[:, inputs["input_ids"].shape[1] :] +response = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] +print(response) +``` + +### Thinking mode + +Set `enable_thinking=True` when applying the chat template to leave the model's reasoning block open. Set it to `False` +to request the immediate-answer prompt format. + +```python +inputs = processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + enable_thinking=True, +).to(model.device, dtype=model.dtype) + +generated_ids = model.generate(**inputs, max_new_tokens=1024) +``` + +`enable_thinking` changes the chat template; it is not a generation argument and does not impose a token budget. + +## Ovis2_5Config + +[[autodoc]] Ovis2_5Config + +## Ovis2_5VisionConfig + +[[autodoc]] Ovis2_5VisionConfig + +## Ovis2_5Processor + +[[autodoc]] Ovis2_5Processor + - __call__ + +## Ovis2_5ImageProcessor + +[[autodoc]] Ovis2_5ImageProcessor + - preprocess + +## Ovis2_5ImageProcessorPil + +[[autodoc]] Ovis2_5ImageProcessorPil + - preprocess + +## Ovis2_5VideoProcessor + +[[autodoc]] Ovis2_5VideoProcessor + - preprocess + +## Ovis2_5VisionModel + +[[autodoc]] Ovis2_5VisionModel + - forward + +## Ovis2_5Model + +[[autodoc]] Ovis2_5Model + - forward + - get_image_features + - get_video_features + +## Ovis2_5ForConditionalGeneration + +[[autodoc]] Ovis2_5ForConditionalGeneration + - forward + - get_image_features + - get_video_features diff --git a/src/transformers/conversion_mapping.py b/src/transformers/conversion_mapping.py index 2b73b499c91b..8997bd5994d4 100755 --- a/src/transformers/conversion_mapping.py +++ b/src/transformers/conversion_mapping.py @@ -185,6 +185,18 @@ def _build_checkpoint_conversion_mapping(): WeightRenaming(r"\.mlp\.up_proj\.", ".mlp.fc1."), WeightRenaming(r"\.mlp\.down_proj\.", ".mlp.fc2."), ], + "ovis2_5": [ + WeightRenaming(r"^llm\.model\.", "model.language_model."), + WeightRenaming(r"^llm\.lm_head\.", "lm_head."), + WeightRenaming(r"^visual_tokenizer\.vit\.vision_model\.embeddings\.", "model.vision_tower.embeddings."), + WeightRenaming(r"^visual_tokenizer\.vit\.vision_model\.encoder\.layers\.", "model.vision_tower.layers."), + WeightRenaming( + r"^visual_tokenizer\.vit\.vision_model\.post_layernorm\.", "model.vision_tower.post_layernorm." + ), + WeightRenaming(r"^visual_tokenizer\.head\.0\.", "model.multi_modal_projector.visual_tokenizer.linear."), + WeightRenaming(r"^visual_tokenizer\.head\.1\.", "model.multi_modal_projector.visual_tokenizer.norm."), + WeightRenaming(r"^vte\.", "model.multi_modal_projector.visual_embeddings_table."), + ], "inkling_mm_model": [ WeightRenaming(source_patterns=r"model\.llm\.layers", target_patterns=r"model.language_model.layers"), WeightRenaming( diff --git a/src/transformers/exporters/exporter_dynamo.py b/src/transformers/exporters/exporter_dynamo.py index 8ec81a5c6860..be3aad816af1 100644 --- a/src/transformers/exporters/exporter_dynamo.py +++ b/src/transformers/exporters/exporter_dynamo.py @@ -438,6 +438,7 @@ def _to_batched(t): # Separate `_proj` + `(cos, sin)` rotary + `.out_proj` (tuple return) "transformers.models.lfm3_vl.modeling_lfm3_vl.Lfm3VlVisionAttention.forward", "transformers.models.video_llama_3.modeling_video_llama_3.VideoLlama3VisionAttention.forward", + "transformers.models.ovis2_5.modeling_ovis2_5.Ovis2_5VisionAttention.forward", "transformers.models.paddleocr_vl.modeling_paddleocr_vl.PaddleOCRVisionAttention.forward", "transformers.models.minimax_m3_vl.modeling_minimax_m3_vl.MiniMaxM3VLVisionAttention.forward", "transformers.models.inkling2.modeling_inkling2.Inkling2VisionAttention.forward", diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index e1bdd0916349..4d0e616c9d32 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -373,6 +373,7 @@ from .openai_privacy_filter import * from .opt import * from .ovis2 import * + from .ovis2_5 import * from .owlv2 import * from .owlvit import * from .paddleocr_vl import * diff --git a/src/transformers/models/auto/auto_mappings.py b/src/transformers/models/auto/auto_mappings.py index 6a80c92d5600..ec3e6fbe12f8 100644 --- a/src/transformers/models/auto/auto_mappings.py +++ b/src/transformers/models/auto/auto_mappings.py @@ -514,6 +514,8 @@ ("openai_privacy_filter", "OpenAIPrivacyFilterConfig"), ("opt", "OPTConfig"), ("ovis2", "Ovis2Config"), + ("ovis2_5", "Ovis2_5Config"), + ("ovis2_5_vision", "Ovis2_5VisionConfig"), ("ovis2_vision", "Ovis2VisionConfig"), ("owlv2", "Owlv2Config"), ("owlv2_text_model", "Owlv2TextConfig"), @@ -1002,6 +1004,7 @@ ("nllb-moe", "nllb_moe"), ("omdet-turbo", "omdet_turbo"), ("openai-gpt", "openai"), + ("ovis2_5_vision", "ovis2_5"), ("ovis2_vision", "ovis2"), ("owlv2_text_model", "owlv2"), ("owlv2_vision_model", "owlv2"), @@ -1150,6 +1153,7 @@ ("molmo2", {"torchvision": "Molmo2VideoProcessor"}), ("muse_glimmer", {"torchvision": "MuseGlimmerVideoProcessor"}), ("nemotron_h_omni", {"torchvision": "NemotronH_Omni_Reasoning_V3VideoProcessor"}), + ("ovis2_5", {"torchvision": "Ovis2_5VideoProcessor"}), ("pe_video", {"torchvision": "PeVideoVideoProcessor"}), ("perception_lm", {"torchvision": "PerceptionLMVideoProcessor"}), ("qwen2_vl", {"torchvision": "Qwen2VLVideoProcessor"}), @@ -1308,6 +1312,7 @@ ("omdet-turbo", "OmDetTurboProcessor"), ("oneformer", "OneFormerProcessor"), ("ovis2", "Ovis2Processor"), + ("ovis2_5", "Ovis2_5Processor"), ("owlv2", "Owlv2Processor"), ("owlvit", "OwlViTProcessor"), ("paddleocr_vl", "PaddleOCRVLProcessor"), @@ -1450,6 +1455,7 @@ ("nougat", {"pil": "NougatImageProcessorPil", "torchvision": "NougatImageProcessor"}), ("oneformer", {"pil": "OneFormerImageProcessorPil", "torchvision": "OneFormerImageProcessor"}), ("ovis2", {"pil": "Ovis2ImageProcessorPil", "torchvision": "Ovis2ImageProcessor"}), + ("ovis2_5", {"pil": "Ovis2_5ImageProcessorPil", "torchvision": "Ovis2_5ImageProcessor"}), ("owlv2", {"pil": "Owlv2ImageProcessorPil", "torchvision": "Owlv2ImageProcessor"}), ("owlvit", {"pil": "OwlViTImageProcessorPil", "torchvision": "OwlViTImageProcessor"}), ("paddleocr_vl", {"pil": "PaddleOCRVLImageProcessorPil", "torchvision": "PaddleOCRVLImageProcessor"}), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 56a0ed1ad4c9..da67071e0611 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -435,6 +435,8 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("openai_privacy_filter", "OpenAIPrivacyFilterModel"), ("opt", "OPTModel"), ("ovis2", "Ovis2Model"), + ("ovis2_5", "Ovis2_5Model"), + ("ovis2_5_vision", "Ovis2_5VisionModel"), ("owlv2", "Owlv2Model"), ("owlvit", "OwlViTModel"), ("paddleocr_vl", "PaddleOCRVLModel"), @@ -1230,6 +1232,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("muse_spark", "MuseSparkForConditionalGeneration"), ("nemotron_h_omni", "NemotronH_Omni_Reasoning_V3"), ("ovis2", "Ovis2ForConditionalGeneration"), + ("ovis2_5", "Ovis2_5ForConditionalGeneration"), ("paddleocr_vl", "PaddleOCRVLForConditionalGeneration"), ("paligemma", "PaliGemmaForConditionalGeneration"), ("perception_lm", "PerceptionLMForConditionalGeneration"), diff --git a/src/transformers/models/ovis2_5/__init__.py b/src/transformers/models/ovis2_5/__init__.py new file mode 100644 index 000000000000..91d50f10ec64 --- /dev/null +++ b/src/transformers/models/ovis2_5/__init__.py @@ -0,0 +1,37 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from the modules in `src/transformers/models/ovis2_5`. +# Do NOT edit this file manually as any edits will be overwritten by auto-generation of the file. +# A module is picked up once it exports a name through `__all__`. +# Regenerate the file with: `python utils/check_inits.py --fix_and_overwrite` +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .configuration_ovis2_5 import * + from .image_processing_ovis2_5 import * + from .image_processing_pil_ovis2_5 import * + from .modeling_ovis2_5 import * + from .processing_ovis2_5 import * + from .video_processing_ovis2_5 import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/src/transformers/models/ovis2_5/configuration_ovis2_5.py b/src/transformers/models/ovis2_5/configuration_ovis2_5.py new file mode 100644 index 000000000000..e4cca641ec53 --- /dev/null +++ b/src/transformers/models/ovis2_5/configuration_ovis2_5.py @@ -0,0 +1,124 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/ovis2_5/modular_ovis2_5.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_ovis2_5.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from huggingface_hub.dataclasses import strict + +from ...configuration_utils import PreTrainedConfig, SubConfigSpec +from ...utils import auto_docstring +from ..auto import AutoConfig + + +@auto_docstring(checkpoint="ovis-community/Ovis2.5-2B") +@strict +class Ovis2_5VisionConfig(PreTrainedConfig): + r""" + window_size (`int`, *optional*, defaults to 112): + Window size, in input pixels, used by windowed vision-attention layers. + temporal_patch_size (`int`, *optional*, defaults to 1): + Number of consecutive video frames represented by one temporal patch. + vocab_size (`int`, *optional*, defaults to 65536): + Size of the visual-token vocabulary, including four learned visual-boundary indicators. + num_visual_indicator_tokens (`int`, *optional*, defaults to 4): + Number of visual-vocabulary rows reserved for image/video boundary indicators. + """ + + model_type = "ovis2_5_vision" + base_config_key = "vision_config" + hidden_size: int = 1152 + intermediate_size: int = 4304 + num_hidden_layers: int = 27 + num_attention_heads: int = 16 + num_channels: int = 3 + patch_size: int = 16 + hidden_act: str = "gelu_pytorch_tanh" + layer_norm_eps: float = 1e-6 + attention_dropout: float | int = 0.0 + spatial_merge_size: int = 2 + default_rope_type = "axial" + + image_size: int = 512 + window_size: int = 112 + layer_types: list[str] | tuple[str, ...] | None = None + temporal_patch_size: int = 1 + vocab_size: int = 65536 + num_visual_indicator_tokens: int = 4 + initializer_range: float = 0.02 + max_position_embeddings: int = 1024 + rope_parameters: dict | None = None + + def __post_init__(self, **kwargs): + full_attention_indexes = kwargs.get("fullatt_block_indexes") + if self.layer_types is None: + if full_attention_indexes is None: + self.layer_types = ["full_attention"] * self.num_hidden_layers + else: + if isinstance(full_attention_indexes, str): + full_attention_indexes = [int(index) for index in full_attention_indexes.split("|") if index] + full_attention_indexes = set(full_attention_indexes) + self.layer_types = [ + "full_attention" if layer_index in full_attention_indexes else "sliding_attention" + for layer_index in range(self.num_hidden_layers) + ] + else: + self.layer_types = list(self.layer_types) + super().__post_init__(**kwargs) + + +@auto_docstring(checkpoint="ovis-community/Ovis2.5-2B") +@strict +class Ovis2_5Config(PreTrainedConfig): + r""" + image_start_token_id (`int`, *optional*, defaults to 151670): + Text-vocabulary token replaced by the learned image-begin visual embedding. + image_end_token_id (`int`, *optional*, defaults to 151671): + Text-vocabulary token replaced by the learned image-end visual embedding. + video_start_token_id (`int`, *optional*, defaults to 151672): + Text-vocabulary token replaced by the learned video-begin visual embedding. + video_end_token_id (`int`, *optional*, defaults to 151673): + Text-vocabulary token replaced by the learned video-end visual embedding. + """ + + model_type = "ovis2_5" + sub_configs_defaults = { + "vision_config": Ovis2_5VisionConfig, + "text_config": SubConfigSpec(config_class=AutoConfig, model_type="qwen3"), + } + keys_to_ignore_at_inference = ["cache"] + + text_config: PreTrainedConfig | dict | None = None + vision_config: PreTrainedConfig | dict | None = None + image_token_id: int = 151669 + video_token_id: int = 151669 + image_start_token_id: int = 151670 + image_end_token_id: int = 151671 + video_start_token_id: int = 151672 + video_end_token_id: int = 151673 + tie_word_embeddings: bool = False + + def __post_init__(self, **kwargs): + if isinstance(self.vision_config, dict): + # Released configs label this remote-code tower as `siglip2_navit`; the native tower type is fixed here. + self.vision_config.pop("model_type", None) + self.vision_config.pop("preserve_original_pe", None) + self.vision_config.pop("use_rope", None) + self.vision_config.pop("num_patches", None) + super().__post_init__(**kwargs) + + +__all__ = ["Ovis2_5Config", "Ovis2_5VisionConfig"] diff --git a/src/transformers/models/ovis2_5/image_processing_ovis2_5.py b/src/transformers/models/ovis2_5/image_processing_ovis2_5.py new file mode 100644 index 000000000000..e6e4d9bf0090 --- /dev/null +++ b/src/transformers/models/ovis2_5/image_processing_ovis2_5.py @@ -0,0 +1,271 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/ovis2_5/modular_ovis2_5.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_ovis2_5.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import torch +from torchvision.transforms.v2 import functional as tvF + +from ...image_processing_backends import TorchvisionBackend +from ...image_processing_utils import BatchFeature +from ...image_transforms import group_images_by_shape, reorder_images +from ...image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, ImageInput, PILImageResampling, SizeDict +from ...processing_utils import ImagesKwargs, Unpack +from ...utils import TensorType, auto_docstring + + +class Ovis2_5ImageProcessorKwargs(ImagesKwargs, total=False): + r""" + patch_size (`int`, *optional*, defaults to 16): + The spatial patch size of the vision encoder. + temporal_patch_size (`int`, *optional*, defaults to 1): + The temporal patch size of the vision encoder. + merge_size (`int`, *optional*, defaults to 2): + The spatial merge size used by the visual tokenizer. + """ + + patch_size: int + temporal_patch_size: int + merge_size: int + + +def smart_resize( + height: int, + width: int, + factor: int = 32, + min_pixels: int = 448 * 448, + max_pixels: int = 1344 * 1792, +) -> tuple[int, int]: + """Rescales the image so that the following conditions are met: + + 1. Both dimensions (height and width) are divisible by 'factor'. + + 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. + + 3. The aspect ratio of the image is maintained as closely as possible. + + """ + # Unlike Qwen, Ovis expands dimensions below the factor and clamps aspect ratios above 200. + if height < factor or width < factor: + if height < width: + width = round(factor / height * width) + height = factor + else: + height = round(factor / width * height) + width = factor + elif max(height, width) / min(height, width) > 200: + if height > width: + height = 200 * width + else: + width = 200 * height + + h_bar = round(height / factor) * factor + w_bar = round(width / factor) * factor + if h_bar * w_bar > max_pixels: + beta = math.sqrt((height * width) / max_pixels) + h_bar = math.floor(height / beta / factor) * factor + w_bar = math.floor(width / beta / factor) * factor + elif h_bar * w_bar < min_pixels: + beta = math.sqrt(min_pixels / (height * width)) + h_bar = math.ceil(height * beta / factor) * factor + w_bar = math.ceil(width * beta / factor) * factor + return h_bar, w_bar + + +@auto_docstring +class Ovis2_5ImageProcessor(TorchvisionBackend): + do_resize = True + resample = PILImageResampling.BILINEAR + size = {"shortest_edge": 448 * 448, "longest_edge": 1344 * 1792} + default_to_square = False + do_rescale = True + rescale_factor = 1 / 255 + do_normalize = True + image_mean = IMAGENET_STANDARD_MEAN + image_std = IMAGENET_STANDARD_STD + do_convert_rgb = True + patch_size = 16 + temporal_patch_size = 1 + merge_size = 2 + valid_kwargs = Ovis2_5ImageProcessorKwargs + model_input_names = ["pixel_values", "image_grid_thw"] + + @auto_docstring + def preprocess(self, images: ImageInput, **kwargs: Unpack[Ovis2_5ImageProcessorKwargs]) -> BatchFeature: + return super().preprocess(images, **kwargs) + + def resize( + self, + images: torch.Tensor, + size: SizeDict, + resample: PILImageResampling | int | None, + factor: int, + temporal_factor: int, + **kwargs, + ) -> torch.Tensor: + """Resize dynamically based on input image aspect ratio.""" + if not size.shortest_edge or not size.longest_edge: + raise ValueError(f"`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got {size}.") + + # Ovis2.5 uses its custom smart_resize policy instead of GLM4V's default resize. + height, width = images.shape[-2:] + resized_height, resized_width = smart_resize( + height, + width, + factor=factor, + min_pixels=size.shortest_edge, + max_pixels=size.longest_edge, + ) + return super().resize( + image=images, + size=SizeDict(height=resized_height, width=resized_width), + resample=resample, + ) + + def patchify( + self, + images: "torch.Tensor", + patch_size: int, + merge_size: int, + temporal_patch_size: int, + ) -> tuple["torch.Tensor", int, int]: + """Patchifies each image into flat layout of shape (`seq_len`, `patch_dim`) so we can concat dynamically shaped pixels.""" + batch_size, channel, resized_height, resized_width = images.shape + grid_h, grid_w = resized_height // patch_size, resized_width // patch_size + patches = images.reshape( + batch_size, + channel, + grid_h // merge_size, + merge_size, + patch_size, + grid_w // merge_size, + merge_size, + patch_size, + ) + patches = patches.permute(0, 2, 5, 3, 6, 1, 4, 7) + flatten_patches = ( + patches.unsqueeze(6) + .expand(-1, -1, -1, -1, -1, -1, temporal_patch_size, -1, -1) + .reshape( + batch_size, + grid_h * grid_w, + channel * temporal_patch_size * patch_size * patch_size, + ) + ) + return flatten_patches, grid_h, grid_w + + def _preprocess( + self, + images: list["torch.Tensor"], + do_resize: bool, + size: SizeDict, + resample: "PILImageResampling | tvF.InterpolationMode | int | None", + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: float | list[float] | None, + image_std: float | list[float] | None, + patch_size: int, + temporal_patch_size: int, + merge_size: int, + disable_grouping: bool | None, + return_tensors: str | TensorType | None, + **kwargs, + ) -> BatchFeature: + """ + Preprocess an image or batch of images. + """ + + grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping) + resized_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + if do_resize: + stacked_images = self.resize( + images=stacked_images, + size=size, + resample=resample, + factor=patch_size * merge_size, + temporal_factor=temporal_patch_size, + ) + resized_images_grouped[shape] = stacked_images + + resized_images = reorder_images(resized_images_grouped, grouped_images_index) + + grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping) + processed_images_grouped = {} + processed_grids = {} + + for shape, stacked_images in grouped_images.items(): + stacked_images = self.rescale_and_normalize( + stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std + ) + + patches, grid_h, grid_w = self.patchify( + stacked_images, + patch_size=patch_size, + merge_size=merge_size, + temporal_patch_size=temporal_patch_size, + ) + + processed_images_grouped[shape] = patches + processed_grids[shape] = [[1, grid_h, grid_w]] * len(stacked_images) + + processed_images = reorder_images(processed_images_grouped, grouped_images_index) + processed_grids = reorder_images(processed_grids, grouped_images_index) + + pixel_values = processed_images[0] if len(processed_images) == 1 else torch.cat(processed_images, dim=0) + image_grid_thw = torch.tensor(processed_grids) + + return BatchFeature( + data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}, tensor_type=return_tensors + ) + + def get_number_of_image_patches(self, height: int, width: int, images_kwargs: dict | None = None) -> int: + """ + A utility that returns number of image patches for a given image size. + + Args: + height (`int`): + Height of the input image. + width (`int`): + Width of the input image. + images_kwargs (`dict`, *optional*) + Any kwargs to override defaults of the image processor. + Returns: + `int`: Number of image patches per image. + """ + images_kwargs = images_kwargs or {} + patch_size = images_kwargs.get("patch_size", self.patch_size) + merge_size = images_kwargs.get("merge_size", self.merge_size) + size = images_kwargs.get("size", self.size) + min_pixels = size["shortest_edge"] if isinstance(size, dict) else size.shortest_edge + max_pixels = size["longest_edge"] if isinstance(size, dict) else size.longest_edge + # Ovis2.5 uses the same custom smart_resize policy to compute the patch count. + resized_height, resized_width = smart_resize( + height, + width, + factor=patch_size * merge_size, + min_pixels=min_pixels, + max_pixels=max_pixels, + ) + return (resized_height // patch_size) * (resized_width // patch_size) + + +__all__ = ["Ovis2_5ImageProcessor"] diff --git a/src/transformers/models/ovis2_5/image_processing_pil_ovis2_5.py b/src/transformers/models/ovis2_5/image_processing_pil_ovis2_5.py new file mode 100644 index 000000000000..fafeb54bd624 --- /dev/null +++ b/src/transformers/models/ovis2_5/image_processing_pil_ovis2_5.py @@ -0,0 +1,267 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/ovis2_5/modular_ovis2_5.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_ovis2_5.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Any + +import numpy as np + +from ...image_processing_backends import PilBackend +from ...image_processing_utils import BatchFeature +from ...image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, ImageInput, PILImageResampling, SizeDict +from ...processing_utils import ImagesKwargs, Unpack +from ...utils import TensorType, auto_docstring + + +class Ovis2_5ImageProcessorKwargs(ImagesKwargs, total=False): + r""" + patch_size (`int`, *optional*, defaults to 16): + The spatial patch size of the vision encoder. + temporal_patch_size (`int`, *optional*, defaults to 1): + The temporal patch size of the vision encoder. + merge_size (`int`, *optional*, defaults to 2): + The spatial merge size used by the visual tokenizer. + """ + + patch_size: int + temporal_patch_size: int + merge_size: int + + +def smart_resize( + height: int, + width: int, + factor: int = 32, + min_pixels: int = 448 * 448, + max_pixels: int = 1344 * 1792, +) -> tuple[int, int]: + """Rescales the image so that the following conditions are met: + + 1. Both dimensions (height and width) are divisible by 'factor'. + + 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. + + 3. The aspect ratio of the image is maintained as closely as possible. + + """ + # Unlike Qwen, Ovis expands dimensions below the factor and clamps aspect ratios above 200. + if height < factor or width < factor: + if height < width: + width = round(factor / height * width) + height = factor + else: + height = round(factor / width * height) + width = factor + elif max(height, width) / min(height, width) > 200: + if height > width: + height = 200 * width + else: + width = 200 * height + + h_bar = round(height / factor) * factor + w_bar = round(width / factor) * factor + if h_bar * w_bar > max_pixels: + beta = math.sqrt((height * width) / max_pixels) + h_bar = math.floor(height / beta / factor) * factor + w_bar = math.floor(width / beta / factor) * factor + elif h_bar * w_bar < min_pixels: + beta = math.sqrt(min_pixels / (height * width)) + h_bar = math.ceil(height * beta / factor) * factor + w_bar = math.ceil(width * beta / factor) * factor + return h_bar, w_bar + + +@auto_docstring +class Ovis2_5ImageProcessorPil(PilBackend): + do_resize = True + resample = PILImageResampling.BILINEAR + size = {"shortest_edge": 448 * 448, "longest_edge": 1344 * 1792} + default_to_square = False + do_rescale = True + rescale_factor = 1 / 255 + do_normalize = True + image_mean = IMAGENET_STANDARD_MEAN + image_std = IMAGENET_STANDARD_STD + do_convert_rgb = True + patch_size = 16 + temporal_patch_size = 1 + merge_size = 2 + valid_kwargs = Ovis2_5ImageProcessorKwargs + model_input_names = ["pixel_values", "image_grid_thw"] + + @auto_docstring + def preprocess(self, images: ImageInput, **kwargs: Unpack[Ovis2_5ImageProcessorKwargs]) -> BatchFeature: + return super().preprocess(images, **kwargs) + + def resize( + self, + image: Any, + size: SizeDict, + resample: PILImageResampling | int | None, + factor: int, + temporal_factor: int, + **kwargs, + ) -> Any: + """Resize dynamically based on input image aspect ratio.""" + if not size.shortest_edge or not size.longest_edge: + raise ValueError(f"`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got {size}.") + + # Ovis2.5 uses its custom smart_resize policy instead of GLM4V's default resize. + height, width = image.shape[-2:] + resized_height, resized_width = smart_resize( + height, + width, + factor=factor, + min_pixels=size.shortest_edge, + max_pixels=size.longest_edge, + ) + return super().resize( + image=image, + size=SizeDict(height=resized_height, width=resized_width), + resample=resample, + ) + + def patchify( + self, + image: np.ndarray, + patch_size: int, + merge_size: int, + temporal_patch_size: int, + ) -> tuple[np.ndarray, int, int]: + """Patchifies each image into flat layout of shape (`seq_len`, `patch_dim`) so we can concat dynamically shaped pixels.""" + # Ensure float32 for patch processing + image = np.asarray(image, dtype=np.float32) + channel, resized_height, resized_width = image.shape + grid_h, grid_w = resized_height // patch_size, resized_width // patch_size + + patches = image.reshape( + channel, + grid_h // merge_size, + merge_size, + patch_size, + grid_w // merge_size, + merge_size, + patch_size, + ) + # (gh, gw, mh, mw, C, ph, pw) + patches = np.transpose(patches, (1, 4, 2, 5, 0, 3, 6)) + + # expand temporal_patch_size as a broadcast (zero-copy) + patches = np.broadcast_to( + patches[:, :, :, :, :, None, :, :], + (*patches.shape[:5], temporal_patch_size, *patches.shape[5:]), + ) + + flatten_patches = patches.reshape( + grid_h * grid_w, + channel * temporal_patch_size * patch_size * patch_size, + ) + return flatten_patches, grid_h, grid_w + + def _preprocess( + self, + images: list[np.ndarray], + do_resize: bool, + size: SizeDict, + resample: "PILImageResampling | None", + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: float | list[float] | None, + image_std: float | list[float] | None, + patch_size: int, + temporal_patch_size: int, + merge_size: int, + return_tensors: str | TensorType | None, + **kwargs, + ) -> BatchFeature: + """ + Preprocess images one by one for PIL backend. + """ + processed_images = [] + processed_grids = [] + + for image in images: + if do_resize: + image = self.resize( + image, + size=size, + resample=resample, + factor=patch_size * merge_size, + temporal_factor=temporal_patch_size, + ) + + # Rescale and normalize + if do_rescale: + image = self.rescale(image, rescale_factor) + if do_normalize: + image = self.normalize(image, image_mean, image_std) + + patches, grid_h, grid_w = self.patchify( + image, + patch_size=patch_size, + merge_size=merge_size, + temporal_patch_size=temporal_patch_size, + ) + + # Remove batch dimension and append: shape is (seq_len, hidden_dim) + processed_images.append(patches) + processed_grids.append([1, grid_h, grid_w]) + + # Concatenate all images along sequence dimension: (total_seq_len, hidden_dim) + pixel_values = np.concatenate(processed_images, axis=0) + image_grid_thw = np.array(processed_grids) + + return BatchFeature( + data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}, tensor_type=return_tensors + ) + + def get_number_of_image_patches(self, height: int, width: int, images_kwargs: dict | None = None) -> int: + """ + A utility that returns number of image patches for a given image size. + + Args: + height (`int`): + Height of the input image. + width (`int`): + Width of the input image. + images_kwargs (`dict`, *optional*) + Any kwargs to override defaults of the image processor. + Returns: + `int`: Number of image patches per image. + """ + images_kwargs = images_kwargs or {} + patch_size = images_kwargs.get("patch_size", self.patch_size) + merge_size = images_kwargs.get("merge_size", self.merge_size) + size = images_kwargs.get("size", self.size) + min_pixels = size["shortest_edge"] if isinstance(size, dict) else size.shortest_edge + max_pixels = size["longest_edge"] if isinstance(size, dict) else size.longest_edge + # Ovis2.5 uses the same custom smart_resize policy to compute the patch count. + resized_height, resized_width = smart_resize( + height, + width, + factor=patch_size * merge_size, + min_pixels=min_pixels, + max_pixels=max_pixels, + ) + return (resized_height // patch_size) * (resized_width // patch_size) + + +__all__ = ["Ovis2_5ImageProcessorPil"] diff --git a/src/transformers/models/ovis2_5/modeling_ovis2_5.py b/src/transformers/models/ovis2_5/modeling_ovis2_5.py new file mode 100644 index 000000000000..973ed2bdb7ab --- /dev/null +++ b/src/transformers/models/ovis2_5/modeling_ovis2_5.py @@ -0,0 +1,975 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/ovis2_5/modular_ovis2_5.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_ovis2_5.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Callable +from dataclasses import dataclass + +import torch +from torch import nn + +from ... import initialization as init +from ...activations import ACT2FN +from ...cache_utils import Cache +from ...generation import GenerationMixin +from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, CausalLMOutputWithPast +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel, slice_logits_to_keep +from ...processing_utils import Unpack +from ...utils import ( + TransformersKwargs, + auto_docstring, + can_return_tuple, + torch_compilable_check, +) +from ...utils.deprecation import deprecate_kwarg +from ...utils.generic import ( + accepts_precomputed_kwargs, + get_max_seqlen, + is_flash_attention_requested, + merge_with_config_defaults, +) +from ...utils.output_capturing import capture_outputs +from ...vision_utils import ( + get_vision_attention_seqlens, + get_vision_interpolation_indices_and_weights, + get_vision_position_ids, + get_vision_window_index, +) +from ..auto import AutoModel +from .configuration_ovis2_5 import Ovis2_5Config, Ovis2_5VisionConfig + + +class Ovis2_5VisionRotaryEmbedding(nn.Module): + """ + Simple axial 2D rope with same freqs used for H and W grids. The freqs are + pre-computed using `head-dim//4` which is later used to concat H and W positions. + The final angles rotate over the whole head dim, no partial rotation involved. + """ + + def __init__(self, config: Ovis2_5VisionConfig): + super().__init__() + self.config = config + + self.rope_type = self.config.rope_parameters["rope_type"] + rope_init_fn: Callable = self.compute_axial_rope_parameters + if self.rope_type != "axial": + raise ValueError(f"{self.__class__.__name__} supports only axial rope, but requested {self.rope_type}") + inv_freq, self.attention_scaling = rope_init_fn(self.config) + + self.inv_freq = nn.Buffer(inv_freq, persistent=False) + self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False) + + @staticmethod + def compute_axial_rope_parameters( + config: Ovis2_5VisionConfig, device=None, **kwargs + ) -> tuple[torch.Tensor, float]: + """ + Computes the inverse frequencies according to the original RoPE implementation + Args: + config ([`~transformers.PreTrainedConfig`]): + The model configuration. + Returns: + Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + """ + base = config.rope_parameters["rope_theta"] + dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + spatial_dim = dim // 2 + + attention_factor = 1.0 # Unused in this type of RoPE + inv_freq = 1.0 / (base ** (torch.arange(0, spatial_dim, 2, dtype=torch.float) / spatial_dim)) + return inv_freq.to(device), attention_factor + + @torch.no_grad() + def forward(self, x, position_ids): + # position_ids: (2, N) — row 0 = h coords, row 1 = w coords + freqs = position_ids[..., None].float() * self.inv_freq.to(device=x.device, dtype=torch.float) + cos = freqs.cos() * self.attention_scaling + sin = freqs.sin() * self.attention_scaling + + cos = self.recomposition_frequencies(cos) + sin = self.recomposition_frequencies(sin) + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + def recomposition_frequencies(self, freq): + """ + Recompose the frequencies into the final spatial layout used per each grid. + """ + freq_h, freq_w = freq[:, 0], freq[:, 1] + freq_hw = torch.cat([freq_h, freq_w], dim=-1) + return torch.cat([freq_hw, freq_hw], dim=-1) + + +class Ovis2_5VisionEmbeddings(nn.Module): + """Patch embeddings and interpolated learned positions for packed Ovis2.5 inputs.""" + + def __init__(self, config: Ovis2_5VisionConfig): + super().__init__() + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.patch_embedding = nn.Conv2d( + in_channels=config.num_channels, + out_channels=self.embed_dim, + kernel_size=self.patch_size, + stride=self.patch_size, + padding="valid", + ) + + self.num_patches_per_side = self.image_size // self.patch_size + self.num_patches = self.num_patches_per_side**2 + self.num_positions = self.num_patches + self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) + self.config = config + # How the (square) learned position grid is resampled to each image's grid. + self.num_grid_per_side = int(self.num_positions**0.5) + self.interpolation_align_corners = False + self.interpolation_mode = "bicubic" + self.spatial_merge_size = config.spatial_merge_size + + def forward( + self, + pixel_values: torch.FloatTensor, + grid_thw: torch.LongTensor, + **kwargs: Unpack[TransformersKwargs], + ) -> torch.Tensor: + """ + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, sequence_length, image_channels, patch_size, patch_size)`): + The tensors corresponding to the input images. + grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ + interp_indices, interp_weights = get_vision_interpolation_indices_and_weights( + grid_thw, + self.num_grid_per_side, + mode=self.interpolation_mode, + align_corners=self.interpolation_align_corners, + spatial_merge_size=self.spatial_merge_size, + kwargs=kwargs, + ) + pixel_values = pixel_values.view( + -1, + 1, + self.config.num_channels * self.config.temporal_patch_size, + self.patch_size, + self.patch_size, + ) + batch_size, sequence_len, channel, height, width = pixel_values.shape + target_dtype = self.patch_embedding.weight.dtype + pixel_values = pixel_values.reshape(batch_size * sequence_len, channel, height, width) + patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype)) # shape = [*, width, grid, grid] + embeddings = patch_embeds.flatten(-2).squeeze(-1) + embeddings = embeddings.reshape(batch_size * sequence_len, -1) + + interp_weights = interp_weights.to(self.position_embedding.weight.dtype) + pos_embeds = (self.position_embedding(interp_indices) * interp_weights[:, :, None]).sum(1) + embeddings = embeddings + pos_embeds.to(embeddings.dtype) + + return embeddings + + +class Ovis2_5VisionMLP(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.activation_fn = ACT2FN[config.hidden_act] + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size) + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states = self.fc2(hidden_states) + return hidden_states + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.Tensor | None, + scaling: float, + dropout: float = 0.0, + **kwargs, +): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def apply_rotary_pos_emb_vision( + q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + orig_q_dtype = q.dtype + orig_k_dtype = k.dtype + q, k = q.float(), k.float() + cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float() + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + q_embed = q_embed.to(orig_q_dtype) + k_embed = k_embed.to(orig_k_dtype) + return q_embed, k_embed + + +class Ovis2_5VisionAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" + f" {self.num_heads})." + ) + self.is_causal = False + + self.k_proj = nn.Linear(self.embed_dim, self.embed_dim) + self.v_proj = nn.Linear(self.embed_dim, self.embed_dim) + self.q_proj = nn.Linear(self.embed_dim, self.embed_dim) + self.out_proj = nn.Linear(self.embed_dim, self.embed_dim) + self.num_key_value_groups = 1 + self.scaling = self.head_dim**-0.5 + self.attention_dropout = config.attention_dropout + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + max_seqlen: int | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple[torch.Tensor, torch.Tensor | None]: + """ + Args: + hidden_states (`torch.Tensor`): + Input to the layer of shape `(seq_len, embed_dim)`. + cu_seqlens (`torch.Tensor` of shape `(num_images_or_videos + 1,)`): + The cumulative sequence lengths of each image or video feature. + position_embeddings (`tuple(torch.Tensor, torch.Tensor)` of shape `(num_patches, head_dim // 2)`): + The cosine and sine position embeddings for vision attention. + """ + seq_length = hidden_states.shape[0] + query_states = self.q_proj(hidden_states).view(seq_length, self.num_heads, self.head_dim) + key_states = self.k_proj(hidden_states).view(seq_length, self.num_heads, self.head_dim) + value_states = self.v_proj(hidden_states).view(seq_length, self.num_heads, self.head_dim) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin) + + query_states = query_states.transpose(0, 1).unsqueeze(0) + key_states = key_states.transpose(0, 1).unsqueeze(0) + value_states = value_states.transpose(0, 1).unsqueeze(0) + + attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( + self.config._attn_implementation, eager_attention_forward + ) + + if is_flash_attention_requested(self.config): + # Flash Attention 2: Use cu_seqlens for variable length attention + max_seqlen = get_max_seqlen(cu_seqlens, self.config, kwargs={"max_seqlen": max_seqlen}) + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask=None, + scaling=self.scaling, + dropout=0.0 if not self.training else self.attention_dropout, + cu_seq_lens_q=cu_seqlens, + cu_seq_lens_k=cu_seqlens, + max_length_q=max_seqlen, + max_length_k=max_seqlen, + is_causal=False, + **kwargs, + ) + else: + # Other implementations: Process each chunk separately + lengths = cu_seqlens[1:] - cu_seqlens[:-1] + splits = [ + torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states) + ] + + attn_outputs, attn_weights = [], [] + for q, k, v in zip(*splits): + attn_output, attn_weight = attention_interface( + self, + q, + k, + v, + attention_mask=None, + scaling=self.scaling, + dropout=0.0 if not self.training else self.attention_dropout, + is_causal=False, + **kwargs, + ) + attn_outputs.append(attn_output) + attn_weights.append(attn_weight) + + attn_output = torch.cat(attn_outputs, dim=1) + + attn_output = attn_output.reshape(seq_length, -1).contiguous() + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights + + +class Ovis2_5VisionEncoderLayer(GradientCheckpointingLayer): + def __init__(self, config: Ovis2_5VisionConfig): + super().__init__() + self.embed_dim = config.hidden_size + self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps) + self.self_attn = Ovis2_5VisionAttention(config=config) + self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps) + self.mlp = Ovis2_5VisionMLP(config=config) + + @auto_docstring + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + **kwargs: Unpack[TransformersKwargs], + ) -> torch.Tensor: + r""" + cu_seqlens (`torch.Tensor` of shape `(num_images_or_videos + 1,)`): + The cumulative sequence lengths of each image or video feature. + position_embeddings (`tuple(torch.Tensor, torch.Tensor)` of shape `(num_patches, head_dim // 2)`): + The cosine and sine position embeddings for vision attention. + """ + residual = hidden_states + + hidden_states = self.layer_norm1(hidden_states) + hidden_states, _ = self.self_attn( + hidden_states, + cu_seqlens=cu_seqlens, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + +class Ovis2_5VisualTokenProjector(nn.Module): + def __init__(self, config: Ovis2_5VisionConfig): + super().__init__() + self.spatial_merge_unit = config.spatial_merge_size**2 + self.num_visual_indicator_tokens = config.num_visual_indicator_tokens + self.vocab_size = config.vocab_size + visual_token_vocab_size = config.vocab_size - config.num_visual_indicator_tokens + self.linear = nn.Linear( + config.hidden_size * self.spatial_merge_unit, + visual_token_vocab_size, + bias=False, + ) + self.norm = nn.LayerNorm(visual_token_vocab_size) + self.indicator_padding = nn.Buffer(torch.zeros(config.num_visual_indicator_tokens), persistent=False) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = hidden_states.reshape(hidden_states.shape[0] // self.spatial_merge_unit, -1) + logits = self.norm(self.linear(hidden_states)) + visual_tokens = torch.softmax(logits, dim=-1, dtype=torch.float32).to(logits.dtype) + indicator_padding = self.indicator_padding.expand(visual_tokens.shape[0], -1) + return torch.cat((visual_tokens, indicator_padding), dim=-1) + + def get_visual_indicator_token_ids(self, device: torch.device) -> torch.Tensor: + return torch.arange( + self.vocab_size - self.num_visual_indicator_tokens, + self.vocab_size, + dtype=torch.long, + device=device, + ) + + +class Ovis2_5MultiModalProjector(nn.Module): + def __init__(self, config: Ovis2_5VisionConfig, text_hidden_size: int): + super().__init__() + self.visual_tokenizer = Ovis2_5VisualTokenProjector(config) + self.visual_embeddings_table = nn.Embedding(config.vocab_size, text_hidden_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + visual_tokens = self.visual_tokenizer(hidden_states) + return torch.matmul(visual_tokens, self.visual_embeddings_table.weight) + + def get_visual_indicator_features(self, device: torch.device | None = None) -> torch.Tensor: + indicator_token_ids = self.visual_tokenizer.get_visual_indicator_token_ids( + device or self.visual_embeddings_table.weight.device + ) + return self.visual_embeddings_table(indicator_token_ids) + + +@auto_docstring +@dataclass +class Ovis2_5VisualFeaturesOutput(BaseModelOutputWithPooling): + r""" + visual_indicator_features (`torch.FloatTensor` of shape `(4, hidden_size)`, *optional*): + Learned image-begin, image-end, video-begin, and video-end embeddings, in that order. + """ + + visual_indicator_features: torch.FloatTensor | None = None + + +@auto_docstring( + custom_intro=""" + Base class for Ovis2_5 outputs, with hidden states and attentions. + """ +) +@dataclass +class Ovis2_5ModelOutputWithPast(BaseModelOutputWithPast): + r""" + image_pooler_output (`torch.FloatTensor`, *optional*): + A `torch.FloatTensor` of size `(num_image_tokens, hidden_size)`. + Image features produced by the vision encoder and projected to the language model hidden size. + video_pooler_output (`torch.FloatTensor`, *optional*): + A `torch.FloatTensor` of size `(num_video_tokens, hidden_size)`. + Video features produced by the vision encoder and projected to the language model hidden size. + """ + + image_pooler_output: torch.FloatTensor | None = None + video_pooler_output: torch.FloatTensor | None = None + + +@auto_docstring +class Ovis2_5PreTrainedModel(PreTrainedModel): + config: Ovis2_5Config + base_model_prefix = "model" + input_modalities = ("image", "video", "text") + supports_gradient_checkpointing = True + _no_split_modules = ["Ovis2_5VisionEncoderLayer"] + _skip_keys_device_placement = ["cache"] + _supports_flash_attn = True + _supports_sdpa = True + # Vision encoding runs eagerly before fullgraph-compiled cached text decoding. + _can_compile_fullgraph = True + _supports_attention_backend = True + _supports_cache_class = True + _supports_flex_attn = True + + def _init_weights(self, module): + super()._init_weights(module) + if isinstance(module, Ovis2_5VisualTokenProjector): + init.zeros_(module.indicator_padding) + + +@auto_docstring(custom_intro="The Ovis2.5 vision tower, without the visual tokenizer or language model.") +class Ovis2_5VisionModel(Ovis2_5PreTrainedModel): + config: Ovis2_5VisionConfig + main_input_name = "pixel_values" + input_modalities = ("image", "video") + _can_record_outputs = { + "hidden_states": Ovis2_5VisionEncoderLayer, + "attentions": Ovis2_5VisionAttention, + } + _input_embed_layer = "patch_embedding" + + def __init__(self, config: Ovis2_5VisionConfig): + super().__init__(config) + self.spatial_merge_size = config.spatial_merge_size + self.window_size = config.window_size + self.patch_size = config.patch_size + self.embeddings = Ovis2_5VisionEmbeddings(config) + self.layers = nn.ModuleList([Ovis2_5VisionEncoderLayer(config) for _ in range(config.num_hidden_layers)]) + self.post_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.rotary_emb = Ovis2_5VisionRotaryEmbedding(config) + self.post_init() + + @merge_with_config_defaults + @capture_outputs(tie_last_hidden_states=False) + @auto_docstring + def forward( + self, + pixel_values: torch.FloatTensor, + grid_thw: torch.LongTensor, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPooling: + r""" + grid_thw (`torch.LongTensor` of shape `(num_images_or_videos, 3)`): + Temporal, height, and width patch-grid dimensions for each packed image or video. + """ + hidden_states = self.embeddings(pixel_values, grid_thw, **kwargs) + spatial_merge_unit = self.spatial_merge_size**2 + window_index, cu_window_seqlens = get_vision_window_index( + grid_thw, + spatial_merge_size=self.spatial_merge_size, + window_size=self.window_size, + patch_size=self.patch_size, + kwargs=kwargs, + ) + cu_seqlens, max_seqlen = get_vision_attention_seqlens(grid_thw, self.config, kwargs=kwargs) + max_window_seqlen = get_max_seqlen( + cu_window_seqlens, self.config, kwargs=kwargs, kwarg_name="max_window_seqlen" + ) + + sequence_length = hidden_states.shape[0] + hidden_states = hidden_states.reshape(sequence_length // spatial_merge_unit, spatial_merge_unit, -1) + hidden_states = hidden_states[window_index].reshape(sequence_length, -1) + + position_ids = get_vision_position_ids(grid_thw, self.spatial_merge_size, kwargs=kwargs) + position_ids = position_ids.reshape(sequence_length // spatial_merge_unit, spatial_merge_unit, -1) + position_ids = position_ids[window_index].reshape(sequence_length, -1) + position_embeddings = self.rotary_emb(hidden_states, position_ids) + + cu_seqlens_mapping = { + "full_attention": (cu_seqlens, max_seqlen), + "sliding_attention": (cu_window_seqlens, max_window_seqlen), + } + for layer_index, encoder_layer in enumerate(self.layers): + layer_cu_seqlens, layer_max_seqlen = cu_seqlens_mapping[self.config.layer_types[layer_index]] + hidden_states = encoder_layer( + hidden_states, + cu_seqlens=layer_cu_seqlens, + position_embeddings=position_embeddings, + max_seqlen=layer_max_seqlen, + **kwargs, + ) + + reverse_indices = torch.argsort(window_index) + hidden_states = hidden_states.reshape(sequence_length // spatial_merge_unit, spatial_merge_unit, -1) + pre_layernorm_hidden_state = hidden_states[reverse_indices].reshape(sequence_length, -1) + last_hidden_state = self.post_layernorm(pre_layernorm_hidden_state) + # The released visual tokenizer consumes the final encoder state before this output normalization. + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pre_layernorm_hidden_state, + ) + + +@auto_docstring(custom_intro="The bare Ovis2.5 multimodal model, without the language modeling head.") +class Ovis2_5Model(Ovis2_5PreTrainedModel): + def __init__(self, config: Ovis2_5Config): + super().__init__(config) + self.vision_tower = Ovis2_5VisionModel(config.vision_config) + self.multi_modal_projector = Ovis2_5MultiModalProjector( + config.vision_config, + config.text_config.hidden_size, + ) + self.language_model = AutoModel.from_config(config.text_config) + self.post_init() + + @accepts_precomputed_kwargs(modality="image") + @can_return_tuple + @auto_docstring(custom_intro="Encodes images into Ovis2.5 visual embeddings.") + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: torch.LongTensor, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | Ovis2_5VisualFeaturesOutput: + vision_outputs = self.vision_tower( + pixel_values=pixel_values, + grid_thw=image_grid_thw, + return_dict=True, + **kwargs, + ) + visual_features = self.multi_modal_projector(vision_outputs.pooler_output) + visual_indicator_features = self.multi_modal_projector.get_visual_indicator_features(visual_features.device) + split_sizes = (image_grid_thw.prod(dim=1) // self.vision_tower.config.spatial_merge_size**2).tolist() + return Ovis2_5VisualFeaturesOutput( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=torch.split(visual_features, split_sizes), + hidden_states=vision_outputs.hidden_states, + attentions=vision_outputs.attentions, + visual_indicator_features=visual_indicator_features, + ) + + def get_placeholder_mask( + self, + input_ids: torch.LongTensor | None, + inputs_embeds: torch.FloatTensor, + image_features: torch.FloatTensor | None = None, + video_features: torch.FloatTensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + Obtain image and video placeholder masks and check that their feature counts match the placeholders. + """ + if input_ids is None: + image_mask = inputs_embeds == self.get_input_embeddings()( + torch.full((), self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + image_mask = image_mask.all(dim=-1) + video_mask = inputs_embeds == self.get_input_embeddings()( + torch.full((), self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + video_mask = video_mask.all(dim=-1) + else: + image_mask = input_ids == self.config.image_token_id + video_mask = input_ids == self.config.video_token_id + + image_mask = image_mask.unsqueeze(-1).to(inputs_embeds.device) + if image_features is not None: + torch_compilable_check( + image_mask.sum() * inputs_embeds.shape[-1] == image_features.numel(), + lambda: ( + f"Image features and image tokens do not match, tokens: {image_mask.sum()}, " + f"features: {image_features.shape[0]}" + ), + ) + + video_mask = video_mask.unsqueeze(-1).to(inputs_embeds.device) + if video_features is not None: + torch_compilable_check( + video_mask.sum() * inputs_embeds.shape[-1] == video_features.numel(), + lambda: ( + f"Video features and video tokens do not match, tokens: {video_mask.sum()}, " + f"features: {video_features.shape[0]}" + ), + ) + + return image_mask, video_mask + + @can_return_tuple + @auto_docstring + @deprecate_kwarg("past_key_values", "5.40.0", new_name="cache") + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + cache: Cache | None = None, + inputs_embeds: torch.FloatTensor | None = None, + use_cache: bool | None = None, + pixel_values: torch.FloatTensor | None = None, + image_grid_thw: torch.LongTensor | None = None, + pixel_values_videos: torch.FloatTensor | None = None, + video_grid_thw: torch.LongTensor | None = None, + mm_encoder_outputs: dict[str, BaseModelOutputWithPooling] | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | Ovis2_5ModelOutputWithPast: + if (input_ids is None) == (inputs_embeds is None): + raise ValueError("You must specify exactly one of `input_ids` or `inputs_embeds`.") + if pixel_values is not None and pixel_values_videos is not None: + raise ValueError("Ovis2.5 supports images or video in one request, but not both.") + if (pixel_values is not None or pixel_values_videos is not None) and mm_encoder_outputs is not None: + raise ValueError( + "You cannot specify both pixel_values/pixel_values_videos and mm_encoder_outputs at the same time" + ) + mm_encoder_outputs = mm_encoder_outputs if mm_encoder_outputs is not None else {} + if mm_encoder_outputs.get("image") is not None and mm_encoder_outputs.get("video") is not None: + raise ValueError("Ovis2.5 supports images or video in one request, but not both.") + if inputs_embeds is None: + inputs_embeds = self.get_input_embeddings()(input_ids) + + if mm_encoder_outputs.get("image") is None and pixel_values is not None: + mm_encoder_outputs["image"] = self.get_image_features( + pixel_values=pixel_values, + image_grid_thw=image_grid_thw, + return_dict=True, + **kwargs, + ) + if mm_encoder_outputs.get("video") is None and pixel_values_videos is not None: + mm_encoder_outputs["video"] = self.get_video_features( + pixel_values_videos=pixel_values_videos, + video_grid_thw=video_grid_thw, + return_dict=True, + **kwargs, + ) + + image_features = None + video_features = None + if mm_encoder_outputs.get("image") is not None: + image_outputs = mm_encoder_outputs["image"] + image_features = torch.cat(image_outputs.pooler_output, dim=0) + inputs_embeds = self._merge_visual_features( + input_ids, + inputs_embeds, + image_features, + image_outputs.visual_indicator_features, + num_visual_inputs=len(image_outputs.pooler_output), + is_video=False, + ) + + if mm_encoder_outputs.get("video") is not None: + video_outputs = mm_encoder_outputs["video"] + video_features = torch.cat(video_outputs.pooler_output, dim=0) + inputs_embeds = self._merge_visual_features( + input_ids, + inputs_embeds, + video_features, + video_outputs.visual_indicator_features, + num_visual_inputs=len(video_outputs.pooler_output), + is_video=True, + ) + + outputs = self.language_model( + attention_mask=attention_mask, + position_ids=position_ids, + cache=cache, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + return_dict=True, + **kwargs, + ) + return Ovis2_5ModelOutputWithPast( + last_hidden_state=outputs.last_hidden_state, + cache=outputs.cache, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + image_pooler_output=image_features, + video_pooler_output=video_features, + ) + + @accepts_precomputed_kwargs(modality="video") + @can_return_tuple + @auto_docstring(custom_intro="Encodes videos into Ovis2.5 visual embeddings.") + def get_video_features( + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: torch.LongTensor, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | Ovis2_5VisualFeaturesOutput: + return self.get_image_features( + pixel_values=pixel_values_videos, + image_grid_thw=video_grid_thw, + **kwargs, + ) + + def _merge_visual_features( + self, + input_ids: torch.LongTensor | None, + inputs_embeds: torch.Tensor, + visual_features: torch.Tensor, + visual_indicator_features: torch.Tensor | None, + num_visual_inputs: int, + is_video: bool, + ) -> torch.Tensor: + """Replace visual placeholders and start/end tokens with their visual embeddings.""" + image_mask, video_mask = self.get_placeholder_mask( + input_ids, + inputs_embeds=inputs_embeds, + image_features=None if is_video else visual_features, + video_features=visual_features if is_video else None, + ) + visual_mask = video_mask if is_video else image_mask + inputs_embeds = inputs_embeds.masked_scatter( + visual_mask, + visual_features.to(inputs_embeds.device, inputs_embeds.dtype), + ) + + if visual_indicator_features is None: + return inputs_embeds + + if is_video: + boundary_token_ids = (self.config.video_start_token_id, self.config.video_end_token_id) + indicator_indexes = (2, 3) + else: + boundary_token_ids = (self.config.image_start_token_id, self.config.image_end_token_id) + indicator_indexes = (0, 1) + + for boundary_token_id, indicator_index in zip(boundary_token_ids, indicator_indexes): + if input_ids is None: + boundary_embedding = self.get_input_embeddings()( + torch.tensor(boundary_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + boundary_mask = (inputs_embeds == boundary_embedding).all(dim=-1) + else: + boundary_mask = input_ids == boundary_token_id + num_boundary_tokens = boundary_mask.sum() + torch_compilable_check( + num_boundary_tokens == num_visual_inputs, + lambda: ( + f"Expected {num_visual_inputs} visual boundary tokens with id {boundary_token_id}, but found " + f"{num_boundary_tokens}." + ), + ) + boundary_features = visual_indicator_features[indicator_index].to( + inputs_embeds.device, + inputs_embeds.dtype, + ) + inputs_embeds = torch.where( + boundary_mask.unsqueeze(-1), + boundary_features.expand_as(inputs_embeds), + inputs_embeds, + ) + + return inputs_embeds + + +@auto_docstring(custom_intro="The Ovis2.5 multimodal model with a language modeling head.") +class Ovis2_5ForConditionalGeneration(Ovis2_5PreTrainedModel, GenerationMixin): + """Ovis2.5 multimodal conditional generation model.""" + + _tied_weights_keys = {"lm_head.weight": "model.language_model.embed_tokens.weight"} + _tp_plan = {"lm_head": "colwise_gather_output"} + # Reference: fix gemma3 grad acc #37208 + accepts_loss_kwargs = False + + input_modalities = ("image", "video", "text") + + def __init__(self, config: Ovis2_5Config): + super().__init__(config) + # Exaone uses an underscored model class, while Ovis exposes `Ovis2_5Model`. + self.model = Ovis2_5Model(config) + self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False) + + self.post_init() + + @auto_docstring + def get_video_features( + self, + pixel_values_videos: torch.Tensor, + video_grid_thw: torch.LongTensor | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values_videos (`torch.Tensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + """ + return self.model.get_video_features(pixel_values_videos, video_grid_thw, **kwargs) + + @auto_docstring + def get_image_features( + self, + pixel_values: torch.Tensor, + image_grid_thw: torch.LongTensor | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.Tensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + """ + return self.model.get_image_features(pixel_values, image_grid_thw, **kwargs) + + @can_return_tuple + @auto_docstring + @deprecate_kwarg("past_key_values", "5.40.0", new_name="cache") + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + cache: Cache | None = None, + inputs_embeds: torch.Tensor | None = None, + labels: torch.LongTensor | None = None, + use_cache: bool | None = None, + pixel_values: torch.Tensor | None = None, + pixel_values_videos: torch.Tensor | None = None, + image_grid_thw: torch.LongTensor | None = None, + video_grid_thw: torch.LongTensor | None = None, + logits_to_keep: int | torch.Tensor = 0, + mm_encoder_outputs: dict[str, BaseModelOutputWithPooling] | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | CausalLMOutputWithPast: + r""" + Example: + + ```python + >>> from transformers import AutoProcessor, Ovis2_5ForConditionalGeneration + >>> import torch + + >>> model = Ovis2_5ForConditionalGeneration.from_pretrained("LGAI-EXAONE/EXAONE-4.5-33B") + >>> processor = AutoProcessor.from_pretrained("LGAI-EXAONE/EXAONE-4.5-33B") + + >>> messages = [ + ... { + ... "role": "user", + ... "content": [ + ... {"type": "image", "image": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"}, + ... {"type": "text", "text": "Describe the image."}, + ... ], + ... } + ... ] + >>> inputs = processor.apply_chat_template( + ... messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt" + ... ) + >>> inputs = {k: v.to(model.device) if isinstance(v, torch.Tensor) else v for k, v in inputs.items()} + >>> generated_ids = model.generate(**inputs, max_new_tokens=64) + ``` + """ + outputs = self.model( + input_ids=input_ids, + pixel_values=pixel_values, + pixel_values_videos=pixel_values_videos, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + position_ids=position_ids, + attention_mask=attention_mask, + cache=cache, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + mm_encoder_outputs=mm_encoder_outputs, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + hidden_states = slice_logits_to_keep(hidden_states, logits_to_keep) + logits = self.lm_head(hidden_states) + + loss = None + if labels is not None: + loss = self.loss_function( + logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size, **kwargs + ) + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + cache=outputs.cache, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation(self, input_ids, **kwargs): + model_inputs = super().prepare_inputs_for_generation(input_ids, **kwargs) + # Force recomputation of 2D-RoPE and ignore rope_deltas + model_inputs["position_ids"] = None + return model_inputs + + +__all__ = ["Ovis2_5VisionModel", "Ovis2_5PreTrainedModel", "Ovis2_5Model", "Ovis2_5ForConditionalGeneration"] diff --git a/src/transformers/models/ovis2_5/modular_ovis2_5.py b/src/transformers/models/ovis2_5/modular_ovis2_5.py new file mode 100644 index 000000000000..15ea3226cae2 --- /dev/null +++ b/src/transformers/models/ovis2_5/modular_ovis2_5.py @@ -0,0 +1,972 @@ +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch Ovis2.5 model.""" + +import math +from dataclasses import dataclass +from typing import Any + +import torch +from huggingface_hub.dataclasses import strict +from torch import nn + +from ... import initialization as init +from ...cache_utils import Cache +from ...configuration_utils import PreTrainedConfig, SubConfigSpec +from ...image_processing_backends import PilBackend, TorchvisionBackend +from ...image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, PILImageResampling, SizeDict +from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import BaseModelOutputWithPooling +from ...modeling_utils import PreTrainedModel +from ...processing_utils import Unpack, VideosKwargs +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, torch_compilable_check +from ...utils.deprecation import deprecate_kwarg +from ...utils.generic import accepts_precomputed_kwargs, get_max_seqlen, merge_with_config_defaults +from ...utils.output_capturing import capture_outputs +from ...vision_utils import ( + get_vision_attention_seqlens, + get_vision_interpolation_indices_and_weights, + get_vision_position_ids, + get_vision_window_index, +) +from ..auto import AutoConfig +from ..cosmos3_edge.configuration_cosmos3_edge import Cosmos3EdgeVisionConfig +from ..exaone4_5.modeling_exaone4_5 import Exaone4_5_ForConditionalGeneration +from ..glm4v.image_processing_glm4v import Glm4vImageProcessor, Glm4vImageProcessorKwargs +from ..glm4v.image_processing_pil_glm4v import Glm4vImageProcessorPil +from ..glm4v.video_processing_glm4v import Glm4vVideoProcessor +from ..lfm3_vl.modeling_lfm3_vl import Lfm3VlModelOutputWithPast +from ..llava.modeling_llava import LlavaModel +from ..minicpmv4_6.modeling_minicpmv4_6 import MiniCPMV4_6VisionEmbeddings +from ..muse_glimmer.modeling_muse_glimmer import MuseGlimmerVisionModel, MuseGlimmerVisionRotaryEmbedding +from ..video_llama_3.modeling_video_llama_3 import ( + VideoLlama3PreTrainedModel, + VideoLlama3VisionAttention, + VideoLlama3VisionEncoderLayer, + VideoLlama3VisionMLP, +) + + +class Ovis2_5ImageProcessorKwargs(Glm4vImageProcessorKwargs, total=False): + r""" + patch_size (`int`, *optional*, defaults to 16): + The spatial patch size of the vision encoder. + temporal_patch_size (`int`, *optional*, defaults to 1): + The temporal patch size of the vision encoder. + merge_size (`int`, *optional*, defaults to 2): + The spatial merge size used by the visual tokenizer. + """ + + +def smart_resize( + height: int, + width: int, + factor: int = 32, + min_pixels: int = 448 * 448, + max_pixels: int = 1344 * 1792, +) -> tuple[int, int]: + """Rescales the image so that the following conditions are met: + + 1. Both dimensions (height and width) are divisible by 'factor'. + + 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. + + 3. The aspect ratio of the image is maintained as closely as possible. + + """ + # Unlike Qwen, Ovis expands dimensions below the factor and clamps aspect ratios above 200. + if height < factor or width < factor: + if height < width: + width = round(factor / height * width) + height = factor + else: + height = round(factor / width * height) + width = factor + elif max(height, width) / min(height, width) > 200: + if height > width: + height = 200 * width + else: + width = 200 * height + + h_bar = round(height / factor) * factor + w_bar = round(width / factor) * factor + if h_bar * w_bar > max_pixels: + beta = math.sqrt((height * width) / max_pixels) + h_bar = math.floor(height / beta / factor) * factor + w_bar = math.floor(width / beta / factor) * factor + elif h_bar * w_bar < min_pixels: + beta = math.sqrt(min_pixels / (height * width)) + h_bar = math.ceil(height * beta / factor) * factor + w_bar = math.ceil(width * beta / factor) * factor + return h_bar, w_bar + + +class Ovis2_5ImageProcessor(Glm4vImageProcessor): + resample = PILImageResampling.BILINEAR + size = {"shortest_edge": 448 * 448, "longest_edge": 1344 * 1792} + image_mean = IMAGENET_STANDARD_MEAN + image_std = IMAGENET_STANDARD_STD + patch_size = 16 + temporal_patch_size = 1 + merge_size = 2 + valid_kwargs = Ovis2_5ImageProcessorKwargs + + def resize( + self, + images: torch.Tensor, + size: SizeDict, + resample: PILImageResampling | int | None, + factor: int, + temporal_factor: int, + **kwargs, + ) -> torch.Tensor: + """Resize dynamically based on input image aspect ratio.""" + if not size.shortest_edge or not size.longest_edge: + raise ValueError(f"`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got {size}.") + + # Ovis2.5 uses its custom smart_resize policy instead of GLM4V's default resize. + height, width = images.shape[-2:] + resized_height, resized_width = smart_resize( + height, + width, + factor=factor, + min_pixels=size.shortest_edge, + max_pixels=size.longest_edge, + ) + return TorchvisionBackend.resize( + self, + image=images, + size=SizeDict(height=resized_height, width=resized_width), + resample=resample, + ) + + def get_number_of_image_patches(self, height: int, width: int, images_kwargs: dict | None = None) -> int: + """ + A utility that returns number of image patches for a given image size. + + Args: + height (`int`): + Height of the input image. + width (`int`): + Width of the input image. + images_kwargs (`dict`, *optional*) + Any kwargs to override defaults of the image processor. + Returns: + `int`: Number of image patches per image. + """ + images_kwargs = images_kwargs or {} + patch_size = images_kwargs.get("patch_size", self.patch_size) + merge_size = images_kwargs.get("merge_size", self.merge_size) + size = images_kwargs.get("size", self.size) + min_pixels = size["shortest_edge"] if isinstance(size, dict) else size.shortest_edge + max_pixels = size["longest_edge"] if isinstance(size, dict) else size.longest_edge + # Ovis2.5 uses the same custom smart_resize policy to compute the patch count. + resized_height, resized_width = smart_resize( + height, + width, + factor=patch_size * merge_size, + min_pixels=min_pixels, + max_pixels=max_pixels, + ) + return (resized_height // patch_size) * (resized_width // patch_size) + + +class Ovis2_5ImageProcessorPil(Glm4vImageProcessorPil): + resample = PILImageResampling.BILINEAR + size = {"shortest_edge": 448 * 448, "longest_edge": 1344 * 1792} + image_mean = IMAGENET_STANDARD_MEAN + image_std = IMAGENET_STANDARD_STD + patch_size = 16 + temporal_patch_size = 1 + merge_size = 2 + valid_kwargs = Ovis2_5ImageProcessorKwargs + + def resize( + self, + image: Any, + size: SizeDict, + resample: PILImageResampling | int | None, + factor: int, + temporal_factor: int, + **kwargs, + ) -> Any: + """Resize dynamically based on input image aspect ratio.""" + if not size.shortest_edge or not size.longest_edge: + raise ValueError(f"`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got {size}.") + + # Ovis2.5 uses its custom smart_resize policy instead of GLM4V's default resize. + height, width = image.shape[-2:] + resized_height, resized_width = smart_resize( + height, + width, + factor=factor, + min_pixels=size.shortest_edge, + max_pixels=size.longest_edge, + ) + return PilBackend.resize( + self, + image=image, + size=SizeDict(height=resized_height, width=resized_width), + resample=resample, + ) + + def get_number_of_image_patches(self, height: int, width: int, images_kwargs: dict | None = None) -> int: + """ + A utility that returns number of image patches for a given image size. + + Args: + height (`int`): + Height of the input image. + width (`int`): + Width of the input image. + images_kwargs (`dict`, *optional*) + Any kwargs to override defaults of the image processor. + Returns: + `int`: Number of image patches per image. + """ + images_kwargs = images_kwargs or {} + patch_size = images_kwargs.get("patch_size", self.patch_size) + merge_size = images_kwargs.get("merge_size", self.merge_size) + size = images_kwargs.get("size", self.size) + min_pixels = size["shortest_edge"] if isinstance(size, dict) else size.shortest_edge + max_pixels = size["longest_edge"] if isinstance(size, dict) else size.longest_edge + # Ovis2.5 uses the same custom smart_resize policy to compute the patch count. + resized_height, resized_width = smart_resize( + height, + width, + factor=patch_size * merge_size, + min_pixels=min_pixels, + max_pixels=max_pixels, + ) + return (resized_height // patch_size) * (resized_width // patch_size) + + +class Ovis2_5VideoProcessorInitKwargs(VideosKwargs, total=False): + r""" + patch_size (`int`, *optional*, defaults to 16): + The spatial patch size used by the vision encoder. + temporal_patch_size (`int`, *optional*, defaults to 1): + The temporal patch size used by the vision encoder. + merge_size (`int`, *optional*, defaults to 2): + The spatial merge size between the vision encoder and language model. + """ + + patch_size: int + temporal_patch_size: int + merge_size: int + + +class Ovis2_5VideoProcessor(Glm4vVideoProcessor): + resample = PILImageResampling.BILINEAR + size = {"shortest_edge": 448 * 448, "longest_edge": 1344 * 1792} + image_mean = IMAGENET_STANDARD_MEAN + image_std = IMAGENET_STANDARD_STD + do_sample_frames = False + patch_size = 16 + temporal_patch_size = 1 + merge_size = 2 + max_image_size = AttributeError() + max_duration = AttributeError() + num_frames = AttributeError() + fps = AttributeError() + valid_kwargs = Ovis2_5VideoProcessorInitKwargs + + def sample_frames(self, **super_kwargs): + raise AttributeError() + + def get_num_of_video_patches(self, **super_kwargs): + raise AttributeError() + + def resize( + self, + videos: torch.Tensor, + size: SizeDict, + resample: PILImageResampling | int | None, + factor: int, + temporal_factor: int, + **kwargs, + ) -> torch.Tensor: + """Resize each frame with the released Ovis2.5 spatial policy.""" + if not size.shortest_edge or not size.longest_edge: + raise ValueError(f"`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got {size}.") + + height, width = videos.shape[-2:] + # Match Ovis2.5 image resizing; GLM4V also factors in the video frame count. + resized_height, resized_width = smart_resize( + height=height, + width=width, + factor=factor, + min_pixels=size.shortest_edge, + max_pixels=size.longest_edge, + ) + return TorchvisionBackend.resize( + self, + image=videos, + size=SizeDict(height=resized_height, width=resized_width), + resample=resample, + ) + + def get_number_of_video_patches( + self, + num_frames: int, + height: int, + width: int, + videos_kwargs: dict | None = None, + ) -> int: + """ + A utility that returns the number of video patches for a given video size. + + Used by `Ovis2_5Processor._get_num_multimodal_tokens` to estimate visual token counts from video dimensions. + + Args: + num_frames (`int`): + Number of frames in the input video. + height (`int`): + Height of the input video frames. + width (`int`): + Width of the input video frames. + videos_kwargs (`dict`, *optional*): + Any kwargs to override defaults of the video processor. + Returns: + `int`: Number of video patches per video. + """ + if num_frames <= 0: + raise ValueError(f"`num_frames` must be positive, got {num_frames}.") + + videos_kwargs = videos_kwargs or {} + patch_size = videos_kwargs.get("patch_size", self.patch_size) + temporal_patch_size = videos_kwargs.get("temporal_patch_size", self.temporal_patch_size) + merge_size = videos_kwargs.get("merge_size", self.merge_size) + do_resize = videos_kwargs.get("do_resize", self.do_resize) + + resized_height, resized_width = height, width + if do_resize: + size = videos_kwargs.get("size", self.size) + min_pixels = size["shortest_edge"] if isinstance(size, dict) else size.shortest_edge + max_pixels = size["longest_edge"] if isinstance(size, dict) else size.longest_edge + + # Mirror `resize` so the estimate matches the processed video grid. + resized_height, resized_width = smart_resize( + height, + width, + factor=patch_size * merge_size, + min_pixels=min_pixels, + max_pixels=max_pixels, + ) + + factor = patch_size * merge_size + if resized_height % factor != 0 or resized_width % factor != 0: + raise ValueError( + "Ovis2.5 videos must have height and width divisible by " + f"`patch_size * merge_size` ({factor}), got ({resized_height}, {resized_width})." + ) + + # Match `patchify`'s last-frame padding when the frame count is not divisible by the temporal patch size. + num_temporal_patches = math.ceil(num_frames / temporal_patch_size) + return num_temporal_patches * (resized_height // patch_size) * (resized_width // patch_size) + + +@auto_docstring(checkpoint="ovis-community/Ovis2.5-2B") +@strict +class Ovis2_5VisionConfig(Cosmos3EdgeVisionConfig): + r""" + window_size (`int`, *optional*, defaults to 112): + Window size, in input pixels, used by windowed vision-attention layers. + temporal_patch_size (`int`, *optional*, defaults to 1): + Number of consecutive video frames represented by one temporal patch. + vocab_size (`int`, *optional*, defaults to 65536): + Size of the visual-token vocabulary, including four learned visual-boundary indicators. + num_visual_indicator_tokens (`int`, *optional*, defaults to 4): + Number of visual-vocabulary rows reserved for image/video boundary indicators. + """ + + model_type = "ovis2_5_vision" + default_rope_type = "axial" + + image_size: int = 512 + patch_size: int = 16 + window_size: int = 112 + layer_types: list[str] | tuple[str, ...] | None = None + temporal_patch_size: int = 1 + vocab_size: int = 65536 + num_visual_indicator_tokens: int = 4 + initializer_range: float = 0.02 + max_position_embeddings: int = 1024 + rope_parameters: dict | None = None + num_patches = AttributeError() + + def __post_init__(self, **kwargs): + full_attention_indexes = kwargs.get("fullatt_block_indexes") + if self.layer_types is None: + if full_attention_indexes is None: + self.layer_types = ["full_attention"] * self.num_hidden_layers + else: + if isinstance(full_attention_indexes, str): + full_attention_indexes = [int(index) for index in full_attention_indexes.split("|") if index] + full_attention_indexes = set(full_attention_indexes) + self.layer_types = [ + "full_attention" if layer_index in full_attention_indexes else "sliding_attention" + for layer_index in range(self.num_hidden_layers) + ] + else: + self.layer_types = list(self.layer_types) + PreTrainedConfig.__post_init__(self, **kwargs) + + +@auto_docstring(checkpoint="ovis-community/Ovis2.5-2B") +@strict +class Ovis2_5Config(PreTrainedConfig): + r""" + image_start_token_id (`int`, *optional*, defaults to 151670): + Text-vocabulary token replaced by the learned image-begin visual embedding. + image_end_token_id (`int`, *optional*, defaults to 151671): + Text-vocabulary token replaced by the learned image-end visual embedding. + video_start_token_id (`int`, *optional*, defaults to 151672): + Text-vocabulary token replaced by the learned video-begin visual embedding. + video_end_token_id (`int`, *optional*, defaults to 151673): + Text-vocabulary token replaced by the learned video-end visual embedding. + """ + + model_type = "ovis2_5" + sub_configs_defaults = { + "vision_config": Ovis2_5VisionConfig, + "text_config": SubConfigSpec(config_class=AutoConfig, model_type="qwen3"), + } + keys_to_ignore_at_inference = ["cache"] + + text_config: PreTrainedConfig | dict | None = None + vision_config: PreTrainedConfig | dict | None = None + image_token_id: int = 151669 + video_token_id: int = 151669 + image_start_token_id: int = 151670 + image_end_token_id: int = 151671 + video_start_token_id: int = 151672 + video_end_token_id: int = 151673 + tie_word_embeddings: bool = False + + def __post_init__(self, **kwargs): + if isinstance(self.vision_config, dict): + # Released configs label this remote-code tower as `siglip2_navit`; the native tower type is fixed here. + self.vision_config.pop("model_type", None) + self.vision_config.pop("preserve_original_pe", None) + self.vision_config.pop("use_rope", None) + self.vision_config.pop("num_patches", None) + super().__post_init__(**kwargs) + + +class Ovis2_5VisionRotaryEmbedding(MuseGlimmerVisionRotaryEmbedding): + pass + + +class Ovis2_5VisionEmbeddings(MiniCPMV4_6VisionEmbeddings): + """Patch embeddings and interpolated learned positions for packed Ovis2.5 inputs.""" + + def __init__(self, config: Ovis2_5VisionConfig): + super().__init__(config) + self.config = config + # How the (square) learned position grid is resampled to each image's grid. + self.num_grid_per_side = int(self.num_positions**0.5) + self.interpolation_align_corners = False + self.interpolation_mode = "bicubic" + self.spatial_merge_size = config.spatial_merge_size + + def forward( + self, + pixel_values: torch.FloatTensor, + grid_thw: torch.LongTensor, + **kwargs: Unpack[TransformersKwargs], + ) -> torch.Tensor: + """ + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, sequence_length, image_channels, patch_size, patch_size)`): + The tensors corresponding to the input images. + grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ + interp_indices, interp_weights = get_vision_interpolation_indices_and_weights( + grid_thw, + self.num_grid_per_side, + mode=self.interpolation_mode, + align_corners=self.interpolation_align_corners, + spatial_merge_size=self.spatial_merge_size, + kwargs=kwargs, + ) + pixel_values = pixel_values.view( + -1, + 1, + self.config.num_channels * self.config.temporal_patch_size, + self.patch_size, + self.patch_size, + ) + batch_size, sequence_len, channel, height, width = pixel_values.shape + target_dtype = self.patch_embedding.weight.dtype + pixel_values = pixel_values.reshape(batch_size * sequence_len, channel, height, width) + patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype)) # shape = [*, width, grid, grid] + embeddings = patch_embeds.flatten(-2).squeeze(-1) + embeddings = embeddings.reshape(batch_size * sequence_len, -1) + + interp_weights = interp_weights.to(self.position_embedding.weight.dtype) + pos_embeds = (self.position_embedding(interp_indices) * interp_weights[:, :, None]).sum(1) + embeddings = embeddings + pos_embeds.to(embeddings.dtype) + + return embeddings + + +class Ovis2_5VisionMLP(VideoLlama3VisionMLP): + pass + + +class Ovis2_5VisionAttention(VideoLlama3VisionAttention): + pass + + +class Ovis2_5VisionEncoderLayer(VideoLlama3VisionEncoderLayer, GradientCheckpointingLayer): + pass + + +class Ovis2_5VisualTokenProjector(nn.Module): + def __init__(self, config: Ovis2_5VisionConfig): + super().__init__() + self.spatial_merge_unit = config.spatial_merge_size**2 + self.num_visual_indicator_tokens = config.num_visual_indicator_tokens + self.vocab_size = config.vocab_size + visual_token_vocab_size = config.vocab_size - config.num_visual_indicator_tokens + self.linear = nn.Linear( + config.hidden_size * self.spatial_merge_unit, + visual_token_vocab_size, + bias=False, + ) + self.norm = nn.LayerNorm(visual_token_vocab_size) + self.indicator_padding = nn.Buffer(torch.zeros(config.num_visual_indicator_tokens), persistent=False) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = hidden_states.reshape(hidden_states.shape[0] // self.spatial_merge_unit, -1) + logits = self.norm(self.linear(hidden_states)) + visual_tokens = torch.softmax(logits, dim=-1, dtype=torch.float32).to(logits.dtype) + indicator_padding = self.indicator_padding.expand(visual_tokens.shape[0], -1) + return torch.cat((visual_tokens, indicator_padding), dim=-1) + + def get_visual_indicator_token_ids(self, device: torch.device) -> torch.Tensor: + return torch.arange( + self.vocab_size - self.num_visual_indicator_tokens, + self.vocab_size, + dtype=torch.long, + device=device, + ) + + +class Ovis2_5MultiModalProjector(nn.Module): + def __init__(self, config: Ovis2_5VisionConfig, text_hidden_size: int): + super().__init__() + self.visual_tokenizer = Ovis2_5VisualTokenProjector(config) + self.visual_embeddings_table = nn.Embedding(config.vocab_size, text_hidden_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + visual_tokens = self.visual_tokenizer(hidden_states) + return torch.matmul(visual_tokens, self.visual_embeddings_table.weight) + + def get_visual_indicator_features(self, device: torch.device | None = None) -> torch.Tensor: + indicator_token_ids = self.visual_tokenizer.get_visual_indicator_token_ids( + device or self.visual_embeddings_table.weight.device + ) + return self.visual_embeddings_table(indicator_token_ids) + + +@auto_docstring +@dataclass +class Ovis2_5VisualFeaturesOutput(BaseModelOutputWithPooling): + r""" + visual_indicator_features (`torch.FloatTensor` of shape `(4, hidden_size)`, *optional*): + Learned image-begin, image-end, video-begin, and video-end embeddings, in that order. + """ + + visual_indicator_features: torch.FloatTensor | None = None + + +class Ovis2_5ModelOutputWithPast(Lfm3VlModelOutputWithPast): + pass + + +@auto_docstring +class Ovis2_5PreTrainedModel(VideoLlama3PreTrainedModel): + config: Ovis2_5Config + _no_split_modules = ["Ovis2_5VisionEncoderLayer"] + _supports_cache_class = True + _supports_flex_attn = True + # Vision encoding runs eagerly before fullgraph-compiled cached text decoding. + _can_compile_fullgraph = True + + def _init_weights(self, module): + PreTrainedModel._init_weights(self, module) + if isinstance(module, Ovis2_5VisualTokenProjector): + init.zeros_(module.indicator_padding) + + +@auto_docstring(custom_intro="The Ovis2.5 vision tower, without the visual tokenizer or language model.") +class Ovis2_5VisionModel(Ovis2_5PreTrainedModel, MuseGlimmerVisionModel): + config: Ovis2_5VisionConfig + main_input_name = "pixel_values" + input_modalities = ("image", "video") + _input_embed_layer = "patch_embedding" + _can_record_outputs = { + "hidden_states": Ovis2_5VisionEncoderLayer, + "attentions": Ovis2_5VisionAttention, + } + + def __init__(self, config: Ovis2_5VisionConfig): + Ovis2_5PreTrainedModel.__init__(self, config) + self.spatial_merge_size = config.spatial_merge_size + self.window_size = config.window_size + self.patch_size = config.patch_size + self.embeddings = Ovis2_5VisionEmbeddings(config) + self.layers = nn.ModuleList([Ovis2_5VisionEncoderLayer(config) for _ in range(config.num_hidden_layers)]) + self.post_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.rotary_emb = Ovis2_5VisionRotaryEmbedding(config) + self.post_init() + + def pixel_shuffle(self): + raise AttributeError("Not needed for Ovis2.5") + + # Unlike Muse, Ovis has no input LayerNorm or pixel shuffle, and does not flip or offset RoPE positions. + # Its visual tokenizer also needs encoder features before the final LayerNorm. + @merge_with_config_defaults + @capture_outputs(tie_last_hidden_states=False) + @auto_docstring + def forward( + self, + pixel_values: torch.FloatTensor, + grid_thw: torch.LongTensor, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPooling: + r""" + grid_thw (`torch.LongTensor` of shape `(num_images_or_videos, 3)`): + Temporal, height, and width patch-grid dimensions for each packed image or video. + """ + hidden_states = self.embeddings(pixel_values, grid_thw, **kwargs) + spatial_merge_unit = self.spatial_merge_size**2 + window_index, cu_window_seqlens = get_vision_window_index( + grid_thw, + spatial_merge_size=self.spatial_merge_size, + window_size=self.window_size, + patch_size=self.patch_size, + kwargs=kwargs, + ) + cu_seqlens, max_seqlen = get_vision_attention_seqlens(grid_thw, self.config, kwargs=kwargs) + max_window_seqlen = get_max_seqlen( + cu_window_seqlens, self.config, kwargs=kwargs, kwarg_name="max_window_seqlen" + ) + + sequence_length = hidden_states.shape[0] + hidden_states = hidden_states.reshape(sequence_length // spatial_merge_unit, spatial_merge_unit, -1) + hidden_states = hidden_states[window_index].reshape(sequence_length, -1) + + position_ids = get_vision_position_ids(grid_thw, self.spatial_merge_size, kwargs=kwargs) + position_ids = position_ids.reshape(sequence_length // spatial_merge_unit, spatial_merge_unit, -1) + position_ids = position_ids[window_index].reshape(sequence_length, -1) + position_embeddings = self.rotary_emb(hidden_states, position_ids) + + cu_seqlens_mapping = { + "full_attention": (cu_seqlens, max_seqlen), + "sliding_attention": (cu_window_seqlens, max_window_seqlen), + } + for layer_index, encoder_layer in enumerate(self.layers): + layer_cu_seqlens, layer_max_seqlen = cu_seqlens_mapping[self.config.layer_types[layer_index]] + hidden_states = encoder_layer( + hidden_states, + cu_seqlens=layer_cu_seqlens, + position_embeddings=position_embeddings, + max_seqlen=layer_max_seqlen, + **kwargs, + ) + + reverse_indices = torch.argsort(window_index) + hidden_states = hidden_states.reshape(sequence_length // spatial_merge_unit, spatial_merge_unit, -1) + pre_layernorm_hidden_state = hidden_states[reverse_indices].reshape(sequence_length, -1) + last_hidden_state = self.post_layernorm(pre_layernorm_hidden_state) + # The released visual tokenizer consumes the final encoder state before this output normalization. + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pre_layernorm_hidden_state, + ) + + +@auto_docstring(custom_intro="The bare Ovis2.5 multimodal model, without the language modeling head.") +class Ovis2_5Model(LlavaModel): + def __init__(self, config: Ovis2_5Config): + super().__init__(config) + self.vision_tower = Ovis2_5VisionModel(config.vision_config) + self.multi_modal_projector = Ovis2_5MultiModalProjector( + config.vision_config, + config.text_config.hidden_size, + ) + + @accepts_precomputed_kwargs(modality="image") + @can_return_tuple + @auto_docstring(custom_intro="Encodes images into Ovis2.5 visual embeddings.") + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: torch.LongTensor, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | Ovis2_5VisualFeaturesOutput: + vision_outputs = self.vision_tower( + pixel_values=pixel_values, + grid_thw=image_grid_thw, + return_dict=True, + **kwargs, + ) + visual_features = self.multi_modal_projector(vision_outputs.pooler_output) + visual_indicator_features = self.multi_modal_projector.get_visual_indicator_features(visual_features.device) + split_sizes = (image_grid_thw.prod(dim=1) // self.vision_tower.config.spatial_merge_size**2).tolist() + return Ovis2_5VisualFeaturesOutput( + last_hidden_state=vision_outputs.last_hidden_state, + pooler_output=torch.split(visual_features, split_sizes), + hidden_states=vision_outputs.hidden_states, + attentions=vision_outputs.attentions, + visual_indicator_features=visual_indicator_features, + ) + + @accepts_precomputed_kwargs(modality="video") + @can_return_tuple + @auto_docstring(custom_intro="Encodes videos into Ovis2.5 visual embeddings.") + def get_video_features( + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: torch.LongTensor, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | Ovis2_5VisualFeaturesOutput: + return self.get_image_features( + pixel_values=pixel_values_videos, + image_grid_thw=video_grid_thw, + **kwargs, + ) + + def get_placeholder_mask( + self, + input_ids: torch.LongTensor | None, + inputs_embeds: torch.FloatTensor, + image_features: torch.FloatTensor | None = None, + video_features: torch.FloatTensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + Obtain image and video placeholder masks and check that their feature counts match the placeholders. + """ + if input_ids is None: + image_mask = inputs_embeds == self.get_input_embeddings()( + torch.full((), self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + image_mask = image_mask.all(dim=-1) + video_mask = inputs_embeds == self.get_input_embeddings()( + torch.full((), self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + video_mask = video_mask.all(dim=-1) + else: + image_mask = input_ids == self.config.image_token_id + video_mask = input_ids == self.config.video_token_id + + image_mask = image_mask.unsqueeze(-1).to(inputs_embeds.device) + if image_features is not None: + torch_compilable_check( + image_mask.sum() * inputs_embeds.shape[-1] == image_features.numel(), + lambda: ( + f"Image features and image tokens do not match, tokens: {image_mask.sum()}, " + f"features: {image_features.shape[0]}" + ), + ) + + video_mask = video_mask.unsqueeze(-1).to(inputs_embeds.device) + if video_features is not None: + torch_compilable_check( + video_mask.sum() * inputs_embeds.shape[-1] == video_features.numel(), + lambda: ( + f"Video features and video tokens do not match, tokens: {video_mask.sum()}, " + f"features: {video_features.shape[0]}" + ), + ) + + return image_mask, video_mask + + def _merge_visual_features( + self, + input_ids: torch.LongTensor | None, + inputs_embeds: torch.Tensor, + visual_features: torch.Tensor, + visual_indicator_features: torch.Tensor | None, + num_visual_inputs: int, + is_video: bool, + ) -> torch.Tensor: + """Replace visual placeholders and start/end tokens with their visual embeddings.""" + image_mask, video_mask = self.get_placeholder_mask( + input_ids, + inputs_embeds=inputs_embeds, + image_features=None if is_video else visual_features, + video_features=visual_features if is_video else None, + ) + visual_mask = video_mask if is_video else image_mask + inputs_embeds = inputs_embeds.masked_scatter( + visual_mask, + visual_features.to(inputs_embeds.device, inputs_embeds.dtype), + ) + + if visual_indicator_features is None: + return inputs_embeds + + if is_video: + boundary_token_ids = (self.config.video_start_token_id, self.config.video_end_token_id) + indicator_indexes = (2, 3) + else: + boundary_token_ids = (self.config.image_start_token_id, self.config.image_end_token_id) + indicator_indexes = (0, 1) + + for boundary_token_id, indicator_index in zip(boundary_token_ids, indicator_indexes): + if input_ids is None: + boundary_embedding = self.get_input_embeddings()( + torch.tensor(boundary_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + boundary_mask = (inputs_embeds == boundary_embedding).all(dim=-1) + else: + boundary_mask = input_ids == boundary_token_id + num_boundary_tokens = boundary_mask.sum() + torch_compilable_check( + num_boundary_tokens == num_visual_inputs, + lambda: ( + f"Expected {num_visual_inputs} visual boundary tokens with id {boundary_token_id}, but found " + f"{num_boundary_tokens}." + ), + ) + boundary_features = visual_indicator_features[indicator_index].to( + inputs_embeds.device, + inputs_embeds.dtype, + ) + inputs_embeds = torch.where( + boundary_mask.unsqueeze(-1), + boundary_features.expand_as(inputs_embeds), + inputs_embeds, + ) + + return inputs_embeds + + @can_return_tuple + @auto_docstring + @deprecate_kwarg("past_key_values", "5.40.0", new_name="cache") + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + cache: Cache | None = None, + inputs_embeds: torch.FloatTensor | None = None, + use_cache: bool | None = None, + pixel_values: torch.FloatTensor | None = None, + image_grid_thw: torch.LongTensor | None = None, + pixel_values_videos: torch.FloatTensor | None = None, + video_grid_thw: torch.LongTensor | None = None, + mm_encoder_outputs: dict[str, BaseModelOutputWithPooling] | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | Ovis2_5ModelOutputWithPast: + if (input_ids is None) == (inputs_embeds is None): + raise ValueError("You must specify exactly one of `input_ids` or `inputs_embeds`.") + if pixel_values is not None and pixel_values_videos is not None: + raise ValueError("Ovis2.5 supports images or video in one request, but not both.") + if (pixel_values is not None or pixel_values_videos is not None) and mm_encoder_outputs is not None: + raise ValueError( + "You cannot specify both pixel_values/pixel_values_videos and mm_encoder_outputs at the same time" + ) + mm_encoder_outputs = mm_encoder_outputs if mm_encoder_outputs is not None else {} + if mm_encoder_outputs.get("image") is not None and mm_encoder_outputs.get("video") is not None: + raise ValueError("Ovis2.5 supports images or video in one request, but not both.") + if inputs_embeds is None: + inputs_embeds = self.get_input_embeddings()(input_ids) + + if mm_encoder_outputs.get("image") is None and pixel_values is not None: + mm_encoder_outputs["image"] = self.get_image_features( + pixel_values=pixel_values, + image_grid_thw=image_grid_thw, + return_dict=True, + **kwargs, + ) + if mm_encoder_outputs.get("video") is None and pixel_values_videos is not None: + mm_encoder_outputs["video"] = self.get_video_features( + pixel_values_videos=pixel_values_videos, + video_grid_thw=video_grid_thw, + return_dict=True, + **kwargs, + ) + + image_features = None + video_features = None + if mm_encoder_outputs.get("image") is not None: + image_outputs = mm_encoder_outputs["image"] + image_features = torch.cat(image_outputs.pooler_output, dim=0) + inputs_embeds = self._merge_visual_features( + input_ids, + inputs_embeds, + image_features, + image_outputs.visual_indicator_features, + num_visual_inputs=len(image_outputs.pooler_output), + is_video=False, + ) + + if mm_encoder_outputs.get("video") is not None: + video_outputs = mm_encoder_outputs["video"] + video_features = torch.cat(video_outputs.pooler_output, dim=0) + inputs_embeds = self._merge_visual_features( + input_ids, + inputs_embeds, + video_features, + video_outputs.visual_indicator_features, + num_visual_inputs=len(video_outputs.pooler_output), + is_video=True, + ) + + outputs = self.language_model( + attention_mask=attention_mask, + position_ids=position_ids, + cache=cache, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + return_dict=True, + **kwargs, + ) + return Ovis2_5ModelOutputWithPast( + last_hidden_state=outputs.last_hidden_state, + cache=outputs.cache, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + image_pooler_output=image_features, + video_pooler_output=video_features, + ) + + +@auto_docstring(custom_intro="The Ovis2.5 multimodal model with a language modeling head.") +class Ovis2_5ForConditionalGeneration(Ovis2_5PreTrainedModel, Exaone4_5_ForConditionalGeneration): + """Ovis2.5 multimodal conditional generation model.""" + + def __init__(self, config: Ovis2_5Config): + super().__init__(config) + # Exaone uses an underscored model class, while Ovis exposes `Ovis2_5Model`. + self.model = Ovis2_5Model(config) + + +__all__ = [ + "Ovis2_5Config", + "Ovis2_5ImageProcessor", + "Ovis2_5ImageProcessorPil", + "Ovis2_5VideoProcessor", + "Ovis2_5VisionConfig", + "Ovis2_5VisionModel", + "Ovis2_5PreTrainedModel", + "Ovis2_5Model", + "Ovis2_5ForConditionalGeneration", +] diff --git a/src/transformers/models/ovis2_5/processing_ovis2_5.py b/src/transformers/models/ovis2_5/processing_ovis2_5.py new file mode 100644 index 000000000000..e93aaedcd5eb --- /dev/null +++ b/src/transformers/models/ovis2_5/processing_ovis2_5.py @@ -0,0 +1,111 @@ +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Processor class for Ovis2.5.""" + +from ...image_utils import make_flat_list_of_images +from ...processing_utils import BatchFeature, MultiModalData, ProcessorMixin +from ...utils import auto_docstring +from ...video_utils import make_batched_videos + + +@auto_docstring +class Ovis2_5Processor(ProcessorMixin): + def __init__( + self, + image_processor=None, + tokenizer=None, + video_processor=None, + chat_template=None, + **kwargs, + ): + self.image_token = "" + self.video_token = "