Add Phi3.5 Vision Model

docs/source/en/_toctree.yml

@@ -1141,6 +1141,8 @@
         title: Perceiver
       - local: model_doc/perception_lm
         title: PerceptionLM
+      - local: model_doc/phi3_v
+        title: Phi-3.5-vision
       - local: model_doc/phi4_multimodal
         title: Phi4 Multimodal
       - local: model_doc/pix2struct

docs/source/en/model_doc/phi3_v.md

@@ -0,0 +1,95 @@
+<!--Copyright 2025 The HuggingFace Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+⚠️ Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be
+rendered properly in your Markdown viewer.
+
+-->
+*This model was released on 2024-06-18 and added to Hugging Face Transformers on 2025-11-15.*
+
+<div style="float: right;">
+    <div class="flex flex-wrap space-x-1">
+        <img alt="PyTorch" src="https://img.shields.io/badge/PyTorch-DE3412?style=flat&logo=pytorch&logoColor=white">
+        <img alt="FlashAttention" src="https://img.shields.io/badge/%E2%9A%A1%EF%B8%8E%20FlashAttention-eae0c8?style=flat">
+        <img alt="SDPA" src="https://img.shields.io/badge/SDPA-DE3412?style=flat&logo=pytorch&logoColor=white">
+    </div>
+</div>
+
+# Phi-3.5 Vision
+
+*Phi-3.5 vision is a lightweight, state-of-the-art open multimodal model built upon datasets which include - synthetic data and filtered publicly available websites - with a focus on very high-quality, reasoning dense data both on text and vision. The model belongs to the Phi-3 model family, and the multimodal version comes with 128K context length (in tokens) it can support. The model underwent a rigorous enhancement process, incorporating both supervised fine-tuning and direct preference optimization to ensure precise instruction adherence and robust safety measures.*
+
+
+## Usage Example
+
+The example below demonstrates how to generate text with `Phi3VForConditionalGeneration`.
+
+
+```py
+import torch
+from transformers import Phi3VForConditionalGeneration, Phi3VProcessor
+
+model_id = "yaswanthgali/Phi-3.5-vision-instruct"
+
+# Prepare inputsfor generation.
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {'type':'image', 'url': 'http://images.cocodataset.org/val2017/000000039769.jpg'},
+            {'type':"text", "text":"What do you see in this image?."}
+        ]
+    },
+]
+
+processor = Phi3VProcessor.from_pretrained(model_id)
+model = Phi3VForConditionalGeneration.from_pretrained(model_id,     
+        dtype=torch.bfloat16,
+        device_map="auto")
+
+inputs = processor.apply_chat_template(
+    messages,
+    add_generation_prompt=True,
+    generation_mode="text",
+    tokenize=True,
+    return_dict=True,
+    return_tensors="pt",
+).to(model.device, dtype=torch.bfloat16)
+
+output = model.generate(**inputs, max_new_tokens=20,do_sample=True)
+text = processor.decode(output[0], skip_special_tokens=True)
+print(text)
+```
+
+## Phi3VConfig
+
+[[autodoc]] Phi3VConfig
+
+## Phi3VImageProcessorFast
+
+[[autodoc]] Phi3VImageProcessorFast
+
+## Phi3VProcessor
+
+[[autodoc]] Phi3VProcessor
+
+## Phi3VModel
+
+[[autodoc]] Phi3VModel
+    - forward
+
+## Phi3VForConditionalGeneration
+
+[[autodoc]] Phi3VForConditionalGeneration
+    - forward
+    - generate
+
+

src/transformers/models/__init__.py

@@ -282,6 +282,7 @@
     from .persimmon import *
     from .phi import *
     from .phi3 import *
+    from .phi3_v import *
     from .phi4_multimodal import *
     from .phimoe import *
     from .phobert import *

src/transformers/models/auto/configuration_auto.py

@@ -320,6 +320,7 @@
         ("persimmon", "PersimmonConfig"),
         ("phi", "PhiConfig"),
         ("phi3", "Phi3Config"),
+        ("phi3_v", "Phi3VConfig"),
         ("phi4_multimodal", "Phi4MultimodalConfig"),
         ("phimoe", "PhimoeConfig"),
         ("pix2struct", "Pix2StructConfig"),
@@ -784,6 +785,7 @@
         ("persimmon", "Persimmon"),
         ("phi", "Phi"),
         ("phi3", "Phi3"),
+        ("phi3_v", "Phi3V"),
         ("phi4_multimodal", "Phi4Multimodal"),
         ("phimoe", "Phimoe"),
         ("phobert", "PhoBERT"),

src/transformers/models/auto/image_processing_auto.py

@@ -157,6 +157,7 @@
             ("paligemma", ("SiglipImageProcessor", "SiglipImageProcessorFast")),
             ("perceiver", ("PerceiverImageProcessor", "PerceiverImageProcessorFast")),
             ("perception_lm", (None, "PerceptionLMImageProcessorFast")),
+            ("phi3_v", (None, "Phi3VImageProcessorFast")),
             ("phi4_multimodal", (None, "Phi4MultimodalImageProcessorFast")),
             ("pix2struct", ("Pix2StructImageProcessor", "Pix2StructImageProcessorFast")),
             ("pixio", ("BitImageProcessor", "BitImageProcessorFast")),

src/transformers/models/auto/modeling_auto.py

@@ -318,6 +318,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("persimmon", "PersimmonModel"),
         ("phi", "PhiModel"),
         ("phi3", "Phi3Model"),
+        ("phi3_v", "Phi3VModel"),
         ("phi4_multimodal", "Phi4MultimodalModel"),
         ("phimoe", "PhimoeModel"),
         ("pixio", "PixioModel"),
@@ -509,6 +510,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("nllb-moe", "NllbMoeForConditionalGeneration"),
         ("openai-gpt", "OpenAIGPTLMHeadModel"),
         ("paligemma", "PaliGemmaForConditionalGeneration"),
+        ("phi3_v", "Phi3VForConditionalGeneration"),
         ("qwen2_audio", "Qwen2AudioForConditionalGeneration"),
         ("roberta", "RobertaForMaskedLM"),
         ("roberta-prelayernorm", "RobertaPreLayerNormForMaskedLM"),
@@ -1039,6 +1041,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("paddleocr_vl", "PaddleOCRVLForConditionalGeneration"),
         ("paligemma", "PaliGemmaForConditionalGeneration"),
         ("perception_lm", "PerceptionLMForConditionalGeneration"),
+        ("phi3_v", "Phi3VForConditionalGeneration"),
         ("pix2struct", "Pix2StructForConditionalGeneration"),
         ("pixtral", "LlavaForConditionalGeneration"),
         ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"),

src/transformers/models/phi3_v/__init__.py

@@ -0,0 +1,29 @@
+# Copyright 2025 Microsoft and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_phi3_v import *
+    from .image_processing_phi3_v_fast import *
+    from .modeling_phi3_v import *
+    from .processing_phi3_v import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

src/transformers/models/phi3_v/configuration_phi3_v.py

@@ -0,0 +1,107 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/phi3_v/modular_phi3_v.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_phi3_v.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# coding=utf-8
+# Copyright 2025 Microsoft and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...configuration_utils import PretrainedConfig
+from ...utils import logging
+from ..auto import CONFIG_MAPPING, AutoConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+class Phi3VConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`Phi3VModel`]. It is used to instantiate an
+    Phi3 Vision model according to the specified arguments, defining the model architecture. Instantiating a configuration
+    with the defaults will yield a similar configuration to that of the Phi-3.5 vision model.
+
+    e.g. [microsoft/Phi-3.5-vision-instruct](https://huggingface.co/microsoft/Phi-3.5-vision-instruct)
+
+    Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PreTrainedConfig`] for more information.
+
+    Args:
+        vision_config (`Union[AutoConfig, dict]`,  *optional*, defaults to `CLIPVisionConfig`):
+            The config object or dictionary of the vision backbone.
+        text_config (`Union[AutoConfig, dict]`, *optional*, defaults to `Phi3Config`):
+            The config object or dictionary of the text backbone.
+        image_token_id (`int`, *optional*, defaults to 32044):
+            Token index of a placeholder image token.
+
+    Example:
+
+    ```python
+    >>> from transformers import Phi3VForConditionalGeneration, Phi3VConfig, Phi3Config, CLIPVisionConfig
+
+    >>> # Initializing vision config
+    >>> vision_config = CLIPVisionConfig()
+
+    >>> # Initializing text config
+    >>> text_config = Phi3Config()
+
+    >>> # Initializing config
+    >>> configuration = Phi3VConfig(vision_config=vision_config, text_config=text_config)
+
+    >>> # Initializing a model from the Phi-3 style configuration
+    >>> model = Phi3VForConditionalGeneration(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "phi3_v"
+
+    sub_configs = {"vision_config": AutoConfig, "text_config": AutoConfig}
+
+    def __init__(self, vision_config=None, text_config=None, image_token_id=32044, **kwargs):
+        if text_config is None:
+            logger.info("`text_config` is None. Initializing with default values")
+            self.text_config = CONFIG_MAPPING["phi3"]()
+        elif isinstance(text_config, dict):
+            text_config["model_type"] = text_config.get("model_type", "phi3")
+            self.text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config)
+        elif isinstance(text_config, PretrainedConfig):
+            self.text_config = text_config
+        else:
+            raise ValueError(
+                f"Invalid type for `text_config`. Must be either `dict` or `LlamaConfig`."
+                f" Type found: {type(text_config)}"
+            )
+
+        if vision_config is None:
+            logger.info("`vision_config` is None. Initializing with default CLIPVisionConfig values")
+            self.vision_config = CONFIG_MAPPING["clip_vision_model"]()
+        elif isinstance(vision_config, dict):
+            vision_config["model_type"] = vision_config.get("model_type", "clip_vision_model")
+            self.vision_config = CONFIG_MAPPING[vision_config["model_type"]](**vision_config)
+        elif isinstance(vision_config, PretrainedConfig):
+            self.vision_config = vision_config
+        else:
+            raise ValueError(
+                f"Invalid type for `vision_config`. Must be either `dict` or `CLIPVisionConfig`."
+                f" Type found: {type(vision_config)}"
+            )
+
+        self.image_token_id = image_token_id
+        super().__init__(**kwargs)
+
+
+__all__ = ["Phi3VConfig"]