Add LLaVA-OneVision-1.5 model and related configurations

.gitattributes

@@ -310,6 +310,8 @@ src/transformers/models/llava_next_video/configuration_*.py linguist-generated=t
 src/transformers/models/llava_next_video/modeling_*.py linguist-generated=true
 src/transformers/models/llava_onevision/image_*.py linguist-generated=true
 src/transformers/models/llava_onevision/modeling_*.py linguist-generated=true
+src/transformers/models/llava_onevision1_5/configuration_*.py linguist-generated=true
+src/transformers/models/llava_onevision1_5/modeling_*.py linguist-generated=true
 src/transformers/models/longcat_flash/modeling_*.py linguist-generated=true
 src/transformers/models/lw_detr/configuration_*.py linguist-generated=true
 src/transformers/models/lw_detr/modeling_*.py linguist-generated=true

docs/source/en/_toctree.yml

@@ -1461,6 +1461,8 @@
         title: LLaVa-NeXT-Video
       - local: model_doc/llava_onevision
         title: LLaVA-Onevision
+      - local: model_doc/llava_onevision1_5
+        title: LLaVA-OneVision-1.5
       - local: model_doc/lxmert
         title: LXMERT
       - local: model_doc/matcha

docs/source/en/model_doc/llava_onevision1_5.md

@@ -0,0 +1,70 @@
+<!--Copyright 2026 The HuggingFace Team. All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+⚠️ Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be
+rendered properly in your Markdown viewer.
+-->
+*This model was contributed to Hugging Face Transformers on 2026-11-11.*
+
+# LLaVA-OneVision-1.5
+
+## Overview
+
+LLaVA-OneVision-1.5 was introduced in [LLaVA-OneVision-1.5: Fully Open Framework for Democratized Multimodal Training](https://huggingface.co/papers/2509.23661).
+It combines a native resolution vision encoder with a Qwen3 language backbone. The vision encoder adds a learned class token to each image, applies full attention, removes the class token, and merges spatial patches into visual features for the language model. The language backbone uses ordinary one dimensional rotary positions for the combined image and text sequence.
+
+Transformers supports the [4B checkpoint](https://huggingface.co/Deep-VLM/LLaVA-OneVision-1.5-4B-Instruct-hf). Use left padding for batched generation.
+
+The paper describes the work as follows:
+
+*The [paper's abstract](https://arxiv.org/abs/2509.23661) describes an open framework for training vision-language models from scratch at a reduced cost. Its release includes a concept-balanced collection of 85 million pretraining examples, a curated instruction dataset containing 22 million examples, and a training system that uses offline parallel packing. The authors report a training budget of $16,000. In their evaluation, the 8B model exceeds Qwen2.5-VL-7B on 18 out of 27 benchmarks, while the 4B model exceeds Qwen2.5-VL-3B on all 27. The report also describes a lightweight reinforcement learning stage that improves multimodal reasoning through chain-of-thought responses.*
+
+## Usage
+
+```python
+from transformers import AutoModelForImageTextToText, AutoProcessor
+
+model_id = "Deep-VLM/LLaVA-OneVision-1.5-4B-Instruct-hf"
+processor = AutoProcessor.from_pretrained(model_id)
+model = AutoModelForImageTextToText.from_pretrained(model_id, device_map="auto")
+
+messages = [{"role": "user", "content": [
+    {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
+    {"type": "text", "text": "Describe the image."},
+]}]
+
+inputs = processor.apply_chat_template(
+    messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt"
+).to(model.device)
+
+output = model.generate(**inputs, max_new_tokens=64)
+
+processor.batch_decode(output[:, inputs.input_ids.shape[1]:], skip_special_tokens=True)
+```
+
+## LlavaOnevision15Config
+
+[[autodoc]] LlavaOnevision15Config
+
+## LlavaOnevision15VisionConfig
+
+[[autodoc]] LlavaOnevision15VisionConfig
+
+## LlavaOnevision15VisionModel
+
+[[autodoc]] LlavaOnevision15VisionModel
+    - forward
+
+## LlavaOnevision15Model
+
+[[autodoc]] LlavaOnevision15Model
+    - forward
+
+## LlavaOnevision15ForConditionalGeneration
+
+[[autodoc]] LlavaOnevision15ForConditionalGeneration
+    - forward

src/transformers/conversion_mapping.py

@@ -133,6 +133,7 @@
     "MaskFormerDetrDecoder": "DetrModel",
     "Qwen2_5_VLModel": "Qwen2VLModel",
     "Qwen2_5_VLForConditionalGeneration": "Qwen2VLForConditionalGeneration",
+    "LlavaOnevision15ForConditionalGeneration": "Qwen2VLForConditionalGeneration",
     "qwen2_5_omni_thinker": "Qwen2_5OmniThinkerForConditionalGeneration",
     "Qwen3OmniMoeThinkerModel": "Qwen2_5OmniThinkerModel",
     "Qwen3OmniMoeThinkerForConditionalGeneration": "Qwen2_5OmniThinkerForConditionalGeneration",
@@ -183,6 +184,16 @@ def _build_checkpoint_conversion_mapping():
             WeightRenaming(r"\.mlp\.up_proj\.", ".mlp.fc1."),
             WeightRenaming(r"\.mlp\.down_proj\.", ".mlp.fc2."),
         ],
+        "LlavaOnevision15Model": [
+            WeightRenaming(r"^model\.(?!language_model\.|visual\.)", "language_model."),
+        ],
+        "llava_onevision1_5_vision": [
+            WeightRenaming(r"merger\.fc1\.", "merger.linear_fc1."),
+            WeightRenaming(r"merger\.fc2\.", "merger.linear_fc2."),
+            WeightRenaming(r"merger\.mlp\.0\.", "merger.linear_fc1."),
+            WeightRenaming(r"merger\.mlp\.2\.", "merger.linear_fc2."),
+            WeightRenaming(r"merger\.ln_q\.", "merger.norm."),
+        ],
         "inkling_mm_model": [
             WeightRenaming(source_patterns=r"model\.llm\.layers", target_patterns=r"model.language_model.layers"),
             WeightRenaming(

src/transformers/exporters/exporter_dynamo.py

@@ -407,6 +407,7 @@ def _to_batched(t):
     # Combined `qkv` + `(cos, sin)` rotary + `.proj`
     "transformers.models.qwen2_vl.modeling_qwen2_vl.VisionAttention.forward",
     "transformers.models.qwen2_5_vl.modeling_qwen2_5_vl.Qwen2_5_VLVisionAttention.forward",
+    "transformers.models.llava_onevision1_5.modeling_llava_onevision1_5.LlavaOnevision15VisionAttention.forward",
     "transformers.models.qwen3_vl.modeling_qwen3_vl.Qwen3VLVisionAttention.forward",
     "transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe.Qwen3VLMoeVisionAttention.forward",
     "transformers.models.qwen3_5.modeling_qwen3_5.Qwen3_5VisionAttention.forward",

src/transformers/exporters/utils.py

@@ -520,6 +520,9 @@ def _prepare_grid_thw_vision_inputs(model: torch.nn.Module, inputs: dict[str, An
     include_temporal = getattr(model, "include_temporal", False)
     inputs["position_ids"] = get_vision_position_ids(grid_thw, spatial_merge_size, include_temporal=include_temporal)
 
+    if hasattr(module, "get_vision_frame_lengths"):
+        inputs["frame_lengths"] = module.get_vision_frame_lengths(grid_thw)
+
     window_size = _find_submodule_attr(model, "window_size")
     patch_size = _find_submodule_attr(model, "patch_size")
     if window_size is not None and patch_size is not None:

src/transformers/models/auto/image_processing_auto.py

@@ -114,6 +114,7 @@
             ("lfm3_vl", {"torchvision": "Qwen2VLImageProcessor", "pil": "Qwen2VLImageProcessorPil"}),
             ("lighton_ocr", {"torchvision": "PixtralImageProcessor", "pil": "PixtralImageProcessorPil"}),
             ("llava_next_video", {"torchvision": "LlavaNextImageProcessor", "pil": "LlavaNextImageProcessorPil"}),
+            ("llava_onevision1_5", {"torchvision": "Qwen2VLImageProcessor", "pil": "Qwen2VLImageProcessorPil"}),
             ("lw_detr", {"torchvision": "DeformableDetrImageProcessor", "pil": "DeformableDetrImageProcessorPil"}),
             ("metaclip_2", {"torchvision": "CLIPImageProcessor", "pil": "CLIPImageProcessorPil"}),
             ("mgp-str", {"torchvision": "ViTImageProcessor", "pil": "ViTImageProcessorPil"}),

src/transformers/models/auto/modeling_auto.py

@@ -321,6 +321,8 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("llava_next", "LlavaNextModel"),
         ("llava_next_video", "LlavaNextVideoModel"),
         ("llava_onevision", "LlavaOnevisionModel"),
+        ("llava_onevision1_5", "LlavaOnevision15Model"),
+        ("llava_onevision1_5_vision", "LlavaOnevision15VisionModel"),
         ("longcat_flash", "LongcatFlashModel"),
         ("longformer", "LongformerModel"),
         ("longt5", "LongT5Model"),
@@ -1198,6 +1200,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("llava_next", "LlavaNextForConditionalGeneration"),
         ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),
         ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),
+        ("llava_onevision1_5", "LlavaOnevision15ForConditionalGeneration"),
         ("minicpm_ocr", "MiniCPMOcrForConditionalGeneration"),
         ("minicpmv4_6", "MiniCPMV4_6ForConditionalGeneration"),
         ("minicpmv4_7", "MiniCPMV4_7ForConditionalGeneration"),

src/transformers/models/auto/processing_auto.py

@@ -68,6 +68,7 @@
             ("inkling_mm_model", "InklingProcessor"),
             ("lasr_ctc", "LasrProcessor"),
             ("lasr_encoder", "LasrProcessor"),
+            ("llava_onevision1_5", "Qwen2_5_VLProcessor"),
             ("metaclip_2", "CLIPProcessor"),
             ("mistral3", "PixtralProcessor"),
             ("mistral_large4", "PixtralProcessor"),

src/transformers/models/auto/tokenization_auto.py

@@ -187,6 +187,7 @@
         ("led", "LEDTokenizer" if is_tokenizers_available() else None),
         ("lighton_ocr", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
         ("lilt", "RobertaTokenizer" if is_tokenizers_available() else None),
+        ("llava_onevision1_5", "Qwen2Tokenizer" if is_tokenizers_available() else None),
         ("longformer", "RobertaTokenizer" if is_tokenizers_available() else None),
         ("luke", "LukeTokenizer"),
         ("lxmert", "LxmertTokenizer" if is_tokenizers_available() else None),

src/transformers/models/auto/video_processing_auto.py

@@ -61,6 +61,7 @@
             ("hyperclovax_vision_v2", {"torchvision": "Qwen2VLVideoProcessor"}),
             ("instructblip", {"torchvision": "InstructBlipVideoVideoProcessor"}),
             ("lfm3_vl", {"torchvision": "Qwen3VLVideoProcessor"}),
+            ("llava_onevision1_5", {"torchvision": "Qwen2VLVideoProcessor"}),
             ("minicpmv4_7", {"torchvision": "MiniCPMV4_6VideoProcessor"}),
             ("muse_spark", {"torchvision": "MuseGlimmerVideoProcessor"}),
             ("pe_audio_video", {"torchvision": "PeVideoVideoProcessor"}),