Add old InternVL2-1B/2B support to the InternVL conversion script #45092

src/transformers/conversion_mapping.py

@@ -22,6 +22,7 @@
     Concatenate,
     ErnieFuseAndSplitTextVisionExperts,
     MergeModulelist,
+    SplitInternLM2QKV,
     Transpose,
     WeightConverter,
     WeightRenaming,
@@ -84,6 +85,42 @@
 
 def _build_checkpoint_conversion_mapping():
     mapping = {
+        "internvl_chat": [
+            WeightRenaming(r"^mlp1\.0\.", "model.multi_modal_projector.layer_norm."),
+            WeightRenaming(r"^mlp1\.1\.", "model.multi_modal_projector.linear_1."),
+            WeightRenaming(r"^mlp1\.3\.", "model.multi_modal_projector.linear_2."),
+            WeightRenaming(r"^vision_model\.", "model.vision_tower."),
+            WeightRenaming(r"\.encoder\.layers\.", ".encoder.layer."),
+            WeightRenaming(r"\.class_embedding", ".cls_token"),
+            WeightRenaming(r"\.position_embedding", ".position_embeddings"),
+            WeightRenaming(r"\.patch_embedding\.", ".patch_embeddings.projection."),
+            WeightRenaming(r"\.ls1$", ".lambda_1"),
+            WeightRenaming(r"\.ls2$", ".lambda_2"),
+            WeightRenaming(r"\.attn\.proj\.", ".attention.projection_layer."),
+            WeightRenaming(r"\.attn\.", ".attention."),
+            WeightRenaming(r"\.norm1\.", ".layernorm_before."),
+            WeightRenaming(r"\.norm2\.", ".layernorm_after."),
+            WeightConverter(
+                source_patterns=r"\.attention\.qkv\.",
+                target_patterns=[".attention.q_proj.", ".attention.k_proj.", ".attention.v_proj."],
+                operations=[Chunk(dim=0)],
+            ),
+            WeightRenaming(r"^language_model\.model\.", "model.language_model."),
+            WeightRenaming(r"^language_model\.lm_head\.", "lm_head."),
+            WeightRenaming(r"^language_model\.output\.", "lm_head."),
+            WeightRenaming(r"\.tok_embeddings\.", ".embed_tokens."),
+            WeightRenaming(r"\.attention\.wo\.", ".self_attn.o_proj."),
+            WeightRenaming(r"\.feed_forward\.w1\.", ".mlp.gate_proj."),
+            WeightRenaming(r"\.feed_forward\.w2\.", ".mlp.down_proj."),
+            WeightRenaming(r"\.feed_forward\.w3\.", ".mlp.up_proj."),
+            WeightRenaming(r"\.attention_norm\.", ".input_layernorm."),
+            WeightRenaming(r"\.ffn_norm\.", ".post_attention_layernorm."),
+            WeightConverter(
+                source_patterns=r"\.attention\.wqkv\.",
+                target_patterns=[".self_attn.q_proj.", ".self_attn.k_proj.", ".self_attn.v_proj."],
+                operations=[SplitInternLM2QKV()],
+            ),
+        ],
         "llava": [
             WeightRenaming(source_patterns=r"language_model.model", target_patterns="language_model"),
             WeightRenaming(source_patterns=r"language_model.lm_head", target_patterns="lm_head"),

src/transformers/core_model_loading.py

@@ -177,6 +177,98 @@ def reverse_op(self) -> ConversionOps:
         return Chunk(self.dim)
 
 
+class SplitInternLM2QKV(ConversionOps):
+    """Split InternLM2 fused grouped-query attention weights into q/k/v projections."""
+
+    @staticmethod
+    def _get_text_config(config):
+        return getattr(config, "text_config", config)
+
+    @torch.no_grad
+    def convert(
+        self, input_dict: dict[str, torch.Tensor], source_patterns: list[str], target_patterns: list[str], **kwargs
+    ) -> dict[str, torch.Tensor]:
+        if len(input_dict) > 1 or len(target_patterns) != 3:
+            raise ValueError("SplitInternLM2QKV expects one source tensor and exactly three target patterns.")
+
+        text_config = self._get_text_config(kwargs.get("config"))
+        if text_config is None:
+            raise ValueError("SplitInternLM2QKV requires a config to infer the grouped-query attention layout.")
+
+        tensor = next(iter(input_dict.values()))
+        tensor = tensor[0] if isinstance(tensor, list) else tensor
+
+        num_attention_heads = text_config.num_attention_heads
+        num_key_value_heads = text_config.num_key_value_heads
+        head_dim = getattr(text_config, "head_dim", None) or (text_config.hidden_size // num_attention_heads)
+        num_key_value_groups = num_attention_heads // num_key_value_heads
+
+        expected_qkv_rows = num_key_value_heads * (num_key_value_groups + 2) * head_dim
+        if tensor.shape[0] != expected_qkv_rows:
+            raise ValueError(
+                "Expected fused wqkv rows="
+                f"{expected_qkv_rows}, but got {tensor.shape[0]} for {kwargs.get('full_layer_name')}."
+            )
+
+        qkv = tensor.reshape(num_key_value_heads, num_key_value_groups + 2, head_dim, tensor.shape[1])
+        q_proj = qkv[:, :num_key_value_groups].reshape(-1, tensor.shape[1]).contiguous()
+        k_proj = qkv[:, -2].reshape(-1, tensor.shape[1]).contiguous()
+        v_proj = qkv[:, -1].reshape(-1, tensor.shape[1]).contiguous()
+
+        return dict(zip(target_patterns, [q_proj, k_proj, v_proj]))
+
+    @property
+    def reverse_op(self) -> ConversionOps:
+        return FuseInternLM2QKV()
+
+
+class FuseInternLM2QKV(ConversionOps):
+    """Fuse q/k/v projections into the legacy InternLM2 grouped-query attention layout."""
+
+    @staticmethod
+    def _get_text_config(config):
+        return getattr(config, "text_config", config)
+
+    @torch.no_grad
+    def convert(
+        self,
+        input_dict: dict[str, list[torch.Tensor]],
+        source_patterns: list[str],
+        target_patterns: list[str],
+        **kwargs,
+    ) -> dict[str, torch.Tensor]:
+        if len(source_patterns) != 3 or len(target_patterns) != 1:
+            raise ValueError("FuseInternLM2QKV expects three source patterns and one target pattern.")
+
+        text_config = self._get_text_config(kwargs.get("config"))
+        if text_config is None:
+            raise ValueError("FuseInternLM2QKV requires a config to infer the grouped-query attention layout.")
+
+        num_attention_heads = text_config.num_attention_heads
+        num_key_value_heads = text_config.num_key_value_heads
+        head_dim = getattr(text_config, "head_dim", None) or (text_config.hidden_size // num_attention_heads)
+        num_key_value_groups = num_attention_heads // num_key_value_heads
+
+        q_proj, k_proj, v_proj = (
+            input_dict[source_pattern][0]
+            if isinstance(input_dict[source_pattern], list)
+            else input_dict[source_pattern]
+            for source_pattern in source_patterns
+        )
+        hidden_size = q_proj.shape[1]
+
+        q_proj = q_proj.reshape(num_key_value_heads, num_key_value_groups, head_dim, hidden_size)
+        k_proj = k_proj.reshape(num_key_value_heads, 1, head_dim, hidden_size)
+        v_proj = v_proj.reshape(num_key_value_heads, 1, head_dim, hidden_size)
+        fused = torch.cat([q_proj, k_proj, v_proj], dim=1).reshape(-1, hidden_size).contiguous()
+
+        return {target_patterns[0]: fused}
+
+    @property
+    def reverse_op(self) -> ConversionOps:
+        return SplitInternLM2QKV()
+
+
 class MergeModulelist(ConversionOps):
     """
     Merge a list of tensors into a single tensor along the first dimension.

src/transformers/models/auto/configuration_auto.py

@@ -233,6 +233,7 @@
         ("instructblip", "InstructBlipConfig"),
         ("instructblipvideo", "InstructBlipVideoConfig"),
         ("internvl", "InternVLConfig"),
+        ("internvl_chat", "InternVLConfig"),
         ("internvl_vision", "InternVLVisionConfig"),
         ("jais2", "Jais2Config"),
         ("jamba", "JambaConfig"),
@@ -748,6 +749,7 @@
         ("instructblip", "InstructBLIP"),
         ("instructblipvideo", "InstructBlipVideo"),
         ("internvl", "InternVL"),
+        ("internvl_chat", "InternVL"),
         ("internvl_vision", "InternVLVision"),
         ("jais2", "Jais2"),
         ("jamba", "Jamba"),
@@ -1120,6 +1122,7 @@
         ("rt_detr_resnet", "rt_detr"),
         ("granitevision", "llava_next"),
         ("internvl_vision", "internvl"),
+        ("internvl_chat", "internvl"),
         ("qwen2_5_vl_text", "qwen2_5_vl"),
         ("qwen2_vl_text", "qwen2_vl"),
         ("qwen3_vl_text", "qwen3_vl"),
@@ -1468,6 +1471,11 @@ def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike[str],
         code_revision = kwargs.pop("code_revision", None)
 
         config_dict, unused_kwargs = PreTrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
+        if config_dict.get("model_type") == "internvl_chat":
+            # Legacy InternVL checkpoints still advertise remote code, but Transformers now supports
+            # loading them natively through the local InternVL implementation.
+            config_dict = dict(config_dict)
+            config_dict.pop("auto_map", None)
         has_remote_code = "auto_map" in config_dict and "AutoConfig" in config_dict["auto_map"]
         has_local_code = "model_type" in config_dict and config_dict["model_type"] in CONFIG_MAPPING
         explicit_local_code = has_local_code and not CONFIG_MAPPING[config_dict["model_type"]].__module__.startswith(

src/transformers/models/auto/image_processing_auto.py

@@ -54,6 +54,23 @@
     "SmolVLMImageProcessor",
 ]
 
+
+def _is_legacy_internvl_chat_config(config: PreTrainedConfig | None) -> bool:
+    return isinstance(config, PreTrainedConfig) and getattr(config, "model_type", None) == "internvl_chat"
+
+
+def _get_legacy_internvl_processor_init_kwargs(config: PreTrainedConfig) -> dict[str, dict[str, int]]:
+    image_size = getattr(config, "force_image_size", None) or getattr(config.vision_config, "image_size", None)
+    if isinstance(image_size, (list, tuple)):
+        size = {"height": image_size[0], "width": image_size[1]}
+    elif image_size is not None:
+        size = {"height": image_size, "width": image_size}
+    else:
+        return {}
+
+    return {"size": size, "crop_size": size}
+
+
 if TYPE_CHECKING:
     # This significantly improves completion suggestion performance when
     # the transformers package is used with Microsoft's Pylance language server.
@@ -144,6 +161,7 @@
             ("imagegpt", {"torchvision": "ImageGPTImageProcessor", "pil": "ImageGPTImageProcessorPil"}),
             ("instructblip", {"torchvision": "BlipImageProcessor", "pil": "BlipImageProcessorPil"}),
             ("internvl", {"torchvision": "GotOcr2ImageProcessor", "pil": "GotOcr2ImageProcessorPil"}),
+            ("internvl_chat", {"torchvision": "GotOcr2ImageProcessor", "pil": "GotOcr2ImageProcessorPil"}),
             ("janus", {"torchvision": "JanusImageProcessor", "pil": "JanusImageProcessorPil"}),
             ("kosmos-2", {"torchvision": "CLIPImageProcessor", "pil": "CLIPImageProcessorPil"}),
             ("kosmos-2.5", {"torchvision": "Kosmos2_5ImageProcessor", "pil": "Kosmos2_5ImageProcessorPil"}),
@@ -664,31 +682,80 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
         else:
             image_processor_filename = IMAGE_PROCESSOR_NAME
 
+        def _get_raw_model_type():
+            if isinstance(config, PreTrainedConfig):
+                return getattr(config, "model_type", None)
+
+            resolved_config_file = cached_file(
+                pretrained_model_name_or_path,
+                CONFIG_NAME,
+                _raise_exceptions_for_gated_repo=False,
+                _raise_exceptions_for_missing_entries=False,
+                _raise_exceptions_for_connection_errors=False,
+                **kwargs,
+            )
+            if resolved_config_file is None:
+                return None
+
+            raw_config_dict = safe_load_json_file(resolved_config_file)
+            return raw_config_dict.get("model_type")
+
+        def _ensure_config():
+            nonlocal config
+            if not isinstance(config, PreTrainedConfig):
+                config = AutoConfig.from_pretrained(
+                    pretrained_model_name_or_path,
+                    trust_remote_code=trust_remote_code,
+                    **kwargs,
+                )
+            return config
+
+        def _maybe_build_legacy_internvl_image_processor(image_processor_class):
+            if not missing_image_processor_dict:
+                return None
+
+            if _get_raw_model_type() == "internvl_chat":
+                config_obj = _ensure_config()
+                return image_processor_class(**_get_legacy_internvl_processor_init_kwargs(config_obj))
+            return None
+
         # Load the image processor config
 
+        missing_image_processor_dict = False
         try:
             config_dict, _ = ImageProcessingMixin.get_image_processor_dict(
                 pretrained_model_name_or_path, image_processor_filename=image_processor_filename, **kwargs
             )
         except Exception as initial_exception:
+            missing_image_processor_dict = True
             # Fallback for Hub TimmWrapper checkpoints (image processing in config.json, not preprocessor_config.json)
             try:
                 config_dict, _ = ImageProcessingMixin.get_image_processor_dict(
                     pretrained_model_name_or_path, image_processor_filename=CONFIG_NAME, **kwargs
                 )
             except Exception:
-                raise initial_exception
-
-            if not is_timm_config_dict(config_dict):
-                raise initial_exception
-
-        image_processor_type = config_dict.get("image_processor_type", None)
+                if _get_raw_model_type() == "internvl_chat":
+                    config_dict = {}
+                else:
+                    raise initial_exception
+
+            if config_dict and not is_timm_config_dict(config_dict):
+                if _get_raw_model_type() == "internvl_chat":
+                    config_dict = {}
+                else:
+                    raise initial_exception
+
+        image_processor_type = config_dict.get("image_processor_type")
         image_processor_auto_map = None
         if "AutoImageProcessor" in config_dict.get("auto_map", {}):
             image_processor_auto_map = config_dict["auto_map"]["AutoImageProcessor"]
 
+        # Legacy `internvl_chat` checkpoints serialize a generic CLIP feature extractor in
+        # `preprocessor_config.json`, but should now use the native InternVL image processor.
+        prefer_model_config_mapping = _get_raw_model_type() == "internvl_chat"
+
         # Backward compat: infer from feature extractor config
-        if image_processor_type is None and image_processor_auto_map is None:
+        if image_processor_type is None and image_processor_auto_map is None and not prefer_model_config_mapping:
             feature_extractor_class = config_dict.pop("feature_extractor_type", None)
             if feature_extractor_class is not None:
                 image_processor_type = feature_extractor_class.replace("FeatureExtractor", "ImageProcessor")
@@ -698,12 +765,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
 
         # If not in image processor config, try the model config
         if image_processor_type is None and image_processor_auto_map is None:
-            if not isinstance(config, PreTrainedConfig):
-                config = AutoConfig.from_pretrained(
-                    pretrained_model_name_or_path,
-                    trust_remote_code=trust_remote_code,
-                    **kwargs,
-                )
+            _ensure_config()
             image_processor_type = getattr(config, "image_processor_type", None)
             if hasattr(config, "auto_map") and "AutoImageProcessor" in config.auto_map:
                 image_processor_auto_map = config.auto_map["AutoImageProcessor"]
@@ -740,13 +802,19 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
             image_processor_class.register_for_auto_class()
             return image_processor_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
         elif image_processor_class is not None:
+            legacy_processor = _maybe_build_legacy_internvl_image_processor(image_processor_class)
+            if legacy_processor is not None:
+                return legacy_processor
             return image_processor_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
         # Last try: we use the IMAGE_PROCESSOR_MAPPING.
         elif type(config) in IMAGE_PROCESSOR_MAPPING:
             image_processor_mapping = IMAGE_PROCESSOR_MAPPING[type(config)]
             image_processor_class = _load_class_with_fallback(image_processor_mapping, backend)
 
             if image_processor_class is not None:
+                legacy_processor = _maybe_build_legacy_internvl_image_processor(image_processor_class)
+                if legacy_processor is not None:
+                    return legacy_processor
                 return image_processor_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
 
             available = [k for k, v in image_processor_mapping.items() if v is not None]

src/transformers/models/auto/modeling_auto.py

@@ -230,6 +230,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("instructblip", "InstructBlipModel"),
         ("instructblipvideo", "InstructBlipVideoModel"),
         ("internvl", "InternVLModel"),
+        ("internvl_chat", "InternVLModel"),
         ("internvl_vision", "InternVLVisionModel"),
         ("jais2", "Jais2Model"),
         ("jamba", "JambaModel"),
@@ -981,6 +982,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("instructblip", "InstructBlipForConditionalGeneration"),
         ("instructblipvideo", "InstructBlipVideoForConditionalGeneration"),
         ("internvl", "InternVLForConditionalGeneration"),
+        ("internvl_chat", "InternVLForConditionalGeneration"),
         ("janus", "JanusForConditionalGeneration"),
         ("kosmos-2", "Kosmos2ForConditionalGeneration"),
         ("kosmos-2.5", "Kosmos2_5ForConditionalGeneration"),

src/transformers/models/auto/processing_auto.py

@@ -99,6 +99,7 @@
             ("instructblip", "InstructBlipProcessor"),
             ("instructblipvideo", "InstructBlipVideoProcessor"),
             ("internvl", "InternVLProcessor"),
+            ("internvl_chat", "InternVLProcessor"),
             ("janus", "JanusProcessor"),
             ("kosmos-2", "Kosmos2Processor"),
             ("kosmos-2.5", "Kosmos2_5Processor"),