Add native CogVLM2-Video support

docs/source/en/_toctree.yml

@@ -832,7 +832,7 @@
       - local: model_doc/nomic_bert
         title: NomicBERT
       - local: model_doc/nystromformer
-        title: Nyströmformer
+        title: Nystr?mformer
       - local: model_doc/olmo
         title: OLMo
       - local: model_doc/olmo2
@@ -1281,6 +1281,8 @@
         title: CLIPSeg
       - local: model_doc/clvp
         title: CLVP
+      - local: model_doc/cogvlm2
+        title: CogVLM2
       - local: model_doc/cwm
         title: Code World Model (CWM)
       - local: model_doc/cohere2_vision

docs/source/en/model_doc/cogvlm2.md

@@ -0,0 +1,95 @@
+<!--Copyright 2026 The HuggingFace Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+-->
+
+# CogVLM2
+
+## Overview
+
+CogVLM2-Video is a video-language model from Zhipu AI / THUDM. The released
+CogVLM2-Video Llama 3 checkpoint combines an EVA2-CLIP visual encoder with a
+Llama 3-style causal language model.
+
+Each video frame is encoded into 64 spatial visual tokens. A beginning-of-image
+and end-of-image embedding are added around those tokens, producing 66 visual
+tokens per frame. CogVLM2 uses a compressed position-id scheme for visual spans
+so that long visual token runs do not consume one language position per token.
+
+The native Transformers implementation supports the modern cache and generation
+APIs and does not depend on the original Triton rotary kernel.
+
+## Inference
+
+Once the checkpoint metadata has been updated for native Transformers support,
+CogVLM2-Video can be loaded without remote code:
+
+```python
+from transformers import AutoModelForCausalLM, AutoProcessor
+
+
+model_id = "zai-org/cogvlm2-video-llama3-chat"
+
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    dtype="auto",
+    device_map="auto",
+)
+processor = AutoProcessor.from_pretrained(model_id)
+
+inputs = processor(
+    text="What happens in this video?",
+    videos="path/to/video.mp4",
+    return_tensors="pt",
+).to(model.device)
+
+output_ids = model.generate(**inputs, max_new_tokens=128)
+print(processor.tokenizer.decode(output_ids[0], skip_special_tokens=True))
+```
+
+The original checkpoint was published with remote-code metadata and without a
+`model_type` field. Existing Hub repositories need their configuration and
+processor metadata updated to use native loading after this model is released.
+
+## CogVLM2Config
+
+[[autodoc]] CogVLM2Config
+
+## CogVLM2VisionConfig
+
+[[autodoc]] CogVLM2VisionConfig
+
+## CogVLM2VideoProcessor
+
+[[autodoc]] CogVLM2VideoProcessor
+    - preprocess
+
+## CogVLM2Processor
+
+[[autodoc]] CogVLM2Processor
+    - __call__
+
+## CogVLM2VisionModel
+
+[[autodoc]] CogVLM2VisionModel
+    - forward
+
+## CogVLM2Model
+
+[[autodoc]] CogVLM2Model
+    - forward
+    - get_video_features
+
+## CogVLM2ForConditionalGeneration
+
+[[autodoc]] CogVLM2ForConditionalGeneration
+    - forward
+    - prepare_inputs_for_generation

src/transformers/models/auto/auto_mappings.py

@@ -1,8 +1,8 @@
-#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#                ????????????????????????????????????????????????????????????????????????????????????????????????
 #             This file was automatically generated from existing config files and their `model_type`s. Do NOT edit this file
 #               manually as any edits will be overwritten by auto-generation of the file. If any change should be done,
 #          please add the correct `cls.model_type` in your config class and run `python utils/check_auto.py --fix_and_overwrite`.
-#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#                ????????????????????????????????????????????????????????????????????????????????????????????????
 # Copyright 2026 The HuggingFace Inc. team.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -97,6 +97,8 @@
         ("clvp_decoder", "ClvpDecoderConfig"),
         ("clvp_encoder", "ClvpEncoderConfig"),
         ("codegen", "CodeGenConfig"),
+        ("cogvlm2", "CogVLM2Config"),
+        ("cogvlm2_vision", "CogVLM2VisionConfig"),
         ("cohere", "CohereConfig"),
         ("cohere2", "Cohere2Config"),
         ("cohere2_moe", "Cohere2MoeConfig"),
@@ -976,6 +978,7 @@
         ("tipsv2_vision_model", "tipsv2"),
         ("unispeech-sat", "unispeech_sat"),
         ("uvdoc_backbone", "uvdoc"),
+        ("cogvlm2_vision", "cogvlm2"),
         ("video_llama_3_vision", "video_llama_3"),
         ("videoprism_text_model", "videoprism"),
         ("videoprism_vision_model", "videoprism"),
@@ -996,6 +999,7 @@
 
 VIDEO_PROCESSOR_MAPPING_NAMES = OrderedDict(
     [
+        ("cogvlm2", "CogVLM2VideoProcessor"),
         ("cohere_compass", "CohereCompassVideoProcessor"),
         ("cosmos3_edge", "Cosmos3EdgeVideoProcessor"),
         ("ernie4_5_vl_moe", "Ernie4_5_VLMoeVideoProcessor"),
@@ -1082,6 +1086,7 @@
         ("clip", "CLIPProcessor"),
         ("clipseg", "CLIPSegProcessor"),
         ("clvp", "ClvpProcessor"),
+        ("cogvlm2", "CogVLM2Processor"),
         ("cohere2_vision", "Cohere2VisionProcessor"),
         ("cohere_asr", "CohereAsrProcessor"),
         ("cohere_compass", "CohereCompassProcessor"),

src/transformers/models/auto/modeling_auto.py

@@ -91,6 +91,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("clipseg", "CLIPSegModel"),
         ("clvp", "ClvpModelForConditionalGeneration"),
         ("codegen", "CodeGenModel"),
+        ("cogvlm2", "CogVLM2Model"),
         ("cohere", "CohereModel"),
         ("cohere2", "Cohere2Model"),
         ("cohere2_moe", "Cohere2MoeModel"),
@@ -723,6 +724,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("blt", "BltForCausalLM"),
         ("camembert", "CamembertForCausalLM"),
         ("codegen", "CodeGenForCausalLM"),
+        ("cogvlm2", "CogVLM2ForConditionalGeneration"),
         ("cohere", "CohereForCausalLM"),
         ("cohere2", "Cohere2ForCausalLM"),
         ("cohere2_moe", "Cohere2MoeForCausalLM"),

src/transformers/models/cogvlm2/__init__.py

@@ -0,0 +1,29 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_cogvlm2 import *
+    from .modeling_cogvlm2 import *
+    from .processing_cogvlm2 import *
+    from .video_processing_cogvlm2 import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

src/transformers/models/cogvlm2/configuration_cogvlm2.py

@@ -0,0 +1,116 @@
+#                ????????????????????????????????????????????????????????????????????????????????????????????????
+#           This file was automatically generated from src/transformers/models/cogvlm2/modular_cogvlm2.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_cogvlm2.py file directly. One of our CI enforces this.
+#                ????????????????????????????????????????????????????????????????????????????????????????????????
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...configuration_utils import PreTrainedConfig
+
+
+class CogVLM2VisionConfig(PreTrainedConfig):
+    model_type = "cogvlm2_vision"
+    base_config_key = "vision_config"
+
+    def __init__(
+        self,
+        hidden_size=1792,
+        image_size=224,
+        in_channels=3,
+        intermediate_size=15360,
+        layer_norm_eps=1e-6,
+        num_heads=16,
+        num_hidden_layers=63,
+        num_positions=257,
+        patch_size=14,
+        dropout_prob=0.0,
+        hidden_act="gelu",
+        **kwargs,
+    ):
+        self.hidden_size = hidden_size
+        self.image_size = image_size
+        self.in_channels = in_channels
+        self.intermediate_size = intermediate_size
+        self.layer_norm_eps = layer_norm_eps
+        self.num_heads = num_heads
+        self.num_hidden_layers = num_hidden_layers
+        self.num_positions = num_positions
+        self.patch_size = patch_size
+        self.dropout_prob = dropout_prob
+        self.hidden_act = hidden_act
+        super().__init__(**kwargs)
+
+
+class CogVLM2Config(PreTrainedConfig):
+    model_type = "cogvlm2"
+    sub_configs = {"vision_config": CogVLM2VisionConfig}
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=128256,
+        hidden_size=4096,
+        intermediate_size=14336,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_multi_query_heads=8,
+        hidden_act="silu",
+        max_position_embeddings=2048,
+        initializer_range=0.02,
+        rms_norm_eps=1e-5,
+        template_version="base",
+        vision_config=None,
+        rope_theta=500000.0,
+        pad_token_id=128002,
+        bos_token_id=128000,
+        eos_token_id=128001,
+        tie_word_embeddings=False,
+        use_cache=True,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_multi_query_heads = num_multi_query_heads
+        self.num_key_value_heads = num_multi_query_heads
+        self.hidden_act = hidden_act
+        self.max_position_embeddings = max_position_embeddings
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.template_version = template_version
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
+
+        if vision_config is None:
+            self.vision_config = CogVLM2VisionConfig()
+        elif isinstance(vision_config, dict):
+            self.vision_config = CogVLM2VisionConfig(**vision_config)
+        else:
+            self.vision_config = vision_config
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+
+__all__ = ["CogVLM2VisionConfig", "CogVLM2Config"]