diff --git a/src/transformers/models/fuyu/image_processing_fuyu.py b/src/transformers/models/fuyu/image_processing_fuyu.py index 25cd58716f6d..234afdfba2e1 100644 --- a/src/transformers/models/fuyu/image_processing_fuyu.py +++ b/src/transformers/models/fuyu/image_processing_fuyu.py @@ -302,10 +302,7 @@ def preprocess_with_tokenizer_info( patch_size (`dict[str, int]`, *optional*): Size of the patches. """ - requires_backends(self, ["torch"]) - logger.warning( - "`image_processor.preprocess_with_tokenizer_info` is deprecated and will be removed in future versions." - ) + logger.warning("`image_processor.preprocess_with_tokenizer_info` is deprecated and will be removed in v.5.20.") if patch_size is None: if isinstance(self.patch_size, SizeDict): diff --git a/src/transformers/models/fuyu/image_processing_pil_fuyu.py b/src/transformers/models/fuyu/image_processing_pil_fuyu.py index 0615909becb4..1dbdc5aee064 100644 --- a/src/transformers/models/fuyu/image_processing_pil_fuyu.py +++ b/src/transformers/models/fuyu/image_processing_pil_fuyu.py @@ -14,7 +14,6 @@ """Image processor class for Fuyu.""" import math -from typing import TYPE_CHECKING import numpy as np @@ -35,8 +34,6 @@ logger = logging.get_logger(__name__) -if TYPE_CHECKING: - import torch if is_torch_available(): import torch @@ -243,20 +240,15 @@ def get_num_patches(self, image_height: int, image_width: int, patch_size: SizeD num_patches = num_patches_per_dim_h * num_patches_per_dim_w return num_patches - def patchify_image( - self, image: "np.ndarray | torch.Tensor", patch_size: SizeDict | None = None - ) -> "np.ndarray | torch.Tensor": + def patchify_image(self, image: "np.ndarray", patch_size: SizeDict | None = None) -> "np.ndarray": """ Convert an image into a tensor of patches using numpy operations. Args: - image (`np.ndarray` or `torch.Tensor`): + image (`np.ndarray`): Image to convert. Shape: [batch, channels, height, width] or [channels, height, width] patch_size (`SizeDict`, *optional*): Dictionary in the format `{"height": int, "width": int}` specifying the size of the patches. """ - requires_backends(self, ["torch"]) - import torch - if patch_size is None: if isinstance(self.patch_size, SizeDict): patch_size = self.patch_size @@ -264,25 +256,16 @@ def patchify_image( patch_size = SizeDict(**self.patch_size) patch_height, patch_width = patch_size.height, patch_size.width - # Handle torch tensors by converting to numpy - is_torch = isinstance(image, torch.Tensor) - if is_torch: - image_np = image.cpu().numpy() - device = image.device - else: - image_np = image - device = None - # Handle batch dimension - if len(image_np.shape) == 4: - batch_size, channels, height, width = image_np.shape - elif len(image_np.shape) == 3: + if len(image.shape) == 4: + batch_size, channels, height, width = image.shape + elif len(image.shape) == 3: batch_size = 1 - channels, height, width = image_np.shape - image_np = image_np[np.newaxis, ...] + channels, height, width = image.shape + image = image[np.newaxis, ...] else: raise ValueError( - f"Expected image shape [batch, channels, height, width] or [channels, height, width], got {image_np.shape}" + f"Expected image shape [batch, channels, height, width] or [channels, height, width], got {image.shape}" ) # Extract patches using numpy operations to match torch unfold behavior exactly @@ -300,7 +283,7 @@ def patchify_image( # After reshape: (num_patches, channels * patch_height * patch_width) # Reshape to extract patches: (channels, num_patches_h, patch_height, num_patches_w, patch_width) - img_reshaped = image_np[b].reshape(channels, num_patches_h, patch_height, num_patches_w, patch_width) + img_reshaped = image[b].reshape(channels, num_patches_h, patch_height, num_patches_w, patch_width) # Transpose to (channels, num_patches, patch_height, patch_width) where num_patches = num_patches_h * num_patches_w img_reshaped = img_reshaped.transpose(0, 1, 3, 2, 4).reshape( channels, num_patches, patch_height, patch_width @@ -312,11 +295,6 @@ def patchify_image( patches_list.append(patches) patches_array = np.stack(patches_list, axis=0) if batch_size > 1 else patches_list[0] - - # Convert back to torch if input was torch - if is_torch: - patches_array = torch.from_numpy(patches_array).to(device) - return patches_array def preprocess_with_tokenizer_info( @@ -355,9 +333,7 @@ def preprocess_with_tokenizer_info( requires_backends(self, ["torch"]) import torch - logger.warning( - "`image_processor.preprocess_with_tokenizer_info` is deprecated and will be removed in future versions." - ) + logger.warning("`image_processor.preprocess_with_tokenizer_info` is deprecated and will be removed in v5.20.") if patch_size is None: if isinstance(self.patch_size, SizeDict): @@ -400,8 +376,8 @@ def preprocess_with_tokenizer_info( [num_patches], image_placeholder_id, dtype=torch.int32, device=image_input.device ) # Patchify the image - convert to numpy, patchify, convert back - image_np = image.cpu().numpy() - patches_np = self.patchify_image(image_np, patch_size=patch_size) + image = image.cpu().numpy() + patches_np = self.patchify_image(image, patch_size=patch_size) patches = torch.from_numpy(patches_np).to(image_input.device) assert num_patches == patches.shape[0] if variable_sized: diff --git a/src/transformers/models/fuyu/modeling_fuyu.py b/src/transformers/models/fuyu/modeling_fuyu.py index 265df3cdabcc..50000f76d695 100644 --- a/src/transformers/models/fuyu/modeling_fuyu.py +++ b/src/transformers/models/fuyu/modeling_fuyu.py @@ -23,6 +23,7 @@ from ...models.auto.modeling_auto import AutoModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging, torch_compilable_check +from ...utils.deprecation import deprecate_kwarg from .configuration_fuyu import FuyuConfig @@ -144,11 +145,12 @@ def get_placeholder_mask( @can_return_tuple @auto_docstring + @deprecate_kwarg("image_patches", version="v5.24", new_name="pixel_values") def forward( self, input_ids: torch.LongTensor | None = None, # [batch_size, num_total_patches, patch_size_ x patch_size x num_channels ] - image_patches: torch.Tensor | None = None, + pixel_values: torch.Tensor | None = None, attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, @@ -157,11 +159,6 @@ def forward( mm_encoder_outputs: dict[str, BaseModelOutputWithPooling] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | CausalLMOutputWithPast: - r""" - image_patches (`torch.FloatTensor` of shape `(batch_size, num_total_patches, patch_size_ x patch_size x num_channels)`, *optional*): - Image patches to be used as continuous embeddings. The patches are flattened and then projected to the - hidden size of the model. - """ if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -179,8 +176,8 @@ def forward( position_ids = position_ids.unsqueeze(0) mm_encoder_outputs = mm_encoder_outputs if mm_encoder_outputs is not None else {} - if mm_encoder_outputs.get("image") is None and image_patches is not None: - mm_encoder_outputs["image"] = self.get_image_features(image_patches, return_dict=True) + if mm_encoder_outputs.get("image") is None and pixel_values is not None: + mm_encoder_outputs["image"] = self.get_image_features(pixel_values, return_dict=True) if mm_encoder_outputs.get("image") is not None: patch_embeddings = mm_encoder_outputs["image"].last_hidden_state.to( @@ -219,11 +216,14 @@ def __init__(self, config: FuyuConfig): @can_return_tuple @auto_docstring + @deprecate_kwarg("image_patches", version="v5.23", new_name="pixel_values") + # Keep both image inputs in sig because `generate()` relies on inspect to filter out inputs def forward( self, input_ids: torch.LongTensor | None = None, # [batch_size, num_total_patches, patch_size_ x patch_size x num_channels ] image_patches: torch.Tensor | None = None, + pixel_values: torch.Tensor | None = None, attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, @@ -270,7 +270,7 @@ def forward( outputs = self.model( input_ids=input_ids, - image_patches=image_patches, + pixel_values=pixel_values, inputs_embeds=inputs_embeds, attention_mask=attention_mask, position_ids=position_ids, diff --git a/src/transformers/models/fuyu/processing_fuyu.py b/src/transformers/models/fuyu/processing_fuyu.py index fe09a1637919..1d59978ebd2f 100644 --- a/src/transformers/models/fuyu/processing_fuyu.py +++ b/src/transformers/models/fuyu/processing_fuyu.py @@ -240,9 +240,14 @@ def __call__( self, images: ImageInput | None = None, text: str | list[str] | TextInput | PreTokenizedInput | None = None, + return_legacy_image_output: bool = True, **kwargs: Unpack[FuyuProcessorKwargs], ) -> "BatchFeature": r""" + return_legacy_image_output (`bool`, *optional*, defaults to `True`): + Whether to return image inputs in legacy format or not. Note that legacy format is + deprecated and will be removed in v5.23. + Returns: [`FuyuBatchEncoding`]: A [`FuyuBatchEncoding`] with the following fields: @@ -278,7 +283,9 @@ def __call__( processed_images = {} images_replacements = [] if images is not None and hasattr(self, "image_processor"): - processed_images, images_replacements = self._process_images(images, **merged_kwargs["images_kwargs"]) + processed_images, images_replacements = self._process_images( + images, return_legacy_image_output=return_legacy_image_output, **merged_kwargs["images_kwargs"] + ) text_inputs = {} return_tensors = merged_kwargs["text_kwargs"].get("return_tensors", None) @@ -392,7 +399,7 @@ def validate_inputs( if text is None and images is None: raise ValueError("You must provide either `text` or `images`.") - def _process_images(self, images: ImageInput, **kwargs): + def _process_images(self, images: ImageInput, return_legacy_image_output: bool = True, **kwargs): processed_images = self.image_processor(images, **kwargs) batch_image_patches = [] @@ -424,7 +431,14 @@ def _process_images(self, images: ImageInput, **kwargs): ) image_replacements.append(replacement_text) - processed_images["image_patches"] = torch.cat(batch_image_patches, dim=0) + if return_legacy_image_output: + processed_images["image_patches"] = torch.cat(batch_image_patches, dim=0) + logger.warning_once( + "returning `image_patches` is deprecated, and we'll be returning `pixel_values` from v5.23. " + "Please pass `return_legacy_image_output=False` to get `pixel_values` in inputs." + ) + else: + processed_images["pixel_values"] = torch.cat(batch_image_patches, dim=0) return processed_images, image_replacements def replace_image_token(self, image_inputs: dict, image_idx: int, **kwargs) -> str: diff --git a/src/transformers/models/video_llava/image_processing_video_llava.py b/src/transformers/models/video_llava/image_processing_video_llava.py index d462a00d0260..62718726966c 100644 --- a/src/transformers/models/video_llava/image_processing_video_llava.py +++ b/src/transformers/models/video_llava/image_processing_video_llava.py @@ -13,6 +13,8 @@ # limitations under the License. """Image processor class for Video-LLaVA.""" +from copy import deepcopy + import numpy as np from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict @@ -35,12 +37,23 @@ valid_images, validate_preprocess_arguments, ) +from ...processing_utils import ImagesKwargs from ...utils import TensorType, filter_out_non_signature_kwargs, logging logger = logging.get_logger(__name__) +class VideoLlavaImageProcessorKwargs(ImagesKwargs, total=False): + """ + return_legacy_image_output (`bool`, *optional*, defaults to `True`): + Whether to return outputs in lagacy format or not. From v5.20 + the legacy path will be dropped. + """ + + return_legacy_image_output: bool + + class VideoLlavaImageProcessor(BaseImageProcessor): r""" Constructs a CLIP image processor. @@ -81,6 +94,7 @@ class VideoLlavaImageProcessor(BaseImageProcessor): """ model_input_names = ["pixel_values"] + valid_kwargs = VideoLlavaImageProcessorKwargs def __init__( self, @@ -111,8 +125,8 @@ def __init__( self.do_rescale = do_rescale self.rescale_factor = rescale_factor self.do_normalize = do_normalize - self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN - self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD + self.image_mean = image_mean if image_mean is not None else deepcopy(OPENAI_CLIP_MEAN) + self.image_std = image_std if image_std is not None else deepcopy(OPENAI_CLIP_STD) self.do_convert_rgb = do_convert_rgb def resize( @@ -182,6 +196,7 @@ def preprocess( return_tensors: str | TensorType | None = None, data_format: ChannelDimension | None = ChannelDimension.FIRST, input_data_format: str | ChannelDimension | None = None, + return_legacy_image_output: bool = True, ) -> BatchFeature: """ Preprocess an image or batch of images. @@ -272,7 +287,14 @@ def preprocess( ) for image in images ] - data = {"pixel_values_images": pixel_values_images} + if return_legacy_image_output: + data = {"pixel_values_images": pixel_values_images} + logger.warning_once( + "returning `pixel_values_images` is deprecated, and we'll be returning `pixel_values` from v5.23. " + "Please pass `return_legacy_image_output=False` to get `pixel_values` in inputs." + ) + else: + data = {"pixel_values": pixel_values_images} encoded_outputs = BatchFeature(data, tensor_type=return_tensors) return encoded_outputs diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index 40b4cef83e28..d2261d62fa87 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -27,6 +27,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, logging, torch_compilable_check +from ...utils.deprecation import deprecate_kwarg from ...utils.generic import can_return_tuple, merge_with_config_defaults from ..auto import AutoModel from .configuration_video_llava import VideoLlavaConfig @@ -166,9 +167,11 @@ def __init__(self, config: VideoLlavaConfig): @auto_docstring( custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection." ) + @deprecate_kwarg("pixel_values_images", version="v5.23", new_name="pixel_values") def get_image_features( self, - pixel_values_images: torch.FloatTensor, + pixel_values: torch.FloatTensor | None = None, + pixel_values_images: torch.FloatTensor | None = None, vision_feature_layer: int | list[int] | list[int] | None = None, vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], @@ -176,17 +179,13 @@ def get_image_features( r""" pixel_values_images (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`) The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` """ kwargs["output_hidden_states"] = True # Ignore arg on purpose image_outputs = self.image_tower( - pixel_values_images, + pixel_values, return_dict=True, **kwargs, ) @@ -223,10 +222,6 @@ def get_video_features( r""" pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_frames, channels, height, width)`) The tensors corresponding to the input videos. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. """ batch_size_vid, num_frames, channels, height, width = pixel_values_videos.shape @@ -297,10 +292,11 @@ def get_placeholder_mask( @merge_with_config_defaults @can_return_tuple @auto_docstring + @deprecate_kwarg("pixel_values_images", version="v5.23", new_name="pixel_values") def forward( self, input_ids: torch.LongTensor | None = None, - pixel_values_images: torch.FloatTensor | None = None, + pixel_values: torch.FloatTensor | None = None, pixel_values_videos: torch.FloatTensor | None = None, attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, @@ -312,27 +308,21 @@ def forward( mm_encoder_outputs: dict[str, BaseModelOutputWithPooling] | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple | VideoLlavaModelOutputWithPast: - r""" - pixel_values_images (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`VideoLlavaImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses - [`VideoLlavaImageProcessor`] for processing images). - """ if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") - if (pixel_values_images is not None or pixel_values_videos is not None) and mm_encoder_outputs is not None: + if (pixel_values is not None or pixel_values_videos is not None) and mm_encoder_outputs is not None: raise ValueError( - "You cannot specify both `pixel_values_images/pixel_values_videos` and `mm_encoder_outputs` at the same time" + "You cannot specify both `pixel_values/pixel_values_videos` and `mm_encoder_outputs` at the same time" ) if inputs_embeds is None: inputs_embeds = self.get_input_embeddings()(input_ids) mm_encoder_outputs = mm_encoder_outputs if mm_encoder_outputs is not None else {} - if mm_encoder_outputs.get("image") is None and pixel_values_images is not None: + if mm_encoder_outputs.get("image") is None and pixel_values is not None: mm_encoder_outputs["image"] = self.get_image_features( - pixel_values_images, + pixel_values, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, return_dict=True, @@ -397,26 +387,21 @@ def get_output_embeddings(self) -> nn.Module: @merge_with_config_defaults @can_return_tuple @auto_docstring + @deprecate_kwarg("pixel_values_images", version="v5.23", new_name="pixel_values") def get_image_features( self, - pixel_values_images: torch.FloatTensor, + pixel_values: torch.FloatTensor, vision_feature_layer: int | list[int] | list[int] | None = None, vision_feature_select_strategy: str | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPooling: r""" - pixel_values_images (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - vision_feature_layer (`Union[int, list[int]]`, *optional*): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. vision_feature_select_strategy (`str`, *optional*): The feature selection strategy used to select the vision feature from the vision backbone. Can be one of `"default"` or `"full"` """ return self.model.get_image_features( - pixel_values_images=pixel_values_images, + pixel_values=pixel_values, vision_feature_layer=vision_feature_layer, vision_feature_select_strategy=vision_feature_select_strategy, **kwargs, @@ -425,9 +410,12 @@ def get_image_features( @merge_with_config_defaults @can_return_tuple @auto_docstring + @deprecate_kwarg("pixel_values_images", version="v5.23", new_name="pixel_values") + # Keep both image inputs in sig because `generate()` relies on inspect to filter out inputs def forward( self, input_ids: torch.LongTensor | None = None, + pixel_values: torch.FloatTensor | None = None, pixel_values_images: torch.FloatTensor | None = None, pixel_values_videos: torch.FloatTensor | None = None, attention_mask: torch.Tensor | None = None, @@ -516,7 +504,7 @@ def forward( """ outputs = self.model( input_ids=input_ids, - pixel_values_images=pixel_values_images, + pixel_values=pixel_values, pixel_values_videos=pixel_values_videos, attention_mask=attention_mask, position_ids=position_ids, diff --git a/src/transformers/models/video_llava/processing_video_llava.py b/src/transformers/models/video_llava/processing_video_llava.py index 3c4feb20e515..958797256518 100644 --- a/src/transformers/models/video_llava/processing_video_llava.py +++ b/src/transformers/models/video_llava/processing_video_llava.py @@ -74,7 +74,8 @@ def __init__( super().__init__(image_processor, video_processor, tokenizer, chat_template=chat_template) def replace_image_token(self, image_inputs: dict, image_idx: int, **kwargs) -> str: - image = image_inputs["pixel_values_images"][image_idx] + new_pixel_values = image_inputs.get("pixel_values") + image = image_inputs.get("pixel_values_images", new_pixel_values)[image_idx] height, width = get_image_size(to_numpy_array(image)) num_image_tokens = (height // self.patch_size) * (width // self.patch_size) num_image_tokens += self.num_additional_image_tokens diff --git a/tests/models/fuyu/test_modeling_fuyu.py b/tests/models/fuyu/test_modeling_fuyu.py index 6f06c8d5f68d..aab33e330fbd 100644 --- a/tests/models/fuyu/test_modeling_fuyu.py +++ b/tests/models/fuyu/test_modeling_fuyu.py @@ -196,6 +196,23 @@ def test_mismatching_image_patches(self): with self.assertRaises(ValueError): _ = model(input_ids=input_ids, image_patches=image_patches) + def test_nonlegacy_input_name(self): + """ + Tests that we support legacy `image_patches` and the recommended `pixel_values` as input. + """ + config, input_dict = self.model_tester.prepare_config_and_inputs_for_common() + nonlegacy_input_dict = input_dict.copy() + nonlegacy_input_dict["pixel_values"] = nonlegacy_input_dict["image_patches"] + del nonlegacy_input_dict["image_patches"] + for model_class in self.all_model_classes: + model = model_class(config).to(torch_device) + model.eval() + with torch.no_grad(): + out1 = model(**input_dict) + out2 = model(**nonlegacy_input_dict) + + torch.testing.assert_close(out1[0], out2[0]) + @parameterized.expand([("random",), ("same",)]) @pytest.mark.generate @unittest.skip("Fuyu doesn't support assisted generation due to the need to crop/extend image patches indices") diff --git a/tests/models/fuyu/test_processing_fuyu.py b/tests/models/fuyu/test_processing_fuyu.py index 8ccec2bde669..c9e64d8c93b3 100644 --- a/tests/models/fuyu/test_processing_fuyu.py +++ b/tests/models/fuyu/test_processing_fuyu.py @@ -208,6 +208,21 @@ def test_get_num_multimodal_tokens_matches_processor_call(self): num_image_tokens_from_helper = processor._get_num_multimodal_tokens(image_sizes=image_sizes) self.assertListEqual(num_image_tokens_from_call, num_image_tokens_from_helper["num_image_tokens"]) + def test_legacy_processor_inputs(self): + """ + Tests that `return_legacy_image_output` actually takes effect. + """ + processor = self.get_processor() + text = self.prepare_text_inputs(batch_size=1, modalities="image") + image_inputs = self.prepare_images_inputs(batch_size=1) + inputs = processor(text=text, images=image_inputs, return_legacy_image_output=False, return_tensors="pt") + self.assertTrue("pixel_values" in inputs) + self.assertTrue("image_patches" not in inputs) + + inputs = processor(text=text, images=image_inputs, return_legacy_image_output=True, return_tensors="pt") + self.assertTrue("pixel_values" not in inputs) + self.assertTrue("image_patches" in inputs) + @require_torch class TestProcessImagesForModelInput(unittest.TestCase): diff --git a/tests/models/video_llava/test_image_processing_video_llava.py b/tests/models/video_llava/test_image_processing_video_llava.py new file mode 100644 index 000000000000..bb86a49562de --- /dev/null +++ b/tests/models/video_llava/test_image_processing_video_llava.py @@ -0,0 +1,168 @@ +# Copyright 2026 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest + +import numpy as np + +from transformers.testing_utils import require_torch, require_vision +from transformers.utils import is_torch_available, is_vision_available + +from ...test_image_processing_common import ImageProcessingTester, ImageProcessingTestMixin + + +if is_vision_available(): + from PIL import Image + +if is_torch_available(): + import torch + + +class VideoLlavaImageProcessingTester(ImageProcessingTester): + def __init__(self, **kwargs): + # Image processor init kwargs + kwargs.setdefault("size", {"shortest_edge": 20}) + kwargs.setdefault("crop_size", {"height": 18, "width": 18}) + + super().__init__(**kwargs) + + +@require_torch +@require_vision +class VideoLlavaImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase): + image_processor_tester_class = VideoLlavaImageProcessingTester + + @unittest.skip("Model doesn't support backends yet!") + def test_new_models_require_torchvision_backend(self): + pass + + @unittest.skip("Model doesn't support backends yet!") + def test_pil_can_load_without_torchvision(self): + pass + + def test_call_pil(self): + for image_processing_class in self.image_processing_classes.values(): + # Initialize image_processing + image_processing = image_processing_class(**self.image_processor_dict) + # create random PIL images + image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False) + for image in image_inputs: + self.assertIsInstance(image, Image.Image) + + # Test not batched input + encoded_images = image_processing(image_inputs[0], return_tensors="pt").pixel_values_images + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]]) + self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape)) + + # Test batched + encoded_images = image_processing(image_inputs, return_tensors="pt").pixel_values_images + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs) + self.assertEqual( + tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape) + ) + + def test_call_numpy(self): + for image_processing_class in self.image_processing_classes.values(): + # Initialize image_processing + image_processing = image_processing_class(**self.image_processor_dict) + # create random numpy tensors + image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, numpify=True) + for image in image_inputs: + self.assertIsInstance(image, np.ndarray) + + # Test not batched input + encoded_images = image_processing(image_inputs[0], return_tensors="pt").pixel_values_images + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]]) + self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape)) + + # Test batched + encoded_images = image_processing(image_inputs, return_tensors="pt").pixel_values_images + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs) + self.assertEqual( + tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape) + ) + + def test_call_pytorch(self): + for image_processing_class in self.image_processing_classes.values(): + # Initialize image_processing + image_processing = image_processing_class(**self.image_processor_dict) + # create random PyTorch tensors + image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, torchify=True) + + for image in image_inputs: + self.assertIsInstance(image, torch.Tensor) + + # Test not batched input + encoded_images = image_processing(image_inputs[0], return_tensors="pt").pixel_values_images + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]]) + self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape)) + + # Test batched + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs) + encoded_images = image_processing(image_inputs, return_tensors="pt").pixel_values_images + self.assertEqual( + tuple(encoded_images.shape), + (self.image_processor_tester.batch_size, *expected_output_image_shape), + ) + + def test_call_numpy_4_channels(self): + for image_processing_class in self.image_processing_classes.values(): + # Test that can process images which have an arbitrary number of channels + # Initialize image_processing + image_processor = image_processing_class(**self.image_processor_dict) + + # create random numpy tensors + self.image_processor_tester.num_channels = 4 + image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, numpify=True) + + # Test not batched input + encoded_images = image_processor( + image_inputs[0], + return_tensors="pt", + input_data_format="channels_last", + image_mean=[0.0, 0.0, 0.0, 0.0], + image_std=[1.0, 1.0, 1.0, 1.0], + ).pixel_values_images + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]]) + self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape)) + + # Test batched + encoded_images = image_processor( + image_inputs, + return_tensors="pt", + input_data_format="channels_last", + image_mean=[0.0, 0.0, 0.0, 0.0], + image_std=[1.0, 1.0, 1.0, 1.0], + ).pixel_values_images + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs) + self.assertEqual( + tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape) + ) + + def test_legacy_processor_inputs(self): + """ + Tests that `return_legacy_image_output` actually takes effect. + """ + for image_processing_class in self.image_processing_classes.values(): + image_processing = image_processing_class(**self.image_processor_dict) + image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False) + + inputs = image_processing(image_inputs, return_legacy_image_output=False, return_tensors="pt") + self.assertTrue("pixel_values" in inputs) + self.assertTrue("pixel_values_images" not in inputs) + + inputs = image_processing(image_inputs, return_legacy_image_output=True, return_tensors="pt") + self.assertTrue("pixel_values" not in inputs) + self.assertTrue("pixel_values_images" in inputs) diff --git a/tests/models/video_llava/test_modeling_video_llava.py b/tests/models/video_llava/test_modeling_video_llava.py index b3c298657f0f..523eda113a8a 100644 --- a/tests/models/video_llava/test_modeling_video_llava.py +++ b/tests/models/video_llava/test_modeling_video_llava.py @@ -235,6 +235,23 @@ def test_training_gradient_checkpointing_use_reentrant_true(self): def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): pass + def test_nonlegacy_input_name(self): + """ + Tests that we support legacy `pixel_values_images` and the recommended `pixel_values` as input. + """ + config, input_dict = self.model_tester.prepare_config_and_inputs_for_common() + nonlegacy_input_dict = input_dict.copy() + nonlegacy_input_dict["pixel_values"] = nonlegacy_input_dict["pixel_values_images"] + del nonlegacy_input_dict["pixel_values_images"] + for model_class in self.all_model_classes: + model = model_class(config).to(torch_device) + model.eval() + with torch.no_grad(): + out1 = model(**input_dict) + out2 = model(**nonlegacy_input_dict) + + torch.testing.assert_close(out1[0], out2[0]) + @run_test_using_subprocess def test_mixed_input(self): config, inputs = self.model_tester.prepare_config_and_inputs_for_common()