Deprecate old input names

src/transformers/models/fuyu/image_processing_fuyu.py

@@ -302,10 +302,7 @@ def preprocess_with_tokenizer_info(
             patch_size (`dict[str, int]`, *optional*):
                 Size of the patches.
         """
-        requires_backends(self, ["torch"])
-        logger.warning(
-            "`image_processor.preprocess_with_tokenizer_info` is deprecated and will be removed in future versions."
-        )
+        logger.warning("`image_processor.preprocess_with_tokenizer_info` is deprecated and will be removed in v.5.20.")
 
         if patch_size is None:
             if isinstance(self.patch_size, SizeDict):

src/transformers/models/fuyu/image_processing_pil_fuyu.py

@@ -14,7 +14,6 @@
 """Image processor class for Fuyu."""
 
 import math
-from typing import TYPE_CHECKING
 
 import numpy as np
 
@@ -35,8 +34,6 @@
 
 logger = logging.get_logger(__name__)
 
-if TYPE_CHECKING:
-    import torch
 
 if is_torch_available():
     import torch
@@ -243,46 +240,32 @@ def get_num_patches(self, image_height: int, image_width: int, patch_size: SizeD
         num_patches = num_patches_per_dim_h * num_patches_per_dim_w
         return num_patches
 
-    def patchify_image(
-        self, image: "np.ndarray | torch.Tensor", patch_size: SizeDict | None = None
-    ) -> "np.ndarray | torch.Tensor":
+    def patchify_image(self, image: "np.ndarray", patch_size: SizeDict | None = None) -> "np.ndarray":
         """
         Convert an image into a tensor of patches using numpy operations.
         Args:
-            image (`np.ndarray` or `torch.Tensor`):
+            image (`np.ndarray`):
                 Image to convert. Shape: [batch, channels, height, width] or [channels, height, width]
             patch_size (`SizeDict`, *optional*):
                 Dictionary in the format `{"height": int, "width": int}` specifying the size of the patches.
         """
-        requires_backends(self, ["torch"])
-        import torch
-
         if patch_size is None:
             if isinstance(self.patch_size, SizeDict):
                 patch_size = self.patch_size
             else:
                 patch_size = SizeDict(**self.patch_size)
         patch_height, patch_width = patch_size.height, patch_size.width
 
-        # Handle torch tensors by converting to numpy
-        is_torch = isinstance(image, torch.Tensor)
-        if is_torch:
-            image_np = image.cpu().numpy()
-            device = image.device
-        else:
-            image_np = image
-            device = None
-
         # Handle batch dimension
-        if len(image_np.shape) == 4:
-            batch_size, channels, height, width = image_np.shape
-        elif len(image_np.shape) == 3:
+        if len(image.shape) == 4:
+            batch_size, channels, height, width = image.shape
+        elif len(image.shape) == 3:
             batch_size = 1
-            channels, height, width = image_np.shape
-            image_np = image_np[np.newaxis, ...]
+            channels, height, width = image.shape
+            image = image[np.newaxis, ...]
         else:
             raise ValueError(
-                f"Expected image shape [batch, channels, height, width] or [channels, height, width], got {image_np.shape}"
+                f"Expected image shape [batch, channels, height, width] or [channels, height, width], got {image.shape}"
             )
 
         # Extract patches using numpy operations to match torch unfold behavior exactly
@@ -300,7 +283,7 @@ def patchify_image(
             # After reshape: (num_patches, channels * patch_height * patch_width)
 
             # Reshape to extract patches: (channels, num_patches_h, patch_height, num_patches_w, patch_width)
-            img_reshaped = image_np[b].reshape(channels, num_patches_h, patch_height, num_patches_w, patch_width)
+            img_reshaped = image[b].reshape(channels, num_patches_h, patch_height, num_patches_w, patch_width)
             # Transpose to (channels, num_patches, patch_height, patch_width) where num_patches = num_patches_h * num_patches_w
             img_reshaped = img_reshaped.transpose(0, 1, 3, 2, 4).reshape(
                 channels, num_patches, patch_height, patch_width
@@ -312,11 +295,6 @@ def patchify_image(
             patches_list.append(patches)
 
         patches_array = np.stack(patches_list, axis=0) if batch_size > 1 else patches_list[0]
-
-        # Convert back to torch if input was torch
-        if is_torch:
-            patches_array = torch.from_numpy(patches_array).to(device)
-
         return patches_array
 
     def preprocess_with_tokenizer_info(
@@ -355,9 +333,7 @@ def preprocess_with_tokenizer_info(
         requires_backends(self, ["torch"])
         import torch
 
-        logger.warning(
-            "`image_processor.preprocess_with_tokenizer_info` is deprecated and will be removed in future versions."
-        )
+        logger.warning("`image_processor.preprocess_with_tokenizer_info` is deprecated and will be removed in v5.20.")
 
         if patch_size is None:
             if isinstance(self.patch_size, SizeDict):
@@ -400,8 +376,8 @@ def preprocess_with_tokenizer_info(
                         [num_patches], image_placeholder_id, dtype=torch.int32, device=image_input.device
                     )
                     # Patchify the image - convert to numpy, patchify, convert back
-                    image_np = image.cpu().numpy()
-                    patches_np = self.patchify_image(image_np, patch_size=patch_size)
+                    image = image.cpu().numpy()
+                    patches_np = self.patchify_image(image, patch_size=patch_size)
                     patches = torch.from_numpy(patches_np).to(image_input.device)
                     assert num_patches == patches.shape[0]
                     if variable_sized:

src/transformers/models/fuyu/modeling_fuyu.py

@@ -23,6 +23,7 @@
 from ...models.auto.modeling_auto import AutoModel
 from ...processing_utils import Unpack
 from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging, torch_compilable_check
+from ...utils.deprecation import deprecate_kwarg
 from .configuration_fuyu import FuyuConfig
 
 
@@ -144,11 +145,12 @@ def get_placeholder_mask(
 
     @can_return_tuple
     @auto_docstring
+    @deprecate_kwarg("image_patches", version="v5.24", new_name="pixel_values")
     def forward(
         self,
         input_ids: torch.LongTensor | None = None,
         # [batch_size, num_total_patches, patch_size_ x patch_size x num_channels ]
-        image_patches: torch.Tensor | None = None,
+        pixel_values: torch.Tensor | None = None,
         attention_mask: torch.Tensor | None = None,
         position_ids: torch.LongTensor | None = None,
         past_key_values: Cache | None = None,
@@ -157,11 +159,6 @@ def forward(
         mm_encoder_outputs: dict[str, BaseModelOutputWithPooling] | None = None,
         **kwargs: Unpack[TransformersKwargs],
     ) -> tuple | CausalLMOutputWithPast:
-        r"""
-        image_patches (`torch.FloatTensor` of shape `(batch_size, num_total_patches, patch_size_ x patch_size x num_channels)`, *optional*):
-            Image patches to be used as continuous embeddings. The patches are flattened and then projected to the
-            hidden size of the model.
-        """
         if (input_ids is None) ^ (inputs_embeds is not None):
             raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
 
@@ -179,8 +176,8 @@ def forward(
             position_ids = position_ids.unsqueeze(0)
 
         mm_encoder_outputs = mm_encoder_outputs if mm_encoder_outputs is not None else {}
-        if mm_encoder_outputs.get("image") is None and image_patches is not None:
-            mm_encoder_outputs["image"] = self.get_image_features(image_patches, return_dict=True)
+        if mm_encoder_outputs.get("image") is None and pixel_values is not None:
+            mm_encoder_outputs["image"] = self.get_image_features(pixel_values, return_dict=True)
 
         if mm_encoder_outputs.get("image") is not None:
             patch_embeddings = mm_encoder_outputs["image"].last_hidden_state.to(
@@ -219,11 +216,14 @@ def __init__(self, config: FuyuConfig):
 
     @can_return_tuple
     @auto_docstring
+    @deprecate_kwarg("image_patches", version="v5.23", new_name="pixel_values")
+    # Keep both image inputs in sig because `generate()` relies on inspect to filter out inputs
     def forward(
         self,
         input_ids: torch.LongTensor | None = None,
         # [batch_size, num_total_patches, patch_size_ x patch_size x num_channels ]
         image_patches: torch.Tensor | None = None,
+        pixel_values: torch.Tensor | None = None,
         attention_mask: torch.Tensor | None = None,
         position_ids: torch.LongTensor | None = None,
         past_key_values: Cache | None = None,
@@ -270,7 +270,7 @@ def forward(
 
         outputs = self.model(
             input_ids=input_ids,
-            image_patches=image_patches,
+            pixel_values=pixel_values,
             inputs_embeds=inputs_embeds,
             attention_mask=attention_mask,
             position_ids=position_ids,

src/transformers/models/fuyu/processing_fuyu.py

@@ -240,9 +240,14 @@ def __call__(
         self,
         images: ImageInput | None = None,
         text: str | list[str] | TextInput | PreTokenizedInput | None = None,
+        return_legacy_image_output: bool = True,
         **kwargs: Unpack[FuyuProcessorKwargs],
     ) -> "BatchFeature":
         r"""
+        return_legacy_image_output (`bool`, *optional*, defaults to `True`):
+            Whether to return image inputs in legacy format or not. Note that legacy format is
+            deprecated and will be removed in v5.23.
+
         Returns:
             [`FuyuBatchEncoding`]: A [`FuyuBatchEncoding`] with the following fields:
 
@@ -278,7 +283,9 @@ def __call__(
         processed_images = {}
         images_replacements = []
         if images is not None and hasattr(self, "image_processor"):
-            processed_images, images_replacements = self._process_images(images, **merged_kwargs["images_kwargs"])
+            processed_images, images_replacements = self._process_images(
+                images, return_legacy_image_output=return_legacy_image_output, **merged_kwargs["images_kwargs"]
+            )
 
         text_inputs = {}
         return_tensors = merged_kwargs["text_kwargs"].get("return_tensors", None)
@@ -392,7 +399,7 @@ def validate_inputs(
         if text is None and images is None:
             raise ValueError("You must provide either `text` or `images`.")
 
-    def _process_images(self, images: ImageInput, **kwargs):
+    def _process_images(self, images: ImageInput, return_legacy_image_output: bool = True, **kwargs):
         processed_images = self.image_processor(images, **kwargs)
 
         batch_image_patches = []
@@ -424,7 +431,14 @@ def _process_images(self, images: ImageInput, **kwargs):
             )
             image_replacements.append(replacement_text)
 
-        processed_images["image_patches"] = torch.cat(batch_image_patches, dim=0)
+        if return_legacy_image_output:
+            processed_images["image_patches"] = torch.cat(batch_image_patches, dim=0)
+            logger.warning_once(
+                "returning `image_patches` is deprecated, and we'll be returning `pixel_values` from v5.23. "
+                "Please pass `return_legacy_image_output=False` to get `pixel_values` in inputs."
+            )
+        else:
+            processed_images["pixel_values"] = torch.cat(batch_image_patches, dim=0)
         return processed_images, image_replacements
 
     def replace_image_token(self, image_inputs: dict, image_idx: int, **kwargs) -> str:

src/transformers/models/video_llava/image_processing_video_llava.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Image processor class for Video-LLaVA."""
 
+from copy import deepcopy
+
 import numpy as np
 
 from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
@@ -35,12 +37,23 @@
     valid_images,
     validate_preprocess_arguments,
 )
+from ...processing_utils import ImagesKwargs
 from ...utils import TensorType, filter_out_non_signature_kwargs, logging
 
 
 logger = logging.get_logger(__name__)
 
 
+class VideoLlavaImageProcessorKwargs(ImagesKwargs, total=False):
+    """
+    return_legacy_image_output (`bool`, *optional*, defaults to `True`):
+        Whether to return outputs in lagacy format or not. From v5.20
+        the legacy path will be dropped.
+    """
+
+    return_legacy_image_output: bool
+
+
 class VideoLlavaImageProcessor(BaseImageProcessor):
     r"""
     Constructs a CLIP image processor.
@@ -81,6 +94,7 @@ class VideoLlavaImageProcessor(BaseImageProcessor):
     """
 
     model_input_names = ["pixel_values"]
+    valid_kwargs = VideoLlavaImageProcessorKwargs
 
     def __init__(
         self,
@@ -111,8 +125,8 @@ def __init__(
         self.do_rescale = do_rescale
         self.rescale_factor = rescale_factor
         self.do_normalize = do_normalize
-        self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
-        self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
+        self.image_mean = image_mean if image_mean is not None else deepcopy(OPENAI_CLIP_MEAN)
+        self.image_std = image_std if image_std is not None else deepcopy(OPENAI_CLIP_STD)
         self.do_convert_rgb = do_convert_rgb
 
     def resize(
@@ -182,6 +196,7 @@ def preprocess(
         return_tensors: str | TensorType | None = None,
         data_format: ChannelDimension | None = ChannelDimension.FIRST,
         input_data_format: str | ChannelDimension | None = None,
+        return_legacy_image_output: bool = True,
     ) -> BatchFeature:
         """
         Preprocess an image or batch of images.
@@ -272,7 +287,14 @@ def preprocess(
             )
             for image in images
         ]
-        data = {"pixel_values_images": pixel_values_images}
+        if return_legacy_image_output:
+            data = {"pixel_values_images": pixel_values_images}
+            logger.warning_once(
+                "returning `pixel_values_images` is deprecated, and we'll be returning `pixel_values` from v5.23. "
+                "Please pass `return_legacy_image_output=False` to get `pixel_values` in inputs."
+            )
+        else:
+            data = {"pixel_values": pixel_values_images}
         encoded_outputs = BatchFeature(data, tensor_type=return_tensors)
 
         return encoded_outputs