diff --git a/src/transformers/image_utils.py b/src/transformers/image_utils.py index d55bf710c06c..2a5778b2c752 100644 --- a/src/transformers/image_utils.py +++ b/src/transformers/image_utils.py @@ -233,7 +233,7 @@ def make_flat_list_of_images( if ( isinstance(images, (list, tuple)) and all(isinstance(images_i, (list, tuple)) for images_i in images) - and all(is_valid_list_of_images(images_i) for images_i in images) + and all(is_valid_list_of_images(images_i) or not images_i for images_i in images) ): return [img for img_list in images for img in img_list] @@ -255,7 +255,7 @@ def make_flat_list_of_images( def make_nested_list_of_images( images: Union[list[ImageInput], ImageInput], expected_ndims: int = 3, -) -> ImageInput: +) -> list[ImageInput]: """ Ensure that the output is a nested list of images. Args: @@ -270,7 +270,7 @@ def make_nested_list_of_images( if ( isinstance(images, (list, tuple)) and all(isinstance(images_i, (list, tuple)) for images_i in images) - and all(is_valid_list_of_images(images_i) for images_i in images) + and all(is_valid_list_of_images(images_i) or not images_i for images_i in images) ): return images diff --git a/src/transformers/models/beit/image_processing_beit.py b/src/transformers/models/beit/image_processing_beit.py index 4f549bcff6d0..7fde9c8a8c90 100644 --- a/src/transformers/models/beit/image_processing_beit.py +++ b/src/transformers/models/beit/image_processing_beit.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -390,10 +390,10 @@ def preprocess( image_std = image_std if image_std is not None else self.image_std do_reduce_labels = do_reduce_labels if do_reduce_labels is not None else self.do_reduce_labels - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if segmentation_maps is not None and not valid_images(segmentation_maps): raise ValueError( diff --git a/src/transformers/models/bit/image_processing_bit.py b/src/transformers/models/bit/image_processing_bit.py index 0bcdb7075a23..6ebd9267b6fb 100644 --- a/src/transformers/models/bit/image_processing_bit.py +++ b/src/transformers/models/bit/image_processing_bit.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -256,7 +256,7 @@ def preprocess( image_std = image_std if image_std is not None else self.image_std do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/chinese_clip/image_processing_chinese_clip.py b/src/transformers/models/chinese_clip/image_processing_chinese_clip.py index 701168625c47..3b2464e9371d 100644 --- a/src/transformers/models/chinese_clip/image_processing_chinese_clip.py +++ b/src/transformers/models/chinese_clip/image_processing_chinese_clip.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -250,7 +250,7 @@ def preprocess( image_std = image_std if image_std is not None else self.image_std do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py index 2b0a287f21a4..9221c463fe85 100644 --- a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py +++ b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py @@ -46,7 +46,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_annotations, @@ -1390,7 +1390,7 @@ def preprocess( pad_size = self.pad_size if pad_size is None else pad_size format = self.format if format is None else format - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/convnext/image_processing_convnext.py b/src/transformers/models/convnext/image_processing_convnext.py index 487913d56ebd..299cabd5d4d9 100644 --- a/src/transformers/models/convnext/image_processing_convnext.py +++ b/src/transformers/models/convnext/image_processing_convnext.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -262,7 +262,7 @@ def preprocess( size = size if size is not None else self.size size = get_size_dict(size, default_to_square=False) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py index c069a872ca6c..c6875eb9b8f8 100644 --- a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py +++ b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py @@ -46,7 +46,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_annotations, @@ -1388,7 +1388,7 @@ def preprocess( pad_size = self.pad_size if pad_size is None else pad_size format = self.format if format is None else format - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/deit/image_processing_deit.py b/src/transformers/models/deit/image_processing_deit.py index b5a7641b654f..1e2f6c3b5ae5 100644 --- a/src/transformers/models/deit/image_processing_deit.py +++ b/src/transformers/models/deit/image_processing_deit.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -240,7 +240,7 @@ def preprocess( crop_size = crop_size if crop_size is not None else self.crop_size crop_size = get_size_dict(crop_size, param_name="crop_size") - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/deprecated/vit_hybrid/image_processing_vit_hybrid.py b/src/transformers/models/deprecated/vit_hybrid/image_processing_vit_hybrid.py index f1e5a9c8e6d9..f274d8c058f9 100644 --- a/src/transformers/models/deprecated/vit_hybrid/image_processing_vit_hybrid.py +++ b/src/transformers/models/deprecated/vit_hybrid/image_processing_vit_hybrid.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_kwargs, @@ -273,7 +273,7 @@ def preprocess( image_std = image_std if image_std is not None else self.image_std do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - images = make_list_of_images(images) + images = make_flat_list_of_images(images) validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self._valid_processor_keys) diff --git a/src/transformers/models/depth_pro/image_processing_depth_pro.py b/src/transformers/models/depth_pro/image_processing_depth_pro.py index c3686231b4fe..47f224f248bd 100644 --- a/src/transformers/models/depth_pro/image_processing_depth_pro.py +++ b/src/transformers/models/depth_pro/image_processing_depth_pro.py @@ -35,7 +35,7 @@ infer_channel_dimension_format, is_scaled_image, is_torch_available, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -258,7 +258,7 @@ def preprocess( size = size if size is not None else self.size - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/detr/image_processing_detr.py b/src/transformers/models/detr/image_processing_detr.py index fe4d85304ab4..f29bd48a5934 100644 --- a/src/transformers/models/detr/image_processing_detr.py +++ b/src/transformers/models/detr/image_processing_detr.py @@ -45,7 +45,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_annotations, @@ -1363,7 +1363,7 @@ def preprocess( pad_size = self.pad_size if pad_size is None else pad_size format = self.format if format is None else format - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/donut/image_processing_donut.py b/src/transformers/models/donut/image_processing_donut.py index e8a48dcabef6..d6c963803743 100644 --- a/src/transformers/models/donut/image_processing_donut.py +++ b/src/transformers/models/donut/image_processing_donut.py @@ -35,7 +35,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -399,7 +399,7 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/dpt/image_processing_dpt.py b/src/transformers/models/dpt/image_processing_dpt.py index 9535a700c728..4a312e3005f3 100644 --- a/src/transformers/models/dpt/image_processing_dpt.py +++ b/src/transformers/models/dpt/image_processing_dpt.py @@ -39,7 +39,7 @@ is_scaled_image, is_torch_available, is_torch_tensor, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -525,10 +525,10 @@ def preprocess( size_divisor = size_divisor if size_divisor is not None else self.size_divisor do_reduce_labels = do_reduce_labels if do_reduce_labels is not None else self.do_reduce_labels - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/efficientnet/image_processing_efficientnet.py b/src/transformers/models/efficientnet/image_processing_efficientnet.py index 880bfd21995e..ea822d75ca27 100644 --- a/src/transformers/models/efficientnet/image_processing_efficientnet.py +++ b/src/transformers/models/efficientnet/image_processing_efficientnet.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -295,7 +295,7 @@ def preprocess( crop_size = crop_size if crop_size is not None else self.crop_size crop_size = get_size_dict(crop_size, param_name="crop_size") - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/emu3/image_processing_emu3.py b/src/transformers/models/emu3/image_processing_emu3.py index 5a480351307a..05431d796a30 100644 --- a/src/transformers/models/emu3/image_processing_emu3.py +++ b/src/transformers/models/emu3/image_processing_emu3.py @@ -31,8 +31,8 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - is_valid_image, - make_list_of_images, + make_flat_list_of_images, + make_nested_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -46,29 +46,6 @@ logger = logging.get_logger(__name__) -def make_batched_images(images) -> list[list[ImageInput]]: - """ - Accepts images in list or nested list format, and makes a list of images for preprocessing. - - Args: - images (`Union[list[list[ImageInput]], list[ImageInput], ImageInput]`): - The input image. - - Returns: - list: A list of images. - """ - if isinstance(images, (list, tuple)) and isinstance(images[0], (list, tuple)) and is_valid_image(images[0][0]): - return [img for img_list in images for img in img_list] - - elif isinstance(images, (list, tuple)) and is_valid_image(images[0]): - return images - - elif is_valid_image(images): - return [images] - - raise ValueError(f"Could not make batched images from {images}") - - def smart_resize( height: int, width: int, factor: int = 28, min_pixels: int = 56 * 56, max_pixels: int = 14 * 14 * 4 * 1280 ): @@ -211,7 +188,7 @@ def _preprocess( - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. """ - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if do_convert_rgb: images = [convert_to_rgb(image) for image in images] @@ -382,7 +359,7 @@ def preprocess( if images is not None: images = self.fetch_images(images) - images = make_batched_images(images) + images = make_nested_list_of_images(images) if images is not None and not valid_images(images): raise ValueError( @@ -402,20 +379,21 @@ def preprocess( pixel_values = [] for image in images: - image = self._preprocess( - image, - do_resize=do_resize, - resample=resample, - do_rescale=do_rescale, - rescale_factor=rescale_factor, - do_normalize=do_normalize, - image_mean=image_mean, - image_std=image_std, - data_format=data_format, - do_convert_rgb=do_convert_rgb, - input_data_format=input_data_format, - ) - pixel_values.extend(image) + if image: + image = self._preprocess( + image, + do_resize=do_resize, + resample=resample, + do_rescale=do_rescale, + rescale_factor=rescale_factor, + do_normalize=do_normalize, + image_mean=image_mean, + image_std=image_std, + data_format=data_format, + do_convert_rgb=do_convert_rgb, + input_data_format=input_data_format, + ) + pixel_values.extend(image) image_sizes = [image.shape[-2:] for image in pixel_values] if do_pad: @@ -471,7 +449,7 @@ def postprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if isinstance(images[0], Image.Image): return images if len(images) > 1 else images[0] diff --git a/src/transformers/models/eomt/image_processing_eomt.py b/src/transformers/models/eomt/image_processing_eomt.py index 05131c543d83..a5f482ef9b40 100644 --- a/src/transformers/models/eomt/image_processing_eomt.py +++ b/src/transformers/models/eomt/image_processing_eomt.py @@ -32,7 +32,6 @@ get_image_size, infer_channel_dimension_format, make_flat_list_of_images, - make_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -615,7 +614,7 @@ def preprocess( ) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) segmentation_maps = [to_numpy_array(mask) for mask in segmentation_maps] segmentation_maps = [ diff --git a/src/transformers/models/flava/image_processing_flava.py b/src/transformers/models/flava/image_processing_flava.py index fc8f47d87ce9..9dcc2c85d479 100644 --- a/src/transformers/models/flava/image_processing_flava.py +++ b/src/transformers/models/flava/image_processing_flava.py @@ -32,7 +32,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -637,7 +637,7 @@ def preprocess( codebook_image_mean = codebook_image_mean if codebook_image_mean is not None else self.codebook_image_mean codebook_image_std = codebook_image_std if codebook_image_std is not None else self.codebook_image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/fuyu/image_processing_fuyu.py b/src/transformers/models/fuyu/image_processing_fuyu.py index bab6e3b47b6d..29af98ed5072 100644 --- a/src/transformers/models/fuyu/image_processing_fuyu.py +++ b/src/transformers/models/fuyu/image_processing_fuyu.py @@ -464,7 +464,11 @@ def preprocess( # All transformations expect numpy arrays. batch_images = [[to_numpy_array(image) for image in images] for images in batch_images] - if do_rescale and is_scaled_image(batch_images[0][0]): + # Search for the first image in the image list. + # NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682 + first_image_in_list = [images for images in batch_images if images][0][0] + + if do_rescale and is_scaled_image(first_image_in_list): logger.warning_once( "It looks like you are trying to rescale already rescaled images. If the input" " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." @@ -472,9 +476,11 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(batch_images[0][0]) + input_data_format = infer_channel_dimension_format(first_image_in_list) - original_image_sizes = [get_image_size(images[0], channel_dim=input_data_format) for images in batch_images] + original_image_sizes = [ + get_image_size(images[0], channel_dim=input_data_format) for images in batch_images if images + ] size = get_size_dict(size) # for BC if do_resize: @@ -483,7 +489,7 @@ def preprocess( for images in batch_images ] - image_sizes = [get_image_size(images[0], channel_dim=input_data_format) for images in batch_images] + image_sizes = [get_image_size(images[0], channel_dim=input_data_format) for images in batch_images if images] image_unpadded_heights = [[image_size[0]] for image_size in image_sizes] image_unpadded_widths = [[image_size[1]] for image_size in image_sizes] diff --git a/src/transformers/models/fuyu/processing_fuyu.py b/src/transformers/models/fuyu/processing_fuyu.py index 8c75879e8b69..59b8738076fc 100644 --- a/src/transformers/models/fuyu/processing_fuyu.py +++ b/src/transformers/models/fuyu/processing_fuyu.py @@ -561,7 +561,7 @@ def __call__( # --- Use self.tokenizer to get the ids of special tokens to insert into image ids --- - tensor_batch_images = torch.stack([img[0] for img in batch_images]).unsqueeze(1) + tensor_batch_images = torch.stack([img[0] for img in batch_images if img]).unsqueeze(1) # --- Use self.image_processor again to obtain the full token ids and batch inputs --- all_encodings = [] diff --git a/src/transformers/models/glm4v/image_processing_glm4v.py b/src/transformers/models/glm4v/image_processing_glm4v.py index db2691c38dc7..2e2f246bc333 100644 --- a/src/transformers/models/glm4v/image_processing_glm4v.py +++ b/src/transformers/models/glm4v/image_processing_glm4v.py @@ -35,7 +35,6 @@ infer_channel_dimension_format, is_scaled_image, make_flat_list_of_images, - make_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -219,7 +218,7 @@ def _preprocess( - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. """ - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if do_convert_rgb: images = [convert_to_rgb(image) for image in images] diff --git a/src/transformers/models/glpn/image_processing_glpn.py b/src/transformers/models/glpn/image_processing_glpn.py index 958230a307f3..e3e0255e2b47 100644 --- a/src/transformers/models/glpn/image_processing_glpn.py +++ b/src/transformers/models/glpn/image_processing_glpn.py @@ -34,7 +34,7 @@ infer_channel_dimension_format, is_scaled_image, is_torch_available, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -186,7 +186,7 @@ def preprocess( size_divisor = size_divisor if size_divisor is not None else self.size_divisor resample = resample if resample is not None else self.resample - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/grounding_dino/image_processing_grounding_dino.py b/src/transformers/models/grounding_dino/image_processing_grounding_dino.py index 4f16a5387a05..24e3c8b3f987 100644 --- a/src/transformers/models/grounding_dino/image_processing_grounding_dino.py +++ b/src/transformers/models/grounding_dino/image_processing_grounding_dino.py @@ -44,7 +44,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_annotations, @@ -1426,7 +1426,7 @@ def preprocess( pad_size = self.pad_size if pad_size is None else pad_size format = self.format if format is None else format - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/idefics/image_processing_idefics.py b/src/transformers/models/idefics/image_processing_idefics.py index 190e1d31dc78..fe9085331cde 100644 --- a/src/transformers/models/idefics/image_processing_idefics.py +++ b/src/transformers/models/idefics/image_processing_idefics.py @@ -24,7 +24,7 @@ ChannelDimension, ImageInput, PILImageResampling, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -152,7 +152,7 @@ def preprocess( return [] images = self.fetch_images(images) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/idefics/processing_idefics.py b/src/transformers/models/idefics/processing_idefics.py index 259f35feb7ec..5f20669c8a8f 100644 --- a/src/transformers/models/idefics/processing_idefics.py +++ b/src/transformers/models/idefics/processing_idefics.py @@ -358,9 +358,10 @@ def __call__( if not all(isinstance(i, str) for i in text): raise ValueError("When using the image-text-to-text behavior, the prompts should only contain text.") if isinstance(images[0], (list, tuple)): - # if nested images, nest text as well - text = [[i] for i in text] - prompts = list(zip(images, text)) + # if nested images, un-nest each sublist and create `prompts` + prompts = [[sample, *image_list] for image_list, sample in zip(images, text)] + else: + prompts = list(zip(images, text)) output_kwargs = self._merge_kwargs( IdeficsProcessorKwargs, diff --git a/src/transformers/models/idefics2/image_processing_idefics2.py b/src/transformers/models/idefics2/image_processing_idefics2.py index 2e564708a078..51ed8f13a6ce 100644 --- a/src/transformers/models/idefics2/image_processing_idefics2.py +++ b/src/transformers/models/idefics2/image_processing_idefics2.py @@ -497,8 +497,11 @@ def preprocess( # All transformations expect numpy arrays. images_list = [[to_numpy_array(image) for image in images] for images in images_list] + # Search for the first image in the image list. + # NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682 + first_image_in_list = [images for images in images_list if images][0][0] - if do_rescale and is_scaled_image(images_list[0][0]): + if do_rescale and is_scaled_image(first_image_in_list): logger.warning_once( "It looks like you are trying to rescale already rescaled images. If the input" " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." @@ -506,7 +509,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images_list[0][0]) + input_data_format = infer_channel_dimension_format(first_image_in_list) if do_image_splitting: new_images_list = [] diff --git a/src/transformers/models/idefics3/image_processing_idefics3.py b/src/transformers/models/idefics3/image_processing_idefics3.py index feae13ab2e45..24afcdd6e02d 100644 --- a/src/transformers/models/idefics3/image_processing_idefics3.py +++ b/src/transformers/models/idefics3/image_processing_idefics3.py @@ -526,7 +526,7 @@ def _pad_image( def pad( self, - images: list[np.ndarray], + images: list[list[np.ndarray]], constant_values: Union[float, Iterable[float]] = 0, return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, @@ -537,7 +537,7 @@ def pad( For a list of images, for each images, pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width. For each sample in the batch, pads the sample with empty images to the max_number of images per sample in the batch. Optionally returns a pixel mask. Args: - images (`list[np.ndarray]`): + images (`list[list[np.ndarray]]`): List of list of images to pad. Pads to the largest height and width in the batch. constant_values (`float` or `Iterable[float]`, *optional*): The value to use for the padding if `mode` is `"constant"`. @@ -565,11 +565,13 @@ def pad( else input_data_format ) data_format = input_data_format if data_format is None else data_format + # filter out empty image lists, then take first image of the first sample + first_image_in_list = [sample_images for sample_images in images if sample_images][0][0] if input_data_format == ChannelDimension.FIRST: - n_channels = images[0][0].shape[0] + n_channels = first_image_in_list.shape[0] elif input_data_format == ChannelDimension.LAST: - n_channels = images[0][0].shape[-1] + n_channels = first_image_in_list.shape[-1] else: raise ValueError("Invalid channel dimension format.") @@ -715,6 +717,9 @@ def preprocess( # All transformations expect numpy arrays. images_list = [[to_numpy_array(image) for image in images] for images in images_list] + # Search for the first image in the image list. + # NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682 + first_image_in_list = [images for images in images_list if images][0][0] # Extra channel dimension for grayscale images if input_data_format in [ChannelDimension.LAST, None]: @@ -726,7 +731,7 @@ def preprocess( [np.expand_dims(img, axis=0) if img.ndim == 2 else img for img in images] for images in images_list ] - if do_rescale and is_scaled_image(images_list[0][0]): + if do_rescale and is_scaled_image(first_image_in_list): logger.warning_once( "It looks like you are trying to rescale already rescaled images. If the input" " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." @@ -734,7 +739,7 @@ def preprocess( # We assume that all images have the same channel dimension format. if input_data_format is None: - input_data_format = infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4)) + input_data_format = infer_channel_dimension_format(first_image_in_list, num_channels=(1, 3, 4)) if do_resize: images_list = [ diff --git a/src/transformers/models/imagegpt/image_processing_imagegpt.py b/src/transformers/models/imagegpt/image_processing_imagegpt.py index 5d3e207dd3eb..7d55a9a9beb8 100644 --- a/src/transformers/models/imagegpt/image_processing_imagegpt.py +++ b/src/transformers/models/imagegpt/image_processing_imagegpt.py @@ -26,7 +26,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -238,7 +238,7 @@ def preprocess( clusters = clusters if clusters is not None else self.clusters clusters = np.array(clusters) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/janus/image_processing_janus.py b/src/transformers/models/janus/image_processing_janus.py index ac2012c62b04..3e045d0aa264 100644 --- a/src/transformers/models/janus/image_processing_janus.py +++ b/src/transformers/models/janus/image_processing_janus.py @@ -36,7 +36,6 @@ infer_channel_dimension_format, is_scaled_image, make_flat_list_of_images, - make_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -434,7 +433,7 @@ def postprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) # Ensures input is a list + images = make_flat_list_of_images(images) # Ensures input is a list if isinstance(images[0], PIL.Image.Image): return images if len(images) > 1 else images[0] diff --git a/src/transformers/models/janus/modular_janus.py b/src/transformers/models/janus/modular_janus.py index d959b7e7b04d..d18ea256e6d9 100644 --- a/src/transformers/models/janus/modular_janus.py +++ b/src/transformers/models/janus/modular_janus.py @@ -36,7 +36,7 @@ PILImageResampling, get_image_size, infer_channel_dimension_format, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, ) from ...modeling_outputs import ModelOutput @@ -1519,7 +1519,7 @@ def postprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) # Ensures input is a list + images = make_flat_list_of_images(images) # Ensures input is a list if isinstance(images[0], PIL.Image.Image): return images if len(images) > 1 else images[0] diff --git a/src/transformers/models/kosmos2/processing_kosmos2.py b/src/transformers/models/kosmos2/processing_kosmos2.py index cb6f63e2e618..0289f82b650d 100644 --- a/src/transformers/models/kosmos2/processing_kosmos2.py +++ b/src/transformers/models/kosmos2/processing_kosmos2.py @@ -20,7 +20,7 @@ from typing import Optional, Union from ...image_processing_utils import BatchFeature -from ...image_utils import ImageInput, is_batched +from ...image_utils import ImageInput from ...processing_utils import ImagesKwargs, ProcessingKwargs, ProcessorMixin, TextKwargs, Unpack from ...tokenization_utils import AddedToken from ...tokenization_utils_base import BatchEncoding, TextInput @@ -372,7 +372,7 @@ def preprocess_examples( if images is None: images = [None] * len(texts) - elif not is_batched(images): + elif not isinstance(images, list): images = [images] if len(texts) != len(images): raise ValueError( diff --git a/src/transformers/models/kosmos2_5/image_processing_kosmos2_5.py b/src/transformers/models/kosmos2_5/image_processing_kosmos2_5.py index e679050d337f..e68ad2462d07 100644 --- a/src/transformers/models/kosmos2_5/image_processing_kosmos2_5.py +++ b/src/transformers/models/kosmos2_5/image_processing_kosmos2_5.py @@ -30,7 +30,7 @@ ImageInput, get_image_size, infer_channel_dimension_format, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -296,7 +296,7 @@ def preprocess( if kwargs.get("data_format") is not None: raise ValueError("data_format is not an accepted input as the outputs are ") - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py index dfb61c352dc3..42b00b2f5c3e 100644 --- a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py +++ b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py @@ -25,7 +25,7 @@ ImageInput, PILImageResampling, infer_channel_dimension_format, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -251,7 +251,7 @@ def preprocess( ocr_lang = ocr_lang if ocr_lang is not None else self.ocr_lang tesseract_config = tesseract_config if tesseract_config is not None else self.tesseract_config - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py index 5d6b690bb619..8189abf67311 100644 --- a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py +++ b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py @@ -29,7 +29,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -306,7 +306,7 @@ def preprocess( apply_ocr = apply_ocr if apply_ocr is not None else self.apply_ocr ocr_lang = ocr_lang if ocr_lang is not None else self.ocr_lang tesseract_config = tesseract_config if tesseract_config is not None else self.tesseract_config - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/levit/image_processing_levit.py b/src/transformers/models/levit/image_processing_levit.py index 7832dbe5f43b..bece4f1a00c2 100644 --- a/src/transformers/models/levit/image_processing_levit.py +++ b/src/transformers/models/levit/image_processing_levit.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -255,7 +255,7 @@ def preprocess( size = get_size_dict(size, default_to_square=False) crop_size = crop_size if crop_size is not None else self.crop_size crop_size = get_size_dict(crop_size, param_name="crop_size") - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/llava/image_processing_llava.py b/src/transformers/models/llava/image_processing_llava.py index a18a9e66ed8a..bbf8ab1421f8 100644 --- a/src/transformers/models/llava/image_processing_llava.py +++ b/src/transformers/models/llava/image_processing_llava.py @@ -34,7 +34,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_kwargs, @@ -368,7 +368,7 @@ def preprocess( validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self._valid_processor_keys) images = self.fetch_images(images) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/llava_next/image_processing_llava_next.py b/src/transformers/models/llava_next/image_processing_llava_next.py index 87ac7f1ce61a..5a5891b3c62b 100644 --- a/src/transformers/models/llava_next/image_processing_llava_next.py +++ b/src/transformers/models/llava_next/image_processing_llava_next.py @@ -44,7 +44,6 @@ infer_channel_dimension_format, is_scaled_image, make_flat_list_of_images, - make_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -373,7 +372,7 @@ def _preprocess( - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. """ - images = make_list_of_images(images) + images = make_flat_list_of_images(images) all_images = [] for image in images: diff --git a/src/transformers/models/llava_next_video/image_processing_llava_next_video.py b/src/transformers/models/llava_next_video/image_processing_llava_next_video.py index 5508f5dcdd4b..787691682999 100644 --- a/src/transformers/models/llava_next_video/image_processing_llava_next_video.py +++ b/src/transformers/models/llava_next_video/image_processing_llava_next_video.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, validate_preprocess_arguments, ) @@ -234,7 +234,7 @@ def _preprocess( - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. """ - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if do_convert_rgb: images = [convert_to_rgb(image) for image in images] diff --git a/src/transformers/models/mask2former/image_processing_mask2former.py b/src/transformers/models/mask2former/image_processing_mask2former.py index 2ccc3bb67bf7..5f9639d0f9db 100644 --- a/src/transformers/models/mask2former/image_processing_mask2former.py +++ b/src/transformers/models/mask2former/image_processing_mask2former.py @@ -36,7 +36,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -761,9 +761,9 @@ def preprocess( "torch.Tensor, tf.Tensor or jax.ndarray." ) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if segmentation_maps is not None and len(images) != len(segmentation_maps): raise ValueError("Images and segmentation maps must have the same length.") diff --git a/src/transformers/models/maskformer/image_processing_maskformer.py b/src/transformers/models/maskformer/image_processing_maskformer.py index 8896dad73442..5437af6df9ec 100644 --- a/src/transformers/models/maskformer/image_processing_maskformer.py +++ b/src/transformers/models/maskformer/image_processing_maskformer.py @@ -37,7 +37,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -764,9 +764,9 @@ def preprocess( "torch.Tensor, tf.Tensor or jax.ndarray." ) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if segmentation_maps is not None and len(images) != len(segmentation_maps): raise ValueError("Images and segmentation maps must have the same length.") diff --git a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py index 449b0b6f7bcd..8b0715787545 100644 --- a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py +++ b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py @@ -32,7 +32,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -245,7 +245,7 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py index 41d765ea314a..7f261c5c45c3 100644 --- a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py +++ b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py @@ -32,7 +32,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -407,10 +407,10 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/mobilevit/image_processing_mobilevit.py b/src/transformers/models/mobilevit/image_processing_mobilevit.py index 86707613764f..1207ec0da399 100644 --- a/src/transformers/models/mobilevit/image_processing_mobilevit.py +++ b/src/transformers/models/mobilevit/image_processing_mobilevit.py @@ -26,7 +26,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -404,12 +404,12 @@ def preprocess( do_reduce_labels = do_reduce_labels if do_reduce_labels is not None else self.do_reduce_labels - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/nougat/image_processing_nougat.py b/src/transformers/models/nougat/image_processing_nougat.py index 827686a6066d..79aa421d3864 100644 --- a/src/transformers/models/nougat/image_processing_nougat.py +++ b/src/transformers/models/nougat/image_processing_nougat.py @@ -35,7 +35,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -448,7 +448,7 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/oneformer/image_processing_oneformer.py b/src/transformers/models/oneformer/image_processing_oneformer.py index 5a66918af17e..d88b373418c6 100644 --- a/src/transformers/models/oneformer/image_processing_oneformer.py +++ b/src/transformers/models/oneformer/image_processing_oneformer.py @@ -39,7 +39,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -722,9 +722,9 @@ def preprocess( "torch.Tensor, tf.Tensor or jax.ndarray." ) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if segmentation_maps is not None and len(images) != len(segmentation_maps): raise ValueError("Images and segmentation maps must have the same length.") diff --git a/src/transformers/models/owlv2/image_processing_owlv2.py b/src/transformers/models/owlv2/image_processing_owlv2.py index 0db195ba160e..c228dc4a0625 100644 --- a/src/transformers/models/owlv2/image_processing_owlv2.py +++ b/src/transformers/models/owlv2/image_processing_owlv2.py @@ -34,7 +34,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -434,7 +434,7 @@ def preprocess( size = size if size is not None else self.size size = get_size_dict(size) # for BC - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/owlvit/image_processing_owlvit.py b/src/transformers/models/owlvit/image_processing_owlvit.py index 4bf9306c5640..8040248039d4 100644 --- a/src/transformers/models/owlvit/image_processing_owlvit.py +++ b/src/transformers/models/owlvit/image_processing_owlvit.py @@ -35,7 +35,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -382,7 +382,7 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/perceiver/image_processing_perceiver.py b/src/transformers/models/perceiver/image_processing_perceiver.py index 7550bc9471dc..850671324712 100644 --- a/src/transformers/models/perceiver/image_processing_perceiver.py +++ b/src/transformers/models/perceiver/image_processing_perceiver.py @@ -29,7 +29,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -286,7 +286,7 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/pix2struct/image_processing_pix2struct.py b/src/transformers/models/pix2struct/image_processing_pix2struct.py index 9e7e4ae977bd..111179796d8a 100644 --- a/src/transformers/models/pix2struct/image_processing_pix2struct.py +++ b/src/transformers/models/pix2struct/image_processing_pix2struct.py @@ -28,7 +28,7 @@ ImageInput, get_image_size, infer_channel_dimension_format, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -407,7 +407,7 @@ def preprocess( if kwargs.get("data_format") is not None: raise ValueError("data_format is not an accepted input as the outputs are ") - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/poolformer/image_processing_poolformer.py b/src/transformers/models/poolformer/image_processing_poolformer.py index ea37c22e7557..c11a3fddf4a4 100644 --- a/src/transformers/models/poolformer/image_processing_poolformer.py +++ b/src/transformers/models/poolformer/image_processing_poolformer.py @@ -32,7 +32,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -291,7 +291,7 @@ def preprocess( crop_size = crop_size if crop_size is not None else self.crop_size crop_size = get_size_dict(crop_size, param_name="crop_size") - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/prompt_depth_anything/image_processing_prompt_depth_anything.py b/src/transformers/models/prompt_depth_anything/image_processing_prompt_depth_anything.py index e283411b4e98..b5323c308ca4 100644 --- a/src/transformers/models/prompt_depth_anything/image_processing_prompt_depth_anything.py +++ b/src/transformers/models/prompt_depth_anything/image_processing_prompt_depth_anything.py @@ -34,7 +34,7 @@ infer_channel_dimension_format, is_scaled_image, is_torch_available, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -368,7 +368,7 @@ def preprocess( do_pad = do_pad if do_pad is not None else self.do_pad size_divisor = size_divisor if size_divisor is not None else self.size_divisor - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( @@ -432,7 +432,7 @@ def preprocess( if prompt_depth is not None: # prompt_depth is a list of images with shape (height, width) # we need to convert it to a list of images with shape (1, height, width) - prompt_depths = make_list_of_images(prompt_depth, expected_ndims=2) + prompt_depths = make_flat_list_of_images(prompt_depth, expected_ndims=2) # Validate prompt_depths has same length as images if len(prompt_depths) != len(images): diff --git a/src/transformers/models/pvt/image_processing_pvt.py b/src/transformers/models/pvt/image_processing_pvt.py index e9ebecd503a4..d0c594e47e36 100644 --- a/src/transformers/models/pvt/image_processing_pvt.py +++ b/src/transformers/models/pvt/image_processing_pvt.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -216,7 +216,7 @@ def preprocess( size = size if size is not None else self.size size_dict = get_size_dict(size) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py b/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py index dbde67be1e69..cfc76f495f83 100644 --- a/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py @@ -40,7 +40,6 @@ infer_channel_dimension_format, is_scaled_image, make_flat_list_of_images, - make_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -224,7 +223,7 @@ def _preprocess( - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. """ - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if do_convert_rgb: images = [convert_to_rgb(image) for image in images] diff --git a/src/transformers/models/rt_detr/image_processing_rt_detr.py b/src/transformers/models/rt_detr/image_processing_rt_detr.py index 42d8efef38a9..de61a8019047 100644 --- a/src/transformers/models/rt_detr/image_processing_rt_detr.py +++ b/src/transformers/models/rt_detr/image_processing_rt_detr.py @@ -42,7 +42,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_annotations, @@ -893,7 +893,7 @@ def preprocess( pad_size = self.pad_size if pad_size is None else pad_size format = self.format if format is None else format - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( @@ -926,7 +926,7 @@ def preprocess( if annotations is not None: validate_annotations(format, SUPPORTED_ANNOTATION_FORMATS, annotations) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " diff --git a/src/transformers/models/sam/image_processing_sam.py b/src/transformers/models/sam/image_processing_sam.py index 7591da5af94d..ea149b256f7b 100644 --- a/src/transformers/models/sam/image_processing_sam.py +++ b/src/transformers/models/sam/image_processing_sam.py @@ -32,7 +32,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -494,7 +494,7 @@ def preprocess( mask_pad_size = get_size_dict(mask_pad_size, default_to_square=True) do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( @@ -503,7 +503,7 @@ def preprocess( ) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if not valid_images(segmentation_maps): raise ValueError( diff --git a/src/transformers/models/segformer/image_processing_segformer.py b/src/transformers/models/segformer/image_processing_segformer.py index 9cb05f811857..a651795ca0cf 100644 --- a/src/transformers/models/segformer/image_processing_segformer.py +++ b/src/transformers/models/segformer/image_processing_segformer.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -368,10 +368,10 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/seggpt/image_processing_seggpt.py b/src/transformers/models/seggpt/image_processing_seggpt.py index b05206713ca9..c60cad80d7fd 100644 --- a/src/transformers/models/seggpt/image_processing_seggpt.py +++ b/src/transformers/models/seggpt/image_processing_seggpt.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -323,7 +323,7 @@ def _preprocess_step( size_dict = get_size_dict(size) # If segmentation map is passed we expect 2D images - images = make_list_of_images(images, expected_ndims=2 if do_convert_rgb else 3) + images = make_flat_list_of_images(images, expected_ndims=2 if do_convert_rgb else 3) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/shieldgemma2/processing_shieldgemma2.py b/src/transformers/models/shieldgemma2/processing_shieldgemma2.py index 69d8e6a89e58..412f231bef91 100644 --- a/src/transformers/models/shieldgemma2/processing_shieldgemma2.py +++ b/src/transformers/models/shieldgemma2/processing_shieldgemma2.py @@ -154,19 +154,36 @@ def __call__( messages = [] expanded_images = [] for img in images: + if not isinstance(img, list): + img = [img] + elif len(img) > 1: + raise ValueError(f"SheildGemma can process at most one image per sample, but got {len(img)} images") + for policy in policies: - messages.append( - [ - { - "role": "user", - "content": [ - {"type": "image"}, - {"type": "text", "text": policy_definitions[policy]}, - ], - } - ] - ) - expanded_images.append([img]) + if img: + messages.append( + [ + { + "role": "user", + "content": [ + {"type": "image"}, + {"type": "text", "text": policy_definitions[policy]}, + ], + } + ] + ) + else: + messages.append( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": policy_definitions[policy]}, + ], + } + ] + ) + expanded_images.append(img) text = self.apply_chat_template(messages, tokenize=False) return super().__call__(images=expanded_images, text=text, **kwargs) diff --git a/src/transformers/models/smolvlm/image_processing_smolvlm.py b/src/transformers/models/smolvlm/image_processing_smolvlm.py index d92d9a3385eb..12ae2cfb8e82 100644 --- a/src/transformers/models/smolvlm/image_processing_smolvlm.py +++ b/src/transformers/models/smolvlm/image_processing_smolvlm.py @@ -523,7 +523,7 @@ def _pad_image( def pad( self, - images: list[np.ndarray], + images: list[list[np.ndarray]], constant_values: Union[float, Iterable[float]] = 0, return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, @@ -534,7 +534,7 @@ def pad( For a list of images, for each images, pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width. For each sample in the batch, pads the sample with empty images to the max_number of images per sample in the batch. Optionally returns a pixel mask. Args: - images (`list[np.ndarray]`): + images (`list[list[np.ndarray]]`): List of list of images to pad. Pads to the largest height and width in the batch. constant_values (`float` or `Iterable[float]`, *optional*): The value to use for the padding if `mode` is `"constant"`. @@ -562,11 +562,13 @@ def pad( else input_data_format ) data_format = input_data_format if data_format is None else data_format + # filter out empty image lists, then take first image of the first sample + first_image_in_list = [sample_images for sample_images in images if sample_images][0][0] if input_data_format == ChannelDimension.FIRST: - n_channels = images[0][0].shape[0] + n_channels = first_image_in_list.shape[0] elif input_data_format == ChannelDimension.LAST: - n_channels = images[0][0].shape[-1] + n_channels = first_image_in_list.shape[-1] else: raise ValueError("Invalid channel dimension format.") @@ -712,6 +714,9 @@ def preprocess( # All transformations expect numpy arrays. images_list = [[to_numpy_array(image) for image in images] for images in images_list] + # Search for the first image in the image list. + # NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682 + first_image_in_list = [images for images in images_list if images][0][0] # Extra channel dimension for grayscale images if input_data_format in [ChannelDimension.LAST, None]: @@ -723,7 +728,7 @@ def preprocess( [np.expand_dims(img, axis=0) if img.ndim == 2 else img for img in images] for images in images_list ] - if do_rescale and is_scaled_image(images_list[0][0]): + if do_rescale and is_scaled_image(first_image_in_list): logger.warning_once( "It looks like you are trying to rescale already rescaled images. If the input" " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." @@ -731,7 +736,7 @@ def preprocess( # We assume that all images have the same channel dimension format. if input_data_format is None: - input_data_format = infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4)) + input_data_format = infer_channel_dimension_format(first_image_in_list, num_channels=(1, 3, 4)) if do_resize: images_list = [ diff --git a/src/transformers/models/superpoint/image_processing_superpoint.py b/src/transformers/models/superpoint/image_processing_superpoint.py index e6167160c29d..7894e34e227d 100644 --- a/src/transformers/models/superpoint/image_processing_superpoint.py +++ b/src/transformers/models/superpoint/image_processing_superpoint.py @@ -26,7 +26,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -245,7 +245,7 @@ def preprocess( size = size if size is not None else self.size size = get_size_dict(size, default_to_square=False) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/swin2sr/image_processing_swin2sr.py b/src/transformers/models/swin2sr/image_processing_swin2sr.py index 8257ca9d8eaa..d36ce936c2f1 100644 --- a/src/transformers/models/swin2sr/image_processing_swin2sr.py +++ b/src/transformers/models/swin2sr/image_processing_swin2sr.py @@ -25,7 +25,7 @@ ImageInput, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -159,7 +159,7 @@ def preprocess( do_pad = do_pad if do_pad is not None else self.do_pad pad_size = pad_size if pad_size is not None else self.pad_size - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/textnet/image_processing_textnet.py b/src/transformers/models/textnet/image_processing_textnet.py index f274c6a6a746..cff4b991d8f3 100644 --- a/src/transformers/models/textnet/image_processing_textnet.py +++ b/src/transformers/models/textnet/image_processing_textnet.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_kwargs, @@ -290,7 +290,7 @@ def preprocess( validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self._valid_processor_keys) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/timm_wrapper/image_processing_timm_wrapper.py b/src/transformers/models/timm_wrapper/image_processing_timm_wrapper.py index 94ca3efedb17..89c8883cbd6a 100644 --- a/src/transformers/models/timm_wrapper/image_processing_timm_wrapper.py +++ b/src/transformers/models/timm_wrapper/image_processing_timm_wrapper.py @@ -20,7 +20,7 @@ from ...image_processing_utils import BaseImageProcessor, BatchFeature from ...image_transforms import to_pil_image -from ...image_utils import ImageInput, make_list_of_images +from ...image_utils import ImageInput, make_flat_list_of_images from ...utils import TensorType, logging, requires_backends from ...utils.import_utils import is_timm_available, is_torch_available, requires @@ -121,7 +121,7 @@ def preprocess( # Add batch dimension if a single image images = images.unsqueeze(0) if images.ndim == 3 else images else: - images = make_list_of_images(images) + images = make_flat_list_of_images(images) images = [to_pil_image(image) for image in images] images = torch.stack([self.val_transforms(image) for image in images]) diff --git a/src/transformers/models/tvp/image_processing_tvp_fast.py b/src/transformers/models/tvp/image_processing_tvp_fast.py index 3eec7e2d9c3d..90c39ea49fb0 100644 --- a/src/transformers/models/tvp/image_processing_tvp_fast.py +++ b/src/transformers/models/tvp/image_processing_tvp_fast.py @@ -101,9 +101,6 @@ def preprocess( videos: Union[ImageInput, list[ImageInput], list[list[ImageInput]]], **kwargs: Unpack[TvpFastImageProcessorKwargs], ) -> BatchFeature: - """ - Preprocess videos using the fast image processor. - """ return super().preprocess(videos, **kwargs) def _further_process_kwargs( diff --git a/src/transformers/models/video_llava/image_processing_video_llava.py b/src/transformers/models/video_llava/image_processing_video_llava.py index 47b51869f600..eb9fbfa17888 100644 --- a/src/transformers/models/video_llava/image_processing_video_llava.py +++ b/src/transformers/models/video_llava/image_processing_video_llava.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -258,7 +258,7 @@ def preprocess( if images is not None: images = self.fetch_images(images) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if images is not None and not valid_images(images): raise ValueError( diff --git a/src/transformers/models/vilt/image_processing_vilt.py b/src/transformers/models/vilt/image_processing_vilt.py index 6c9a727f0f9a..a624d957013f 100644 --- a/src/transformers/models/vilt/image_processing_vilt.py +++ b/src/transformers/models/vilt/image_processing_vilt.py @@ -30,7 +30,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -417,7 +417,7 @@ def preprocess( size = size if size is not None else self.size size = get_size_dict(size, default_to_square=False) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/vit/image_processing_vit.py b/src/transformers/models/vit/image_processing_vit.py index 8da5f302d475..27de1076cf54 100644 --- a/src/transformers/models/vit/image_processing_vit.py +++ b/src/transformers/models/vit/image_processing_vit.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -225,7 +225,7 @@ def preprocess( size = size if size is not None else self.size size_dict = get_size_dict(size) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/vitmatte/image_processing_vitmatte.py b/src/transformers/models/vitmatte/image_processing_vitmatte.py index 15d3ec604601..891fdb457359 100644 --- a/src/transformers/models/vitmatte/image_processing_vitmatte.py +++ b/src/transformers/models/vitmatte/image_processing_vitmatte.py @@ -28,7 +28,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -195,8 +195,8 @@ def preprocess( image_std = image_std if image_std is not None else self.image_std size_divisibility = size_divisibility if size_divisibility is not None else self.size_divisibility - images = make_list_of_images(images) - trimaps = make_list_of_images(trimaps, expected_ndims=2) + images = make_flat_list_of_images(images) + trimaps = make_flat_list_of_images(trimaps, expected_ndims=2) if not valid_images(trimaps): raise ValueError( diff --git a/src/transformers/models/vitpose/image_processing_vitpose.py b/src/transformers/models/vitpose/image_processing_vitpose.py index 32c048a8a9d0..6466ccc1d22f 100644 --- a/src/transformers/models/vitpose/image_processing_vitpose.py +++ b/src/transformers/models/vitpose/image_processing_vitpose.py @@ -29,7 +29,7 @@ ImageInput, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -484,7 +484,7 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/yolos/image_processing_yolos.py b/src/transformers/models/yolos/image_processing_yolos.py index 4a716036d400..48a1300191af 100644 --- a/src/transformers/models/yolos/image_processing_yolos.py +++ b/src/transformers/models/yolos/image_processing_yolos.py @@ -44,7 +44,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_annotations, @@ -1301,7 +1301,7 @@ def preprocess( format = self.format if format is None else format validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self._valid_processor_keys) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/zoedepth/image_processing_zoedepth.py b/src/transformers/models/zoedepth/image_processing_zoedepth.py index 2fb125e2b793..90e69105be01 100644 --- a/src/transformers/models/zoedepth/image_processing_zoedepth.py +++ b/src/transformers/models/zoedepth/image_processing_zoedepth.py @@ -35,7 +35,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -385,7 +385,7 @@ def preprocess( image_std = image_std if image_std is not None else self.image_std do_pad = do_pad if do_pad is not None else self.do_pad - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/processing_utils.py b/src/transformers/processing_utils.py index a4fe2f61b1fd..abb8d4476af7 100644 --- a/src/transformers/processing_utils.py +++ b/src/transformers/processing_utils.py @@ -1603,10 +1603,8 @@ def apply_chat_template( # Currently all processors can accept nested list of batches, but not flat list of visuals # So we'll make a batched list of images and let the processor handle it - if image_fnames: - batch_images.append(image_fnames) - if video_fnames: - batch_videos.append(video_fnames) + batch_images.append(image_fnames) + batch_videos.append(video_fnames) prompt, generation_indices = render_jinja_template( conversations=conversations, @@ -1636,10 +1634,12 @@ def apply_chat_template( ): kwargs["do_sample_frames"] = True + images_exist = any(len(im) > 0 for im_list in batch_images for im in im_list) + videos_exist = any(len(vid) > 0 for vid_list in batch_videos for vid in vid_list) out = self( text=prompt, - images=batch_images if batch_images else None, - videos=batch_videos if batch_videos else None, + images=batch_images if images_exist else None, + videos=batch_videos if videos_exist else None, audio=batch_audios if batch_audios else None, **kwargs, ) diff --git a/src/transformers/video_utils.py b/src/transformers/video_utils.py index 7576b05857df..e03cda179d3c 100644 --- a/src/transformers/video_utils.py +++ b/src/transformers/video_utils.py @@ -115,7 +115,7 @@ def is_valid_video_frame(frame): def is_valid_video(video): if not isinstance(video, (list, tuple)): return (is_numpy_array(video) or is_torch_tensor(video)) and video.ndim == 4 - return all(is_valid_video_frame(frame) for frame in video) + return video and all(is_valid_video_frame(frame) for frame in video) def valid_videos(videos): @@ -181,7 +181,7 @@ def make_batched_videos(videos) -> list[Union["np.ndarray", "torch.Tensor", "URL """ # Early exit for deeply nested list of image frame paths. We shouldn't flatten them try: - if isinstance(videos[0][0][0], str): + if isinstance(videos[0][0], list) and isinstance(videos[0][0][0], str): return [image_paths for sublist in videos for image_paths in sublist] except (IndexError, TypeError): pass @@ -202,7 +202,7 @@ def make_batched_videos(videos) -> list[Union["np.ndarray", "torch.Tensor", "URL for item in videos: if isinstance(item, str) or is_valid_video(item): flat_videos_list.append(item) - elif isinstance(item, list): + elif isinstance(item, list) and item: flat_videos_list.extend(make_batched_videos(item)) flat_videos_list = convert_pil_frames_to_video(flat_videos_list) diff --git a/tests/models/colpali/test_processing_colpali.py b/tests/models/colpali/test_processing_colpali.py index 35a74a11a00b..221836db8423 100644 --- a/tests/models/colpali/test_processing_colpali.py +++ b/tests/models/colpali/test_processing_colpali.py @@ -282,3 +282,7 @@ def test_model_input_names(self): inputs = processor(images=image_input) self.assertSetEqual(set(inputs.keys()), set(processor.model_input_names)) + + @unittest.skip("ColPali can't process text+image inputs at the same time") + def test_processor_text_has_no_visual(self): + pass diff --git a/tests/models/colqwen2/test_processing_colqwen2.py b/tests/models/colqwen2/test_processing_colqwen2.py index 560f87a56d91..7346c0d5079c 100644 --- a/tests/models/colqwen2/test_processing_colqwen2.py +++ b/tests/models/colqwen2/test_processing_colqwen2.py @@ -281,3 +281,7 @@ def test_model_input_names(self): inputs = processor(images=image_input) self.assertSetEqual(set(inputs.keys()), set(processor.model_input_names)) + + @unittest.skip("ColPali can't process text+image inputs at the same time") + def test_processor_text_has_no_visual(self): + pass diff --git a/tests/models/fuyu/test_processing_fuyu.py b/tests/models/fuyu/test_processing_fuyu.py index 6f38e9a0e81b..8ed9b20d168f 100644 --- a/tests/models/fuyu/test_processing_fuyu.py +++ b/tests/models/fuyu/test_processing_fuyu.py @@ -347,6 +347,28 @@ def test_unstructured_kwargs_batched(self): self.assertEqual(len(inputs["input_ids"][0]), 7) + def test_processor_text_has_no_visual(self): + # Overwritten: Fuyu has a complicated processing so we don't check id values + processor = self.get_processor() + + text = self.prepare_text_inputs(batch_size=3, modalities="image") + image_inputs = self.prepare_image_inputs(batch_size=3) + processing_kwargs = {"return_tensors": "pt", "padding": True, "multi_page": True} + + # Call with nested list of vision inputs + image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] + inputs_dict_nested = {"text": text, "images": image_inputs_nested} + inputs = processor(**inputs_dict_nested, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs) + + # Call with one of the samples with no associated vision input + plain_text = "lower newer" + image_inputs_nested[0] = [] + text[0] = plain_text + inputs_dict_no_vision = {"text": text, "images": image_inputs_nested} + inputs_nested = processor(**inputs_dict_no_vision, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs_nested) + @require_torch class TestImageTextProcessingUtils(unittest.TestCase): diff --git a/tests/models/got_ocr2/test_processing_got_ocr2.py b/tests/models/got_ocr2/test_processing_got_ocr2.py index 0719d211ddad..ffa0f97cd4e4 100644 --- a/tests/models/got_ocr2/test_processing_got_ocr2.py +++ b/tests/models/got_ocr2/test_processing_got_ocr2.py @@ -78,3 +78,27 @@ def test_ocr_queries(self): inputs = processor(image_input, return_tensors="pt", crop_to_patches=True, max_patches=6) self.assertEqual(inputs["input_ids"].shape, (1, 1826)) self.assertEqual(inputs["pixel_values"].shape, (7, 3, 384, 384)) + + def test_processor_text_has_no_visual(self): + # Overwritten: requires `multi_page` kwarg to process nested vision inputs + processor = self.get_processor() + + text = self.prepare_text_inputs(batch_size=3, modalities="image") + image_inputs = self.prepare_image_inputs(batch_size=3) + processing_kwargs = {"return_tensors": "pt", "padding": True, "multi_page": True} + + # Call with nested list of vision inputs + image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] + inputs_dict_nested = {"text": text, "images": image_inputs_nested} + inputs = processor(**inputs_dict_nested, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs) + + # Call with one of the samples with no associated vision input + plain_text = "lower newer" + image_inputs_nested[0] = [] + text[0] = plain_text + inputs_dict_no_vision = {"text": text, "images": image_inputs_nested} + inputs_nested = processor(**inputs_dict_no_vision, **processing_kwargs) + self.assertListEqual( + inputs[self.text_input_name][1:].tolist(), inputs_nested[self.text_input_name][1:].tolist() + ) diff --git a/tests/models/grounding_dino/test_processing_grounding_dino.py b/tests/models/grounding_dino/test_processing_grounding_dino.py index a2d1e5427b52..1c007dc2d0b4 100644 --- a/tests/models/grounding_dino/test_processing_grounding_dino.py +++ b/tests/models/grounding_dino/test_processing_grounding_dino.py @@ -280,3 +280,30 @@ def test_text_preprocessing_equivalence(self): torch.allclose(inputs1["input_ids"], inputs2["input_ids"]), f"Input ids are not equal for batched input: {inputs1['input_ids']} != {inputs2['input_ids']}", ) + + def test_processor_text_has_no_visual(self): + # Overwritten: text inputs have to be nested as well + processor = self.get_processor() + + text = self.prepare_text_inputs(batch_size=3, modalities="image") + image_inputs = self.prepare_image_inputs(batch_size=3) + processing_kwargs = {"return_tensors": "pt", "padding": True} + + # Call with nested list of vision inputs + image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] + inputs_dict_nested = {"text": text, "images": image_inputs_nested} + inputs = processor(**inputs_dict_nested, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs) + + # Call with one of the samples with no associated vision input + plain_text = ["lower newer"] + image_inputs_nested[0] = [] + text[0] = plain_text + inputs_dict_no_vision = {"text": text, "images": image_inputs_nested} + inputs_nested = processor(**inputs_dict_no_vision, **processing_kwargs) + + # Check that text samples are same and are expanded with placeholder tokens correctly. First sample + # has no vision input associated, so we skip it and check it has no vision + self.assertListEqual( + inputs[self.text_input_name][1:].tolist(), inputs_nested[self.text_input_name][1:].tolist() + ) diff --git a/tests/models/idefics2/test_processing_idefics2.py b/tests/models/idefics2/test_processing_idefics2.py index a39d14d4f17e..84a1f8229201 100644 --- a/tests/models/idefics2/test_processing_idefics2.py +++ b/tests/models/idefics2/test_processing_idefics2.py @@ -285,17 +285,11 @@ def test_process_interleaved_images_prompts_image_error(self): "In this other sentence we try some good things", ] images = [[self.image1], []] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - images = [[], [self.image2]] with self.assertRaises(ValueError): processor(text=text, images=images, padding=True) images = [self.image1, self.image2] with self.assertRaises(ValueError): processor(text=text, images=images, padding=True) - images = [self.image1] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) def test_apply_chat_template(self): # Message contains content which a mix of lists with images and image urls and string diff --git a/tests/models/idefics3/test_processing_idefics3.py b/tests/models/idefics3/test_processing_idefics3.py index 780a6899d540..1940ab1a9171 100644 --- a/tests/models/idefics3/test_processing_idefics3.py +++ b/tests/models/idefics3/test_processing_idefics3.py @@ -321,17 +321,11 @@ def test_process_interleaved_images_prompts_image_error(self): "In this other sentence we try some good things", ] images = [[self.image1], []] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - images = [[], [self.image2]] with self.assertRaises(ValueError): processor(text=text, images=images, padding=True) images = [self.image1, self.image2] with self.assertRaises(ValueError): processor(text=text, images=images, padding=True) - images = [self.image1] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) def test_apply_chat_template(self): # Message contains content which a mix of lists with images and image urls and string diff --git a/tests/models/mllama/test_processing_mllama.py b/tests/models/mllama/test_processing_mllama.py index e5e2ce680cbd..b75cf8e5cff7 100644 --- a/tests/models/mllama/test_processing_mllama.py +++ b/tests/models/mllama/test_processing_mllama.py @@ -16,6 +16,7 @@ import shutil import tempfile import unittest +from typing import Optional import numpy as np @@ -56,6 +57,14 @@ def tearDownClass(cls): def prepare_processor_dict(self): return {"chat_template": "{% for message in messages %}{% if loop.index0 == 0 %}{{ bos_token }}{% endif %}{{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }}{% if message['content'] is string %}{{ message['content'] }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' %}{{ '<|image|>' }}{% elif content['type'] == 'text' %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}{{ '<|eot_id|>' }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}"} # fmt: skip + # Override as Mllama needs images to be an explicitly nested batch + def prepare_image_inputs(self, batch_size: Optional[int] = None): + """This function prepares a list of PIL images for testing""" + images = super().prepare_image_inputs(batch_size) + if isinstance(images, (list, tuple)): + images = [[image] for image in images] + return images + def test_chat_template_is_saved(self): processor_loaded = self.processor_class.from_pretrained(self.tmpdirname) processor_dict_loaded = json.loads(processor_loaded.to_json_string()) @@ -344,7 +353,6 @@ def test_unstructured_kwargs_batched(self): input_str = self.prepare_text_inputs(batch_size=2, modalities="image") image_input = self.prepare_image_inputs(batch_size=2) - image_input = [[image_input[0]], [image_input[1]]] inputs = processor( text=input_str, images=image_input, @@ -368,7 +376,6 @@ def test_special_mm_token_truncation(self): input_str = self.prepare_text_inputs(batch_size=2, modalities="image") image_input = self.prepare_image_inputs(batch_size=2) - image_input = [[image_input[0]], [image_input[1]]] _ = processor( text=input_str, images=image_input, @@ -386,3 +393,7 @@ def test_special_mm_token_truncation(self): padding=True, max_length=3, ) + + @unittest.skip("Mllama can't process inouts with no image ttogether with multimodal inputs") + def test_processor_text_has_no_visual(self): + pass diff --git a/tests/models/shieldgemma2/test_processing_shieldgemma2.py b/tests/models/shieldgemma2/test_processing_shieldgemma2.py index 0bbe17e65607..b9f8b58ad3f6 100644 --- a/tests/models/shieldgemma2/test_processing_shieldgemma2.py +++ b/tests/models/shieldgemma2/test_processing_shieldgemma2.py @@ -210,3 +210,25 @@ def test_kwargs_overrides_default_image_processor_kwargs(self): @unittest.skip("ShieldGemma requires images in input, and fails in text-only processing") def test_apply_chat_template_assistant_mask(self): pass + + def test_processor_text_has_no_visual(self): + # Overwritten: Shieldgemma has a complicated processing so we don't check id values + processor = self.get_processor() + + text = self.prepare_text_inputs(batch_size=3, modalities="image") + image_inputs = self.prepare_image_inputs(batch_size=3) + processing_kwargs = {"return_tensors": "pt", "padding": True, "multi_page": True} + + # Call with nested list of vision inputs + image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] + inputs_dict_nested = {"text": text, "images": image_inputs_nested} + inputs = processor(**inputs_dict_nested, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs) + + # Call with one of the samples with no associated vision input + plain_text = "lower newer" + image_inputs_nested[0] = [] + text[0] = plain_text + inputs_dict_no_vision = {"text": text, "images": image_inputs_nested} + inputs_nested = processor(**inputs_dict_no_vision, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs_nested) diff --git a/tests/models/smolvlm/test_processing_smolvlm.py b/tests/models/smolvlm/test_processing_smolvlm.py index 4c6af0dd1e9e..1634c53f132a 100644 --- a/tests/models/smolvlm/test_processing_smolvlm.py +++ b/tests/models/smolvlm/test_processing_smolvlm.py @@ -94,6 +94,14 @@ def prepare_processor_dict(): "chat_template": "<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '' }}{% endif %}{% endfor %}\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}", } + # Override as SmolVLM needs images/video to be an explicitly nested batch + def prepare_image_inputs(self, batch_size: Optional[int] = None): + """This function prepares a list of PIL images for testing""" + images = super().prepare_image_inputs(batch_size) + if isinstance(images, (list, tuple)): + images = [[image] for image in images] + return images + def prepare_video_inputs(self, batch_size: Optional[int] = None): """This function prepares a list of numpy videos.""" video_input = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)] * 8 @@ -334,17 +342,11 @@ def test_process_interleaved_images_prompts_image_error(self): "In this other sentence we try some good things", ] images = [[self.image1], []] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - images = [[], [self.image2]] with self.assertRaises(ValueError): processor(text=text, images=images, padding=True) images = [self.image1, self.image2] with self.assertRaises(ValueError): processor(text=text, images=images, padding=True) - images = [self.image1] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) def test_apply_chat_template(self): # Message contains content which a mix of lists with images and image urls and string @@ -452,7 +454,6 @@ def test_unstructured_kwargs_batched(self): input_str = self.prepare_text_inputs(batch_size=2, modalities="image") image_input = self.prepare_image_inputs(batch_size=2) - image_input = [[image_input[0]], [image_input[1]]] inputs = processor( text=input_str, images=image_input, @@ -574,7 +575,6 @@ def test_special_mm_token_truncation(self): input_str = self.prepare_text_inputs(batch_size=2, modalities="image") image_input = self.prepare_image_inputs(batch_size=2) - image_input = [[image_input[0]], [image_input[1]]] _ = processor( text=input_str, images=image_input, diff --git a/tests/models/udop/test_processing_udop.py b/tests/models/udop/test_processing_udop.py index de1fa89a7ebf..78cb2c36b9f3 100644 --- a/tests/models/udop/test_processing_udop.py +++ b/tests/models/udop/test_processing_udop.py @@ -196,6 +196,10 @@ def preprocess_data(examples): self.assertEqual(len(train_data["pixel_values"]), len(train_data["input_ids"])) + @unittest.skip("We will not support batch input with and without images for UDOP!") + def test_processor_text_has_no_visual(self): + pass + # different use cases tests @require_sentencepiece diff --git a/tests/test_processing_common.py b/tests/test_processing_common.py index 7845b926e06c..e1b36cd2445f 100644 --- a/tests/test_processing_common.py +++ b/tests/test_processing_common.py @@ -170,6 +170,7 @@ def prepare_image_inputs(self, batch_size: Optional[int] = None): def prepare_video_inputs(self, batch_size: Optional[int] = None): """This function prepares a list of numpy videos.""" video_input = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)] * 8 + video_input = np.array(video_input) if batch_size is None: return video_input return [video_input] * batch_size @@ -264,6 +265,90 @@ def test_model_input_names(self): self.assertSetEqual(set(inputs.keys()), set(processor.model_input_names)) + def test_processor_text_has_no_visual(self): + """ + Tests that multimodal models can process batch of inputs where samples can + be with images/videos or without. See https://github.com/huggingface/transformers/issues/40263 + """ + processor = self.get_processor() + call_signature = inspect.signature(processor.__call__) + input_args = [param.name for param in call_signature.parameters.values() if param.annotation != param.empty] + + if not ("text" in input_args and ("images" in input_args and "videos" in input_args)): + self.skipTest(f"{self.processor_class} doesn't support several vision modalities with text.") + + # Prepare inputs and filter by input signature. Make sure to use a high batch size, we'll set some + # samples to text-only later + text = self.prepare_text_inputs(batch_size=3, modalities=["image", "video"]) + image_inputs = self.prepare_image_inputs(batch_size=3) + video_inputs = self.prepare_video_inputs(batch_size=3) + inputs_dict = {"text": text, "images": image_inputs, "videos": video_inputs} + inputs_dict = {k: v for k, v in inputs_dict.items() if k in input_args} + + processing_kwargs = {"return_tensors": "pt", "padding": True} + if "videos" in inputs_dict: + processing_kwargs["do_sample_frames"] = False + + # Firts call processor with all inputs and use nested input type, which is the format supported by all multimodal processors + image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] + video_inputs_nested = [[video] for video in video_inputs] + inputs_dict_nested = {"text": text, "images": image_inputs_nested, "videos": video_inputs_nested} + inputs_dict_nested = {k: v for k, v in inputs_dict_nested.items() if k in input_args} + inputs = processor(**inputs_dict_nested, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs) + + # Now call with one of the samples with no associated vision input. Let's set the first input to be a plain text + # with no placeholder tokens and no images/videos. The final format would be `images = [[], [image2], [image3]]` + plain_text = "lower newer" + image_inputs_nested[0] = [] + video_inputs_nested[0] = [] + text[0] = plain_text + inputs_dict_no_vision = {"text": text, "images": image_inputs_nested, "videos": video_inputs_nested} + inputs_dict_no_vision = {k: v for k, v in inputs_dict_no_vision.items() if k in input_args} + inputs_nested = processor(**inputs_dict_no_vision, **processing_kwargs) + + # Check that text samples are same and are expanded with placeholder tokens correctly. First sample + # has no vision input associated, so we skip it and check it has no vision + self.assertListEqual( + inputs[self.text_input_name][1:].tolist(), inputs_nested[self.text_input_name][1:].tolist() + ) + + # Now test if we can apply chat templates with no vision inputs in one of the samples + # NOTE: we don't skip the test as we want the above to be checked even if process has to chat template + if processor.chat_template is not None: + messages = [ + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is the capital of France?"}, + ], + }, + ], + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is the capital of France?"}, + { + "type": "image", + "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/coco_sample.png", + }, + ], + }, + ], + ] + + inputs_chat_template = processor.apply_chat_template( + messages, + add_generation_prompt=False, + tokenize=True, + return_dict=True, + return_tensors="pt", + padding=True, + ) + self.assertTrue(self.text_input_name in inputs_chat_template) + # These kwargs-related tests ensure that processors are correctly instantiated. # they need to be applied only if an image_processor exists.