From f26b2754875fa82e56c475cbb1d8de4f46b9d3b4 Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Thu, 13 Mar 2025 01:45:20 +0800 Subject: [PATCH 01/18] update make nested image list --- src/transformers/image_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/image_utils.py b/src/transformers/image_utils.py index bde61e3803b5..4188b57db266 100644 --- a/src/transformers/image_utils.py +++ b/src/transformers/image_utils.py @@ -277,7 +277,7 @@ def make_nested_list_of_images( if ( isinstance(images, (list, tuple)) and all(isinstance(images_i, (list, tuple)) for images_i in images) - and all(is_valid_list_of_images(images_i) for images_i in images) + and any(is_valid_list_of_images(images_i) for images_i in images) ): return images From 26d2aeb49eadcff9f5314e6d59d74e0a189e3cda Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Thu, 13 Mar 2025 01:56:26 +0800 Subject: [PATCH 02/18] fix make flat list of images --- src/transformers/image_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/image_utils.py b/src/transformers/image_utils.py index 4188b57db266..05ead485f280 100644 --- a/src/transformers/image_utils.py +++ b/src/transformers/image_utils.py @@ -243,7 +243,7 @@ def make_flat_list_of_images( if ( isinstance(images, (list, tuple)) and all(isinstance(images_i, (list, tuple)) for images_i in images) - and all(is_valid_list_of_images(images_i) for images_i in images) + and any(is_valid_list_of_images(images_i) for images_i in images) ): return [img for img_list in images for img in img_list] From 98a64cb94d7f4dfea10be985ab6fe7e03a9cb867 Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Thu, 13 Mar 2025 18:24:06 +0800 Subject: [PATCH 03/18] update type anno --- src/transformers/image_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/image_utils.py b/src/transformers/image_utils.py index 05ead485f280..a3899cdc30e1 100644 --- a/src/transformers/image_utils.py +++ b/src/transformers/image_utils.py @@ -229,7 +229,7 @@ def make_list_of_images(images, expected_ndims: int = 3) -> List[ImageInput]: def make_flat_list_of_images( images: Union[List[ImageInput], ImageInput], -) -> ImageInput: +) -> List[ImageInput]: """ Ensure that the output is a flat list of images. If the input is a single image, it is converted to a list of length 1. If the input is a nested list of images, it is converted to a flat list of images. @@ -264,7 +264,7 @@ def make_flat_list_of_images( def make_nested_list_of_images( images: Union[List[ImageInput], ImageInput], -) -> ImageInput: +) -> List[List[ImageInput]]: """ Ensure that the output is a nested list of images. Args: From 5ab405deb33f0a787ad7e8cf0ee94a0822dcb4e3 Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Fri, 14 Mar 2025 23:20:20 +0800 Subject: [PATCH 04/18] fix image_processing_smolvlm --- .../models/gemma3/image_processing_gemma3.py | 5 +++-- .../idefics2/image_processing_idefics2.py | 11 +++++++---- .../idefics3/image_processing_idefics3.py | 17 +++++++++++------ .../models/smolvlm/image_processing_smolvlm.py | 17 +++++++++++------ .../models/idefics2/test_processor_idefics2.py | 6 ------ .../models/idefics3/test_processor_idefics3.py | 6 ------ tests/models/smolvlm/test_processor_smolvlm.py | 6 ------ 7 files changed, 32 insertions(+), 36 deletions(-) diff --git a/src/transformers/models/gemma3/image_processing_gemma3.py b/src/transformers/models/gemma3/image_processing_gemma3.py index 1b41d702bd69..4374c3e4b7b2 100644 --- a/src/transformers/models/gemma3/image_processing_gemma3.py +++ b/src/transformers/models/gemma3/image_processing_gemma3.py @@ -35,6 +35,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, + make_flat_list_of_images, make_nested_list_of_images, to_numpy_array, valid_images, @@ -358,7 +359,7 @@ def preprocess( # All transformations expect numpy arrays. images_list = [[to_numpy_array(image) for image in images] for images in images_list] - if do_rescale and is_scaled_image(images_list[0][0]): + if do_rescale and is_scaled_image(make_flat_list_of_images(images_list)[0]): logger.warning_once( "It looks like you are trying to rescale already rescaled images. If the input" " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." @@ -366,7 +367,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images_list[0][0]) + input_data_format = infer_channel_dimension_format(make_flat_list_of_images(images_list)[0]) if do_pan_and_scan: images_list_and_num_crops = [ diff --git a/src/transformers/models/idefics2/image_processing_idefics2.py b/src/transformers/models/idefics2/image_processing_idefics2.py index 2a853fc02e7a..b9f326ede761 100644 --- a/src/transformers/models/idefics2/image_processing_idefics2.py +++ b/src/transformers/models/idefics2/image_processing_idefics2.py @@ -29,6 +29,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, + make_flat_list_of_images, make_nested_list_of_images, to_numpy_array, valid_images, @@ -92,7 +93,7 @@ def get_max_height_width( Get the maximum height and width across all images in a batch. """ if input_data_format is None: - input_data_format = infer_channel_dimension_format(images_list[0][0]) + input_data_format = infer_channel_dimension_format(make_flat_list_of_images(images_list)[0]) image_sizes = [] for images in images_list: @@ -316,7 +317,9 @@ def pad( batch_size = len(images) max_num_images = max(len(images_) for images_ in images) input_data_format = ( - infer_channel_dimension_format(images[0][0]) if input_data_format is None else input_data_format + infer_channel_dimension_format(make_flat_list_of_images(images)[0]) + if input_data_format is None + else input_data_format ) data_format = input_data_format if data_format is None else data_format @@ -496,7 +499,7 @@ def preprocess( # All transformations expect numpy arrays. images_list = [[to_numpy_array(image) for image in images] for images in images_list] - if do_rescale and is_scaled_image(images_list[0][0]): + if do_rescale and is_scaled_image(make_flat_list_of_images(images_list)[0]): logger.warning_once( "It looks like you are trying to rescale already rescaled images. If the input" " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." @@ -504,7 +507,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images_list[0][0]) + input_data_format = infer_channel_dimension_format(make_flat_list_of_images(images_list)[0]) if do_image_splitting: new_images_list = [] diff --git a/src/transformers/models/idefics3/image_processing_idefics3.py b/src/transformers/models/idefics3/image_processing_idefics3.py index bf21f06e8c5a..c07cb886095d 100644 --- a/src/transformers/models/idefics3/image_processing_idefics3.py +++ b/src/transformers/models/idefics3/image_processing_idefics3.py @@ -29,6 +29,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, + make_flat_list_of_images, make_nested_list_of_images, to_numpy_array, valid_images, @@ -156,7 +157,9 @@ def get_max_height_width( Get the maximum height and width across all images in a batch. """ if input_data_format is None: - input_data_format = infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4)) + input_data_format = infer_channel_dimension_format( + make_flat_list_of_images(images_list)[0], num_channels=(1, 3, 4) + ) max_height = max_width = float("-inf") for images in images_list: @@ -559,16 +562,16 @@ def pad( batch_size = len(images) max_num_images = max(len(images_) for images_ in images) input_data_format = ( - infer_channel_dimension_format(images[0][0], num_channels=(1, 3, 4)) + infer_channel_dimension_format(make_flat_list_of_images(images)[0], num_channels=(1, 3, 4)) if input_data_format is None else input_data_format ) data_format = input_data_format if data_format is None else data_format if input_data_format == ChannelDimension.FIRST: - n_channels = images[0][0].shape[0] + n_channels = make_flat_list_of_images(images)[0].shape[0] elif input_data_format == ChannelDimension.LAST: - n_channels = images[0][0].shape[-1] + n_channels = make_flat_list_of_images(images)[0].shape[-1] else: raise ValueError("Invalid channel dimension format.") @@ -724,7 +727,7 @@ def preprocess( [np.expand_dims(img, axis=0) if img.ndim == 2 else img for img in images] for images in images_list ] - if do_rescale and is_scaled_image(images_list[0][0]): + if do_rescale and is_scaled_image(make_flat_list_of_images(images_list)[0]): logger.warning_once( "It looks like you are trying to rescale already rescaled images. If the input" " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." @@ -732,7 +735,9 @@ def preprocess( # We assume that all images have the same channel dimension format. if input_data_format is None: - input_data_format = infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4)) + input_data_format = infer_channel_dimension_format( + make_flat_list_of_images(images_list)[0], num_channels=(1, 3, 4) + ) if do_resize: images_list = [ diff --git a/src/transformers/models/smolvlm/image_processing_smolvlm.py b/src/transformers/models/smolvlm/image_processing_smolvlm.py index 93ed18d13449..e44a2bd39503 100644 --- a/src/transformers/models/smolvlm/image_processing_smolvlm.py +++ b/src/transformers/models/smolvlm/image_processing_smolvlm.py @@ -35,6 +35,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, + make_flat_list_of_images, make_nested_list_of_images, to_numpy_array, valid_images, @@ -154,7 +155,9 @@ def get_max_height_width( Get the maximum height and width across all images in a batch. """ if input_data_format is None: - input_data_format = infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4)) + input_data_format = infer_channel_dimension_format( + make_flat_list_of_images(images_list)[0], num_channels=(1, 3, 4) + ) max_height = max_width = float("-inf") for images in images_list: @@ -556,16 +559,16 @@ def pad( batch_size = len(images) max_num_images = max(len(images_) for images_ in images) input_data_format = ( - infer_channel_dimension_format(images[0][0], num_channels=(1, 3, 4)) + infer_channel_dimension_format(make_flat_list_of_images(images)[0], num_channels=(1, 3, 4)) if input_data_format is None else input_data_format ) data_format = input_data_format if data_format is None else data_format if input_data_format == ChannelDimension.FIRST: - n_channels = images[0][0].shape[0] + n_channels = make_flat_list_of_images(images)[0].shape[0] elif input_data_format == ChannelDimension.LAST: - n_channels = images[0][0].shape[-1] + n_channels = make_flat_list_of_images(images)[0].shape[-1] else: raise ValueError("Invalid channel dimension format.") @@ -721,7 +724,7 @@ def preprocess( [np.expand_dims(img, axis=0) if img.ndim == 2 else img for img in images] for images in images_list ] - if do_rescale and is_scaled_image(images_list[0][0]): + if do_rescale and is_scaled_image(make_flat_list_of_images(images_list)[0]): logger.warning_once( "It looks like you are trying to rescale already rescaled images. If the input" " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." @@ -729,7 +732,9 @@ def preprocess( # We assume that all images have the same channel dimension format. if input_data_format is None: - input_data_format = infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4)) + input_data_format = infer_channel_dimension_format( + make_flat_list_of_images(images_list)[0], num_channels=(1, 3, 4) + ) if do_resize: images_list = [ diff --git a/tests/models/idefics2/test_processor_idefics2.py b/tests/models/idefics2/test_processor_idefics2.py index d89004679aef..123730b1bee2 100644 --- a/tests/models/idefics2/test_processor_idefics2.py +++ b/tests/models/idefics2/test_processor_idefics2.py @@ -281,17 +281,11 @@ def test_process_interleaved_images_prompts_image_error(self): "In this other sentence we try some good things", ] images = [[self.image1], []] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - images = [[], [self.image2]] with self.assertRaises(ValueError): processor(text=text, images=images, padding=True) images = [self.image1, self.image2] with self.assertRaises(ValueError): processor(text=text, images=images, padding=True) - images = [self.image1] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) def test_apply_chat_template(self): # Message contains content which a mix of lists with images and image urls and string diff --git a/tests/models/idefics3/test_processor_idefics3.py b/tests/models/idefics3/test_processor_idefics3.py index 36c5d2948449..184f62dedab0 100644 --- a/tests/models/idefics3/test_processor_idefics3.py +++ b/tests/models/idefics3/test_processor_idefics3.py @@ -306,17 +306,11 @@ def test_process_interleaved_images_prompts_image_error(self): "In this other sentence we try some good things", ] images = [[self.image1], []] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - images = [[], [self.image2]] with self.assertRaises(ValueError): processor(text=text, images=images, padding=True) images = [self.image1, self.image2] with self.assertRaises(ValueError): processor(text=text, images=images, padding=True) - images = [self.image1] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) def test_apply_chat_template(self): # Message contains content which a mix of lists with images and image urls and string diff --git a/tests/models/smolvlm/test_processor_smolvlm.py b/tests/models/smolvlm/test_processor_smolvlm.py index d276d92a5011..3288485396c6 100644 --- a/tests/models/smolvlm/test_processor_smolvlm.py +++ b/tests/models/smolvlm/test_processor_smolvlm.py @@ -322,17 +322,11 @@ def test_process_interleaved_images_prompts_image_error(self): "In this other sentence we try some good things", ] images = [[self.image1], []] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - images = [[], [self.image2]] with self.assertRaises(ValueError): processor(text=text, images=images, padding=True) images = [self.image1, self.image2] with self.assertRaises(ValueError): processor(text=text, images=images, padding=True) - images = [self.image1] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) def test_apply_chat_template(self): # Message contains content which a mix of lists with images and image urls and string From a6153a64e09c3d5c902c3ce055128ed24583a33c Mon Sep 17 00:00:00 2001 From: hiyouga Date: Mon, 17 Mar 2025 15:33:29 +0000 Subject: [PATCH 05/18] use first image --- .../models/gemma3/image_processing_gemma3.py | 6 ++++-- .../idefics2/image_processing_idefics2.py | 12 ++++++------ .../idefics3/image_processing_idefics3.py | 19 +++++++++---------- .../smolvlm/image_processing_smolvlm.py | 19 +++++++++---------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/transformers/models/gemma3/image_processing_gemma3.py b/src/transformers/models/gemma3/image_processing_gemma3.py index 4374c3e4b7b2..c4b047d2bbc9 100644 --- a/src/transformers/models/gemma3/image_processing_gemma3.py +++ b/src/transformers/models/gemma3/image_processing_gemma3.py @@ -358,8 +358,10 @@ def preprocess( # All transformations expect numpy arrays. images_list = [[to_numpy_array(image) for image in images] for images in images_list] + # Search for the first image in the image list. + first_image_in_list = make_flat_list_of_images(images_list)[0] - if do_rescale and is_scaled_image(make_flat_list_of_images(images_list)[0]): + if do_rescale and is_scaled_image(first_image_in_list): logger.warning_once( "It looks like you are trying to rescale already rescaled images. If the input" " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." @@ -367,7 +369,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(make_flat_list_of_images(images_list)[0]) + input_data_format = infer_channel_dimension_format(first_image_in_list) if do_pan_and_scan: images_list_and_num_crops = [ diff --git a/src/transformers/models/idefics2/image_processing_idefics2.py b/src/transformers/models/idefics2/image_processing_idefics2.py index b9f326ede761..3ac7349ca243 100644 --- a/src/transformers/models/idefics2/image_processing_idefics2.py +++ b/src/transformers/models/idefics2/image_processing_idefics2.py @@ -93,7 +93,7 @@ def get_max_height_width( Get the maximum height and width across all images in a batch. """ if input_data_format is None: - input_data_format = infer_channel_dimension_format(make_flat_list_of_images(images_list)[0]) + input_data_format = infer_channel_dimension_format(images_list[0][0]) image_sizes = [] for images in images_list: @@ -317,9 +317,7 @@ def pad( batch_size = len(images) max_num_images = max(len(images_) for images_ in images) input_data_format = ( - infer_channel_dimension_format(make_flat_list_of_images(images)[0]) - if input_data_format is None - else input_data_format + infer_channel_dimension_format(images[0][0]) if input_data_format is None else input_data_format ) data_format = input_data_format if data_format is None else data_format @@ -498,8 +496,10 @@ def preprocess( # All transformations expect numpy arrays. images_list = [[to_numpy_array(image) for image in images] for images in images_list] + # Search for the first image in the image list. + first_image_in_list = make_flat_list_of_images(images_list)[0] - if do_rescale and is_scaled_image(make_flat_list_of_images(images_list)[0]): + if do_rescale and is_scaled_image(first_image_in_list): logger.warning_once( "It looks like you are trying to rescale already rescaled images. If the input" " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." @@ -507,7 +507,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(make_flat_list_of_images(images_list)[0]) + input_data_format = infer_channel_dimension_format(first_image_in_list) if do_image_splitting: new_images_list = [] diff --git a/src/transformers/models/idefics3/image_processing_idefics3.py b/src/transformers/models/idefics3/image_processing_idefics3.py index c07cb886095d..872fd1da9b4a 100644 --- a/src/transformers/models/idefics3/image_processing_idefics3.py +++ b/src/transformers/models/idefics3/image_processing_idefics3.py @@ -157,9 +157,7 @@ def get_max_height_width( Get the maximum height and width across all images in a batch. """ if input_data_format is None: - input_data_format = infer_channel_dimension_format( - make_flat_list_of_images(images_list)[0], num_channels=(1, 3, 4) - ) + input_data_format = infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4)) max_height = max_width = float("-inf") for images in images_list: @@ -562,16 +560,17 @@ def pad( batch_size = len(images) max_num_images = max(len(images_) for images_ in images) input_data_format = ( - infer_channel_dimension_format(make_flat_list_of_images(images)[0], num_channels=(1, 3, 4)) + infer_channel_dimension_format(images[0][0], num_channels=(1, 3, 4)) if input_data_format is None else input_data_format ) data_format = input_data_format if data_format is None else data_format + first_image_in_list = make_flat_list_of_images(images)[0] if input_data_format == ChannelDimension.FIRST: - n_channels = make_flat_list_of_images(images)[0].shape[0] + n_channels = first_image_in_list.shape[0] elif input_data_format == ChannelDimension.LAST: - n_channels = make_flat_list_of_images(images)[0].shape[-1] + n_channels = first_image_in_list.shape[-1] else: raise ValueError("Invalid channel dimension format.") @@ -716,6 +715,8 @@ def preprocess( # All transformations expect numpy arrays. images_list = [[to_numpy_array(image) for image in images] for images in images_list] + # Search for the first image in the image list. + first_image_in_list = make_flat_list_of_images(images_list)[0] # Extra channel dimension for grayscale images if input_data_format in [ChannelDimension.LAST, None]: @@ -727,7 +728,7 @@ def preprocess( [np.expand_dims(img, axis=0) if img.ndim == 2 else img for img in images] for images in images_list ] - if do_rescale and is_scaled_image(make_flat_list_of_images(images_list)[0]): + if do_rescale and is_scaled_image(first_image_in_list): logger.warning_once( "It looks like you are trying to rescale already rescaled images. If the input" " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." @@ -735,9 +736,7 @@ def preprocess( # We assume that all images have the same channel dimension format. if input_data_format is None: - input_data_format = infer_channel_dimension_format( - make_flat_list_of_images(images_list)[0], num_channels=(1, 3, 4) - ) + input_data_format = infer_channel_dimension_format(first_image_in_list, num_channels=(1, 3, 4)) if do_resize: images_list = [ diff --git a/src/transformers/models/smolvlm/image_processing_smolvlm.py b/src/transformers/models/smolvlm/image_processing_smolvlm.py index e44a2bd39503..cdbe8e20bbec 100644 --- a/src/transformers/models/smolvlm/image_processing_smolvlm.py +++ b/src/transformers/models/smolvlm/image_processing_smolvlm.py @@ -155,9 +155,7 @@ def get_max_height_width( Get the maximum height and width across all images in a batch. """ if input_data_format is None: - input_data_format = infer_channel_dimension_format( - make_flat_list_of_images(images_list)[0], num_channels=(1, 3, 4) - ) + input_data_format = infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4)) max_height = max_width = float("-inf") for images in images_list: @@ -559,16 +557,17 @@ def pad( batch_size = len(images) max_num_images = max(len(images_) for images_ in images) input_data_format = ( - infer_channel_dimension_format(make_flat_list_of_images(images)[0], num_channels=(1, 3, 4)) + infer_channel_dimension_format(images[0][0], num_channels=(1, 3, 4)) if input_data_format is None else input_data_format ) data_format = input_data_format if data_format is None else data_format + first_image_in_list = make_flat_list_of_images(images)[0] if input_data_format == ChannelDimension.FIRST: - n_channels = make_flat_list_of_images(images)[0].shape[0] + n_channels = first_image_in_list.shape[0] elif input_data_format == ChannelDimension.LAST: - n_channels = make_flat_list_of_images(images)[0].shape[-1] + n_channels = first_image_in_list.shape[-1] else: raise ValueError("Invalid channel dimension format.") @@ -713,6 +712,8 @@ def preprocess( # All transformations expect numpy arrays. images_list = [[to_numpy_array(image) for image in images] for images in images_list] + # Search for the first image in the image list. + first_image_in_list = make_flat_list_of_images(images_list)[0] # Extra channel dimension for grayscale images if input_data_format in [ChannelDimension.LAST, None]: @@ -724,7 +725,7 @@ def preprocess( [np.expand_dims(img, axis=0) if img.ndim == 2 else img for img in images] for images in images_list ] - if do_rescale and is_scaled_image(make_flat_list_of_images(images_list)[0]): + if do_rescale and is_scaled_image(first_image_in_list): logger.warning_once( "It looks like you are trying to rescale already rescaled images. If the input" " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." @@ -732,9 +733,7 @@ def preprocess( # We assume that all images have the same channel dimension format. if input_data_format is None: - input_data_format = infer_channel_dimension_format( - make_flat_list_of_images(images_list)[0], num_channels=(1, 3, 4) - ) + input_data_format = infer_channel_dimension_format(first_image_in_list, num_channels=(1, 3, 4)) if do_resize: images_list = [ From 9596e3a14cb90ef1aeb9bc916ef36e25866e7685 Mon Sep 17 00:00:00 2001 From: hiyouga Date: Mon, 17 Mar 2025 17:18:24 +0000 Subject: [PATCH 06/18] add verbose comment --- src/transformers/models/gemma3/image_processing_gemma3.py | 1 + src/transformers/models/idefics2/image_processing_idefics2.py | 1 + src/transformers/models/idefics3/image_processing_idefics3.py | 1 + src/transformers/models/smolvlm/image_processing_smolvlm.py | 1 + 4 files changed, 4 insertions(+) diff --git a/src/transformers/models/gemma3/image_processing_gemma3.py b/src/transformers/models/gemma3/image_processing_gemma3.py index c4b047d2bbc9..6316a98586a1 100644 --- a/src/transformers/models/gemma3/image_processing_gemma3.py +++ b/src/transformers/models/gemma3/image_processing_gemma3.py @@ -359,6 +359,7 @@ def preprocess( # All transformations expect numpy arrays. images_list = [[to_numpy_array(image) for image in images] for images in images_list] # Search for the first image in the image list. + # NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682 first_image_in_list = make_flat_list_of_images(images_list)[0] if do_rescale and is_scaled_image(first_image_in_list): diff --git a/src/transformers/models/idefics2/image_processing_idefics2.py b/src/transformers/models/idefics2/image_processing_idefics2.py index 3ac7349ca243..5e0a65207dc6 100644 --- a/src/transformers/models/idefics2/image_processing_idefics2.py +++ b/src/transformers/models/idefics2/image_processing_idefics2.py @@ -497,6 +497,7 @@ def preprocess( # All transformations expect numpy arrays. images_list = [[to_numpy_array(image) for image in images] for images in images_list] # Search for the first image in the image list. + # NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682 first_image_in_list = make_flat_list_of_images(images_list)[0] if do_rescale and is_scaled_image(first_image_in_list): diff --git a/src/transformers/models/idefics3/image_processing_idefics3.py b/src/transformers/models/idefics3/image_processing_idefics3.py index 872fd1da9b4a..160664312f6e 100644 --- a/src/transformers/models/idefics3/image_processing_idefics3.py +++ b/src/transformers/models/idefics3/image_processing_idefics3.py @@ -716,6 +716,7 @@ def preprocess( # All transformations expect numpy arrays. images_list = [[to_numpy_array(image) for image in images] for images in images_list] # Search for the first image in the image list. + # NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682 first_image_in_list = make_flat_list_of_images(images_list)[0] # Extra channel dimension for grayscale images diff --git a/src/transformers/models/smolvlm/image_processing_smolvlm.py b/src/transformers/models/smolvlm/image_processing_smolvlm.py index cdbe8e20bbec..0ae6ad61ec2d 100644 --- a/src/transformers/models/smolvlm/image_processing_smolvlm.py +++ b/src/transformers/models/smolvlm/image_processing_smolvlm.py @@ -713,6 +713,7 @@ def preprocess( # All transformations expect numpy arrays. images_list = [[to_numpy_array(image) for image in images] for images in images_list] # Search for the first image in the image list. + # NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682 first_image_in_list = make_flat_list_of_images(images_list)[0] # Extra channel dimension for grayscale images From d86d96382a3eec3a7535b7e99bdc5f3594a709b7 Mon Sep 17 00:00:00 2001 From: hiyouga Date: Mon, 17 Mar 2025 17:59:48 +0000 Subject: [PATCH 07/18] fix images --- src/transformers/image_utils.py | 4 ++-- .../models/gemma3/image_processing_gemma3.py | 3 +-- .../idefics2/image_processing_idefics2.py | 3 +-- .../idefics3/image_processing_idefics3.py | 19 +++++++++---------- .../smolvlm/image_processing_smolvlm.py | 19 +++++++++---------- 5 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/transformers/image_utils.py b/src/transformers/image_utils.py index a3899cdc30e1..66d5ac9c935b 100644 --- a/src/transformers/image_utils.py +++ b/src/transformers/image_utils.py @@ -243,7 +243,7 @@ def make_flat_list_of_images( if ( isinstance(images, (list, tuple)) and all(isinstance(images_i, (list, tuple)) for images_i in images) - and any(is_valid_list_of_images(images_i) for images_i in images) + and all(is_valid_list_of_images(images_i) or not images_i for images_i in images) ): return [img for img_list in images for img in img_list] @@ -277,7 +277,7 @@ def make_nested_list_of_images( if ( isinstance(images, (list, tuple)) and all(isinstance(images_i, (list, tuple)) for images_i in images) - and any(is_valid_list_of_images(images_i) for images_i in images) + and all(is_valid_list_of_images(images_i) or not images_i for images_i in images) ): return images diff --git a/src/transformers/models/gemma3/image_processing_gemma3.py b/src/transformers/models/gemma3/image_processing_gemma3.py index 6316a98586a1..97cb0a666830 100644 --- a/src/transformers/models/gemma3/image_processing_gemma3.py +++ b/src/transformers/models/gemma3/image_processing_gemma3.py @@ -35,7 +35,6 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_flat_list_of_images, make_nested_list_of_images, to_numpy_array, valid_images, @@ -360,7 +359,7 @@ def preprocess( images_list = [[to_numpy_array(image) for image in images] for images in images_list] # Search for the first image in the image list. # NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682 - first_image_in_list = make_flat_list_of_images(images_list)[0] + first_image_in_list = [images for images in images_list if images][0][0] if do_rescale and is_scaled_image(first_image_in_list): logger.warning_once( diff --git a/src/transformers/models/idefics2/image_processing_idefics2.py b/src/transformers/models/idefics2/image_processing_idefics2.py index 5e0a65207dc6..94c59175aecf 100644 --- a/src/transformers/models/idefics2/image_processing_idefics2.py +++ b/src/transformers/models/idefics2/image_processing_idefics2.py @@ -29,7 +29,6 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_flat_list_of_images, make_nested_list_of_images, to_numpy_array, valid_images, @@ -498,7 +497,7 @@ def preprocess( images_list = [[to_numpy_array(image) for image in images] for images in images_list] # Search for the first image in the image list. # NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682 - first_image_in_list = make_flat_list_of_images(images_list)[0] + first_image_in_list = [images for images in images_list if images][0][0] if do_rescale and is_scaled_image(first_image_in_list): logger.warning_once( diff --git a/src/transformers/models/idefics3/image_processing_idefics3.py b/src/transformers/models/idefics3/image_processing_idefics3.py index 160664312f6e..a78db3110856 100644 --- a/src/transformers/models/idefics3/image_processing_idefics3.py +++ b/src/transformers/models/idefics3/image_processing_idefics3.py @@ -29,7 +29,6 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_flat_list_of_images, make_nested_list_of_images, to_numpy_array, valid_images, @@ -526,7 +525,7 @@ def _pad_image( def pad( self, - images: List[np.ndarray], + images_list: List[np.ndarray], constant_values: Union[float, Iterable[float]] = 0, return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, @@ -537,7 +536,7 @@ def pad( For a list of images, for each images, pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width. For each sample in the batch, pads the sample with empty images to the max_number of images per sample in the batch. Optionally returns a pixel mask. Args: - images (`List[np.ndarray]`): + images_list (`List[np.ndarray]`): List of list of images to pad. Pads to the largest height and width in the batch. constant_values (`float` or `Iterable[float]`, *optional*): The value to use for the padding if `mode` is `"constant"`. @@ -555,17 +554,17 @@ def pad( input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images, input_data_format=input_data_format) + pad_size = get_max_height_width(images_list, input_data_format=input_data_format) - batch_size = len(images) - max_num_images = max(len(images_) for images_ in images) + batch_size = len(images_list) + max_num_images = max(len(images) for images in images_list) input_data_format = ( - infer_channel_dimension_format(images[0][0], num_channels=(1, 3, 4)) + infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4)) if input_data_format is None else input_data_format ) data_format = input_data_format if data_format is None else data_format - first_image_in_list = make_flat_list_of_images(images)[0] + first_image_in_list = [images for images in images_list if images][0][0] if input_data_format == ChannelDimension.FIRST: n_channels = first_image_in_list.shape[0] @@ -586,7 +585,7 @@ def empty_image(size, input_data_format): padded_masks = [[np.zeros(pad_size) for _ in range(max_num_images)] for _ in range(batch_size)] for batch_idx in range(batch_size): - for sample_idx, image in enumerate(images[batch_idx]): + for sample_idx, image in enumerate(images_list[batch_idx]): padded_images_list[batch_idx][sample_idx] = self._pad_image( image, pad_size, @@ -717,7 +716,7 @@ def preprocess( images_list = [[to_numpy_array(image) for image in images] for images in images_list] # Search for the first image in the image list. # NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682 - first_image_in_list = make_flat_list_of_images(images_list)[0] + first_image_in_list = [images for images in images_list if images][0][0] # Extra channel dimension for grayscale images if input_data_format in [ChannelDimension.LAST, None]: diff --git a/src/transformers/models/smolvlm/image_processing_smolvlm.py b/src/transformers/models/smolvlm/image_processing_smolvlm.py index 0ae6ad61ec2d..7cce9a9ae292 100644 --- a/src/transformers/models/smolvlm/image_processing_smolvlm.py +++ b/src/transformers/models/smolvlm/image_processing_smolvlm.py @@ -35,7 +35,6 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_flat_list_of_images, make_nested_list_of_images, to_numpy_array, valid_images, @@ -523,7 +522,7 @@ def _pad_image( def pad( self, - images: List[np.ndarray], + images_list: List[np.ndarray], constant_values: Union[float, Iterable[float]] = 0, return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, @@ -534,7 +533,7 @@ def pad( For a list of images, for each images, pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width. For each sample in the batch, pads the sample with empty images to the max_number of images per sample in the batch. Optionally returns a pixel mask. Args: - images (`List[np.ndarray]`): + images_list (`List[np.ndarray]`): List of list of images to pad. Pads to the largest height and width in the batch. constant_values (`float` or `Iterable[float]`, *optional*): The value to use for the padding if `mode` is `"constant"`. @@ -552,17 +551,17 @@ def pad( input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images, input_data_format=input_data_format) + pad_size = get_max_height_width(images_list, input_data_format=input_data_format) - batch_size = len(images) - max_num_images = max(len(images_) for images_ in images) + batch_size = len(images_list) + max_num_images = max(len(images) for images in images_list) input_data_format = ( - infer_channel_dimension_format(images[0][0], num_channels=(1, 3, 4)) + infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4)) if input_data_format is None else input_data_format ) data_format = input_data_format if data_format is None else data_format - first_image_in_list = make_flat_list_of_images(images)[0] + first_image_in_list = [images for images in images_list if images][0][0] if input_data_format == ChannelDimension.FIRST: n_channels = first_image_in_list.shape[0] @@ -583,7 +582,7 @@ def empty_image(size, input_data_format): padded_masks = [[np.zeros(pad_size) for _ in range(max_num_images)] for _ in range(batch_size)] for batch_idx in range(batch_size): - for sample_idx, image in enumerate(images[batch_idx]): + for sample_idx, image in enumerate(images_list[batch_idx]): padded_images_list[batch_idx][sample_idx] = self._pad_image( image, pad_size, @@ -714,7 +713,7 @@ def preprocess( images_list = [[to_numpy_array(image) for image in images] for images in images_list] # Search for the first image in the image list. # NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682 - first_image_in_list = make_flat_list_of_images(images_list)[0] + first_image_in_list = [images for images in images_list if images][0][0] # Extra channel dimension for grayscale images if input_data_format in [ChannelDimension.LAST, None]: From 8b2a747a34bbd2a06183494b06e0b36a6ce34dd2 Mon Sep 17 00:00:00 2001 From: hiyouga Date: Mon, 17 Mar 2025 18:18:07 +0000 Subject: [PATCH 08/18] rollback --- .../models/idefics3/image_processing_idefics3.py | 16 ++++++++-------- .../models/smolvlm/image_processing_smolvlm.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/transformers/models/idefics3/image_processing_idefics3.py b/src/transformers/models/idefics3/image_processing_idefics3.py index a78db3110856..b48dfae2dcd3 100644 --- a/src/transformers/models/idefics3/image_processing_idefics3.py +++ b/src/transformers/models/idefics3/image_processing_idefics3.py @@ -525,7 +525,7 @@ def _pad_image( def pad( self, - images_list: List[np.ndarray], + images: List[np.ndarray], constant_values: Union[float, Iterable[float]] = 0, return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, @@ -536,7 +536,7 @@ def pad( For a list of images, for each images, pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width. For each sample in the batch, pads the sample with empty images to the max_number of images per sample in the batch. Optionally returns a pixel mask. Args: - images_list (`List[np.ndarray]`): + images (`List[List[np.ndarray]]`): List of list of images to pad. Pads to the largest height and width in the batch. constant_values (`float` or `Iterable[float]`, *optional*): The value to use for the padding if `mode` is `"constant"`. @@ -554,17 +554,17 @@ def pad( input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images_list, input_data_format=input_data_format) + pad_size = get_max_height_width(images, input_data_format=input_data_format) - batch_size = len(images_list) - max_num_images = max(len(images) for images in images_list) + batch_size = len(images) + max_num_images = max(len(images_) for images_ in images) input_data_format = ( - infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4)) + infer_channel_dimension_format(images[0][0], num_channels=(1, 3, 4)) if input_data_format is None else input_data_format ) data_format = input_data_format if data_format is None else data_format - first_image_in_list = [images for images in images_list if images][0][0] + first_image_in_list = [images_ for images_ in images if images][0][0] if input_data_format == ChannelDimension.FIRST: n_channels = first_image_in_list.shape[0] @@ -585,7 +585,7 @@ def empty_image(size, input_data_format): padded_masks = [[np.zeros(pad_size) for _ in range(max_num_images)] for _ in range(batch_size)] for batch_idx in range(batch_size): - for sample_idx, image in enumerate(images_list[batch_idx]): + for sample_idx, image in enumerate(images[batch_idx]): padded_images_list[batch_idx][sample_idx] = self._pad_image( image, pad_size, diff --git a/src/transformers/models/smolvlm/image_processing_smolvlm.py b/src/transformers/models/smolvlm/image_processing_smolvlm.py index 7cce9a9ae292..e266cd7f9562 100644 --- a/src/transformers/models/smolvlm/image_processing_smolvlm.py +++ b/src/transformers/models/smolvlm/image_processing_smolvlm.py @@ -522,7 +522,7 @@ def _pad_image( def pad( self, - images_list: List[np.ndarray], + images: List[np.ndarray], constant_values: Union[float, Iterable[float]] = 0, return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, @@ -533,7 +533,7 @@ def pad( For a list of images, for each images, pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width. For each sample in the batch, pads the sample with empty images to the max_number of images per sample in the batch. Optionally returns a pixel mask. Args: - images_list (`List[np.ndarray]`): + images (`List[List[np.ndarray]]`): List of list of images to pad. Pads to the largest height and width in the batch. constant_values (`float` or `Iterable[float]`, *optional*): The value to use for the padding if `mode` is `"constant"`. @@ -551,17 +551,17 @@ def pad( input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images_list, input_data_format=input_data_format) + pad_size = get_max_height_width(images, input_data_format=input_data_format) - batch_size = len(images_list) - max_num_images = max(len(images) for images in images_list) + batch_size = len(images) + max_num_images = max(len(images_) for images_ in images) input_data_format = ( - infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4)) + infer_channel_dimension_format(images[0][0], num_channels=(1, 3, 4)) if input_data_format is None else input_data_format ) data_format = input_data_format if data_format is None else data_format - first_image_in_list = [images for images in images_list if images][0][0] + first_image_in_list = [images_ for images_ in images if images][0][0] if input_data_format == ChannelDimension.FIRST: n_channels = first_image_in_list.shape[0] @@ -582,7 +582,7 @@ def empty_image(size, input_data_format): padded_masks = [[np.zeros(pad_size) for _ in range(max_num_images)] for _ in range(batch_size)] for batch_idx in range(batch_size): - for sample_idx, image in enumerate(images_list[batch_idx]): + for sample_idx, image in enumerate(images[batch_idx]): padded_images_list[batch_idx][sample_idx] = self._pad_image( image, pad_size, From e484f54bacad0358c4572bb7433deca58dd1c016 Mon Sep 17 00:00:00 2001 From: hiyouga Date: Mon, 17 Mar 2025 18:36:37 +0000 Subject: [PATCH 09/18] fix ut --- src/transformers/models/idefics3/image_processing_idefics3.py | 2 +- src/transformers/models/smolvlm/image_processing_smolvlm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/idefics3/image_processing_idefics3.py b/src/transformers/models/idefics3/image_processing_idefics3.py index b48dfae2dcd3..b89a357676dd 100644 --- a/src/transformers/models/idefics3/image_processing_idefics3.py +++ b/src/transformers/models/idefics3/image_processing_idefics3.py @@ -564,7 +564,7 @@ def pad( else input_data_format ) data_format = input_data_format if data_format is None else data_format - first_image_in_list = [images_ for images_ in images if images][0][0] + first_image_in_list = [images_ for images_ in images if images_][0][0] if input_data_format == ChannelDimension.FIRST: n_channels = first_image_in_list.shape[0] diff --git a/src/transformers/models/smolvlm/image_processing_smolvlm.py b/src/transformers/models/smolvlm/image_processing_smolvlm.py index e266cd7f9562..8f371d928c71 100644 --- a/src/transformers/models/smolvlm/image_processing_smolvlm.py +++ b/src/transformers/models/smolvlm/image_processing_smolvlm.py @@ -561,7 +561,7 @@ def pad( else input_data_format ) data_format = input_data_format if data_format is None else data_format - first_image_in_list = [images_ for images_ in images if images][0][0] + first_image_in_list = [images_ for images_ in images if images_][0][0] if input_data_format == ChannelDimension.FIRST: n_channels = first_image_in_list.shape[0] From 07b9418636dcc3cc593382faa6b115f2aff9d9e4 Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Wed, 2 Apr 2025 01:21:22 +0800 Subject: [PATCH 10/18] Update image_processing_smolvlm.py --- src/transformers/models/smolvlm/image_processing_smolvlm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/smolvlm/image_processing_smolvlm.py b/src/transformers/models/smolvlm/image_processing_smolvlm.py index 8f371d928c71..b5461f3fa9bc 100644 --- a/src/transformers/models/smolvlm/image_processing_smolvlm.py +++ b/src/transformers/models/smolvlm/image_processing_smolvlm.py @@ -561,7 +561,8 @@ def pad( else input_data_format ) data_format = input_data_format if data_format is None else data_format - first_image_in_list = [images_ for images_ in images if images_][0][0] + # filter out empty image lists, then take first image of the first sample + first_image_in_list = [sample_images for sample_images in images if sample_images][0][0] if input_data_format == ChannelDimension.FIRST: n_channels = first_image_in_list.shape[0] From f728c4ff082dcb380240becaa646e6130e289646 Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Wed, 2 Apr 2025 01:30:38 +0800 Subject: [PATCH 11/18] Update image_processing_idefics3.py --- src/transformers/models/idefics3/image_processing_idefics3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/idefics3/image_processing_idefics3.py b/src/transformers/models/idefics3/image_processing_idefics3.py index b89a357676dd..9b1c8d6446d5 100644 --- a/src/transformers/models/idefics3/image_processing_idefics3.py +++ b/src/transformers/models/idefics3/image_processing_idefics3.py @@ -564,7 +564,8 @@ def pad( else input_data_format ) data_format = input_data_format if data_format is None else data_format - first_image_in_list = [images_ for images_ in images if images_][0][0] + # filter out empty image lists, then take first image of the first sample + first_image_in_list = [sample_images for sample_images in images if sample_images][0][0] if input_data_format == ChannelDimension.FIRST: n_channels = first_image_in_list.shape[0] From dab0b336c7a20356f582d1a0cfe10b27c447d928 Mon Sep 17 00:00:00 2001 From: raushan Date: Tue, 26 Aug 2025 12:58:09 +0200 Subject: [PATCH 12/18] add tests and fix some processors --- .../models/beit/image_processing_beit.py | 6 +- .../models/bit/image_processing_bit.py | 4 +- .../image_processing_chinese_clip.py | 4 +- .../image_processing_conditional_detr.py | 4 +- .../convnext/image_processing_convnext.py | 4 +- .../image_processing_deepseek_vl_hybrid.py | 4 +- .../modular_deepseek_vl_hybrid.py | 3 +- .../image_processing_deformable_detr.py | 4 +- .../models/deit/image_processing_deit.py | 4 +- .../vit_hybrid/image_processing_vit_hybrid.py | 4 +- .../depth_pro/image_processing_depth_pro.py | 4 +- .../models/detr/image_processing_detr.py | 4 +- .../models/donut/image_processing_donut.py | 4 +- .../models/dpt/image_processing_dpt.py | 6 +- .../image_processing_efficientnet.py | 4 +- .../models/emu3/image_processing_emu3.py | 62 +- .../models/eomt/image_processing_eomt.py | 3 +- .../models/flava/image_processing_flava.py | 4 +- .../models/fuyu/image_processing_fuyu.py | 35 +- .../models/fuyu/processing_fuyu.py | 2 +- .../models/glm4v/image_processing_glm4v.py | 3 +- .../models/glpn/image_processing_glpn.py | 4 +- .../image_processing_grounding_dino.py | 4 +- .../idefics/image_processing_idefics.py | 4 +- .../models/idefics/processing_idefics.py | 7 +- .../imagegpt/image_processing_imagegpt.py | 4 +- .../models/interns1/modeling_interns1.py | 1099 +++++++++++++++++ .../models/janus/image_processing_janus.py | 3 +- .../models/janus/modular_janus.py | 4 +- .../models/kosmos2/processing_kosmos2.py | 4 +- .../kosmos2_5/image_processing_kosmos2_5.py | 4 +- .../layoutlmv2/image_processing_layoutlmv2.py | 4 +- .../layoutlmv3/image_processing_layoutlmv3.py | 4 +- .../models/levit/image_processing_levit.py | 4 +- .../models/llava/image_processing_llava.py | 4 +- .../llava_next/image_processing_llava_next.py | 3 +- .../image_processing_llava_next_video.py | 4 +- .../image_processing_mask2former.py | 6 +- .../maskformer/image_processing_maskformer.py | 6 +- .../image_processing_mobilenet_v1.py | 4 +- .../image_processing_mobilenet_v2.py | 6 +- .../mobilevit/image_processing_mobilevit.py | 8 +- .../models/nougat/image_processing_nougat.py | 4 +- .../oneformer/image_processing_oneformer.py | 6 +- .../models/owlv2/image_processing_owlv2.py | 4 +- .../models/owlvit/image_processing_owlvit.py | 4 +- .../perceiver/image_processing_perceiver.py | 4 +- .../pix2struct/image_processing_pix2struct.py | 4 +- .../pixtral/image_processing_pixtral.py | 4 +- .../models/pixtral/processing_pixtral.py | 17 +- .../poolformer/image_processing_poolformer.py | 4 +- .../image_processing_prompt_depth_anything.py | 6 +- .../models/pvt/image_processing_pvt.py | 4 +- .../qwen2_vl/image_processing_qwen2_vl.py | 3 +- .../rt_detr/image_processing_rt_detr.py | 6 +- .../models/sam/image_processing_sam.py | 6 +- .../segformer/image_processing_segformer.py | 6 +- .../models/seggpt/image_processing_seggpt.py | 4 +- .../shieldgemma2/processing_shieldgemma2.py | 7 +- .../superpoint/image_processing_superpoint.py | 4 +- .../swin2sr/image_processing_swin2sr.py | 4 +- .../textnet/image_processing_textnet.py | 4 +- .../image_processing_timm_wrapper.py | 4 +- .../image_processing_video_llava.py | 4 +- .../models/vilt/image_processing_vilt.py | 4 +- .../models/vit/image_processing_vit.py | 4 +- .../vitmatte/image_processing_vitmatte.py | 6 +- .../vitpose/image_processing_vitpose.py | 4 +- .../models/yolos/image_processing_yolos.py | 4 +- .../zoedepth/image_processing_zoedepth.py | 4 +- src/transformers/processing_utils.py | 16 +- src/transformers/video_utils.py | 6 +- .../got_ocr2/test_processing_got_ocr2.py | 20 + .../test_processing_grounding_dino.py | 20 + tests/models/mllama/test_processing_mllama.py | 9 + .../models/smolvlm/test_processing_smolvlm.py | 8 + tests/test_processing_common.py | 35 + 77 files changed, 1380 insertions(+), 235 deletions(-) create mode 100644 src/transformers/models/interns1/modeling_interns1.py diff --git a/src/transformers/models/beit/image_processing_beit.py b/src/transformers/models/beit/image_processing_beit.py index 4f549bcff6d0..7fde9c8a8c90 100644 --- a/src/transformers/models/beit/image_processing_beit.py +++ b/src/transformers/models/beit/image_processing_beit.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -390,10 +390,10 @@ def preprocess( image_std = image_std if image_std is not None else self.image_std do_reduce_labels = do_reduce_labels if do_reduce_labels is not None else self.do_reduce_labels - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if segmentation_maps is not None and not valid_images(segmentation_maps): raise ValueError( diff --git a/src/transformers/models/bit/image_processing_bit.py b/src/transformers/models/bit/image_processing_bit.py index 0bcdb7075a23..6ebd9267b6fb 100644 --- a/src/transformers/models/bit/image_processing_bit.py +++ b/src/transformers/models/bit/image_processing_bit.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -256,7 +256,7 @@ def preprocess( image_std = image_std if image_std is not None else self.image_std do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/chinese_clip/image_processing_chinese_clip.py b/src/transformers/models/chinese_clip/image_processing_chinese_clip.py index 701168625c47..3b2464e9371d 100644 --- a/src/transformers/models/chinese_clip/image_processing_chinese_clip.py +++ b/src/transformers/models/chinese_clip/image_processing_chinese_clip.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -250,7 +250,7 @@ def preprocess( image_std = image_std if image_std is not None else self.image_std do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py index e3de085ee515..1694ab315290 100644 --- a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py +++ b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py @@ -46,7 +46,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_annotations, @@ -1390,7 +1390,7 @@ def preprocess( pad_size = self.pad_size if pad_size is None else pad_size format = self.format if format is None else format - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/convnext/image_processing_convnext.py b/src/transformers/models/convnext/image_processing_convnext.py index 487913d56ebd..299cabd5d4d9 100644 --- a/src/transformers/models/convnext/image_processing_convnext.py +++ b/src/transformers/models/convnext/image_processing_convnext.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -262,7 +262,7 @@ def preprocess( size = size if size is not None else self.size size = get_size_dict(size, default_to_square=False) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/deepseek_vl_hybrid/image_processing_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/image_processing_deepseek_vl_hybrid.py index 769d47f24199..8a289b772fea 100644 --- a/src/transformers/models/deepseek_vl_hybrid/image_processing_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/image_processing_deepseek_vl_hybrid.py @@ -34,7 +34,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -327,7 +327,7 @@ def preprocess( high_res_size = high_res_size if high_res_size is not None else self.high_res_size high_res_size_dict = get_size_dict(high_res_size) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py index 80292d6b7d6e..5d263bda9849 100644 --- a/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py +++ b/src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py @@ -36,7 +36,6 @@ infer_channel_dimension_format, is_scaled_image, make_flat_list_of_images, - make_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -631,7 +630,7 @@ def preprocess( high_res_size = high_res_size if high_res_size is not None else self.high_res_size high_res_size_dict = get_size_dict(high_res_size) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py index 358a2da919e6..e4ace2e8b562 100644 --- a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py +++ b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py @@ -46,7 +46,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_annotations, @@ -1388,7 +1388,7 @@ def preprocess( pad_size = self.pad_size if pad_size is None else pad_size format = self.format if format is None else format - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/deit/image_processing_deit.py b/src/transformers/models/deit/image_processing_deit.py index b5a7641b654f..1e2f6c3b5ae5 100644 --- a/src/transformers/models/deit/image_processing_deit.py +++ b/src/transformers/models/deit/image_processing_deit.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -240,7 +240,7 @@ def preprocess( crop_size = crop_size if crop_size is not None else self.crop_size crop_size = get_size_dict(crop_size, param_name="crop_size") - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/deprecated/vit_hybrid/image_processing_vit_hybrid.py b/src/transformers/models/deprecated/vit_hybrid/image_processing_vit_hybrid.py index f1e5a9c8e6d9..f274d8c058f9 100644 --- a/src/transformers/models/deprecated/vit_hybrid/image_processing_vit_hybrid.py +++ b/src/transformers/models/deprecated/vit_hybrid/image_processing_vit_hybrid.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_kwargs, @@ -273,7 +273,7 @@ def preprocess( image_std = image_std if image_std is not None else self.image_std do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - images = make_list_of_images(images) + images = make_flat_list_of_images(images) validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self._valid_processor_keys) diff --git a/src/transformers/models/depth_pro/image_processing_depth_pro.py b/src/transformers/models/depth_pro/image_processing_depth_pro.py index c3686231b4fe..47f224f248bd 100644 --- a/src/transformers/models/depth_pro/image_processing_depth_pro.py +++ b/src/transformers/models/depth_pro/image_processing_depth_pro.py @@ -35,7 +35,7 @@ infer_channel_dimension_format, is_scaled_image, is_torch_available, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -258,7 +258,7 @@ def preprocess( size = size if size is not None else self.size - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/detr/image_processing_detr.py b/src/transformers/models/detr/image_processing_detr.py index 039ba1781cbe..4489b0f0e415 100644 --- a/src/transformers/models/detr/image_processing_detr.py +++ b/src/transformers/models/detr/image_processing_detr.py @@ -45,7 +45,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_annotations, @@ -1363,7 +1363,7 @@ def preprocess( pad_size = self.pad_size if pad_size is None else pad_size format = self.format if format is None else format - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/donut/image_processing_donut.py b/src/transformers/models/donut/image_processing_donut.py index e8a48dcabef6..d6c963803743 100644 --- a/src/transformers/models/donut/image_processing_donut.py +++ b/src/transformers/models/donut/image_processing_donut.py @@ -35,7 +35,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -399,7 +399,7 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/dpt/image_processing_dpt.py b/src/transformers/models/dpt/image_processing_dpt.py index 9535a700c728..4a312e3005f3 100644 --- a/src/transformers/models/dpt/image_processing_dpt.py +++ b/src/transformers/models/dpt/image_processing_dpt.py @@ -39,7 +39,7 @@ is_scaled_image, is_torch_available, is_torch_tensor, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -525,10 +525,10 @@ def preprocess( size_divisor = size_divisor if size_divisor is not None else self.size_divisor do_reduce_labels = do_reduce_labels if do_reduce_labels is not None else self.do_reduce_labels - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/efficientnet/image_processing_efficientnet.py b/src/transformers/models/efficientnet/image_processing_efficientnet.py index 880bfd21995e..ea822d75ca27 100644 --- a/src/transformers/models/efficientnet/image_processing_efficientnet.py +++ b/src/transformers/models/efficientnet/image_processing_efficientnet.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -295,7 +295,7 @@ def preprocess( crop_size = crop_size if crop_size is not None else self.crop_size crop_size = get_size_dict(crop_size, param_name="crop_size") - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/emu3/image_processing_emu3.py b/src/transformers/models/emu3/image_processing_emu3.py index b96983b0e858..dbd9db987a1d 100644 --- a/src/transformers/models/emu3/image_processing_emu3.py +++ b/src/transformers/models/emu3/image_processing_emu3.py @@ -31,8 +31,8 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - is_valid_image, - make_list_of_images, + make_flat_list_of_images, + make_nested_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -46,29 +46,6 @@ logger = logging.get_logger(__name__) -def make_batched_images(images) -> list[list[ImageInput]]: - """ - Accepts images in list or nested list format, and makes a list of images for preprocessing. - - Args: - images (`Union[list[list[ImageInput]], list[ImageInput], ImageInput]`): - The input image. - - Returns: - list: A list of images. - """ - if isinstance(images, (list, tuple)) and isinstance(images[0], (list, tuple)) and is_valid_image(images[0][0]): - return [img for img_list in images for img in img_list] - - elif isinstance(images, (list, tuple)) and is_valid_image(images[0]): - return images - - elif is_valid_image(images): - return [images] - - raise ValueError(f"Could not make batched images from {images}") - - def smart_resize( height: int, width: int, factor: int = 28, min_pixels: int = 56 * 56, max_pixels: int = 14 * 14 * 4 * 1280 ): @@ -211,7 +188,7 @@ def _preprocess( - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. """ - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if do_convert_rgb: images = [convert_to_rgb(image) for image in images] @@ -381,7 +358,7 @@ def preprocess( do_pad = do_pad if do_pad is not None else self.do_pad if images is not None: - images = make_batched_images(images) + images = make_nested_list_of_images(images) if images is not None and not valid_images(images): raise ValueError( @@ -401,20 +378,21 @@ def preprocess( pixel_values = [] for image in images: - image = self._preprocess( - image, - do_resize=do_resize, - resample=resample, - do_rescale=do_rescale, - rescale_factor=rescale_factor, - do_normalize=do_normalize, - image_mean=image_mean, - image_std=image_std, - data_format=data_format, - do_convert_rgb=do_convert_rgb, - input_data_format=input_data_format, - ) - pixel_values.extend(image) + if image: + image = self._preprocess( + image, + do_resize=do_resize, + resample=resample, + do_rescale=do_rescale, + rescale_factor=rescale_factor, + do_normalize=do_normalize, + image_mean=image_mean, + image_std=image_std, + data_format=data_format, + do_convert_rgb=do_convert_rgb, + input_data_format=input_data_format, + ) + pixel_values.extend(image) image_sizes = [image.shape[-2:] for image in pixel_values] if do_pad: @@ -470,7 +448,7 @@ def postprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if isinstance(images[0], Image.Image): return images if len(images) > 1 else images[0] diff --git a/src/transformers/models/eomt/image_processing_eomt.py b/src/transformers/models/eomt/image_processing_eomt.py index 37b9b11103ad..a0a338c68bcb 100644 --- a/src/transformers/models/eomt/image_processing_eomt.py +++ b/src/transformers/models/eomt/image_processing_eomt.py @@ -32,7 +32,6 @@ get_image_size, infer_channel_dimension_format, make_flat_list_of_images, - make_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -614,7 +613,7 @@ def preprocess( ) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) segmentation_maps = [to_numpy_array(mask) for mask in segmentation_maps] segmentation_maps = [ diff --git a/src/transformers/models/flava/image_processing_flava.py b/src/transformers/models/flava/image_processing_flava.py index fc8f47d87ce9..9dcc2c85d479 100644 --- a/src/transformers/models/flava/image_processing_flava.py +++ b/src/transformers/models/flava/image_processing_flava.py @@ -32,7 +32,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -637,7 +637,7 @@ def preprocess( codebook_image_mean = codebook_image_mean if codebook_image_mean is not None else self.codebook_image_mean codebook_image_std = codebook_image_std if codebook_image_std is not None else self.codebook_image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/fuyu/image_processing_fuyu.py b/src/transformers/models/fuyu/image_processing_fuyu.py index bab6e3b47b6d..37c2cd19e38d 100644 --- a/src/transformers/models/fuyu/image_processing_fuyu.py +++ b/src/transformers/models/fuyu/image_processing_fuyu.py @@ -27,13 +27,11 @@ ) from ...image_utils import ( ChannelDimension, - ImageInput, PILImageResampling, get_image_size, infer_channel_dimension_format, is_scaled_image, - is_valid_image, - make_list_of_images, + make_nested_list_of_images, to_numpy_array, validate_preprocess_arguments, ) @@ -55,21 +53,6 @@ logger = logging.get_logger(__name__) -def make_list_of_list_of_images( - images: Union[list[list[ImageInput]], list[ImageInput], ImageInput], -) -> list[list[ImageInput]]: - if is_valid_image(images): - return [[images]] - - if isinstance(images, list) and all(isinstance(image, list) for image in images): - return images - - if isinstance(images, list): - return [make_list_of_images(image) for image in images] - - raise ValueError("images must be a list of list of images or a list of images or an image.") - - class FuyuBatchFeature(BatchFeature): """ BatchFeature class for Fuyu image processor and processor. @@ -447,7 +430,7 @@ def preprocess( if isinstance(images, list) and any(isinstance(elem, list) and len(elem) >= 2 for elem in images): raise ValueError("Multiple images for a single sample are not yet supported.") - batch_images = make_list_of_list_of_images(images) + batch_images = make_nested_list_of_images(images) validate_preprocess_arguments( do_rescale=do_rescale, @@ -464,7 +447,11 @@ def preprocess( # All transformations expect numpy arrays. batch_images = [[to_numpy_array(image) for image in images] for images in batch_images] - if do_rescale and is_scaled_image(batch_images[0][0]): + # Search for the first image in the image list. + # NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682 + first_image_in_list = [images for images in batch_images if images][0][0] + + if do_rescale and is_scaled_image(first_image_in_list): logger.warning_once( "It looks like you are trying to rescale already rescaled images. If the input" " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." @@ -472,9 +459,11 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(batch_images[0][0]) + input_data_format = infer_channel_dimension_format(first_image_in_list) - original_image_sizes = [get_image_size(images[0], channel_dim=input_data_format) for images in batch_images] + original_image_sizes = [ + get_image_size(images[0], channel_dim=input_data_format) for images in batch_images if images + ] size = get_size_dict(size) # for BC if do_resize: @@ -483,7 +472,7 @@ def preprocess( for images in batch_images ] - image_sizes = [get_image_size(images[0], channel_dim=input_data_format) for images in batch_images] + image_sizes = [get_image_size(images[0], channel_dim=input_data_format) for images in batch_images if images] image_unpadded_heights = [[image_size[0]] for image_size in image_sizes] image_unpadded_widths = [[image_size[1]] for image_size in image_sizes] diff --git a/src/transformers/models/fuyu/processing_fuyu.py b/src/transformers/models/fuyu/processing_fuyu.py index 8c75879e8b69..59b8738076fc 100644 --- a/src/transformers/models/fuyu/processing_fuyu.py +++ b/src/transformers/models/fuyu/processing_fuyu.py @@ -561,7 +561,7 @@ def __call__( # --- Use self.tokenizer to get the ids of special tokens to insert into image ids --- - tensor_batch_images = torch.stack([img[0] for img in batch_images]).unsqueeze(1) + tensor_batch_images = torch.stack([img[0] for img in batch_images if img]).unsqueeze(1) # --- Use self.image_processor again to obtain the full token ids and batch inputs --- all_encodings = [] diff --git a/src/transformers/models/glm4v/image_processing_glm4v.py b/src/transformers/models/glm4v/image_processing_glm4v.py index 678eb10406a9..edb7908d176f 100644 --- a/src/transformers/models/glm4v/image_processing_glm4v.py +++ b/src/transformers/models/glm4v/image_processing_glm4v.py @@ -35,7 +35,6 @@ infer_channel_dimension_format, is_scaled_image, make_flat_list_of_images, - make_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -219,7 +218,7 @@ def _preprocess( - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. """ - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if do_convert_rgb: images = [convert_to_rgb(image) for image in images] diff --git a/src/transformers/models/glpn/image_processing_glpn.py b/src/transformers/models/glpn/image_processing_glpn.py index 958230a307f3..e3e0255e2b47 100644 --- a/src/transformers/models/glpn/image_processing_glpn.py +++ b/src/transformers/models/glpn/image_processing_glpn.py @@ -34,7 +34,7 @@ infer_channel_dimension_format, is_scaled_image, is_torch_available, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -186,7 +186,7 @@ def preprocess( size_divisor = size_divisor if size_divisor is not None else self.size_divisor resample = resample if resample is not None else self.resample - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/grounding_dino/image_processing_grounding_dino.py b/src/transformers/models/grounding_dino/image_processing_grounding_dino.py index 8ba1f34ae9cc..8b586779cc39 100644 --- a/src/transformers/models/grounding_dino/image_processing_grounding_dino.py +++ b/src/transformers/models/grounding_dino/image_processing_grounding_dino.py @@ -44,7 +44,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_annotations, @@ -1426,7 +1426,7 @@ def preprocess( pad_size = self.pad_size if pad_size is None else pad_size format = self.format if format is None else format - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/idefics/image_processing_idefics.py b/src/transformers/models/idefics/image_processing_idefics.py index 74a13ff5d11e..21b46ecf2c07 100644 --- a/src/transformers/models/idefics/image_processing_idefics.py +++ b/src/transformers/models/idefics/image_processing_idefics.py @@ -24,7 +24,7 @@ ChannelDimension, ImageInput, PILImageResampling, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -151,7 +151,7 @@ def preprocess( if isinstance(images, list) and len(images) == 0: return [] - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/idefics/processing_idefics.py b/src/transformers/models/idefics/processing_idefics.py index 259f35feb7ec..5f20669c8a8f 100644 --- a/src/transformers/models/idefics/processing_idefics.py +++ b/src/transformers/models/idefics/processing_idefics.py @@ -358,9 +358,10 @@ def __call__( if not all(isinstance(i, str) for i in text): raise ValueError("When using the image-text-to-text behavior, the prompts should only contain text.") if isinstance(images[0], (list, tuple)): - # if nested images, nest text as well - text = [[i] for i in text] - prompts = list(zip(images, text)) + # if nested images, un-nest each sublist and create `prompts` + prompts = [[sample, *image_list] for image_list, sample in zip(images, text)] + else: + prompts = list(zip(images, text)) output_kwargs = self._merge_kwargs( IdeficsProcessorKwargs, diff --git a/src/transformers/models/imagegpt/image_processing_imagegpt.py b/src/transformers/models/imagegpt/image_processing_imagegpt.py index 5d3e207dd3eb..7d55a9a9beb8 100644 --- a/src/transformers/models/imagegpt/image_processing_imagegpt.py +++ b/src/transformers/models/imagegpt/image_processing_imagegpt.py @@ -26,7 +26,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -238,7 +238,7 @@ def preprocess( clusters = clusters if clusters is not None else self.clusters clusters = np.array(clusters) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/interns1/modeling_interns1.py b/src/transformers/models/interns1/modeling_interns1.py new file mode 100644 index 000000000000..4f69e38c73fd --- /dev/null +++ b/src/transformers/models/interns1/modeling_interns1.py @@ -0,0 +1,1099 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/interns1/modular_interns1.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_interns1.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# coding=utf-8 +# Copyright 2025 HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections.abc +import copy +from dataclasses import dataclass +from typing import Callable, Optional, Union + +import numpy as np +import torch +import torch.nn as nn + +from ...activations import ACT2FN +from ...cache_utils import Cache +from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub +from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from ...processing_utils import Unpack +from ...utils import ( + ModelOutput, + TransformersKwargs, + auto_docstring, + can_return_tuple, + torch_int, +) +from ...utils.generic import check_model_inputs +from ..auto import AutoModel +from .configuration_interns1 import InternS1Config, InternS1VisionConfig + + +@use_kernel_forward_from_hub("RMSNorm") +class InternS1VisionRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + InternS1VisionRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + scaling: float, + dropout: float = 0.0, + **kwargs, +): + key_states = key + value_states = value + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + # No upcasting of the attention weights to float32 in this implementation + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +class InternS1VisionAttention(nn.Module): + """Attention Class for InternS1 Vision Encoder""" + + def __init__(self, config: InternS1VisionConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" + f" {self.num_heads})." + ) + self.scale = self.head_dim**-0.5 + self.attention_dropout = config.attention_dropout + proj_dropout = config.projection_dropout + qk_norm = config.use_qk_norm + + # Needed for flash attention + self.is_causal = False + + self.q_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=config.attention_bias) + self.k_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=config.attention_bias) + self.v_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=config.attention_bias) + self.projection_layer = nn.Linear(self.embed_dim, self.embed_dim) + self.projection_dropout = nn.Dropout(proj_dropout) if proj_dropout > 0 else nn.Identity() + + self.q_norm = InternS1VisionRMSNorm(self.embed_dim) if qk_norm else nn.Identity() + self.k_norm = InternS1VisionRMSNorm(self.embed_dim) if qk_norm else nn.Identity() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[torch.Tensor] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ): + batch_size, seq_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = self.q_norm(query_states) + key_states = self.k_norm(key_states) + + query_states = query_states.reshape(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.reshape(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scale, + is_causal=False, + **kwargs, + ) + attn_output = attn_output.reshape(batch_size, seq_len, self.embed_dim) + + output = self.projection_layer(attn_output) + output = self.projection_dropout(output) + + return output, attn_weights + + +NORM2FN = {"layer_norm": nn.LayerNorm, "rms_norm": InternS1VisionRMSNorm} + + +@dataclass +@auto_docstring( + custom_intro=""" + Class for outputs of [`InternS1VisionModel`]. + """ +) +class InternS1VisionModelOutputWithPooling(BaseModelOutputWithPooling): + r""" + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if + *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token + will be returned. + """ + + +class InternS1VisionPatchEmbeddings(nn.Module): + """ + This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial + `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a + Transformer. + """ + + def __init__(self, config): + super().__init__() + image_size, patch_size = config.image_size, config.patch_size + num_channels, hidden_size = config.num_channels, config.hidden_size + + num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0]) + patch_shape = (image_size[0] // patch_size[0], image_size[1] // patch_size[1]) + self.image_size = image_size + self.patch_size = patch_size + self.num_channels = num_channels + self.num_patches = num_patches + self.patch_shape = patch_shape + + self.projection = nn.Conv2d(num_channels, hidden_size, kernel_size=patch_size, stride=patch_size) + + def forward(self, pixel_values: torch.Tensor) -> tuple[torch.Tensor, tuple[int, int]]: + batch_size, num_channels, height, width = pixel_values.shape + if num_channels != self.num_channels: + raise ValueError( + "Make sure that the channel dimension of the pixel values match with the one set in the configuration." + ) + # Note: more robust + embeddings = self.projection(pixel_values.to(self.projection.weight.dtype)) + patch_height, patch_width = embeddings.shape[2], embeddings.shape[3] + embeddings = embeddings.flatten(2).transpose(1, 2) + + return embeddings, (patch_height, patch_width) + + +# Based on timm implementation, which can be found here: +# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py +class InternS1VisionEmbeddings(nn.Module): + """ + Construct the CLS token, position and patch embeddings. Optionally, also the mask token. + + """ + + def __init__(self, config: InternS1VisionConfig) -> None: + super().__init__() + + self.cls_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size)) + if config.use_mask_token: + self.mask_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size)) + else: + self.mask_token = None + self.patch_embeddings = InternS1VisionPatchEmbeddings(config) + self.patch_size = config.patch_size + self.image_size = ( + config.image_size + if isinstance(config.image_size, collections.abc.Iterable) + else (config.image_size, config.image_size) + ) + num_patches = self.patch_embeddings.num_patches + if config.use_absolute_position_embeddings: + self.position_embeddings = nn.Parameter(torch.zeros(1, num_patches + 1, config.hidden_size)) + else: + self.position_embeddings = None + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def interpolate_pos_encoding(self, embeddings: torch.Tensor, height: int, width: int) -> torch.Tensor: + """ + This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution + images. This method is also adapted to support torch.jit tracing. + + Adapted from: + - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and + - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 + """ + + num_patches = embeddings.shape[1] - 1 + num_positions = self.position_embeddings.shape[1] - 1 + + # always interpolate when tracing to ensure the exported model works for dynamic input shapes + if not torch.jit.is_tracing() and num_patches == num_positions and height == width: + return self.position_embeddings + + class_pos_embed = self.position_embeddings[:, :1] + patch_pos_embed = self.position_embeddings[:, 1:] + + dim = embeddings.shape[-1] + + new_height = height // self.patch_size[0] + new_width = width // self.patch_size[1] + + sqrt_num_positions = torch_int(num_positions**0.5) + patch_pos_embed = patch_pos_embed.reshape(1, sqrt_num_positions, sqrt_num_positions, dim) + patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2) + + patch_pos_embed = nn.functional.interpolate( + patch_pos_embed, + size=(new_height, new_width), + mode="bicubic", + align_corners=False, + ) + + patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim) + + return torch.cat((class_pos_embed, patch_pos_embed), dim=1) + + def forward( + self, + pixel_values: torch.Tensor, + bool_masked_pos: Optional[torch.BoolTensor] = None, + ) -> torch.Tensor: + _, _, height, width = pixel_values.shape + embeddings, (patch_height, patch_width) = self.patch_embeddings(pixel_values) + batch_size, seq_len, _ = embeddings.size() + + if bool_masked_pos is not None: + mask_tokens = self.mask_token.expand(batch_size, seq_len, -1) + # replace the masked visual tokens by mask_tokens + w = bool_masked_pos.unsqueeze(-1).type_as(mask_tokens) + embeddings = embeddings * (1 - w) + mask_tokens * w + + cls_tokens = self.cls_token.expand(batch_size, -1, -1) + embeddings = torch.cat((cls_tokens, embeddings), dim=1) + + if self.position_embeddings is not None: + embeddings = embeddings + self.interpolate_pos_encoding(embeddings, height, width) + + embeddings = self.dropout(embeddings) + + return embeddings, (patch_height, patch_width) + + +class InternS1VisionMLP(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.activation_fn = ACT2FN[config.hidden_act] + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size) + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states = self.fc2(hidden_states) + return hidden_states + + +def drop_path(input: torch.Tensor, drop_prob: float = 0.0, training: bool = False) -> torch.Tensor: + """ + Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + + Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks, + however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the + layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the + argument. + """ + if drop_prob == 0.0 or not training: + return input + keep_prob = 1 - drop_prob + shape = (input.shape[0],) + (1,) * (input.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = keep_prob + torch.rand(shape, dtype=input.dtype, device=input.device) + random_tensor.floor_() # binarize + output = input.div(keep_prob) * random_tensor + return output + + +class InternS1DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob: Optional[float] = None) -> None: + super().__init__() + self.drop_prob = drop_prob + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return drop_path(hidden_states, self.drop_prob, self.training) + + def extra_repr(self) -> str: + return f"p={self.drop_prob}" + + +class InternS1VisionLayer(GradientCheckpointingLayer): + """This corresponds to the Block class in the timm implementation.""" + + def __init__(self, config: InternS1VisionConfig) -> None: + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = InternS1VisionAttention(config) + self.mlp = InternS1VisionMLP(config) + # InternS1 uses different layernorm implementations for different models + self.layernorm_before = NORM2FN[config.norm_type](config.hidden_size, eps=config.layer_norm_eps) + self.layernorm_after = NORM2FN[config.norm_type](config.hidden_size, eps=config.layer_norm_eps) + + init_values = config.layer_scale_init_value + self.lambda_1 = nn.Parameter(init_values * torch.ones(config.hidden_size), requires_grad=True) + self.lambda_2 = nn.Parameter(init_values * torch.ones(config.hidden_size), requires_grad=True) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + # Note: Compared to InternVL, we have added support for drop_path to facilitate user fine-tuning. + self.drop_path1 = InternS1DropPath(config.drop_path_rate) + self.drop_path2 = InternS1DropPath(config.drop_path_rate) + + def forward( + self, + hidden_states: torch.Tensor, + ) -> Union[tuple[torch.Tensor], tuple[torch.Tensor, torch.Tensor]]: + attention_output, _ = self.attention( + self.layernorm_before(hidden_states), # in InternS1Vision, layernorm is applied before self-attention + ) + + attention_output = self.lambda_1 * attention_output + + # first residual connection + hidden_states = self.drop_path1(attention_output) + hidden_states + + # in InternS1Vision, layernorm is also applied after self-attention + layer_output = self.layernorm_after(hidden_states) + + layer_output = self.mlp(layer_output) + layer_output = self.dropout(layer_output) + + if self.lambda_2 is not None: + layer_output = self.lambda_2 * layer_output + + # second residual connection + layer_output = self.drop_path2(layer_output) + hidden_states + + return layer_output + + +class InternS1VisionEncoder(nn.Module): + def __init__(self, config: InternS1VisionConfig) -> None: + super().__init__() + self.config = config + dpr = np.linspace(0.0, float(config.drop_path_rate), int(config.num_hidden_layers)) + dpr_configs = [] + for idx in range(config.num_hidden_layers): + copy_config = copy.deepcopy(config) + copy_config.drop_path_rate = dpr[idx] + dpr_configs.append(copy_config) + self.layer = nn.ModuleList([InternS1VisionLayer(dpr_configs[idx]) for idx in range(config.num_hidden_layers)]) + self.gradient_checkpointing = False + + @check_model_inputs + def forward( + self, + hidden_states: torch.Tensor, + output_hidden_states: bool = False, + ) -> Union[tuple, BaseModelOutput]: + for i, layer_module in enumerate(self.layer): + hidden_states = layer_module(hidden_states) + + return BaseModelOutput( + last_hidden_state=hidden_states, + ) + + +@auto_docstring +class InternS1VisionPreTrainedModel(PreTrainedModel): + config: InternS1VisionConfig + base_model_prefix = "interns1_vision" + main_input_name = "pixel_values" + supports_gradient_checkpointing = True + _no_split_modules = ["InternS1VisionLayer"] + _supports_sdpa = True + _supports_flash_attn = True + _supports_flex_attn = True + _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": InternS1VisionLayer, + "attentions": InternS1VisionAttention, + } + + def _init_weights(self, module): + """Initialize the weights""" + super()._init_weights(module) + if isinstance(module, InternS1VisionEmbeddings): + module.cls_token.data.zero_() + if module.mask_token is not None: + module.mask_token.data.zero_() + if module.position_embeddings is not None: + module.position_embeddings.data.zero_() + elif isinstance(module, InternS1VisionLayer): + module.lambda_1.data.fill_(self.config.layer_scale_init_value) + module.lambda_2.data.fill_(self.config.layer_scale_init_value) + + +@auto_docstring +class InternS1VisionModel(InternS1VisionPreTrainedModel): + def __init__(self, config: InternS1VisionConfig) -> None: + super().__init__(config) + self.config = config + + self.embeddings = InternS1VisionEmbeddings(config) + self.encoder = InternS1VisionEncoder(config) + + self.layernorm = ( + nn.Identity() if config.use_mean_pooling else nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + ) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embeddings.patch_embeddings + + @can_return_tuple + @auto_docstring + def forward( + self, + pixel_values: torch.Tensor, + bool_masked_pos: Optional[torch.BoolTensor] = None, + ) -> Union[tuple, InternS1VisionModelOutputWithPooling]: + r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + """ + embedding_output, _ = self.embeddings(pixel_values, bool_masked_pos=bool_masked_pos) + + encoder_outputs = self.encoder( + embedding_output, + ) + sequence_output = encoder_outputs[0] + sequence_output = self.layernorm(sequence_output) + + return InternS1VisionModelOutputWithPooling( + last_hidden_state=sequence_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + +@auto_docstring +class InternS1PreTrainedModel(PreTrainedModel): + config: InternS1Config + base_model_prefix = "" + supports_gradient_checkpointing = True + _skip_keys_device_placement = "past_key_values" + + _supports_flash_attn = True + _supports_sdpa = True + + _can_compile_fullgraph = True + _supports_flex_attn = True + _supports_attention_backend = True + _can_record_outputs = { + "router_logits": "SparseMoeBlock", + "hidden_states": "DecoderLayer", + "attentions": "Attention", + } + + +class InternS1MultiModalProjector(nn.Module): + def __init__(self, config: InternS1Config): + super().__init__() + self.layer_norm = nn.LayerNorm(config.vision_config.hidden_size * int(1 / config.downsample_ratio) ** 2) + self.linear_1 = nn.Linear( + config.vision_config.hidden_size * int(1 / config.downsample_ratio) ** 2, config.text_config.hidden_size + ) + self.act = ACT2FN[config.projector_hidden_act] + self.linear_2 = nn.Linear(config.text_config.hidden_size, config.text_config.hidden_size) + + def forward(self, image_features): + hidden_states = self.layer_norm(image_features) + hidden_states = self.linear_1(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states = self.linear_2(hidden_states) + return hidden_states + + +@dataclass +@auto_docstring( + custom_intro=""" + Base class for InternS1 outputs, with hidden states and attentions. + """ +) +class InternS1ModelOutputWithPast(BaseModelOutputWithPast): + r""" + router_logits (`tuple(torch.FloatTensor)`, *optional*, returned when `output_router_probs=True` and `config.add_router_probs=True` is passed or when `config.output_router_probs=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, sequence_length, num_experts)`. + + Raw router logtis (post-softmax) that are computed by MoE routers, these terms are used to compute the auxiliary + loss for Mixture of Experts models. + """ + + image_hidden_states: Optional[torch.FloatTensor] = None + + router_logits: Optional[tuple[torch.FloatTensor]] = None + + +@auto_docstring( + custom_intro=""" + The InternS1 model which consists of a vision backbone and a language model, without a language modeling head. + """ +) +class InternS1Model(InternS1PreTrainedModel): + _checkpoint_conversion_mapping = {} + + def __init__(self, config: InternS1Config): + super().__init__(config) + self.vision_tower = AutoModel.from_config(config.vision_config) + + self.multi_modal_projector = InternS1MultiModalProjector(config) + self.language_model = AutoModel.from_config(config.text_config) + self.post_init() + + def get_input_embeddings(self): + return self.language_model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.language_model.set_input_embeddings(value) + + def set_decoder(self, decoder): + self.language_model = decoder + + def get_decoder(self): + return self.language_model + + def get_image_features( + self, + pixel_values: torch.FloatTensor, + vision_feature_layer: Optional[Union[int, list[int]]] = None, + vision_feature_select_strategy: Optional[str] = None, + **kwargs, + ): + """ + Obtains image last hidden states from the vision tower and apply multimodal projection. + + Args: + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. + vision_feature_layer (`int` or `list[int]`): + Layer index or list of layer indices to extract features from. + Returns: + vision_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`. + """ + vision_feature_layer = ( + vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer + ) + vision_feature_select_strategy = ( + vision_feature_select_strategy + if vision_feature_select_strategy is not None + else self.config.vision_feature_select_strategy + ) + pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility + + downsample_ratio = self.config.downsample_ratio + if vision_feature_layer == -1: + vision_features = self.vision_tower(pixel_values=pixel_values).last_hidden_state + else: + vision_features = self.vision_model(pixel_values=pixel_values).hidden_states[vision_feature_layer] + if vision_feature_select_strategy == "default": + vision_features = vision_features[:, 1:, :] + + # Calculate dimensions based on vision features + channels = vision_features.shape[1] + feature_size = int(channels**0.5) + batch_size = vision_features.shape[0] + + # Reshape tensor to spatial dimensions + vision_features = vision_features.reshape(batch_size, feature_size, feature_size, -1) + + # Apply downsampling using pixel shuffle + vision_features = self.pixel_shuffle(vision_features, scale_factor=downsample_ratio) + + # Reshape tensor to prepare for projection + vision_features = vision_features.reshape(batch_size, -1, vision_features.shape[-1]) + + # Project features through multi-modal projector + vision_features = self.multi_modal_projector(vision_features) + return vision_features + + def get_placeholder_mask( + self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor + ): + """ + Obtains multimodal placeholdr mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is + equal to the length of multimodal features. If the lengths are different, an error is raised. + """ + if input_ids is None: + special_image_mask = inputs_embeds == self.get_input_embeddings()( + torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + special_image_mask = special_image_mask.all(-1) + else: + special_image_mask = input_ids == self.config.image_token_id + + n_image_tokens = special_image_mask.sum() + special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) + n_image_features = image_features.shape[0] * image_features.shape[1] + if inputs_embeds[special_image_mask].numel() != image_features.numel(): + raise ValueError( + f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {n_image_features}" + ) + return special_image_mask + + @check_model_inputs + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor = None, + pixel_values: torch.FloatTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + vision_feature_layer: Optional[Union[int, list[int]]] = None, + vision_feature_select_strategy: Optional[str] = None, + use_cache: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> InternS1ModelOutputWithPast: + vision_feature_layer = ( + vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer + ) + vision_feature_select_strategy = ( + vision_feature_select_strategy + if vision_feature_select_strategy is not None + else self.config.vision_feature_select_strategy + ) + + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds = self.get_input_embeddings()(input_ids) + + if pixel_values is not None: + image_features = self.get_image_features( + pixel_values=pixel_values, + vision_feature_layer=vision_feature_layer, + vision_feature_select_strategy=vision_feature_select_strategy, + ) + image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) + special_image_mask = self.get_placeholder_mask( + input_ids, inputs_embeds=inputs_embeds, image_features=image_features + ) + inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features) + + outputs = self.language_model( + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + cache_position=cache_position, + **kwargs, + ) + + return InternS1ModelOutputWithPast( + last_hidden_state=outputs.last_hidden_state, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + router_logits=outputs.router_logits, + image_hidden_states=image_features if pixel_values is not None else None, + ) + + def pixel_shuffle(self, vision_features: torch.Tensor, scale_factor: float = 0.5): + """Perform pixel shuffle downsampling on vision features. + + Args: + vision_features (`torch.Tensor`): + Input tensor of shape (batch_size, width, height, channels). + scale_factor (`float`, *optional*, defaults to `0.5`): + Factor by which to downsample. Default is 0.5, which halves the dimensions. + + Returns: + vision_features (`torch.Tensor`): + Downsampled tensor of shape (batch_size, height*scale_factor, width*scale_factor, channels/(scale_factor^2)). + """ + batch_size, width, height, channels = vision_features.size() + + if height % scale_factor != 0 or width % scale_factor != 0: + raise ValueError("Height and width must be divisible by scale_factor for proper downsampling.") + + # Reshape to allow downsampling + vision_features = vision_features.view( + batch_size, width, int(height * scale_factor), int(channels / scale_factor) + ) + # Permute dimensions to align downsampled axis correctly + vision_features = vision_features.permute(0, 2, 1, 3).contiguous() + + # Reshape to achieve final downsampled dimensions + vision_features = vision_features.view( + batch_size, int(height * scale_factor), int(width * scale_factor), int(channels / (scale_factor**2)) + ) + + # Swap height and width back for proper orientation + vision_features = vision_features.permute(0, 2, 1, 3).contiguous() + + return vision_features + + +@dataclass +@auto_docstring( + custom_intro=""" + Base class for InternS1 causal language model (or autoregressive) outputs. + """ +) +class InternS1CausalLMOutputWithPast(ModelOutput): + r""" + aux_loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided): + aux_loss for the sparse modules. + router_logits (`tuple(torch.FloatTensor)`, *optional*, returned when `output_router_probs=True` and `config.add_router_probs=True` is passed or when `config.output_router_probs=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, sequence_length, num_experts)`. + + Raw router logtis (post-softmax) that are computed by MoE routers, these terms are used to compute the auxiliary + loss for Mixture of Experts models. + """ + + loss: Optional[torch.FloatTensor] = None + logits: Optional[torch.FloatTensor] = None + past_key_values: Optional[list[torch.FloatTensor]] = None + hidden_states: Optional[tuple[torch.FloatTensor]] = None + attentions: Optional[tuple[torch.FloatTensor]] = None + image_hidden_states: Optional[torch.FloatTensor] = None + + aux_loss: Optional[torch.FloatTensor] = None + router_logits: Optional[tuple[torch.FloatTensor]] = None + + +def load_balancing_loss_func( + gate_logits: Union[torch.Tensor, tuple[torch.Tensor], None], + num_experts: Optional[int] = None, + top_k=2, + attention_mask: Optional[torch.Tensor] = None, +) -> Union[torch.Tensor, int]: + r""" + Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch. + + See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss + function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between + experts is too unbalanced. + + Args: + gate_logits: + Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of + shape [batch_size X sequence_length, num_experts]. + num_experts: + Number of experts + top_k: + The number of experts to route per-token, can be also interpreted as the `top-k` routing + parameter. + attention_mask (`torch.Tensor`, *optional*): + The attention_mask used in forward function + shape [batch_size X sequence_length] if not None. + + Returns: + The auxiliary loss. + """ + if gate_logits is None or not isinstance(gate_logits, tuple): + return 0 + + if isinstance(gate_logits, tuple): + compute_device = gate_logits[0].device + concatenated_gate_logits = torch.cat([layer_gate.to(compute_device) for layer_gate in gate_logits], dim=0) + + routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-1) + + _, selected_experts = torch.topk(routing_weights, top_k, dim=-1) + + expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts) + + if attention_mask is None: + # Compute the percentage of tokens routed to each experts + tokens_per_expert = torch.mean(expert_mask.float(), dim=0) + + # Compute the average probability of routing to these experts + router_prob_per_expert = torch.mean(routing_weights, dim=0) + else: + batch_size, sequence_length = attention_mask.shape + num_hidden_layers = concatenated_gate_logits.shape[0] // (batch_size * sequence_length) + + # Compute the mask that masks all padding tokens as 0 with the same shape of expert_mask + expert_attention_mask = ( + attention_mask[None, :, :, None, None] + .expand((num_hidden_layers, batch_size, sequence_length, top_k, num_experts)) + .reshape(-1, top_k, num_experts) + .to(compute_device) + ) + + # Compute the percentage of tokens routed to each experts + tokens_per_expert = torch.sum(expert_mask.float() * expert_attention_mask, dim=0) / torch.sum( + expert_attention_mask, dim=0 + ) + + # Compute the mask that masks all padding tokens as 0 with the same shape of tokens_per_expert + router_per_expert_attention_mask = ( + attention_mask[None, :, :, None] + .expand((num_hidden_layers, batch_size, sequence_length, num_experts)) + .reshape(-1, num_experts) + .to(compute_device) + ) + + # Compute the average probability of routing to these experts + router_prob_per_expert = torch.sum(routing_weights * router_per_expert_attention_mask, dim=0) / torch.sum( + router_per_expert_attention_mask, dim=0 + ) + + overall_loss = torch.sum(tokens_per_expert * router_prob_per_expert.unsqueeze(0)) + return overall_loss * num_experts + + +@auto_docstring( + custom_intro=""" + The INTERNS1 model which consists of a vision backbone and a language model. + """ +) +class InternS1ForConditionalGeneration(InternS1PreTrainedModel, GenerationMixin): + _checkpoint_conversion_mapping = {} + _tied_weights_keys = ["lm_head.weight"] + + def __init__(self, config: InternS1Config): + super().__init__(config) + self.model = InternS1Model(config) + self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False) + + self.post_init() + + def get_input_embeddings(self): + return self.model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.model.set_input_embeddings(value) + + def get_output_embeddings(self) -> nn.Module: + return self.lm_head + + def set_decoder(self, decoder): + self.model.set_decoder(decoder) + + def get_decoder(self): + return self.model.get_decoder() + + def get_image_features( + self, + pixel_values: torch.FloatTensor, + vision_feature_layer: Optional[Union[int, list[int]]] = None, + vision_feature_select_strategy: Optional[str] = None, + **kwargs, + ): + return self.model.get_image_features( + pixel_values=pixel_values, + vision_feature_layer=vision_feature_layer, + vision_feature_select_strategy=vision_feature_select_strategy, + **kwargs, + ) + + # Make modules available through conditional class for BC + @property + def language_model(self): + return self.model.language_model + + @property + def vision_tower(self): + return self.model.vision_tower + + @property + def multi_modal_projector(self): + return self.model.multi_modal_projector + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor = None, + pixel_values: torch.FloatTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + vision_feature_layer: Optional[Union[int, list[int]]] = None, + vision_feature_select_strategy: Optional[str] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + logits_to_keep: Union[int, torch.Tensor] = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, InternS1CausalLMOutputWithPast]: + r""" + Example: + + ```python + >>> import torch + >>> from transformers import AutoProcessor, AutoModelForImageTextToText + + >>> torch_device = "cuda" + >>> processor = AutoProcessor.from_pretrained("internlm/Intern-S1") + >>> model = AutoModelForImageTextToText.from_pretrained( + ... "internlm/Intern-S1", torch_dtype=torch.bfloat16, device_map=torch_device + ... ) + + >>> messages = [ + ... { + ... "role": "user", + ... "content": [ + ... { + ... "type": "image", + ... "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg", + ... }, + ... { + ... "type": "image", + ... "url": "https://thumbs.dreamstime.com/b/golden-gate-bridge-san-francisco-purple-flowers-california-echium-candicans-36805947.jpg", + ... }, + ... {"type": "text", "text": "These images depict two different landmarks. Can you identify them?"}, + ... ], + ... }, + ... ] + + >>> inputs = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt").to(torch_device) + >>> generate_ids = model.generate(**inputs, max_new_tokens=200) + >>> print(processor.decode(generate_ids[0, inputs["input_ids"].shape[1] :], skip_special_tokens=True)) + ```""" + vision_feature_layer = ( + vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer + ) + vision_feature_select_strategy = ( + vision_feature_select_strategy + if vision_feature_select_strategy is not None + else self.config.vision_feature_select_strategy + ) + + outputs = self.model( + input_ids=input_ids, + pixel_values=pixel_values, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + vision_feature_layer=vision_feature_layer, + vision_feature_select_strategy=vision_feature_select_strategy, + use_cache=use_cache, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep + logits = self.lm_head(hidden_states[:, slice_indices, :]) + + loss = None + if labels is not None: + loss = self.loss_function( + logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size, **kwargs + ) + + aux_loss = None + if hasattr(outputs, "router_logits") and labels is not None: + aux_loss = load_balancing_loss_func( + outputs.router_logits, + self.config.text_config.num_experts, + self.config.text_config.num_experts_per_tok, + attention_mask, + ) + loss += self.config.text_config.router_aux_loss_coef * aux_loss.to(loss.device) + + return InternS1CausalLMOutputWithPast( + loss=loss, + aux_loss=aux_loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + router_logits=outputs.router_logits if hasattr(outputs, "router_logits") else None, + image_hidden_states=outputs.image_hidden_states, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + inputs_embeds=None, + pixel_values=None, + attention_mask=None, + cache_position=None, + logits_to_keep=None, + **kwargs, + ): + # Overwritten -- in specific circumstances we don't want to forward image inputs to the model + + model_inputs = super().prepare_inputs_for_generation( + input_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + cache_position=cache_position, + logits_to_keep=logits_to_keep, + **kwargs, + ) + + if cache_position[0] == 0: + # If we're in cached decoding stage, pixel values should be None because input ids do not contain special image token anymore + # Otherwise we need pixel values to be passed to model + model_inputs["pixel_values"] = pixel_values + + return model_inputs + + +__all__ = [ + "InternS1VisionPreTrainedModel", + "InternS1VisionModel", + "InternS1PreTrainedModel", + "InternS1Model", + "InternS1ForConditionalGeneration", +] diff --git a/src/transformers/models/janus/image_processing_janus.py b/src/transformers/models/janus/image_processing_janus.py index 0535f38a33c5..08d5cd85fe01 100644 --- a/src/transformers/models/janus/image_processing_janus.py +++ b/src/transformers/models/janus/image_processing_janus.py @@ -36,7 +36,6 @@ infer_channel_dimension_format, is_scaled_image, make_flat_list_of_images, - make_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -433,7 +432,7 @@ def postprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) # Ensures input is a list + images = make_flat_list_of_images(images) # Ensures input is a list if isinstance(images[0], PIL.Image.Image): return images if len(images) > 1 else images[0] diff --git a/src/transformers/models/janus/modular_janus.py b/src/transformers/models/janus/modular_janus.py index 2ae65710b2a5..aaf6a95def78 100644 --- a/src/transformers/models/janus/modular_janus.py +++ b/src/transformers/models/janus/modular_janus.py @@ -36,7 +36,7 @@ PILImageResampling, get_image_size, infer_channel_dimension_format, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, ) from ...modeling_outputs import ModelOutput @@ -1519,7 +1519,7 @@ def postprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) # Ensures input is a list + images = make_flat_list_of_images(images) # Ensures input is a list if isinstance(images[0], PIL.Image.Image): return images if len(images) > 1 else images[0] diff --git a/src/transformers/models/kosmos2/processing_kosmos2.py b/src/transformers/models/kosmos2/processing_kosmos2.py index cb6f63e2e618..0289f82b650d 100644 --- a/src/transformers/models/kosmos2/processing_kosmos2.py +++ b/src/transformers/models/kosmos2/processing_kosmos2.py @@ -20,7 +20,7 @@ from typing import Optional, Union from ...image_processing_utils import BatchFeature -from ...image_utils import ImageInput, is_batched +from ...image_utils import ImageInput from ...processing_utils import ImagesKwargs, ProcessingKwargs, ProcessorMixin, TextKwargs, Unpack from ...tokenization_utils import AddedToken from ...tokenization_utils_base import BatchEncoding, TextInput @@ -372,7 +372,7 @@ def preprocess_examples( if images is None: images = [None] * len(texts) - elif not is_batched(images): + elif not isinstance(images, list): images = [images] if len(texts) != len(images): raise ValueError( diff --git a/src/transformers/models/kosmos2_5/image_processing_kosmos2_5.py b/src/transformers/models/kosmos2_5/image_processing_kosmos2_5.py index e679050d337f..e68ad2462d07 100644 --- a/src/transformers/models/kosmos2_5/image_processing_kosmos2_5.py +++ b/src/transformers/models/kosmos2_5/image_processing_kosmos2_5.py @@ -30,7 +30,7 @@ ImageInput, get_image_size, infer_channel_dimension_format, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -296,7 +296,7 @@ def preprocess( if kwargs.get("data_format") is not None: raise ValueError("data_format is not an accepted input as the outputs are ") - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py index dfb61c352dc3..42b00b2f5c3e 100644 --- a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py +++ b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py @@ -25,7 +25,7 @@ ImageInput, PILImageResampling, infer_channel_dimension_format, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -251,7 +251,7 @@ def preprocess( ocr_lang = ocr_lang if ocr_lang is not None else self.ocr_lang tesseract_config = tesseract_config if tesseract_config is not None else self.tesseract_config - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py index 5d6b690bb619..8189abf67311 100644 --- a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py +++ b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py @@ -29,7 +29,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -306,7 +306,7 @@ def preprocess( apply_ocr = apply_ocr if apply_ocr is not None else self.apply_ocr ocr_lang = ocr_lang if ocr_lang is not None else self.ocr_lang tesseract_config = tesseract_config if tesseract_config is not None else self.tesseract_config - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/levit/image_processing_levit.py b/src/transformers/models/levit/image_processing_levit.py index 7832dbe5f43b..bece4f1a00c2 100644 --- a/src/transformers/models/levit/image_processing_levit.py +++ b/src/transformers/models/levit/image_processing_levit.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -255,7 +255,7 @@ def preprocess( size = get_size_dict(size, default_to_square=False) crop_size = crop_size if crop_size is not None else self.crop_size crop_size = get_size_dict(crop_size, param_name="crop_size") - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/llava/image_processing_llava.py b/src/transformers/models/llava/image_processing_llava.py index fa737bd9b9c6..9f9dd615d3b2 100644 --- a/src/transformers/models/llava/image_processing_llava.py +++ b/src/transformers/models/llava/image_processing_llava.py @@ -34,7 +34,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_kwargs, @@ -367,7 +367,7 @@ def preprocess( validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self._valid_processor_keys) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/llava_next/image_processing_llava_next.py b/src/transformers/models/llava_next/image_processing_llava_next.py index 21c47701405d..c53dd51c9ef9 100644 --- a/src/transformers/models/llava_next/image_processing_llava_next.py +++ b/src/transformers/models/llava_next/image_processing_llava_next.py @@ -44,7 +44,6 @@ infer_channel_dimension_format, is_scaled_image, make_flat_list_of_images, - make_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -373,7 +372,7 @@ def _preprocess( - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. """ - images = make_list_of_images(images) + images = make_flat_list_of_images(images) all_images = [] for image in images: diff --git a/src/transformers/models/llava_next_video/image_processing_llava_next_video.py b/src/transformers/models/llava_next_video/image_processing_llava_next_video.py index 2bb7b43e677d..53e57f0fd19f 100644 --- a/src/transformers/models/llava_next_video/image_processing_llava_next_video.py +++ b/src/transformers/models/llava_next_video/image_processing_llava_next_video.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, validate_preprocess_arguments, ) @@ -234,7 +234,7 @@ def _preprocess( - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. """ - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if do_convert_rgb: images = [convert_to_rgb(image) for image in images] diff --git a/src/transformers/models/mask2former/image_processing_mask2former.py b/src/transformers/models/mask2former/image_processing_mask2former.py index 2ccc3bb67bf7..5f9639d0f9db 100644 --- a/src/transformers/models/mask2former/image_processing_mask2former.py +++ b/src/transformers/models/mask2former/image_processing_mask2former.py @@ -36,7 +36,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -761,9 +761,9 @@ def preprocess( "torch.Tensor, tf.Tensor or jax.ndarray." ) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if segmentation_maps is not None and len(images) != len(segmentation_maps): raise ValueError("Images and segmentation maps must have the same length.") diff --git a/src/transformers/models/maskformer/image_processing_maskformer.py b/src/transformers/models/maskformer/image_processing_maskformer.py index 8896dad73442..5437af6df9ec 100644 --- a/src/transformers/models/maskformer/image_processing_maskformer.py +++ b/src/transformers/models/maskformer/image_processing_maskformer.py @@ -37,7 +37,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -764,9 +764,9 @@ def preprocess( "torch.Tensor, tf.Tensor or jax.ndarray." ) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if segmentation_maps is not None and len(images) != len(segmentation_maps): raise ValueError("Images and segmentation maps must have the same length.") diff --git a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py index 449b0b6f7bcd..8b0715787545 100644 --- a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py +++ b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py @@ -32,7 +32,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -245,7 +245,7 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py index 41d765ea314a..7f261c5c45c3 100644 --- a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py +++ b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py @@ -32,7 +32,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -407,10 +407,10 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/mobilevit/image_processing_mobilevit.py b/src/transformers/models/mobilevit/image_processing_mobilevit.py index 86707613764f..1207ec0da399 100644 --- a/src/transformers/models/mobilevit/image_processing_mobilevit.py +++ b/src/transformers/models/mobilevit/image_processing_mobilevit.py @@ -26,7 +26,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -404,12 +404,12 @@ def preprocess( do_reduce_labels = do_reduce_labels if do_reduce_labels is not None else self.do_reduce_labels - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/nougat/image_processing_nougat.py b/src/transformers/models/nougat/image_processing_nougat.py index 827686a6066d..79aa421d3864 100644 --- a/src/transformers/models/nougat/image_processing_nougat.py +++ b/src/transformers/models/nougat/image_processing_nougat.py @@ -35,7 +35,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -448,7 +448,7 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/oneformer/image_processing_oneformer.py b/src/transformers/models/oneformer/image_processing_oneformer.py index 5a66918af17e..d88b373418c6 100644 --- a/src/transformers/models/oneformer/image_processing_oneformer.py +++ b/src/transformers/models/oneformer/image_processing_oneformer.py @@ -39,7 +39,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -722,9 +722,9 @@ def preprocess( "torch.Tensor, tf.Tensor or jax.ndarray." ) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if segmentation_maps is not None and len(images) != len(segmentation_maps): raise ValueError("Images and segmentation maps must have the same length.") diff --git a/src/transformers/models/owlv2/image_processing_owlv2.py b/src/transformers/models/owlv2/image_processing_owlv2.py index 0db195ba160e..c228dc4a0625 100644 --- a/src/transformers/models/owlv2/image_processing_owlv2.py +++ b/src/transformers/models/owlv2/image_processing_owlv2.py @@ -34,7 +34,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -434,7 +434,7 @@ def preprocess( size = size if size is not None else self.size size = get_size_dict(size) # for BC - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/owlvit/image_processing_owlvit.py b/src/transformers/models/owlvit/image_processing_owlvit.py index 4bf9306c5640..8040248039d4 100644 --- a/src/transformers/models/owlvit/image_processing_owlvit.py +++ b/src/transformers/models/owlvit/image_processing_owlvit.py @@ -35,7 +35,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -382,7 +382,7 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/perceiver/image_processing_perceiver.py b/src/transformers/models/perceiver/image_processing_perceiver.py index 7550bc9471dc..850671324712 100644 --- a/src/transformers/models/perceiver/image_processing_perceiver.py +++ b/src/transformers/models/perceiver/image_processing_perceiver.py @@ -29,7 +29,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -286,7 +286,7 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/pix2struct/image_processing_pix2struct.py b/src/transformers/models/pix2struct/image_processing_pix2struct.py index d9002cae4b38..30275bb1dcbd 100644 --- a/src/transformers/models/pix2struct/image_processing_pix2struct.py +++ b/src/transformers/models/pix2struct/image_processing_pix2struct.py @@ -28,7 +28,7 @@ ImageInput, get_image_size, infer_channel_dimension_format, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -407,7 +407,7 @@ def preprocess( if kwargs.get("data_format") is not None: raise ValueError("data_format is not an accepted input as the outputs are ") - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/pixtral/image_processing_pixtral.py b/src/transformers/models/pixtral/image_processing_pixtral.py index 152373b7e548..51ee28dfc5bb 100644 --- a/src/transformers/models/pixtral/image_processing_pixtral.py +++ b/src/transformers/models/pixtral/image_processing_pixtral.py @@ -32,7 +32,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_kwargs, @@ -397,7 +397,7 @@ def preprocess( validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self._valid_processor_keys) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images[0]): raise ValueError( diff --git a/src/transformers/models/pixtral/processing_pixtral.py b/src/transformers/models/pixtral/processing_pixtral.py index b59aa840ff87..42edbe24f1f5 100644 --- a/src/transformers/models/pixtral/processing_pixtral.py +++ b/src/transformers/models/pixtral/processing_pixtral.py @@ -21,7 +21,7 @@ import numpy as np from ...feature_extraction_utils import BatchFeature -from ...image_utils import ImageInput, is_valid_image, load_image +from ...image_utils import ImageInput, is_valid_image from ...processing_utils import ( MultiModalData, ProcessingKwargs, @@ -166,21 +166,6 @@ def __call__( patch_size = self.patch_size * self.spatial_merge_size if images is not None: - if is_image_or_image_url(images): - images = [images] - elif isinstance(images, (list, tuple)) and is_image_or_image_url(images[0]): - pass - elif ( - isinstance(images, (list, tuple)) - and isinstance(images[0], (list, tuple)) - and is_image_or_image_url(images[0][0]) - ): - images = [image for sublist in images for image in sublist] - else: - raise ValueError( - "Invalid input images. Please provide a single image, a list of images, or a list of lists of images." - ) - images = [load_image(im) if isinstance(im, str) else im for im in images] image_inputs = self.image_processor(images, patch_size=patch_size, **output_kwargs["images_kwargs"]) else: image_inputs = {} diff --git a/src/transformers/models/poolformer/image_processing_poolformer.py b/src/transformers/models/poolformer/image_processing_poolformer.py index ea37c22e7557..c11a3fddf4a4 100644 --- a/src/transformers/models/poolformer/image_processing_poolformer.py +++ b/src/transformers/models/poolformer/image_processing_poolformer.py @@ -32,7 +32,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -291,7 +291,7 @@ def preprocess( crop_size = crop_size if crop_size is not None else self.crop_size crop_size = get_size_dict(crop_size, param_name="crop_size") - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/prompt_depth_anything/image_processing_prompt_depth_anything.py b/src/transformers/models/prompt_depth_anything/image_processing_prompt_depth_anything.py index e283411b4e98..b5323c308ca4 100644 --- a/src/transformers/models/prompt_depth_anything/image_processing_prompt_depth_anything.py +++ b/src/transformers/models/prompt_depth_anything/image_processing_prompt_depth_anything.py @@ -34,7 +34,7 @@ infer_channel_dimension_format, is_scaled_image, is_torch_available, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -368,7 +368,7 @@ def preprocess( do_pad = do_pad if do_pad is not None else self.do_pad size_divisor = size_divisor if size_divisor is not None else self.size_divisor - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( @@ -432,7 +432,7 @@ def preprocess( if prompt_depth is not None: # prompt_depth is a list of images with shape (height, width) # we need to convert it to a list of images with shape (1, height, width) - prompt_depths = make_list_of_images(prompt_depth, expected_ndims=2) + prompt_depths = make_flat_list_of_images(prompt_depth, expected_ndims=2) # Validate prompt_depths has same length as images if len(prompt_depths) != len(images): diff --git a/src/transformers/models/pvt/image_processing_pvt.py b/src/transformers/models/pvt/image_processing_pvt.py index e9ebecd503a4..d0c594e47e36 100644 --- a/src/transformers/models/pvt/image_processing_pvt.py +++ b/src/transformers/models/pvt/image_processing_pvt.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -216,7 +216,7 @@ def preprocess( size = size if size is not None else self.size size_dict = get_size_dict(size) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py b/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py index e1529594d076..0780b7a8ad84 100644 --- a/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py @@ -40,7 +40,6 @@ infer_channel_dimension_format, is_scaled_image, make_flat_list_of_images, - make_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -224,7 +223,7 @@ def _preprocess( - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. """ - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if do_convert_rgb: images = [convert_to_rgb(image) for image in images] diff --git a/src/transformers/models/rt_detr/image_processing_rt_detr.py b/src/transformers/models/rt_detr/image_processing_rt_detr.py index 42d8efef38a9..de61a8019047 100644 --- a/src/transformers/models/rt_detr/image_processing_rt_detr.py +++ b/src/transformers/models/rt_detr/image_processing_rt_detr.py @@ -42,7 +42,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_annotations, @@ -893,7 +893,7 @@ def preprocess( pad_size = self.pad_size if pad_size is None else pad_size format = self.format if format is None else format - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( @@ -926,7 +926,7 @@ def preprocess( if annotations is not None: validate_annotations(format, SUPPORTED_ANNOTATION_FORMATS, annotations) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " diff --git a/src/transformers/models/sam/image_processing_sam.py b/src/transformers/models/sam/image_processing_sam.py index 7591da5af94d..ea149b256f7b 100644 --- a/src/transformers/models/sam/image_processing_sam.py +++ b/src/transformers/models/sam/image_processing_sam.py @@ -32,7 +32,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -494,7 +494,7 @@ def preprocess( mask_pad_size = get_size_dict(mask_pad_size, default_to_square=True) do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( @@ -503,7 +503,7 @@ def preprocess( ) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if not valid_images(segmentation_maps): raise ValueError( diff --git a/src/transformers/models/segformer/image_processing_segformer.py b/src/transformers/models/segformer/image_processing_segformer.py index 9cb05f811857..a651795ca0cf 100644 --- a/src/transformers/models/segformer/image_processing_segformer.py +++ b/src/transformers/models/segformer/image_processing_segformer.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -368,10 +368,10 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=2) + segmentation_maps = make_flat_list_of_images(segmentation_maps, expected_ndims=2) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/seggpt/image_processing_seggpt.py b/src/transformers/models/seggpt/image_processing_seggpt.py index b05206713ca9..c60cad80d7fd 100644 --- a/src/transformers/models/seggpt/image_processing_seggpt.py +++ b/src/transformers/models/seggpt/image_processing_seggpt.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -323,7 +323,7 @@ def _preprocess_step( size_dict = get_size_dict(size) # If segmentation map is passed we expect 2D images - images = make_list_of_images(images, expected_ndims=2 if do_convert_rgb else 3) + images = make_flat_list_of_images(images, expected_ndims=2 if do_convert_rgb else 3) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/shieldgemma2/processing_shieldgemma2.py b/src/transformers/models/shieldgemma2/processing_shieldgemma2.py index 69d8e6a89e58..993fce03095b 100644 --- a/src/transformers/models/shieldgemma2/processing_shieldgemma2.py +++ b/src/transformers/models/shieldgemma2/processing_shieldgemma2.py @@ -154,6 +154,11 @@ def __call__( messages = [] expanded_images = [] for img in images: + if not isinstance(img, list): + img = [img] + elif len(img) > 1: + raise ValueError(f"SheildGemma can process at most one image per sample, but got {len(img)} images") + for policy in policies: messages.append( [ @@ -166,7 +171,7 @@ def __call__( } ] ) - expanded_images.append([img]) + expanded_images.append(img) text = self.apply_chat_template(messages, tokenize=False) return super().__call__(images=expanded_images, text=text, **kwargs) diff --git a/src/transformers/models/superpoint/image_processing_superpoint.py b/src/transformers/models/superpoint/image_processing_superpoint.py index e6167160c29d..7894e34e227d 100644 --- a/src/transformers/models/superpoint/image_processing_superpoint.py +++ b/src/transformers/models/superpoint/image_processing_superpoint.py @@ -26,7 +26,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -245,7 +245,7 @@ def preprocess( size = size if size is not None else self.size size = get_size_dict(size, default_to_square=False) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/swin2sr/image_processing_swin2sr.py b/src/transformers/models/swin2sr/image_processing_swin2sr.py index 2832e531602e..425231205cd8 100644 --- a/src/transformers/models/swin2sr/image_processing_swin2sr.py +++ b/src/transformers/models/swin2sr/image_processing_swin2sr.py @@ -25,7 +25,7 @@ ImageInput, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -159,7 +159,7 @@ def preprocess( do_pad = do_pad if do_pad is not None else self.do_pad pad_size = pad_size if pad_size is not None else self.pad_size - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/textnet/image_processing_textnet.py b/src/transformers/models/textnet/image_processing_textnet.py index f274c6a6a746..cff4b991d8f3 100644 --- a/src/transformers/models/textnet/image_processing_textnet.py +++ b/src/transformers/models/textnet/image_processing_textnet.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_kwargs, @@ -290,7 +290,7 @@ def preprocess( validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self._valid_processor_keys) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/timm_wrapper/image_processing_timm_wrapper.py b/src/transformers/models/timm_wrapper/image_processing_timm_wrapper.py index 94ca3efedb17..89c8883cbd6a 100644 --- a/src/transformers/models/timm_wrapper/image_processing_timm_wrapper.py +++ b/src/transformers/models/timm_wrapper/image_processing_timm_wrapper.py @@ -20,7 +20,7 @@ from ...image_processing_utils import BaseImageProcessor, BatchFeature from ...image_transforms import to_pil_image -from ...image_utils import ImageInput, make_list_of_images +from ...image_utils import ImageInput, make_flat_list_of_images from ...utils import TensorType, logging, requires_backends from ...utils.import_utils import is_timm_available, is_torch_available, requires @@ -121,7 +121,7 @@ def preprocess( # Add batch dimension if a single image images = images.unsqueeze(0) if images.ndim == 3 else images else: - images = make_list_of_images(images) + images = make_flat_list_of_images(images) images = [to_pil_image(image) for image in images] images = torch.stack([self.val_transforms(image) for image in images]) diff --git a/src/transformers/models/video_llava/image_processing_video_llava.py b/src/transformers/models/video_llava/image_processing_video_llava.py index fa3aeeeb2b3a..37e7c69b105b 100644 --- a/src/transformers/models/video_llava/image_processing_video_llava.py +++ b/src/transformers/models/video_llava/image_processing_video_llava.py @@ -33,7 +33,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -257,7 +257,7 @@ def preprocess( do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb if images is not None: - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if images is not None and not valid_images(images): raise ValueError( diff --git a/src/transformers/models/vilt/image_processing_vilt.py b/src/transformers/models/vilt/image_processing_vilt.py index 6c9a727f0f9a..a624d957013f 100644 --- a/src/transformers/models/vilt/image_processing_vilt.py +++ b/src/transformers/models/vilt/image_processing_vilt.py @@ -30,7 +30,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -417,7 +417,7 @@ def preprocess( size = size if size is not None else self.size size = get_size_dict(size, default_to_square=False) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/vit/image_processing_vit.py b/src/transformers/models/vit/image_processing_vit.py index 8da5f302d475..27de1076cf54 100644 --- a/src/transformers/models/vit/image_processing_vit.py +++ b/src/transformers/models/vit/image_processing_vit.py @@ -28,7 +28,7 @@ PILImageResampling, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -225,7 +225,7 @@ def preprocess( size = size if size is not None else self.size size_dict = get_size_dict(size) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/vitmatte/image_processing_vitmatte.py b/src/transformers/models/vitmatte/image_processing_vitmatte.py index 15d3ec604601..891fdb457359 100644 --- a/src/transformers/models/vitmatte/image_processing_vitmatte.py +++ b/src/transformers/models/vitmatte/image_processing_vitmatte.py @@ -28,7 +28,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -195,8 +195,8 @@ def preprocess( image_std = image_std if image_std is not None else self.image_std size_divisibility = size_divisibility if size_divisibility is not None else self.size_divisibility - images = make_list_of_images(images) - trimaps = make_list_of_images(trimaps, expected_ndims=2) + images = make_flat_list_of_images(images) + trimaps = make_flat_list_of_images(trimaps, expected_ndims=2) if not valid_images(trimaps): raise ValueError( diff --git a/src/transformers/models/vitpose/image_processing_vitpose.py b/src/transformers/models/vitpose/image_processing_vitpose.py index 32c048a8a9d0..6466ccc1d22f 100644 --- a/src/transformers/models/vitpose/image_processing_vitpose.py +++ b/src/transformers/models/vitpose/image_processing_vitpose.py @@ -29,7 +29,7 @@ ImageInput, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, ) @@ -484,7 +484,7 @@ def preprocess( image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/yolos/image_processing_yolos.py b/src/transformers/models/yolos/image_processing_yolos.py index 4a716036d400..48a1300191af 100644 --- a/src/transformers/models/yolos/image_processing_yolos.py +++ b/src/transformers/models/yolos/image_processing_yolos.py @@ -44,7 +44,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_annotations, @@ -1301,7 +1301,7 @@ def preprocess( format = self.format if format is None else format validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self._valid_processor_keys) - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/models/zoedepth/image_processing_zoedepth.py b/src/transformers/models/zoedepth/image_processing_zoedepth.py index 2fb125e2b793..90e69105be01 100644 --- a/src/transformers/models/zoedepth/image_processing_zoedepth.py +++ b/src/transformers/models/zoedepth/image_processing_zoedepth.py @@ -35,7 +35,7 @@ get_image_size, infer_channel_dimension_format, is_scaled_image, - make_list_of_images, + make_flat_list_of_images, to_numpy_array, valid_images, validate_preprocess_arguments, @@ -385,7 +385,7 @@ def preprocess( image_std = image_std if image_std is not None else self.image_std do_pad = do_pad if do_pad is not None else self.do_pad - images = make_list_of_images(images) + images = make_flat_list_of_images(images) if not valid_images(images): raise ValueError( diff --git a/src/transformers/processing_utils.py b/src/transformers/processing_utils.py index eb836db1ff27..44aaec5f15e2 100644 --- a/src/transformers/processing_utils.py +++ b/src/transformers/processing_utils.py @@ -1601,11 +1601,9 @@ def apply_chat_template( # Currently all processors can accept nested list of batches, but not flat list of visuals # So we'll make a batched list of images and let the processor handle it - if images: - batch_images.append(images) - if videos: - batch_videos.append(videos) - batch_video_metadata.append(video_metadata) + batch_images.append(images) + batch_videos.append(videos) + batch_video_metadata.append(video_metadata) prompt, generation_indices = render_jinja_template( conversations=conversations, @@ -1633,12 +1631,14 @@ def apply_chat_template( if "do_sample_frames" not in kwargs and ("fps" in kwargs or "num_frames" in kwargs): kwargs["do_sample_frames"] = True + flat_images = [im for im_list in batch_images for im in im_list] + flat_videos = [vid for vid_list in batch_videos for vid in vid_list] out = self( text=prompt, - images=batch_images if batch_images else None, - videos=batch_videos if batch_videos else None, + images=batch_images if flat_images else None, + videos=batch_videos if flat_videos else None, audio=batch_audios if batch_audios else None, - video_metadata=batch_video_metadata, + video_metadata=flat_videos, **kwargs, ) diff --git a/src/transformers/video_utils.py b/src/transformers/video_utils.py index ea5a93f97ca7..4fc6301cec91 100644 --- a/src/transformers/video_utils.py +++ b/src/transformers/video_utils.py @@ -89,7 +89,7 @@ def is_valid_video_frame(frame): def is_valid_video(video): if not isinstance(video, (list, tuple)): return (is_numpy_array(video) or is_torch_tensor(video)) and video.ndim == 4 - return all(is_valid_video_frame(frame) for frame in video) + return video and all(is_valid_video_frame(frame) for frame in video) def valid_videos(videos): @@ -167,7 +167,9 @@ def make_batched_videos(videos) -> list[Union["np.ndarray", "torch.Tensor"]]: elif is_valid_image(videos): videos = [np.array(videos)[None, ...]] # nested batch so we need to unflatten - elif isinstance(videos[0], (list, tuple)) and is_valid_video(videos[0][0]): + elif isinstance(videos[0], (list, tuple)) and all( + not videos_list or is_valid_video(videos_list[0]) for videos_list in videos + ): videos = [video for sublist in videos for video in sublist] return convert_pil_frames_to_video(videos) diff --git a/tests/models/got_ocr2/test_processing_got_ocr2.py b/tests/models/got_ocr2/test_processing_got_ocr2.py index 0719d211ddad..b97f31a670a4 100644 --- a/tests/models/got_ocr2/test_processing_got_ocr2.py +++ b/tests/models/got_ocr2/test_processing_got_ocr2.py @@ -78,3 +78,23 @@ def test_ocr_queries(self): inputs = processor(image_input, return_tensors="pt", crop_to_patches=True, max_patches=6) self.assertEqual(inputs["input_ids"].shape, (1, 1826)) self.assertEqual(inputs["pixel_values"].shape, (7, 3, 384, 384)) + + def test_processor_text_has_no_visual(self): + # Overwritten: requires `multi_page` kwarg to process nested vision inputs + processor = self.get_processor() + + text = self.prepare_text_inputs(batch_size=3, modality="image") + image_inputs = self.prepare_image_inputs(batch_size=3) + processing_kwargs = {"return_tensors": "pt", "padding": True, "multi_page": True} + + # Call with nested list of vision inputs + image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] + inputs_dict_nested = {"text": text, "images": image_inputs_nested} + processor(**inputs_dict_nested, **processing_kwargs) + + # Call with one of the samples with no associated vision input + plain_text = "lower newer" + image_inputs_nested[0] = [] + text[0] = plain_text + inputs_dict_no_vision = {"text": text, "images": image_inputs_nested} + processor(**inputs_dict_no_vision, **processing_kwargs) diff --git a/tests/models/grounding_dino/test_processing_grounding_dino.py b/tests/models/grounding_dino/test_processing_grounding_dino.py index 42d9743c1f38..721a2109d592 100644 --- a/tests/models/grounding_dino/test_processing_grounding_dino.py +++ b/tests/models/grounding_dino/test_processing_grounding_dino.py @@ -280,3 +280,23 @@ def test_text_preprocessing_equivalence(self): torch.allclose(inputs1["input_ids"], inputs2["input_ids"]), f"Input ids are not equal for batched input: {inputs1['input_ids']} != {inputs2['input_ids']}", ) + + def test_processor_text_has_no_visual(self): + # Overwritten: text inputs have to be nested as well + processor = self.get_processor() + + text = self.prepare_text_inputs(batch_size=3, modality="image") + image_inputs = self.prepare_image_inputs(batch_size=3) + processing_kwargs = {"return_tensors": "pt", "padding": True} + + # Call with nested list of vision inputs + image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] + inputs_dict_nested = {"text": text, "images": image_inputs_nested} + processor(**inputs_dict_nested, **processing_kwargs) + + # Call with one of the samples with no associated vision input + plain_text = ["lower newer"] + image_inputs_nested[0] = [] + text[0] = plain_text + inputs_dict_no_vision = {"text": text, "images": image_inputs_nested} + processor(**inputs_dict_no_vision, **processing_kwargs) diff --git a/tests/models/mllama/test_processing_mllama.py b/tests/models/mllama/test_processing_mllama.py index f20dd5d277fa..b888edc899a8 100644 --- a/tests/models/mllama/test_processing_mllama.py +++ b/tests/models/mllama/test_processing_mllama.py @@ -16,6 +16,7 @@ import shutil import tempfile import unittest +from typing import Optional import numpy as np @@ -56,6 +57,14 @@ def tearDownClass(cls): def prepare_processor_dict(self): return {"chat_template": "{% for message in messages %}{% if loop.index0 == 0 %}{{ bos_token }}{% endif %}{{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }}{% if message['content'] is string %}{{ message['content'] }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' %}{{ '<|image|>' }}{% elif content['type'] == 'text' %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}{{ '<|eot_id|>' }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}"} # fmt: skip + # Override as Mllama needs images to be an explicitly nested batch + def prepare_image_inputs(self, batch_size: Optional[int] = None): + """This function prepares a list of PIL images for testing""" + images = super().prepare_image_inputs(batch_size) + if isinstance(images, (list, tuple)): + images = [[image] for image in images] + return images + def test_chat_template_is_saved(self): processor_loaded = self.processor_class.from_pretrained(self.tmpdirname) processor_dict_loaded = json.loads(processor_loaded.to_json_string()) diff --git a/tests/models/smolvlm/test_processing_smolvlm.py b/tests/models/smolvlm/test_processing_smolvlm.py index 22d15329a58d..63ca2ea9e267 100644 --- a/tests/models/smolvlm/test_processing_smolvlm.py +++ b/tests/models/smolvlm/test_processing_smolvlm.py @@ -94,6 +94,14 @@ def prepare_processor_dict(): "chat_template": "<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '' }}{% endif %}{% endfor %}\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}", } + # Override as SmolVLM needs images/video to be an explicitly nested batch + def prepare_image_inputs(self, batch_size: Optional[int] = None): + """This function prepares a list of PIL images for testing""" + images = super().prepare_image_inputs(batch_size) + if isinstance(images, (list, tuple)): + images = [[image] for image in images] + return images + def prepare_video_inputs(self, batch_size: Optional[int] = None): """This function prepares a list of numpy videos.""" video_input = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)] * 8 diff --git a/tests/test_processing_common.py b/tests/test_processing_common.py index b4d9d7498552..df42cdfe652d 100644 --- a/tests/test_processing_common.py +++ b/tests/test_processing_common.py @@ -167,6 +167,7 @@ def prepare_image_inputs(self, batch_size: Optional[int] = None): def prepare_video_inputs(self, batch_size: Optional[int] = None): """This function prepares a list of numpy videos.""" video_input = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)] * 8 + video_input = np.array(video_input) if batch_size is None: return video_input return [video_input] * batch_size @@ -231,6 +232,40 @@ def test_model_input_names(self): self.assertSetEqual(set(inputs.keys()), set(processor.model_input_names)) + def test_processor_text_has_no_visual(self): + processor = self.get_processor() + call_signature = inspect.signature(processor.__call__) + input_args = [param.name for param in call_signature.parameters.values()] + + if not ("text" in input_args and ("images" in input_args and "videos" in input_args)): + self.skipTest(f"{self.processor_class} doesn't support several vision modalities with text.") + + text = self.prepare_text_inputs(batch_size=3, modality="image") + image_inputs = self.prepare_image_inputs(batch_size=3) + video_inputs = self.prepare_video_inputs(batch_size=3) + inputs_dict = {"text": text, "images": image_inputs, "videos": video_inputs} + inputs_dict = {k: v for k, v in inputs_dict.items() if k in input_args} + + processing_kwargs = {"return_tensors": "pt", "padding": True} + if "videos" in inputs_dict: + processing_kwargs["do_sample_frames"] = False + + # Call processor with all inputs first. Use nested input type which is the format supported by all multimodal processors + image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] + video_inputs_nested = [[video] for video in video_inputs] + inputs_dict_nested = {"text": text, "images": image_inputs_nested, "videos": video_inputs_nested} + inputs_dict_nested = {k: v for k, v in inputs_dict_nested.items() if k in input_args} + processor(**inputs_dict_nested, **processing_kwargs) + + # Call with one of the samples with no associated vision input + plain_text = "lower newer" + image_inputs_nested[0] = [] + video_inputs_nested[0] = [] + text[0] = plain_text + inputs_dict_no_vision = {"text": text, "images": image_inputs_nested, "videos": video_inputs_nested} + inputs_dict_no_vision = {k: v for k, v in inputs_dict_no_vision.items() if k in input_args} + processor(**inputs_dict_no_vision, **processing_kwargs) + # These kwargs-related tests ensure that processors are correctly instantiated. # they need to be applied only if an image_processor exists. From 8bde8d25788fd651daae9cc5d24d44e569dd6e54 Mon Sep 17 00:00:00 2001 From: raushan Date: Tue, 26 Aug 2025 13:09:05 +0200 Subject: [PATCH 13/18] fix copies --- src/transformers/models/smolvlm/image_processing_smolvlm.py | 2 +- src/transformers/models/tvp/image_processing_tvp_fast.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/transformers/models/smolvlm/image_processing_smolvlm.py b/src/transformers/models/smolvlm/image_processing_smolvlm.py index 8edd0ff7deea..ef133b31ed57 100644 --- a/src/transformers/models/smolvlm/image_processing_smolvlm.py +++ b/src/transformers/models/smolvlm/image_processing_smolvlm.py @@ -534,7 +534,7 @@ def pad( For a list of images, for each images, pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width. For each sample in the batch, pads the sample with empty images to the max_number of images per sample in the batch. Optionally returns a pixel mask. Args: - images (`List[List[np.ndarray]]`): + images (`list[list[np.ndarray]]`): List of list of images to pad. Pads to the largest height and width in the batch. constant_values (`float` or `Iterable[float]`, *optional*): The value to use for the padding if `mode` is `"constant"`. diff --git a/src/transformers/models/tvp/image_processing_tvp_fast.py b/src/transformers/models/tvp/image_processing_tvp_fast.py index 3eec7e2d9c3d..90c39ea49fb0 100644 --- a/src/transformers/models/tvp/image_processing_tvp_fast.py +++ b/src/transformers/models/tvp/image_processing_tvp_fast.py @@ -101,9 +101,6 @@ def preprocess( videos: Union[ImageInput, list[ImageInput], list[list[ImageInput]]], **kwargs: Unpack[TvpFastImageProcessorKwargs], ) -> BatchFeature: - """ - Preprocess videos using the fast image processor. - """ return super().preprocess(videos, **kwargs) def _further_process_kwargs( From c5642099e65b9b9e372900619ce0c431f56e44fa Mon Sep 17 00:00:00 2001 From: raushan Date: Tue, 26 Aug 2025 13:42:14 +0200 Subject: [PATCH 14/18] fix after rebase --- .../models/interns1/modeling_interns1.py | 1099 ----------------- .../shieldgemma2/processing_shieldgemma2.py | 34 +- src/transformers/video_utils.py | 2 +- .../models/colpali/test_processing_colpali.py | 4 + .../colqwen2/test_processing_colqwen2.py | 4 + .../got_ocr2/test_processing_got_ocr2.py | 2 +- .../test_processing_grounding_dino.py | 2 +- tests/models/mllama/test_processing_mllama.py | 4 + tests/test_processing_common.py | 2 +- 9 files changed, 39 insertions(+), 1114 deletions(-) delete mode 100644 src/transformers/models/interns1/modeling_interns1.py diff --git a/src/transformers/models/interns1/modeling_interns1.py b/src/transformers/models/interns1/modeling_interns1.py deleted file mode 100644 index 4f69e38c73fd..000000000000 --- a/src/transformers/models/interns1/modeling_interns1.py +++ /dev/null @@ -1,1099 +0,0 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from src/transformers/models/interns1/modular_interns1.py. -# Do NOT edit this file manually as any edits will be overwritten by the generation of -# the file from the modular. If any change should be done, please apply the change to the -# modular_interns1.py file directly. One of our CI enforces this. -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# coding=utf-8 -# Copyright 2025 HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import collections.abc -import copy -from dataclasses import dataclass -from typing import Callable, Optional, Union - -import numpy as np -import torch -import torch.nn as nn - -from ...activations import ACT2FN -from ...cache_utils import Cache -from ...generation import GenerationMixin -from ...integrations import use_kernel_forward_from_hub -from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling -from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...processing_utils import Unpack -from ...utils import ( - ModelOutput, - TransformersKwargs, - auto_docstring, - can_return_tuple, - torch_int, -) -from ...utils.generic import check_model_inputs -from ..auto import AutoModel -from .configuration_interns1 import InternS1Config, InternS1VisionConfig - - -@use_kernel_forward_from_hub("RMSNorm") -class InternS1VisionRMSNorm(nn.Module): - def __init__(self, hidden_size, eps=1e-6): - """ - InternS1VisionRMSNorm is equivalent to T5LayerNorm - """ - super().__init__() - self.weight = nn.Parameter(torch.ones(hidden_size)) - self.variance_epsilon = eps - - def forward(self, hidden_states): - input_dtype = hidden_states.dtype - hidden_states = hidden_states.to(torch.float32) - variance = hidden_states.pow(2).mean(-1, keepdim=True) - hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) - return self.weight * hidden_states.to(input_dtype) - - def extra_repr(self): - return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" - - -def eager_attention_forward( - module: nn.Module, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - attention_mask: Optional[torch.Tensor], - scaling: float, - dropout: float = 0.0, - **kwargs, -): - key_states = key - value_states = value - - attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling - if attention_mask is not None: - causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] - attn_weights = attn_weights + causal_mask - - # No upcasting of the attention weights to float32 in this implementation - attn_weights = nn.functional.softmax(attn_weights, dim=-1) - attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) - attn_output = torch.matmul(attn_weights, value_states) - attn_output = attn_output.transpose(1, 2).contiguous() - - return attn_output, attn_weights - - -class InternS1VisionAttention(nn.Module): - """Attention Class for InternS1 Vision Encoder""" - - def __init__(self, config: InternS1VisionConfig): - super().__init__() - self.config = config - self.embed_dim = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.embed_dim // self.num_heads - if self.head_dim * self.num_heads != self.embed_dim: - raise ValueError( - f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" - f" {self.num_heads})." - ) - self.scale = self.head_dim**-0.5 - self.attention_dropout = config.attention_dropout - proj_dropout = config.projection_dropout - qk_norm = config.use_qk_norm - - # Needed for flash attention - self.is_causal = False - - self.q_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=config.attention_bias) - self.k_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=config.attention_bias) - self.v_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=config.attention_bias) - self.projection_layer = nn.Linear(self.embed_dim, self.embed_dim) - self.projection_dropout = nn.Dropout(proj_dropout) if proj_dropout > 0 else nn.Identity() - - self.q_norm = InternS1VisionRMSNorm(self.embed_dim) if qk_norm else nn.Identity() - self.k_norm = InternS1VisionRMSNorm(self.embed_dim) if qk_norm else nn.Identity() - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - output_attentions: Optional[torch.Tensor] = None, - **kwargs: Unpack[FlashAttentionKwargs], - ): - batch_size, seq_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = self.q_norm(query_states) - key_states = self.k_norm(key_states) - - query_states = query_states.reshape(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) - key_states = key_states.reshape(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) - - attention_interface: Callable = eager_attention_forward - if self.config._attn_implementation != "eager": - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - - attn_output, attn_weights = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask, - dropout=0.0 if not self.training else self.attention_dropout, - scaling=self.scale, - is_causal=False, - **kwargs, - ) - attn_output = attn_output.reshape(batch_size, seq_len, self.embed_dim) - - output = self.projection_layer(attn_output) - output = self.projection_dropout(output) - - return output, attn_weights - - -NORM2FN = {"layer_norm": nn.LayerNorm, "rms_norm": InternS1VisionRMSNorm} - - -@dataclass -@auto_docstring( - custom_intro=""" - Class for outputs of [`InternS1VisionModel`]. - """ -) -class InternS1VisionModelOutputWithPooling(BaseModelOutputWithPooling): - r""" - pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): - Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if - *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token - will be returned. - """ - - -class InternS1VisionPatchEmbeddings(nn.Module): - """ - This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial - `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a - Transformer. - """ - - def __init__(self, config): - super().__init__() - image_size, patch_size = config.image_size, config.patch_size - num_channels, hidden_size = config.num_channels, config.hidden_size - - num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0]) - patch_shape = (image_size[0] // patch_size[0], image_size[1] // patch_size[1]) - self.image_size = image_size - self.patch_size = patch_size - self.num_channels = num_channels - self.num_patches = num_patches - self.patch_shape = patch_shape - - self.projection = nn.Conv2d(num_channels, hidden_size, kernel_size=patch_size, stride=patch_size) - - def forward(self, pixel_values: torch.Tensor) -> tuple[torch.Tensor, tuple[int, int]]: - batch_size, num_channels, height, width = pixel_values.shape - if num_channels != self.num_channels: - raise ValueError( - "Make sure that the channel dimension of the pixel values match with the one set in the configuration." - ) - # Note: more robust - embeddings = self.projection(pixel_values.to(self.projection.weight.dtype)) - patch_height, patch_width = embeddings.shape[2], embeddings.shape[3] - embeddings = embeddings.flatten(2).transpose(1, 2) - - return embeddings, (patch_height, patch_width) - - -# Based on timm implementation, which can be found here: -# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py -class InternS1VisionEmbeddings(nn.Module): - """ - Construct the CLS token, position and patch embeddings. Optionally, also the mask token. - - """ - - def __init__(self, config: InternS1VisionConfig) -> None: - super().__init__() - - self.cls_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size)) - if config.use_mask_token: - self.mask_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size)) - else: - self.mask_token = None - self.patch_embeddings = InternS1VisionPatchEmbeddings(config) - self.patch_size = config.patch_size - self.image_size = ( - config.image_size - if isinstance(config.image_size, collections.abc.Iterable) - else (config.image_size, config.image_size) - ) - num_patches = self.patch_embeddings.num_patches - if config.use_absolute_position_embeddings: - self.position_embeddings = nn.Parameter(torch.zeros(1, num_patches + 1, config.hidden_size)) - else: - self.position_embeddings = None - self.dropout = nn.Dropout(config.hidden_dropout_prob) - - def interpolate_pos_encoding(self, embeddings: torch.Tensor, height: int, width: int) -> torch.Tensor: - """ - This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution - images. This method is also adapted to support torch.jit tracing. - - Adapted from: - - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and - - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 - """ - - num_patches = embeddings.shape[1] - 1 - num_positions = self.position_embeddings.shape[1] - 1 - - # always interpolate when tracing to ensure the exported model works for dynamic input shapes - if not torch.jit.is_tracing() and num_patches == num_positions and height == width: - return self.position_embeddings - - class_pos_embed = self.position_embeddings[:, :1] - patch_pos_embed = self.position_embeddings[:, 1:] - - dim = embeddings.shape[-1] - - new_height = height // self.patch_size[0] - new_width = width // self.patch_size[1] - - sqrt_num_positions = torch_int(num_positions**0.5) - patch_pos_embed = patch_pos_embed.reshape(1, sqrt_num_positions, sqrt_num_positions, dim) - patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2) - - patch_pos_embed = nn.functional.interpolate( - patch_pos_embed, - size=(new_height, new_width), - mode="bicubic", - align_corners=False, - ) - - patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim) - - return torch.cat((class_pos_embed, patch_pos_embed), dim=1) - - def forward( - self, - pixel_values: torch.Tensor, - bool_masked_pos: Optional[torch.BoolTensor] = None, - ) -> torch.Tensor: - _, _, height, width = pixel_values.shape - embeddings, (patch_height, patch_width) = self.patch_embeddings(pixel_values) - batch_size, seq_len, _ = embeddings.size() - - if bool_masked_pos is not None: - mask_tokens = self.mask_token.expand(batch_size, seq_len, -1) - # replace the masked visual tokens by mask_tokens - w = bool_masked_pos.unsqueeze(-1).type_as(mask_tokens) - embeddings = embeddings * (1 - w) + mask_tokens * w - - cls_tokens = self.cls_token.expand(batch_size, -1, -1) - embeddings = torch.cat((cls_tokens, embeddings), dim=1) - - if self.position_embeddings is not None: - embeddings = embeddings + self.interpolate_pos_encoding(embeddings, height, width) - - embeddings = self.dropout(embeddings) - - return embeddings, (patch_height, patch_width) - - -class InternS1VisionMLP(nn.Module): - def __init__(self, config): - super().__init__() - self.config = config - self.activation_fn = ACT2FN[config.hidden_act] - self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size) - self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - hidden_states = self.fc1(hidden_states) - hidden_states = self.activation_fn(hidden_states) - hidden_states = self.fc2(hidden_states) - return hidden_states - - -def drop_path(input: torch.Tensor, drop_prob: float = 0.0, training: bool = False) -> torch.Tensor: - """ - Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). - - Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks, - however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... - See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the - layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the - argument. - """ - if drop_prob == 0.0 or not training: - return input - keep_prob = 1 - drop_prob - shape = (input.shape[0],) + (1,) * (input.ndim - 1) # work with diff dim tensors, not just 2D ConvNets - random_tensor = keep_prob + torch.rand(shape, dtype=input.dtype, device=input.device) - random_tensor.floor_() # binarize - output = input.div(keep_prob) * random_tensor - return output - - -class InternS1DropPath(nn.Module): - """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" - - def __init__(self, drop_prob: Optional[float] = None) -> None: - super().__init__() - self.drop_prob = drop_prob - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - return drop_path(hidden_states, self.drop_prob, self.training) - - def extra_repr(self) -> str: - return f"p={self.drop_prob}" - - -class InternS1VisionLayer(GradientCheckpointingLayer): - """This corresponds to the Block class in the timm implementation.""" - - def __init__(self, config: InternS1VisionConfig) -> None: - super().__init__() - self.chunk_size_feed_forward = config.chunk_size_feed_forward - self.seq_len_dim = 1 - self.attention = InternS1VisionAttention(config) - self.mlp = InternS1VisionMLP(config) - # InternS1 uses different layernorm implementations for different models - self.layernorm_before = NORM2FN[config.norm_type](config.hidden_size, eps=config.layer_norm_eps) - self.layernorm_after = NORM2FN[config.norm_type](config.hidden_size, eps=config.layer_norm_eps) - - init_values = config.layer_scale_init_value - self.lambda_1 = nn.Parameter(init_values * torch.ones(config.hidden_size), requires_grad=True) - self.lambda_2 = nn.Parameter(init_values * torch.ones(config.hidden_size), requires_grad=True) - self.dropout = nn.Dropout(config.hidden_dropout_prob) - - # Note: Compared to InternVL, we have added support for drop_path to facilitate user fine-tuning. - self.drop_path1 = InternS1DropPath(config.drop_path_rate) - self.drop_path2 = InternS1DropPath(config.drop_path_rate) - - def forward( - self, - hidden_states: torch.Tensor, - ) -> Union[tuple[torch.Tensor], tuple[torch.Tensor, torch.Tensor]]: - attention_output, _ = self.attention( - self.layernorm_before(hidden_states), # in InternS1Vision, layernorm is applied before self-attention - ) - - attention_output = self.lambda_1 * attention_output - - # first residual connection - hidden_states = self.drop_path1(attention_output) + hidden_states - - # in InternS1Vision, layernorm is also applied after self-attention - layer_output = self.layernorm_after(hidden_states) - - layer_output = self.mlp(layer_output) - layer_output = self.dropout(layer_output) - - if self.lambda_2 is not None: - layer_output = self.lambda_2 * layer_output - - # second residual connection - layer_output = self.drop_path2(layer_output) + hidden_states - - return layer_output - - -class InternS1VisionEncoder(nn.Module): - def __init__(self, config: InternS1VisionConfig) -> None: - super().__init__() - self.config = config - dpr = np.linspace(0.0, float(config.drop_path_rate), int(config.num_hidden_layers)) - dpr_configs = [] - for idx in range(config.num_hidden_layers): - copy_config = copy.deepcopy(config) - copy_config.drop_path_rate = dpr[idx] - dpr_configs.append(copy_config) - self.layer = nn.ModuleList([InternS1VisionLayer(dpr_configs[idx]) for idx in range(config.num_hidden_layers)]) - self.gradient_checkpointing = False - - @check_model_inputs - def forward( - self, - hidden_states: torch.Tensor, - output_hidden_states: bool = False, - ) -> Union[tuple, BaseModelOutput]: - for i, layer_module in enumerate(self.layer): - hidden_states = layer_module(hidden_states) - - return BaseModelOutput( - last_hidden_state=hidden_states, - ) - - -@auto_docstring -class InternS1VisionPreTrainedModel(PreTrainedModel): - config: InternS1VisionConfig - base_model_prefix = "interns1_vision" - main_input_name = "pixel_values" - supports_gradient_checkpointing = True - _no_split_modules = ["InternS1VisionLayer"] - _supports_sdpa = True - _supports_flash_attn = True - _supports_flex_attn = True - _supports_attention_backend = True - _can_record_outputs = { - "hidden_states": InternS1VisionLayer, - "attentions": InternS1VisionAttention, - } - - def _init_weights(self, module): - """Initialize the weights""" - super()._init_weights(module) - if isinstance(module, InternS1VisionEmbeddings): - module.cls_token.data.zero_() - if module.mask_token is not None: - module.mask_token.data.zero_() - if module.position_embeddings is not None: - module.position_embeddings.data.zero_() - elif isinstance(module, InternS1VisionLayer): - module.lambda_1.data.fill_(self.config.layer_scale_init_value) - module.lambda_2.data.fill_(self.config.layer_scale_init_value) - - -@auto_docstring -class InternS1VisionModel(InternS1VisionPreTrainedModel): - def __init__(self, config: InternS1VisionConfig) -> None: - super().__init__(config) - self.config = config - - self.embeddings = InternS1VisionEmbeddings(config) - self.encoder = InternS1VisionEncoder(config) - - self.layernorm = ( - nn.Identity() if config.use_mean_pooling else nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) - ) - - # Initialize weights and apply final processing - self.post_init() - - def get_input_embeddings(self): - return self.embeddings.patch_embeddings - - @can_return_tuple - @auto_docstring - def forward( - self, - pixel_values: torch.Tensor, - bool_masked_pos: Optional[torch.BoolTensor] = None, - ) -> Union[tuple, InternS1VisionModelOutputWithPooling]: - r""" - bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): - Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). - """ - embedding_output, _ = self.embeddings(pixel_values, bool_masked_pos=bool_masked_pos) - - encoder_outputs = self.encoder( - embedding_output, - ) - sequence_output = encoder_outputs[0] - sequence_output = self.layernorm(sequence_output) - - return InternS1VisionModelOutputWithPooling( - last_hidden_state=sequence_output, - hidden_states=encoder_outputs.hidden_states, - attentions=encoder_outputs.attentions, - ) - - -@auto_docstring -class InternS1PreTrainedModel(PreTrainedModel): - config: InternS1Config - base_model_prefix = "" - supports_gradient_checkpointing = True - _skip_keys_device_placement = "past_key_values" - - _supports_flash_attn = True - _supports_sdpa = True - - _can_compile_fullgraph = True - _supports_flex_attn = True - _supports_attention_backend = True - _can_record_outputs = { - "router_logits": "SparseMoeBlock", - "hidden_states": "DecoderLayer", - "attentions": "Attention", - } - - -class InternS1MultiModalProjector(nn.Module): - def __init__(self, config: InternS1Config): - super().__init__() - self.layer_norm = nn.LayerNorm(config.vision_config.hidden_size * int(1 / config.downsample_ratio) ** 2) - self.linear_1 = nn.Linear( - config.vision_config.hidden_size * int(1 / config.downsample_ratio) ** 2, config.text_config.hidden_size - ) - self.act = ACT2FN[config.projector_hidden_act] - self.linear_2 = nn.Linear(config.text_config.hidden_size, config.text_config.hidden_size) - - def forward(self, image_features): - hidden_states = self.layer_norm(image_features) - hidden_states = self.linear_1(hidden_states) - hidden_states = self.act(hidden_states) - hidden_states = self.linear_2(hidden_states) - return hidden_states - - -@dataclass -@auto_docstring( - custom_intro=""" - Base class for InternS1 outputs, with hidden states and attentions. - """ -) -class InternS1ModelOutputWithPast(BaseModelOutputWithPast): - r""" - router_logits (`tuple(torch.FloatTensor)`, *optional*, returned when `output_router_probs=True` and `config.add_router_probs=True` is passed or when `config.output_router_probs=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, sequence_length, num_experts)`. - - Raw router logtis (post-softmax) that are computed by MoE routers, these terms are used to compute the auxiliary - loss for Mixture of Experts models. - """ - - image_hidden_states: Optional[torch.FloatTensor] = None - - router_logits: Optional[tuple[torch.FloatTensor]] = None - - -@auto_docstring( - custom_intro=""" - The InternS1 model which consists of a vision backbone and a language model, without a language modeling head. - """ -) -class InternS1Model(InternS1PreTrainedModel): - _checkpoint_conversion_mapping = {} - - def __init__(self, config: InternS1Config): - super().__init__(config) - self.vision_tower = AutoModel.from_config(config.vision_config) - - self.multi_modal_projector = InternS1MultiModalProjector(config) - self.language_model = AutoModel.from_config(config.text_config) - self.post_init() - - def get_input_embeddings(self): - return self.language_model.get_input_embeddings() - - def set_input_embeddings(self, value): - self.language_model.set_input_embeddings(value) - - def set_decoder(self, decoder): - self.language_model = decoder - - def get_decoder(self): - return self.language_model - - def get_image_features( - self, - pixel_values: torch.FloatTensor, - vision_feature_layer: Optional[Union[int, list[int]]] = None, - vision_feature_select_strategy: Optional[str] = None, - **kwargs, - ): - """ - Obtains image last hidden states from the vision tower and apply multimodal projection. - - Args: - pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) - The tensors corresponding to the input images. - vision_feature_layer (`int` or `list[int]`): - Layer index or list of layer indices to extract features from. - Returns: - vision_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`. - """ - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - pixel_values = pixel_values.to(dtype=self.dtype) # fp16 compatibility - - downsample_ratio = self.config.downsample_ratio - if vision_feature_layer == -1: - vision_features = self.vision_tower(pixel_values=pixel_values).last_hidden_state - else: - vision_features = self.vision_model(pixel_values=pixel_values).hidden_states[vision_feature_layer] - if vision_feature_select_strategy == "default": - vision_features = vision_features[:, 1:, :] - - # Calculate dimensions based on vision features - channels = vision_features.shape[1] - feature_size = int(channels**0.5) - batch_size = vision_features.shape[0] - - # Reshape tensor to spatial dimensions - vision_features = vision_features.reshape(batch_size, feature_size, feature_size, -1) - - # Apply downsampling using pixel shuffle - vision_features = self.pixel_shuffle(vision_features, scale_factor=downsample_ratio) - - # Reshape tensor to prepare for projection - vision_features = vision_features.reshape(batch_size, -1, vision_features.shape[-1]) - - # Project features through multi-modal projector - vision_features = self.multi_modal_projector(vision_features) - return vision_features - - def get_placeholder_mask( - self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor - ): - """ - Obtains multimodal placeholdr mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is - equal to the length of multimodal features. If the lengths are different, an error is raised. - """ - if input_ids is None: - special_image_mask = inputs_embeds == self.get_input_embeddings()( - torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device) - ) - special_image_mask = special_image_mask.all(-1) - else: - special_image_mask = input_ids == self.config.image_token_id - - n_image_tokens = special_image_mask.sum() - special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) - n_image_features = image_features.shape[0] * image_features.shape[1] - if inputs_embeds[special_image_mask].numel() != image_features.numel(): - raise ValueError( - f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {n_image_features}" - ) - return special_image_mask - - @check_model_inputs - @auto_docstring - def forward( - self, - input_ids: torch.LongTensor = None, - pixel_values: torch.FloatTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[Cache] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - vision_feature_layer: Optional[Union[int, list[int]]] = None, - vision_feature_select_strategy: Optional[str] = None, - use_cache: Optional[bool] = None, - cache_position: Optional[torch.LongTensor] = None, - **kwargs: Unpack[FlashAttentionKwargs], - ) -> InternS1ModelOutputWithPast: - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - - if (input_ids is None) ^ (inputs_embeds is not None): - raise ValueError("You must specify exactly one of input_ids or inputs_embeds") - - if inputs_embeds is None: - inputs_embeds = self.get_input_embeddings()(input_ids) - - if pixel_values is not None: - image_features = self.get_image_features( - pixel_values=pixel_values, - vision_feature_layer=vision_feature_layer, - vision_feature_select_strategy=vision_feature_select_strategy, - ) - image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) - special_image_mask = self.get_placeholder_mask( - input_ids, inputs_embeds=inputs_embeds, image_features=image_features - ) - inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features) - - outputs = self.language_model( - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - use_cache=use_cache, - cache_position=cache_position, - **kwargs, - ) - - return InternS1ModelOutputWithPast( - last_hidden_state=outputs.last_hidden_state, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - router_logits=outputs.router_logits, - image_hidden_states=image_features if pixel_values is not None else None, - ) - - def pixel_shuffle(self, vision_features: torch.Tensor, scale_factor: float = 0.5): - """Perform pixel shuffle downsampling on vision features. - - Args: - vision_features (`torch.Tensor`): - Input tensor of shape (batch_size, width, height, channels). - scale_factor (`float`, *optional*, defaults to `0.5`): - Factor by which to downsample. Default is 0.5, which halves the dimensions. - - Returns: - vision_features (`torch.Tensor`): - Downsampled tensor of shape (batch_size, height*scale_factor, width*scale_factor, channels/(scale_factor^2)). - """ - batch_size, width, height, channels = vision_features.size() - - if height % scale_factor != 0 or width % scale_factor != 0: - raise ValueError("Height and width must be divisible by scale_factor for proper downsampling.") - - # Reshape to allow downsampling - vision_features = vision_features.view( - batch_size, width, int(height * scale_factor), int(channels / scale_factor) - ) - # Permute dimensions to align downsampled axis correctly - vision_features = vision_features.permute(0, 2, 1, 3).contiguous() - - # Reshape to achieve final downsampled dimensions - vision_features = vision_features.view( - batch_size, int(height * scale_factor), int(width * scale_factor), int(channels / (scale_factor**2)) - ) - - # Swap height and width back for proper orientation - vision_features = vision_features.permute(0, 2, 1, 3).contiguous() - - return vision_features - - -@dataclass -@auto_docstring( - custom_intro=""" - Base class for InternS1 causal language model (or autoregressive) outputs. - """ -) -class InternS1CausalLMOutputWithPast(ModelOutput): - r""" - aux_loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided): - aux_loss for the sparse modules. - router_logits (`tuple(torch.FloatTensor)`, *optional*, returned when `output_router_probs=True` and `config.add_router_probs=True` is passed or when `config.output_router_probs=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, sequence_length, num_experts)`. - - Raw router logtis (post-softmax) that are computed by MoE routers, these terms are used to compute the auxiliary - loss for Mixture of Experts models. - """ - - loss: Optional[torch.FloatTensor] = None - logits: Optional[torch.FloatTensor] = None - past_key_values: Optional[list[torch.FloatTensor]] = None - hidden_states: Optional[tuple[torch.FloatTensor]] = None - attentions: Optional[tuple[torch.FloatTensor]] = None - image_hidden_states: Optional[torch.FloatTensor] = None - - aux_loss: Optional[torch.FloatTensor] = None - router_logits: Optional[tuple[torch.FloatTensor]] = None - - -def load_balancing_loss_func( - gate_logits: Union[torch.Tensor, tuple[torch.Tensor], None], - num_experts: Optional[int] = None, - top_k=2, - attention_mask: Optional[torch.Tensor] = None, -) -> Union[torch.Tensor, int]: - r""" - Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch. - - See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss - function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between - experts is too unbalanced. - - Args: - gate_logits: - Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of - shape [batch_size X sequence_length, num_experts]. - num_experts: - Number of experts - top_k: - The number of experts to route per-token, can be also interpreted as the `top-k` routing - parameter. - attention_mask (`torch.Tensor`, *optional*): - The attention_mask used in forward function - shape [batch_size X sequence_length] if not None. - - Returns: - The auxiliary loss. - """ - if gate_logits is None or not isinstance(gate_logits, tuple): - return 0 - - if isinstance(gate_logits, tuple): - compute_device = gate_logits[0].device - concatenated_gate_logits = torch.cat([layer_gate.to(compute_device) for layer_gate in gate_logits], dim=0) - - routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-1) - - _, selected_experts = torch.topk(routing_weights, top_k, dim=-1) - - expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts) - - if attention_mask is None: - # Compute the percentage of tokens routed to each experts - tokens_per_expert = torch.mean(expert_mask.float(), dim=0) - - # Compute the average probability of routing to these experts - router_prob_per_expert = torch.mean(routing_weights, dim=0) - else: - batch_size, sequence_length = attention_mask.shape - num_hidden_layers = concatenated_gate_logits.shape[0] // (batch_size * sequence_length) - - # Compute the mask that masks all padding tokens as 0 with the same shape of expert_mask - expert_attention_mask = ( - attention_mask[None, :, :, None, None] - .expand((num_hidden_layers, batch_size, sequence_length, top_k, num_experts)) - .reshape(-1, top_k, num_experts) - .to(compute_device) - ) - - # Compute the percentage of tokens routed to each experts - tokens_per_expert = torch.sum(expert_mask.float() * expert_attention_mask, dim=0) / torch.sum( - expert_attention_mask, dim=0 - ) - - # Compute the mask that masks all padding tokens as 0 with the same shape of tokens_per_expert - router_per_expert_attention_mask = ( - attention_mask[None, :, :, None] - .expand((num_hidden_layers, batch_size, sequence_length, num_experts)) - .reshape(-1, num_experts) - .to(compute_device) - ) - - # Compute the average probability of routing to these experts - router_prob_per_expert = torch.sum(routing_weights * router_per_expert_attention_mask, dim=0) / torch.sum( - router_per_expert_attention_mask, dim=0 - ) - - overall_loss = torch.sum(tokens_per_expert * router_prob_per_expert.unsqueeze(0)) - return overall_loss * num_experts - - -@auto_docstring( - custom_intro=""" - The INTERNS1 model which consists of a vision backbone and a language model. - """ -) -class InternS1ForConditionalGeneration(InternS1PreTrainedModel, GenerationMixin): - _checkpoint_conversion_mapping = {} - _tied_weights_keys = ["lm_head.weight"] - - def __init__(self, config: InternS1Config): - super().__init__(config) - self.model = InternS1Model(config) - self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False) - - self.post_init() - - def get_input_embeddings(self): - return self.model.get_input_embeddings() - - def set_input_embeddings(self, value): - self.model.set_input_embeddings(value) - - def get_output_embeddings(self) -> nn.Module: - return self.lm_head - - def set_decoder(self, decoder): - self.model.set_decoder(decoder) - - def get_decoder(self): - return self.model.get_decoder() - - def get_image_features( - self, - pixel_values: torch.FloatTensor, - vision_feature_layer: Optional[Union[int, list[int]]] = None, - vision_feature_select_strategy: Optional[str] = None, - **kwargs, - ): - return self.model.get_image_features( - pixel_values=pixel_values, - vision_feature_layer=vision_feature_layer, - vision_feature_select_strategy=vision_feature_select_strategy, - **kwargs, - ) - - # Make modules available through conditional class for BC - @property - def language_model(self): - return self.model.language_model - - @property - def vision_tower(self): - return self.model.vision_tower - - @property - def multi_modal_projector(self): - return self.model.multi_modal_projector - - @can_return_tuple - @auto_docstring - def forward( - self, - input_ids: torch.LongTensor = None, - pixel_values: torch.FloatTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[Cache] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - vision_feature_layer: Optional[Union[int, list[int]]] = None, - vision_feature_select_strategy: Optional[str] = None, - labels: Optional[torch.LongTensor] = None, - use_cache: Optional[bool] = None, - cache_position: Optional[torch.LongTensor] = None, - logits_to_keep: Union[int, torch.Tensor] = 0, - **kwargs: Unpack[TransformersKwargs], - ) -> Union[tuple, InternS1CausalLMOutputWithPast]: - r""" - Example: - - ```python - >>> import torch - >>> from transformers import AutoProcessor, AutoModelForImageTextToText - - >>> torch_device = "cuda" - >>> processor = AutoProcessor.from_pretrained("internlm/Intern-S1") - >>> model = AutoModelForImageTextToText.from_pretrained( - ... "internlm/Intern-S1", torch_dtype=torch.bfloat16, device_map=torch_device - ... ) - - >>> messages = [ - ... { - ... "role": "user", - ... "content": [ - ... { - ... "type": "image", - ... "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg", - ... }, - ... { - ... "type": "image", - ... "url": "https://thumbs.dreamstime.com/b/golden-gate-bridge-san-francisco-purple-flowers-california-echium-candicans-36805947.jpg", - ... }, - ... {"type": "text", "text": "These images depict two different landmarks. Can you identify them?"}, - ... ], - ... }, - ... ] - - >>> inputs = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt").to(torch_device) - >>> generate_ids = model.generate(**inputs, max_new_tokens=200) - >>> print(processor.decode(generate_ids[0, inputs["input_ids"].shape[1] :], skip_special_tokens=True)) - ```""" - vision_feature_layer = ( - vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer - ) - vision_feature_select_strategy = ( - vision_feature_select_strategy - if vision_feature_select_strategy is not None - else self.config.vision_feature_select_strategy - ) - - outputs = self.model( - input_ids=input_ids, - pixel_values=pixel_values, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - vision_feature_layer=vision_feature_layer, - vision_feature_select_strategy=vision_feature_select_strategy, - use_cache=use_cache, - cache_position=cache_position, - **kwargs, - ) - - hidden_states = outputs.last_hidden_state - # Only compute necessary logits, and do not upcast them to float if we are not computing the loss - slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep - logits = self.lm_head(hidden_states[:, slice_indices, :]) - - loss = None - if labels is not None: - loss = self.loss_function( - logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size, **kwargs - ) - - aux_loss = None - if hasattr(outputs, "router_logits") and labels is not None: - aux_loss = load_balancing_loss_func( - outputs.router_logits, - self.config.text_config.num_experts, - self.config.text_config.num_experts_per_tok, - attention_mask, - ) - loss += self.config.text_config.router_aux_loss_coef * aux_loss.to(loss.device) - - return InternS1CausalLMOutputWithPast( - loss=loss, - aux_loss=aux_loss, - logits=logits, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - router_logits=outputs.router_logits if hasattr(outputs, "router_logits") else None, - image_hidden_states=outputs.image_hidden_states, - ) - - def prepare_inputs_for_generation( - self, - input_ids, - past_key_values=None, - inputs_embeds=None, - pixel_values=None, - attention_mask=None, - cache_position=None, - logits_to_keep=None, - **kwargs, - ): - # Overwritten -- in specific circumstances we don't want to forward image inputs to the model - - model_inputs = super().prepare_inputs_for_generation( - input_ids, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - attention_mask=attention_mask, - cache_position=cache_position, - logits_to_keep=logits_to_keep, - **kwargs, - ) - - if cache_position[0] == 0: - # If we're in cached decoding stage, pixel values should be None because input ids do not contain special image token anymore - # Otherwise we need pixel values to be passed to model - model_inputs["pixel_values"] = pixel_values - - return model_inputs - - -__all__ = [ - "InternS1VisionPreTrainedModel", - "InternS1VisionModel", - "InternS1PreTrainedModel", - "InternS1Model", - "InternS1ForConditionalGeneration", -] diff --git a/src/transformers/models/shieldgemma2/processing_shieldgemma2.py b/src/transformers/models/shieldgemma2/processing_shieldgemma2.py index 993fce03095b..412f231bef91 100644 --- a/src/transformers/models/shieldgemma2/processing_shieldgemma2.py +++ b/src/transformers/models/shieldgemma2/processing_shieldgemma2.py @@ -160,17 +160,29 @@ def __call__( raise ValueError(f"SheildGemma can process at most one image per sample, but got {len(img)} images") for policy in policies: - messages.append( - [ - { - "role": "user", - "content": [ - {"type": "image"}, - {"type": "text", "text": policy_definitions[policy]}, - ], - } - ] - ) + if img: + messages.append( + [ + { + "role": "user", + "content": [ + {"type": "image"}, + {"type": "text", "text": policy_definitions[policy]}, + ], + } + ] + ) + else: + messages.append( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": policy_definitions[policy]}, + ], + } + ] + ) expanded_images.append(img) text = self.apply_chat_template(messages, tokenize=False) diff --git a/src/transformers/video_utils.py b/src/transformers/video_utils.py index bfbe2c7056c2..b2b1b5dc1ea5 100644 --- a/src/transformers/video_utils.py +++ b/src/transformers/video_utils.py @@ -202,7 +202,7 @@ def make_batched_videos(videos) -> list[Union["np.ndarray", "torch.Tensor", "URL for item in videos: if isinstance(item, str) or is_valid_video(item): flat_videos_list.append(item) - elif isinstance(item, list): + elif isinstance(item, list) and item: flat_videos_list.extend(make_batched_videos(item)) flat_videos_list = convert_pil_frames_to_video(flat_videos_list) diff --git a/tests/models/colpali/test_processing_colpali.py b/tests/models/colpali/test_processing_colpali.py index 35a74a11a00b..221836db8423 100644 --- a/tests/models/colpali/test_processing_colpali.py +++ b/tests/models/colpali/test_processing_colpali.py @@ -282,3 +282,7 @@ def test_model_input_names(self): inputs = processor(images=image_input) self.assertSetEqual(set(inputs.keys()), set(processor.model_input_names)) + + @unittest.skip("ColPali can't process text+image inputs at the same time") + def test_processor_text_has_no_visual(self): + pass diff --git a/tests/models/colqwen2/test_processing_colqwen2.py b/tests/models/colqwen2/test_processing_colqwen2.py index 560f87a56d91..7346c0d5079c 100644 --- a/tests/models/colqwen2/test_processing_colqwen2.py +++ b/tests/models/colqwen2/test_processing_colqwen2.py @@ -281,3 +281,7 @@ def test_model_input_names(self): inputs = processor(images=image_input) self.assertSetEqual(set(inputs.keys()), set(processor.model_input_names)) + + @unittest.skip("ColPali can't process text+image inputs at the same time") + def test_processor_text_has_no_visual(self): + pass diff --git a/tests/models/got_ocr2/test_processing_got_ocr2.py b/tests/models/got_ocr2/test_processing_got_ocr2.py index b97f31a670a4..3ae70f27e016 100644 --- a/tests/models/got_ocr2/test_processing_got_ocr2.py +++ b/tests/models/got_ocr2/test_processing_got_ocr2.py @@ -83,7 +83,7 @@ def test_processor_text_has_no_visual(self): # Overwritten: requires `multi_page` kwarg to process nested vision inputs processor = self.get_processor() - text = self.prepare_text_inputs(batch_size=3, modality="image") + text = self.prepare_text_inputs(batch_size=3, modalities="image") image_inputs = self.prepare_image_inputs(batch_size=3) processing_kwargs = {"return_tensors": "pt", "padding": True, "multi_page": True} diff --git a/tests/models/grounding_dino/test_processing_grounding_dino.py b/tests/models/grounding_dino/test_processing_grounding_dino.py index 3720f9a40fdf..f8d953bc9138 100644 --- a/tests/models/grounding_dino/test_processing_grounding_dino.py +++ b/tests/models/grounding_dino/test_processing_grounding_dino.py @@ -285,7 +285,7 @@ def test_processor_text_has_no_visual(self): # Overwritten: text inputs have to be nested as well processor = self.get_processor() - text = self.prepare_text_inputs(batch_size=3, modality="image") + text = self.prepare_text_inputs(batch_size=3, modalities="image") image_inputs = self.prepare_image_inputs(batch_size=3) processing_kwargs = {"return_tensors": "pt", "padding": True} diff --git a/tests/models/mllama/test_processing_mllama.py b/tests/models/mllama/test_processing_mllama.py index 7eaf84491c85..b95b12358718 100644 --- a/tests/models/mllama/test_processing_mllama.py +++ b/tests/models/mllama/test_processing_mllama.py @@ -395,3 +395,7 @@ def test_special_mm_token_truncation(self): padding=True, max_length=3, ) + + @unittest.skip("Mllama can't process inouts with no image ttogether with multimodal inputs") + def test_processor_text_has_no_visual(self): + pass diff --git a/tests/test_processing_common.py b/tests/test_processing_common.py index b9345c451597..d2fc336e05f7 100644 --- a/tests/test_processing_common.py +++ b/tests/test_processing_common.py @@ -243,7 +243,7 @@ def test_processor_text_has_no_visual(self): if not ("text" in input_args and ("images" in input_args and "videos" in input_args)): self.skipTest(f"{self.processor_class} doesn't support several vision modalities with text.") - text = self.prepare_text_inputs(batch_size=3, modality="image") + text = self.prepare_text_inputs(batch_size=3, modalities=["image", "video"]) image_inputs = self.prepare_image_inputs(batch_size=3) video_inputs = self.prepare_video_inputs(batch_size=3) inputs_dict = {"text": text, "images": image_inputs, "videos": video_inputs} From 8e5fab9c2df7215967f38d08634bacaa93eda4ee Mon Sep 17 00:00:00 2001 From: raushan Date: Tue, 26 Aug 2025 15:53:35 +0200 Subject: [PATCH 15/18] make the test cover chat templates --- src/transformers/processing_utils.py | 8 +-- tests/models/fuyu/test_processing_fuyu.py | 22 +++++++ .../got_ocr2/test_processing_got_ocr2.py | 8 ++- .../test_processing_grounding_dino.py | 11 +++- .../test_processing_shieldgemma2.py | 22 +++++++ tests/test_processing_common.py | 58 +++++++++++++++++-- 6 files changed, 117 insertions(+), 12 deletions(-) diff --git a/src/transformers/processing_utils.py b/src/transformers/processing_utils.py index 980de02f5ff9..17ae36c84655 100644 --- a/src/transformers/processing_utils.py +++ b/src/transformers/processing_utils.py @@ -1603,12 +1603,12 @@ def apply_chat_template( if "do_sample_frames" not in kwargs and ("fps" in kwargs or "num_frames" in kwargs): kwargs["do_sample_frames"] = True - flat_images = [im for im_list in batch_images for im in im_list] - flat_videos = [vid for vid_list in batch_videos for vid in vid_list] + images_exist = any(im for im_list in batch_images for im in im_list) + videos_exist = any(vid for vid_list in batch_videos for vid in vid_list) out = self( text=prompt, - images=batch_images if flat_images else None, - videos=batch_videos if flat_videos else None, + images=batch_images if images_exist else None, + videos=batch_videos if videos_exist else None, audio=batch_audios if batch_audios else None, **kwargs, ) diff --git a/tests/models/fuyu/test_processing_fuyu.py b/tests/models/fuyu/test_processing_fuyu.py index 6f38e9a0e81b..8ed9b20d168f 100644 --- a/tests/models/fuyu/test_processing_fuyu.py +++ b/tests/models/fuyu/test_processing_fuyu.py @@ -347,6 +347,28 @@ def test_unstructured_kwargs_batched(self): self.assertEqual(len(inputs["input_ids"][0]), 7) + def test_processor_text_has_no_visual(self): + # Overwritten: Fuyu has a complicated processing so we don't check id values + processor = self.get_processor() + + text = self.prepare_text_inputs(batch_size=3, modalities="image") + image_inputs = self.prepare_image_inputs(batch_size=3) + processing_kwargs = {"return_tensors": "pt", "padding": True, "multi_page": True} + + # Call with nested list of vision inputs + image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] + inputs_dict_nested = {"text": text, "images": image_inputs_nested} + inputs = processor(**inputs_dict_nested, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs) + + # Call with one of the samples with no associated vision input + plain_text = "lower newer" + image_inputs_nested[0] = [] + text[0] = plain_text + inputs_dict_no_vision = {"text": text, "images": image_inputs_nested} + inputs_nested = processor(**inputs_dict_no_vision, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs_nested) + @require_torch class TestImageTextProcessingUtils(unittest.TestCase): diff --git a/tests/models/got_ocr2/test_processing_got_ocr2.py b/tests/models/got_ocr2/test_processing_got_ocr2.py index 3ae70f27e016..ffa0f97cd4e4 100644 --- a/tests/models/got_ocr2/test_processing_got_ocr2.py +++ b/tests/models/got_ocr2/test_processing_got_ocr2.py @@ -90,11 +90,15 @@ def test_processor_text_has_no_visual(self): # Call with nested list of vision inputs image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] inputs_dict_nested = {"text": text, "images": image_inputs_nested} - processor(**inputs_dict_nested, **processing_kwargs) + inputs = processor(**inputs_dict_nested, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs) # Call with one of the samples with no associated vision input plain_text = "lower newer" image_inputs_nested[0] = [] text[0] = plain_text inputs_dict_no_vision = {"text": text, "images": image_inputs_nested} - processor(**inputs_dict_no_vision, **processing_kwargs) + inputs_nested = processor(**inputs_dict_no_vision, **processing_kwargs) + self.assertListEqual( + inputs[self.text_input_name][1:].tolist(), inputs_nested[self.text_input_name][1:].tolist() + ) diff --git a/tests/models/grounding_dino/test_processing_grounding_dino.py b/tests/models/grounding_dino/test_processing_grounding_dino.py index f8d953bc9138..1c007dc2d0b4 100644 --- a/tests/models/grounding_dino/test_processing_grounding_dino.py +++ b/tests/models/grounding_dino/test_processing_grounding_dino.py @@ -292,11 +292,18 @@ def test_processor_text_has_no_visual(self): # Call with nested list of vision inputs image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] inputs_dict_nested = {"text": text, "images": image_inputs_nested} - processor(**inputs_dict_nested, **processing_kwargs) + inputs = processor(**inputs_dict_nested, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs) # Call with one of the samples with no associated vision input plain_text = ["lower newer"] image_inputs_nested[0] = [] text[0] = plain_text inputs_dict_no_vision = {"text": text, "images": image_inputs_nested} - processor(**inputs_dict_no_vision, **processing_kwargs) + inputs_nested = processor(**inputs_dict_no_vision, **processing_kwargs) + + # Check that text samples are same and are expanded with placeholder tokens correctly. First sample + # has no vision input associated, so we skip it and check it has no vision + self.assertListEqual( + inputs[self.text_input_name][1:].tolist(), inputs_nested[self.text_input_name][1:].tolist() + ) diff --git a/tests/models/shieldgemma2/test_processing_shieldgemma2.py b/tests/models/shieldgemma2/test_processing_shieldgemma2.py index 0bbe17e65607..b9f8b58ad3f6 100644 --- a/tests/models/shieldgemma2/test_processing_shieldgemma2.py +++ b/tests/models/shieldgemma2/test_processing_shieldgemma2.py @@ -210,3 +210,25 @@ def test_kwargs_overrides_default_image_processor_kwargs(self): @unittest.skip("ShieldGemma requires images in input, and fails in text-only processing") def test_apply_chat_template_assistant_mask(self): pass + + def test_processor_text_has_no_visual(self): + # Overwritten: Shieldgemma has a complicated processing so we don't check id values + processor = self.get_processor() + + text = self.prepare_text_inputs(batch_size=3, modalities="image") + image_inputs = self.prepare_image_inputs(batch_size=3) + processing_kwargs = {"return_tensors": "pt", "padding": True, "multi_page": True} + + # Call with nested list of vision inputs + image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] + inputs_dict_nested = {"text": text, "images": image_inputs_nested} + inputs = processor(**inputs_dict_nested, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs) + + # Call with one of the samples with no associated vision input + plain_text = "lower newer" + image_inputs_nested[0] = [] + text[0] = plain_text + inputs_dict_no_vision = {"text": text, "images": image_inputs_nested} + inputs_nested = processor(**inputs_dict_no_vision, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs_nested) diff --git a/tests/test_processing_common.py b/tests/test_processing_common.py index d2fc336e05f7..71cd21157e41 100644 --- a/tests/test_processing_common.py +++ b/tests/test_processing_common.py @@ -236,6 +236,10 @@ def test_model_input_names(self): self.assertSetEqual(set(inputs.keys()), set(processor.model_input_names)) def test_processor_text_has_no_visual(self): + """ + Tests that multimodal models can process batch of inputs where samples can + be with images/videos or without. See https://github.com/huggingface/transformers/issues/40263 + """ processor = self.get_processor() call_signature = inspect.signature(processor.__call__) input_args = [param.name for param in call_signature.parameters.values()] @@ -243,6 +247,8 @@ def test_processor_text_has_no_visual(self): if not ("text" in input_args and ("images" in input_args and "videos" in input_args)): self.skipTest(f"{self.processor_class} doesn't support several vision modalities with text.") + # Prepare inputs and filter by input signature. Make sure to use a high batch size, we'll set some + # samples to text-only later text = self.prepare_text_inputs(batch_size=3, modalities=["image", "video"]) image_inputs = self.prepare_image_inputs(batch_size=3) video_inputs = self.prepare_video_inputs(batch_size=3) @@ -253,21 +259,65 @@ def test_processor_text_has_no_visual(self): if "videos" in inputs_dict: processing_kwargs["do_sample_frames"] = False - # Call processor with all inputs first. Use nested input type which is the format supported by all multimodal processors + # Firts call processor with all inputs and use nested input type, which is the format supported by all multimodal processors image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] video_inputs_nested = [[video] for video in video_inputs] inputs_dict_nested = {"text": text, "images": image_inputs_nested, "videos": video_inputs_nested} inputs_dict_nested = {k: v for k, v in inputs_dict_nested.items() if k in input_args} - processor(**inputs_dict_nested, **processing_kwargs) + inputs = processor(**inputs_dict_nested, **processing_kwargs) + self.assertTrue(self.text_input_name in inputs) - # Call with one of the samples with no associated vision input + # Now call with one of the samples with no associated vision input. Let's set the first input to be a plain text + # with no placeholder tokens and no images/videos. The final format would be `images = [[], [image2], [image3]]` plain_text = "lower newer" image_inputs_nested[0] = [] video_inputs_nested[0] = [] text[0] = plain_text inputs_dict_no_vision = {"text": text, "images": image_inputs_nested, "videos": video_inputs_nested} inputs_dict_no_vision = {k: v for k, v in inputs_dict_no_vision.items() if k in input_args} - processor(**inputs_dict_no_vision, **processing_kwargs) + inputs_nested = processor(**inputs_dict_no_vision, **processing_kwargs) + + # Check that text samples are same and are expanded with placeholder tokens correctly. First sample + # has no vision input associated, so we skip it and check it has no vision + self.assertListEqual( + inputs[self.text_input_name][1:].tolist(), inputs_nested[self.text_input_name][1:].tolist() + ) + + # Now test if we can apply chat templates with no vision inputs in one of the samples + # NOTE: we don't skip the test as we want the above to be checked even if process has to chat template + if processor.chat_template is not None: + messages = [ + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is the capital of France?"}, + ], + }, + ], + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is the capital of France?"}, + { + "type": "image", + "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/coco_sample.png", + }, + ], + }, + ], + ] + + inputs_chat_template = processor.apply_chat_template( + messages, + add_generation_prompt=False, + tokenize=True, + return_dict=True, + return_tensors="pt", + padding=True, + ) + self.assertTrue(self.text_input_name in inputs_chat_template) # These kwargs-related tests ensure that processors are correctly instantiated. # they need to be applied only if an image_processor exists. From 2b342df8de16b8225fcbb15c0906ff3aeda4f8fb Mon Sep 17 00:00:00 2001 From: raushan Date: Tue, 26 Aug 2025 16:26:05 +0200 Subject: [PATCH 16/18] sjip udop, no point in fixing it --- tests/models/udop/test_processing_udop.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/models/udop/test_processing_udop.py b/tests/models/udop/test_processing_udop.py index de1fa89a7ebf..78cb2c36b9f3 100644 --- a/tests/models/udop/test_processing_udop.py +++ b/tests/models/udop/test_processing_udop.py @@ -196,6 +196,10 @@ def preprocess_data(examples): self.assertEqual(len(train_data["pixel_values"]), len(train_data["input_ids"])) + @unittest.skip("We will not support batch input with and without images for UDOP!") + def test_processor_text_has_no_visual(self): + pass + # different use cases tests @require_sentencepiece From e71bef35250e9547cb5df78566f48a7f71c38aae Mon Sep 17 00:00:00 2001 From: raushan Date: Mon, 1 Sep 2025 11:39:01 +0200 Subject: [PATCH 17/18] fix after rebase --- src/transformers/processing_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/processing_utils.py b/src/transformers/processing_utils.py index 3e26871f1bf2..abb8d4476af7 100644 --- a/src/transformers/processing_utils.py +++ b/src/transformers/processing_utils.py @@ -1634,8 +1634,8 @@ def apply_chat_template( ): kwargs["do_sample_frames"] = True - images_exist = any(im for im_list in batch_images for im in im_list) - videos_exist = any(vid for vid_list in batch_videos for vid in vid_list) + images_exist = any(len(im) > 0 for im_list in batch_images for im in im_list) + videos_exist = any(len(vid) > 0 for vid_list in batch_videos for vid in vid_list) out = self( text=prompt, images=batch_images if images_exist else None, From 8e4c67a96ed0dc1447805113cee63925d07e71dc Mon Sep 17 00:00:00 2001 From: raushan Date: Mon, 1 Sep 2025 12:47:00 +0200 Subject: [PATCH 18/18] fix a few more tests --- .../models/fuyu/image_processing_fuyu.py | 21 +++++++++++++++++-- src/transformers/video_utils.py | 2 +- tests/models/mllama/test_processing_mllama.py | 2 -- .../models/smolvlm/test_processing_smolvlm.py | 2 -- tests/test_processing_common.py | 2 +- 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/fuyu/image_processing_fuyu.py b/src/transformers/models/fuyu/image_processing_fuyu.py index 37c2cd19e38d..29af98ed5072 100644 --- a/src/transformers/models/fuyu/image_processing_fuyu.py +++ b/src/transformers/models/fuyu/image_processing_fuyu.py @@ -27,11 +27,13 @@ ) from ...image_utils import ( ChannelDimension, + ImageInput, PILImageResampling, get_image_size, infer_channel_dimension_format, is_scaled_image, - make_nested_list_of_images, + is_valid_image, + make_list_of_images, to_numpy_array, validate_preprocess_arguments, ) @@ -53,6 +55,21 @@ logger = logging.get_logger(__name__) +def make_list_of_list_of_images( + images: Union[list[list[ImageInput]], list[ImageInput], ImageInput], +) -> list[list[ImageInput]]: + if is_valid_image(images): + return [[images]] + + if isinstance(images, list) and all(isinstance(image, list) for image in images): + return images + + if isinstance(images, list): + return [make_list_of_images(image) for image in images] + + raise ValueError("images must be a list of list of images or a list of images or an image.") + + class FuyuBatchFeature(BatchFeature): """ BatchFeature class for Fuyu image processor and processor. @@ -430,7 +447,7 @@ def preprocess( if isinstance(images, list) and any(isinstance(elem, list) and len(elem) >= 2 for elem in images): raise ValueError("Multiple images for a single sample are not yet supported.") - batch_images = make_nested_list_of_images(images) + batch_images = make_list_of_list_of_images(images) validate_preprocess_arguments( do_rescale=do_rescale, diff --git a/src/transformers/video_utils.py b/src/transformers/video_utils.py index b2b1b5dc1ea5..e03cda179d3c 100644 --- a/src/transformers/video_utils.py +++ b/src/transformers/video_utils.py @@ -181,7 +181,7 @@ def make_batched_videos(videos) -> list[Union["np.ndarray", "torch.Tensor", "URL """ # Early exit for deeply nested list of image frame paths. We shouldn't flatten them try: - if isinstance(videos[0][0][0], str): + if isinstance(videos[0][0], list) and isinstance(videos[0][0][0], str): return [image_paths for sublist in videos for image_paths in sublist] except (IndexError, TypeError): pass diff --git a/tests/models/mllama/test_processing_mllama.py b/tests/models/mllama/test_processing_mllama.py index b95b12358718..b75cf8e5cff7 100644 --- a/tests/models/mllama/test_processing_mllama.py +++ b/tests/models/mllama/test_processing_mllama.py @@ -353,7 +353,6 @@ def test_unstructured_kwargs_batched(self): input_str = self.prepare_text_inputs(batch_size=2, modalities="image") image_input = self.prepare_image_inputs(batch_size=2) - image_input = [[image_input[0]], [image_input[1]]] inputs = processor( text=input_str, images=image_input, @@ -377,7 +376,6 @@ def test_special_mm_token_truncation(self): input_str = self.prepare_text_inputs(batch_size=2, modalities="image") image_input = self.prepare_image_inputs(batch_size=2) - image_input = [[image_input[0]], [image_input[1]]] _ = processor( text=input_str, images=image_input, diff --git a/tests/models/smolvlm/test_processing_smolvlm.py b/tests/models/smolvlm/test_processing_smolvlm.py index aff733c00603..1634c53f132a 100644 --- a/tests/models/smolvlm/test_processing_smolvlm.py +++ b/tests/models/smolvlm/test_processing_smolvlm.py @@ -454,7 +454,6 @@ def test_unstructured_kwargs_batched(self): input_str = self.prepare_text_inputs(batch_size=2, modalities="image") image_input = self.prepare_image_inputs(batch_size=2) - image_input = [[image_input[0]], [image_input[1]]] inputs = processor( text=input_str, images=image_input, @@ -576,7 +575,6 @@ def test_special_mm_token_truncation(self): input_str = self.prepare_text_inputs(batch_size=2, modalities="image") image_input = self.prepare_image_inputs(batch_size=2) - image_input = [[image_input[0]], [image_input[1]]] _ = processor( text=input_str, images=image_input, diff --git a/tests/test_processing_common.py b/tests/test_processing_common.py index a449c1087bb3..e1b36cd2445f 100644 --- a/tests/test_processing_common.py +++ b/tests/test_processing_common.py @@ -272,7 +272,7 @@ def test_processor_text_has_no_visual(self): """ processor = self.get_processor() call_signature = inspect.signature(processor.__call__) - input_args = [param.name for param in call_signature.parameters.values()] + input_args = [param.name for param in call_signature.parameters.values() if param.annotation != param.empty] if not ("text" in input_args and ("images" in input_args and "videos" in input_args)): self.skipTest(f"{self.processor_class} doesn't support several vision modalities with text.")