Skip to content
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/transformers/image_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ def make_list_of_images(images, expected_ndims: int = 3) -> list[ImageInput]:

def make_flat_list_of_images(
images: Union[list[ImageInput], ImageInput],
) -> ImageInput:
) -> list[ImageInput]:
"""
Ensure that the output is a flat list of images. If the input is a single image, it is converted to a list of length 1.
If the input is a nested list of images, it is converted to a flat list of images.
Expand All @@ -243,7 +243,7 @@ def make_flat_list_of_images(
if (
isinstance(images, (list, tuple))
and all(isinstance(images_i, (list, tuple)) for images_i in images)
and all(is_valid_list_of_images(images_i) for images_i in images)
and all(is_valid_list_of_images(images_i) or not images_i for images_i in images)
):
return [img for img_list in images for img in img_list]

Expand All @@ -264,7 +264,7 @@ def make_flat_list_of_images(

def make_nested_list_of_images(
images: Union[list[ImageInput], ImageInput],
) -> ImageInput:
) -> list[list[ImageInput]]:
"""
Ensure that the output is a nested list of images.
Args:
Expand All @@ -277,7 +277,7 @@ def make_nested_list_of_images(
if (
isinstance(images, (list, tuple))
and all(isinstance(images_i, (list, tuple)) for images_i in images)
and all(is_valid_list_of_images(images_i) for images_i in images)
and all(is_valid_list_of_images(images_i) or not images_i for images_i in images)
):
return images

Expand Down
7 changes: 5 additions & 2 deletions src/transformers/models/idefics2/image_processing_idefics2.py
Original file line number Diff line number Diff line change
Expand Up @@ -495,16 +495,19 @@ def preprocess(

# All transformations expect numpy arrays.
images_list = [[to_numpy_array(image) for image in images] for images in images_list]
# Search for the first image in the image list.
# NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682
first_image_in_list = [images for images in images_list if images][0][0]

if do_rescale and is_scaled_image(images_list[0][0]):
if do_rescale and is_scaled_image(first_image_in_list):
logger.warning_once(
"It looks like you are trying to rescale already rescaled images. If the input"
" images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."
)

if input_data_format is None:
# We assume that all images have the same channel dimension format.
input_data_format = infer_channel_dimension_format(images_list[0][0])
input_data_format = infer_channel_dimension_format(first_image_in_list)

if do_image_splitting:
new_images_list = []
Expand Down
15 changes: 10 additions & 5 deletions src/transformers/models/idefics3/image_processing_idefics3.py
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,7 @@ def pad(
For a list of images, for each images, pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width.
For each sample in the batch, pads the sample with empty images to the max_number of images per sample in the batch. Optionally returns a pixel mask.
Args:
images (`List[np.ndarray]`):
images (`List[List[np.ndarray]]`):
List of list of images to pad. Pads to the largest height and width in the batch.
constant_values (`float` or `Iterable[float]`, *optional*):
The value to use for the padding if `mode` is `"constant"`.
Expand Down Expand Up @@ -564,11 +564,13 @@ def pad(
else input_data_format
)
data_format = input_data_format if data_format is None else data_format
# filter out empty image lists, then take first image of the first sample
first_image_in_list = [sample_images for sample_images in images if sample_images][0][0]

if input_data_format == ChannelDimension.FIRST:
n_channels = images[0][0].shape[0]
n_channels = first_image_in_list.shape[0]
elif input_data_format == ChannelDimension.LAST:
n_channels = images[0][0].shape[-1]
n_channels = first_image_in_list.shape[-1]
else:
raise ValueError("Invalid channel dimension format.")

Expand Down Expand Up @@ -713,6 +715,9 @@ def preprocess(

# All transformations expect numpy arrays.
images_list = [[to_numpy_array(image) for image in images] for images in images_list]
# Search for the first image in the image list.
# NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682
first_image_in_list = [images for images in images_list if images][0][0]

# Extra channel dimension for grayscale images
if input_data_format in [ChannelDimension.LAST, None]:
Expand All @@ -724,15 +729,15 @@ def preprocess(
[np.expand_dims(img, axis=0) if img.ndim == 2 else img for img in images] for images in images_list
]

if do_rescale and is_scaled_image(images_list[0][0]):
if do_rescale and is_scaled_image(first_image_in_list):
logger.warning_once(
"It looks like you are trying to rescale already rescaled images. If the input"
" images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."
)

# We assume that all images have the same channel dimension format.
if input_data_format is None:
input_data_format = infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4))
input_data_format = infer_channel_dimension_format(first_image_in_list, num_channels=(1, 3, 4))

if do_resize:
images_list = [
Expand Down
15 changes: 10 additions & 5 deletions src/transformers/models/smolvlm/image_processing_smolvlm.py
Original file line number Diff line number Diff line change
Expand Up @@ -533,7 +533,7 @@ def pad(
For a list of images, for each images, pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width.
For each sample in the batch, pads the sample with empty images to the max_number of images per sample in the batch. Optionally returns a pixel mask.
Args:
images (`List[np.ndarray]`):
images (`List[List[np.ndarray]]`):
List of list of images to pad. Pads to the largest height and width in the batch.
constant_values (`float` or `Iterable[float]`, *optional*):
The value to use for the padding if `mode` is `"constant"`.
Expand Down Expand Up @@ -561,11 +561,13 @@ def pad(
else input_data_format
)
data_format = input_data_format if data_format is None else data_format
# filter out empty image lists, then take first image of the first sample
first_image_in_list = [sample_images for sample_images in images if sample_images][0][0]

if input_data_format == ChannelDimension.FIRST:
n_channels = images[0][0].shape[0]
n_channels = first_image_in_list.shape[0]
elif input_data_format == ChannelDimension.LAST:
n_channels = images[0][0].shape[-1]
n_channels = first_image_in_list.shape[-1]
else:
raise ValueError("Invalid channel dimension format.")

Expand Down Expand Up @@ -710,6 +712,9 @@ def preprocess(

# All transformations expect numpy arrays.
images_list = [[to_numpy_array(image) for image in images] for images in images_list]
# Search for the first image in the image list.
# NOTE: we can't slice the first image with images_list[0][0] if the first batch contains no images. See #36682
first_image_in_list = [images for images in images_list if images][0][0]

# Extra channel dimension for grayscale images
if input_data_format in [ChannelDimension.LAST, None]:
Expand All @@ -721,15 +726,15 @@ def preprocess(
[np.expand_dims(img, axis=0) if img.ndim == 2 else img for img in images] for images in images_list
]

if do_rescale and is_scaled_image(images_list[0][0]):
if do_rescale and is_scaled_image(first_image_in_list):
logger.warning_once(
"It looks like you are trying to rescale already rescaled images. If the input"
" images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."
)

# We assume that all images have the same channel dimension format.
if input_data_format is None:
input_data_format = infer_channel_dimension_format(images_list[0][0], num_channels=(1, 3, 4))
input_data_format = infer_channel_dimension_format(first_image_in_list, num_channels=(1, 3, 4))

if do_resize:
images_list = [
Expand Down
6 changes: 0 additions & 6 deletions tests/models/idefics2/test_processor_idefics2.py
Original file line number Diff line number Diff line change
Expand Up @@ -281,17 +281,11 @@ def test_process_interleaved_images_prompts_image_error(self):
"In this other sentence we try some good things<image>",
]
images = [[self.image1], []]
with self.assertRaises(ValueError):
processor(text=text, images=images, padding=True)
images = [[], [self.image2]]
with self.assertRaises(ValueError):
processor(text=text, images=images, padding=True)
images = [self.image1, self.image2]
with self.assertRaises(ValueError):
processor(text=text, images=images, padding=True)
images = [self.image1]
with self.assertRaises(ValueError):
processor(text=text, images=images, padding=True)
Comment on lines -292 to -

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

didn't get why this doesn't throw error anymore, IMO passing flat images is ambiguous, and we throw errors instead of trying to infer which text corresponds to which image


def test_apply_chat_template(self):
# Message contains content which a mix of lists with images and image urls and string
Expand Down
6 changes: 0 additions & 6 deletions tests/models/idefics3/test_processor_idefics3.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,17 +306,11 @@ def test_process_interleaved_images_prompts_image_error(self):
"In this other sentence we try some good things<image>",
]
images = [[self.image1], []]
with self.assertRaises(ValueError):
processor(text=text, images=images, padding=True)
images = [[], [self.image2]]
with self.assertRaises(ValueError):
processor(text=text, images=images, padding=True)
images = [self.image1, self.image2]
with self.assertRaises(ValueError):
processor(text=text, images=images, padding=True)
images = [self.image1]
with self.assertRaises(ValueError):
processor(text=text, images=images, padding=True)

def test_apply_chat_template(self):
# Message contains content which a mix of lists with images and image urls and string
Expand Down
6 changes: 0 additions & 6 deletions tests/models/smolvlm/test_processor_smolvlm.py
Original file line number Diff line number Diff line change
Expand Up @@ -322,17 +322,11 @@ def test_process_interleaved_images_prompts_image_error(self):
"In this other sentence we try some good things<image>",
]
images = [[self.image1], []]
with self.assertRaises(ValueError):
processor(text=text, images=images, padding=True)
images = [[], [self.image2]]
with self.assertRaises(ValueError):
processor(text=text, images=images, padding=True)
images = [self.image1, self.image2]
with self.assertRaises(ValueError):
processor(text=text, images=images, padding=True)
images = [self.image1]
with self.assertRaises(ValueError):
processor(text=text, images=images, padding=True)

def test_apply_chat_template(self):
# Message contains content which a mix of lists with images and image urls and string
Expand Down