Skip to content
7 changes: 4 additions & 3 deletions src/transformers/models/idefics2/image_processing_idefics2.py
Original file line number Diff line number Diff line change
Expand Up @@ -258,18 +258,19 @@ def _preprocess(
if do_pad:
max_num_images = max(len(images_) for images_ in processed_images)
max_height, max_width = get_max_height_width(processed_images)
first_image = next(image for images in processed_images for image in images)

processed_images_padded = torch.zeros(
len(processed_images),
max_num_images,
*(processed_images[0][0].shape[0], max_height, max_width),
device=processed_images[0][0].device,
*(first_image.shape[0], max_height, max_width),
device=first_image.device,
)
pixel_attention_masks = torch.zeros(
len(processed_images),
max_num_images,
*(max_height, max_width),
device=processed_images[0][0].device,
device=first_image.device,
)
for i, images in enumerate(processed_images):
for j, image in enumerate(images):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ def _preprocess(
if do_pad:
max_num_images = max(len(images_) for images_ in images)
max_height, max_width = get_max_height_width(images)
num_channels = images[0][0].shape[0]
num_channels = next(image for batch_images in images for image in batch_images).shape[0]

padded_images_list = [
[np.zeros((num_channels, max_height, max_width), dtype=np.float32) for _ in range(max_num_images)]
Expand Down
2 changes: 2 additions & 0 deletions src/transformers/models/pi0/modular_pi0.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,8 @@ def __call__(
padded_pixel_values = torch.zeros(len(batched_images), max_num_cameras, 3, self.height, self.width)

for batch, sample_images in enumerate(batched_images):
if not sample_images:
continue
processed = self.image_processor(sample_images, return_tensors="pt", **output_kwargs["images_kwargs"])

num_cameras = len(sample_images)
Expand Down
2 changes: 2 additions & 0 deletions src/transformers/models/pi0/processing_pi0.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,8 @@ def __call__(
padded_pixel_values = torch.zeros(len(batched_images), max_num_cameras, 3, self.height, self.width)

for batch, sample_images in enumerate(batched_images):
if not sample_images:
continue
processed = self.image_processor(sample_images, return_tensors="pt", **output_kwargs["images_kwargs"])

num_cameras = len(sample_images)
Expand Down
12 changes: 12 additions & 0 deletions tests/models/glm_image/test_processing_glm_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,18 @@ def test_model_input_names(self):

self.assertSetEqual(set(inputs.keys()), set(processor.model_input_names))

def test_processor_text_has_no_visual(self):
# GLM-Image generates images and requires homogeneous source-image counts within a batch.
processor = self.get_processor()
images = self.prepare_images_inputs(batch_size=2, nested=True)
text = self.prepare_text_inputs(batch_size=2, modalities=["image"])
processor(images=images, text=text, padding=True, return_tensors="pt")

images[0] = []
text[0] = "lower newer"
with self.assertRaisesRegex(ValueError, "all samples must have the same number of source images"):
processor(images=images, text=text, padding=True, return_tensors="pt")

@unittest.skip("tiny model has too little tokens and collapses everything to UNK which is not defined")
def test_replacement_offsets(self):
pass
Expand Down
30 changes: 30 additions & 0 deletions tests/models/idefics2/test_image_processing_idefics2.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import unittest

import numpy as np
from parameterized import parameterized

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available
Expand Down Expand Up @@ -319,6 +320,35 @@ def test_pixel_attention_mask(self):
result = image_processing(equal_size_inputs, return_tensors="pt")
self.assertNotIn("pixel_attention_mask", result)

@parameterized.expand([("pil",), ("numpy",), ("torch",)])
def test_padding_with_text_only_examples(self, input_type):
images = [
np.arange(8 * 10 * 3, dtype=np.uint8).reshape(8, 10, 3),
np.arange(12 * 6 * 3, dtype=np.uint8).reshape(12, 6, 3),
]
if input_type == "pil":
images = [Image.fromarray(image) for image in images]
elif input_type == "torch":
images = [torch.from_numpy(image).permute(2, 0, 1) for image in images]

for backend, image_processing_class in self.image_processing_classes.items():
for do_image_splitting in (False, True):
Comment thread
zucchini-nlp marked this conversation as resolved.
Resolved
image_processor = image_processing_class(do_resize=False, do_image_splitting=do_image_splitting)
reference = image_processor([[images[0]], images], return_tensors="pt")
for empty_index in range(3):
with self.subTest(backend=backend, splitting=do_image_splitting, empty_index=empty_index):
batch = [[images[0]], images]
batch.insert(empty_index, [])
result = image_processor(batch, return_tensors="pt")
populated_indices = [i for i in range(3) if i != empty_index]

torch.testing.assert_close(result.pixel_values[populated_indices], reference.pixel_values)
torch.testing.assert_close(
result.pixel_attention_mask[populated_indices], reference.pixel_attention_mask
)
self.assertEqual(result.pixel_values[empty_index].count_nonzero().item(), 0)
self.assertEqual(result.pixel_attention_mask[empty_index].count_nonzero().item(), 0)

def test_convert_rgb(self):
for image_processing_class in self.image_processing_classes.values():
rgba_image = Image.new("RGBA", (100, 100), (255, 0, 0, 128))
Expand Down
14 changes: 13 additions & 1 deletion tests/models/neomme/test_processing_neomme.py
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ def test_apply_chat_template_rejects_unsupported_inputs(self):
with self.subTest(name=name), self.assertRaisesRegex(TemplateError, error):
processor.apply_chat_template(messages, task=task, tokenize=True)

def test_apply_chat_template_supports_mixed_document_batch(self):
def test_processor_text_has_no_visual(self):
processor = self.get_processor()
self._set_retrieval_chat_template(processor)
image = Image.fromarray(np.random.randint(0, 255, (8, 8, 3), dtype=np.uint8))
Expand All @@ -360,6 +360,18 @@ def test_apply_chat_template_supports_mixed_document_batch(self):
self.assertEqual(inputs["position_ids"].shape[1], 2)
self.assertIn("pixel_values", inputs)

direct_inputs = processor(
text=[
processor.tokenizer.document_token + "hello",
processor.tokenizer.document_token + processor.image_token,
],
images=[[], [image]],
padding=True,
return_tensors="pt",
)
for key in ("input_ids", "attention_mask", "pixel_values", "position_ids"):
torch.testing.assert_close(inputs[key], direct_inputs[key])

def test_apply_chat_template_rejects_assistant_mask(self):
processor = self.get_processor()
self._set_retrieval_chat_template(processor)
Expand Down
24 changes: 24 additions & 0 deletions tests/models/pi0/test_processing_pi0.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,30 @@ def test_multi_camera_padding_and_masks(self):
self.assertEqual(outputs["pixel_values"].shape[:2], torch.Size([2, 2]))
self.assertTrue(torch.equal(outputs["pixel_attention_mask"], torch.tensor([[True, True], [True, False]])))

@require_torch
def test_padding_with_text_only_samples(self):
processor = self.get_processor()
processor.image_seq_length = 2
images = [[self.prepare_images_inputs(), self.prepare_images_inputs()], [self.prepare_images_inputs()]]
text = ["task a", "task b"]
Comment thread
zucchini-nlp marked this conversation as resolved.
Resolved
reference = processor(images=images, text=text, return_tensors="pt")

for empty_index in range(3):
with self.subTest(empty_index=empty_index):
batch_images = images[:empty_index] + [[]] + images[empty_index:]
batch_text = text[:empty_index] + ["text only"] + text[empty_index:]
outputs = processor(images=batch_images, text=batch_text, return_tensors="pt")
populated_indices = [index for index in range(3) if index != empty_index]

torch.testing.assert_close(outputs.pixel_values[populated_indices], reference.pixel_values)
torch.testing.assert_close(
outputs.pixel_attention_mask[populated_indices], reference.pixel_attention_mask
)
torch.testing.assert_close(outputs.input_ids[populated_indices], reference.input_ids)
self.assertEqual(outputs.pixel_values[empty_index].count_nonzero().item(), 0)
self.assertFalse(outputs.pixel_attention_mask[empty_index].any().item())
self.assertNotIn(processor.image_token_id, outputs.input_ids[empty_index].tolist())

@require_torch
def test_newline_normalization(self):
processor = self.get_processor()
Expand Down
34 changes: 18 additions & 16 deletions tests/test_processing_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -789,38 +789,40 @@ def test_processor_text_has_no_visual(self):
call_signature = inspect.signature(processor.__call__)
input_args = [param.name for param in call_signature.parameters.values() if param.annotation != param.empty]

if not ("text" in input_args and ("images" in input_args and "videos" in input_args)):
self.skipTest(f"{self.processor_class} doesn't support several vision modalities with text.")
if not (
"text" in input_args
and "images" in input_args
and hasattr(processor, "image_processor")
and hasattr(processor, "tokenizer")
):
self.skipTest(f"{self.processor_class} doesn't support images with text.")

# Prepare inputs and filter by input signature. Make sure to use a high batch size, we'll set some
# samples to text-only later
text = self.prepare_text_inputs(batch_size=3, modalities=["image", "video"])
image_inputs = self.prepare_images_inputs(batch_size=3)
video_inputs = self.prepare_videos_inputs(batch_size=3)
inputs_dict = {"text": text, "images": image_inputs, "videos": video_inputs}
inputs_dict = {k: v for k, v in inputs_dict.items() if k in input_args}
image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs]
inputs_dict_nested = {"images": image_inputs_nested}
modalities = ["image"]

processing_kwargs = {"return_tensors": "pt", "padding": True}
# Shouldn't sample when input is a decoded video without metadata (fpx/duration/etc.)
if "videos" in inputs_dict:
if "videos" in input_args and hasattr(processor, "video_processor"):
video_inputs = self.prepare_videos_inputs(batch_size=3)
inputs_dict_nested["videos"] = [[video] for video in video_inputs]
modalities.append("video")
Comment thread
zucchini-nlp marked this conversation as resolved.
Resolved
processing_kwargs["do_sample_frames"] = False

# First call processor with all inputs and use nested input type, which is the format supported by all multimodal processors
image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs]
video_inputs_nested = [[video] for video in video_inputs]
inputs_dict_nested = {"text": text, "images": image_inputs_nested, "videos": video_inputs_nested}
inputs_dict_nested = {k: v for k, v in inputs_dict_nested.items() if k in input_args}
text = self.prepare_text_inputs(batch_size=3, modalities=modalities)
inputs_dict_nested["text"] = text
inputs = processor(**inputs_dict_nested, **processing_kwargs)
self.assertTrue(self.text_input_name in inputs)

# Now call with one of the samples with no associated vision input. Let's set the first input to be a plain text
# with no placeholder tokens and no images/videos. The final format would be `images = [[], [image2], [image3]]`
plain_text = "lower newer"
image_inputs_nested[0] = []
video_inputs_nested[0] = []
text[0] = plain_text
inputs_dict_no_vision = {"text": text, "images": image_inputs_nested, "videos": video_inputs_nested}
inputs_dict_no_vision = {k: v for k, v in inputs_dict_no_vision.items() if k in input_args}
inputs_dict_no_vision = {key: [[]] + value[1:] for key, value in inputs_dict_nested.items() if key != "text"}
inputs_dict_no_vision["text"] = [plain_text] + text[1:]
inputs_nested = processor(**inputs_dict_no_vision, **processing_kwargs)

# Check that text samples are same and are expanded with placeholder tokens correctly. First sample
Expand Down
Loading