From c0c7698eee4f47626202f721afd6c79a4b0e5838 Mon Sep 17 00:00:00 2001 From: Arnav Sharma <73969466+Arnavsharma2@users.noreply.github.com> Date: Sat, 12 Sep 2026 23:43:22 -0500 Subject: [PATCH 1/5] Fix Idefics2 padding for leading text-only examples --- .../idefics2/image_processing_idefics2.py | 7 +++-- .../idefics2/image_processing_pil_idefics2.py | 2 +- .../test_image_processing_idefics2.py | 30 +++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/idefics2/image_processing_idefics2.py b/src/transformers/models/idefics2/image_processing_idefics2.py index 61f2037ce276..4933ef266323 100644 --- a/src/transformers/models/idefics2/image_processing_idefics2.py +++ b/src/transformers/models/idefics2/image_processing_idefics2.py @@ -258,18 +258,19 @@ def _preprocess( if do_pad: max_num_images = max(len(images_) for images_ in processed_images) max_height, max_width = get_max_height_width(processed_images) + first_image = next(image for images in processed_images for image in images) processed_images_padded = torch.zeros( len(processed_images), max_num_images, - *(processed_images[0][0].shape[0], max_height, max_width), - device=processed_images[0][0].device, + *(first_image.shape[0], max_height, max_width), + device=first_image.device, ) pixel_attention_masks = torch.zeros( len(processed_images), max_num_images, *(max_height, max_width), - device=processed_images[0][0].device, + device=first_image.device, ) for i, images in enumerate(processed_images): for j, image in enumerate(images): diff --git a/src/transformers/models/idefics2/image_processing_pil_idefics2.py b/src/transformers/models/idefics2/image_processing_pil_idefics2.py index 641bdf0d2682..86cf44a2ce50 100644 --- a/src/transformers/models/idefics2/image_processing_pil_idefics2.py +++ b/src/transformers/models/idefics2/image_processing_pil_idefics2.py @@ -246,7 +246,7 @@ def _preprocess( if do_pad: max_num_images = max(len(images_) for images_ in images) max_height, max_width = get_max_height_width(images) - num_channels = images[0][0].shape[0] + num_channels = next(image for batch_images in images for image in batch_images).shape[0] padded_images_list = [ [np.zeros((num_channels, max_height, max_width), dtype=np.float32) for _ in range(max_num_images)] diff --git a/tests/models/idefics2/test_image_processing_idefics2.py b/tests/models/idefics2/test_image_processing_idefics2.py index dfb1b9db52cf..dafb805143ca 100644 --- a/tests/models/idefics2/test_image_processing_idefics2.py +++ b/tests/models/idefics2/test_image_processing_idefics2.py @@ -15,6 +15,7 @@ import unittest import numpy as np +from parameterized import parameterized from transformers.testing_utils import require_torch, require_vision from transformers.utils import is_torch_available, is_vision_available @@ -319,6 +320,35 @@ def test_pixel_attention_mask(self): result = image_processing(equal_size_inputs, return_tensors="pt") self.assertNotIn("pixel_attention_mask", result) + @parameterized.expand([("pil",), ("numpy",), ("torch",)]) + def test_padding_with_text_only_examples(self, input_type): + images = [ + np.arange(8 * 10 * 3, dtype=np.uint8).reshape(8, 10, 3), + np.arange(12 * 6 * 3, dtype=np.uint8).reshape(12, 6, 3), + ] + if input_type == "pil": + images = [Image.fromarray(image) for image in images] + elif input_type == "torch": + images = [torch.from_numpy(image).permute(2, 0, 1) for image in images] + + for backend, image_processing_class in self.image_processing_classes.items(): + for do_image_splitting in (False, True): + image_processor = image_processing_class(do_resize=False, do_image_splitting=do_image_splitting) + reference = image_processor([[images[0]], images], return_tensors="pt") + for empty_index in range(3): + with self.subTest(backend=backend, splitting=do_image_splitting, empty_index=empty_index): + batch = [[images[0]], images] + batch.insert(empty_index, []) + result = image_processor(batch, return_tensors="pt") + populated_indices = [i for i in range(3) if i != empty_index] + + torch.testing.assert_close(result.pixel_values[populated_indices], reference.pixel_values) + torch.testing.assert_close( + result.pixel_attention_mask[populated_indices], reference.pixel_attention_mask + ) + self.assertEqual(result.pixel_values[empty_index].count_nonzero().item(), 0) + self.assertEqual(result.pixel_attention_mask[empty_index].count_nonzero().item(), 0) + def test_convert_rgb(self): for image_processing_class in self.image_processing_classes.values(): rgba_image = Image.new("RGBA", (100, 100), (255, 0, 0, 128)) From 7c78a93cba6aeec301ed5f59523085a3fc7e827a Mon Sep 17 00:00:00 2001 From: Arnav Sharma <73969466+Arnavsharma2@users.noreply.github.com> Date: Mon, 14 Sep 2026 09:15:01 -0500 Subject: [PATCH 2/5] Test mixed text-only batches for image-only processors --- tests/test_processing_common.py | 34 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/tests/test_processing_common.py b/tests/test_processing_common.py index 7f7fedb36b2e..47409c87ee53 100644 --- a/tests/test_processing_common.py +++ b/tests/test_processing_common.py @@ -789,38 +789,40 @@ def test_processor_text_has_no_visual(self): call_signature = inspect.signature(processor.__call__) input_args = [param.name for param in call_signature.parameters.values() if param.annotation != param.empty] - if not ("text" in input_args and ("images" in input_args and "videos" in input_args)): - self.skipTest(f"{self.processor_class} doesn't support several vision modalities with text.") + if not ( + "text" in input_args + and "images" in input_args + and hasattr(processor, "image_processor") + and hasattr(processor, "tokenizer") + ): + self.skipTest(f"{self.processor_class} doesn't support images with text.") # Prepare inputs and filter by input signature. Make sure to use a high batch size, we'll set some # samples to text-only later - text = self.prepare_text_inputs(batch_size=3, modalities=["image", "video"]) image_inputs = self.prepare_images_inputs(batch_size=3) - video_inputs = self.prepare_videos_inputs(batch_size=3) - inputs_dict = {"text": text, "images": image_inputs, "videos": video_inputs} - inputs_dict = {k: v for k, v in inputs_dict.items() if k in input_args} + image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] + inputs_dict_nested = {"images": image_inputs_nested} + modalities = ["image"] processing_kwargs = {"return_tensors": "pt", "padding": True} # Shouldn't sample when input is a decoded video without metadata (fpx/duration/etc.) - if "videos" in inputs_dict: + if "videos" in input_args and hasattr(processor, "video_processor"): + video_inputs = self.prepare_videos_inputs(batch_size=3) + inputs_dict_nested["videos"] = [[video] for video in video_inputs] + modalities.append("video") processing_kwargs["do_sample_frames"] = False # First call processor with all inputs and use nested input type, which is the format supported by all multimodal processors - image_inputs_nested = [[image] if not isinstance(image, list) else image for image in image_inputs] - video_inputs_nested = [[video] for video in video_inputs] - inputs_dict_nested = {"text": text, "images": image_inputs_nested, "videos": video_inputs_nested} - inputs_dict_nested = {k: v for k, v in inputs_dict_nested.items() if k in input_args} + text = self.prepare_text_inputs(batch_size=3, modalities=modalities) + inputs_dict_nested["text"] = text inputs = processor(**inputs_dict_nested, **processing_kwargs) self.assertTrue(self.text_input_name in inputs) # Now call with one of the samples with no associated vision input. Let's set the first input to be a plain text # with no placeholder tokens and no images/videos. The final format would be `images = [[], [image2], [image3]]` plain_text = "lower newer" - image_inputs_nested[0] = [] - video_inputs_nested[0] = [] - text[0] = plain_text - inputs_dict_no_vision = {"text": text, "images": image_inputs_nested, "videos": video_inputs_nested} - inputs_dict_no_vision = {k: v for k, v in inputs_dict_no_vision.items() if k in input_args} + inputs_dict_no_vision = {key: [[]] + value[1:] for key, value in inputs_dict_nested.items() if key != "text"} + inputs_dict_no_vision["text"] = [plain_text] + text[1:] inputs_nested = processor(**inputs_dict_no_vision, **processing_kwargs) # Check that text samples are same and are expanded with placeholder tokens correctly. First sample From 0fa7ce94cff12a075499d4dbaf1608febea42a15 Mon Sep 17 00:00:00 2001 From: Arnav Sharma <73969466+Arnavsharma2@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:19:37 -0500 Subject: [PATCH 3/5] Fix PI0 empty-image batches and adapt specialized processor tests --- src/transformers/models/pi0/modular_pi0.py | 2 ++ src/transformers/models/pi0/processing_pi0.py | 2 ++ .../glm_image/test_processing_glm_image.py | 12 ++++++++++ tests/models/neomme/test_processing_neomme.py | 14 ++++++++++- tests/models/pi0/test_processing_pi0.py | 24 +++++++++++++++++++ 5 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/pi0/modular_pi0.py b/src/transformers/models/pi0/modular_pi0.py index 20cb86888a2a..af488b760d91 100644 --- a/src/transformers/models/pi0/modular_pi0.py +++ b/src/transformers/models/pi0/modular_pi0.py @@ -163,6 +163,8 @@ def __call__( padded_pixel_values = torch.zeros(len(batched_images), max_num_cameras, 3, self.height, self.width) for batch, sample_images in enumerate(batched_images): + if not sample_images: + continue processed = self.image_processor(sample_images, return_tensors="pt", **output_kwargs["images_kwargs"]) num_cameras = len(sample_images) diff --git a/src/transformers/models/pi0/processing_pi0.py b/src/transformers/models/pi0/processing_pi0.py index 586ae69ea034..d56ee354b0e5 100644 --- a/src/transformers/models/pi0/processing_pi0.py +++ b/src/transformers/models/pi0/processing_pi0.py @@ -147,6 +147,8 @@ def __call__( padded_pixel_values = torch.zeros(len(batched_images), max_num_cameras, 3, self.height, self.width) for batch, sample_images in enumerate(batched_images): + if not sample_images: + continue processed = self.image_processor(sample_images, return_tensors="pt", **output_kwargs["images_kwargs"]) num_cameras = len(sample_images) diff --git a/tests/models/glm_image/test_processing_glm_image.py b/tests/models/glm_image/test_processing_glm_image.py index d314a1ab4e69..a5d25e5c5668 100644 --- a/tests/models/glm_image/test_processing_glm_image.py +++ b/tests/models/glm_image/test_processing_glm_image.py @@ -59,6 +59,18 @@ def test_model_input_names(self): self.assertSetEqual(set(inputs.keys()), set(processor.model_input_names)) + def test_processor_text_has_no_visual(self): + # GLM-Image generates images and requires homogeneous source-image counts within a batch. + processor = self.get_processor() + images = self.prepare_images_inputs(batch_size=2, nested=True) + text = self.prepare_text_inputs(batch_size=2, modalities=["image"]) + processor(images=images, text=text, padding=True, return_tensors="pt") + + images[0] = [] + text[0] = "lower newer" + with self.assertRaisesRegex(ValueError, "all samples must have the same number of source images"): + processor(images=images, text=text, padding=True, return_tensors="pt") + @unittest.skip("tiny model has too little tokens and collapses everything to UNK which is not defined") def test_replacement_offsets(self): pass diff --git a/tests/models/neomme/test_processing_neomme.py b/tests/models/neomme/test_processing_neomme.py index a7d3f97c2ef0..45d0da5bb26e 100644 --- a/tests/models/neomme/test_processing_neomme.py +++ b/tests/models/neomme/test_processing_neomme.py @@ -337,7 +337,7 @@ def test_apply_chat_template_rejects_unsupported_inputs(self): with self.subTest(name=name), self.assertRaisesRegex(TemplateError, error): processor.apply_chat_template(messages, task=task, tokenize=True) - def test_apply_chat_template_supports_mixed_document_batch(self): + def test_processor_text_has_no_visual(self): processor = self.get_processor() self._set_retrieval_chat_template(processor) image = Image.fromarray(np.random.randint(0, 255, (8, 8, 3), dtype=np.uint8)) @@ -360,6 +360,18 @@ def test_apply_chat_template_supports_mixed_document_batch(self): self.assertEqual(inputs["position_ids"].shape[1], 2) self.assertIn("pixel_values", inputs) + direct_inputs = processor( + text=[ + processor.tokenizer.document_token + "hello", + processor.tokenizer.document_token + processor.image_token, + ], + images=[[], [image]], + padding=True, + return_tensors="pt", + ) + for key in ("input_ids", "attention_mask", "pixel_values", "position_ids"): + torch.testing.assert_close(inputs[key], direct_inputs[key]) + def test_apply_chat_template_rejects_assistant_mask(self): processor = self.get_processor() self._set_retrieval_chat_template(processor) diff --git a/tests/models/pi0/test_processing_pi0.py b/tests/models/pi0/test_processing_pi0.py index caf867aca14c..0d5cfd361688 100644 --- a/tests/models/pi0/test_processing_pi0.py +++ b/tests/models/pi0/test_processing_pi0.py @@ -95,6 +95,30 @@ def test_multi_camera_padding_and_masks(self): self.assertEqual(outputs["pixel_values"].shape[:2], torch.Size([2, 2])) self.assertTrue(torch.equal(outputs["pixel_attention_mask"], torch.tensor([[True, True], [True, False]]))) + @require_torch + def test_padding_with_text_only_samples(self): + processor = self.get_processor() + processor.image_seq_length = 2 + images = [[self.prepare_images_inputs(), self.prepare_images_inputs()], [self.prepare_images_inputs()]] + text = ["task a", "task b"] + reference = processor(images=images, text=text, return_tensors="pt") + + for empty_index in range(3): + with self.subTest(empty_index=empty_index): + batch_images = images[:empty_index] + [[]] + images[empty_index:] + batch_text = text[:empty_index] + ["text only"] + text[empty_index:] + outputs = processor(images=batch_images, text=batch_text, return_tensors="pt") + populated_indices = [index for index in range(3) if index != empty_index] + + torch.testing.assert_close(outputs.pixel_values[populated_indices], reference.pixel_values) + torch.testing.assert_close( + outputs.pixel_attention_mask[populated_indices], reference.pixel_attention_mask + ) + torch.testing.assert_close(outputs.input_ids[populated_indices], reference.input_ids) + self.assertEqual(outputs.pixel_values[empty_index].count_nonzero().item(), 0) + self.assertFalse(outputs.pixel_attention_mask[empty_index].any().item()) + self.assertNotIn(processor.image_token_id, outputs.input_ids[empty_index].tolist()) + @require_torch def test_newline_normalization(self): processor = self.get_processor() From 5dafe55d4b5828d474ab96846126bf902f96225e Mon Sep 17 00:00:00 2001 From: Arnav Sharma <73969466+Arnavsharma2@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:37:47 -0500 Subject: [PATCH 4/5] Remove redundant mixed text-only batch tests --- .../test_image_processing_idefics2.py | 30 ------------------- tests/models/pi0/test_processing_pi0.py | 24 --------------- 2 files changed, 54 deletions(-) diff --git a/tests/models/idefics2/test_image_processing_idefics2.py b/tests/models/idefics2/test_image_processing_idefics2.py index dafb805143ca..dfb1b9db52cf 100644 --- a/tests/models/idefics2/test_image_processing_idefics2.py +++ b/tests/models/idefics2/test_image_processing_idefics2.py @@ -15,7 +15,6 @@ import unittest import numpy as np -from parameterized import parameterized from transformers.testing_utils import require_torch, require_vision from transformers.utils import is_torch_available, is_vision_available @@ -320,35 +319,6 @@ def test_pixel_attention_mask(self): result = image_processing(equal_size_inputs, return_tensors="pt") self.assertNotIn("pixel_attention_mask", result) - @parameterized.expand([("pil",), ("numpy",), ("torch",)]) - def test_padding_with_text_only_examples(self, input_type): - images = [ - np.arange(8 * 10 * 3, dtype=np.uint8).reshape(8, 10, 3), - np.arange(12 * 6 * 3, dtype=np.uint8).reshape(12, 6, 3), - ] - if input_type == "pil": - images = [Image.fromarray(image) for image in images] - elif input_type == "torch": - images = [torch.from_numpy(image).permute(2, 0, 1) for image in images] - - for backend, image_processing_class in self.image_processing_classes.items(): - for do_image_splitting in (False, True): - image_processor = image_processing_class(do_resize=False, do_image_splitting=do_image_splitting) - reference = image_processor([[images[0]], images], return_tensors="pt") - for empty_index in range(3): - with self.subTest(backend=backend, splitting=do_image_splitting, empty_index=empty_index): - batch = [[images[0]], images] - batch.insert(empty_index, []) - result = image_processor(batch, return_tensors="pt") - populated_indices = [i for i in range(3) if i != empty_index] - - torch.testing.assert_close(result.pixel_values[populated_indices], reference.pixel_values) - torch.testing.assert_close( - result.pixel_attention_mask[populated_indices], reference.pixel_attention_mask - ) - self.assertEqual(result.pixel_values[empty_index].count_nonzero().item(), 0) - self.assertEqual(result.pixel_attention_mask[empty_index].count_nonzero().item(), 0) - def test_convert_rgb(self): for image_processing_class in self.image_processing_classes.values(): rgba_image = Image.new("RGBA", (100, 100), (255, 0, 0, 128)) diff --git a/tests/models/pi0/test_processing_pi0.py b/tests/models/pi0/test_processing_pi0.py index 0d5cfd361688..caf867aca14c 100644 --- a/tests/models/pi0/test_processing_pi0.py +++ b/tests/models/pi0/test_processing_pi0.py @@ -95,30 +95,6 @@ def test_multi_camera_padding_and_masks(self): self.assertEqual(outputs["pixel_values"].shape[:2], torch.Size([2, 2])) self.assertTrue(torch.equal(outputs["pixel_attention_mask"], torch.tensor([[True, True], [True, False]]))) - @require_torch - def test_padding_with_text_only_samples(self): - processor = self.get_processor() - processor.image_seq_length = 2 - images = [[self.prepare_images_inputs(), self.prepare_images_inputs()], [self.prepare_images_inputs()]] - text = ["task a", "task b"] - reference = processor(images=images, text=text, return_tensors="pt") - - for empty_index in range(3): - with self.subTest(empty_index=empty_index): - batch_images = images[:empty_index] + [[]] + images[empty_index:] - batch_text = text[:empty_index] + ["text only"] + text[empty_index:] - outputs = processor(images=batch_images, text=batch_text, return_tensors="pt") - populated_indices = [index for index in range(3) if index != empty_index] - - torch.testing.assert_close(outputs.pixel_values[populated_indices], reference.pixel_values) - torch.testing.assert_close( - outputs.pixel_attention_mask[populated_indices], reference.pixel_attention_mask - ) - torch.testing.assert_close(outputs.input_ids[populated_indices], reference.input_ids) - self.assertEqual(outputs.pixel_values[empty_index].count_nonzero().item(), 0) - self.assertFalse(outputs.pixel_attention_mask[empty_index].any().item()) - self.assertNotIn(processor.image_token_id, outputs.input_ids[empty_index].tolist()) - @require_torch def test_newline_normalization(self): processor = self.get_processor() From 6ec2a44ad156259e2f27ab8bb58bd9870942561c Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 17 Sep 2026 12:10:46 +0200 Subject: [PATCH 5/5] ci fix --- src/transformers/models/lighton_ocr/modular_lighton_ocr.py | 3 +++ src/transformers/models/lighton_ocr/processing_lighton_ocr.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/transformers/models/lighton_ocr/modular_lighton_ocr.py b/src/transformers/models/lighton_ocr/modular_lighton_ocr.py index 1e6951dec3ac..fc35f2af82d3 100644 --- a/src/transformers/models/lighton_ocr/modular_lighton_ocr.py +++ b/src/transformers/models/lighton_ocr/modular_lighton_ocr.py @@ -119,6 +119,9 @@ class LightOnOcrProcessorKwargs(ProcessingKwargs, total=False): "padding": False, "return_mm_token_type_ids": False, }, + "images_kwargs": { + "do_pad": True, + }, "common_kwargs": { "return_tensors": "pt", }, diff --git a/src/transformers/models/lighton_ocr/processing_lighton_ocr.py b/src/transformers/models/lighton_ocr/processing_lighton_ocr.py index fe379d3ea5b5..0d7f262b8ef1 100644 --- a/src/transformers/models/lighton_ocr/processing_lighton_ocr.py +++ b/src/transformers/models/lighton_ocr/processing_lighton_ocr.py @@ -32,6 +32,9 @@ class LightOnOcrProcessorKwargs(ProcessingKwargs, total=False): "padding": False, "return_mm_token_type_ids": False, }, + "images_kwargs": { + "do_pad": True, + }, "common_kwargs": { "return_tensors": "pt", },