Fix!

src/transformers/models/qianfan_ocr/modular_qianfan_ocr.py

@@ -20,7 +20,7 @@
 from huggingface_hub.dataclasses import strict
 
 from ...configuration_utils import SubConfigSpec
-from ...image_utils import ImageInput
+from ...image_utils import ImageInput, make_flat_list_of_images
 from ...modeling_outputs import BaseModelOutputWithPooling
 from ...processing_utils import ProcessorMixin, Unpack
 from ...tokenization_utils_base import PreTokenizedInput, TextInput
@@ -421,13 +421,36 @@ def validate_inputs(
         videos=None,
         **kwargs: Unpack[QianfanOCRProcessorKwargs],
     ):
-        super().validate_inputs(images=images, text=text, videos=videos, **kwargs)
         if text is None:
             raise ValueError("You have to specify text.")
 
         if videos is not None:
             raise ValueError("QianfanOCR does not support video input.")
 
+        if images is None and text is None:
+            raise ValueError(f"You need to provide at least one input to call {self.__class__.__name__}")
+
+        # Check that number of placeholders in text is same as number of mm-data for models
+        # that have a special multimodal placeholder token defined
+        if text is not None:
+            if not (
+                isinstance(text, str) or (isinstance(text, list) and all(isinstance(sample, str) for sample in text))
+            ):
+                raise ValueError("`text` must be a string or list of strings.")
+
+            n_images_in_text = [sample.count(self.image_placeholder_token) for sample in text]
+            if images is not None:
+                images = make_flat_list_of_images(images)
+                if sum(n_images_in_text) != len(images):
+                    raise ValueError(
+                        f"The total number of {self.image_placeholder_token} tokens in the prompts should be the same as the number of images passed."
+                        f" Found total {sum(n_images_in_text)} {self.image_placeholder_token} tokens and {len(images)} images."
+                    )
+            elif any(n_images_in_text):
+                raise ValueError(
+                    f"Found {sum(n_images_in_text)} {self.image_placeholder_token} tokens in the text but no images were passed."
+                )
+
     def replace_video_token(self, video_inputs: dict, video_idx: int, **kwargs) -> str:
         raise NotImplementedError("QianfanOCR does not support video input")
 

tests/models/phi4_multimodal/test_modeling_phi4_multimodal.py

@@ -31,6 +31,7 @@
     is_torch_available,
     is_vision_available,
 )
+from transformers.audio_utils import load_audio
 from transformers.testing_utils import (
     Expectations,
     cleanup,
@@ -41,7 +42,6 @@
     slow,
     torch_device,
 )
-from transformers.utils import is_torchcodec_available
 
 from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
@@ -57,10 +57,6 @@
     from PIL import Image
 
 
-if is_torchcodec_available():
-    import torchcodec
-
-
 class Phi4MultimodalModelTester:
     def __init__(
         self,
@@ -291,9 +287,8 @@ def setUp(self):
         self.assistant_token = "<|assistant|>"
         self.end_token = "<|end|>"
         self.image = Image.open(requests.get(self.image_url, stream=True).raw)
-        audio_bytes = requests.get(self.audio_url, stream=True).raw.data
-        samples = torchcodec.decoders.AudioDecoder(audio_bytes).get_all_samples()
-        self.audio, self.sampling_rate = samples.data, samples.sample_rate
+        self.sampling_rate = 16000
+        self.audio = load_audio(self.audio_url, sampling_rate=self.sampling_rate)
 
         cleanup(torch_device, gc_collect=True)
 
@@ -306,7 +301,7 @@ def test_text_only_generation(self):
         )
 
         prompt = f"{self.user_token}What is the answer for 1+1? Explain it.{self.end_token}{self.assistant_token}"
-        inputs = self.processor(prompt, images=None, return_tensors="pt").to(torch_device)
+        inputs = self.processor(text=prompt, images=None, return_tensors="pt").to(torch_device)
 
         output = model.generate(
             **inputs,
@@ -326,7 +321,7 @@ def test_vision_text_generation(self):
         )
 
         prompt = f"{self.user_token}<|image|>What is shown in this image?{self.end_token}{self.assistant_token}"
-        inputs = self.processor(prompt, images=self.image, return_tensors="pt").to(torch_device)
+        inputs = self.processor(text=prompt, images=self.image, return_tensors="pt").to(torch_device)
 
         output = model.generate(
             **inputs,
@@ -360,7 +355,7 @@ def test_multi_image_vision_text_generation(self):
             placeholder += "<|image|>"
 
         prompt = f"{self.user_token}{placeholder}Summarize the deck of slides.{self.end_token}{self.assistant_token}"
-        inputs = self.processor(prompt, images, return_tensors="pt").to(torch_device)
+        inputs = self.processor(text=prompt, images=images, return_tensors="pt").to(torch_device)
 
         output = model.generate(
             **inputs,
@@ -380,9 +375,9 @@ def test_audio_text_generation(self):
         )
 
         prompt = f"{self.user_token}<|audio|>What is happening in this audio?{self.end_token}{self.assistant_token}"
-        inputs = self.processor(prompt, audio=self.audio, sampling_rate=self.sampling_rate, return_tensors="pt").to(
-            torch_device
-        )
+        inputs = self.processor(
+            text=prompt, audio=self.audio, sampling_rate=self.sampling_rate, return_tensors="pt"
+        ).to(torch_device)
 
         output = model.generate(
             **inputs,