diff --git a/src/transformers/models/qianfan_ocr/modular_qianfan_ocr.py b/src/transformers/models/qianfan_ocr/modular_qianfan_ocr.py index 6fe402485fb1..549efbabc556 100644 --- a/src/transformers/models/qianfan_ocr/modular_qianfan_ocr.py +++ b/src/transformers/models/qianfan_ocr/modular_qianfan_ocr.py @@ -20,7 +20,7 @@ from huggingface_hub.dataclasses import strict from ...configuration_utils import SubConfigSpec -from ...image_utils import ImageInput +from ...image_utils import ImageInput, make_flat_list_of_images from ...modeling_outputs import BaseModelOutputWithPooling from ...processing_utils import ProcessorMixin, Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput @@ -421,13 +421,36 @@ def validate_inputs( videos=None, **kwargs: Unpack[QianfanOCRProcessorKwargs], ): - super().validate_inputs(images=images, text=text, videos=videos, **kwargs) if text is None: raise ValueError("You have to specify text.") if videos is not None: raise ValueError("QianfanOCR does not support video input.") + if images is None and text is None: + raise ValueError(f"You need to provide at least one input to call {self.__class__.__name__}") + + # Check that number of placeholders in text is same as number of mm-data for models + # that have a special multimodal placeholder token defined + if text is not None: + if not ( + isinstance(text, str) or (isinstance(text, list) and all(isinstance(sample, str) for sample in text)) + ): + raise ValueError("`text` must be a string or list of strings.") + + n_images_in_text = [sample.count(self.image_placeholder_token) for sample in text] + if images is not None: + images = make_flat_list_of_images(images) + if sum(n_images_in_text) != len(images): + raise ValueError( + f"The total number of {self.image_placeholder_token} tokens in the prompts should be the same as the number of images passed." + f" Found total {sum(n_images_in_text)} {self.image_placeholder_token} tokens and {len(images)} images." + ) + elif any(n_images_in_text): + raise ValueError( + f"Found {sum(n_images_in_text)} {self.image_placeholder_token} tokens in the text but no images were passed." + ) + def replace_video_token(self, video_inputs: dict, video_idx: int, **kwargs) -> str: raise NotImplementedError("QianfanOCR does not support video input") diff --git a/src/transformers/models/qianfan_ocr/processing_qianfan_ocr.py b/src/transformers/models/qianfan_ocr/processing_qianfan_ocr.py index d9c7ccd28c15..5266ae1119a7 100644 --- a/src/transformers/models/qianfan_ocr/processing_qianfan_ocr.py +++ b/src/transformers/models/qianfan_ocr/processing_qianfan_ocr.py @@ -21,7 +21,7 @@ import re from ...image_processing_utils import BatchFeature -from ...image_utils import ImageInput +from ...image_utils import ImageInput, make_flat_list_of_images from ...processing_utils import MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput from ...utils import auto_docstring @@ -106,13 +106,36 @@ def validate_inputs( videos=None, **kwargs: Unpack[QianfanOCRProcessorKwargs], ): - super().validate_inputs(images=images, text=text, videos=videos, **kwargs) if text is None: raise ValueError("You have to specify text.") if videos is not None: raise ValueError("QianfanOCR does not support video input.") + if images is None and text is None: + raise ValueError(f"You need to provide at least one input to call {self.__class__.__name__}") + + # Check that number of placeholders in text is same as number of mm-data for models + # that have a special multimodal placeholder token defined + if text is not None: + if not ( + isinstance(text, str) or (isinstance(text, list) and all(isinstance(sample, str) for sample in text)) + ): + raise ValueError("`text` must be a string or list of strings.") + + n_images_in_text = [sample.count(self.image_placeholder_token) for sample in text] + if images is not None: + images = make_flat_list_of_images(images) + if sum(n_images_in_text) != len(images): + raise ValueError( + f"The total number of {self.image_placeholder_token} tokens in the prompts should be the same as the number of images passed." + f" Found total {sum(n_images_in_text)} {self.image_placeholder_token} tokens and {len(images)} images." + ) + elif any(n_images_in_text): + raise ValueError( + f"Found {sum(n_images_in_text)} {self.image_placeholder_token} tokens in the text but no images were passed." + ) + def replace_image_token(self, image_inputs: dict, image_idx: int, **kwargs) -> str: image_num_patches = image_inputs["num_patches"] return f"{self.start_image_token}{self.image_token * self.image_seq_length * image_num_patches[image_idx]}{self.end_image_token}" diff --git a/tests/models/phi4_multimodal/test_modeling_phi4_multimodal.py b/tests/models/phi4_multimodal/test_modeling_phi4_multimodal.py index d20eabfa60a4..7e982f0fca6e 100644 --- a/tests/models/phi4_multimodal/test_modeling_phi4_multimodal.py +++ b/tests/models/phi4_multimodal/test_modeling_phi4_multimodal.py @@ -31,6 +31,7 @@ is_torch_available, is_vision_available, ) +from transformers.audio_utils import load_audio from transformers.testing_utils import ( Expectations, cleanup, @@ -41,7 +42,6 @@ slow, torch_device, ) -from transformers.utils import is_torchcodec_available from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester @@ -57,10 +57,6 @@ from PIL import Image -if is_torchcodec_available(): - import torchcodec - - class Phi4MultimodalModelTester: def __init__( self, @@ -291,9 +287,8 @@ def setUp(self): self.assistant_token = "<|assistant|>" self.end_token = "<|end|>" self.image = Image.open(requests.get(self.image_url, stream=True).raw) - audio_bytes = requests.get(self.audio_url, stream=True).raw.data - samples = torchcodec.decoders.AudioDecoder(audio_bytes).get_all_samples() - self.audio, self.sampling_rate = samples.data, samples.sample_rate + self.sampling_rate = 16000 + self.audio = load_audio(self.audio_url, sampling_rate=self.sampling_rate) cleanup(torch_device, gc_collect=True) @@ -306,7 +301,7 @@ def test_text_only_generation(self): ) prompt = f"{self.user_token}What is the answer for 1+1? Explain it.{self.end_token}{self.assistant_token}" - inputs = self.processor(prompt, images=None, return_tensors="pt").to(torch_device) + inputs = self.processor(text=prompt, images=None, return_tensors="pt").to(torch_device) output = model.generate( **inputs, @@ -326,7 +321,7 @@ def test_vision_text_generation(self): ) prompt = f"{self.user_token}<|image|>What is shown in this image?{self.end_token}{self.assistant_token}" - inputs = self.processor(prompt, images=self.image, return_tensors="pt").to(torch_device) + inputs = self.processor(text=prompt, images=self.image, return_tensors="pt").to(torch_device) output = model.generate( **inputs, @@ -360,7 +355,7 @@ def test_multi_image_vision_text_generation(self): placeholder += "<|image|>" prompt = f"{self.user_token}{placeholder}Summarize the deck of slides.{self.end_token}{self.assistant_token}" - inputs = self.processor(prompt, images, return_tensors="pt").to(torch_device) + inputs = self.processor(text=prompt, images=images, return_tensors="pt").to(torch_device) output = model.generate( **inputs, @@ -380,9 +375,9 @@ def test_audio_text_generation(self): ) prompt = f"{self.user_token}<|audio|>What is happening in this audio?{self.end_token}{self.assistant_token}" - inputs = self.processor(prompt, audio=self.audio, sampling_rate=self.sampling_rate, return_tensors="pt").to( - torch_device - ) + inputs = self.processor( + text=prompt, audio=self.audio, sampling_rate=self.sampling_rate, return_tensors="pt" + ).to(torch_device) output = model.generate( **inputs,