From 703f75a124f11a35b478c2e028349f8633534b16 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 18 Dec 2025 20:44:48 +0100 Subject: [PATCH] make initial commit --- src/transformers/feature_extraction_utils.py | 26 +++++++---- .../image_processing_utils_fast.py | 8 ++-- .../models/llava/image_processing_llava.py | 7 ++- .../llava/image_processing_llava_fast.py | 3 +- .../models/llava/processing_llava.py | 7 ++- src/transformers/processing_utils.py | 6 +++ .../tokenization_mistral_common.py | 10 ++++- src/transformers/tokenization_python.py | 8 +++- src/transformers/tokenization_utils_base.py | 26 ++++++++--- .../tokenization_utils_tokenizers.py | 7 ++- src/transformers/video_processing_utils.py | 5 ++- tests/test_feature_extraction_common.py | 43 +++++++++++++++++- tests/test_image_processing_common.py | 20 +++++++++ tests/test_processing_common.py | 45 +++++++++++++++++++ tests/test_tokenization_common.py | 16 +++++++ tests/test_video_processing_common.py | 20 +++++++++ 16 files changed, 226 insertions(+), 31 deletions(-) diff --git a/src/transformers/feature_extraction_utils.py b/src/transformers/feature_extraction_utils.py index 20d46a0d58e5..dec8fd9717ad 100644 --- a/src/transformers/feature_extraction_utils.py +++ b/src/transformers/feature_extraction_utils.py @@ -44,6 +44,8 @@ if TYPE_CHECKING: + import torch.device + from .feature_extraction_sequence_utils import SequenceFeatureExtractor @@ -70,6 +72,8 @@ class BatchFeature(UserDict): initialization. skip_tensor_conversion (`list[str]` or `set[str]`, *optional*): List or set of keys that should NOT be converted to tensors, even when `tensor_type` is specified. + device (`Union[torch.device, str]`, *optional*): + The device to create tensots on when tensor_type is "pt". """ def __init__( @@ -77,9 +81,12 @@ def __init__( data: Optional[dict[str, Any]] = None, tensor_type: Union[None, str, TensorType] = None, skip_tensor_conversion: Optional[Union[list[str], set[str]]] = None, + device: Optional[Union["torch.device", str]] = None, ): super().__init__(data) - self.convert_to_tensors(tensor_type=tensor_type, skip_tensor_conversion=skip_tensor_conversion) + if tensor_type != "pt" and device is not None: + raise ValueError(f"Inputs cannot be moved to a device={device} when `tensor_type` is not Pytorch tensors") + self.convert_to_tensors(tensor_type=tensor_type, skip_tensor_conversion=skip_tensor_conversion, device=device) def __getitem__(self, item: str) -> Any: """ @@ -117,13 +124,13 @@ def _get_is_as_tensor_fns(self, tensor_type: Optional[Union[str, TensorType]] = raise ImportError("Unable to convert output to PyTorch tensors format, PyTorch is not installed.") import torch - def as_tensor(value): + def as_tensor(value, device=None): if torch.is_tensor(value): - return value + return value.to(device=device) # stack list of tensors if tensor_type is PyTorch (# torch.tensor() does not support list of tensors) if isinstance(value, (list, tuple)) and len(value) > 0 and torch.is_tensor(value[0]): - return torch.stack(value) + return torch.stack(value).to(device=device) # convert list of numpy arrays to numpy array (stack) if tensor_type is Numpy if isinstance(value, (list, tuple)) and len(value) > 0: @@ -136,14 +143,14 @@ def as_tensor(value): ): value = np.array(value) if isinstance(value, np.ndarray): - return torch.from_numpy(value) + return torch.from_numpy(value).to(device=device) else: - return torch.tensor(value) + return torch.tensor(value, device=device) is_tensor = torch.is_tensor else: - def as_tensor(value, dtype=None): + def as_tensor(value, dtype=None, device=None): if isinstance(value, (list, tuple)) and isinstance(value[0], (list, tuple, np.ndarray)): value_lens = [len(val) for val in value] if len(set(value_lens)) > 1 and dtype is None: @@ -158,6 +165,7 @@ def convert_to_tensors( self, tensor_type: Optional[Union[str, TensorType]] = None, skip_tensor_conversion: Optional[Union[list[str], set[str]]] = None, + device: Optional[Union["torch.device", str]] = None, ): """ Convert the inner content to tensors. @@ -168,6 +176,8 @@ def convert_to_tensors( `None`, no modification is done. skip_tensor_conversion (`list[str]` or `set[str]`, *optional*): List or set of keys that should NOT be converted to tensors, even when `tensor_type` is specified. + device (`Union[torch.device, str]`, *optional*): + The device to create tensots on when tensor_type is "pt". Note: Values that don't have an array-like structure (e.g., strings, dicts, lists of strings) are @@ -191,7 +201,7 @@ def convert_to_tensors( try: if not is_tensor(value): - tensor = as_tensor(value) + tensor = as_tensor(value, device=device) self[key] = tensor except Exception as e: if key == "overflowing_values": diff --git a/src/transformers/image_processing_utils_fast.py b/src/transformers/image_processing_utils_fast.py index 3f6f6c82b3fc..d0003e58a40d 100644 --- a/src/transformers/image_processing_utils_fast.py +++ b/src/transformers/image_processing_utils_fast.py @@ -852,7 +852,6 @@ def preprocess(self, images: ImageInput, *args, **kwargs: Unpack[ImagesKwargs]) # Extract parameters that are only used for preparing the input images do_convert_rgb = kwargs.pop("do_convert_rgb") input_data_format = kwargs.pop("input_data_format") - device = kwargs.pop("device") # Update kwargs that need further processing before being validated kwargs = self._further_process_kwargs(**kwargs) @@ -864,7 +863,7 @@ def preprocess(self, images: ImageInput, *args, **kwargs: Unpack[ImagesKwargs]) kwargs.pop("data_format") return self._preprocess_image_like_inputs( - images, *args, do_convert_rgb=do_convert_rgb, input_data_format=input_data_format, device=device, **kwargs + images, *args, do_convert_rgb=do_convert_rgb, input_data_format=input_data_format, **kwargs ) def _preprocess_image_like_inputs( @@ -885,7 +884,7 @@ def _preprocess_image_like_inputs( images = self._prepare_image_like_inputs( images=images, do_convert_rgb=do_convert_rgb, input_data_format=input_data_format, device=device ) - return self._preprocess(images, *args, **kwargs) + return self._preprocess(images, *args, device=device, **kwargs) def _preprocess( self, @@ -904,6 +903,7 @@ def _preprocess( pad_size: Optional[SizeDict], disable_grouping: Optional[bool], return_tensors: Optional[Union[str, TensorType]], + device: torch.device | str | None, **kwargs, ) -> BatchFeature: # Group images by size for batched resizing @@ -932,7 +932,7 @@ def _preprocess( if do_pad: processed_images = self.pad(processed_images, pad_size=pad_size, disable_grouping=disable_grouping) - return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors) + return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors, device=device) def to_dict(self): encoder_dict = super().to_dict() diff --git a/src/transformers/models/llava/image_processing_llava.py b/src/transformers/models/llava/image_processing_llava.py index 7c24557e5da5..5e5cf25c2fa7 100644 --- a/src/transformers/models/llava/image_processing_llava.py +++ b/src/transformers/models/llava/image_processing_llava.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for LLaVa.""" -from typing import Optional, Union +from typing import TYPE_CHECKING, Optional, Union import numpy as np @@ -45,6 +45,8 @@ logger = logging.get_logger(__name__) +if TYPE_CHECKING: + import torch.device if is_vision_available(): import PIL @@ -294,6 +296,7 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, + device: Optional[Union["torch.device", str]] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -426,7 +429,7 @@ def preprocess( image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) processed_images.append(image) - return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors) + return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors, device=device) __all__ = ["LlavaImageProcessor"] diff --git a/src/transformers/models/llava/image_processing_llava_fast.py b/src/transformers/models/llava/image_processing_llava_fast.py index e2f941d7ac49..e83c091eaffd 100644 --- a/src/transformers/models/llava/image_processing_llava_fast.py +++ b/src/transformers/models/llava/image_processing_llava_fast.py @@ -112,6 +112,7 @@ def _preprocess( image_std: Optional[Union[float, list[float]]], disable_grouping: Optional[bool], return_tensors: Optional[Union[str, TensorType]], + device: torch.device | str | None, **kwargs, ) -> BatchFeature: # Group images by size for batched resizing @@ -150,7 +151,7 @@ def _preprocess( processed_images = reorder_images(processed_images_grouped, grouped_images_index) - return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors) + return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors, device=device) __all__ = ["LlavaImageProcessorFast"] diff --git a/src/transformers/models/llava/processing_llava.py b/src/transformers/models/llava/processing_llava.py index a11e80280b74..2aac938a6b03 100644 --- a/src/transformers/models/llava/processing_llava.py +++ b/src/transformers/models/llava/processing_llava.py @@ -156,6 +156,7 @@ def __call__( prompt_strings.append(sample) return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None) + device = output_kwargs["text_kwargs"].get("device") return_mm_token_type_ids = output_kwargs["text_kwargs"].pop("return_mm_token_type_ids", False) text_inputs = self.tokenizer(prompt_strings, **output_kwargs["text_kwargs"], return_tensors=None) self._check_special_mm_tokens(prompt_strings, text_inputs, modalities=["image"]) @@ -166,7 +167,11 @@ def __call__( mm_token_type_ids[array_ids == self.image_token_id] = 1 text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist() - return BatchFeature(data={**text_inputs, **image_inputs}, tensor_type=return_tensors) + return BatchFeature( + data={**text_inputs, **image_inputs}, + tensor_type=return_tensors, + device=device, + ) def _get_num_multimodal_tokens(self, image_sizes=None, **kwargs): """ diff --git a/src/transformers/processing_utils.py b/src/transformers/processing_utils.py index f1ed964704f1..0f438451e72c 100644 --- a/src/transformers/processing_utils.py +++ b/src/transformers/processing_utils.py @@ -197,6 +197,8 @@ class TextKwargs(TypedDict, total=False): If set, will return tensors of a particular framework. Acceptable values are: - `'pt'`: Return PyTorch `torch.Tensor` objects. - `'np'`: Return NumPy `np.ndarray` objects. + device (`Union[str, torch.Tensor]`, *optional*): + The device to use for processing (e.g. "cpu", "cuda"), only relevant for fast image processing. """ text_pair: Optional[Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]]] @@ -219,6 +221,7 @@ class TextKwargs(TypedDict, total=False): padding_side: Optional[Literal["left", "right"]] return_mm_token_type_ids: Optional[bool] return_tensors: Annotated[Optional[Union[str, TensorType]], tensor_type_validator()] + device: Annotated[Optional[Union[str, "torch.device"]], device_validator()] class ImagesKwargs(TypedDict, total=False): @@ -402,6 +405,8 @@ class AudioKwargs(TypedDict, total=False): If set, will return tensors of a particular framework. Acceptable values are: - `'pt'`: Return PyTorch `torch.Tensor` objects. - `'np'`: Return NumPy `np.ndarray` objects. + device (`Union[str, torch.Tensor]`, *optional*): + The device to use for processing (e.g. "cpu", "cuda"), only relevant for fast image processing. """ sampling_rate: Annotated[Optional[int], positive_int()] @@ -412,6 +417,7 @@ class AudioKwargs(TypedDict, total=False): pad_to_multiple_of: Annotated[Optional[int], positive_int()] return_attention_mask: Optional[bool] return_tensors: Annotated[Optional[Union[str, TensorType]], tensor_type_validator()] + device: Annotated[Optional[Union[str, "torch.device"]], device_validator()] class ProcessingKwargs(TypedDict, total=False): diff --git a/src/transformers/tokenization_mistral_common.py b/src/transformers/tokenization_mistral_common.py index 3bfb52c1ab46..3622e49a1bfd 100644 --- a/src/transformers/tokenization_mistral_common.py +++ b/src/transformers/tokenization_mistral_common.py @@ -18,7 +18,7 @@ from collections.abc import Callable, Mapping, Sized from enum import Enum from pathlib import Path -from typing import Any, Union, overload +from typing import TYPE_CHECKING, Any, Optional, Union, overload import numpy as np from huggingface_hub import create_repo @@ -39,6 +39,10 @@ from transformers.utils.import_utils import is_mistral_common_available, is_torch_available, requires +if TYPE_CHECKING: + import torch.device + + if is_mistral_common_available(): from mistral_common.protocol.instruct.request import ChatCompletionRequest from mistral_common.protocol.instruct.validator import ValidationMode @@ -745,6 +749,7 @@ def _encode_plus( return_special_tokens_mask: bool = False, return_length: bool = False, verbose: bool = True, + device: Optional[Union["torch.device", str]] = None, ) -> BatchEncoding: def get_input_ids(text): if isinstance(text, str): @@ -943,6 +948,7 @@ def prepare_for_model( return_length: bool = False, verbose: bool = True, prepend_batch_axis: bool = False, + device: Optional[Union["torch.device", str]] = None, **kwargs, ) -> BatchEncoding: """ @@ -1018,7 +1024,7 @@ def prepare_for_model( encoded_inputs["length"] = len(encoded_inputs["input_ids"]) batch_outputs = BatchEncoding( - encoded_inputs, tensor_type=return_tensors, prepend_batch_axis=prepend_batch_axis + encoded_inputs, tensor_type=return_tensors, prepend_batch_axis=prepend_batch_axis, device=device ) return batch_outputs diff --git a/src/transformers/tokenization_python.py b/src/transformers/tokenization_python.py index d320cb5e9382..6387f0af2b67 100644 --- a/src/transformers/tokenization_python.py +++ b/src/transformers/tokenization_python.py @@ -19,7 +19,7 @@ import bisect import unicodedata from collections import OrderedDict -from typing import Any, overload +from typing import TYPE_CHECKING, Any, Optional, Union, overload from .tokenization_utils_base import ( INIT_TOKENIZER_DOCSTRING, @@ -34,6 +34,9 @@ from .utils import PaddingStrategy, TensorType, add_end_docstrings, logging +if TYPE_CHECKING: + import torch.device + logger = logging.get_logger(__name__) # Slow tokenizers are saved in a vocabulary plus three separated files @@ -711,6 +714,7 @@ def _encode_plus( return_special_tokens_mask: bool = False, return_length: bool = False, verbose: bool = True, + device: Optional[Union["torch.device", str]] = None, **kwargs, ) -> BatchEncoding: # Detect batched inputs (list of sequences) @@ -789,7 +793,7 @@ def _encode_plus( return_attention_mask=return_attention_mask, ) - return BatchEncoding(batch_outputs, tensor_type=return_tensors) + return BatchEncoding(batch_outputs, tensor_type=return_tensors, device=device) # Single sequence handling def get_input_ids(text): diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 540683aff3e5..2a444f901b8a 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -215,6 +215,8 @@ class BatchEncoding(UserDict): n_sequences (`Optional[int]`, *optional*): You can give a tensor_type here to convert the lists of integers in PyTorch/Numpy Tensors at initialization. + device (`Union[torch.device, str]`, *optional*): + The device to create tensots on when tensor_type is "pt". """ def __init__( @@ -224,9 +226,13 @@ def __init__( tensor_type: Union[None, str, TensorType] = None, prepend_batch_axis: bool = False, n_sequences: Optional[int] = None, + device: Optional[Union[torch.device, str]] = None, ): super().__init__(data) + if tensor_type != "pt" and device is not None: + raise ValueError(f"Inputs cannot be moved to a device={device} when `tensor_type` is not Pytorch tensors") + # If encoding is not None, the fast tokenization is used if encoding is not None and isinstance(encoding, EncodingFast): encoding = [encoding] @@ -238,7 +244,7 @@ def __init__( self._n_sequences = n_sequences - self.convert_to_tensors(tensor_type=tensor_type, prepend_batch_axis=prepend_batch_axis) + self.convert_to_tensors(tensor_type=tensor_type, prepend_batch_axis=prepend_batch_axis, device=device) @property def n_sequences(self) -> Optional[int]: @@ -663,7 +669,10 @@ def char_to_word(self, batch_or_char_index: int, char_index: Optional[int] = Non return self._encodings[batch_index].char_to_word(char_index, sequence_index) def convert_to_tensors( - self, tensor_type: Optional[Union[str, TensorType]] = None, prepend_batch_axis: bool = False + self, + tensor_type: Optional[Union[str, TensorType]] = None, + prepend_batch_axis: bool = False, + device: Optional[Union[torch.device, str]] = None, ): """ Convert the inner content to tensors. @@ -674,6 +683,8 @@ def convert_to_tensors( `None`, no modification is done. prepend_batch_axis (`int`, *optional*, defaults to `False`): Whether or not to add the batch dimension during the conversion. + device (`Union[torch.device, str]`, *optional*): + The device to create tensots on when tensor_type is "pt". """ if tensor_type is None: return self @@ -687,12 +698,12 @@ def convert_to_tensors( raise ImportError("Unable to convert output to PyTorch tensors format, PyTorch is not installed.") import torch - def as_tensor(value, dtype=None): + def as_tensor(value, dtype=None, device=None): if isinstance(value, list) and len(value) > 0 and isinstance(value[0], np.ndarray): return torch.from_numpy(np.array(value)) if len(flatten(value)) == 0 and dtype is None: dtype = torch.int64 - return torch.tensor(value, dtype=dtype) + return torch.tensor(value, dtype=dtype, device=device) is_tensor = torch.is_tensor @@ -701,7 +712,7 @@ def as_tensor(value, dtype=None): raise ImportError("Unable to convert output to MLX tensors format, MLX is not installed.") import mlx.core as mx - def as_tensor(value, dtype=None): + def as_tensor(value, dtype=None, device=None): if len(flatten(value)) == 0 and dtype is None: dtype = mx.int32 return mx.array(value, dtype=dtype) @@ -710,7 +721,7 @@ def is_tensor(obj): return isinstance(obj, mx.array) else: - def as_tensor(value, dtype=None): + def as_tensor(value, dtype=None, device=None): if ( isinstance(value, (list, tuple)) and len(value) > 0 @@ -733,7 +744,7 @@ def as_tensor(value, dtype=None): value = [value] if not is_tensor(value): - tensor = as_tensor(value) + tensor = as_tensor(value, device=device) # Removing this for now in favor of controlling the shape with `prepend_batch_axis` # # at-least2d @@ -2616,6 +2627,7 @@ def _encode_plus( return_length: bool = False, verbose: bool = True, split_special_tokens: bool = False, + device: Optional[Union[torch.device, str]] = None, **kwargs, ) -> BatchEncoding: raise NotImplementedError diff --git a/src/transformers/tokenization_utils_tokenizers.py b/src/transformers/tokenization_utils_tokenizers.py index ccc30f133ec3..f53f21c04732 100644 --- a/src/transformers/tokenization_utils_tokenizers.py +++ b/src/transformers/tokenization_utils_tokenizers.py @@ -22,7 +22,7 @@ from collections import defaultdict from collections.abc import Iterable from shutil import copyfile -from typing import Any, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union import tokenizers.pre_tokenizers as pre_tokenizers_fast from huggingface_hub import is_offline_mode @@ -46,6 +46,9 @@ from .utils import PaddingStrategy, add_end_docstrings, logging +if TYPE_CHECKING: + import torch.device + logger = logging.get_logger(__name__) # Fast tokenizers (provided by HuggingFace tokenizer's library) can be saved in a single file @@ -748,6 +751,7 @@ def _encode_plus( return_length: bool = False, verbose: bool = True, split_special_tokens: Optional[bool] = None, + device: Optional[Union["torch.device", str]] = None, **kwargs, ) -> BatchEncoding: # Input validation (from _call_one) @@ -877,6 +881,7 @@ def _is_valid_text_input(t): for key, value in batched_output.items() }, batched_output.encodings, + device=device, ) return batched_output diff --git a/src/transformers/video_processing_utils.py b/src/transformers/video_processing_utils.py index e62693efdd83..a0b0a1023b27 100644 --- a/src/transformers/video_processing_utils.py +++ b/src/transformers/video_processing_utils.py @@ -376,7 +376,7 @@ def preprocess( input_data_format = kwargs.pop("input_data_format") do_sample_frames = kwargs.pop("do_sample_frames") - device = kwargs.pop("device") + device = kwargs.get("device") video_metadata = kwargs.pop("video_metadata") sample_indices_fn = partial(self.sample_frames, **kwargs) if do_sample_frames else None @@ -415,6 +415,7 @@ def _preprocess( image_mean: Optional[Union[float, list[float]]], image_std: Optional[Union[float, list[float]]], return_tensors: Optional[Union[str, TensorType]] = None, + device: torch.device | str | None = None, **kwargs, ) -> BatchFeature: # Group videos by size for batched resizing @@ -443,7 +444,7 @@ def _preprocess( processed_videos = reorder_videos(processed_videos_grouped, grouped_videos_index) - return BatchFeature(data={"pixel_values_videos": processed_videos}, tensor_type=return_tensors) + return BatchFeature(data={"pixel_values_videos": processed_videos}, tensor_type=return_tensors, device=device) @classmethod def from_pretrained( diff --git a/tests/test_feature_extraction_common.py b/tests/test_feature_extraction_common.py index be90ee316e9a..cca9e9e16f29 100644 --- a/tests/test_feature_extraction_common.py +++ b/tests/test_feature_extraction_common.py @@ -15,14 +15,37 @@ import json import os +import random import tempfile -from transformers.testing_utils import check_json_file_has_correct_format +import numpy as np + +from transformers.testing_utils import check_json_file_has_correct_format, require_torch +from transformers.utils import is_torch_available + + +if is_torch_available(): + import torch class FeatureExtractionSavingTestMixin: test_cast_dtype = None + def prepare_audio_inputs(batch_size, seq_len, scale=1.0, rng=None, torchify=False, numpify=False): + rng = random.Random() + values = [] + for batch_idx in range(batch_size): + values.append([]) + for _ in range(seq_len): + values[-1].append(rng.random() * scale) + + if numpify: + values = np.array(values) + elif torchify: + values = torch.tensor(values) + + return values + def test_feat_extract_to_json_string(self): feat_extract = self.feature_extraction_class(**self.feat_extract_dict) obj = json.loads(feat_extract.to_json_string()) @@ -52,3 +75,21 @@ def test_feat_extract_from_and_save_pretrained(self): def test_init_without_params(self): feat_extract = self.feature_extraction_class() self.assertIsNotNone(feat_extract) + + @require_torch + def test_call_with_device(self): + feature_extractor = self.feature_extraction_class(**self.feat_extract_dict) + list_audio_inputs = self.prepare_audio_inputs(numpify=True) + numpy_audio_inputs = self.prepare_audio_inputs(numpify=True) + torch_audio_inputs = self.prepare_audio_inputs(torchify=True) + + for inputs in [list_audio_inputs, numpy_audio_inputs, torch_audio_inputs]: + encoded_audio = feature_extractor(inputs, return_tensors="pt") + for key in encoded_audio: + self.assertIsInstance(encoded_audio[key], torch.Tensor) + self.assertTrue(encoded_audio[key].device.type == "cpu") + + encoded_audio = feature_extractor(inputs, return_tensors="pt", device="cuda") + for key in encoded_audio: + self.assertIsInstance(encoded_audio[key], torch.Tensor) + self.assertTrue(encoded_audio[key].device.type == "cuda") diff --git a/tests/test_image_processing_common.py b/tests/test_image_processing_common.py index 9a7f0996bf01..e0bf5dbdfbbc 100644 --- a/tests/test_image_processing_common.py +++ b/tests/test_image_processing_common.py @@ -441,6 +441,26 @@ def test_cast_dtype_device(self): self.assertEqual(encoding.pixel_values.dtype, torch.float16) self.assertEqual(encoding.input_ids.dtype, torch.long) + def test_call_with_device(self): + for image_processing_class in self.image_processor_list: + image_processing = image_processing_class(**self.image_processor_dict) + pil_image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False) + numpy_image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, numpify=True) + torch_image_inputs = self.image_processor_tester.prepare_image_inputs( + equal_resolution=False, torchify=True + ) + + for inputs in [pil_image_inputs, numpy_image_inputs, torch_image_inputs]: + encoded_images = image_processing(inputs, return_tensors="pt") + for key in encoded_images: + self.assertIsInstance(encoded_images[key], torch.Tensor) + self.assertTrue(encoded_images[key].device.type == "cpu") + + encoded_images = image_processing(inputs, return_tensors="pt", device="cuda") + for key in encoded_images: + self.assertIsInstance(encoded_images[key], torch.Tensor) + self.assertTrue(encoded_images[key].device.type == "cuda") + def test_call_pil(self): for image_processing_class in self.image_processor_list: # Initialize image_processing diff --git a/tests/test_processing_common.py b/tests/test_processing_common.py index cf506c7323f0..5a89d3918bf3 100644 --- a/tests/test_processing_common.py +++ b/tests/test_processing_common.py @@ -829,6 +829,51 @@ def test_processor_text_has_no_visual(self): ) self.assertTrue(self.text_input_name in inputs_chat_template) + def test_batch_feature_device_placement(self): + processor = self.get_processor() + attributes = self.processor_class.get_attributes() + + # Map attributes to input parameter names, prepare methods, and output key names + attr_to_input_param = { + "tokenizer": ("text", "prepare_text_inputs"), + "image_processor": ("images", "prepare_image_inputs"), + "video_processor": ("videos", "prepare_video_inputs"), + "feature_extractor": ("audio", "prepare_audio_inputs"), + } + + # Prepare inputs dynamically based on processor attributes + processor_inputs = {} + + for attr in attributes: + if attr in attr_to_input_param: + param_name, prepare_method_name = attr_to_input_param[attr] + # Call the prepare method + prepare_method = getattr(self, prepare_method_name) + if param_name == "text": + modalities = [] + if "image_processor" in attributes: + modalities.append("image") + if "video_processor" in attributes: + modalities.append("video") + if "audio_processor" in attributes or "feature_extractor" in attributes: + modalities.append("audio") + processor_inputs[param_name] = prepare_method(modalities=modalities) + else: + processor_inputs[param_name] = prepare_method() + + inputs = processor(**processor_inputs, return_tensors="pt") + for key in inputs: + self.assertIsInstance(inputs[key], torch.Tensor) + self.assertTrue(inputs[key].device.type == "cpu") + + inputs = processor(**processor_inputs, return_tensors="pt", device="cuda") + for key in inputs: + self.assertIsInstance(inputs[key], torch.Tensor) + self.assertTrue(inputs[key].device.type == "cuda") + + with self.assertRaises(ValueError): + inputs = processor(**processor_inputs, return_tensors="np", device="cuda") + # These kwargs-related tests ensure that processors are correctly instantiated. # they need to be applied only if an image_processor exists. diff --git a/tests/test_tokenization_common.py b/tests/test_tokenization_common.py index f901630ee3d2..2262285c2609 100644 --- a/tests/test_tokenization_common.py +++ b/tests/test_tokenization_common.py @@ -599,6 +599,22 @@ def convert_batch_to_list_format(batch_encode_plus_sequences): for i in range(len(batch_encode_plus_sequences["input_ids"])) ] + def test_tokenize_with_device(self): + tokenizer = self.get_tokenizer(do_lower_case=True) + + inputs = tokenizer("lower newer", return_tensors="pt") + for key in inputs: + self.assertIsInstance(inputs[key], torch.Tensor) + self.assertTrue(inputs[key].device.type == "cpu") + + inputs = tokenizer("lower newer", return_tensors="pt", device="cuda") + for key in inputs: + self.assertIsInstance(inputs[key], torch.Tensor) + self.assertTrue(inputs[key].device.type == "cuda") + + with self.assertRaises(ValueError): + inputs = tokenizer("lower newer", return_tensors="np", device="cuda") + # TODO: this test can be combined with `test_sentencepiece_tokenize_and_convert_tokens_to_string` after the latter is extended to all tokenizers. def test_tokenize_special_tokens(self): """Test `tokenize` with special tokens.""" diff --git a/tests/test_video_processing_common.py b/tests/test_video_processing_common.py index b2faa846204a..0a27556e01ee 100644 --- a/tests/test_video_processing_common.py +++ b/tests/test_video_processing_common.py @@ -258,6 +258,26 @@ def test_cast_dtype_device(self): self.assertEqual(encoding[self.input_name].dtype, torch.float16) self.assertEqual(encoding.input_ids.dtype, torch.long) + def test_call_with_device(self): + for video_processing_class in self.video_processor_list: + video_processing = video_processing_class(**self.video_processor_dict) + pil_video_inputs = self.video_processor_tester.prepare_video_inputs(equal_resolution=False) + numpy_video_inputs = self.video_processor_tester.prepare_video_inputs(equal_resolution=False, numpify=True) + torch_video_inputs = self.video_processor_tester.prepare_video_inputs( + equal_resolution=False, torchify=True + ) + + for inputs in [pil_video_inputs, numpy_video_inputs, torch_video_inputs]: + encoded_videos = video_processing(inputs, return_tensors="pt") + for key in encoded_videos: + self.assertIsInstance(encoded_videos[key], torch.Tensor) + self.assertTrue(encoded_videos[key].device.type == "cpu") + + encoded_videos = video_processing(inputs, return_tensors="pt", device="cuda") + for key in encoded_videos: + self.assertIsInstance(encoded_videos[key], torch.Tensor) + self.assertTrue(encoded_videos[key].device.type == "cuda") + def test_call_pil(self): for video_processing_class in self.video_processor_list: # Initialize video_processing