Remove infer_device

docs/source/en/cache_explanation.md

@@ -98,9 +98,10 @@ The example below demonstrates how to create a generation loop with [`DynamicCac
 
 ```py
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, DynamicCache, infer_device
+from transformers import AutoTokenizer, AutoModelForCausalLM, DynamicCache
+from accelerate import Accelerator
 
-device = f"{infer_device()}:0"
+device = Accelerator().device
 
 model_id = "meta-llama/Llama-2-7b-chat-hf"
 model = AutoModelForCausalLM.from_pretrained(model_id, dtype=torch.bfloat16, device_map=device)
@@ -143,9 +144,10 @@ The generation loop usually takes care of the cache position, but if you're writ
 
 ```py
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, DynamicCache, infer_device
+from transformers import AutoTokenizer, AutoModelForCausalLM, DynamicCache
+from accelerate import Accelerator
 
-device = f"{infer_device()}:0"
+device = Accelerator().device
 
 model_id = "meta-llama/Llama-2-7b-chat-hf"
 model = AutoModelForCausalLM.from_pretrained(model_id, dtype=torch.bfloat16, device_map=device)

docs/source/en/generation_strategies.md

@@ -32,9 +32,10 @@ Greedy search works well for tasks with relatively short outputs where creativit
 
 ```py
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, infer_device
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from accelerate import Accelerator
 
-device = infer_device()
+device = Accelerator().device
 
 tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
 inputs = tokenizer("Hugging Face is an open-source company", return_tensors="pt").to(device)
@@ -54,9 +55,10 @@ Enable multinomial sampling with `do_sample=True` and `num_beams=1`.
 
 ```py
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, infer_device
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from accelerate import Accelerator
 
-device = infer_device()
+device = Accelerator().device
 
 tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
 inputs = tokenizer("Hugging Face is an open-source company", return_tensors="pt").to(device)
@@ -79,9 +81,10 @@ Enable beam search with the `num_beams` parameter (should be greater than 1 othe
 
 ```py
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, infer_device
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from accelerate import Accelerator
 
-device = infer_device()
+device = Accelerator().device
 
 tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
 inputs = tokenizer("Hugging Face is an open-source company", return_tensors="pt").to(device)
@@ -166,9 +169,10 @@ Enable prompt lookup decoding with the `prompt_lookup_num_tokens` parameter.
 
 ```py
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, infer_device
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from accelerate import Accelerator
 
-device = infer_device()
+device = Accelerator().device
 
 tokenizer = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM-1.7B")
 model = AutoModelForCausalLM.from_pretrained("HuggingFaceTB/SmolLM-1.7B", dtype=torch.float16).to(device)

docs/source/en/internal/file_utils.md

@@ -43,4 +43,3 @@ Most of those are only useful if you are studying the general code in the librar
 ## Other Utilities
 
 [[autodoc]] utils._LazyModule
-[[autodoc]] pytorch_utils.infer_device

docs/source/en/kv_cache.md

@@ -124,11 +124,12 @@ The example below shows how you can fallback to an offloaded cache if you run ou
 
 ```py
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, infer_device
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from accelerate import Accelerator
 
 def resilient_generate(model, *args, **kwargs):
     oom = False
-    device = infer_device()
+    device = Accelerator().device
     torch_device_module = getattr(torch, device, torch.cuda)
     try:
         return model.generate(*args, **kwargs)

docs/source/en/llm_optims.md

@@ -114,7 +114,8 @@ print(tokenizer.batch_decode(outputs, skip_special_tokens=True))
 Another option for using [`StaticCache`] is to pass it to a models forward pass using the same `past_key_values` argument. This allows you to write your own custom decoding function to decode the next token given the current token, position, and cache position of previously generated tokens.
 
 ```py
-from transformers import LlamaTokenizer, LlamaForCausalLM, StaticCache, logging, infer_device
+from transformers import LlamaTokenizer, LlamaForCausalLM, StaticCache, logging
+from accelerate import Accelerator
 from transformers.testing_utils import CaptureLogger
 import torch
 
@@ -124,7 +125,7 @@ prompts = [
 ]
 
 NUM_TOKENS_TO_GENERATE = 40
-torch_device = infer_device()
+torch_device = Accelerator().device
 
 tokenizer = LlamaTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf", pad_token="</s>", padding_side="right")
 model = LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf", device_map="sequential")
@@ -208,10 +209,11 @@ Enable speculative decoding by loading an assistant model and passing it to [`~G
 <hfoption id="greedy search">
 
 ```py
-from transformers import AutoModelForCausalLM, AutoTokenizer, infer_device
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from accelerate import Accelerator
 import torch
 
-device = infer_device()
+device = Accelerator().device
 
 tokenizer = AutoTokenizer.from_pretrained("facebook/opt-1.3b")
 inputs = tokenizer("Einstein's theory of relativity states", return_tensors="pt").to(device)
@@ -229,10 +231,11 @@ tokenizer.batch_decode(outputs, skip_special_tokens=True)
 For speculative sampling decoding, add the [do_sample](https://hf.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig.do_sample) and [temperature](https://hf.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig.temperature) parameters to [`~GenerationMixin.generate`].
 
 ```py
-from transformers import AutoModelForCausalLM, AutoTokenizer, infer_device
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from accelerate import Accelerator
 import torch
 
-device = infer_device()
+device = Accelerator().device
 
 tokenizer = AutoTokenizer.from_pretrained("facebook/opt-1.3b")
 inputs = tokenizer("Einstein's theory of relativity states", return_tensors="pt").to(device)
@@ -257,10 +260,11 @@ To enable prompt lookup decoding, specify the number of tokens that should be ov
 <hfoption id="greedy decoding">
 
 ```py
-from transformers import AutoModelForCausalLM, AutoTokenizer, infer_device
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from accelerate import Accelerator
 import torch
 
-device = infer_device()
+device = Accelerator().device
 
 tokenizer = AutoTokenizer.from_pretrained("facebook/opt-1.3b")
 inputs = tokenizer("The second law of thermodynamics states", return_tensors="pt").to(device)
@@ -278,10 +282,11 @@ print(tokenizer.batch_decode(outputs, skip_special_tokens=True))
 For prompt lookup decoding with sampling, add the [do_sample](https://hf.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig.do_sample) and [temperature](https://hf.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig.temperature) parameters to [`~GenerationMixin.generate`].
 
 ```py
-from transformers import AutoModelForCausalLM, AutoTokenizer, infer_device
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from accelerate import Accelerator
 import torch
 
-device = infer_device()
+device = Accelerator().device
 
 tokenizer = AutoTokenizer.from_pretrained("facebook/opt-1.3b")
 inputs = tokenizer("The second law of thermodynamics states", return_tensors="pt").to(device)

docs/source/en/model_doc/bark.md

@@ -43,10 +43,11 @@ Bark can be optimized with just a few extra lines of code, which **significantly
 You can speed up inference and reduce memory footprint by 50% simply by loading the model in half-precision.
 
 ```python
-from transformers import BarkModel, infer_device
+from transformers import BarkModel
+from accelerate import Accelerator
 import torch
 
-device = infer_device()
+device = Accelerator().device
 model = BarkModel.from_pretrained("suno/bark-small", dtype=torch.float16).to(device)
 ```
 
@@ -98,10 +99,11 @@ To put this into perspective, on an NVIDIA A100 and when generating 400 semantic
 You can combine optimization techniques, and use CPU offload, half-precision and Flash Attention 2 all at once.
 
 ```python
-from transformers import BarkModel, infer_device
+from transformers import BarkModel
+from accelerate import Accelerator
 import torch
 
-device = infer_device()
+device = Accelerator().device
 
 # load in fp16 and use Flash Attention 2
 model = BarkModel.from_pretrained("suno/bark-small", dtype=torch.float16, attn_implementation="flash_attention_2").to(device)

docs/source/en/model_doc/colqwen2.md

@@ -107,10 +107,11 @@ import requests
 import torch
 from PIL import Image
 
-from transformers import BitsAndBytesConfig, ColQwen2ForRetrieval, ColQwen2Processor, infer_device
+from transformers import BitsAndBytesConfig, ColQwen2ForRetrieval, ColQwen2Processor
+from accelerate import Accelerator
 
 model_name = "vidore/colqwen2-v1.0-hf"
-device = infer_device()
+device = Accelerator().device
 
 # 4-bit quantization configuration
 bnb_config = BitsAndBytesConfig(

docs/source/en/model_doc/csm.md

@@ -38,10 +38,11 @@ CSM can be used to simply generate speech from a text prompt:
 
 ```python
 import torch
-from transformers import CsmForConditionalGeneration, AutoProcessor, infer_device
+from transformers import CsmForConditionalGeneration, AutoProcessor
+from accelerate import Accelerator
 
 model_id = "sesame/csm-1b"
-device = infer_device()
+device = Accelerator().device
 
 # load the model and the processor
 processor = AutoProcessor.from_pretrained(model_id)
@@ -72,11 +73,12 @@ CSM can be used to generate speech given a conversation, allowing consistency in
 
 ```python
 import torch
-from transformers import CsmForConditionalGeneration, AutoProcessor, infer_device
+from transformers import CsmForConditionalGeneration, AutoProcessor
+from accelerate import Accelerator
 from datasets import load_dataset, Audio
 
 model_id = "sesame/csm-1b"
-device = infer_device()
+device = Accelerator().device
 
 # load the model and the processor
 processor = AutoProcessor.from_pretrained(model_id)
@@ -117,11 +119,12 @@ CSM supports batched inference!
 
 ```python
 import torch
-from transformers import CsmForConditionalGeneration, AutoProcessor, infer_device
+from transformers import CsmForConditionalGeneration, AutoProcessor
+from accelerate import Accelerator
 from datasets import load_dataset, Audio
 
 model_id = "sesame/csm-1b"
-device = infer_device()
+device = Accelerator().device
 
 # load the model and the processor
 processor = AutoProcessor.from_pretrained(model_id)
@@ -306,11 +309,12 @@ print("="*50)
 CSM Transformers integration supports training!
 
 ```python
-from transformers import CsmForConditionalGeneration, AutoProcessor, infer_device
+from transformers import CsmForConditionalGeneration, AutoProcessor
+from accelerate import Accelerator
 from datasets import load_dataset, Audio
 
 model_id = "sesame/csm-1b"
-device = infer_device()
+device = Accelerator().device
 
 # load the model and the processor
 processor = AutoProcessor.from_pretrained(model_id)

docs/source/en/model_doc/depth_pro.md

@@ -46,9 +46,10 @@ The DepthPro model processes an input image by first downsampling it at multiple
 >>> import requests
 >>> from PIL import Image
 >>> import torch
->>> from transformers import DepthProImageProcessorFast, DepthProForDepthEstimation, infer_device
+>>> from transformers import DepthProImageProcessorFast, DepthProForDepthEstimation
+from accelerate import Accelerator
 
->>> device = infer_device()
+>>> device = Accelerator().device
 
 >>> url = 'http://images.cocodataset.org/val2017/000000039769.jpg'
 >>> image = Image.open(requests.get(url, stream=True).raw)

docs/source/en/model_doc/dia.md

@@ -42,9 +42,10 @@ tokens and decodes them back into audio.
 ### Generation with Text
 
 ```python
-from transformers import AutoProcessor, DiaForConditionalGeneration, infer_device
+from transformers import AutoProcessor, DiaForConditionalGeneration
+from accelerate import Accelerator
 
-torch_device = infer_device()
+torch_device = Accelerator().device
 model_checkpoint = "nari-labs/Dia-1.6B-0626"
 
 text = ["[S1] Dia is an open weights text to dialogue model."]
@@ -64,9 +65,10 @@ processor.save_audio(outputs, "example.wav")
 
 ```python
 from datasets import load_dataset, Audio
-from transformers import AutoProcessor, DiaForConditionalGeneration, infer_device
+from transformers import AutoProcessor, DiaForConditionalGeneration
+from accelerate import Accelerator
 
-torch_device = infer_device()
+torch_device = Accelerator().device
 model_checkpoint = "nari-labs/Dia-1.6B-0626"
 
 ds = load_dataset("hf-internal-testing/dailytalk-dummy", split="train")
@@ -91,9 +93,10 @@ processor.save_audio(outputs, "example_with_audio.wav")
 
 ```python
 from datasets import load_dataset, Audio
-from transformers import AutoProcessor, DiaForConditionalGeneration, infer_device
+from transformers import AutoProcessor, DiaForConditionalGeneration
+from accelerate import Accelerator
 
-torch_device = infer_device()
+torch_device = Accelerator().device
 model_checkpoint = "nari-labs/Dia-1.6B-0626"
 
 ds = load_dataset("hf-internal-testing/dailytalk-dummy", split="train")

docs/source/en/model_doc/donut.md

@@ -119,14 +119,15 @@ print(answer)
 
     ```py
     >>> import re
-    >>> from transformers import DonutProcessor, VisionEncoderDecoderModel, infer_device
+    >>> from transformers import DonutProcessor, VisionEncoderDecoderModel
+from accelerate import Accelerator
     >>> from datasets import load_dataset
     >>> import torch
 
     >>> processor = DonutProcessor.from_pretrained("naver-clova-ix/donut-base-finetuned-rvlcdip")
     >>> model = VisionEncoderDecoderModel.from_pretrained("naver-clova-ix/donut-base-finetuned-rvlcdip")
 
-    >>> device = infer_device()
+    >>> device = Accelerator().device
     >>> model.to(device)  # doctest: +IGNORE_RESULT
 
     >>> # load document image
@@ -161,14 +162,15 @@ print(answer)
 
     ```py
     >>> import re
-    >>> from transformers import DonutProcessor, VisionEncoderDecoderModel, infer_device
+    >>> from transformers import DonutProcessor, VisionEncoderDecoderModel
+from accelerate import Accelerator
     >>> from datasets import load_dataset
     >>> import torch
 
     >>> processor = DonutProcessor.from_pretrained("naver-clova-ix/donut-base-finetuned-cord-v2")
     >>> model = VisionEncoderDecoderModel.from_pretrained("naver-clova-ix/donut-base-finetuned-cord-v2")
 
-    >>> device = infer_device()
+    >>> device = Accelerator().device
     >>> model.to(device)  # doctest: +IGNORE_RESULT
 
     >>> # load document image

docs/source/en/model_doc/edgetam.md

@@ -61,12 +61,13 @@ EdgeTAM can be used for automatic mask generation to segment all objects in an i
 You can segment objects by providing a single point click on the object you want to segment:
 
 ```python
->>> from transformers import Sam2Processor, EdgeTamModel, infer_device
+>>> from transformers import Sam2Processor, EdgeTamModel
+from accelerate import Accelerator
 >>> import torch
 >>> from PIL import Image
 >>> import requests
 
->>> device = infer_device()
+>>> device = Accelerator().device
 
 >>> model = EdgeTamModel.from_pretrained("yonigozlan/edgetam-1").to(device)
 >>> processor = Sam2Processor.from_pretrained("yonigozlan/edgetam-1")
@@ -157,12 +158,13 @@ IoU scores: tensor([0.7616, 0.9465], device='cuda:0')
 Process multiple images simultaneously for improved efficiency:
 
 ```python
->>> from transformers import Sam2Processor, EdgeTamModel, infer_device
+>>> from transformers import Sam2Processor, EdgeTamModel
+from accelerate import Accelerator
 >>> import torch
 >>> from PIL import Image
 >>> import requests
 
->>> device = infer_device()
+>>> device = Accelerator().device
 
 >>> model = EdgeTamModel.from_pretrained("yonigozlan/edgetam-1").to(device)
 >>> processor = Sam2Processor.from_pretrained("yonigozlan/edgetam-1")