use TokenizersBackend (#42894)

* us `TokenizersBackend`

* fixes

* pioritize mapping

* pioritize mapping

* only use mapping for some models

* fix fallback

* undo debug thing

* add case to tokenizersbackend init

* add default bos eos token to tok backend

* set bos eos

* fix more models

* mistrla idefics

* fix stopping criteria test

* fix stopping criteria test

* try stopping criteria fix

* rebase

* update tokenizer model for stopping criteria test

* fix tuple mapping for ministral

* ignore `tokenizer_class` as it is always wrong

* up

* try to fix idefics

* fix unispeech and maybe other: fallback if conversion was not possible to the saveclass

* nits

* fixup

* TIL that it was ALSO saved in config.json...

* arf

* fallback to tok config if no config json

* people who map to Llama probably don't even want llama either..

* processors to load tokbackend

* auto fix order

* try diff order

* mistral fix for weird chars

* reorder

* random fix attempt for failing tests that are failing locally so idk how to check these

* trying an older commit

* fix mistral

* map unispeech

* try something out

* update

* nits

* trying to be a little bit more restrictive

* token type ids for tokenizers should be explicits... let's see which test fail this and we'll add to the specific classes?

* Nit

* idefics 1-2 are actually the only ones that should map to llama force

* small fixes

* fix layout

* fixup

* fix some tests

* 1 nit

* aria fix

* style

* canine

* fixup

* very small test

* style

* update to tokenizersbackend

---------

Co-authored-by: [email protected] <[email protected]>
Co-authored-by: [email protected] <[email protected]>
Co-authored-by: [email protected] <[email protected]>
Co-authored-by: [email protected] <[email protected]>
Co-authored-by: [email protected] <[email protected]>
Co-authored-by: itazap <[email protected]>
Co-authored-by: Ita Zaporozhets <[email protected]>
Co-authored-by: [email protected] <[email protected]>
Co-authored-by: [email protected] <[email protected]>

Author: 9 people

docs/source/en/model_doc/parakeet.md

@@ -188,9 +188,9 @@ outputs = model(**inputs)
 outputs.loss.backward()
 ```
 
-## ParakeetTokenizerFast
+## ParakeetTokenizer
 
-[[autodoc]] ParakeetTokenizerFast
+[[autodoc]] ParakeetTokenizer
 
 ## ParakeetFeatureExtractor
 

src/transformers/integrations/accelerate.py

@@ -293,7 +293,7 @@ def _get_device_map(
         # especially if the model uses WeightConverter (because there will be some uncontrollable cpu memory spikes during
         # the conversions before we resave the weights). In those cases, it's better to offload to disk a bit more
         # if we were in-between, as otherwise we blow-up cpu memory
-        if max_memory is None:
+        if max_memory is None and "cpu" in inferred_max_memory:
             inferred_max_memory["cpu"] *= 0.90
 
         if hf_quantizer is not None:

src/transformers/models/auto/tokenization_auto.py

@@ -160,13 +160,6 @@ def __init__(
 
         self._tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=add_prefix_space)
         self._tokenizer.decoder = decoders.ByteLevel()
-        self._tokenizer.post_processor = processors.RobertaProcessing(
-            sep=(str(eos_token), self._vocab.get(str(eos_token), 2)),
-            cls=(str(bos_token), self._vocab.get(str(bos_token), 0)),
-            add_prefix_space=add_prefix_space,
-            trim_offsets=True,
-        )
-
         super().__init__(
             bos_token=bos_token,
             eos_token=eos_token,
@@ -178,6 +171,12 @@ def __init__(
             add_prefix_space=add_prefix_space,
             **kwargs,
         )
+        self._tokenizer.post_processor = processors.RobertaProcessing(
+            sep=(str(eos_token), self.eos_token_id),
+            cls=(str(bos_token), self.bos_token_id),
+            add_prefix_space=add_prefix_space,
+            trim_offsets=True,
+        )
 
 
 __all__ = ["BlenderbotTokenizer"]

src/transformers/models/blenderbot/tokenization_blenderbot.py

@@ -67,6 +67,8 @@ class CanineTokenizer(PreTrainedTokenizer):
                 The maximum sentence length the model accepts.
     """
 
+    model_input_names = ["input_ids", "attention_mask", "token_type_ids"]
+
     def __init__(
         self,
         bos_token=chr(CLS),

src/transformers/models/canine/tokenization_canine.py

@@ -158,7 +158,7 @@ def __init__(
                 unk_token=str(unk_token),
             )
         )
-        prepend_scheme = "first" if self.add_prefix_space else "none"
+        prepend_scheme = "first" if self.add_prefix_space else "never"
         self._tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(
             replacement="▁", prepend_scheme=prepend_scheme, split=False
         )

src/transformers/models/code_llama/tokenization_code_llama.py

@@ -160,6 +160,7 @@ class LayoutLMv2Tokenizer(TokenizersBackend):
 
     vocab_files_names = VOCAB_FILES_NAMES
     model = models.WordPiece
+    model_input_names = ["input_ids", "attention_mask", "token_type_ids"]
 
     def __init__(
         self,

src/transformers/models/layoutlmv2/tokenization_layoutlmv2.py

@@ -441,31 +441,26 @@ def __init__(
         )
         self._tokenizer.decoder = decoders.ByteLevel(add_prefix_space=True, trim_offsets=True, use_regex=True)
 
-        # Set up post processor with bos and eos tokens
-        bos_token_id = self._vocab.get(str(bos_token), 0)
-        eos_token_id = self._vocab.get(str(eos_token), 2)
-        pad_token_id = self._vocab.get(str(pad_token), 1)
+        super().__init__(
+            errors=errors,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            **kwargs,
+        )
         self._tokenizer.post_processor = processors.TemplateProcessing(
             single=f"{bos_token}:0 $A:0 {eos_token}:0",
             pair="$A:0 $B:1",
             special_tokens=[
-                (str(eos_token), eos_token_id),
-                (str(bos_token), bos_token_id),
+                (str(eos_token), self.eos_token_id),
+                (str(bos_token), self.bos_token_id),
             ],
         )
 
         # Enable truncation and padding
         self._tokenizer.enable_truncation(max_length=4096)
-        self._tokenizer.enable_padding(length=4096, pad_id=pad_token_id, pad_token=str(pad_token))
-
-        super().__init__(
-            errors=errors,
-            unk_token=unk_token,
-            bos_token=bos_token,
-            eos_token=eos_token,
-            pad_token=pad_token,
-            **kwargs,
-        )
+        self._tokenizer.enable_padding(length=4096, pad_id=self.pad_token_id, pad_token=str(pad_token))
 
     def remove_hallucinated_references(self, text: str) -> str:
         """

src/transformers/models/nougat/tokenization_nougat.py

@@ -30,7 +30,7 @@
     ParakeetFeatureExtractor,
     ParakeetForCTC,
     ParakeetProcessor,
-    ParakeetTokenizerFast,
+    ParakeetTokenizer,
 )
 from transformers.convert_slow_tokenizer import ParakeetConverter
 from transformers.utils.hub import cached_file
@@ -151,7 +151,7 @@ def extract_nemo_archive(nemo_file_path: str, extract_dir: str) -> dict[str, str
 
 def write_processor(nemo_config: dict, model_files, output_dir, push_to_repo_id=None):
     tokenizer_converted = ParakeetConverter(model_files["tokenizer_model_file"]).converted()
-    tokenizer_converted_fast = ParakeetTokenizerFast(
+    tokenizer_converted_fast = ParakeetTokenizer(
         tokenizer_object=tokenizer_converted,
         clean_up_tokenization_spaces=False,
     )

src/transformers/models/parakeet/convert_nemo_to_hf.py

@@ -16,10 +16,10 @@
 import itertools
 from typing import Optional, Union
 
-from ...tokenization_utils_tokenizers import PreTrainedTokenizerFast
+from ...tokenization_utils_tokenizers import TokenizersBackend
 
 
-class ParakeetTokenizerFast(PreTrainedTokenizerFast):
+class ParakeetTokenizer(TokenizersBackend):
     """
     Inherits all methods from [`PreTrainedTokenizerFast`]. Users should refer to this superclass for more information regarding those methods,
     except for `_decode` which is overridden to adapt it to CTC decoding:
@@ -51,4 +51,4 @@ def _decode(
         )
 
 
-__all__ = ["ParakeetTokenizerFast"]
+__all__ = ["ParakeetTokenizer"]