Restore TokenizersBackend override for DeepSeek V3/R1 tokenizer dispatch

src/transformers/models/auto/tokenization_auto.py

@@ -715,13 +715,24 @@ def from_pretrained(
             and (TOKENIZER_MAPPING_NAMES.get(config_model_type).removesuffix("Fast"))
             != (tokenizer_config_class.removesuffix("Fast"))
         ):
-            tokenizer_class = tokenizer_class_from_name(tokenizer_config_class)
-            if tokenizer_class is not None and tokenizer_class.__name__ not in (
-                "TokenizersBackend",
-                "PythonBackend",
-                "PreTrainedTokenizerFast",
-            ):
-                return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+            mapped_tokenizer_class = TOKENIZER_MAPPING_NAMES.get(config_model_type)
+            # When `MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS` (or an explicit registration)
+            # pins a model_type to `TokenizersBackend`, the `tokenizer_class` declared in the
+            # Hub's `tokenizer_config.json` is known to be wrong (e.g. DeepSeek-V3/R1 which
+            # ship `tokenizer_class: LlamaTokenizerFast` over a ByteLevel `tokenizer.json`,
+            # but `LlamaTokenizerFast.__init__` would clobber the pre-tokenizer with
+            # Metaspace and silently break round-trip). Honor the override and skip the
+            # specialized class path entirely.
+            forced_tokenizers_backend = mapped_tokenizer_class == "TokenizersBackend"
+
+            if not forced_tokenizers_backend:
+                tokenizer_class = tokenizer_class_from_name(tokenizer_config_class)
+                if tokenizer_class is not None and tokenizer_class.__name__ not in (
+                    "TokenizersBackend",
+                    "PythonBackend",
+                    "PreTrainedTokenizerFast",
+                ):
+                    return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
 
             if TokenizersBackend is not None:
                 return TokenizersBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)

tests/models/auto/test_tokenization_auto.py

@@ -765,3 +765,17 @@ def test_mismatched_model_type_uses_config_tokenizer_class_without_sentencepiece
                 revision="f8d333a098d19b4fd9a8b18f94170487ad3f821d",
             )
             self.assertEqual(tokenizer.__class__.__name__, "NllbTokenizer")
+
+    @require_tokenizers
+    def test_models_with_incorrect_hub_tokenizer_class_use_tokenizers_backend(self):
+        """Regression test for https://github.com/huggingface/transformers/issues/45488.
+
+        DeepSeek-V3/R1 declare `tokenizer_class: LlamaTokenizerFast` in `tokenizer_config.json`
+        but ship a ByteLevel `tokenizer.json`. `LlamaTokenizerFast.__init__` overwrites the
+        pre-tokenizer with `Metaspace`, dropping all spaces from round-trip. The
+        `MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS` override pins these model types to
+        `TokenizersBackend`; the dispatch in `AutoTokenizer.from_pretrained` must honor it.
+        """
+        tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-R1")
+        self.assertEqual(tokenizer.__class__.__name__, "TokenizersBackend")
+        self.assertEqual(tokenizer.decode(tokenizer.encode("hello world", add_special_tokens=False)), "hello world")