diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 6d0adc8473a6..ff61e84c87eb 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -715,13 +715,24 @@ def from_pretrained( and (TOKENIZER_MAPPING_NAMES.get(config_model_type).removesuffix("Fast")) != (tokenizer_config_class.removesuffix("Fast")) ): - tokenizer_class = tokenizer_class_from_name(tokenizer_config_class) - if tokenizer_class is not None and tokenizer_class.__name__ not in ( - "TokenizersBackend", - "PythonBackend", - "PreTrainedTokenizerFast", - ): - return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + mapped_tokenizer_class = TOKENIZER_MAPPING_NAMES.get(config_model_type) + # When `MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS` (or an explicit registration) + # pins a model_type to `TokenizersBackend`, the `tokenizer_class` declared in the + # Hub's `tokenizer_config.json` is known to be wrong (e.g. DeepSeek-V3/R1 which + # ship `tokenizer_class: LlamaTokenizerFast` over a ByteLevel `tokenizer.json`, + # but `LlamaTokenizerFast.__init__` would clobber the pre-tokenizer with + # Metaspace and silently break round-trip). Honor the override and skip the + # specialized class path entirely. + forced_tokenizers_backend = mapped_tokenizer_class == "TokenizersBackend" + + if not forced_tokenizers_backend: + tokenizer_class = tokenizer_class_from_name(tokenizer_config_class) + if tokenizer_class is not None and tokenizer_class.__name__ not in ( + "TokenizersBackend", + "PythonBackend", + "PreTrainedTokenizerFast", + ): + return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) if TokenizersBackend is not None: return TokenizersBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) diff --git a/tests/models/auto/test_tokenization_auto.py b/tests/models/auto/test_tokenization_auto.py index 5e584a55b21f..c9ea0d0211de 100644 --- a/tests/models/auto/test_tokenization_auto.py +++ b/tests/models/auto/test_tokenization_auto.py @@ -765,3 +765,17 @@ def test_mismatched_model_type_uses_config_tokenizer_class_without_sentencepiece revision="f8d333a098d19b4fd9a8b18f94170487ad3f821d", ) self.assertEqual(tokenizer.__class__.__name__, "NllbTokenizer") + + @require_tokenizers + def test_models_with_incorrect_hub_tokenizer_class_use_tokenizers_backend(self): + """Regression test for https://github.com/huggingface/transformers/issues/45488. + + DeepSeek-V3/R1 declare `tokenizer_class: LlamaTokenizerFast` in `tokenizer_config.json` + but ship a ByteLevel `tokenizer.json`. `LlamaTokenizerFast.__init__` overwrites the + pre-tokenizer with `Metaspace`, dropping all spaces from round-trip. The + `MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS` override pins these model types to + `TokenizersBackend`; the dispatch in `AutoTokenizer.from_pretrained` must honor it. + """ + tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-R1") + self.assertEqual(tokenizer.__class__.__name__, "TokenizersBackend") + self.assertEqual(tokenizer.decode(tokenizer.encode("hello world", add_special_tokens=False)), "hello world")