diff --git a/src/transformers/tokenization_utils_tokenizers.py b/src/transformers/tokenization_utils_tokenizers.py index 85ebabd017ab..5365e9c1b6e1 100644 --- a/src/transformers/tokenization_utils_tokenizers.py +++ b/src/transformers/tokenization_utils_tokenizers.py @@ -1324,7 +1324,7 @@ def is_base_mistral(model_id: str) -> bool: # Detect if we can skip the mistral fix by # a) having a non-mistral tokenizer # b) fixed version of transformers - if transformers_version and version.parse(transformers_version) <= version.parse("4.57.2"): + if transformers_version and version.parse(transformers_version) < version.parse("5.0.0"): if ( is_local and transformers_model_type is not None @@ -1338,7 +1338,7 @@ def is_base_mistral(model_id: str) -> bool: ] ): return tokenizer - elif transformers_version and version.parse(transformers_version) > version.parse("4.57.3"): + elif transformers_version and version.parse(transformers_version) >= version.parse("5.0.0"): return tokenizer mistral_config_detected = True diff --git a/tests/models/auto/test_tokenization_auto.py b/tests/models/auto/test_tokenization_auto.py index 2bc79a3f82d6..0d8c099ca097 100644 --- a/tests/models/auto/test_tokenization_auto.py +++ b/tests/models/auto/test_tokenization_auto.py @@ -267,35 +267,65 @@ def test_auto_tokenizer_from_local_folder(self): self.assertEqual(tokenizer2.vocab_size, 12) def test_auto_tokenizer_from_local_folder_mistral_detection(self): - """See #42374 for reference, ensuring proper mistral detection on local tokenizers""" + """See #42374 and #45444 for reference, ensuring proper mistral detection on local tokenizers""" tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-235B-A22B-Thinking-2507") config = Qwen3MoeConfig.from_pretrained("Qwen/Qwen3-235B-A22B-Thinking-2507") self.assertIsInstance(tokenizer, (Qwen2Tokenizer, Qwen2TokenizerFast)) + mistral_warning = ( + "with an incorrect regex pattern: " + "https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84" + "#69121093e8b480e709447d5e" + ) + logger = logging.get_logger("transformers.tokenization_utils_tokenizers") + with tempfile.TemporaryDirectory() as tmp_dir: tokenizer.save_pretrained(tmp_dir) + config_path = os.path.join(tmp_dir, "config.json") - # Case 1: Tokenizer with no config associated - logger = logging.get_logger("transformers.tokenization_utils_base") + def _write_config(**overrides): + config_dict = config.to_diff_dict() + for key, value in overrides.items(): + if value is None: + config_dict.pop(key, None) + else: + config_dict[key] = value + with open(config_path, "w", encoding="utf-8") as f: + json.dump(config_dict, f, indent=2, sort_keys=True) + + # Case 1: Tokenizer with no config associated must not warn with CaptureLogger(logger) as cl: AutoTokenizer.from_pretrained(tmp_dir) - self.assertNotIn( - "with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e", - cl.out, - ) - - # Case 2: Tokenizer with config associated - # Needed to be saved along the tokenizer to detect (non)mistral - # for a version where the regex bug occurs - config_dict = config.to_diff_dict() - config_dict["transformers_version"] = "4.57.2" + self.assertNotIn(mistral_warning, cl.out) + + # Case 2: Non-mistral local config must not warn for any `transformers_version` + for saved_version in ("4.57.2", "4.57.3", "4.57.6", "5.0.1"): + _write_config(transformers_version=saved_version) + with CaptureLogger(logger) as cl: + tokenizer2 = AutoTokenizer.from_pretrained(tmp_dir) + self.assertNotIn( + mistral_warning, + cl.out, + msg=f"Unexpected mistral regex warning for non-mistral config (transformers_version={saved_version!r})", + ) - # Manually saving to avoid versioning clashes - config_path = os.path.join(tmp_dir, "config.json") - with open(config_path, "w", encoding="utf-8") as f: - json.dump(config_dict, f, indent=2, sort_keys=True) + # Case 3: Mistral-family local config saved by an affected transformers release + # must still warn, even up to 4.57.6 + for saved_version in ("4.57.3", "4.57.6"): + _write_config(model_type="mistral", transformers_version=saved_version) + with CaptureLogger(logger) as cl: + AutoTokenizer.from_pretrained(tmp_dir) + self.assertIn( + mistral_warning, + cl.out, + msg=f"Missing mistral regex warning for mistral config (transformers_version={saved_version!r})", + ) - tokenizer2 = AutoTokenizer.from_pretrained(tmp_dir) + # Case 4: Mistral-family local config saved by a fixed transformers release must not warn + _write_config(model_type="mistral", transformers_version="5.0.1") + with CaptureLogger(logger) as cl: + AutoTokenizer.from_pretrained(tmp_dir) + self.assertNotIn(mistral_warning, cl.out) self.assertIsInstance(tokenizer2, tokenizer.__class__) self.assertTrue(tokenizer2.vocab_size > 100_000)