From bbf78d55d6ba59f54029064bf4c52d16ce1b77cd Mon Sep 17 00:00:00 2001 From: itazap Date: Wed, 18 Feb 2026 11:36:01 +0100 Subject: [PATCH 1/2] AutoTokenizer ignores config when model_type is None --- src/transformers/models/auto/tokenization_auto.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 2d3993d0fe3a..6b6ff939a50c 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -649,7 +649,8 @@ def from_pretrained( and tokenizer_config_class is not None and config_model_type is not None and config_model_type != "" - and TOKENIZER_MAPPING_NAMES.get(config_model_type, "").replace("Fast", "") + and TOKENIZER_MAPPING_NAMES.get(config_model_type) is not None + and TOKENIZER_MAPPING_NAMES.get(config_model_type).replace("Fast", "") != tokenizer_config_class.replace("Fast", "") ): # new model, but we ignore it unless the model type is the same From af9d764c6f13e4d2b1009b8d89e9d7c8d3d07f7f Mon Sep 17 00:00:00 2001 From: itazap Date: Wed, 18 Feb 2026 12:20:47 +0100 Subject: [PATCH 2/2] update tests --- tests/models/aria/test_processing_aria.py | 2 +- tests/models/auto/test_tokenization_auto.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/models/aria/test_processing_aria.py b/tests/models/aria/test_processing_aria.py index 0f64fdcf59ff..a196f9bed1a9 100644 --- a/tests/models/aria/test_processing_aria.py +++ b/tests/models/aria/test_processing_aria.py @@ -149,7 +149,7 @@ def test_process_interleaved_images_prompts_no_image_splitting(self): # Pad the first input to match the second input pad_len = len(expected_input_ids_2) - len(expected_input_ids_1) - expected_attention_mask = [ [1] * len(expected_input_ids_1) + [0] * pad_len, [1] * (len(expected_input_ids_2))] + expected_attention_mask = [ [0] * pad_len + [1] * len(expected_input_ids_1), [1] * (len(expected_input_ids_2))] self.assertEqual( inputs["attention_mask"], diff --git a/tests/models/auto/test_tokenization_auto.py b/tests/models/auto/test_tokenization_auto.py index 48f7c509673c..4ace3a39da08 100644 --- a/tests/models/auto/test_tokenization_auto.py +++ b/tests/models/auto/test_tokenization_auto.py @@ -621,15 +621,9 @@ def test_tokenization_class_priority(self): tok2 = AutoTokenizer.from_pretrained(tmp_dir) self.assertTrue(tok2.__class__ == HerbertTokenizer) - tok = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM2-135M-Instruct") - self.assertTrue(tok.__class__ == TokenizersBackend) - tok = AutoProcessor.from_pretrained("mistralai/Ministral-3-8B-Instruct-2512-BF16").tokenizer self.assertTrue(tok.__class__ == TokenizersBackend) - tok = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM2-135M-Instruct") - self.assertTrue(tok.__class__ == TokenizersBackend) - def test_custom_tokenizer_with_mismatched_tokenizer_class(self): nop_tokenizer_code = """ import transformers