From 72ef7ac6eefb1be04483124bf0c1073273f9a84e Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Wed, 1 Apr 2026 17:12:55 +0200 Subject: [PATCH 1/2] Fix tokenizer `explicit_local_code` detection Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- src/transformers/models/auto/tokenization_auto.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 0e50f2a0041d..2ccae521e558 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -738,11 +738,15 @@ def from_pretrained( or tokenizer_class_from_name(tokenizer_config_class + "Fast") is not None ) ) - explicit_local_code = has_local_code and ( - tokenizer_config_class is not None - and not ( - tokenizer_class_from_name(tokenizer_config_class).__module__.startswith("transformers.") - and tokenizer_class_from_name(tokenizer_config_class + "Fast").__module__.startswith("transformers.") + explicit_local_code = ( + has_local_code + and type(config) not in TOKENIZER_MAPPING + and ( + tokenizer_config_class is not None + and not ( + tokenizer_class_from_name(tokenizer_config_class) + or tokenizer_class_from_name(tokenizer_config_class + "Fast") + ).__module__.startswith("transformers.") ) ) # V5: Skip remote tokenizer for custom models with incorrect hub tokenizer class From cf7e056fd11fe16b5b4faf042cc34306f38e17e8 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Wed, 1 Apr 2026 17:20:06 +0200 Subject: [PATCH 2/2] Fix image processor `explicit_local_code` detection Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- src/transformers/models/auto/image_processing_auto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index 2a9e2720106e..1e868161c9f2 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -725,7 +725,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs): has_remote_code = image_processor_auto_map is not None has_local_code = image_processor_class is not None or type(config) in IMAGE_PROCESSOR_MAPPING explicit_local_code = has_local_code and not ( - image_processor_class or IMAGE_PROCESSOR_MAPPING[type(config)] + image_processor_class or _load_class_with_fallback(IMAGE_PROCESSOR_MAPPING[type(config)], backend) ).__module__.startswith("transformers.") if has_remote_code: class_ref = _resolve_auto_map_class_ref(image_processor_auto_map, backend)