From fb5b721e6c3cd21738a24586334496f275f20031 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Tue, 12 May 2026 12:15:49 +0200 Subject: [PATCH 1/2] FIX Transformers weight conversion PEFT model prefix In convert_peft_adapter_state_dict_for_transformers, we pass PEFT model to get_model_conversion_mapping. After the changes in https://github.com/huggingface/transformers/pull/45661, a prefix check was introduced, which fails with PEFT models, since they have 'base_model.model' as an extra prefix. https://github.com/huggingface/transformers/blob/a25b8efa0a3220da89493a72f57081aa5720291f/src/transformers/conversion_mapping.py#L1110-L1115 However, this bug was not immediately visible because the prefixes to check (and submodules) were determined once at Transformers model creation (i.e. before the model is wrapped by PEFT) and stored inside of _named_pretrained_submodules. However, https://github.com/huggingface/transformers/pull/45889 removed this attribute and now dynamically loops over the modules. At that point, the prefix error surfaces, which leads to failures in the PEFT CI. This PR now changes the PEFT call to pass the Transformers base model to get_model_conversion_mapping instead of the PEFT model. I could confirm that the tests now pass on main and also with Transformers from before 45661. Note that this error could also affect other models that are not PEFT and that wrap the Transformers module, if they use weight conversion. A more robust fix could thus be on the Transformers side to account for this possibility and make the scope_prefix check account for this possibility. --- src/peft/utils/transformers_weight_conversion.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/peft/utils/transformers_weight_conversion.py b/src/peft/utils/transformers_weight_conversion.py index 3350175481..57ba8b0211 100644 --- a/src/peft/utils/transformers_weight_conversion.py +++ b/src/peft/utils/transformers_weight_conversion.py @@ -508,7 +508,10 @@ def convert_peft_adapter_state_dict_for_transformers( # no architecture-level conversion registered for this model return adapter_state_dict - weight_conversions = get_model_conversion_mapping(model) + # pass the base model to get_model_conversion_mapping, as Transformers may not correctly deal with base_model.model + # prefix, see the introduction of scope_prefix in https://github.com/huggingface/transformers/pull/45661 + base_model = model.get_base_model() if hasattr(model, "get_base_model") else model + weight_conversions = get_model_conversion_mapping(base_model) peft_weight_conversions = build_peft_weight_mapping( weight_conversions, adapter_name=adapter_name, peft_config=peft_config ) From f66aa94b3277d3fe318f06e02f11a9467b2d50c4 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Wed, 27 May 2026 17:42:04 +0200 Subject: [PATCH 2/2] Additional fix to deal with undoing key renaming --- .../utils/transformers_weight_conversion.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/peft/utils/transformers_weight_conversion.py b/src/peft/utils/transformers_weight_conversion.py index 508839c866..935e31f602 100644 --- a/src/peft/utils/transformers_weight_conversion.py +++ b/src/peft/utils/transformers_weight_conversion.py @@ -538,14 +538,29 @@ def convert_peft_adapter_state_dict_for_transformers( param_name_to_load: dict[str, WeightRenaming | WeightConverter] = {} + def str_rreplace(s, old, new, count=-1): + # this is the same as str.replace(...) but counting from the right side instead of left side + return new.join(s.rsplit(old, count)) + # Mirror transformers.core_model_loading flow: stable ordering + same rename logic. # https://github.com/huggingface/transformers/blob/1bd97f246318456c1b87cf8ef8dc043ec1a53fff/src/transformers/core_model_loading.py#L997 state_items = sorted(lora_like_state_dict.items(), key=lambda kv: dot_natural_key(kv[0])) for original_key, tensor in state_items: renamed_key, source_pattern = rename_source_key(original_key, renamings, converters) + renamed_key_without_adapter_name = str_rreplace(renamed_key, f".{adapter_name}", "", count=1) + new_key_is_suffix_of_old_key = original_key.endswith(renamed_key_without_adapter_name) + # https://github.com/huggingface/transformers/pull/44396 introduced code to undo the renaming: # https://github.com/huggingface/transformers/blob/027d1a97025295a1346c2eb5c361259e69eedfe7/src/transformers/core_model_loading.py#L1361-L1365 - if renamed_key not in adapter_state_dict and original_key in adapter_state_dict: + # This logic is copied here but with an extra caveat: We only undo the key renaming if it is due to a missing + # prefix, e.g. "text_model.foo.bar" => "foo.bar". This is why we check if the renamed key is a suffix of the + # original key. However, strictly checking for the suffix is not enough because the renamed key also introduces + # the adapater name, which is not in the original key. Therefore, we need to remove the adapter name first. + if ( + (renamed_key not in adapter_state_dict) + and (original_key in adapter_state_dict) + and new_key_is_suffix_of_old_key + ): # Key should probably not have been renamed but we might need the `prefix` to be added. renamed_key, source_pattern = rename_source_key( original_key, [], [], prefix=prefix, meta_state_dict=adapter_state_dict