From 1219f839e0d982e98fffcade922f068f8d0622e8 Mon Sep 17 00:00:00 2001 From: Austin Varga Date: Tue, 7 Apr 2026 02:13:40 +0000 Subject: [PATCH 1/4] fix(ernie4_5_vl_moe): resolve config loading failures for ERNIE-4.5-VL MoE models Three issues prevented AutoConfig from loading baidu/ERNIE-4.5-VL-28B-A3B-Paddle: 1. model_type mismatch: the published checkpoint uses "ernie4_5_moe_vl" but transformers registers the class as "ernie4_5_vl_moe". Add "ernie4_5_moe_vl" alias in SPECIAL_MODEL_TYPE_TO_MODULE_NAME, CONFIG_MAPPING_NAMES, and MODEL_NAMES_MAPPING so AutoConfig resolves it to Ernie4_5_VLMoeConfig. 2. rope_theta validation failure: PreTrainedConfig.__init__ only triggered convert_rope_params_to_dict when rope_theta was present in **kwargs, but Ernie4_5Config.__init__ consumes rope_theta as a named parameter before calling super().__init__(). Also check getattr(self, "rope_theta", None) so the RoPE standardization path fires correctly. 3. moe_num_experts type error: Ernie4_5_VLMoeTextConfig declared the field as int | None but the checkpoint supplies a list [64, 64] for per-layer expert counts. Widen the type to int | list[int] | None. --- src/transformers/configuration_utils.py | 4 +++- src/transformers/models/auto/configuration_auto.py | 3 +++ .../models/ernie4_5_vl_moe/configuration_ernie4_5_vl_moe.py | 6 +++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py index 97d6b94b57aa..e3eddc5e6cd6 100755 --- a/src/transformers/configuration_utils.py +++ b/src/transformers/configuration_utils.py @@ -265,7 +265,9 @@ def __post_init__(self, **kwargs): # BC for rotary embeddings. We will pop out legacy keys from kwargs and rename to new format if hasattr(self, "rope_parameters"): kwargs = self.convert_rope_params_to_dict(**kwargs) - elif kwargs.get("rope_scaling") and kwargs.get("rope_theta"): + elif kwargs.get("rope_scaling") and ( + kwargs.get("rope_theta") or getattr(self, "rope_theta", None) is not None + ): logger.warning( f"{self.__class__.__name__} got `key=rope_scaling` in kwargs but hasn't set it as attribute. " "For RoPE standardization you need to set `self.rope_parameters` in model's config. " diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 2c0fe88d0e74..8877071f8e95 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -145,6 +145,7 @@ ("ernie", "ErnieConfig"), ("ernie4_5", "Ernie4_5Config"), ("ernie4_5_moe", "Ernie4_5_MoeConfig"), + ("ernie4_5_moe_vl", "Ernie4_5_VLMoeConfig"), ("ernie4_5_vl_moe", "Ernie4_5_VLMoeConfig"), ("esm", "EsmConfig"), ("eurobert", "EuroBertConfig"), @@ -661,6 +662,7 @@ ("ernie", "ERNIE"), ("ernie4_5", "Ernie4_5"), ("ernie4_5_moe", "Ernie4_5_MoE"), + ("ernie4_5_moe_vl", "Ernie4_5_VLMoE"), ("ernie4_5_vl_moe", "Ernie4_5_VLMoE"), ("esm", "ESM"), ("eurobert", "EuroBERT"), @@ -1081,6 +1083,7 @@ SPECIAL_MODEL_TYPE_TO_MODULE_NAME = OrderedDict[str, str]( [ + ("ernie4_5_moe_vl", "ernie4_5_vl_moe"), ("audioflamingo3_encoder", "audioflamingo3"), ("musicflamingo_encoder", "musicflamingo"), ("openai-gpt", "openai"), diff --git a/src/transformers/models/ernie4_5_vl_moe/configuration_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/configuration_ernie4_5_vl_moe.py index e4eea836f107..4d16d9061fd3 100644 --- a/src/transformers/models/ernie4_5_vl_moe/configuration_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/configuration_ernie4_5_vl_moe.py @@ -67,8 +67,8 @@ class Ernie4_5_VLMoeTextConfig(PreTrainedConfig): Whether to use a bias in any of the projections including mlp and attention for example moe_k (`int`, *optional*, defaults to 6): Number of selected experts. - moe_num_experts (`int`, *optional*, defaults to 64): - Number of routed experts. + moe_num_experts (`int` or `list[int]`, *optional*, defaults to 64): + Number of routed experts. Can be a list to specify per-layer expert counts. moe_num_shared_experts (`int`, *optional*, defaults to 2): The number of experts that are shared for all MoE forwards. moe_norm_min (`float`, *optional*, defaults to 1e-12): @@ -119,7 +119,7 @@ class Ernie4_5_VLMoeTextConfig(PreTrainedConfig): use_bias: bool | None = False moe_intermediate_size: list[int] | None = None moe_k: int | None = 6 - moe_num_experts: int | None = 64 + moe_num_experts: int | list[int] | None = 64 moe_num_shared_experts: int | None = 2 moe_norm_min: float | None = 1e-12 output_router_logits: bool | None = False From a770099cb08f5009e751f417486f05ee82c4de84 Mon Sep 17 00:00:00 2001 From: Austin Varga Date: Tue, 7 Apr 2026 17:50:46 +0000 Subject: [PATCH 2/4] fix: override moe_num_experts type in modular source to match generated file --- .../models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index 42bbb44b70a5..6aaf75aeda51 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -117,7 +117,7 @@ class Ernie4_5_VLMoeTextConfig(Ernie4_5_MoeConfig): Whether to use a bias in any of the projections including mlp and attention for example moe_k (`int`, *optional*, defaults to 6): Number of selected experts. - moe_num_experts (`int`, *optional*, defaults to 64): + moe_num_experts (`int` or `list[int]`, *optional*, defaults to 64): Number of routed experts. moe_num_shared_experts (`int`, *optional*, defaults to 2): The number of experts that are shared for all MoE forwards. @@ -149,6 +149,7 @@ class Ernie4_5_VLMoeTextConfig(Ernie4_5_MoeConfig): pad_token_id: int | None = None eos_token_id: int | list[int] | None = None bos_token_id: int | None = None + moe_num_experts: int | list[int] | None = 64 moe_layer_end_index = AttributeError() moe_layer_interval = AttributeError() moe_layer_start_index = AttributeError() From e169dc9fbc1f3cb962b2ea06294d291c221b2afc Mon Sep 17 00:00:00 2001 From: Austin Varga Date: Tue, 7 Apr 2026 18:07:25 +0000 Subject: [PATCH 3/4] fix: sync modular docstring with generated config --- .../models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index 6aaf75aeda51..cf7c94e8b27f 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -118,7 +118,7 @@ class Ernie4_5_VLMoeTextConfig(Ernie4_5_MoeConfig): moe_k (`int`, *optional*, defaults to 6): Number of selected experts. moe_num_experts (`int` or `list[int]`, *optional*, defaults to 64): - Number of routed experts. + Number of routed experts. Can be a list to specify per-layer expert counts. moe_num_shared_experts (`int`, *optional*, defaults to 2): The number of experts that are shared for all MoE forwards. moe_norm_min (`float`, *optional*, defaults to 1e-12): From c558547cb35f95829a0ca717794f9af48b11cb98 Mon Sep 17 00:00:00 2001 From: Austin Varga Date: Tue, 7 Apr 2026 12:41:42 -0600 Subject: [PATCH 4/4] revert: remove unnecessary rope_theta getattr check per maintainer feedback --- src/transformers/configuration_utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py index e3eddc5e6cd6..97d6b94b57aa 100755 --- a/src/transformers/configuration_utils.py +++ b/src/transformers/configuration_utils.py @@ -265,9 +265,7 @@ def __post_init__(self, **kwargs): # BC for rotary embeddings. We will pop out legacy keys from kwargs and rename to new format if hasattr(self, "rope_parameters"): kwargs = self.convert_rope_params_to_dict(**kwargs) - elif kwargs.get("rope_scaling") and ( - kwargs.get("rope_theta") or getattr(self, "rope_theta", None) is not None - ): + elif kwargs.get("rope_scaling") and kwargs.get("rope_theta"): logger.warning( f"{self.__class__.__name__} got `key=rope_scaling` in kwargs but hasn't set it as attribute. " "For RoPE standardization you need to set `self.rope_parameters` in model's config. "