diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 2c0fe88d0e74..8877071f8e95 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -145,6 +145,7 @@ ("ernie", "ErnieConfig"), ("ernie4_5", "Ernie4_5Config"), ("ernie4_5_moe", "Ernie4_5_MoeConfig"), + ("ernie4_5_moe_vl", "Ernie4_5_VLMoeConfig"), ("ernie4_5_vl_moe", "Ernie4_5_VLMoeConfig"), ("esm", "EsmConfig"), ("eurobert", "EuroBertConfig"), @@ -661,6 +662,7 @@ ("ernie", "ERNIE"), ("ernie4_5", "Ernie4_5"), ("ernie4_5_moe", "Ernie4_5_MoE"), + ("ernie4_5_moe_vl", "Ernie4_5_VLMoE"), ("ernie4_5_vl_moe", "Ernie4_5_VLMoE"), ("esm", "ESM"), ("eurobert", "EuroBERT"), @@ -1081,6 +1083,7 @@ SPECIAL_MODEL_TYPE_TO_MODULE_NAME = OrderedDict[str, str]( [ + ("ernie4_5_moe_vl", "ernie4_5_vl_moe"), ("audioflamingo3_encoder", "audioflamingo3"), ("musicflamingo_encoder", "musicflamingo"), ("openai-gpt", "openai"), diff --git a/src/transformers/models/ernie4_5_vl_moe/configuration_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/configuration_ernie4_5_vl_moe.py index e4eea836f107..4d16d9061fd3 100644 --- a/src/transformers/models/ernie4_5_vl_moe/configuration_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/configuration_ernie4_5_vl_moe.py @@ -67,8 +67,8 @@ class Ernie4_5_VLMoeTextConfig(PreTrainedConfig): Whether to use a bias in any of the projections including mlp and attention for example moe_k (`int`, *optional*, defaults to 6): Number of selected experts. - moe_num_experts (`int`, *optional*, defaults to 64): - Number of routed experts. + moe_num_experts (`int` or `list[int]`, *optional*, defaults to 64): + Number of routed experts. Can be a list to specify per-layer expert counts. moe_num_shared_experts (`int`, *optional*, defaults to 2): The number of experts that are shared for all MoE forwards. moe_norm_min (`float`, *optional*, defaults to 1e-12): @@ -119,7 +119,7 @@ class Ernie4_5_VLMoeTextConfig(PreTrainedConfig): use_bias: bool | None = False moe_intermediate_size: list[int] | None = None moe_k: int | None = 6 - moe_num_experts: int | None = 64 + moe_num_experts: int | list[int] | None = 64 moe_num_shared_experts: int | None = 2 moe_norm_min: float | None = 1e-12 output_router_logits: bool | None = False diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index 42bbb44b70a5..cf7c94e8b27f 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -117,8 +117,8 @@ class Ernie4_5_VLMoeTextConfig(Ernie4_5_MoeConfig): Whether to use a bias in any of the projections including mlp and attention for example moe_k (`int`, *optional*, defaults to 6): Number of selected experts. - moe_num_experts (`int`, *optional*, defaults to 64): - Number of routed experts. + moe_num_experts (`int` or `list[int]`, *optional*, defaults to 64): + Number of routed experts. Can be a list to specify per-layer expert counts. moe_num_shared_experts (`int`, *optional*, defaults to 2): The number of experts that are shared for all MoE forwards. moe_norm_min (`float`, *optional*, defaults to 1e-12): @@ -149,6 +149,7 @@ class Ernie4_5_VLMoeTextConfig(Ernie4_5_MoeConfig): pad_token_id: int | None = None eos_token_id: int | list[int] | None = None bos_token_id: int | None = None + moe_num_experts: int | list[int] | None = 64 moe_layer_end_index = AttributeError() moe_layer_interval = AttributeError() moe_layer_start_index = AttributeError()