Skip to content

Commit 04dc993

Browse files
authored
Merge 114cc5c into d28b793
2 parents d28b793 + 114cc5c commit 04dc993

59 files changed

Lines changed: 132 additions & 145 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎docs/source/en/expert_parallelism.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ rendered properly in your Markdown viewer.
1919

2020
## DistributedConfig
2121

22-
Enable expert parallelism with the [`DistributedConfig`] class and the `ep_size` argument. Most models route with masking and all-reduce, which requires `ep_size=tp_size` so every rank in an expert group receives the same tokens. Models whose plan uses [token dispatch](#token-dispatch), such as Qwen3 MoE, can set `ep_size` independently of `tp_size`.
22+
Enable expert parallelism with the [`DistributedConfig`] class and the `ep_size` argument. Most MoE models default to [token dispatch](#token-dispatch), so `ep_size` can be set independently of `tp_size`. A few, such as Llama 4 and Gemma 4, still default to masking and all-reduce (`"ep_router"` and `"moe_tp_experts"` in `model.ep_plan`). Masking is also available on any model with an `ep_plan` override, and requires `ep_size=tp_size` so every rank in an expert group receives the same tokens.
2323

2424
```py
2525
import os
@@ -74,7 +74,7 @@ distributed_config = DistributedConfig(
7474

7575
Providing a plan does not infer parallel sizes: set `tp_size` and `ep_size` explicitly.
7676

77-
Qwen3 MoE defaults to `"ep_dispatch_experts"`. To use masking and all-reduce instead, set `ep_size=tp_size` and override both the router and the expert forward rules:
77+
Most MoE models default to `"ep_dispatch_experts"`. To use masking and all-reduce instead, set `ep_size=tp_size` and override both the router and the expert forward rules (the router module name depends on the model, e.g. `mlp.router` on gpt-oss):
7878

7979
```py
8080
distributed_config = DistributedConfig(
@@ -155,7 +155,7 @@ These configurations each use eight GPUs:
155155

156156
| Configuration | Result |
157157
| :--- | :--- |
158-
| `DistributedConfig(tp_size=4, fsdp_size=2, ep_size=4)` | Dispatch with TP groups of four, each slicing its batch in four (Qwen3 MoE default plan); unless you specify a ep_plan to use the legacy masked EP |
158+
| `DistributedConfig(tp_size=4, fsdp_size=2, ep_size=4)` | Dispatch with TP groups of four, each slicing its batch in four (default plan); unless you specify a ep_plan to use the legacy masked EP |
159159
| `DistributedConfig(tp_size=1, fsdp_size=8, ep_size=4)` | Dispatch with an independent batch on each rank, no slicing, and experts FSDP-sharded across pairs of ranks. |
160160
| `DistributedConfig(tp_size=2, fsdp_size=4, ep_size=4)` | Dispatch with a TP pair per batch, each pair slicing its batch in two; two batches per expert group. |
161161
| `DistributedConfig(tp_size=8, ep_size=8)` | Dispatch with every rank sharing one batch, sliced in eight, or masking and all-reduce for a masked plan. |

‎src/transformers/distributed/mixin.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ class DistributedMixin:
5252
"""Distributed orchestration and save/load hooks for [`PreTrainedModel`]."""
5353

5454
_device_mesh = None
55+
_is_distributed_loading_by_transformers = False
5556
_mesh_manager: TransformersDeviceMesh | None = None
5657
_tp_plan: dict[str, str] | None = None
5758
_ep_plan: dict[str, str] | None = None
@@ -60,6 +61,14 @@ class DistributedMixin:
6061
_pp_plan: dict[str, tuple[str, str]] | None = None
6162
_fsdp_plan: dict[str, str] | None = None
6263

64+
@property
65+
def is_distributed_loading_by_transformers(self) -> bool:
66+
"""
67+
Whether `from_pretrained(distributed_config=...)` sharded the model at load time (each rank reads only its
68+
shard through `DtensorShardOperation`), rather than a wrapper such as Accelerate sharding it afterwards.
69+
"""
70+
return self._is_distributed_loading_by_transformers
71+
6372
def init_parallel_plans(self) -> None:
6473
"""Copy class-level plans onto the instance and merge config/children contributions."""
6574
model_cls = type(self)
@@ -192,6 +201,7 @@ def maybe_distribute_model(
192201
model.config.distributed_config = distributed_config
193202
model._mesh_manager = mesh_manager
194203
model._device_mesh = mesh_manager.get_mesh(("pp", "fsdp", "tp"))
204+
model._is_distributed_loading_by_transformers = True
195205
model._tp_size = distributed_config.tp_size
196206
model._fsdp_size = distributed_config.fsdp_size
197207

‎src/transformers/models/afmoe/configuration_afmoe.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,9 @@ class AfmoeConfig(PreTrainedConfig):
6262
"norm": (["hidden_states"], ["hidden_states"]),
6363
}
6464
base_model_ep_plan = {
65-
"layers.*.mlp.router": "ep_router",
6665
"layers.*.mlp.experts.gate_up_proj": "grouped_gemm",
6766
"layers.*.mlp.experts.down_proj": "grouped_gemm",
68-
"layers.*.mlp.experts": "moe_tp_experts",
67+
"layers.*.mlp.experts": "ep_dispatch_experts",
6968
}
7069

7170
vocab_size: int = 200192

‎src/transformers/models/axk1/configuration_axk1.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,9 @@ class AXK1Config(PreTrainedConfig):
6969
"norm": (["hidden_states"], ["hidden_states"]),
7070
}
7171
base_model_ep_plan = {
72-
"layers.*.mlp.gate": "ep_router",
7372
"layers.*.mlp.experts.gate_up_proj": "grouped_gemm",
7473
"layers.*.mlp.experts.down_proj": "grouped_gemm",
75-
"layers.*.mlp.experts": "moe_tp_experts",
74+
"layers.*.mlp.experts": "ep_dispatch_experts",
7675
}
7776
attribute_map = {
7877
"num_local_experts": "n_routed_experts",

‎src/transformers/models/axk2/configuration_axk2.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,10 +82,9 @@ class AXK2Config(PreTrainedConfig):
8282
"norm": (["hidden_states"], ["hidden_states"]),
8383
}
8484
base_model_ep_plan = {
85-
"layers.*.mlp.gate": "ep_router",
8685
"layers.*.mlp.experts.gate_up_proj": "grouped_gemm",
8786
"layers.*.mlp.experts.down_proj": "grouped_gemm",
88-
"layers.*.mlp.experts": "moe_tp_experts",
87+
"layers.*.mlp.experts": "ep_dispatch_experts",
8988
}
9089

9190
attribute_map = {"num_local_experts": "n_routed_experts"}

‎src/transformers/models/cohere2_moe/configuration_cohere2_moe.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,9 @@ class Cohere2MoeConfig(PreTrainedConfig):
8585
"norm": (["hidden_states"], ["hidden_states"]),
8686
}
8787
base_model_ep_plan = {
88-
"layers.*.mlp.gate": "ep_router",
8988
"layers.*.mlp.experts.gate_up_proj": "grouped_gemm",
9089
"layers.*.mlp.experts.down_proj": "grouped_gemm",
91-
"layers.*.mlp.experts": "moe_tp_experts",
90+
"layers.*.mlp.experts": "ep_dispatch_experts",
9291
}
9392

9493
vocab_size: int = 256000

‎src/transformers/models/deepseek_ocr2/configuration_deepseek_ocr2.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -238,10 +238,9 @@ class DeepseekOcr2TextConfig(PreTrainedConfig):
238238
mlp_bias: bool = False
239239
head_dim: int | None = None
240240
base_model_ep_plan = {
241-
"layers.*.mlp.gate": "ep_router",
242241
"layers.*.mlp.experts.gate_up_proj": "grouped_gemm",
243242
"layers.*.mlp.experts.down_proj": "grouped_gemm",
244-
"layers.*.mlp.experts": "moe_tp_experts",
243+
"layers.*.mlp.experts": "ep_dispatch_experts",
245244
}
246245
attribute_map = {
247246
"num_experts": "n_routed_experts",

‎src/transformers/models/deepseek_v2/configuration_deepseek_v2.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,9 @@ class DeepseekV2Config(PreTrainedConfig):
9595
mlp_bias: bool = False
9696
head_dim: int | None = None
9797
base_model_ep_plan = {
98-
"layers.*.mlp.gate": "ep_router",
9998
"layers.*.mlp.experts.gate_up_proj": "grouped_gemm",
10099
"layers.*.mlp.experts.down_proj": "grouped_gemm",
101-
"layers.*.mlp.experts": "moe_tp_experts",
100+
"layers.*.mlp.experts": "ep_dispatch_experts",
102101
}
103102
attribute_map = {
104103
"num_experts": "n_routed_experts",

‎src/transformers/models/deepseek_v2/modular_deepseek_v2.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,10 +86,9 @@ class DeepseekV2Config(LlamaConfig):
8686
"layers.*.mlp.down_proj": "rowwise",
8787
}
8888
base_model_ep_plan = {
89-
"layers.*.mlp.gate": "ep_router",
9089
"layers.*.mlp.experts.gate_up_proj": "grouped_gemm",
9190
"layers.*.mlp.experts.down_proj": "grouped_gemm",
92-
"layers.*.mlp.experts": "moe_tp_experts",
91+
"layers.*.mlp.experts": "ep_dispatch_experts",
9392
}
9493

9594
model_type = "deepseek_v2"

‎src/transformers/models/deepseek_v3/configuration_deepseek_v3.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,9 @@ class DeepseekV3Config(PreTrainedConfig):
6969
"norm": (["hidden_states"], ["hidden_states"]),
7070
}
7171
base_model_ep_plan = {
72-
"layers.*.mlp.gate": "ep_router",
7372
"layers.*.mlp.experts.gate_up_proj": "grouped_gemm",
7473
"layers.*.mlp.experts.down_proj": "grouped_gemm",
75-
"layers.*.mlp.experts": "moe_tp_experts",
74+
"layers.*.mlp.experts": "ep_dispatch_experts",
7675
}
7776

7877
attribute_map = {

0 commit comments

Comments
 (0)