Repository navigation
Expert-parallel token dispatch: each rank trains on its own part of the batch #48518
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 36 commits
8af1a00
1f0b7bf
4636d42
89fa55c
96769d9
c74c122
1c66acb
38aefbf
e72b117
449acd2
9d17bd7
6f2d9c4
3a8da11
1b07199
7063c45
dc1fdee
f2a3c26
3ab94ff
d188144
b9bf58b
39cdce5
9f86230
930c146
15588e0
51f281d
101bfea
2d9046b
42606af
d8eb1fc
23920e3
e149869
e139e8a
cf74929
0e21b59
18144ab
c9a4216
8f2f9cb
3c8f600
e7be30a
8dbccad
35ed430
79fe11d
0817429
34e63c9
75fff71
acba6bc
69668c3
9cfb2d3
2884e2f
4ce4700
28cb482
a6e36c0
1be3a7f
db78860
a6f5824
b56e1d6
cb02b27
6a86e18
79988c3
5120748
0147840
d604a2a
b169a75
3439765
4f2a6b9
86561b6
56047f4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -20,7 +20,7 @@ | |
| from ..utils import is_torch_available, is_torch_distributed_available, is_torch_greater_or_equal, logging, strtobool | ||
| from ..utils.quantization_config import QuantizationMethod | ||
| from .tensor_parallel import replace_layer_number_by_wildcard | ||
| from .utils import _is_torch_distributed_initialized | ||
| from .utils import _is_torch_distributed_initialized, is_dtensor | ||
|
|
||
|
|
||
| if TYPE_CHECKING: | ||
|
|
@@ -185,13 +185,19 @@ def verify_fsdp_plan(module_names: list[str], fsdp_plan: dict[str, str] | None) | |
|
|
||
|
|
||
| def apply_fully_sharded_data_parallelism( | ||
| model: nn.Module, fsdp_mesh: torch.distributed.device_mesh.DeviceMesh | ||
| model: nn.Module, | ||
| fsdp_mesh: torch.distributed.device_mesh.DeviceMesh, | ||
| expert_mesh: torch.distributed.device_mesh.DeviceMesh | None = None, | ||
| ) -> nn.Module: | ||
| """ | ||
| Apply FSDP2 (fully_shard) to a model. | ||
|
|
||
| Torch availability, distributed initialization and the version requirement | ||
| are asserted upstream by `initialize_fully_sharded_data_parallelism`. | ||
|
|
||
| With expert-parallel token dispatch `fsdp_mesh` spans the expert-parallel ranks, which the experts are already | ||
| sharded across: the experts are fully sharded across `expert_mesh` in their own group, and passed to FSDP2 as | ||
| `ignored_params` when `expert_mesh` is `None`. | ||
| """ | ||
| fsdp_plan = dict(getattr(model, "_fsdp_plan", None) or {}) | ||
| if not fsdp_plan: | ||
|
|
@@ -206,6 +212,20 @@ def apply_fully_sharded_data_parallelism( | |
| adapted_fsdp_plan = _resolve_tied_embed_lm_head_plan(fsdp_plan, model) | ||
| reshard_targets, no_reshard_targets = expand_fsdp_plan(model, adapted_fsdp_plan) | ||
|
|
||
| if distributed_config is not None and distributed_config.expert_parallel_dispatch: | ||
| if not is_torch_greater_or_equal("2.7"): | ||
| raise OSError("Expert-parallel token dispatch requires `torch>=2.7`.") | ||
| # The DTensor parameters are the expert-parallel experts: `maybe_distribute_model` rewrote the expert | ||
| # parallel plan to shard only them. | ||
| expert_modules = [ | ||
| module for module in model.modules() if any(is_dtensor(p) for p in module.parameters(recurse=False)) | ||
|
qgallouedec marked this conversation as resolved.
Resolved
|
||
| ] | ||
| if expert_mesh is not None: | ||
| for module in expert_modules: | ||
| fully_shard(module, mesh=expert_mesh, reshard_after_forward=True, **fsdp_policy_kwargs) | ||
| else: | ||
| fsdp_policy_kwargs["ignored_params"] = {p for module in expert_modules for p in module.parameters()} | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No, the way to say "we want only these FSDP" is here: https://github.com/VI-Arthur/transformers/blob/fix-ep-eager-waste/src/transformers/distributed/mixin.py#L202-L202 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This needs a small update but: fsdp_plan = dict(getattr(model, "_fsdp_plan", None) or {})setting a custom plan will make sure onlyl these are used There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should of course be in the distributed config
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, the DTensor sniff is fragile: it happens to be exact only because the block above rewrites You said you'd take this one, so I'll leave it to you 👍
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. can you explain a bit why we need to
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It is the Under dispatch the trunk is DP over the whole mesh, so Without ignoring them they land in those param groups, and FSDP2 tries to concatenate its mesh with the parameter's own one and dies at load. Nothing transformers-specific: # torchrun --nproc_per_node 2 repro.py
import torch, torch.distributed as dist
from torch.distributed.fsdp import fully_shard
from torch.distributed.tensor import Shard, distribute_tensor
dist.init_process_group("gloo")
mesh = torch.distributed.init_device_mesh("cpu", (dist.get_world_size(),))
class Block(torch.nn.Module):
def __init__(self):
super().__init__()
self.trunk = torch.nn.Linear(8, 8, bias=False)
self.experts = torch.nn.Linear(8, 8, bias=False)
block = Block()
block.experts.weight = torch.nn.Parameter(distribute_tensor(block.experts.weight.data, mesh, [Shard(0)]))
fully_shard(block, mesh=mesh)The mesh (The branch just above is the other case: when
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
qgallouedec marked this conversation as resolved.
Resolved
|
||
|
|
||
| for module_name, module in reshard_targets: | ||
| fully_shard(module, mesh=fsdp_mesh, reshard_after_forward=True, **fsdp_policy_kwargs) | ||
| logger.debug(f"Applied fully_shard to {module_name} (reshard=True)") | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -56,6 +56,8 @@ class DistributedMixin: | |
| _tp_plan: dict[str, str] | None = None | ||
| _ep_plan: dict[str, str] | None = None | ||
| _tp_size = None | ||
| _fsdp_size = None | ||
| _expert_parallel_dispatch = False | ||
| _pp_plan: dict[str, tuple[str, str]] | None = None | ||
| _fsdp_plan: dict[str, str] | None = None | ||
|
|
||
|
|
@@ -163,17 +165,23 @@ def prepare_distribute_model( | |
| f"is not equal to world_size ({world_size})" | ||
| ) | ||
|
|
||
| if distributed_config.tp_size > 1: | ||
| if distributed_config.tp_plan is None: | ||
| distributed_config.tp_plan = "auto" | ||
| if distributed_config.tp_size > 1 and distributed_config.tp_plan is None: | ||
| distributed_config.tp_plan = "auto" | ||
|
|
||
| if distributed_config.fsdp_size > 1: | ||
| # Builds a 2-D (fsdp, tp) mesh when tensor/expert parallelism is also requested. | ||
| if device_mesh is not None: | ||
| raise ValueError( | ||
| "`device_mesh` cannot be passed together with `fsdp_size > 1`: the mesh is built here." | ||
| ) | ||
| device_map, device_mesh = initialize_fully_sharded_data_parallelism(distributed_config) | ||
| elif distributed_config.tp_size > 1: | ||
| device_map, device_mesh = initialize_tensor_parallelism( | ||
| distributed_config.tp_plan, | ||
| tp_size=distributed_config.tp_size, | ||
| device_mesh=device_mesh, | ||
| device_map=device_map, | ||
| ) | ||
| elif distributed_config.fsdp_size > 1: | ||
| device_map, device_mesh = initialize_fully_sharded_data_parallelism(distributed_config) | ||
| elif distributed_config.pp_size > 1: | ||
| device_map, device_mesh = initialize_pipeline_parallelism(distributed_config) | ||
|
|
||
|
|
@@ -190,19 +198,52 @@ def maybe_distribute_model( | |
| if device_mesh is not None: | ||
| model.config.distributed_config = distributed_config | ||
| model._device_mesh = device_mesh | ||
| # The Trainer mirrors these into accelerate's `ParallelismConfig`; without them accelerate | ||
| # sees unaccounted ranks and falls back to DDP, which rejects the DTensor parameters. | ||
| model._tp_size = distributed_config.tp_size | ||
| model._fsdp_size = distributed_config.fsdp_size | ||
| model._expert_parallel_dispatch = distributed_config.expert_parallel_dispatch | ||
|
|
||
| # Both may apply: the tensor/expert parallel plan shards across `tp` first, then FSDP2 | ||
| # shards every parameter (the `tp`-sharded ones included) across `fsdp`. | ||
| if distributed_config.tp_size > 1: | ||
| tp_mesh = device_mesh["tp"] if device_mesh.ndim > 1 else device_mesh | ||
| if isinstance(distributed_config.tp_plan, dict): | ||
| model.tp_plan = distributed_config.tp_plan | ||
| if distributed_config.expert_parallel_dispatch: | ||
| # Every rank trains on its own part of the batch, so only the experts can be sharded across the | ||
| # group: the experts get the dispatch style, the router keeps its global ids and scores, and | ||
| # whatever else the plan shards stays replicated, data-parallel like the rest of the trunk. | ||
| replicated = sorted( | ||
| name | ||
| for name, style in model.tp_plan.items() | ||
| if style not in ("ep_router", "grouped_gemm", "moe_tp_experts") | ||
| ) | ||
|
Comment on lines
+199
to
+201
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. pretty sure default is replicate, so if you pass the ep plan hard coded, it won't follow the one form the config. You don't need any of that in that case no?
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. it's not setting a default, it's dropping entries.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agreed it's the wrong place to hardcode the style names though. If it moves behind |
||
| if replicated: | ||
| logger.warning( | ||
| "`expert_parallel_dispatch=True` shards only the experts, so these expert parallel plan " | ||
| f"entries are ignored and their modules stay replicated: {replicated}." | ||
| ) | ||
| # `tp_plan` reads `_ep_plan` under expert parallelism, so that is the plan to rewrite. | ||
| model._ep_plan = { | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think it is better to do the dispatch of plan at the same place where at this stage of the code, we should only be reading & applying the
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if you do that, you can probably check for replicate by just doing
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agreed, it should not be modifying the plan here. @ArthurZucker made the same point in #48518 (comment) and said he would take it, so I left it alone. Happy to move it into the property as you describe if he would rather not.
(a property is read more than once, so it would need |
||
| name: "ep_dispatch_experts" if style == "moe_tp_experts" else style | ||
| for name, style in model.tp_plan.items() | ||
| if style in ("grouped_gemm", "moe_tp_experts") | ||
| } | ||
| model = apply_tensor_parallelism(model, tp_mesh) | ||
|
|
||
| if distributed_config.expert_parallel_dispatch: | ||
| # Every expert-parallel rank trains on its own part of the batch, so the parameters outside the | ||
| # experts are data-parallel across the whole mesh: FSDP2 shards them across all of it and owns | ||
| # their gradient reduction. The experts stay sharded across `tp` and, if any, across `fsdp`. | ||
| trunk_mesh = device_mesh["fsdp_tp"] if device_mesh.ndim > 1 else device_mesh | ||
| expert_mesh = device_mesh["fsdp"] if device_mesh.ndim > 1 else None | ||
| model = apply_fully_sharded_data_parallelism(model, trunk_mesh, expert_mesh=expert_mesh) | ||
| elif distributed_config.fsdp_size > 1: | ||
| fsdp_mesh = device_mesh["fsdp"] if device_mesh.ndim > 1 else device_mesh | ||
| model = apply_fully_sharded_data_parallelism(model, fsdp_mesh) | ||
|
|
||
| elif distributed_config.pp_size > 1: | ||
| if distributed_config.pp_size > 1: | ||
| pp_mesh = device_mesh["pp"] if device_mesh.ndim > 1 else device_mesh | ||
| model = apply_pipeline_parallelism(model, pp_mesh) | ||
| return model | ||
|
|
@@ -265,6 +306,16 @@ def gather_sharded_state_dict_for_save( | |
| if distributed_config is None: | ||
| return state_dict | ||
|
|
||
| if distributed_config.fsdp_size > 1 or distributed_config.expert_parallel_dispatch: | ||
| # Also covers the 2-D (fsdp, tp) mesh and token dispatch: every parameter is FSDP-managed, and | ||
| # the full state dict is only materialized on rank 0. | ||
| if not _is_torch_distributed_initialized(): | ||
| raise ValueError( | ||
| "Saving an FSDP-wrapped model requires torch.distributed to be initialized. " | ||
| "Call save_pretrained from every rank after init_process_group." | ||
| ) | ||
| return gather_full_state_dict(model_to_save) | ||
|
|
||
| if distributed_config.tp_size > 1: | ||
| state_dict = gather_state_dict_for_save( | ||
| state_dict, self._tp_plan, self._device_mesh, distributed_config.tp_size | ||
|
|
@@ -273,14 +324,6 @@ def gather_sharded_state_dict_for_save( | |
| state_dict = {} | ||
| return state_dict | ||
|
|
||
| if distributed_config.fsdp_size > 1: | ||
| if not _is_torch_distributed_initialized(): | ||
| raise ValueError( | ||
| "Saving an FSDP-wrapped model requires torch.distributed to be initialized. " | ||
| "Call save_pretrained from every rank after init_process_group." | ||
| ) | ||
| return gather_full_state_dict(model_to_save) | ||
|
|
||
| return state_dict | ||
|
|
||
| def barrier_after_gathered_checkpoint_save(self, distributed_config: DistributedConfig | None) -> None: | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.