From 8af1a002aaabce1a058dc53a2f00abfb249ef1e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Fri, 21 Aug 2026 23:10:29 +0000 Subject: [PATCH 01/19] Fix NaN gradients in expert-parallel training: mask uninitialized grouped_mm rows Under EP, sentinel token-expert slots sit beyond offsets[-1] and torch._grouped_mm leaves those output rows (fwd output and bwd d_input) uninitialized. The forward relied on a single post-mask plus a single pre-mask, letting NaN/Inf from uninitialized memory transit the activation and down-projection backward. The gate product's backward (act_fn(gate) * up) turns 0 x Inf into NaN (torch.autograd anomaly mode names this exact Mul), and it escapes into finite gradients: full fine-tuning of any EP-sharded MoE produced nan grad_norm on the second step (the first step survives only because freshly-allocated CUDA memory happens to be zeroed) and the loss collapsed to 0. Mask the sentinel-tail rows after each grouped GEMM instead. Full fine-tuning of OLMoE-1B-7B under ep=4 now matches the single-GPU loss trajectory. --- src/transformers/integrations/moe.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/transformers/integrations/moe.py b/src/transformers/integrations/moe.py index b9b288f4a696..bd0adeaaaf89 100644 --- a/src/transformers/integrations/moe.py +++ b/src/transformers/integrations/moe.py @@ -441,6 +441,11 @@ def grouped_mm_experts_forward( selected_hidden_states_g, selected_weights, offsets, bias=selected_biases, is_transposed=self.is_transposed ) # (S, 2 * intermediate_dim) or (S, intermediate_dim) depending on whether we have gating + # Mask the sentinel-tail rows the kernel left uninitialized, in forward AND backward: relying on + # the single post-mask lets NaN/Inf from the uninitialized rows transit through the activation and + # down-projection backward, where it can escape into finite gradients. + proj_out = proj_out.masked_fill(sentinel_mask, 0.0) + # Apply gating or activation if self.has_gate: # for gated experts we apply the custom/default gating mechanism @@ -458,6 +463,9 @@ def grouped_mm_experts_forward( proj_out, selected_weights, offsets, bias=selected_biases, is_transposed=self.is_transposed ) # (S, hidden_dim) + # Same as above: zero the uninitialized sentinel-tail rows before anything consumes them. + proj_out = proj_out.masked_fill(sentinel_mask, 0.0) + # Apply routing weights weighted_out = proj_out * sample_weights_g.unsqueeze(-1) # (S, hidden_dim) From 1f0b7bf661b05ae788b51bec43cc95a4cdbb853f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Fri, 21 Aug 2026 23:56:29 +0000 Subject: [PATCH 02/19] Fix wrong gradients for all non-expert parameters in expert-parallel training Under EP the router hook zeroes the routing scores of non-local experts, so in backward each rank's score gradient covers only the slots of its local experts, and nothing sums the per-rank partial gradients: the gate weights and, through the gate's input, every parameter upstream of each MoE block receive gradients missing the contributions that flow through remote experts. The existing _AllReduceBackward on the experts' hidden input covers the dispatch branch, and the top_k_weights branch is explicitly skipped when is_expert_parallel -- but under EP it is exactly as partial as under TP-MoE. Measured against a single-GPU reference (OLMoE-1B-7B, one batch, fp32 so rounding noise vanishes): before the fix, 3/179 parameters agree (relative max-abs errors 0.3-2.5 on attention, norms, embeddings and router gates, 10-100x above the run-to-run noise floor; only the last layer's experts and the final norm -- the parameters backward reaches before crossing an expert block -- are correct). After the fix: 179/179 agree, max relative error 2.7e-5. Fix: allreduce-sum the score gradient in the EP router hook, before the non-local mask (each slot has exactly one owning rank, so the sum is exact). --- src/transformers/distributed/tensor_parallel.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/transformers/distributed/tensor_parallel.py b/src/transformers/distributed/tensor_parallel.py index 3dc23a07a304..2bb1e7038260 100644 --- a/src/transformers/distributed/tensor_parallel.py +++ b/src/transformers/distributed/tensor_parallel.py @@ -709,6 +709,12 @@ def transform_output_post_forward(self, module, output, mesh): num_local_experts = num_experts // ep_size router_logits, router_scores, router_indices, *extra_outputs = output + # The gradient reaching each rank's scores covers only the slots of its local experts (the + # non-local slots are zeroed by the mask below in backward). Sum the per-rank partial + # gradients BEFORE the mask so the gate weights and the hidden states upstream of the router + # receive the full gradient — each slot has exactly one owning rank, so the sum is exact. + process_group = mesh.get_group() if mesh.ndim == 1 else mesh.get_group("tp") + router_scores = _AllReduceBackward.apply(router_scores, process_group) non_local_mask = (router_indices // num_local_experts) != ep_rank router_scores = router_scores.masked_fill(non_local_mask, 0.0) router_indices = router_indices.masked_fill(non_local_mask, -1) From 4636d4253ca784d3e279e72d682b71ebee6e5622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Sat, 22 Aug 2026 01:08:56 +0000 Subject: [PATCH 03/19] Trim comments --- src/transformers/distributed/tensor_parallel.py | 6 ++---- src/transformers/integrations/moe.py | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/transformers/distributed/tensor_parallel.py b/src/transformers/distributed/tensor_parallel.py index 2bb1e7038260..bc7364502c6b 100644 --- a/src/transformers/distributed/tensor_parallel.py +++ b/src/transformers/distributed/tensor_parallel.py @@ -709,10 +709,8 @@ def transform_output_post_forward(self, module, output, mesh): num_local_experts = num_experts // ep_size router_logits, router_scores, router_indices, *extra_outputs = output - # The gradient reaching each rank's scores covers only the slots of its local experts (the - # non-local slots are zeroed by the mask below in backward). Sum the per-rank partial - # gradients BEFORE the mask so the gate weights and the hidden states upstream of the router - # receive the full gradient — each slot has exactly one owning rank, so the sum is exact. + # Each rank's score gradient covers only its local experts' slots; sum the per-rank partials + # before the mask (each slot has exactly one owning rank, so the sum is exact). process_group = mesh.get_group() if mesh.ndim == 1 else mesh.get_group("tp") router_scores = _AllReduceBackward.apply(router_scores, process_group) non_local_mask = (router_indices // num_local_experts) != ep_rank diff --git a/src/transformers/integrations/moe.py b/src/transformers/integrations/moe.py index bd0adeaaaf89..49312469d43a 100644 --- a/src/transformers/integrations/moe.py +++ b/src/transformers/integrations/moe.py @@ -441,9 +441,7 @@ def grouped_mm_experts_forward( selected_hidden_states_g, selected_weights, offsets, bias=selected_biases, is_transposed=self.is_transposed ) # (S, 2 * intermediate_dim) or (S, intermediate_dim) depending on whether we have gating - # Mask the sentinel-tail rows the kernel left uninitialized, in forward AND backward: relying on - # the single post-mask lets NaN/Inf from the uninitialized rows transit through the activation and - # down-projection backward, where it can escape into finite gradients. + # Zero the sentinel-tail rows the kernel left uninitialized (fwd output and bwd `d_input`). proj_out = proj_out.masked_fill(sentinel_mask, 0.0) # Apply gating or activation @@ -463,7 +461,7 @@ def grouped_mm_experts_forward( proj_out, selected_weights, offsets, bias=selected_biases, is_transposed=self.is_transposed ) # (S, hidden_dim) - # Same as above: zero the uninitialized sentinel-tail rows before anything consumes them. + # Same: zero the uninitialized sentinel-tail rows. proj_out = proj_out.masked_fill(sentinel_mask, 0.0) # Apply routing weights From 89fa55ca91ec52c19a17c74d75cdae7b80c587ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Thu, 3 Sep 2026 02:17:41 +0000 Subject: [PATCH 04/19] Gate the router-score backward all-reduce on grad mode; drop the post-mask superseded by the per-mm masks --- src/transformers/distributed/tensor_parallel.py | 5 +++-- src/transformers/integrations/moe.py | 3 --- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/transformers/distributed/tensor_parallel.py b/src/transformers/distributed/tensor_parallel.py index bc7364502c6b..6a9455637a33 100644 --- a/src/transformers/distributed/tensor_parallel.py +++ b/src/transformers/distributed/tensor_parallel.py @@ -711,8 +711,9 @@ def transform_output_post_forward(self, module, output, mesh): router_logits, router_scores, router_indices, *extra_outputs = output # Each rank's score gradient covers only its local experts' slots; sum the per-rank partials # before the mask (each slot has exactly one owning rank, so the sum is exact). - process_group = mesh.get_group() if mesh.ndim == 1 else mesh.get_group("tp") - router_scores = _AllReduceBackward.apply(router_scores, process_group) + if torch.is_grad_enabled() and router_scores.requires_grad: + process_group = mesh.get_group() if mesh.ndim == 1 else mesh.get_group("tp") + router_scores = _AllReduceBackward.apply(router_scores, process_group) non_local_mask = (router_indices // num_local_experts) != ep_rank router_scores = router_scores.masked_fill(non_local_mask, 0.0) router_indices = router_indices.masked_fill(non_local_mask, -1) diff --git a/src/transformers/integrations/moe.py b/src/transformers/integrations/moe.py index 49312469d43a..fb3b736c0301 100644 --- a/src/transformers/integrations/moe.py +++ b/src/transformers/integrations/moe.py @@ -467,9 +467,6 @@ def grouped_mm_experts_forward( # Apply routing weights weighted_out = proj_out * sample_weights_g.unsqueeze(-1) # (S, hidden_dim) - # Post-mask (fwd path). - weighted_out.masked_fill_(sentinel_mask, 0.0) - # Restore original order inv_perm = torch.empty_like(perm) inv_perm[perm] = torch.arange(perm.size(0), device=device) From 96769d9972449eb4874553aa7cb3cb968e02f1b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:22:19 -0400 Subject: [PATCH 05/19] Fix expert parallelism through Trainer (#48208) --- src/transformers/distributed/mixin.py | 1 + src/transformers/trainer.py | 55 ++++++++++++++++++++++++++- src/transformers/trainer_optimizer.py | 17 +++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/src/transformers/distributed/mixin.py b/src/transformers/distributed/mixin.py index eb1bafd7355f..2de4ebfcdf07 100644 --- a/src/transformers/distributed/mixin.py +++ b/src/transformers/distributed/mixin.py @@ -190,6 +190,7 @@ def maybe_distribute_model( if device_mesh is not None: model.config.distributed_config = distributed_config model._device_mesh = device_mesh + model._tp_size = distributed_config.tp_size if distributed_config.tp_size > 1: tp_mesh = device_mesh["tp"] if device_mesh.ndim > 1 else device_mesh diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 7a5005146858..5b2781b51916 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -610,6 +610,8 @@ def __init__( self._train_batch_size = args.train_batch_size # Guards one-time LR scheduler creation in create_optimizer_and_scheduler self._created_lr_scheduler = False + # Resolved lazily at the first gradient clip; see `_has_mixed_mesh_grads`. + self._mixed_mesh_grads: bool | None = None self.control = self.callback_handler.on_init_end(self.args, self.state, self.control) @@ -2613,17 +2615,68 @@ def _track_num_input_tokens(self, inputs): input_tokens = torch.as_tensor(input_tokens, device=self.args.device, dtype=torch.int64) self.state.num_input_tokens_seen += self.accelerator.gather(input_tokens).sum().item() + def _mixed_mesh_grad_norm(self, model, max_norm): + """Gradient norm (and clip) when only some parameters are sharded, as under expert parallelism. + + Expert parallelism shards the expert weights and leaves everything else replicated, so + `model.parameters()` holds a mix of `DTensor` and plain tensors and `_foreach_norm` cannot span + both. Take the two groups separately: the sharded gradients contribute their local norms summed + across the mesh, the replicated ones are identical on every rank and are counted once. + """ + from torch.distributed.tensor import DTensor + + sharded, replicated = [], [] + for param in model.parameters(): + if param.grad is None: + continue + (sharded if isinstance(param.grad, DTensor) else replicated).append(param.grad) + + device = (sharded or replicated)[0].device + replicated_sq = torch.zeros((), device=device, dtype=torch.float32) + if replicated: + replicated_sq = torch.linalg.vector_norm(torch.stack([g.norm(2) for g in replicated])) ** 2 + + sharded_sq = torch.zeros((), device=device, dtype=torch.float32) + if sharded: + local = [g.to_local() for g in sharded] + sharded_sq = torch.linalg.vector_norm(torch.stack([g.norm(2) for g in local])) ** 2 + # Sum the per-shard contributions over the mesh the experts are sharded on. + torch.distributed.all_reduce(sharded_sq, group=sharded[0].device_mesh.get_group()) + + total_norm = (replicated_sq + sharded_sq).sqrt() + if max_norm != float("inf"): + clip = (max_norm / (total_norm + 1e-6)).clamp(max=1.0) + for g in replicated: + g.mul_(clip) + for g in sharded: + g.to_local().mul_(clip) + return total_norm + + def _has_mixed_mesh_grads(self, model) -> bool: + # Static for the life of the run (sharding never changes after setup), so scan the + # parameters only on the first call. + if self._mixed_mesh_grads is None: + from .trainer_optimizer import has_mixed_dtensor + + self._mixed_mesh_grads = has_mixed_dtensor(p.grad for p in model.parameters() if p.grad is not None) + return self._mixed_mesh_grads + def _clip_grad_norm(self, model): """Clip gradients to max_grad_norm. Returns the pre-clip gradient norm.""" if is_sagemaker_mp_enabled() and self.args.fp16: return self.optimizer.clip_master_grads(self.args.max_grad_norm) + if self._has_mixed_mesh_grads(model): + return self._mixed_mesh_grad_norm(model, self.args.max_grad_norm) return self.accelerator.clip_grad_norm_(model.parameters(), self.args.max_grad_norm) def _get_grad_norm(self, model, grad_norm=None): """Return the gradient norm as a Python float.""" if grad_norm is None: # Compute norm without clipping (inf means no actual clipping happens) - grad_norm = self.accelerator.clip_grad_norm_(model.parameters(), float("inf")) + if self._has_mixed_mesh_grads(model): + grad_norm = self._mixed_mesh_grad_norm(model, float("inf")) + else: + grad_norm = self.accelerator.clip_grad_norm_(model.parameters(), float("inf")) if self.accelerator.distributed_type == DistributedType.DEEPSPEED: if hasattr(grad_norm, "item"): diff --git a/src/transformers/trainer_optimizer.py b/src/transformers/trainer_optimizer.py index de63c614f156..bff836a4ca45 100644 --- a/src/transformers/trainer_optimizer.py +++ b/src/transformers/trainer_optimizer.py @@ -205,9 +205,26 @@ def _get_adamw_torch(ctx: OptimizerContext) -> tuple[Any, dict[str, Any]]: ctx.optimizer_kwargs.update(ctx.adam_kwargs) if ctx.args.optim == OptimizerNames.ADAMW_TORCH_FUSED: ctx.optimizer_kwargs.update({"fused": True}) + # HACK(ep-2d): expert parallelism leaves only the expert weights as DTensor, so the fused and + # foreach kernels cannot span the parameter set. Step per parameter instead. + if ctx.model is not None and has_mixed_dtensor(p for p in ctx.model.parameters() if p.requires_grad): + ctx.optimizer_kwargs.update({"fused": False, "foreach": False}) return AdamW, ctx.optimizer_kwargs +def has_mixed_dtensor(tensors) -> bool: + """Whether `tensors` mixes `DTensor` and plain tensors, as expert parallelism leaves a model: + experts sharded, everything else replicated.""" + from torch.distributed.tensor import DTensor + + tensors = list(tensors) + return ( + bool(tensors) + and any(isinstance(t, DTensor) for t in tensors) + and not all(isinstance(t, DTensor) for t in tensors) + ) + + def _get_adamw_torch_xla(ctx: OptimizerContext) -> tuple[Any, dict[str, Any]]: """Get Torch XLA syncfree AdamW optimizer.""" try: From c74c1228fb9e0ae515cd433dab2ca24e8aba14e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Thu, 3 Sep 2026 19:45:04 +0000 Subject: [PATCH 06/19] Enable FSDP2 + expert parallelism via a 2-D (fsdp, tp) device mesh DistributedConfig(tp_size=E, fsdp_size=D, enable_expert_parallel=True) builds a 2-D mesh: experts are sharded across tp, everything else is fully sharded across fsdp. The Trainer mirrors both dimensions into accelerate's ParallelismConfig, averages the expert gradients over fsdp (FSDP2 only reduces what it shards), computes the gradient norm across parameters on different meshes, and gathers the DTensor state dict on save. --- src/transformers/distributed/mixin.py | 18 +++-- src/transformers/distributed/utils.py | 6 ++ src/transformers/modeling_utils.py | 8 ++ src/transformers/trainer.py | 110 ++++++++++++++++++++------ src/transformers/trainer_optimizer.py | 18 ++--- tests/test_fsdp_mixin.py | 80 +++++++++++++++++-- 6 files changed, 192 insertions(+), 48 deletions(-) diff --git a/src/transformers/distributed/mixin.py b/src/transformers/distributed/mixin.py index 2de4ebfcdf07..ff7834cf62ef 100644 --- a/src/transformers/distributed/mixin.py +++ b/src/transformers/distributed/mixin.py @@ -56,6 +56,7 @@ class DistributedMixin: _tp_plan: dict[str, str] | None = None _ep_plan: dict[str, str] | None = None _tp_size = None + _fsdp_size = None _pp_plan: dict[str, tuple[str, str]] | None = None _fsdp_plan: dict[str, str] | None = None @@ -163,7 +164,12 @@ def prepare_distribute_model( f"is not equal to world_size ({world_size})" ) - if distributed_config.tp_size > 1: + if distributed_config.fsdp_size > 1: + # Builds a 2-D (fsdp, tp) mesh when tensor/expert parallelism is also requested. + device_map, device_mesh = initialize_fully_sharded_data_parallelism(distributed_config) + if distributed_config.tp_size > 1 and distributed_config.tp_plan is None: + distributed_config.tp_plan = "auto" + elif distributed_config.tp_size > 1: if distributed_config.tp_plan is None: distributed_config.tp_plan = "auto" device_map, device_mesh = initialize_tensor_parallelism( @@ -172,8 +178,6 @@ def prepare_distribute_model( device_mesh=device_mesh, device_map=device_map, ) - elif distributed_config.fsdp_size > 1: - device_map, device_mesh = initialize_fully_sharded_data_parallelism(distributed_config) elif distributed_config.pp_size > 1: device_map, device_mesh = initialize_pipeline_parallelism(distributed_config) @@ -190,19 +194,23 @@ def maybe_distribute_model( if device_mesh is not None: model.config.distributed_config = distributed_config model._device_mesh = device_mesh + # The Trainer mirrors these into accelerate's `ParallelismConfig`; without them accelerate + # sees unaccounted ranks and falls back to DDP, which rejects the DTensor parameters. model._tp_size = distributed_config.tp_size + model._fsdp_size = distributed_config.fsdp_size + # Both may apply: experts/attention are sharded across `tp`, the rest across `fsdp`. if distributed_config.tp_size > 1: tp_mesh = device_mesh["tp"] if device_mesh.ndim > 1 else device_mesh if isinstance(distributed_config.tp_plan, dict): model.tp_plan = distributed_config.tp_plan model = apply_tensor_parallelism(model, tp_mesh) - elif distributed_config.fsdp_size > 1: + if distributed_config.fsdp_size > 1: fsdp_mesh = device_mesh["fsdp"] if device_mesh.ndim > 1 else device_mesh model = apply_fully_sharded_data_parallelism(model, fsdp_mesh) - elif distributed_config.pp_size > 1: + if distributed_config.pp_size > 1: pp_mesh = device_mesh["pp"] if device_mesh.ndim > 1 else device_mesh model = apply_pipeline_parallelism(model, pp_mesh) return model diff --git a/src/transformers/distributed/utils.py b/src/transformers/distributed/utils.py index 49f0f1c41814..50cf27bda817 100644 --- a/src/transformers/distributed/utils.py +++ b/src/transformers/distributed/utils.py @@ -193,11 +193,17 @@ def initialize_fully_sharded_data_parallelism(distributed_config: DistributedCon device_map = torch.device(device_type) fsdp_size = distributed_config.fsdp_size + tp_size = distributed_config.tp_size or 1 + # `fsdp` is the outer dimension so that the `tp` ranks of a group are contiguous, which is what + # the expert all-to-all and the TP collectives want. dims, names = [], [] if fsdp_size > 1: dims.append(fsdp_size) names.append("fsdp") + if tp_size > 1: + dims.append(tp_size) + names.append("tp") # Build the N-dimensional device mesh mesh = torch.distributed.init_device_mesh(device_type, tuple(dims), mesh_dim_names=tuple(names)) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index a7da91270ff4..334ec8955476 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -4611,6 +4611,14 @@ def tp_size(self): # if None, the model didn't undergo tensor parallel sharding return self._tp_size + @property + def fsdp_size(self): + """ + Returns the model's FSDP sharding degree. + """ + # if None, the model didn't undergo FSDP sharding + return self._fsdp_size + @property def supports_pp_plan(self): # Check if model has a PP plan diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 5b2781b51916..49c370beeaeb 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -473,6 +473,9 @@ def __init__( or self.is_fsdp_xla_enabled or self.is_fsdp_enabled or is_sagemaker_mp_enabled() + # Sharded at load time (`DistributedConfig`): the model manages its own placement, and + # `.to()` on FSDP2-managed (possibly CPU-offloaded) parameters raises in `_apply`. + or getattr(model, "_device_mesh", None) is not None ): self.place_model_on_device = False else: @@ -748,7 +751,16 @@ def _build_accelerator_args(self, **kwargs) -> dict[str, Any]: ) args["parallelism_config"] = self.args.parallelism_config - if getattr(self.model, "tp_size", None) is not None and self.model.tp_size > 1: + model_fsdp_size = getattr(self.model, "fsdp_size", None) or 1 + if model_fsdp_size > 1: + # 2-D mesh: accelerate has to know about both dimensions or it will try to wrap the + # already-sharded model in DDP. + from accelerate import ParallelismConfig + + args["parallelism_config"] = ParallelismConfig( + dp_shard_size=model_fsdp_size, tp_size=getattr(self.model, "tp_size", None) or 1 + ) + elif getattr(self.model, "tp_size", None) is not None and self.model.tp_size > 1: if self.args.parallelism_config is None: if is_accelerate_available("1.12.0"): if self.args.parallelism_config is None: @@ -1859,6 +1871,7 @@ def _run_epoch( self._track_num_input_tokens(inputs) if do_sync_step: + self._sync_replicated_grads(model) grad_norm = None if self.args.max_grad_norm > 0: grad_norm = self._clip_grad_norm(model) @@ -2615,41 +2628,64 @@ def _track_num_input_tokens(self, inputs): input_tokens = torch.as_tensor(input_tokens, device=self.args.device, dtype=torch.int64) self.state.num_input_tokens_seen += self.accelerator.gather(input_tokens).sum().item() - def _mixed_mesh_grad_norm(self, model, max_norm): - """Gradient norm (and clip) when only some parameters are sharded, as under expert parallelism. - - Expert parallelism shards the expert weights and leaves everything else replicated, so - `model.parameters()` holds a mix of `DTensor` and plain tensors and `_foreach_norm` cannot span - both. Take the two groups separately: the sharded gradients contribute their local norms summed - across the mesh, the replicated ones are identical on every rank and are counted once. - """ + def _sync_replicated_grads(self, model): + """Average the gradients of parameters replicated along the data-parallel dimension (e.g. the + EP-sharded experts under a 2-D mesh) -- FSDP2 only reduces what it shards.""" from torch.distributed.tensor import DTensor - sharded, replicated = [], [] + mesh = getattr(model, "_device_mesh", None) or getattr(getattr(model, "module", None), "_device_mesh", None) + if mesh is None or mesh.ndim < 2 or "fsdp" not in (mesh.mesh_dim_names or ()): + return + dp_group = mesh["fsdp"].get_group() + dp_size = mesh["fsdp"].size() + if dp_size < 2: + return + for param in model.parameters(): if param.grad is None: continue - (sharded if isinstance(param.grad, DTensor) else replicated).append(param.grad) + grad = param.grad + if isinstance(grad, DTensor): + # Already reduced over `fsdp` if that dimension is part of the gradient's own mesh. + if "fsdp" in (grad.device_mesh.mesh_dim_names or ()): + continue + grad = grad.to_local() + torch.distributed.all_reduce(grad, group=dp_group) + grad.div_(dp_size) - device = (sharded or replicated)[0].device - replicated_sq = torch.zeros((), device=device, dtype=torch.float32) - if replicated: - replicated_sq = torch.linalg.vector_norm(torch.stack([g.norm(2) for g in replicated])) ** 2 + def _mixed_mesh_grad_norm(self, model, max_norm): + """Gradient norm (and clip) when parameters live on different device meshes: sum each gradient's + local squared norm, discounting the mesh dimensions it is replicated over, reduce once.""" + from torch.distributed.tensor import DTensor + + grads = [p.grad for p in model.parameters() if p.grad is not None] + if not grads: + return torch.zeros((), device=self.args.device) + + world = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1 + total_sq = torch.zeros((), device=grads[0].device, dtype=torch.float32) + for g in grads: + if isinstance(g, DTensor): + local = g.to_local() + # Ranks along a replicated mesh dimension all hold the same values. + replicas = 1 + for dim, placement in enumerate(g.placements): + if placement.is_replicate(): + replicas *= g.device_mesh.size(dim) + # Dimensions of the world this tensor's mesh does not span also hold copies. + replicas *= world // g.device_mesh.size() + else: + local, replicas = g, world # a plain tensor is the same on every rank + total_sq += local.detach().float().pow(2).sum() / replicas - sharded_sq = torch.zeros((), device=device, dtype=torch.float32) - if sharded: - local = [g.to_local() for g in sharded] - sharded_sq = torch.linalg.vector_norm(torch.stack([g.norm(2) for g in local])) ** 2 - # Sum the per-shard contributions over the mesh the experts are sharded on. - torch.distributed.all_reduce(sharded_sq, group=sharded[0].device_mesh.get_group()) + if world > 1: + torch.distributed.all_reduce(total_sq) + total_norm = total_sq.sqrt() - total_norm = (replicated_sq + sharded_sq).sqrt() if max_norm != float("inf"): clip = (max_norm / (total_norm + 1e-6)).clamp(max=1.0) - for g in replicated: - g.mul_(clip) - for g in sharded: - g.to_local().mul_(clip) + for g in grads: + (g.to_local() if isinstance(g, DTensor) else g).mul_(clip) return total_norm def _has_mixed_mesh_grads(self, model) -> bool: @@ -3978,6 +4014,28 @@ def save_model(self, output_dir: str | None = None, _internal_call: bool = False remove_dummy_checkpoint(self.args.should_save, output_dir, [WEIGHTS_NAME, SAFE_WEIGHTS_NAME]) self.model_wrapped.save_checkpoint(output_dir) + elif getattr( + self.accelerator.unwrap_model(self.model, keep_torch_compile=False), "_device_mesh", None + ) is not None and not _is_peft_model(self.model): + # The model was sharded at load time (`DistributedConfig`). Gathering its DTensor state + # dict is collective, so every rank must participate; only the main process writes. + # (PEFT models fall through to the adapter-only save below.) + from .distributed.tensor_parallel import gather_state_dict_for_save + + unwrapped = self.accelerator.unwrap_model(self.model, keep_torch_compile=False) + gather_start = time.time() + state_dict = gather_state_dict_for_save(unwrapped.state_dict(), None, None, 0) + gather_s = time.time() - gather_start + if self.args.should_save: + write_start = time.time() + # `save_pretrained` ends with `barrier_after_gathered_checkpoint_save`. + self._save(output_dir, state_dict=state_dict) + logger.info(f"Sharded-model save: gather {gather_s:.1f}s, write {time.time() - write_start:.1f}s") + else: + # Match the writer's end-of-save barrier so no rank runs ahead into the next + # step's collectives while the main process is still writing. + unwrapped.barrier_after_gathered_checkpoint_save(unwrapped.config.distributed_config) + elif self.args.should_save: self._save(output_dir) diff --git a/src/transformers/trainer_optimizer.py b/src/transformers/trainer_optimizer.py index bff836a4ca45..bc78622cdc29 100644 --- a/src/transformers/trainer_optimizer.py +++ b/src/transformers/trainer_optimizer.py @@ -205,24 +205,22 @@ def _get_adamw_torch(ctx: OptimizerContext) -> tuple[Any, dict[str, Any]]: ctx.optimizer_kwargs.update(ctx.adam_kwargs) if ctx.args.optim == OptimizerNames.ADAMW_TORCH_FUSED: ctx.optimizer_kwargs.update({"fused": True}) - # HACK(ep-2d): expert parallelism leaves only the expert weights as DTensor, so the fused and - # foreach kernels cannot span the parameter set. Step per parameter instead. + # Expert parallelism leaves the parameters on different meshes, so the fused and foreach kernels + # cannot span the parameter set. Step per parameter instead. if ctx.model is not None and has_mixed_dtensor(p for p in ctx.model.parameters() if p.requires_grad): ctx.optimizer_kwargs.update({"fused": False, "foreach": False}) return AdamW, ctx.optimizer_kwargs def has_mixed_dtensor(tensors) -> bool: - """Whether `tensors` mixes `DTensor` and plain tensors, as expert parallelism leaves a model: - experts sharded, everything else replicated.""" + """Whether `tensors` do not all share one device mesh, so whole-set ops (fused/foreach kernels, + `clip_grad_norm_`) cannot span them. Expert parallelism leaves the experts sharded and everything + else as plain tensors; under a 2-D (fsdp, tp) mesh everything is a `DTensor`, but the experts live + on the `tp` mesh and the rest on the `fsdp` mesh.""" from torch.distributed.tensor import DTensor - tensors = list(tensors) - return ( - bool(tensors) - and any(isinstance(t, DTensor) for t in tensors) - and not all(isinstance(t, DTensor) for t in tensors) - ) + meshes = {t.device_mesh if isinstance(t, DTensor) else None for t in tensors} + return len(meshes) > 1 def _get_adamw_torch_xla(ctx: OptimizerContext) -> tuple[Any, dict[str, Any]]: diff --git a/tests/test_fsdp_mixin.py b/tests/test_fsdp_mixin.py index 51dd0c7b20e5..ea658341ce40 100644 --- a/tests/test_fsdp_mixin.py +++ b/tests/test_fsdp_mixin.py @@ -494,6 +494,60 @@ def _test_fsdp2_plan_vs_ddp_impl(rank, config_class, config_dict, tie_word_embed logger.debug("DDP and FSDP2 comparison checks passed.") +def _test_fsdp2_expert_parallel_2d_vs_ddp_impl(rank, config_class, config_dict, dtype=None): + """DDP vs a 2-D (fsdp, tp) mesh with expert parallelism on `tp`: same batch on every rank, so the + sharded run has to trace the DDP run step by step and end on the same weights.""" + init_test_logger() + + if dtype is None: + dtype = torch.float32 + + device = _get_rank_device(rank) + config = config_class.from_dict(config_dict) + batches = _build_repeated_training_batches(config, device, NUM_STEPS) + + with _deterministic_init_model_dir(rank, config, dtype) as init_model_dir: + ddp_losses, _, ddp_state_dict = train_ddp(rank, batches, LR, device, dtype, init_model_dir) + + _set_determinism(SEED) + world_size = dist.get_world_size() + model = AutoModelForCausalLM.from_pretrained( + init_model_dir, + torch_dtype=dtype, + distributed_config=DistributedConfig(tp_size=2, fsdp_size=world_size // 2, enable_expert_parallel=True), + ) + assert model.tp_size == 2 and model.fsdp_size == world_size // 2 + assert model._device_mesh.mesh_dim_names == ("fsdp", "tp") + model.train() + optimizer = torch.optim.Adam(model.parameters(), lr=LR, foreach=False) + # `clip_grad_norm_` cannot span parameters living on different meshes; the Trainer has its own + # norm for that case, so only the losses and the final weights are compared here. + losses, _ = _run_training_steps(model, optimizer, batches, track_grad_norms=False) + state_dict = gather_full_state_dict(model) + + for step in range(len(ddp_losses)): + torch.testing.assert_close( + torch.tensor(ddp_losses[step]), + torch.tensor(losses[step]), + rtol=DDP_FSDP_RTOL, + atol=DDP_FSDP_ATOL, + msg=f"Loss mismatch at step {step}: DDP={ddp_losses[step]}, FSDP2+EP={losses[step]}", + ) + + for key in ddp_state_dict: + assert key in state_dict, f"Key {key} missing from FSDP2+EP state dict" + torch.testing.assert_close( + ddp_state_dict[key], + state_dict[key], + rtol=DDP_FSDP_RTOL, + atol=DDP_FSDP_ATOL, + msg=f"Weight mismatch for {key}: DDP vs FSDP2+EP", + ) + + if rank == 0: + logger.debug("DDP and FSDP2+EP (2-D mesh) comparison checks passed.") + + # ============================================================================= # Mixin class # ============================================================================= @@ -509,10 +563,11 @@ def model_tester(self): """The model tester instance (e.g., CausalLMModelTester).""" ... - def _skip_if_insufficient_devices(self): + def _skip_if_insufficient_devices(self, world_size=None): + world_size = world_size or self.fsdp_nproc_per_node available_workers = _get_available_fsdp_workers() - if available_workers < self.fsdp_nproc_per_node: - self.skipTest(f"Need at least {self.fsdp_nproc_per_node} FSDP workers, have {available_workers}") + if available_workers < world_size: + self.skipTest(f"Need at least {world_size} FSDP workers, have {available_workers}") def _skip_if_mps(self): if torch._C._get_accelerator().type == "mps": @@ -558,9 +613,10 @@ def _get_tiny_config(self): config.vocab_size_per_layer_input = config.vocab_size return type(config), config.to_diff_dict() - def _run_fsdp2_distributed_test(self, test_name, test_impl, *test_args, **test_kwargs): + def _run_fsdp2_distributed_test(self, test_name, test_impl, *test_args, world_size=None, **test_kwargs): + world_size = world_size or self.fsdp_nproc_per_node self._skip_if_mps() - self._skip_if_insufficient_devices() + self._skip_if_insufficient_devices(world_size) self._skip_if_fsdp_distributed_not_enabled() config_class, config_dict = self._get_tiny_config() @@ -575,8 +631,8 @@ def _run_fsdp2_distributed_test(self, test_name, test_impl, *test_args, **test_k try: mp.spawn( _fsdp_global_wrapper, - args=(test_name, test_impl, func_args, test_kwargs, self.fsdp_nproc_per_node, port, results_file), - nprocs=self.fsdp_nproc_per_node, + args=(test_name, test_impl, func_args, test_kwargs, world_size, port, results_file), + nprocs=world_size, ) with open(results_file) as f: @@ -636,3 +692,13 @@ def test_fsdp2_plan_vs_ddp(self, label): _test_fsdp2_plan_vs_ddp_impl, label == "tied", ) + + @is_fsdp_test + def test_fsdp2_expert_parallel_2d_vs_ddp(self): + """Training on a 2-D (fsdp, tp) mesh with expert parallelism traces DDP step by step.""" + config = self.model_tester.get_config() + if getattr(config, "base_model_ep_plan", None) is None: + self.skipTest("Model does not have an expert parallel plan (base_model_ep_plan)") + self._run_fsdp2_distributed_test( + "fsdp2_expert_parallel_2d_vs_ddp", _test_fsdp2_expert_parallel_2d_vs_ddp_impl, world_size=4 + ) From 1c66acbc5adf0133ecebf025886658b21fc84c99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Thu, 3 Sep 2026 19:54:59 +0000 Subject: [PATCH 07/19] Gather the sharded state dict through the model's own save helpers --- src/transformers/trainer.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 49c370beeaeb..f4d0baa3f714 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -4018,23 +4018,20 @@ def save_model(self, output_dir: str | None = None, _internal_call: bool = False self.accelerator.unwrap_model(self.model, keep_torch_compile=False), "_device_mesh", None ) is not None and not _is_peft_model(self.model): # The model was sharded at load time (`DistributedConfig`). Gathering its DTensor state - # dict is collective, so every rank must participate; only the main process writes. + # dict is collective, so every rank must take part; only the main process writes. # (PEFT models fall through to the adapter-only save below.) - from .distributed.tensor_parallel import gather_state_dict_for_save - unwrapped = self.accelerator.unwrap_model(self.model, keep_torch_compile=False) - gather_start = time.time() - state_dict = gather_state_dict_for_save(unwrapped.state_dict(), None, None, 0) - gather_s = time.time() - gather_start + distributed_config = unwrapped.config.distributed_config + state_dict = unwrapped.gather_sharded_state_dict_for_save( + unwrapped, unwrapped.state_dict(), distributed_config, save_on_this_rank=self.args.should_save + ) if self.args.should_save: - write_start = time.time() # `save_pretrained` ends with `barrier_after_gathered_checkpoint_save`. self._save(output_dir, state_dict=state_dict) - logger.info(f"Sharded-model save: gather {gather_s:.1f}s, write {time.time() - write_start:.1f}s") else: # Match the writer's end-of-save barrier so no rank runs ahead into the next # step's collectives while the main process is still writing. - unwrapped.barrier_after_gathered_checkpoint_save(unwrapped.config.distributed_config) + unwrapped.barrier_after_gathered_checkpoint_save(distributed_config) elif self.args.should_save: self._save(output_dir) From 38aefbf2214d18dc60b0d7bf717afc90c8b65551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Thu, 3 Sep 2026 19:55:18 +0000 Subject: [PATCH 08/19] Document combining expert parallelism with FSDP2 --- docs/source/en/expert_parallelism.md | 26 ++++++++++++++++++++++++++ docs/source/en/fsdp.md | 3 +++ 2 files changed, 29 insertions(+) diff --git a/docs/source/en/expert_parallelism.md b/docs/source/en/expert_parallelism.md index f66d993495e6..7bab29ddf965 100644 --- a/docs/source/en/expert_parallelism.md +++ b/docs/source/en/expert_parallelism.md @@ -50,4 +50,30 @@ Launch your inference script with [torchrun](https://pytorch.org/docs/stable/ela torchrun --nproc-per-node 8 your_script.py ``` +## Combining with FSDP2 + +Expert parallelism only shards the experts. Everything else (attention, embeddings, norms) and its optimizer state is replicated on every expert-parallel rank, which is what limits the model size you can train. Set `fsdp_size` together with `tp_size` to shard the rest of the model with [FSDP2](./fsdp) on a second mesh dimension. + +```py +distributed_config = DistributedConfig( + tp_size=8, # expert parallel size + fsdp_size=2, # data parallel shards + enable_expert_parallel=True, +) +model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-30B-A3B", distributed_config=distributed_config) +``` + +The model is loaded on a 2-D `(fsdp, tp)` device mesh: the experts are sharded across the `tp` dimension, all other parameters are fully sharded across the `fsdp` dimension, and `tp_size * fsdp_size` must equal the number of processes. The experts see the same batch on every `fsdp` rank, so their gradients are averaged over that dimension by the [`Trainer`] (FSDP2 only reduces what it shards). Nothing else changes: train with the [`Trainer`] as usual, and `save_model` gathers the sharded weights and writes a regular checkpoint. + +On 8 GPUs, full fine-tuning of Qwen3-30B-A3B in bf16 at sequence length 2048: + +| configuration | tokens/s/GPU | peak memory/GPU | +|---|---|---| +| `tp_size=8` | 3023 | 40.9 GB | +| `tp_size=4, fsdp_size=2` | 2326 | 34.2 GB | +| `tp_size=2, fsdp_size=4` | 2244 | 32.3 GB | + +> [!WARNING] +> Resuming from a checkpoint is not supported yet for models sharded at load time. + [[autodoc]] DistributedConfig diff --git a/docs/source/en/fsdp.md b/docs/source/en/fsdp.md index e0727b77d897..dbbd629f22d1 100644 --- a/docs/source/en/fsdp.md +++ b/docs/source/en/fsdp.md @@ -121,6 +121,9 @@ TrainingArguments( +> [!TIP] +> For mixture-of-experts models, `fsdp_size` can be combined with `tp_size` and `enable_expert_parallel=True` to shard the experts across one mesh dimension and everything else across the other. See [expert parallelism](./expert_parallelism#combining-with-fsdp2). + ## Next steps - See [DDP](./ddp) for data-parallel training when your model fits on one GPU. From e72b117d836a815b6adfbd8f476e6cdd8a11ec54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Thu, 3 Sep 2026 20:10:32 +0000 Subject: [PATCH 09/19] Address review: drop the dead expert-gradient sync, torch-native mixed-mesh norm, single collective save FSDP2 composes over the tp-sharded experts and shards them across fsdp as well, so nothing is replicated over fsdp and the Trainer-side gradient averaging never ran; remove it and describe the actual layout. The gradient norm is now one get_total_norm per mesh, each reduced over its own mesh. save_model runs save_pretrained on every rank so its gather is collective and only rank 0 writes; the FSDP branch of the gather (full state dict on rank 0 only) now also covers the 2-D mesh. ParallelismConfig keeps a user-supplied config and only claims what the model was loaded with; pipeline parallelism is rejected together with tp/fsdp; optimizer checkpoints are refused for models sharded at load time since they cannot be resumed. --- docs/source/en/expert_parallelism.md | 8 +- .../distributed/configuration_utils.py | 7 +- src/transformers/distributed/mixin.py | 32 ++-- src/transformers/distributed/utils.py | 2 +- src/transformers/trainer.py | 139 ++++++------------ src/transformers/trainer_optimizer.py | 2 +- tests/test_fsdp_mixin.py | 3 +- 7 files changed, 76 insertions(+), 117 deletions(-) diff --git a/docs/source/en/expert_parallelism.md b/docs/source/en/expert_parallelism.md index 7bab29ddf965..e79e8c2082f4 100644 --- a/docs/source/en/expert_parallelism.md +++ b/docs/source/en/expert_parallelism.md @@ -52,18 +52,18 @@ torchrun --nproc-per-node 8 your_script.py ## Combining with FSDP2 -Expert parallelism only shards the experts. Everything else (attention, embeddings, norms) and its optimizer state is replicated on every expert-parallel rank, which is what limits the model size you can train. Set `fsdp_size` together with `tp_size` to shard the rest of the model with [FSDP2](./fsdp) on a second mesh dimension. +Expert parallelism only shards the experts. Everything else (attention, embeddings, norms) and its optimizer state is replicated on every expert-parallel rank, which is what limits the model size you can train. Set `fsdp_size` together with `tp_size` to add [FSDP2](./fsdp) on a second mesh dimension. ```py distributed_config = DistributedConfig( - tp_size=8, # expert parallel size + tp_size=4, # expert parallel size fsdp_size=2, # data parallel shards enable_expert_parallel=True, ) model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-30B-A3B", distributed_config=distributed_config) ``` -The model is loaded on a 2-D `(fsdp, tp)` device mesh: the experts are sharded across the `tp` dimension, all other parameters are fully sharded across the `fsdp` dimension, and `tp_size * fsdp_size` must equal the number of processes. The experts see the same batch on every `fsdp` rank, so their gradients are averaged over that dimension by the [`Trainer`] (FSDP2 only reduces what it shards). Nothing else changes: train with the [`Trainer`] as usual, and `save_model` gathers the sharded weights and writes a regular checkpoint. +The model is loaded on a 2-D `(fsdp, tp)` device mesh, and `tp_size * fsdp_size` must equal the number of processes. The expert parallel plan shards the experts across `tp`, then FSDP2 shards every parameter, experts included, across `fsdp` and owns their gradient reduction. Each `fsdp` rank trains on its own part of the batch. Nothing else changes: train with the [`Trainer`] as usual (it computes the gradient norm across the two meshes and steps the optimizer per parameter), and `save_model` gathers the sharded weights and writes a regular checkpoint. On 8 GPUs, full fine-tuning of Qwen3-30B-A3B in bf16 at sequence length 2048: @@ -74,6 +74,6 @@ On 8 GPUs, full fine-tuning of Qwen3-30B-A3B in bf16 at sequence length 2048: | `tp_size=2, fsdp_size=4` | 2244 | 32.3 GB | > [!WARNING] -> Resuming from a checkpoint is not supported yet for models sharded at load time. +> Resuming from a checkpoint is not supported yet for models sharded at load time, so the [`Trainer`] only accepts `save_only_model=True` or `save_strategy="no"` for them. [[autodoc]] DistributedConfig diff --git a/src/transformers/distributed/configuration_utils.py b/src/transformers/distributed/configuration_utils.py index a18a6f9a1dda..e43e5e479281 100644 --- a/src/transformers/distributed/configuration_utils.py +++ b/src/transformers/distributed/configuration_utils.py @@ -73,11 +73,10 @@ def __post_init__(self): elif self.tp_size is None: self.tp_size = 1 - if self.tp_size > 1 and self.fsdp_size > 1 and self.pp_size > 1: + if self.pp_size > 1 and (self.tp_size > 1 or self.fsdp_size > 1): raise ValueError( - "FSDP+TP+PP is not supported yet. " - "Use DistributedConfig(fsdp_size=N) or DistributedConfig(tp_size=N) or DistributedConfig(pp_size=N), not all three. " - "Only 1D support is available for now." + "Pipeline parallelism cannot be combined with tensor or FSDP parallelism yet. " + "Use DistributedConfig(pp_size=N) on its own, or DistributedConfig(tp_size=N, fsdp_size=M)." ) @classmethod diff --git a/src/transformers/distributed/mixin.py b/src/transformers/distributed/mixin.py index ff7834cf62ef..f2dd2d4510a3 100644 --- a/src/transformers/distributed/mixin.py +++ b/src/transformers/distributed/mixin.py @@ -164,14 +164,17 @@ def prepare_distribute_model( f"is not equal to world_size ({world_size})" ) + if distributed_config.tp_size > 1 and distributed_config.tp_plan is None: + distributed_config.tp_plan = "auto" + if distributed_config.fsdp_size > 1: # Builds a 2-D (fsdp, tp) mesh when tensor/expert parallelism is also requested. + if device_mesh is not None: + raise ValueError( + "`device_mesh` cannot be passed together with `fsdp_size > 1`: the mesh is built here." + ) device_map, device_mesh = initialize_fully_sharded_data_parallelism(distributed_config) - if distributed_config.tp_size > 1 and distributed_config.tp_plan is None: - distributed_config.tp_plan = "auto" elif distributed_config.tp_size > 1: - if distributed_config.tp_plan is None: - distributed_config.tp_plan = "auto" device_map, device_mesh = initialize_tensor_parallelism( distributed_config.tp_plan, tp_size=distributed_config.tp_size, @@ -199,7 +202,8 @@ def maybe_distribute_model( model._tp_size = distributed_config.tp_size model._fsdp_size = distributed_config.fsdp_size - # Both may apply: experts/attention are sharded across `tp`, the rest across `fsdp`. + # Both may apply: the tensor/expert parallel plan shards across `tp` first, then FSDP2 + # shards every parameter (the `tp`-sharded ones included) across `fsdp`. if distributed_config.tp_size > 1: tp_mesh = device_mesh["tp"] if device_mesh.ndim > 1 else device_mesh if isinstance(distributed_config.tp_plan, dict): @@ -273,15 +277,9 @@ def gather_sharded_state_dict_for_save( if distributed_config is None: return state_dict - if distributed_config.tp_size > 1: - state_dict = gather_state_dict_for_save( - state_dict, self._tp_plan, self._device_mesh, distributed_config.tp_size - ) - if not save_on_this_rank: - state_dict = {} - return state_dict - if distributed_config.fsdp_size > 1: + # Also covers the 2-D (fsdp, tp) mesh: every parameter is FSDP-managed, and the full + # state dict is only materialized on rank 0. if not _is_torch_distributed_initialized(): raise ValueError( "Saving an FSDP-wrapped model requires torch.distributed to be initialized. " @@ -289,6 +287,14 @@ def gather_sharded_state_dict_for_save( ) return gather_full_state_dict(model_to_save) + if distributed_config.tp_size > 1: + state_dict = gather_state_dict_for_save( + state_dict, self._tp_plan, self._device_mesh, distributed_config.tp_size + ) + if not save_on_this_rank: + state_dict = {} + return state_dict + return state_dict def barrier_after_gathered_checkpoint_save(self, distributed_config: DistributedConfig | None) -> None: diff --git a/src/transformers/distributed/utils.py b/src/transformers/distributed/utils.py index 50cf27bda817..6feb97c5e730 100644 --- a/src/transformers/distributed/utils.py +++ b/src/transformers/distributed/utils.py @@ -193,7 +193,7 @@ def initialize_fully_sharded_data_parallelism(distributed_config: DistributedCon device_map = torch.device(device_type) fsdp_size = distributed_config.fsdp_size - tp_size = distributed_config.tp_size or 1 + tp_size = distributed_config.tp_size # `fsdp` is the outer dimension so that the `tp` ranks of a group are contiguous, which is what # the expert all-to-all and the TP collectives want. diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index f4d0baa3f714..c19f8b400c74 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -28,6 +28,7 @@ import tempfile import time import warnings +from collections import defaultdict from collections.abc import Callable, Iterator, Mapping from functools import partial from pathlib import Path @@ -615,6 +616,15 @@ def __init__( self._created_lr_scheduler = False # Resolved lazily at the first gradient clip; see `_has_mixed_mesh_grads`. self._mixed_mesh_grads: bool | None = None + if ( + getattr(model, "_device_mesh", None) is not None + and args.save_strategy != SaveStrategy.NO + and not args.save_only_model + ): + raise ValueError( + "Resuming is not supported for models sharded at load time (`DistributedConfig`), so their " + "optimizer state cannot be checkpointed. Pass `save_only_model=True` or `save_strategy='no'`." + ) self.control = self.callback_handler.on_init_end(self.args, self.state, self.control) @@ -751,26 +761,21 @@ def _build_accelerator_args(self, **kwargs) -> dict[str, Any]: ) args["parallelism_config"] = self.args.parallelism_config + model_tp_size = getattr(self.model, "tp_size", None) or 1 model_fsdp_size = getattr(self.model, "fsdp_size", None) or 1 - if model_fsdp_size > 1: - # 2-D mesh: accelerate has to know about both dimensions or it will try to wrap the - # already-sharded model in DDP. - from accelerate import ParallelismConfig - - args["parallelism_config"] = ParallelismConfig( - dp_shard_size=model_fsdp_size, tp_size=getattr(self.model, "tp_size", None) or 1 - ) - elif getattr(self.model, "tp_size", None) is not None and self.model.tp_size > 1: - if self.args.parallelism_config is None: - if is_accelerate_available("1.12.0"): - if self.args.parallelism_config is None: - from accelerate import ParallelismConfig - - args["parallelism_config"] = ParallelismConfig(tp_size=self.model.tp_size) - else: - raise ValueError("Requires accelerate>1.12.0 to use Tensor Parallelism.") - elif args["parallelism_config"].tp_size != self.model.tp_size: - args["parallelism_config"].tp_size = self.model.tp_size + if model_tp_size > 1: + # Sharded at load time (tensor/expert parallelism, optionally with FSDP2 on a second mesh + # dimension): accelerate has to know both sizes, or it sees unaccounted ranks and wraps + # the DTensor model in DDP, which raises. + if not is_accelerate_available("1.12.0"): + raise ValueError("Requires accelerate>1.12.0 to use Tensor Parallelism.") + if args.get("parallelism_config") is None: + from accelerate import ParallelismConfig + + args["parallelism_config"] = ParallelismConfig(tp_size=model_tp_size, dp_shard_size=model_fsdp_size) + else: + args["parallelism_config"].tp_size = model_tp_size + args["parallelism_config"].dp_shard_size = model_fsdp_size if is_accelerate_available("1.2.0"): # it we don't have the correct version, we will rely on env var instead that were set in TrainingArguments @@ -1871,7 +1876,6 @@ def _run_epoch( self._track_num_input_tokens(inputs) if do_sync_step: - self._sync_replicated_grads(model) grad_norm = None if self.args.max_grad_norm > 0: grad_norm = self._clip_grad_norm(model) @@ -2628,64 +2632,26 @@ def _track_num_input_tokens(self, inputs): input_tokens = torch.as_tensor(input_tokens, device=self.args.device, dtype=torch.int64) self.state.num_input_tokens_seen += self.accelerator.gather(input_tokens).sum().item() - def _sync_replicated_grads(self, model): - """Average the gradients of parameters replicated along the data-parallel dimension (e.g. the - EP-sharded experts under a 2-D mesh) -- FSDP2 only reduces what it shards.""" - from torch.distributed.tensor import DTensor - - mesh = getattr(model, "_device_mesh", None) or getattr(getattr(model, "module", None), "_device_mesh", None) - if mesh is None or mesh.ndim < 2 or "fsdp" not in (mesh.mesh_dim_names or ()): - return - dp_group = mesh["fsdp"].get_group() - dp_size = mesh["fsdp"].size() - if dp_size < 2: - return - - for param in model.parameters(): - if param.grad is None: - continue - grad = param.grad - if isinstance(grad, DTensor): - # Already reduced over `fsdp` if that dimension is part of the gradient's own mesh. - if "fsdp" in (grad.device_mesh.mesh_dim_names or ()): - continue - grad = grad.to_local() - torch.distributed.all_reduce(grad, group=dp_group) - grad.div_(dp_size) - def _mixed_mesh_grad_norm(self, model, max_norm): - """Gradient norm (and clip) when parameters live on different device meshes: sum each gradient's - local squared norm, discounting the mesh dimensions it is replicated over, reduce once.""" + """Gradient norm (and clip) when the gradients live on different device meshes, which + `clip_grad_norm_` cannot span: one norm per mesh, each already reduced over its own mesh.""" from torch.distributed.tensor import DTensor + from torch.nn.utils import clip_grads_with_norm_, get_total_norm - grads = [p.grad for p in model.parameters() if p.grad is not None] - if not grads: - return torch.zeros((), device=self.args.device) - - world = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1 - total_sq = torch.zeros((), device=grads[0].device, dtype=torch.float32) - for g in grads: - if isinstance(g, DTensor): - local = g.to_local() - # Ranks along a replicated mesh dimension all hold the same values. - replicas = 1 - for dim, placement in enumerate(g.placements): - if placement.is_replicate(): - replicas *= g.device_mesh.size(dim) - # Dimensions of the world this tensor's mesh does not span also hold copies. - replicas *= world // g.device_mesh.size() - else: - local, replicas = g, world # a plain tensor is the same on every rank - total_sq += local.detach().float().pow(2).sum() / replicas + params_by_mesh = defaultdict(list) + for param in model.parameters(): + if param.grad is not None: + params_by_mesh[param.grad.device_mesh if isinstance(param.grad, DTensor) else None].append(param) - if world > 1: - torch.distributed.all_reduce(total_sq) - total_norm = total_sq.sqrt() + norms = [] + for params in params_by_mesh.values(): + norm = get_total_norm([p.grad for p in params]) + norms.append(norm.full_tensor() if isinstance(norm, DTensor) else norm) + total_norm = torch.linalg.vector_norm(torch.stack(norms)) if max_norm != float("inf"): - clip = (max_norm / (total_norm + 1e-6)).clamp(max=1.0) - for g in grads: - (g.to_local() if isinstance(g, DTensor) else g).mul_(clip) + for params in params_by_mesh.values(): + clip_grads_with_norm_(params, max_norm, total_norm) return total_norm def _has_mixed_mesh_grads(self, model) -> bool: @@ -4014,24 +3980,11 @@ def save_model(self, output_dir: str | None = None, _internal_call: bool = False remove_dummy_checkpoint(self.args.should_save, output_dir, [WEIGHTS_NAME, SAFE_WEIGHTS_NAME]) self.model_wrapped.save_checkpoint(output_dir) - elif getattr( - self.accelerator.unwrap_model(self.model, keep_torch_compile=False), "_device_mesh", None - ) is not None and not _is_peft_model(self.model): - # The model was sharded at load time (`DistributedConfig`). Gathering its DTensor state - # dict is collective, so every rank must take part; only the main process writes. - # (PEFT models fall through to the adapter-only save below.) - unwrapped = self.accelerator.unwrap_model(self.model, keep_torch_compile=False) - distributed_config = unwrapped.config.distributed_config - state_dict = unwrapped.gather_sharded_state_dict_for_save( - unwrapped, unwrapped.state_dict(), distributed_config, save_on_this_rank=self.args.should_save - ) - if self.args.should_save: - # `save_pretrained` ends with `barrier_after_gathered_checkpoint_save`. - self._save(output_dir, state_dict=state_dict) - else: - # Match the writer's end-of-save barrier so no rank runs ahead into the next - # step's collectives while the main process is still writing. - unwrapped.barrier_after_gathered_checkpoint_save(distributed_config) + elif getattr(self.model, "_device_mesh", None) is not None and not _is_peft_model(self.model): + # Sharded at load time (`DistributedConfig`): gathering the weights inside `save_pretrained` + # is collective, so every rank saves; only the main process writes, the others leave at the + # closing barrier. (PEFT models fall through to the adapter-only save below.) + self._save(output_dir) elif self.args.should_save: self._save(output_dir) @@ -4042,10 +3995,10 @@ def save_model(self, output_dir: str | None = None, _internal_call: bool = False def _save(self, output_dir: str | None = None, state_dict: dict | None = None) -> None: """Save model weights, configuration, and processing class to `output_dir`.""" - # If we are executing this function, we are the process zero, so we don't check for that. output_dir = output_dir if output_dir is not None else self.args.output_dir os.makedirs(output_dir, exist_ok=True) - logger.info(f"Saving model checkpoint to {output_dir}") + if self.args.should_save: + logger.info(f"Saving model checkpoint to {output_dir}") supported_classes = (PreTrainedModel,) if not is_peft_available() else (PreTrainedModel, PeftModel) # Save a trained model and configuration using `save_pretrained()`. @@ -4066,6 +4019,8 @@ def _save(self, output_dir: str | None = None, state_dict: dict | None = None) - else: self.model.save_pretrained(output_dir, state_dict=state_dict) + if not self.args.should_save: + return # a non-writer rank of a model sharded at load time, only here for the collectives above if self.processing_class is not None: self.processing_class.save_pretrained(output_dir) elif ( diff --git a/src/transformers/trainer_optimizer.py b/src/transformers/trainer_optimizer.py index bc78622cdc29..d5acb6fd2dd6 100644 --- a/src/transformers/trainer_optimizer.py +++ b/src/transformers/trainer_optimizer.py @@ -216,7 +216,7 @@ def has_mixed_dtensor(tensors) -> bool: """Whether `tensors` do not all share one device mesh, so whole-set ops (fused/foreach kernels, `clip_grad_norm_`) cannot span them. Expert parallelism leaves the experts sharded and everything else as plain tensors; under a 2-D (fsdp, tp) mesh everything is a `DTensor`, but the experts live - on the `tp` mesh and the rest on the `fsdp` mesh.""" + on the full mesh and the rest on the `fsdp` sub-mesh.""" from torch.distributed.tensor import DTensor meshes = {t.device_mesh if isinstance(t, DTensor) else None for t in tensors} diff --git a/tests/test_fsdp_mixin.py b/tests/test_fsdp_mixin.py index ea658341ce40..6f31f2d7429e 100644 --- a/tests/test_fsdp_mixin.py +++ b/tests/test_fsdp_mixin.py @@ -563,8 +563,7 @@ def model_tester(self): """The model tester instance (e.g., CausalLMModelTester).""" ... - def _skip_if_insufficient_devices(self, world_size=None): - world_size = world_size or self.fsdp_nproc_per_node + def _skip_if_insufficient_devices(self, world_size): available_workers = _get_available_fsdp_workers() if available_workers < world_size: self.skipTest(f"Need at least {world_size} FSDP workers, have {available_workers}") From 449acd211d512dbb3aed02ae4cb86f491e778e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Thu, 3 Sep 2026 20:16:20 +0000 Subject: [PATCH 10/19] Give each device mesh its own optimizer param group instead of stepping per parameter Fused/foreach AdamW cannot span parameters on different meshes, but it can run per mesh: 189 ms/step per-parameter vs 41 ms fused per mesh group on Qwen3-30B-A3B at tp=4 x fsdp=2. --- src/transformers/trainer.py | 15 +++++++++++++-- src/transformers/trainer_optimizer.py | 6 +----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index c19f8b400c74..16bea712cbd5 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -97,6 +97,7 @@ _OPTIMIZER_HANDLERS, OptimizerContext, _parse_optim_args, + has_mixed_dtensor, is_optimizer_factory, ) from .trainer_pt_utils import ( @@ -1271,6 +1272,18 @@ def create_optimizer(self, model=None) -> torch.optim.Optimizer: "weight_decay": 0.0, }, ] + if has_mixed_dtensor(p for group in optimizer_grouped_parameters for p in group["params"]): + # Parameters on different device meshes (expert parallelism, alone or with FSDP2 on a 2-D + # mesh) cannot share one fused/foreach kernel call: give each mesh its own param group. + from torch.distributed.tensor import DTensor + + split_groups = [] + for group in optimizer_grouped_parameters: + by_mesh = defaultdict(list) + for p in group["params"]: + by_mesh[p.device_mesh if isinstance(p, DTensor) else None].append(p) + split_groups.extend({**group, "params": params} for params in by_mesh.values()) + optimizer_grouped_parameters = split_groups if self.optimizer_cls_and_kwargs is not None: optimizer_cls, optimizer_kwargs = self.optimizer_cls_and_kwargs @@ -2658,8 +2671,6 @@ def _has_mixed_mesh_grads(self, model) -> bool: # Static for the life of the run (sharding never changes after setup), so scan the # parameters only on the first call. if self._mixed_mesh_grads is None: - from .trainer_optimizer import has_mixed_dtensor - self._mixed_mesh_grads = has_mixed_dtensor(p.grad for p in model.parameters() if p.grad is not None) return self._mixed_mesh_grads diff --git a/src/transformers/trainer_optimizer.py b/src/transformers/trainer_optimizer.py index d5acb6fd2dd6..c4347a5c0d36 100644 --- a/src/transformers/trainer_optimizer.py +++ b/src/transformers/trainer_optimizer.py @@ -205,16 +205,12 @@ def _get_adamw_torch(ctx: OptimizerContext) -> tuple[Any, dict[str, Any]]: ctx.optimizer_kwargs.update(ctx.adam_kwargs) if ctx.args.optim == OptimizerNames.ADAMW_TORCH_FUSED: ctx.optimizer_kwargs.update({"fused": True}) - # Expert parallelism leaves the parameters on different meshes, so the fused and foreach kernels - # cannot span the parameter set. Step per parameter instead. - if ctx.model is not None and has_mixed_dtensor(p for p in ctx.model.parameters() if p.requires_grad): - ctx.optimizer_kwargs.update({"fused": False, "foreach": False}) return AdamW, ctx.optimizer_kwargs def has_mixed_dtensor(tensors) -> bool: """Whether `tensors` do not all share one device mesh, so whole-set ops (fused/foreach kernels, - `clip_grad_norm_`) cannot span them. Expert parallelism leaves the experts sharded and everything + `clip_grad_norm_`) cannot span them; the Trainer then groups by mesh. Expert parallelism leaves the experts sharded and everything else as plain tensors; under a 2-D (fsdp, tp) mesh everything is a `DTensor`, but the experts live on the full mesh and the rest on the `fsdp` sub-mesh.""" from torch.distributed.tensor import DTensor From 9d17bd7aacf922fae1bb634fdd9be2cd051fa4ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Thu, 3 Sep 2026 20:19:13 +0000 Subject: [PATCH 11/19] Test the 2-D mesh with a batch split across the fsdp ranks and compare gradient norms --- tests/test_fsdp_mixin.py | 61 ++++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/tests/test_fsdp_mixin.py b/tests/test_fsdp_mixin.py index 6f31f2d7429e..5863140e4f3b 100644 --- a/tests/test_fsdp_mixin.py +++ b/tests/test_fsdp_mixin.py @@ -494,9 +494,26 @@ def _test_fsdp2_plan_vs_ddp_impl(rank, config_class, config_dict, tie_word_embed logger.debug("DDP and FSDP2 comparison checks passed.") +def _grad_norm_across_meshes(model): + """Total gradient norm of parameters living on different device meshes (what the Trainer does).""" + from torch.distributed.tensor import DTensor + from torch.nn.utils import get_total_norm + + grads_by_mesh = {} + for param in model.parameters(): + if param.grad is not None: + grads_by_mesh.setdefault(param.grad.device_mesh if isinstance(param.grad, DTensor) else None, []).append( + param.grad + ) + norms = [get_total_norm(grads) for grads in grads_by_mesh.values()] + norms = [n.full_tensor() if isinstance(n, DTensor) else n for n in norms] + return torch.linalg.vector_norm(torch.stack(norms)) + + def _test_fsdp2_expert_parallel_2d_vs_ddp_impl(rank, config_class, config_dict, dtype=None): - """DDP vs a 2-D (fsdp, tp) mesh with expert parallelism on `tp`: same batch on every rank, so the - sharded run has to trace the DDP run step by step and end on the same weights.""" + """DDP vs a 2-D (fsdp, tp) mesh with expert parallelism on `tp`. DDP sees the whole batch on every + rank; each `fsdp` rank of the 2-D run sees its own slice of it, so FSDP2's reduction over `fsdp` + is exercised. Losses, gradient norms and final weights have to match step by step.""" init_test_logger() if dtype is None: @@ -504,25 +521,39 @@ def _test_fsdp2_expert_parallel_2d_vs_ddp_impl(rank, config_class, config_dict, device = _get_rank_device(rank) config = config_class.from_dict(config_dict) - batches = _build_repeated_training_batches(config, device, NUM_STEPS) + world_size = dist.get_world_size() + dp = world_size // 2 + generator = torch.Generator(device=device) + generator.manual_seed(SEED) + input_ids = torch.randint(0, config.vocab_size, (dp * BATCH_SIZE, SEQ_LEN), device=device, generator=generator) + batches = [(input_ids, input_ids.clone())] * NUM_STEPS with _deterministic_init_model_dir(rank, config, dtype) as init_model_dir: - ddp_losses, _, ddp_state_dict = train_ddp(rank, batches, LR, device, dtype, init_model_dir) + ddp_losses, ddp_grad_norms, ddp_state_dict = train_ddp(rank, batches, LR, device, dtype, init_model_dir) _set_determinism(SEED) - world_size = dist.get_world_size() model = AutoModelForCausalLM.from_pretrained( init_model_dir, torch_dtype=dtype, - distributed_config=DistributedConfig(tp_size=2, fsdp_size=world_size // 2, enable_expert_parallel=True), + distributed_config=DistributedConfig(tp_size=2, fsdp_size=dp, enable_expert_parallel=True), ) - assert model.tp_size == 2 and model.fsdp_size == world_size // 2 + assert model.tp_size == 2 and model.fsdp_size == dp assert model._device_mesh.mesh_dim_names == ("fsdp", "tp") model.train() optimizer = torch.optim.Adam(model.parameters(), lr=LR, foreach=False) - # `clip_grad_norm_` cannot span parameters living on different meshes; the Trainer has its own - # norm for that case, so only the losses and the final weights are compared here. - losses, _ = _run_training_steps(model, optimizer, batches, track_grad_norms=False) + dp_rank = model._device_mesh["fsdp"].get_local_rank() + dp_group = model._device_mesh["fsdp"].get_group() + losses, grad_norms = [], [] + for ids, labels in batches: + rows = slice(dp_rank * BATCH_SIZE, (dp_rank + 1) * BATCH_SIZE) + optimizer.zero_grad() + loss = model(input_ids=ids[rows], labels=labels[rows], use_cache=False).loss + loss.backward() + grad_norms.append(_grad_norm_across_meshes(model).item()) + optimizer.step() + loss = loss.detach() + dist.all_reduce(loss, group=dp_group) + losses.append(loss.item() / dp) state_dict = gather_full_state_dict(model) for step in range(len(ddp_losses)): @@ -533,6 +564,13 @@ def _test_fsdp2_expert_parallel_2d_vs_ddp_impl(rank, config_class, config_dict, atol=DDP_FSDP_ATOL, msg=f"Loss mismatch at step {step}: DDP={ddp_losses[step]}, FSDP2+EP={losses[step]}", ) + torch.testing.assert_close( + torch.tensor(ddp_grad_norms[step]), + torch.tensor(grad_norms[step]), + rtol=DDP_FSDP_RTOL, + atol=DDP_FSDP_ATOL, + msg=f"Grad norm mismatch at step {step}: DDP={ddp_grad_norms[step]}, FSDP2+EP={grad_norms[step]}", + ) for key in ddp_state_dict: assert key in state_dict, f"Key {key} missing from FSDP2+EP state dict" @@ -694,7 +732,8 @@ def test_fsdp2_plan_vs_ddp(self, label): @is_fsdp_test def test_fsdp2_expert_parallel_2d_vs_ddp(self): - """Training on a 2-D (fsdp, tp) mesh with expert parallelism traces DDP step by step.""" + """Training on a 2-D (fsdp, tp) mesh with expert parallelism, each fsdp rank on its own slice of + the batch, traces DDP on the whole batch step by step.""" config = self.model_tester.get_config() if getattr(config, "base_model_ep_plan", None) is None: self.skipTest("Model does not have an expert parallel plan (base_model_ep_plan)") From 6f2d9c4bf87ba5f0a1b4e715bc281217a3a8f73b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Thu, 3 Sep 2026 20:19:59 +0000 Subject: [PATCH 12/19] Refresh the throughput table --- docs/source/en/expert_parallelism.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/en/expert_parallelism.md b/docs/source/en/expert_parallelism.md index e79e8c2082f4..6e45e28bd27d 100644 --- a/docs/source/en/expert_parallelism.md +++ b/docs/source/en/expert_parallelism.md @@ -69,9 +69,9 @@ On 8 GPUs, full fine-tuning of Qwen3-30B-A3B in bf16 at sequence length 2048: | configuration | tokens/s/GPU | peak memory/GPU | |---|---|---| -| `tp_size=8` | 3023 | 40.9 GB | -| `tp_size=4, fsdp_size=2` | 2326 | 34.2 GB | -| `tp_size=2, fsdp_size=4` | 2244 | 32.3 GB | +| `tp_size=8` | 3485 | 38.6 GB | +| `tp_size=4, fsdp_size=2` | 2900 | 34.2 GB | +| `tp_size=2, fsdp_size=4` | 2830 | 32.3 GB | > [!WARNING] > Resuming from a checkpoint is not supported yet for models sharded at load time, so the [`Trainer`] only accepts `save_only_model=True` or `save_strategy="no"` for them. From 3a8da11954d9c58ce09a5b3b000620ff9143f016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Thu, 3 Sep 2026 23:33:32 +0000 Subject: [PATCH 13/19] Cosmetic: docstring layout, comment placement, doc link --- docs/source/en/expert_parallelism.md | 2 +- src/transformers/trainer.py | 10 +++++++--- src/transformers/trainer_optimizer.py | 10 ++++++---- tests/test_fsdp_mixin.py | 21 ++++++++++++--------- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/docs/source/en/expert_parallelism.md b/docs/source/en/expert_parallelism.md index 6e45e28bd27d..c62eef296119 100644 --- a/docs/source/en/expert_parallelism.md +++ b/docs/source/en/expert_parallelism.md @@ -63,7 +63,7 @@ distributed_config = DistributedConfig( model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-30B-A3B", distributed_config=distributed_config) ``` -The model is loaded on a 2-D `(fsdp, tp)` device mesh, and `tp_size * fsdp_size` must equal the number of processes. The expert parallel plan shards the experts across `tp`, then FSDP2 shards every parameter, experts included, across `fsdp` and owns their gradient reduction. Each `fsdp` rank trains on its own part of the batch. Nothing else changes: train with the [`Trainer`] as usual (it computes the gradient norm across the two meshes and steps the optimizer per parameter), and `save_model` gathers the sharded weights and writes a regular checkpoint. +The model is loaded on a 2-D `(fsdp, tp)` device mesh, and `tp_size * fsdp_size` must equal the number of processes. The expert parallel plan shards the experts across `tp`, then FSDP2 shards every parameter, experts included, across `fsdp` and owns their gradient reduction. Each `fsdp` rank trains on its own part of the batch. Nothing else changes: train with the [`Trainer`] as usual (it computes the gradient norm across the two meshes and gives each mesh its own optimizer param group), and [`~Trainer.save_model`] gathers the sharded weights and writes a regular checkpoint. On 8 GPUs, full fine-tuning of Qwen3-30B-A3B in bf16 at sequence length 2048: diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 16bea712cbd5..b08ba795bfda 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -2646,8 +2646,10 @@ def _track_num_input_tokens(self, inputs): self.state.num_input_tokens_seen += self.accelerator.gather(input_tokens).sum().item() def _mixed_mesh_grad_norm(self, model, max_norm): - """Gradient norm (and clip) when the gradients live on different device meshes, which - `clip_grad_norm_` cannot span: one norm per mesh, each already reduced over its own mesh.""" + """ + Gradient norm (and clip) when the gradients live on different device meshes, which `clip_grad_norm_` cannot + span: one norm per mesh, each already reduced over its own mesh. + """ from torch.distributed.tensor import DTensor from torch.nn.utils import clip_grads_with_norm_, get_total_norm @@ -4030,8 +4032,10 @@ def _save(self, output_dir: str | None = None, state_dict: dict | None = None) - else: self.model.save_pretrained(output_dir, state_dict=state_dict) + # A non-writer rank of a model sharded at load time is only here for the collectives above. if not self.args.should_save: - return # a non-writer rank of a model sharded at load time, only here for the collectives above + return + if self.processing_class is not None: self.processing_class.save_pretrained(output_dir) elif ( diff --git a/src/transformers/trainer_optimizer.py b/src/transformers/trainer_optimizer.py index c4347a5c0d36..13ab9edc18da 100644 --- a/src/transformers/trainer_optimizer.py +++ b/src/transformers/trainer_optimizer.py @@ -209,10 +209,12 @@ def _get_adamw_torch(ctx: OptimizerContext) -> tuple[Any, dict[str, Any]]: def has_mixed_dtensor(tensors) -> bool: - """Whether `tensors` do not all share one device mesh, so whole-set ops (fused/foreach kernels, - `clip_grad_norm_`) cannot span them; the Trainer then groups by mesh. Expert parallelism leaves the experts sharded and everything - else as plain tensors; under a 2-D (fsdp, tp) mesh everything is a `DTensor`, but the experts live - on the full mesh and the rest on the `fsdp` sub-mesh.""" + """ + Whether `tensors` do not all share one device mesh, so whole-set ops (fused/foreach kernels, `clip_grad_norm_`) + cannot span them and the Trainer groups them by mesh. Expert parallelism leaves the experts sharded and everything + else as plain tensors; under a 2-D (fsdp, tp) mesh everything is a `DTensor`, but the experts live on the full mesh + and the rest on the `fsdp` sub-mesh. + """ from torch.distributed.tensor import DTensor meshes = {t.device_mesh if isinstance(t, DTensor) else None for t in tensors} diff --git a/tests/test_fsdp_mixin.py b/tests/test_fsdp_mixin.py index 5863140e4f3b..e780b74f56c9 100644 --- a/tests/test_fsdp_mixin.py +++ b/tests/test_fsdp_mixin.py @@ -23,6 +23,7 @@ import time import traceback from abc import ABC, abstractmethod +from collections import defaultdict from contextlib import contextmanager from parameterized import parameterized @@ -499,21 +500,21 @@ def _grad_norm_across_meshes(model): from torch.distributed.tensor import DTensor from torch.nn.utils import get_total_norm - grads_by_mesh = {} + grads_by_mesh = defaultdict(list) for param in model.parameters(): if param.grad is not None: - grads_by_mesh.setdefault(param.grad.device_mesh if isinstance(param.grad, DTensor) else None, []).append( - param.grad - ) + grads_by_mesh[param.grad.device_mesh if isinstance(param.grad, DTensor) else None].append(param.grad) norms = [get_total_norm(grads) for grads in grads_by_mesh.values()] norms = [n.full_tensor() if isinstance(n, DTensor) else n for n in norms] return torch.linalg.vector_norm(torch.stack(norms)) def _test_fsdp2_expert_parallel_2d_vs_ddp_impl(rank, config_class, config_dict, dtype=None): - """DDP vs a 2-D (fsdp, tp) mesh with expert parallelism on `tp`. DDP sees the whole batch on every - rank; each `fsdp` rank of the 2-D run sees its own slice of it, so FSDP2's reduction over `fsdp` - is exercised. Losses, gradient norms and final weights have to match step by step.""" + """ + DDP vs a 2-D (fsdp, tp) mesh with expert parallelism on `tp`. DDP sees the whole batch on every rank; each `fsdp` + rank of the 2-D run sees its own slice of it, so FSDP2's reduction over `fsdp` is exercised. Losses, gradient norms + and final weights have to match step by step. + """ init_test_logger() if dtype is None: @@ -732,8 +733,10 @@ def test_fsdp2_plan_vs_ddp(self, label): @is_fsdp_test def test_fsdp2_expert_parallel_2d_vs_ddp(self): - """Training on a 2-D (fsdp, tp) mesh with expert parallelism, each fsdp rank on its own slice of - the batch, traces DDP on the whole batch step by step.""" + """ + Training on a 2-D (fsdp, tp) mesh with expert parallelism, each fsdp rank on its own slice of the batch, + traces DDP on the whole batch step by step. + """ config = self.model_tester.get_config() if getattr(config, "base_model_ep_plan", None) is None: self.skipTest("Model does not have an expert parallel plan (base_model_ep_plan)") From 75fff7166656c08b66b264247783c68fc6e5edf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:31:09 -0400 Subject: [PATCH 14/19] Update docs/source/en/expert_parallelism.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- docs/source/en/expert_parallelism.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/expert_parallelism.md b/docs/source/en/expert_parallelism.md index c62eef296119..ba2a704c8676 100644 --- a/docs/source/en/expert_parallelism.md +++ b/docs/source/en/expert_parallelism.md @@ -65,7 +65,7 @@ model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-30B-A3B", distributed_c The model is loaded on a 2-D `(fsdp, tp)` device mesh, and `tp_size * fsdp_size` must equal the number of processes. The expert parallel plan shards the experts across `tp`, then FSDP2 shards every parameter, experts included, across `fsdp` and owns their gradient reduction. Each `fsdp` rank trains on its own part of the batch. Nothing else changes: train with the [`Trainer`] as usual (it computes the gradient norm across the two meshes and gives each mesh its own optimizer param group), and [`~Trainer.save_model`] gathers the sharded weights and writes a regular checkpoint. -On 8 GPUs, full fine-tuning of Qwen3-30B-A3B in bf16 at sequence length 2048: +The table below compares EP-only training with 2D EP+FSDP2 on 8xH100 GPUs. The workload is full fine-tuning of Qwen3-30B-A3B in bf16 at sequence length 2048. More FSDP shards cut peak memory, and tokens/s drop some because FSDP2 all-gathers and reduce-scatters the experts across `fsdp`. | configuration | tokens/s/GPU | peak memory/GPU | |---|---|---| From acba6bc97a0e809178ac43aecdae5de9c0062ea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:31:17 -0400 Subject: [PATCH 15/19] Update docs/source/en/expert_parallelism.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- docs/source/en/expert_parallelism.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/en/expert_parallelism.md b/docs/source/en/expert_parallelism.md index ba2a704c8676..170ea6ba9813 100644 --- a/docs/source/en/expert_parallelism.md +++ b/docs/source/en/expert_parallelism.md @@ -55,6 +55,9 @@ torchrun --nproc-per-node 8 your_script.py Expert parallelism only shards the experts. Everything else (attention, embeddings, norms) and its optimizer state is replicated on every expert-parallel rank, which is what limits the model size you can train. Set `fsdp_size` together with `tp_size` to add [FSDP2](./fsdp) on a second mesh dimension. ```py +from transformers import AutoModelForCausalLM +from transformers.distributed import DistributedConfig + distributed_config = DistributedConfig( tp_size=4, # expert parallel size fsdp_size=2, # data parallel shards From 69668c3c29413ce5b01e8d2a2470d8f46c812795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:31:25 -0400 Subject: [PATCH 16/19] Update docs/source/en/expert_parallelism.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- docs/source/en/expert_parallelism.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/en/expert_parallelism.md b/docs/source/en/expert_parallelism.md index 170ea6ba9813..ec673f61f56a 100644 --- a/docs/source/en/expert_parallelism.md +++ b/docs/source/en/expert_parallelism.md @@ -66,7 +66,9 @@ distributed_config = DistributedConfig( model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-30B-A3B", distributed_config=distributed_config) ``` -The model is loaded on a 2-D `(fsdp, tp)` device mesh, and `tp_size * fsdp_size` must equal the number of processes. The expert parallel plan shards the experts across `tp`, then FSDP2 shards every parameter, experts included, across `fsdp` and owns their gradient reduction. Each `fsdp` rank trains on its own part of the batch. Nothing else changes: train with the [`Trainer`] as usual (it computes the gradient norm across the two meshes and gives each mesh its own optimizer param group), and [`~Trainer.save_model`] gathers the sharded weights and writes a regular checkpoint. +The model is loaded on a 2D `(fsdp, tp)` device mesh, and `tp_size * fsdp_size` must equal the number of processes. The expert parallel plan shards the experts across `tp`, then FSDP2 shards every parameter, experts included, across `fsdp` and owns their gradient reduction. Each `fsdp` rank trains on its own part of the batch. + +Load the model as usual, then train with [`Trainer`]. It takes the gradient norm across both meshes and gives each mesh its own optimizer param group. [`~Trainer.save_model`] gathers sharded weights into a regular checkpoint. This requires `accelerate>=1.12` so the `Trainer` can mirror `tp_size` and `fsdp_size` into [`~Accelerate.ParallelismConfig`]. The table below compares EP-only training with 2D EP+FSDP2 on 8xH100 GPUs. The workload is full fine-tuning of Qwen3-30B-A3B in bf16 at sequence length 2048. More FSDP shards cut peak memory, and tokens/s drop some because FSDP2 all-gathers and reduce-scatters the experts across `fsdp`. From 9cfb2d3b09035e380713b157380c1105e5bcca21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:31:42 -0400 Subject: [PATCH 17/19] Update docs/source/en/expert_parallelism.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- docs/source/en/expert_parallelism.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/expert_parallelism.md b/docs/source/en/expert_parallelism.md index ec673f61f56a..307d8f8d9ee1 100644 --- a/docs/source/en/expert_parallelism.md +++ b/docs/source/en/expert_parallelism.md @@ -52,7 +52,7 @@ torchrun --nproc-per-node 8 your_script.py ## Combining with FSDP2 -Expert parallelism only shards the experts. Everything else (attention, embeddings, norms) and its optimizer state is replicated on every expert-parallel rank, which is what limits the model size you can train. Set `fsdp_size` together with `tp_size` to add [FSDP2](./fsdp) on a second mesh dimension. +Expert parallelism only shards the experts. Everything else (attention, embeddings, norms) and its optimizer state is replicated on every expert-parallel rank, which limits how large a model you can train. Add [FSDP2](./fsdp) on a second mesh dimension with `fsdp_size`, and keep using `tp_size` for the expert parallel width (`tp_size` is the EP size). ```py from transformers import AutoModelForCausalLM From 5120748873b981d38c5f48087ed1ac0b0f8e5375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Sat, 12 Sep 2026 14:40:52 +0000 Subject: [PATCH 18/19] Drop the comment on the parallel sizes --- src/transformers/distributed/mixin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/transformers/distributed/mixin.py b/src/transformers/distributed/mixin.py index 957289aeffa9..646b8ac102cc 100644 --- a/src/transformers/distributed/mixin.py +++ b/src/transformers/distributed/mixin.py @@ -172,8 +172,6 @@ def maybe_distribute_model( if device_mesh is not None: model.config.distributed_config = distributed_config model._device_mesh = device_mesh - # The Trainer mirrors these into accelerate's `ParallelismConfig`; without them accelerate - # sees unaccounted ranks and falls back to DDP, which rejects the DTensor parameters. model._tp_size = distributed_config.tp_size model._fsdp_size = distributed_config.fsdp_size From 0147840fa50ef57164bcab05794232511a4538fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Sat, 12 Sep 2026 14:45:10 +0000 Subject: [PATCH 19/19] Trim the parallelism config comment --- src/transformers/trainer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index b08ba795bfda..a4c927f34c12 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -766,8 +766,7 @@ def _build_accelerator_args(self, **kwargs) -> dict[str, Any]: model_fsdp_size = getattr(self.model, "fsdp_size", None) or 1 if model_tp_size > 1: # Sharded at load time (tensor/expert parallelism, optionally with FSDP2 on a second mesh - # dimension): accelerate has to know both sizes, or it sees unaccounted ranks and wraps - # the DTensor model in DDP, which raises. + # dimension): accelerate has to know both sizes. if not is_accelerate_available("1.12.0"): raise ValueError("Requires accelerate>1.12.0 to use Tensor Parallelism.") if args.get("parallelism_config") is None: