Repository navigation
EP + Trainer integration on top of DistributedConfig (#45028) #46126
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -150,19 +150,37 @@ def distribute_model(model, distributed_config: DistributedConfig, device_mesh) | |
|
|
||
| @torch.no_grad() | ||
| def clip_grad_norm(parameters, max_norm: float, norm_type: float = 2.0): | ||
| """Grad-norm clip that works when params live on different DTensor meshes. | ||
|
|
||
| ``torch.nn.utils.get_total_norm`` stacks per-grad norms; that fails when grads | ||
| live on different meshes (e.g. TP-wrapped params on the (fsdp, tp) mesh and | ||
| FSDP-only params on the (fsdp,) sub-mesh). We sidestep it by replicating each | ||
| DTensor grad to a plain local tensor, computing the norm over those, and | ||
| scaling the original DTensor grads in place — the placement of the original | ||
| grads doesn't matter for the per-element clip. | ||
| """ | ||
| grads = [p.grad for p in parameters if p.grad is not None] | ||
| local_grads = [_replicate_dtensor(g).to_local() if isinstance(g, DTensor) else g for g in grads] | ||
| total_norm = torch.nn.utils.get_total_norm(local_grads, norm_type=norm_type) | ||
| torch.nn.utils.clip_grads_with_norm_(grads, max_norm=max_norm, total_norm=total_norm) | ||
| """Mesh-aware grad-norm clip with O(1) extra memory per parameter.""" | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. remove unrelated if unrelated
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Oh yes I created I different PR : #46113 for the grad clipping issue |
||
| parameters = [p for p in parameters if p.grad is not None] | ||
| if not parameters: | ||
| return torch.tensor(0.0) | ||
|
|
||
| norm_p = float(norm_type) | ||
| device = parameters[0].grad.device | ||
| accum = torch.zeros((), dtype=torch.float32, device=device) | ||
|
|
||
| for p in parameters: | ||
| g = p.grad | ||
| if isinstance(g, DTensor): | ||
| partial = g.to_local().detach().to(torch.float32).abs().pow_(norm_p).sum() | ||
| mesh = g.device_mesh | ||
| for dim_idx, placement in enumerate(g.placements): | ||
| if placement.is_replicate(): | ||
| continue | ||
| group = mesh.get_group(dim_idx) if mesh.ndim > 1 else mesh.get_group() | ||
| torch.distributed.all_reduce(partial, op=torch.distributed.ReduceOp.SUM, group=group) | ||
| accum.add_(partial) | ||
| else: | ||
| accum.add_(g.detach().to(torch.float32).abs().pow_(norm_p).sum()) | ||
|
|
||
| total_norm = accum.pow_(1.0 / norm_p) | ||
| clip_coef = (max_norm / (total_norm + 1e-6)).clamp(max=1.0) | ||
| for p in parameters: | ||
| g = p.grad | ||
| if isinstance(g, DTensor): | ||
| g._local_tensor.mul_(clip_coef) | ||
| else: | ||
| g.mul_(clip_coef) | ||
| return total_norm | ||
|
|
||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -86,9 +86,14 @@ class Qwen3MoeConfig(PreTrainedConfig): | |
| # This allows EP to scale beyond num_kv_heads (not constrained by 4 for Qwen3-30B). | ||
| base_model_ep_plan = { | ||
| "layers.*.mlp.gate": "ep_router", | ||
| "layers.*.mlp.experts.gate_up_proj": "grouped_gemm", | ||
| "layers.*.mlp.experts.down_proj": "grouped_gemm", | ||
| "layers.*.mlp.experts": "moe_experts_allreduce", | ||
|
Comment on lines
-89
to
-91
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this is what we want |
||
| # `moe_experts_ep_allreduce` shards on the experts dim (dim 0) so each rank | ||
| # holds num_experts/ep_size experts at full hidden_in. The `grouped_gemm` | ||
| # entries that used to be here are intentionally dropped: the resolver in | ||
| # `apply_tensor_parallel` only iterates modules (not parameters), so plan | ||
| # keys naming `gate_up_proj` / `down_proj` (which are nn.Parameter, not | ||
| # nn.Module) were unreachable. Sharding is now handled by `MoEExpertsParallel` | ||
| # via its `shard_plan` on the parent experts module. | ||
| "layers.*.mlp.experts": "moe_experts_ep_allreduce", | ||
| } | ||
| base_model_pp_plan = { | ||
| "embed_tokens": (["input_ids"], ["inputs_embeds"]), | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -56,6 +56,7 @@ | |
| from .data.data_collator import DataCollator, DataCollatorWithPadding, default_data_collator | ||
| from .debug_utils import DebugOption, DebugUnderflowOverflow | ||
| from .distributed.fsdp import get_fsdp_ckpt_kwargs, update_fsdp_plugin_peft | ||
| from .distributed.utils import clip_grad_norm, maybe_disable_foreach_and_fused_for_mixed_dtensor_groups | ||
| from .feature_extraction_sequence_utils import SequenceFeatureExtractor | ||
| from .feature_extraction_utils import FeatureExtractionMixin | ||
| from .hyperparameter_search import ALL_HYPERPARAMETER_SEARCH_BACKENDS, default_hp_search_backend | ||
|
|
@@ -465,6 +466,7 @@ def __init__( | |
| or (args.fp16_full_eval or args.bf16_full_eval) | ||
| or self.is_fsdp_xla_enabled | ||
| or self.is_fsdp_enabled | ||
| or self.is_native_distributed | ||
| or is_sagemaker_mp_enabled() | ||
| ): | ||
| self.place_model_on_device = False | ||
|
|
@@ -665,7 +667,7 @@ def _validate_args(self) -> None: | |
| " `Trainer`. Make sure the lines `import torch_xla.core.xla_model as xm` and" | ||
| " `model.to(xm.xla_device())` is performed before the optimizer creation in your script." | ||
| ) | ||
| if (self.is_fsdp_xla_enabled or self.is_fsdp_enabled) and ( | ||
| if (self.is_fsdp_xla_enabled or self.is_fsdp_enabled or self.is_native_distributed) and ( | ||
| self.optimizer is not None or self.lr_scheduler is not None | ||
| ): | ||
| raise RuntimeError( | ||
|
|
@@ -816,9 +818,25 @@ def create_accelerator_and_postprocess(self) -> None: | |
| self.gather_function, use_gather_object=self.args.eval_use_gather_object | ||
| ) | ||
|
|
||
| # deepspeed and accelerate flags covering both trainer args and accelerate launcher | ||
| self.is_deepspeed_enabled = getattr(self.accelerator.state, "deepspeed_plugin", None) is not None | ||
| self.is_fsdp_enabled = getattr(self.accelerator.state, "fsdp_plugin", None) is not None | ||
| # Model wrapped via `from_pretrained(distributed_config=...)` (native TP/FSDP2 path). | ||
| # When set, transformers owns distribution and accelerate's plugins (if any) are ignored. | ||
| self.is_native_distributed = getattr(self.model.config, "distributed_config", None) is not None | ||
|
|
||
| # deepspeed and accelerate flags covering both trainer args and accelerate launcher. | ||
| # Treated as exclusive with `is_native_distributed`: if transformers already manages | ||
| # distribution, the accelerate plugin (e.g. left over from `accelerate launch --use_fsdp`) | ||
| # is not the responsible party — skip its code paths to avoid operating on a model | ||
| # accelerate didn't wrap. | ||
| _ds = getattr(self.accelerator.state, "deepspeed_plugin", None) is not None | ||
| _fsdp = getattr(self.accelerator.state, "fsdp_plugin", None) is not None | ||
| if self.is_native_distributed and (_ds or _fsdp): | ||
| logger.warning_once( | ||
| "`distributed_config` is set on the model AND accelerate has a " | ||
| f"{'deepspeed' if _ds else 'fsdp'}_plugin in state. Trusting `distributed_config` " | ||
| "(transformers-native) and ignoring the accelerate plugin." | ||
| ) | ||
| self.is_deepspeed_enabled = _ds and not self.is_native_distributed | ||
| self.is_fsdp_enabled = _fsdp and not self.is_native_distributed | ||
|
Comment on lines
+825
to
+839
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. not super clean IMO but in trainer soooo
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hah, agreed on cleanliness, but let me defend this 😆. I just tried to localize variables that clearly indicate which mode we are running, so we can do stuff like |
||
|
|
||
| # post accelerator creation setup | ||
| if self.is_fsdp_enabled: | ||
|
|
@@ -835,7 +853,7 @@ def create_accelerator_and_postprocess(self) -> None: | |
| # `save_only_model` can't be used with DeepSpeed/FSDP along with `load_best_model_at_end` | ||
| if ( | ||
| self.args.save_only_model | ||
| and (self.is_deepspeed_enabled or self.is_fsdp_enabled) | ||
| and (self.is_deepspeed_enabled or self.is_fsdp_enabled or self.is_native_distributed) | ||
| and self.args.load_best_model_at_end | ||
| ): | ||
| wrapper = "DeepSpeed" if self.is_deepspeed_enabled else "FSDP" | ||
|
|
@@ -1214,6 +1232,10 @@ def create_optimizer(self, model=None) -> torch.optim.Optimizer: | |
| if is_sagemaker_mp_enabled(): | ||
| self.optimizer = smp.DistributedOptimizer(self.optimizer) | ||
|
|
||
| # Adam fused/foreach rejects mixed Tensor/DTensor groups; the helper is a | ||
| # no-op when groups are uniform. | ||
| maybe_disable_foreach_and_fused_for_mixed_dtensor_groups(self.optimizer) | ||
|
|
||
| return self.optimizer | ||
|
|
||
| def create_scheduler( | ||
|
|
@@ -1408,7 +1430,12 @@ def train( | |
| # Load model checkpoint before accelerator.prepare() for regular models, | ||
| # so that buffers and parameters are on the right device after prepare. | ||
| # Deepspeed/FSDP models are loaded after prepare in _prepare_for_training. | ||
| if not is_sagemaker_mp_enabled() and not self.is_deepspeed_enabled and not self.is_fsdp_enabled: | ||
| if ( | ||
| not is_sagemaker_mp_enabled() | ||
| and not self.is_deepspeed_enabled | ||
| and not self.is_fsdp_enabled | ||
| and not self.is_native_distributed | ||
| ): | ||
| self._load_from_checkpoint(resume_from_checkpoint) | ||
| state = TrainerState.load_from_json(os.path.join(resume_from_checkpoint, TRAINER_STATE_NAME)) | ||
| if state.train_batch_size is not None and args.auto_find_batch_size: | ||
|
|
@@ -1606,7 +1633,7 @@ def _prepare_for_training(self, max_steps, train_dataloader, resume_from_checkpo | |
| # updating self.model_wrapped | ||
| self.model_wrapped = model | ||
|
|
||
| if self.is_fsdp_enabled or self.is_fsdp_xla_enabled: | ||
| if self.is_fsdp_enabled or self.is_fsdp_xla_enabled or self.is_native_distributed: | ||
| # breaking convention for FSDP model | ||
| # TODO: check if this is really needed | ||
| self.model = self.model_wrapped = model | ||
|
|
@@ -1621,7 +1648,7 @@ def _prepare_for_training(self, max_steps, train_dataloader, resume_from_checkpo | |
| # self.model_wrapped is DDP(Transformers Model), Deepspeed(Transformers Model), | ||
| # FSDP(Transformers Model), Dynamo Optimized Module(Transformers Model) etc. | ||
|
|
||
| if self.is_fsdp_enabled: | ||
| if self.is_fsdp_enabled or self.is_native_distributed: | ||
| # Fix `got mixed torch.Tensor and DTensor` error in model.generate() for FSDP2 with LoRA | ||
| if hasattr(self.model, "generate"): | ||
| dist.fsdp.register_fsdp_forward_method(self.model, "generate") | ||
|
|
@@ -2015,7 +2042,15 @@ def compute_loss( | |
| and (self.model_accepts_loss_kwargs or self.compute_loss_func) | ||
| and num_items_in_batch is not None | ||
| ): | ||
| loss *= self.accelerator.num_processes if self.args.n_gpu <= 1 else self.args.n_gpu | ||
| # TP/EP-as-TP ranks see replicated batches — divide out the tp dim from | ||
| # whichever path is active (accelerate ParallelismConfig or DistributedConfig). | ||
| loss_scale = self.accelerator.num_processes | ||
| pc = getattr(self.accelerator, "parallelism_config", None) | ||
| if pc is not None and getattr(pc, "tp_size", 1) > 1: | ||
| loss_scale //= pc.tp_size | ||
| elif self.is_native_distributed and "tp" in (self.model.device_mesh.mesh_dim_names or ()): | ||
| loss_scale //= self.model.device_mesh["tp"].size() | ||
| loss *= loss_scale if self.args.n_gpu <= 1 else self.args.n_gpu | ||
|
|
||
| return (loss, outputs) if return_outputs else loss | ||
|
|
||
|
|
@@ -2444,7 +2479,7 @@ def _update_auto_batch_size(self, batch_size): | |
| # frees the wrapped model and resets it back to the unwrapped base model | ||
| release_memory(self.model_wrapped) | ||
|
|
||
| if self.is_fsdp_enabled: | ||
| if self.is_fsdp_enabled or self.is_native_distributed: | ||
| # Remove FSDP wrapping from sub-models because self.model points to the wrapped model in FSDP case | ||
| self.model = unwrap_model(self.model, recursive=True) | ||
|
|
||
|
|
@@ -2495,6 +2530,9 @@ def _clip_grad_norm(self, model): | |
| """Clip gradients to max_grad_norm. Returns the pre-clip gradient norm.""" | ||
| if is_sagemaker_mp_enabled() and self.args.fp16: | ||
| return self.optimizer.clip_master_grads(self.args.max_grad_norm) | ||
| # Mesh-aware streaming clip — handles cross-mesh DTensor grads and plain tensors. | ||
| if self.is_native_distributed: | ||
| return clip_grad_norm(model.parameters(), self.args.max_grad_norm) | ||
| return self.accelerator.clip_grad_norm_(model.parameters(), self.args.max_grad_norm) | ||
|
|
||
| def _get_grad_norm(self, model, grad_norm=None): | ||
|
|
@@ -3318,7 +3356,7 @@ def _load_from_checkpoint(self, resume_from_checkpoint: str, model: nn.Module | | |
| else [] | ||
| ) | ||
|
|
||
| if is_fsdp_ckpt and not self.is_fsdp_enabled: | ||
| if is_fsdp_ckpt and not (self.is_fsdp_enabled or self.is_native_distributed): | ||
| raise ValueError(f"Checkpoint found at {resume_from_checkpoint} is only supported when using PyTorch FSDP") | ||
|
|
||
| if not ( | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No no no no nonoooooo this is the worst pattern that was ever introduced in the lib, sorry you have to "pay the price" but EP should be back to how we did it:
gate_up_projgets "gemm" anddown_projas well, te module getsep_allreduce!Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
no worries 🤣 ! Ok, I was just following the exact implementation pattern:
transformers/src/transformers/distributed/tensor_parallel.py
Line 447 in a65bf6c
Just sharding on the dim 0 for the experts. If I go back to :
There is some implementation logic that uses the new
TensorParallelStylethat needs to be rewritten, I guess