Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 63 additions & 1 deletion src/transformers/distributed/tensor_parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,56 @@ def transform_output_post_forward(self, module, output, mesh):
return output.to_local()


class RouterParallel(TensorParallelStyle):
"""Hooks the routing gate's output to mask non-local expert scores under EP.

Forward: route module returns `(router_logits, router_scores, router_indices)`.
Each EP rank computes the same logits/scores/indices (gate weight is replicated),
then this hook drops the entries that route to experts owned by other ranks:

- Zero `router_scores` where `(router_indices // num_local_experts) != ep_rank`.
- Remap `router_indices` to local-expert indices via `fmod(router_indices, num_local_experts)`.
- Mark non-local slots with a sentinel = `num_local_experts` so downstream
`one_hot` / `grouped_mm` paths know to skip them.

Gate weight stays replicated (no sharding) — see `_apply` below.

Adapted from the prod `RouterParallel` at
`transformers/integrations/tensor_parallel.py::RouterParallel` (same semantics).
"""

def _apply(self, module, mesh):
# Keep the gate weight replicated on the EP mesh. The `TensorParallelStyle`
# base wraps `module.forward` to call our `transform_output_post_forward`.
return super()._apply(module, mesh)

def transform_output_post_forward(self, module, output, mesh):
ep_rank = mesh.get_local_rank()
ep_size = mesh.size()
num_experts = getattr(module, "num_experts", None)
if num_experts is None:
num_experts = getattr(getattr(module, "config", None), "num_experts", None)
if num_experts is None:
raise AttributeError(
f"Router module {type(module).__name__} is missing `num_experts` and `config.num_experts`"
)
if num_experts % ep_size != 0:
raise ValueError(f"num_experts ({num_experts}) must be divisible by ep_size ({ep_size})")
num_local_experts = num_experts // ep_size

router_logits, router_scores, router_indices = output
non_local_mask = (router_indices // num_local_experts) != ep_rank
router_scores = router_scores.masked_fill(non_local_mask, 0.0)
router_indices = router_indices.masked_fill(non_local_mask, -1)
# `-1 % 1 == 0`, so plain fmod doesn't work when num_local_experts == 1.
if num_local_experts > 1:
router_indices = torch.fmod(router_indices, num_local_experts)
else:
router_indices = router_indices.masked_fill(router_indices > 0, 0).masked_fill(router_indices < 0, -1)
router_indices = router_indices.masked_fill(router_indices == -1, num_local_experts)
return router_logits, router_scores, router_indices


class ParallelInterface(GeneralInterface):
"""Registry of named TP styles. Configs and modeling files reference these by string name.

Expand Down Expand Up @@ -451,6 +501,18 @@ class ParallelInterface(GeneralInterface):
"down_proj": Shard(-1),
},
),
# MoE — EP variant: shards on the experts dim(0) so each rank holds
# `num_experts/ep_size` experts at full hidden_in.
"moe_experts_ep_allreduce": MoEExpertsParallel(
output_layouts=Replicate(),
shard_plan={
"gate_up_proj": Shard(0),
"down_proj": Shard(0),
},
),
Comment on lines +506 to +512

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No no no no nonoooooo this is the worst pattern that was ever introduced in the lib, sorry you have to "pay the price" but EP should be back to how we did it: gate_up_proj gets "gemm" and down_proj as well, te module gets ep_allreduce!

@AmineDiro AmineDiro May 27, 2026 •

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no worries 🤣 ! Ok, I was just following the exact implementation pattern:

"moe_experts_allreduce": MoEExpertsParallel(

Just sharding on the dim 0 for the experts. If I go back to :

        "layers.*.mlp.experts.gate_up_proj": "grouped_gemm",
        "layers.*.mlp.experts.down_proj": "grouped_gemm",
        "layers.*.mlp.experts": "moe_experts_allreduce",

There is some implementation logic that uses the new TensorParallelStyle that needs to be rewritten, I guess

# EP — routing gate. Replicated weight; output hook masks non-local
# expert scores and remaps global indices to local. See `RouterParallel`.
"ep_router": RouterParallel(),
}
if is_torch_available() and is_torch_greater_or_equal("2.5") and _torch_distributed_available
else {}
Expand All @@ -477,7 +539,7 @@ def apply_tensor_parallel(model, tp_mesh, tp_plan):
if enable_sp:
tp_plan = dict(model._sp_plan or {})
else:
tp_plan = dict(model._tp_plan or {})
tp_plan = dict(model.tp_plan or {})

# tie_weights() replaces lm_head.weight with embed_tokens.weight after TP is applied.
# If embed_tokens isn't in the plan, sharding lm_head as a DTensor causes tie to
Expand Down
44 changes: 31 additions & 13 deletions src/transformers/distributed/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,19 +150,37 @@ def distribute_model(model, distributed_config: DistributedConfig, device_mesh)

@torch.no_grad()
def clip_grad_norm(parameters, max_norm: float, norm_type: float = 2.0):
"""Grad-norm clip that works when params live on different DTensor meshes.

``torch.nn.utils.get_total_norm`` stacks per-grad norms; that fails when grads
live on different meshes (e.g. TP-wrapped params on the (fsdp, tp) mesh and
FSDP-only params on the (fsdp,) sub-mesh). We sidestep it by replicating each
DTensor grad to a plain local tensor, computing the norm over those, and
scaling the original DTensor grads in place — the placement of the original
grads doesn't matter for the per-element clip.
"""
grads = [p.grad for p in parameters if p.grad is not None]
local_grads = [_replicate_dtensor(g).to_local() if isinstance(g, DTensor) else g for g in grads]
total_norm = torch.nn.utils.get_total_norm(local_grads, norm_type=norm_type)
torch.nn.utils.clip_grads_with_norm_(grads, max_norm=max_norm, total_norm=total_norm)
"""Mesh-aware grad-norm clip with O(1) extra memory per parameter."""

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove unrelated if unrelated

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh yes I created I different PR : #46113 for the grad clipping issue

parameters = [p for p in parameters if p.grad is not None]
if not parameters:
return torch.tensor(0.0)

norm_p = float(norm_type)
device = parameters[0].grad.device
accum = torch.zeros((), dtype=torch.float32, device=device)

for p in parameters:
g = p.grad
if isinstance(g, DTensor):
partial = g.to_local().detach().to(torch.float32).abs().pow_(norm_p).sum()
mesh = g.device_mesh
for dim_idx, placement in enumerate(g.placements):
if placement.is_replicate():
continue
group = mesh.get_group(dim_idx) if mesh.ndim > 1 else mesh.get_group()
torch.distributed.all_reduce(partial, op=torch.distributed.ReduceOp.SUM, group=group)
accum.add_(partial)
else:
accum.add_(g.detach().to(torch.float32).abs().pow_(norm_p).sum())

total_norm = accum.pow_(1.0 / norm_p)
clip_coef = (max_norm / (total_norm + 1e-6)).clamp(max=1.0)
for p in parameters:
g = p.grad
if isinstance(g, DTensor):
g._local_tensor.mul_(clip_coef)
else:
g.mul_(clip_coef)
return total_norm


Expand Down
11 changes: 8 additions & 3 deletions src/transformers/models/qwen3_moe/configuration_qwen3_moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,14 @@ class Qwen3MoeConfig(PreTrainedConfig):
# This allows EP to scale beyond num_kv_heads (not constrained by 4 for Qwen3-30B).
base_model_ep_plan = {
"layers.*.mlp.gate": "ep_router",
"layers.*.mlp.experts.gate_up_proj": "grouped_gemm",
"layers.*.mlp.experts.down_proj": "grouped_gemm",
"layers.*.mlp.experts": "moe_experts_allreduce",
Comment on lines -89 to -91

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is what we want

# `moe_experts_ep_allreduce` shards on the experts dim (dim 0) so each rank
# holds num_experts/ep_size experts at full hidden_in. The `grouped_gemm`
# entries that used to be here are intentionally dropped: the resolver in
# `apply_tensor_parallel` only iterates modules (not parameters), so plan
# keys naming `gate_up_proj` / `down_proj` (which are nn.Parameter, not
# nn.Module) were unreachable. Sharding is now handled by `MoEExpertsParallel`
# via its `shard_plan` on the parent experts module.
"layers.*.mlp.experts": "moe_experts_ep_allreduce",
}
base_model_pp_plan = {
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
Expand Down
60 changes: 49 additions & 11 deletions src/transformers/trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
from .data.data_collator import DataCollator, DataCollatorWithPadding, default_data_collator
from .debug_utils import DebugOption, DebugUnderflowOverflow
from .distributed.fsdp import get_fsdp_ckpt_kwargs, update_fsdp_plugin_peft
from .distributed.utils import clip_grad_norm, maybe_disable_foreach_and_fused_for_mixed_dtensor_groups
from .feature_extraction_sequence_utils import SequenceFeatureExtractor
from .feature_extraction_utils import FeatureExtractionMixin
from .hyperparameter_search import ALL_HYPERPARAMETER_SEARCH_BACKENDS, default_hp_search_backend
Expand Down Expand Up @@ -465,6 +466,7 @@ def __init__(
or (args.fp16_full_eval or args.bf16_full_eval)
or self.is_fsdp_xla_enabled
or self.is_fsdp_enabled
or self.is_native_distributed
or is_sagemaker_mp_enabled()
):
self.place_model_on_device = False
Expand Down Expand Up @@ -665,7 +667,7 @@ def _validate_args(self) -> None:
" `Trainer`. Make sure the lines `import torch_xla.core.xla_model as xm` and"
" `model.to(xm.xla_device())` is performed before the optimizer creation in your script."
)
if (self.is_fsdp_xla_enabled or self.is_fsdp_enabled) and (
if (self.is_fsdp_xla_enabled or self.is_fsdp_enabled or self.is_native_distributed) and (
self.optimizer is not None or self.lr_scheduler is not None
):
raise RuntimeError(
Expand Down Expand Up @@ -816,9 +818,25 @@ def create_accelerator_and_postprocess(self) -> None:
self.gather_function, use_gather_object=self.args.eval_use_gather_object
)

# deepspeed and accelerate flags covering both trainer args and accelerate launcher
self.is_deepspeed_enabled = getattr(self.accelerator.state, "deepspeed_plugin", None) is not None
self.is_fsdp_enabled = getattr(self.accelerator.state, "fsdp_plugin", None) is not None
# Model wrapped via `from_pretrained(distributed_config=...)` (native TP/FSDP2 path).
# When set, transformers owns distribution and accelerate's plugins (if any) are ignored.
self.is_native_distributed = getattr(self.model.config, "distributed_config", None) is not None

# deepspeed and accelerate flags covering both trainer args and accelerate launcher.
# Treated as exclusive with `is_native_distributed`: if transformers already manages
# distribution, the accelerate plugin (e.g. left over from `accelerate launch --use_fsdp`)
# is not the responsible party — skip its code paths to avoid operating on a model
# accelerate didn't wrap.
_ds = getattr(self.accelerator.state, "deepspeed_plugin", None) is not None
_fsdp = getattr(self.accelerator.state, "fsdp_plugin", None) is not None
if self.is_native_distributed and (_ds or _fsdp):
logger.warning_once(
"`distributed_config` is set on the model AND accelerate has a "
f"{'deepspeed' if _ds else 'fsdp'}_plugin in state. Trusting `distributed_config` "
"(transformers-native) and ignoring the accelerate plugin."
)
self.is_deepspeed_enabled = _ds and not self.is_native_distributed
self.is_fsdp_enabled = _fsdp and not self.is_native_distributed
Comment on lines +825 to +839

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not super clean IMO but in trainer soooo

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hah, agreed on cleanliness, but let me defend this 😆. I just tried to localize variables that clearly indicate which mode we are running, so we can do stuff like self.is_deepspeed_enabled or self.is_fsdp_enabled or self.is_native_distributed. Because we can be running in fsdp from accelerate of Zero3 from native and maybe have passed the DistributedConfig, we just want to avoid overlap and be explicit in the condition.
But happy to change this if you have a better idea!


# post accelerator creation setup
if self.is_fsdp_enabled:
Expand All @@ -835,7 +853,7 @@ def create_accelerator_and_postprocess(self) -> None:
# `save_only_model` can't be used with DeepSpeed/FSDP along with `load_best_model_at_end`
if (
self.args.save_only_model
and (self.is_deepspeed_enabled or self.is_fsdp_enabled)
and (self.is_deepspeed_enabled or self.is_fsdp_enabled or self.is_native_distributed)
and self.args.load_best_model_at_end
):
wrapper = "DeepSpeed" if self.is_deepspeed_enabled else "FSDP"
Expand Down Expand Up @@ -1214,6 +1232,10 @@ def create_optimizer(self, model=None) -> torch.optim.Optimizer:
if is_sagemaker_mp_enabled():
self.optimizer = smp.DistributedOptimizer(self.optimizer)

# Adam fused/foreach rejects mixed Tensor/DTensor groups; the helper is a
# no-op when groups are uniform.
maybe_disable_foreach_and_fused_for_mixed_dtensor_groups(self.optimizer)

return self.optimizer

def create_scheduler(
Expand Down Expand Up @@ -1408,7 +1430,12 @@ def train(
# Load model checkpoint before accelerator.prepare() for regular models,
# so that buffers and parameters are on the right device after prepare.
# Deepspeed/FSDP models are loaded after prepare in _prepare_for_training.
if not is_sagemaker_mp_enabled() and not self.is_deepspeed_enabled and not self.is_fsdp_enabled:
if (
not is_sagemaker_mp_enabled()
and not self.is_deepspeed_enabled
and not self.is_fsdp_enabled
and not self.is_native_distributed
):
self._load_from_checkpoint(resume_from_checkpoint)
state = TrainerState.load_from_json(os.path.join(resume_from_checkpoint, TRAINER_STATE_NAME))
if state.train_batch_size is not None and args.auto_find_batch_size:
Expand Down Expand Up @@ -1606,7 +1633,7 @@ def _prepare_for_training(self, max_steps, train_dataloader, resume_from_checkpo
# updating self.model_wrapped
self.model_wrapped = model

if self.is_fsdp_enabled or self.is_fsdp_xla_enabled:
if self.is_fsdp_enabled or self.is_fsdp_xla_enabled or self.is_native_distributed:
# breaking convention for FSDP model
# TODO: check if this is really needed
self.model = self.model_wrapped = model
Expand All @@ -1621,7 +1648,7 @@ def _prepare_for_training(self, max_steps, train_dataloader, resume_from_checkpo
# self.model_wrapped is DDP(Transformers Model), Deepspeed(Transformers Model),
# FSDP(Transformers Model), Dynamo Optimized Module(Transformers Model) etc.

if self.is_fsdp_enabled:
if self.is_fsdp_enabled or self.is_native_distributed:
# Fix `got mixed torch.Tensor and DTensor` error in model.generate() for FSDP2 with LoRA
if hasattr(self.model, "generate"):
dist.fsdp.register_fsdp_forward_method(self.model, "generate")
Expand Down Expand Up @@ -2015,7 +2042,15 @@ def compute_loss(
and (self.model_accepts_loss_kwargs or self.compute_loss_func)
and num_items_in_batch is not None
):
loss *= self.accelerator.num_processes if self.args.n_gpu <= 1 else self.args.n_gpu
# TP/EP-as-TP ranks see replicated batches — divide out the tp dim from
# whichever path is active (accelerate ParallelismConfig or DistributedConfig).
loss_scale = self.accelerator.num_processes
pc = getattr(self.accelerator, "parallelism_config", None)
if pc is not None and getattr(pc, "tp_size", 1) > 1:
loss_scale //= pc.tp_size
elif self.is_native_distributed and "tp" in (self.model.device_mesh.mesh_dim_names or ()):
loss_scale //= self.model.device_mesh["tp"].size()
loss *= loss_scale if self.args.n_gpu <= 1 else self.args.n_gpu

return (loss, outputs) if return_outputs else loss

Expand Down Expand Up @@ -2444,7 +2479,7 @@ def _update_auto_batch_size(self, batch_size):
# frees the wrapped model and resets it back to the unwrapped base model
release_memory(self.model_wrapped)

if self.is_fsdp_enabled:
if self.is_fsdp_enabled or self.is_native_distributed:
# Remove FSDP wrapping from sub-models because self.model points to the wrapped model in FSDP case
self.model = unwrap_model(self.model, recursive=True)

Expand Down Expand Up @@ -2495,6 +2530,9 @@ def _clip_grad_norm(self, model):
"""Clip gradients to max_grad_norm. Returns the pre-clip gradient norm."""
if is_sagemaker_mp_enabled() and self.args.fp16:
return self.optimizer.clip_master_grads(self.args.max_grad_norm)
# Mesh-aware streaming clip — handles cross-mesh DTensor grads and plain tensors.
if self.is_native_distributed:
return clip_grad_norm(model.parameters(), self.args.max_grad_norm)
return self.accelerator.clip_grad_norm_(model.parameters(), self.args.max_grad_norm)

def _get_grad_norm(self, model, grad_norm=None):
Expand Down Expand Up @@ -3318,7 +3356,7 @@ def _load_from_checkpoint(self, resume_from_checkpoint: str, model: nn.Module |
else []
)

if is_fsdp_ckpt and not self.is_fsdp_enabled:
if is_fsdp_ckpt and not (self.is_fsdp_enabled or self.is_native_distributed):
raise ValueError(f"Checkpoint found at {resume_from_checkpoint} is only supported when using PyTorch FSDP")

if not (
Expand Down
Loading