Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion src/megatron/bridge/peft/dora.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from megatron.bridge.peft.base import PEFT
from megatron.bridge.peft.dora_layers import DoRALinear, ParallelLinearDoRAAdapter
from megatron.bridge.peft.module_matcher import ModuleMatcher
from megatron.bridge.peft.utils import get_adapter_attributes_from_linear
from megatron.bridge.peft.utils import get_adapter_attributes_from_linear, is_expert_linear


logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -90,7 +90,22 @@ def transform(self, m: nn.Module, name: Optional[str] = None, prefix: Optional[s

if (ans := self.match(m, name, prefix)) is not None:
_, full_name = ans
if is_expert_linear(full_name):
raise NotImplementedError(
f"DoRA does not support MoE expert linears (matched {full_name}): the DoRA "
"adapter is built without is_expert, so its sharding and collectives do not "
"compose with expert parallelism (TP, ETP, or EP > 1). Exclude expert "
"modules from target_modules to proceed."
)
attrs = get_adapter_attributes_from_linear(m)
if attrs.disable_tensor_parallel_comm and attrs.base_linear_is_parallel:
raise NotImplementedError(
f"DoRA does not support TP-sharded linears whose own tensor-parallel "
f"communication is suppressed (matched {full_name}; e.g. shared experts "
"under moe_shared_expert_overlap): DoRA's per-rank magnitude norm does not "
"compose with the downstream TP reduction. Exclude the module from "
"target_modules. A replicated base (e.g. a duplicated TELinear) is fine."
)
logger.info(f"Adding DoRA to: {full_name}")
adapter = ParallelLinearDoRAAdapter(
attrs.in_features,
Expand Down
13 changes: 13 additions & 0 deletions src/megatron/bridge/peft/lora_layers.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,19 @@ class to provide a specific implementation of the forward method.
@property
def weight(self) -> torch.Tensor:
"""Return the effective base weight, including the LoRA delta when enabled."""
from megatron.bridge.peft.utils import GroupedExpertLinearAdapter, SharedOuterGroupedExpertAdapter

if isinstance(self.adapter, (GroupedExpertLinearAdapter, SharedOuterGroupedExpertAdapter)):
# Grouped bases expose weight0..N (no single 2D to_wrap.weight), and these
# adapters carry 3D per-expert weights with no `tp_group` attribute — the
# group-less merge below would silently merge only the local shard. The
# merge path for grouped adapters is the export-side materializer
# (models/conversion/peft_bridge.py), not this property.
raise NotImplementedError(
f"LoRALinear.weight is not supported for grouped expert adapters "
f"({type(self.adapter).__name__} on {self.adapter.base_linear_name}); "
"use the checkpoint/export merge path instead."
)
base_weight = self.to_wrap.weight
if not self._adapter_enabled:
return base_weight
Expand Down
32 changes: 32 additions & 0 deletions src/megatron/bridge/peft/multi_lora_layers.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,18 @@ def __init__(
# column-sharded adapter output must be gathered before the residual
# add; a column-parallel base keeps the [tokens, out/tp] shard.
self._gather_output = attrs.input_is_parallel or not attrs.base_linear_is_parallel
# Shared-expert fc2 under moe_shared_expert_overlap: the base's own TP
# collectives are suppressed (attrs.disable_tensor_parallel_comm) and
# SharedExpertMLP.post_forward_comm sums the full-width partials across
# the dense TP group. A gathered delta would be counted once per rank;
# zero-embedding this rank's shard instead gives each output element
# exactly one non-zero contributor, so the downstream sum counts it once.
# Mirrors ParallelLinearAdapter._shared_expert_row_parallel.
self._suppressed_comm_row_parallel = bool(
attrs.input_is_parallel and attrs.disable_tensor_parallel_comm and attrs.base_linear_is_parallel
)
if self._suppressed_comm_row_parallel:
self._gather_output = False

# ModuleList of ParallelLinearAdapters gives per-adapter optimizer state
# isolation, clean checkpoint serialization, and bridge export compatibility.
Expand Down Expand Up @@ -155,6 +167,21 @@ def __init__(
"rank_values", torch.full((n_adapters,), dim, dtype=dtype, device=device), persistent=False
)

def _embed_row_parallel_shard(self, out: torch.Tensor) -> torch.Tensor:
"""Zero-embed this rank's output shard at its dense-TP offset.

Padding is differentiable and its adjoint is the matching slice, so the
downstream ``post_forward_comm`` sum reproduces the delta exactly once with
no ``1/tp`` factor and no gradient compensation.
"""
tp_size = parallel_state.get_tensor_model_parallel_world_size()
if tp_size <= 1:
return out
rank = parallel_state.get_tensor_model_parallel_rank()
shard_width = out.shape[-1]
left = rank * shard_width
return nn.functional.pad(out, (left, (tp_size - 1) * shard_width - left))

def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
linear_output, bias, layernorm_output = self.base_linear_forward(x, *args, **kwargs)

Expand Down Expand Up @@ -226,8 +253,13 @@ def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> Tuple[torch.Ten
# Match the wrapped base linear's output layout: row-parallel base
# produces a fully-summed [tokens, h_out] tensor (which we then SP
# scatter); column-parallel base keeps the [tokens, h_out/tp] shard.
# A suppressed-comm row-parallel base (shared experts under overlap)
# emits a full-width partial that post_forward_comm sums across TP, so
# the delta must be contributed as a zero-embedded local shard instead.
if self._gather_output:
out = gather_from_tensor_model_parallel_region(out)
elif self._suppressed_comm_row_parallel:
out = self._embed_row_parallel_shard(out)

if not self.disable_sequence_parallel_comm and self.input_is_parallel:
if self.use_a2a:
Expand Down
Loading