From e0b1a2723725dda35145600baad4412c84fc754c Mon Sep 17 00:00:00 2001 From: Tigist Diriba Date: Tue, 18 Aug 2026 11:48:54 +0300 Subject: [PATCH 1/2] test(peft): demonstrate expert-LoRA ETP>1 exactness defects (fail-first) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expert LoRA computes wrong results whenever Megatron suppresses a linear's own tensor-parallel collectives: at expert_tensor_parallel_size > 1 the default adapter's fc2 delta is built from unsummed partials and over-counted by the dispatcher's expert-TP reduce, its fc1 dL/dA silently loses every cross-rank term under an exact forward, the grouped per-expert adapter's bare all_gather severs autograd entirely, and shared-expert fc2 deltas under moe_shared_expert_overlap (plain TP > 1) are counted once per rank — same defect in MultiLoRALinear. Three test layers, each defect demonstrated as xfail(strict=True) so this commit lands green while recording the failures: torch-only closed-form algebra with negative controls; a CPU suite driving the real adapters through real MCore explicit_expert_comm suppression (one-rank gloo init plus mock group sizes; only the collectives are faked, and each fake asserts it received exactly the tensor the real collective would see); and a two-rank TP=2 x ETP=2 real-NCCL suite with its L0 launch script. Verified on 2x H100: every demonstration fails on its intended assertion against this tree (5 passed / 8 xfailed CPU; 4 passed / 7 xfailed distributed). The algebra files and the defect analysis originate from Mohammad Taufeeque's earlier investigation; the files are ported verbatim apart from docstrings. Co-authored-by: Taufeeque <9taufeeque9@gmail.com> Co-Authored-By: Claude Fable 5 Signed-off-by: Tigist Diriba --- .../L0_Launch_peft_expert_etp_distributed.sh | 30 + tests/unit_tests/peft/test_dora.py | 96 ++- ...olumn_parallel_adapter_backward_algebra.py | 97 +++ tests/unit_tests/peft/test_expert_lora_etp.py | 637 ++++++++++++++++++ .../peft/test_expert_lora_etp_distributed.py | 525 +++++++++++++++ ...est_expert_row_parallel_adapter_algebra.py | 107 +++ 6 files changed, 1491 insertions(+), 1 deletion(-) create mode 100755 tests/functional_tests/launch_scripts/h100/active/L0_Launch_peft_expert_etp_distributed.sh create mode 100644 tests/unit_tests/peft/test_expert_column_parallel_adapter_backward_algebra.py create mode 100644 tests/unit_tests/peft/test_expert_lora_etp.py create mode 100644 tests/unit_tests/peft/test_expert_lora_etp_distributed.py create mode 100644 tests/unit_tests/peft/test_expert_row_parallel_adapter_algebra.py diff --git a/tests/functional_tests/launch_scripts/h100/active/L0_Launch_peft_expert_etp_distributed.sh b/tests/functional_tests/launch_scripts/h100/active/L0_Launch_peft_expert_etp_distributed.sh new file mode 100755 index 0000000000..e728547c74 --- /dev/null +++ b/tests/functional_tests/launch_scripts/h100/active/L0_Launch_peft_expert_etp_distributed.sh @@ -0,0 +1,30 @@ +# CI_TIMEOUT=15 +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +set -xeuo pipefail + +REPO_ROOT=$(cd "$(dirname "$0")/../../../../.." && pwd) + +export CUDA_VISIBLE_DEVICES="0,1" + +cd "${REPO_ROOT}" + +uv run python -m torch.distributed.run --nproc_per_node=2 --nnodes=1 -m coverage run \ + --data-file="${REPO_ROOT}/.coverage" --source="${REPO_ROOT}" --parallel-mode \ + -m pytest -o log_cli=true -o log_cli_level=INFO -v -s -x -m "not pleasefixme" --tb=short -rA \ + tests/unit_tests/peft/test_expert_lora_etp_distributed.py + +coverage combine -q diff --git a/tests/unit_tests/peft/test_dora.py b/tests/unit_tests/peft/test_dora.py index 4faf5a8348..ccc4dfc0fb 100644 --- a/tests/unit_tests/peft/test_dora.py +++ b/tests/unit_tests/peft/test_dora.py @@ -33,7 +33,7 @@ from megatron.bridge.peft.dora import DoRA from megatron.bridge.peft.dora_layers import DoRALinear, ParallelLinearDoRAAdapter from megatron.bridge.peft.utils import AdapterAttributes -from tests.unit_tests.peft.test_utils import MockModelParallelConfig +from tests.unit_tests.peft.test_utils import MockModelParallelConfig, MockRowParallelLinear class SimpleModel(nn.Module): @@ -397,6 +397,100 @@ def test_exclude_modules_functionality(self): # This functionality should be available through ModuleMatcher assert hasattr(dora, "exclude_modules") + @staticmethod + def _fake_dora_linear_cls(): + """A real class stand-in for DoRALinear. + + ``DoRA.transform``'s first statement is ``isinstance(m, DoRALinear)``, and + ``isinstance`` against a Mock raises TypeError — so the patch must be a genuine + class. A real ``DoRALinear`` would compute weight norms from the (mocked) + adapter at init, hence the stand-in. + """ + + class _FakeDoRALinear(nn.Module): + def __init__(self, to_wrap, adapter): + super().__init__() + self.to_wrap = to_wrap + self.adapter = adapter + + return _FakeDoRALinear + + @pytest.mark.xfail( + strict=True, + reason="dora-guard: transform silently builds a dense-group, is_expert=False adapter on an " + "MoE expert linear — its sharding and collectives do not compose with expert " + "parallelism at TP, ETP, or EP > 1", + ) + def test_dora_refuses_expert_linears(self): + """DoRA must refuse expert linears loudly instead of mis-building an adapter.""" + # Demonstrates the missing guard fail-first; the strict marker is deleted by the fix change. + module = MockRowParallelLinear(16, 16) + module.config = MockModelParallelConfig() + dora = DoRA(dim=4, alpha=8) + + with ( + patch("megatron.bridge.peft.dora.ParallelLinearDoRAAdapter"), + patch("megatron.bridge.peft.dora.DoRALinear", self._fake_dora_linear_cls()), + ): + with pytest.raises(NotImplementedError, match="expert"): + dora.transform(module, name="linear_fc2", prefix="decoder.layers.0.mlp.experts") + + @pytest.mark.xfail( + strict=True, + reason="dora-guard: a TP-sharded base with suppressed tensor-parallel communication " + "(shared experts under moe_shared_expert_overlap) reaches DoRA unguarded — its " + "per-rank magnitude norm does not compose with the downstream TP reduction", + ) + def test_dora_refuses_suppressed_comm_parallel_bases(self): + """DoRA must refuse suppressed-comm TP-sharded bases (e.g. shared-expert overlap).""" + # Demonstrates the missing guard fail-first; the strict marker is deleted by the fix change. + module = MockRowParallelLinear(16, 16) + module.config = MockModelParallelConfig() + dora = DoRA(dim=4, alpha=8) + attrs = AdapterAttributes( + input_is_parallel=True, + in_features=16, + out_features=16, + disable_tensor_parallel_comm=True, + disable_sequence_parallel_comm=True, + base_linear_is_parallel=True, + ) + + with ( + patch("megatron.bridge.peft.dora.get_adapter_attributes_from_linear", return_value=attrs), + patch("megatron.bridge.peft.dora.ParallelLinearDoRAAdapter"), + patch("megatron.bridge.peft.dora.DoRALinear", self._fake_dora_linear_cls()), + ): + with pytest.raises(NotImplementedError, match="suppressed"): + dora.transform(module, name="linear_fc2", prefix="decoder.layers.0.mlp.shared_experts") + + def test_dora_still_transforms_replicated_suppressed_comm_bases(self): + """Scoping pin: a replicated base (base_linear_is_parallel=False) stays transformable. + + Suppressed-comm refusal must not extend to duplicated TELinear bases — there is no + shard to mis-count. Passes today and must keep passing after the refusals land. + """ + module = MockRowParallelLinear(16, 16) + module.config = MockModelParallelConfig() + dora = DoRA(dim=4, alpha=8) + attrs = AdapterAttributes( + input_is_parallel=True, + in_features=16, + out_features=16, + disable_tensor_parallel_comm=True, + disable_sequence_parallel_comm=True, + base_linear_is_parallel=False, + ) + fake_cls = self._fake_dora_linear_cls() + + with ( + patch("megatron.bridge.peft.dora.get_adapter_attributes_from_linear", return_value=attrs), + patch("megatron.bridge.peft.dora.ParallelLinearDoRAAdapter"), + patch("megatron.bridge.peft.dora.DoRALinear", fake_cls), + ): + wrapped = dora.transform(module, name="linear_fc2", prefix="decoder.layers.0.mlp.shared_experts") + assert isinstance(wrapped, fake_cls), "replicated suppressed-comm base must still transform" + def test_dora_transform_idempotent(self): """Test that DoRA transform is idempotent (applying twice has same effect as applying once).""" test_module = nn.Linear(512, 256) diff --git a/tests/unit_tests/peft/test_expert_column_parallel_adapter_backward_algebra.py b/tests/unit_tests/peft/test_expert_column_parallel_adapter_backward_algebra.py new file mode 100644 index 0000000000..7ef95a00f5 --- /dev/null +++ b/tests/unit_tests/peft/test_expert_column_parallel_adapter_backward_algebra.py @@ -0,0 +1,97 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Closed-form check of the expert COLUMN-parallel adapter (experts.linear_fc1) at ETP > 1. + +Its forward is exact and its `dL/dB` is already correct, so of the quantities modeled here +only `dL/dA` can tell the fixed path from the broken one -- which is why a forward-only check +calls the broken version green. (`dL/dx` also discriminates, but it flows through the +dispatcher and is out of scope for this file.) + +`A` is sharded over the low-rank dim and gathered to a full `z`; `B` is sharded over the output +and consumes that full `z`, so the true `dL/dz` sums over every rank. `explicit_expert_comm` +forces `allreduce_dgrad=False` on the adapter's `linear_out`, so without a compensating +backward all-reduce rank r sees only its own term. Production wiring lives in +test_expert_lora_etp.py; real collectives in test_expert_lora_etp_distributed.py. + +The reference comes from autograd on the merged-weight formulation, independently of how the +sharded arms are assembled -- otherwise the fixed arm would be compared against its own +definition and could not fail. +""" + +import torch + + +T, IN, DIM, OUT, ETP = 7, 12, 8, 16, 4 +DIM_SHARD, OUT_SHARD = DIM // ETP, OUT // ETP +TOL = 1e-12 + + +def _fixtures(): + x = torch.randn(T, IN, dtype=torch.float64, generator=torch.Generator().manual_seed(0)) + a = torch.randn(DIM, IN, dtype=torch.float64, generator=torch.Generator().manual_seed(1)) + b = torch.randn(OUT, DIM, dtype=torch.float64, generator=torch.Generator().manual_seed(2)) + g = [ + torch.randn(T, OUT_SHARD, dtype=torch.float64, generator=torch.Generator().manual_seed(10 + r)) + for r in range(ETP) + ] + return x, a, b, g + + +def _reference_grads(x, a0, b0, g): + a, b = a0.clone().requires_grad_(True), b0.clone().requires_grad_(True) + outs = [x @ (b[r * OUT_SHARD : (r + 1) * OUT_SHARD, :] @ a).t() for r in range(ETP)] + torch.autograd.backward(outs, g) + return a.grad.clone(), b.grad.clone() + + +def _da_from(per_rank_dz, x): + """`dL/dA_r` is built from rank r's slice of whatever `dL/dz` that rank holds.""" + return torch.cat([per_rank_dz[r][:, r * DIM_SHARD : (r + 1) * DIM_SHARD].t() @ x for r in range(ETP)], dim=0) + + +def _local_dz(a0, b0, g): + shards = [b0[r * OUT_SHARD : (r + 1) * OUT_SHARD, :] for r in range(ETP)] + return [g[r] @ shards[r] for r in range(ETP)] + + +def test_dL_dA_matches_merged_weight_after_the_backward_all_reduce(): + x, a, b, g = _fixtures() + ref_da, _ = _reference_grads(x, a, b, g) + summed = sum(_local_dz(a, b, g)) + got = _da_from([summed] * ETP, x) + assert (got - ref_da).abs().max() / ref_da.abs().max() < TOL + + +def test_dL_dB_was_already_correct(): + x, a, b, g = _fixtures() + _, ref_db = _reference_grads(x, a, b, g) + z = x @ a.t() + got = torch.cat([g[r].t() @ z for r in range(ETP)], dim=0) + assert (got - ref_db).abs().max() / ref_db.abs().max() < TOL + + +def test_without_the_all_reduce_dL_dA_loses_the_cross_rank_terms(): + """Negative control: the defect this file exists to catch. + + ``_local_dz`` is the fc1 backward ``ParallelLinearAdapter(is_expert=True)`` ships at + ETP > 1 (``allreduce_dgrad`` suppressed under ``explicit_expert_comm``): a green forward + with every cross-rank ``dL/dA`` term missing. It also subsumes + ``GroupedExpertLinearAdapter``'s fc1 defect, whose severed autograd yields ``dL/dA = 0`` + -- a rel-err > 0.1 trivially. + """ + x, a, b, g = _fixtures() + ref_da, _ = _reference_grads(x, a, b, g) + got = _da_from(_local_dz(a, b, g), x) + assert (got - ref_da).abs().max() / ref_da.abs().max() > 0.1 diff --git a/tests/unit_tests/peft/test_expert_lora_etp.py b/tests/unit_tests/peft/test_expert_lora_etp.py new file mode 100644 index 0000000000..c73be0d215 --- /dev/null +++ b/tests/unit_tests/peft/test_expert_lora_etp.py @@ -0,0 +1,637 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fail-first wiring and exactness tests for expert LoRA at expert-TP > 1. + +Each defect-demonstrating test carries ``@pytest.mark.xfail(strict=True, reason=": ...")`` +and fails against the current adapter code — the marker is the fail-first record. The fix +commit deletes the markers; ``strict=True`` turns the resulting XPASS into a suite failure, so +the fix cannot merge with any demonstration left unfixed. Behavior pins are plain tests. + +The harness runs the real shipped adapters through the real MCore suppression logic on CPU: a +one-rank gloo init plus a mock process group reporting ``size() == 2`` makes MCore's +``get_pg_size`` (megatron/core/utils.py) return 2, so ``ColumnParallelLinear`` / +``RowParallelLinear`` construct expert-TP-sharded weights with ``explicit_expert_comm=True`` +and the defective forward/backward paths execute unpatched. Only the collectives are replaced, +by mathematically faithful fakes that assert their own inputs (the closed-form ground truth for +these fakes is tests/unit_tests/peft/test_expert_row_parallel_adapter_algebra.py and +test_expert_column_parallel_adapter_backward_algebra.py). Real collectives are covered by +tests/unit_tests/peft/test_expert_lora_etp_distributed.py on 2 GPUs. + +The fix's completion ops live in ``megatron.bridge.peft.utils`` only after the fix commit; +``_patch_fix_ops_if_present`` patches them when they exist so the same test bodies hold in both +worlds (pre-fix: the names are absent and the defective path runs; post-fix: the fakes model +the new collectives faithfully, including their backward semantics). +""" + +import contextlib +import datetime +import os +from unittest.mock import patch + +import pytest +import torch +import torch.distributed as dist +from megatron.core.model_parallel_config import ModelParallelConfig + +import megatron.bridge.peft.utils as peft_utils +from megatron.bridge.peft.utils import GroupedExpertLinearAdapter, ParallelLinearAdapter +from tests.unit_tests.peft.test_utils import MockModelParallelConfig, make_mock_pg_collection + + +ETP = 2 +TOKENS, IN, DIM, OUT = 6, 8, 4, 8 +IN_SHARD, DIM_SHARD, OUT_SHARD = IN // ETP, DIM // ETP, OUT // ETP +TOL = 1e-10 + +# Patch site for the collectives MCore's linear layers call (name-imports in that module). +_MCORE_LAYERS = "megatron.core.tensor_parallel.layers" + + +@pytest.fixture(scope="module", autouse=True) +def _gloo_single_rank(): + """One-rank gloo backend so MCore reads mock process-group sizes instead of 1.""" + + created = not dist.is_initialized() + if created: + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", "29512") + dist.init_process_group(backend="gloo", world_size=1, rank=0, timeout=datetime.timedelta(minutes=5)) + yield + if created and dist.is_initialized(): + dist.destroy_process_group() + + +def _etp2_config() -> ModelParallelConfig: + """Real config: fp64 for closed-form tolerances, CPU init, simulated ETP=2.""" + + return ModelParallelConfig( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + expert_model_parallel_size=1, + expert_tensor_parallel_size=ETP, + sequence_parallel=False, + params_dtype=torch.float64, + use_cpu_initialization=True, + gradient_accumulation_fusion=False, + ) + + +def _tp2_config() -> ModelParallelConfig: + """Real config for the dense-TP=2 shared-expert-overlap flavor.""" + + return ModelParallelConfig( + tensor_model_parallel_size=ETP, + pipeline_model_parallel_size=1, + expert_model_parallel_size=1, + expert_tensor_parallel_size=1, + sequence_parallel=False, + params_dtype=torch.float64, + use_cpu_initialization=True, + gradient_accumulation_fusion=False, + ) + + +def _seeded(shape, seed): + return torch.randn(*shape, dtype=torch.float64, generator=torch.Generator().manual_seed(seed)) + + +def _fake_gather(rank, slots): + """Faithful model of MCore's forward all-gather (backward = own-rank slice, no sum). + + ``torch.cat`` over constants with the live tensor at this rank's slot reproduces both + directions of ``_GatherFromModelParallelRegion``: forward concatenation, and a backward + that routes only the own-rank slice of the incoming gradient — no cross-rank sum. + """ + + def fake(t, group=None): + assert torch.allclose(t.detach(), slots[rank], atol=TOL), "gather fake fed wrong shard" + return torch.cat([slots[s] if s != rank else t for s in range(ETP)], dim=-1) + + return fake + + +def _fake_reduce(other_partial): + """Faithful model of a forward all-reduce (sum); backward of reduce_from is identity.""" + + def fake(t, group=None): + return t + other_partial + + return fake + + +class _BackwardAddConstant(torch.autograd.Function): + """Forward identity; backward adds a precomputed other-rank gradient term. + + Models ``copy_to_tensor_model_parallel_region``'s backward all-reduce in a + single-process simulation, where the other simulated rank's contribution is a + closed-form constant rather than a live collective. + """ + + @staticmethod + def forward(ctx, t, other_grad): + ctx.save_for_backward(other_grad) + return t + + @staticmethod + def backward(ctx, grad): + (other_grad,) = ctx.saved_tensors + return grad + other_grad, None + + +def _fake_copy_to(other_grad): + def fake(t, group=None): + return _BackwardAddConstant.apply(t, other_grad) + + return fake + + +class _FakeAllGatherLastDim(torch.autograd.Function): + """Forward AG (cat with the live own-rank slot); backward reduce-scatter-sum. + + Models ``all_gather_last_dim_from_tensor_parallel_region``: the backward sums + gradient slices across ranks, then hands this rank its own slice. The other + rank's slice-sum contribution enters as a precomputed constant. + """ + + @staticmethod + def forward(ctx, t, rank, slots, other_grad_slice): + ctx.rank = rank + ctx.width = t.shape[-1] + ctx.save_for_backward(other_grad_slice) + parts = [slots[s] if s != rank else t for s in range(ETP)] + return torch.cat(parts, dim=-1) + + @staticmethod + def backward(ctx, grad): + (other_grad_slice,) = ctx.saved_tensors + own = grad[..., ctx.rank * ctx.width : (ctx.rank + 1) * ctx.width] + return own + other_grad_slice, None, None, None + + +def _fake_ag_last_dim(rank, slots, other_grad_slice): + def fake(t, group=None): + assert torch.allclose(t.detach(), slots[rank], atol=TOL), "AG fake fed wrong shard" + return _FakeAllGatherLastDim.apply(t, rank, slots, other_grad_slice) + + return fake + + +def _patch_fix_ops_if_present(stack, *, copy_to=None, reduce_from=None, ag_last_dim=None): + """Patch the fix's completion ops at their bridge import site — only once they exist. + + Pre-fix (fail-first form) the names are absent from ``megatron.bridge.peft.utils`` and the + defective path runs unpatched; post-fix these fakes model the new collectives. + """ + + for name, fake in ( + ("copy_to_tensor_model_parallel_region", copy_to), + ("reduce_from_tensor_model_parallel_region", reduce_from), + ("all_gather_last_dim_from_tensor_parallel_region", ag_last_dim), + ): + if fake is not None and hasattr(peft_utils, name): + stack.enter_context(patch.object(peft_utils, name, fake)) + + +# --------------------------------------------------------------------------- +# 2a/2d flag: linear_out gather_output truth table (construction-level) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("is_expert", "input_is_parallel", "base_linear_is_parallel", "dtc", "expected_gather"), + [ + pytest.param( + True, + True, + True, + True, + False, + marks=pytest.mark.xfail( + strict=True, + reason="expert-fc2: linear_out keeps gather_output=True — the gathered delta is " + "summed once per rank by the dispatcher's expert-TP reduce", + ), + id="expert-fc2", + ), + pytest.param( + False, + True, + True, + True, + False, + marks=pytest.mark.xfail( + strict=True, + reason="shared-overlap: linear_out keeps gather_output=True — the gathered " + "delta is summed TP times by SharedExpertMLP.post_forward_comm", + ), + id="shared-overlap-fc2", + ), + pytest.param(False, True, True, False, True, id="dense-fc2-keeps-gather"), + pytest.param(True, False, True, True, False, id="expert-fc1"), + pytest.param(False, False, True, True, False, id="shared-overlap-fc1"), + pytest.param(True, True, False, True, True, id="replicated-base-keeps-gather"), + ], +) +def test_lin_out_gather_output_truth_table( + is_expert, input_is_parallel, base_linear_is_parallel, dtc, expected_gather +): + """Pin the constructed gather_output flag per flavor (post-fix contract).""" + + config = MockModelParallelConfig() + config.expert_tensor_parallel_size = ETP + config.tensor_model_parallel_size = ETP + with ( + patch.object(peft_utils, "ColumnParallelLinear") as mock_col, + patch.object(peft_utils, "RowParallelLinear"), + ): + ParallelLinearAdapter( + IN, + OUT, + DIM, + base_linear_name="decoder.layers.0.mlp.experts.local_experts.0.linear_fc2", + activation="identity", + input_is_parallel=input_is_parallel, + is_expert=is_expert, + disable_tensor_parallel_comm=dtc, + base_linear_is_parallel=base_linear_is_parallel, + model_parallel_config=config, + pg_collection=make_mock_pg_collection(etp_size=ETP, tp_size=ETP), + ) + # linear_out is always the last ColumnParallelLinear constructed. + assert mock_col.call_args.kwargs["gather_output"] is expected_gather + + +# --------------------------------------------------------------------------- +# 2a: default-path expert fc2 — missing reduce + full delta where a partial term is due +# --------------------------------------------------------------------------- + + +def _build_expert_adapter(rank, *, input_is_parallel): + """Real ParallelLinearAdapter(is_expert=True) as simulated ETP rank ``rank``.""" + + return ParallelLinearAdapter( + IN, + OUT, + DIM, + base_linear_name="decoder.layers.0.mlp.experts.local_experts.0.linear_fc2" + if input_is_parallel + else "decoder.layers.0.mlp.experts.local_experts.0.linear_fc1", + activation="identity", + input_is_parallel=input_is_parallel, + is_expert=True, + alpha=DIM, # alpha/dim == 1 keeps the closed forms scale-free + disable_tensor_parallel_comm=True, + model_parallel_config=_etp2_config(), + pg_collection=make_mock_pg_collection(etp_size=ETP, etp_rank=rank), + ) + + +def _expert_fc2_fixtures(): + x = _seeded((TOKENS, IN), 0) + a = _seeded((DIM, IN), 1) + b = _seeded((OUT, DIM), 2) + z_partials = [ + x[:, s * IN_SHARD : (s + 1) * IN_SHARD] @ a[:, s * IN_SHARD : (s + 1) * IN_SHARD].t() for s in range(ETP) + ] + # What each rank's broken pipeline emits into the (pre-fix) output gather. + out_slots = [z_partials[s] @ b[s * OUT_SHARD : (s + 1) * OUT_SHARD, :].t() for s in range(ETP)] + return x, a, b, z_partials, out_slots + + +@pytest.mark.xfail( + strict=True, + reason="expert-fc2: A@h stays a per-rank partial (row-parallel reduce suppressed under " + "explicit_expert_comm) and the gathered delta is summed once per ETP rank — " + "combined == 2 * cat_s((x_s @ A_s^T) @ B_s^T), not x @ (B@A)^T", +) +def test_expert_fc2_adapter_is_exact_under_the_dispatcher_etp_sum(): + """Forward exactness of the default expert fc2 adapter under the dispatcher's ETP sum.""" + + x, a, b, z_partials, out_slots = _expert_fc2_fixtures() + outputs = [] + for rank in range(ETP): + adapter = _build_expert_adapter(rank, input_is_parallel=True) + adapter.linear_in.weight.data.copy_(a[:, rank * IN_SHARD : (rank + 1) * IN_SHARD]) + adapter.linear_out.weight.data.copy_(b[rank * OUT_SHARD : (rank + 1) * OUT_SHARD, :]) + x_r = x[:, rank * IN_SHARD : (rank + 1) * IN_SHARD] + with contextlib.ExitStack() as stack: + stack.enter_context( + patch(f"{_MCORE_LAYERS}.gather_from_tensor_model_parallel_region", _fake_gather(rank, out_slots)) + ) + _patch_fix_ops_if_present( + stack, + copy_to=_fake_copy_to(torch.zeros(TOKENS, DIM, dtype=torch.float64)), + reduce_from=_fake_reduce(z_partials[1 - rank]), + ) + outputs.append(adapter(x_r)) + combined = outputs[0] + outputs[1] # the dispatcher's expert-TP sum, per token + assert torch.allclose(combined, x @ (b @ a).t(), atol=TOL), (combined / (x @ (b @ a).t())).flatten()[:4] + + +# --------------------------------------------------------------------------- +# 2b: default-path expert fc1 — green forward, broken dL/dA (V4) +# --------------------------------------------------------------------------- + + +def _expert_fc1_fixtures(): + x = _seeded((TOKENS, IN), 0) + a = _seeded((DIM, IN), 1) + b = _seeded((OUT, DIM), 2) + g = [_seeded((TOKENS, OUT_SHARD), 10 + r) for r in range(ETP)] + z_slots = [x @ a[s * DIM_SHARD : (s + 1) * DIM_SHARD, :].t() for s in range(ETP)] + return x, a, b, g, z_slots + + +def _run_expert_fc1(rank, x, a, b, g, z_slots): + adapter = _build_expert_adapter(rank, input_is_parallel=False) + adapter.linear_in.weight.data.copy_(a[rank * DIM_SHARD : (rank + 1) * DIM_SHARD, :]) + adapter.linear_out.weight.data.copy_(b[rank * OUT_SHARD : (rank + 1) * OUT_SHARD, :]) + # The compensating backward all-reduce sums dL/dz across ranks; the other rank's + # term is the closed-form constant g_other @ B_other. + other_dz = g[1 - rank] @ b[(1 - rank) * OUT_SHARD : (2 - rank) * OUT_SHARD, :] + with contextlib.ExitStack() as stack: + stack.enter_context( + patch(f"{_MCORE_LAYERS}.gather_from_tensor_model_parallel_region", _fake_gather(rank, z_slots)) + ) + _patch_fix_ops_if_present(stack, copy_to=_fake_copy_to(other_dz)) + out = adapter(x) + (out * g[rank]).sum().backward() + return adapter, out + + +def test_expert_fc1_forward_and_dL_dB_are_exact(): + """Pins: the fc1 forward is exact and dL/dB is unaffected — the green-forward trap.""" + + x, a, b, g, z_slots = _expert_fc1_fixtures() + z = x @ a.t() + for rank in range(ETP): + adapter, out = _run_expert_fc1(rank, x, a, b, g, z_slots) + b_r = b[rank * OUT_SHARD : (rank + 1) * OUT_SHARD, :] + assert torch.allclose(out, z @ b_r.t(), atol=TOL) + assert torch.allclose(adapter.linear_out.weight.grad, g[rank].t() @ z, atol=TOL) + + +@pytest.mark.xfail( + strict=True, + reason="expert-fc1-dgrad: allreduce_dgrad is forced False under explicit_expert_comm, so dL/dA_r keeps " + "only the own-rank term (g_r @ B_r) and loses every cross-rank term", +) +def test_expert_fc1_dL_dA_recovers_cross_rank_terms(): + x, a, b, g, z_slots = _expert_fc1_fixtures() + dz_sum = sum(g[s] @ b[s * OUT_SHARD : (s + 1) * OUT_SHARD, :] for s in range(ETP)) + for rank in range(ETP): + adapter, _ = _run_expert_fc1(rank, x, a, b, g, z_slots) + want = dz_sum[:, rank * DIM_SHARD : (rank + 1) * DIM_SHARD].t() @ x + got = adapter.linear_in.weight.grad + assert torch.allclose(got, want, atol=TOL), (got - want).abs().max() / want.abs().max() + + +# --------------------------------------------------------------------------- +# 2c: GroupedExpertLinearAdapter — severed autograd (V2) +# --------------------------------------------------------------------------- + +N_EXPERTS = 2 +# Uneven but ETP-divisible: the production per-expert path pads each expert's tokens to a +# multiple of ETP, and the closed-form slot constants must match the tensors the collectives +# actually see. The pad path itself is exercised by the distributed file's empty-split case +# and the algebra files' non-divisible token count. +SPLITS = [2, 4] + + +def _build_grouped_adapter(rank, *, input_is_parallel): + return GroupedExpertLinearAdapter( + in_features=IN, + out_features=OUT, + dim=DIM, + num_local_experts=N_EXPERTS, + base_linear_name="decoder.layers.0.mlp.experts.linear_fc2" + if input_is_parallel + else "decoder.layers.0.mlp.experts.linear_fc1", + activation="identity", + input_is_parallel=input_is_parallel, + alpha=DIM, + model_parallel_config=_etp2_config(), + params_device=torch.device("cpu"), + params_dtype=torch.float64, + pg_collection=make_mock_pg_collection(etp_size=ETP, etp_rank=rank), + ) + + +def _grouped_fixtures(): + x = _seeded((sum(SPLITS), IN), 0) + a = [_seeded((DIM, IN), 1 + e) for e in range(N_EXPERTS)] + b = [_seeded((OUT, DIM), 11 + e) for e in range(N_EXPERTS)] + return x, a, b + + +def _expert_inputs(x): + outs, start = [], 0 + for split in SPLITS: + outs.append(x.narrow(0, start, split)) + start += split + return outs + + +def _sequential_slot_fake(expected_slot_lists, rank): + """Bare-collective fake: fills the production empty_like buffers, call-order sequenced. + + The severance under test is the production ``empty_like`` + ``cat`` — the fake only + supplies exact values (with a faithfulness assert on the live input). + """ + + calls = iter(expected_slot_lists) + + def fake(gathered, tensor, group=None): + slots = next(calls) + assert torch.allclose(tensor.detach(), slots[rank], atol=TOL), "all_gather fake fed wrong shard" + for s in range(ETP): + gathered[s].copy_(slots[s]) + + return fake + + +@pytest.mark.xfail( + strict=True, + reason="grouped-autograd: _gather_along_last_dim is a bare torch.distributed.all_gather into empty_like " + "buffers — invisible to autograd, so dL/dA is None on the fc1 path", +) +def test_grouped_fc1_gather_severs_dL_dA(): + x, a, b = _grouped_fixtures() + rank = 0 + adapter = _build_grouped_adapter(rank, input_is_parallel=False) + for e in range(N_EXPERTS): + adapter.linear_in.weight.data[e].copy_(a[e][rank * DIM_SHARD : (rank + 1) * DIM_SHARD, :]) + adapter.linear_out.weight.data[e].copy_(b[e][rank * OUT_SHARD : (rank + 1) * OUT_SHARD, :]) + x = x.clone().requires_grad_(True) + hidden_slots = [ + [x_e.detach() @ a[e][s * DIM_SHARD : (s + 1) * DIM_SHARD, :].t() for s in range(ETP)] + for e, x_e in enumerate(_expert_inputs(x)) + ] + g = _seeded((sum(SPLITS), OUT_SHARD), 20) + with contextlib.ExitStack() as stack: + stack.enter_context(patch("torch.distributed.all_gather", _sequential_slot_fake(hidden_slots, rank))) + ag_fakes = iter( + [ + _fake_ag_last_dim(rank, slots, torch.zeros(SPLITS[e], DIM_SHARD, dtype=torch.float64)) + for e, slots in enumerate(hidden_slots) + ] + ) + _patch_fix_ops_if_present(stack, ag_last_dim=lambda t, group=None: next(ag_fakes)(t, group=group)) + out = adapter(x, SPLITS) + (out * g).sum().backward() + # Pin: the fc1 forward is exact even today (concatenated distinct slices). + for e, (x_e, start) in enumerate(zip(_expert_inputs(x), [0, SPLITS[0]])): + want = (x_e.detach() @ a[e].t()) @ b[e][rank * OUT_SHARD : (rank + 1) * OUT_SHARD, :].t() + assert torch.allclose(out[start : start + SPLITS[e]].detach(), want, atol=TOL) + assert adapter.linear_in.weight.grad is not None + assert x.grad is not None + + +@pytest.mark.xfail( + strict=True, + reason="grouped-autograd: the fc2 path gathers after linear_out through the autograd-invisible " + "all_gather — the adapter output carries no graph at all (dL/dA = dL/dB = 0)", +) +def test_grouped_fc2_output_requires_grad(): + x, a, b = _grouped_fixtures() + rank = 0 + adapter = _build_grouped_adapter(rank, input_is_parallel=True) + for e in range(N_EXPERTS): + adapter.linear_in.weight.data[e].copy_(a[e][:, rank * IN_SHARD : (rank + 1) * IN_SHARD]) + adapter.linear_out.weight.data[e].copy_(b[e][rank * OUT_SHARD : (rank + 1) * OUT_SHARD, :]) + x_r = x[:, rank * IN_SHARD : (rank + 1) * IN_SHARD] + out_slots = [ + [ + (x_e[:, s * IN_SHARD : (s + 1) * IN_SHARD] @ a[e][:, s * IN_SHARD : (s + 1) * IN_SHARD].t()) + @ b[e][s * OUT_SHARD : (s + 1) * OUT_SHARD, :].t() + for s in range(ETP) + ] + for e, x_e in enumerate(_expert_inputs(x)) + ] + z_other = [ + x_e[:, (1 - rank) * IN_SHARD : (2 - rank) * IN_SHARD] + @ a[e][:, (1 - rank) * IN_SHARD : (2 - rank) * IN_SHARD].t() + for e, x_e in enumerate(_expert_inputs(x)) + ] + with contextlib.ExitStack() as stack: + stack.enter_context(patch("torch.distributed.all_gather", _sequential_slot_fake(out_slots, rank))) + reduce_fakes = iter([_fake_reduce(z_other[e]) for e in range(N_EXPERTS)]) + _patch_fix_ops_if_present( + stack, + copy_to=_fake_copy_to(torch.zeros(1, dtype=torch.float64)), + reduce_from=lambda t, group=None: next(reduce_fakes)(t, group=group), + ) + out = adapter(x_r, SPLITS) + assert out.requires_grad + + +@pytest.mark.xfail( + strict=True, + reason="grouped-fc2: the adapter emits an unsummed-partial delta gathered identically on " + "every rank — the dispatcher's ETP sum yields 2 * cat_s((x_s A_s^T) B_s^T)", +) +def test_grouped_fc2_is_exact_under_the_dispatcher_etp_sum(): + x, a, b = _grouped_fixtures() + outputs = [] + for rank in range(ETP): + adapter = _build_grouped_adapter(rank, input_is_parallel=True) + for e in range(N_EXPERTS): + adapter.linear_in.weight.data[e].copy_(a[e][:, rank * IN_SHARD : (rank + 1) * IN_SHARD]) + adapter.linear_out.weight.data[e].copy_(b[e][rank * OUT_SHARD : (rank + 1) * OUT_SHARD, :]) + x_r = x[:, rank * IN_SHARD : (rank + 1) * IN_SHARD] + out_slots = [ + [ + (x_e[:, s * IN_SHARD : (s + 1) * IN_SHARD] @ a[e][:, s * IN_SHARD : (s + 1) * IN_SHARD].t()) + @ b[e][s * OUT_SHARD : (s + 1) * OUT_SHARD, :].t() + for s in range(ETP) + ] + for e, x_e in enumerate(_expert_inputs(x)) + ] + z_other = [ + x_e[:, (1 - rank) * IN_SHARD : (2 - rank) * IN_SHARD] + @ a[e][:, (1 - rank) * IN_SHARD : (2 - rank) * IN_SHARD].t() + for e, x_e in enumerate(_expert_inputs(x)) + ] + with contextlib.ExitStack() as stack: + stack.enter_context(patch("torch.distributed.all_gather", _sequential_slot_fake(out_slots, rank))) + reduce_fakes = iter([_fake_reduce(z_other[e]) for e in range(N_EXPERTS)]) + _patch_fix_ops_if_present( + stack, + copy_to=_fake_copy_to(torch.zeros(1, dtype=torch.float64)), + reduce_from=lambda t, group=None: next(reduce_fakes)(t, group=group), + ) + outputs.append(adapter(x_r, SPLITS)) + combined = outputs[0] + outputs[1] + want = torch.cat( + [x_e @ (b[e] @ a[e]).t() for e, x_e in enumerate(_expert_inputs(x))], + dim=0, + ) + assert torch.allclose(combined.detach(), want, atol=TOL), (combined.detach() / want).flatten()[:4] + + +# --------------------------------------------------------------------------- +# 2d: shared-expert-overlap fc2 — delta counted TP times (V5) +# --------------------------------------------------------------------------- + + +def _build_shared_overlap_adapter(rank): + return ParallelLinearAdapter( + IN, + OUT, + DIM, + base_linear_name="decoder.layers.0.mlp.shared_experts.linear_fc2", + activation="identity", + input_is_parallel=True, + is_expert=False, + alpha=DIM, + disable_tensor_parallel_comm=True, + model_parallel_config=_tp2_config(), + pg_collection=make_mock_pg_collection(tp_size=ETP, tp_rank=rank), + ) + + +@pytest.mark.xfail( + strict=True, + reason="shared-overlap: the adapter's linear_in reduce completes z, linear_out gathers the full-width " + "delta on every rank, and SharedExpertMLP.post_forward_comm sums it TP times — " + "combined == 2 * (x @ (B@A)^T)", +) +def test_shared_expert_fc2_delta_is_counted_once_by_post_forward_comm(): + x = _seeded((TOKENS, IN), 0) + a = _seeded((DIM, IN), 1) + b = _seeded((OUT, DIM), 2) + z_partials = [ + x[:, s * IN_SHARD : (s + 1) * IN_SHARD] @ a[:, s * IN_SHARD : (s + 1) * IN_SHARD].t() for s in range(ETP) + ] + z_full = sum(z_partials) + out_slots = [z_full @ b[s * OUT_SHARD : (s + 1) * OUT_SHARD, :].t() for s in range(ETP)] + outputs = [] + for rank in range(ETP): + adapter = _build_shared_overlap_adapter(rank) + adapter.linear_in.weight.data.copy_(a[:, rank * IN_SHARD : (rank + 1) * IN_SHARD]) + adapter.linear_out.weight.data.copy_(b[rank * OUT_SHARD : (rank + 1) * OUT_SHARD, :]) + x_r = x[:, rank * IN_SHARD : (rank + 1) * IN_SHARD] + with contextlib.ExitStack() as stack: + # The adapter's own RowParallelLinear is NOT expert-suppressed here: its forward + # calls the mcore reduce (z completes correctly — only the output side is broken). + stack.enter_context( + patch(f"{_MCORE_LAYERS}.reduce_from_tensor_model_parallel_region", _fake_reduce(z_partials[1 - rank])) + ) + stack.enter_context( + patch(f"{_MCORE_LAYERS}.gather_from_tensor_model_parallel_region", _fake_gather(rank, out_slots)) + ) + outputs.append(adapter(x_r)) + combined = outputs[0] + outputs[1] # SharedExpertMLP.post_forward_comm's dense-TP sum + assert torch.allclose(combined, x @ (b @ a).t(), atol=TOL), (combined / (x @ (b @ a).t())).flatten()[:4] diff --git a/tests/unit_tests/peft/test_expert_lora_etp_distributed.py b/tests/unit_tests/peft/test_expert_lora_etp_distributed.py new file mode 100644 index 0000000000..956ba92dc3 --- /dev/null +++ b/tests/unit_tests/peft/test_expert_lora_etp_distributed.py @@ -0,0 +1,525 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Two-rank real-collective tests for expert LoRA exactness at expert-TP = 2. + +Run with: +uv run python -m torch.distributed.run --nproc_per_node=2 -m pytest \ + tests/unit_tests/peft/test_expert_lora_etp_distributed.py + +This file discharges the one modeling assumption the CPU harness +(tests/unit_tests/peft/test_expert_lora_etp.py) leans on: that its faked collectives are +faithful to MCore + c10d. Here real NCCL groups and real ``parallel_state`` topology drive the +same defect sites. One topology serves every defect: TP=2 gives the dense group for the +shared-expert-overlap case (V5), ETP=2 the expert group for V1/V2/V4, world size 2, DP=1. + +Each defect-demonstrating assertion carries ``@pytest.mark.xfail(strict=True)`` — the +fail-first record; the fix commit deletes the markers. All collectives run unconditionally +before any assert so a failing assert cannot desync the two ranks. +""" + +import os +from collections.abc import Iterator +from contextlib import contextmanager + +import megatron.core.parallel_state as parallel_state +import pytest +import torch +import torch.distributed as dist +from megatron.core.model_parallel_config import ModelParallelConfig +from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed + +from megatron.bridge.peft.utils import GroupedExpertLinearAdapter, ParallelLinearAdapter + + +_WORLD_SIZE = 2 +_ETP_SIZE = 2 +TOKENS, IN, DIM, OUT = 8, 8, 4, 8 +IN_SHARD, DIM_SHARD, OUT_SHARD = IN // _ETP_SIZE, DIM // _ETP_SIZE, OUT // _ETP_SIZE +N_EXPERTS = 2 + + +@contextmanager +def _distributed_tp_etp() -> Iterator[ProcessGroupCollection]: + """Initialize the two-rank TP=2 x ETP=2 topology used by every test here.""" + owns_process_group = not dist.is_initialized() + owns_model_parallel = not parallel_state.model_parallel_is_initialized() + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + torch.cuda.set_device(local_rank) + # TF32 (the H100 default for fp32 matmuls) adds ~1e-3 relative noise, swamping the + # 1e-4 tolerances these closed-form comparisons use. The defect signals here are + # 2x-class, but exact tolerances keep the pins meaningful. + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + + if owns_process_group: + dist.init_process_group(backend="nccl") + if owns_model_parallel: + parallel_state.initialize_model_parallel( + tensor_model_parallel_size=_ETP_SIZE, + pipeline_model_parallel_size=1, + context_parallel_size=1, + expert_model_parallel_size=1, + expert_tensor_parallel_size=_ETP_SIZE, + ) + model_parallel_cuda_manual_seed(2026, force_reset_rng=True) + + try: + yield ProcessGroupCollection.use_mpu_process_groups() + finally: + if owns_model_parallel and parallel_state.model_parallel_is_initialized(): + parallel_state.destroy_model_parallel() + if owns_process_group and dist.is_initialized(): + dist.destroy_process_group() + + +@pytest.fixture(scope="module") +def pg_collection() -> Iterator[ProcessGroupCollection]: + """Provide one shared two-rank TP=2 x ETP=2 topology for this module.""" + if int(os.environ.get("WORLD_SIZE", "1")) != _WORLD_SIZE: + pytest.skip("requires a two-rank torch.distributed launch") + if not torch.cuda.is_available(): + pytest.skip("requires CUDA") + + with _distributed_tp_etp() as pgs: + yield pgs + + +def _config(*, tp_size: int, etp_size: int, regather: bool = False) -> ModelParallelConfig: + return ModelParallelConfig( + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=1, + expert_model_parallel_size=1, + expert_tensor_parallel_size=etp_size, + sequence_parallel=False, + params_dtype=torch.float32, + gradient_accumulation_fusion=False, + ) + + +def _full_weights(device): + """Full (unsharded) fp32 A/B and inputs, identical on every rank by fixed seed.""" + gen = torch.Generator().manual_seed(2026) + x = torch.randn(TOKENS, IN, generator=gen).to(device) + a = torch.randn(DIM, IN, generator=gen).to(device) + b = torch.randn(OUT, DIM, generator=gen).to(device) + g = torch.randn(TOKENS, OUT, generator=gen).to(device) + return x, a, b, g + + +def _copy_expert_shards(adapter, a, b, rank, *, input_is_parallel): + with torch.no_grad(): + if input_is_parallel: + adapter.linear_in.weight.copy_(a[:, rank * IN_SHARD : (rank + 1) * IN_SHARD]) + else: + adapter.linear_in.weight.copy_(a[rank * DIM_SHARD : (rank + 1) * DIM_SHARD, :]) + adapter.linear_out.weight.copy_(b[rank * OUT_SHARD : (rank + 1) * OUT_SHARD, :]) + + +# --------------------------------------------------------------------------- +# V1: default ParallelLinearAdapter, fc2 flavor — forward exactness +# --------------------------------------------------------------------------- + + +@pytest.mark.gpu +@pytest.mark.xfail( + strict=True, + reason="expert-fc2: A@h stays a per-rank partial and the gathered delta is summed once per rank " + "by the dispatcher's expert-TP reduce — real-collective form of the CPU demonstration", +) +def test_default_fc2_adapter_delta_matches_merged_reference(pg_collection) -> None: + device = torch.device("cuda", torch.cuda.current_device()) + rank = parallel_state.get_expert_tensor_parallel_rank() + x, a, b, g = _full_weights(device) + + adapter = ParallelLinearAdapter( + IN, + OUT, + DIM, + base_linear_name="decoder.layers.0.mlp.experts.linear_fc2", + activation="identity", + input_is_parallel=True, + is_expert=True, + alpha=DIM, + disable_tensor_parallel_comm=True, + model_parallel_config=_config(tp_size=_ETP_SIZE, etp_size=_ETP_SIZE), + pg_collection=pg_collection, + ).to(device) + _copy_expert_shards(adapter, a, b, rank, input_is_parallel=True) + + x_r = x[:, rank * IN_SHARD : (rank + 1) * IN_SHARD] + delta = adapter(x_r) + # The dispatcher's expert-TP sum (combine_preprocess reduce), run before any assert. + combined = delta.detach().clone() + dist.all_reduce(combined, group=pg_collection.expt_tp) + + reference = x @ (b @ a).t() + torch.testing.assert_close(combined, reference, rtol=1e-4, atol=1e-4) + + +# --------------------------------------------------------------------------- +# V4: default ParallelLinearAdapter, fc1 flavor — green forward, broken dL/dA +# --------------------------------------------------------------------------- + + +def _run_fc1(pg_collection, *, regather: bool = False): + device = torch.device("cuda", torch.cuda.current_device()) + rank = parallel_state.get_expert_tensor_parallel_rank() + x, a, b, g = _full_weights(device) + + adapter = ParallelLinearAdapter( + IN, + OUT, + DIM, + base_linear_name="decoder.layers.0.mlp.experts.linear_fc1", + activation="identity", + input_is_parallel=False, + is_expert=True, + alpha=DIM, + disable_tensor_parallel_comm=True, + model_parallel_config=_config(tp_size=_ETP_SIZE, etp_size=_ETP_SIZE), + pg_collection=pg_collection, + sequence_parallel_input_regather=regather, + ).to(device) + _copy_expert_shards(adapter, a, b, rank, input_is_parallel=False) + + g_r = g[:, rank * OUT_SHARD : (rank + 1) * OUT_SHARD] + out = adapter(x) + (out * g_r).sum().backward() + + # All collectives before any assert: gather both ranks' dL/dA shards. + da_local = adapter.linear_in.weight.grad.detach().clone() + da_shards = [torch.empty_like(da_local) for _ in range(_ETP_SIZE)] + dist.all_gather(da_shards, da_local, group=pg_collection.expt_tp) + + z = x @ a.t() + dz_full = sum( + g[:, s * OUT_SHARD : (s + 1) * OUT_SHARD] @ b[s * OUT_SHARD : (s + 1) * OUT_SHARD, :] for s in range(_ETP_SIZE) + ) + return adapter, out, g_r, z, dz_full, da_shards, b, x, rank + + +@pytest.mark.gpu +def test_default_fc1_forward_and_dL_dB_are_exact(pg_collection) -> None: + """Pins: the fc1 forward is exact and dL/dB unaffected — the green-forward trap.""" + adapter, out, g_r, z, _, _, b, _, rank = _run_fc1(pg_collection) + b_r = b[rank * OUT_SHARD : (rank + 1) * OUT_SHARD, :] + torch.testing.assert_close(out, z @ b_r.t(), rtol=1e-4, atol=1e-4) + torch.testing.assert_close(adapter.linear_out.weight.grad, g_r.t() @ z, rtol=1e-4, atol=1e-4) + + +@pytest.mark.gpu +@pytest.mark.xfail( + strict=True, + reason="expert-fc1-dgrad: allreduce_dgrad is forced False under explicit_expert_comm — each rank's " + "dL/dA keeps only its own-rank term (~72% relative error measured on 2xH100)", +) +def test_default_fc1_dL_dA_matches_merged_reference(pg_collection) -> None: + _, _, _, _, dz_full, da_shards, _, x, _ = _run_fc1(pg_collection) + for s in range(_ETP_SIZE): + want = dz_full[:, s * DIM_SHARD : (s + 1) * DIM_SHARD].t() @ x + torch.testing.assert_close(da_shards[s], want, rtol=1e-4, atol=1e-4) + + +@pytest.mark.gpu +def test_default_fc1_regather_toggle_is_inert_for_experts(pg_collection) -> None: + """Pin (§6.6): sequence_parallel_input_regather must not change expert-adapter results.""" + _, out_plain, _, _, _, _, _, _, _ = _run_fc1(pg_collection, regather=False) + _, out_regather, _, _, _, _, _, _, _ = _run_fc1(pg_collection, regather=True) + torch.testing.assert_close(out_plain, out_regather, rtol=0, atol=0) + + +# --------------------------------------------------------------------------- +# V2: GroupedExpertLinearAdapter — severed autograd, real bare all_gather +# --------------------------------------------------------------------------- + + +def _build_grouped(pg_collection, *, input_is_parallel): + device = torch.device("cuda", torch.cuda.current_device()) + adapter = GroupedExpertLinearAdapter( + in_features=IN, + out_features=OUT, + dim=DIM, + num_local_experts=N_EXPERTS, + base_linear_name="decoder.layers.0.mlp.experts.linear_fc2" + if input_is_parallel + else "decoder.layers.0.mlp.experts.linear_fc1", + activation="identity", + input_is_parallel=input_is_parallel, + alpha=DIM, + model_parallel_config=_config(tp_size=_ETP_SIZE, etp_size=_ETP_SIZE), + params_device=device, + params_dtype=torch.float32, + pg_collection=pg_collection, + ) + return adapter, device + + +@pytest.mark.gpu +@pytest.mark.xfail( + strict=True, + reason="grouped-autograd: the real bare torch.distributed.all_gather severs autograd — grouped fc2 " + "adapter output carries no graph (dL/dA = dL/dB = 0, silently)", +) +def test_grouped_fc2_output_requires_grad(pg_collection) -> None: + adapter, device = _build_grouped(pg_collection, input_is_parallel=True) + x = torch.randn(TOKENS, IN_SHARD, device=device) + out = adapter(x, [TOKENS // 2, TOKENS - TOKENS // 2]) + dist.barrier() + assert out.requires_grad + + +@pytest.mark.gpu +@pytest.mark.xfail( + strict=True, + reason="grouped-autograd: grouped fc1 forward is exact but the gather severs the graph — dL/dA is None", +) +def test_grouped_fc1_dL_dA_flows(pg_collection) -> None: + adapter, device = _build_grouped(pg_collection, input_is_parallel=False) + x = torch.randn(TOKENS, IN, device=device) + out = adapter(x, [TOKENS // 2, TOKENS - TOKENS // 2]) + out.sum().backward() + dist.barrier() + assert adapter.linear_in.weight.grad is not None + + +def _per_expert_weights(device): + """Deterministic full A/B per expert, identical on every rank by fixed seeds.""" + a, b = [], [] + for e in range(N_EXPERTS): + gen = torch.Generator().manual_seed(3000 + e) + a.append(torch.randn(DIM, IN, generator=gen).to(device)) + b.append(torch.randn(OUT, DIM, generator=gen).to(device)) + return a, b + + +@pytest.mark.gpu +@pytest.mark.xfail( + strict=True, + reason="grouped-fc2: the adapter emits an unsummed-partial delta gathered identically on " + "every rank — the real dispatcher-style expert-TP sum yields 2 * cat_s((x_s A_s^T) B_s^T), " + "not the merged reference", +) +def test_grouped_fc2_delta_matches_merged_reference(pg_collection) -> None: + """Real-collective form of the CPU grouped-fc2 wrong-value demonstration.""" + adapter, device = _build_grouped(pg_collection, input_is_parallel=True) + rank = parallel_state.get_expert_tensor_parallel_rank() + a, b = _per_expert_weights(device) + with torch.no_grad(): + for e in range(N_EXPERTS): + adapter.linear_in.weight[e].copy_(a[e][:, rank * IN_SHARD : (rank + 1) * IN_SHARD]) + adapter.linear_out.weight[e].copy_(b[e][rank * OUT_SHARD : (rank + 1) * OUT_SHARD, :]) + + gen = torch.Generator().manual_seed(2026) + x = torch.randn(TOKENS, IN, generator=gen).to(device) + x_r = x[:, rank * IN_SHARD : (rank + 1) * IN_SHARD] + splits = [TOKENS // 2, TOKENS - TOKENS // 2] + + delta = adapter(x_r, splits) + # The dispatcher's expert-TP sum, run before any assert. + combined = delta.detach().clone() + dist.all_reduce(combined, group=pg_collection.expt_tp) + + want = torch.cat( + [x[e * splits[0] : e * splits[0] + splits[e]] @ (b[e] @ a[e]).t() for e in range(N_EXPERTS)], + dim=0, + ) + torch.testing.assert_close(combined, want, rtol=1e-4, atol=1e-4) + + +# --------------------------------------------------------------------------- +# MultiLoRA V5-analog: dense wrapper on a suppressed-comm row-parallel base +# --------------------------------------------------------------------------- + + +@pytest.mark.gpu +@pytest.mark.xfail( + strict=True, + reason="multi-lora-overlap: the dense wrapper gathers its full-width delta on every " + "rank while the suppressed-comm base's partials are summed downstream across dense TP " + "— the delta is counted exactly TP times", +) +def test_multi_lora_shared_expert_fc2_delta_counted_once(pg_collection) -> None: + """Real-collective MultiLoRALinear on a base whose own TP comms are suppressed. + + Mirrors what ``SharedExpertMLP`` does under ``moe_shared_expert_overlap`` to legacy + linears: sets ``explicit_expert_comm = True`` on the base so it emits full-width + partials that ``post_forward_comm`` later sums across the dense TP group. + """ + from megatron.core.tensor_parallel import RowParallelLinear + + from megatron.bridge.peft.multi_lora_layers import MultiLoRALinear + + device = torch.device("cuda", torch.cuda.current_device()) + rank = parallel_state.get_tensor_model_parallel_rank() + config = _config(tp_size=_ETP_SIZE, etp_size=1) + config.params_dtype = torch.bfloat16 # torch._grouped_mm operates in bf16 + + # torch._grouped_mm requires 16-byte-aligned strides: every bf16 GEMM operand's + # last dim must be a multiple of 8 (same constraint the repo documents for + # GroupedExpertLinearAdapter's grouped_mm fast path). Local dims, sized so that + # in/2, dim, and out/2 are all >= 8. + m_in, m_dim, m_out = 16, 8, 16 + m_in_shard, m_out_shard = m_in // _ETP_SIZE, m_out // _ETP_SIZE + + base = RowParallelLinear( + m_in, + m_out, + config=config, + init_method=torch.nn.init.xavier_normal_, + bias=False, + input_is_parallel=True, + skip_bias_add=True, + tp_group=pg_collection.tp, + ).to(device) + # What shared_experts.py does to legacy linears under moe_shared_expert_overlap. + base.explicit_expert_comm = True + + wrapper = MultiLoRALinear( + to_wrap=base, + n_adapters=2, + dim=m_dim, + alpha=m_dim, # alpha/rank == 1 + full_name="decoder.layers.0.mlp.shared_experts.linear_fc2", + ) + a, b = [], [] + for slot in range(2): + gen = torch.Generator().manual_seed(4000 + slot) + a.append(torch.randn(m_dim, m_in, generator=gen).to(device)) + b.append(torch.randn(m_out, m_dim, generator=gen).to(device)) + with torch.no_grad(): + for s in range(2): + wrapper.adapters[s].linear_in.weight.copy_( + a[s][:, rank * m_in_shard : (rank + 1) * m_in_shard].to(torch.bfloat16) + ) + wrapper.adapters[s].linear_out.weight.copy_( + b[s][rank * m_out_shard : (rank + 1) * m_out_shard, :].to(torch.bfloat16) + ) + wrapper.alpha_values.fill_(1.0) + wrapper.rank_values.fill_(1.0) + + gen = torch.Generator().manual_seed(2026) + x = torch.randn(TOKENS, m_in, generator=gen).to(device=device, dtype=torch.bfloat16) + x_r = x[:, rank * m_in_shard : (rank + 1) * m_in_shard].contiguous() + splits = torch.tensor([TOKENS // 2, TOKENS - TOKENS // 2], dtype=torch.int32, device=device) + wrapper.tokens_per_adapter = splits + wrapper.tokens_per_adapter_total = TOKENS + + out, _ = wrapper(x_r) + base_out, _ = base(x_r) + delta = (out - base_out).detach().clone() + # SharedExpertMLP.post_forward_comm's dense-TP sum, run before any assert. + combined_delta = delta.float() + dist.all_reduce(combined_delta, group=pg_collection.tp) + base_combined = base_out.detach().float().clone() + dist.all_reduce(base_combined, group=pg_collection.tp) + + n0 = TOKENS // 2 + want = torch.cat( + [ + (x[:n0].float() @ (b[0] @ a[0]).t().float()), + (x[n0:].float() @ (b[1] @ a[1]).t().float()), + ], + dim=0, + ) + # bf16 GEMMs: loose per-element tolerance (observed ~7e-2 relative on H100), but an + # order of magnitude tighter than the 2x-overcount defect signal. + torch.testing.assert_close(combined_delta, want, rtol=1.5e-1, atol=5e-1) + + +@pytest.mark.gpu +def test_grouped_adapter_handles_empty_expert_split(pg_collection) -> None: + """Pin (§6.2): a zero-token expert must not hang or error — identical collective + sequences on every ETP rank (the dispatcher replicates the token set across ETP).""" + adapter, device = _build_grouped(pg_collection, input_is_parallel=False) + x = torch.randn(TOKENS, IN, device=device) + out = adapter(x, [0, TOKENS]) + dist.barrier() + assert out.shape[0] == TOKENS + + +# --------------------------------------------------------------------------- +# V5: shared-expert fc2 under moe_shared_expert_overlap — dense-TP overcount +# --------------------------------------------------------------------------- + + +@pytest.mark.gpu +@pytest.mark.xfail( + strict=True, + reason="shared-overlap: the gathered full-width delta is summed once per dense-TP rank by " + "post_forward_comm — combined comes out exactly 2x the merged reference", +) +def test_shared_expert_fc2_delta_counted_once_by_tp_sum(pg_collection) -> None: + device = torch.device("cuda", torch.cuda.current_device()) + rank = parallel_state.get_tensor_model_parallel_rank() + x, a, b, g = _full_weights(device) + + adapter = ParallelLinearAdapter( + IN, + OUT, + DIM, + base_linear_name="decoder.layers.0.mlp.shared_experts.linear_fc2", + activation="identity", + input_is_parallel=True, + is_expert=False, + alpha=DIM, + disable_tensor_parallel_comm=True, + model_parallel_config=_config(tp_size=_ETP_SIZE, etp_size=1), + pg_collection=pg_collection, + ).to(device) + _copy_expert_shards(adapter, a, b, rank, input_is_parallel=True) + + x_r = x[:, rank * IN_SHARD : (rank + 1) * IN_SHARD] + delta = adapter(x_r) + # SharedExpertMLP.post_forward_comm's dense-TP sum, run before any assert. + combined = delta.detach().clone() + dist.all_reduce(combined, group=pg_collection.tp) + + reference = x @ (b @ a).t() + torch.testing.assert_close(combined, reference, rtol=1e-4, atol=1e-4) + + +@pytest.mark.gpu +def test_shared_expert_fc1_is_exact_under_overlap(pg_collection) -> None: + """Pin: the fc1 flavor under overlap already emits the correct output shard today.""" + device = torch.device("cuda", torch.cuda.current_device()) + rank = parallel_state.get_tensor_model_parallel_rank() + x, a, b, _ = _full_weights(device) + + adapter = ParallelLinearAdapter( + IN, + OUT, + DIM, + base_linear_name="decoder.layers.0.mlp.shared_experts.linear_fc1", + activation="identity", + input_is_parallel=False, + is_expert=False, + alpha=DIM, + disable_tensor_parallel_comm=True, + model_parallel_config=_config(tp_size=_ETP_SIZE, etp_size=1), + pg_collection=pg_collection, + ).to(device) + with torch.no_grad(): + adapter.linear_in.weight.copy_(a[rank * DIM_SHARD : (rank + 1) * DIM_SHARD, :]) + adapter.linear_out.weight.copy_(b[rank * OUT_SHARD : (rank + 1) * OUT_SHARD, :]) + + out = adapter(x) + out_shards = [torch.empty_like(out) for _ in range(_ETP_SIZE)] + dist.all_gather(out_shards, out.detach().contiguous(), group=pg_collection.tp) + + z = x @ a.t() + for s in range(_ETP_SIZE): + want = z @ b[s * OUT_SHARD : (s + 1) * OUT_SHARD, :].t() + torch.testing.assert_close(out_shards[s], want, rtol=1e-4, atol=1e-4) diff --git a/tests/unit_tests/peft/test_expert_row_parallel_adapter_algebra.py b/tests/unit_tests/peft/test_expert_row_parallel_adapter_algebra.py new file mode 100644 index 0000000000..791905e646 --- /dev/null +++ b/tests/unit_tests/peft/test_expert_row_parallel_adapter_algebra.py @@ -0,0 +1,107 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Closed-form check of the expert ROW-parallel adapter (experts.linear_fc2) at ETP > 1. + +The adapter must compute what a merged `B @ A` folded into the base weight would, on the +forward pass and on both parameter gradients. (Both arms model the unscaled product; the +`alpha/dim` scaling applied by the shipped classes is elementwise and exercised by +test_expert_lora_etp.py.) This exercises the arithmetic the fix relies on -- per-rank +`A_r @ h_r` summed across the expert-tensor-parallel group, then a zero-embedded `B_r @ z` +that the dispatcher's cross-ETP sum reassembles exactly once -- without needing GPUs or a +process group. These are closed-form models, not the shipped class: the production wiring is +covered by test_expert_lora_etp.py, and the real collectives by +test_expert_lora_etp_distributed.py. + +The token count is deliberately NOT a multiple of ETP so the algebra cannot depend on +divisibility. The pad/unpad path itself is not modeled here; its gradient flow is covered in +test_expert_lora_etp.py. +""" + +import torch + + +T, IN, DIM, OUT, ETP = 7, 12, 4, 8, 4 +IN_SHARD, OUT_SHARD = IN // ETP, OUT // ETP +TOL = 1e-10 + + +def _inputs(): + x = torch.randn(T, IN, dtype=torch.float64, generator=torch.Generator().manual_seed(0)) + g = torch.randn(T, OUT, dtype=torch.float64, generator=torch.Generator().manual_seed(3)) + return x, g + + +def _params(): + a = torch.randn(DIM, IN, dtype=torch.float64, generator=torch.Generator().manual_seed(1)) + b = torch.randn(OUT, DIM, dtype=torch.float64, generator=torch.Generator().manual_seed(2)) + return a.requires_grad_(True), b.requires_grad_(True) + + +def _run(build): + x, g = _inputs() + a, b = _params() + build(x, a, b).backward(g) + return a.grad.clone(), b.grad.clone() + + +def _merged(x, a, b): + """The reference: one dense matmul against the merged weight.""" + return x @ (b @ a).t() + + +def _fixed(x, a, b): + """Per-rank shards, `A @ h` summed across ETP, `B_r @ z` zero-embedded, dispatcher sums.""" + z = sum(x[:, r * IN_SHARD : (r + 1) * IN_SHARD] @ a[:, r * IN_SHARD : (r + 1) * IN_SHARD].t() for r in range(ETP)) + parts = [] + for r in range(ETP): + shard = b[r * OUT_SHARD : (r + 1) * OUT_SHARD, :] @ z.t() + left, right = r * OUT_SHARD, (ETP - 1) * OUT_SHARD - r * OUT_SHARD + parts.append(torch.nn.functional.pad(shard, (0, 0, left, right)).t()) + return sum(parts) + + +def _unfixed(x, a, b): + """`A @ h` left as a per-rank partial, and a gathered delta the dispatcher counts ETP times.""" + z = [x[:, r * IN_SHARD : (r + 1) * IN_SHARD] @ a[:, r * IN_SHARD : (r + 1) * IN_SHARD].t() for r in range(ETP)] + gathered = torch.cat([b[r * OUT_SHARD : (r + 1) * OUT_SHARD, :] @ z[r].t() for r in range(ETP)], dim=0).t() + return ETP * gathered + + +def test_forward_matches_merged_weight(): + x, _ = _inputs() + a, b = _params() + assert torch.allclose(_fixed(x, a, b), _merged(x, a, b), atol=TOL) + + +def test_parameter_gradients_match_merged_weight(): + ref_da, ref_db = _run(_merged) + got_da, got_db = _run(_fixed) + assert torch.allclose(got_da, ref_da, atol=TOL), (got_da - ref_da).abs().max() + assert torch.allclose(got_db, ref_db, atol=TOL), (got_db - ref_db).abs().max() + + +def test_the_unfixed_arithmetic_is_detected(): + """Negative control. Without it the assertions above cannot be shown to discriminate. + + ``_unfixed`` is the arithmetic ``ParallelLinearAdapter(is_expert=True)`` ships for + experts.linear_fc2 at ETP > 1: the suppressed row-parallel reduce leaves ``A_r @ h_r`` + a per-rank partial, and the gathered full-width delta is summed once per rank by the + dispatcher's expert-TP reduce-scatter. The same full-delta-vs-partial-term error (modulo autograd + severance) is ``GroupedExpertLinearAdapter``'s fc2 path. + """ + ref_da, ref_db = _run(_merged) + bad_da, bad_db = _run(_unfixed) + assert (bad_da - ref_da).abs().max() / ref_da.abs().max() > 0.1 + assert (bad_db - ref_db).abs().max() / ref_db.abs().max() > 0.1 From b8b23fd4438a55092fa942f8ad562dbe360f55ff Mon Sep 17 00:00:00 2001 From: Tigist Diriba Date: Tue, 18 Aug 2026 11:49:39 +0300 Subject: [PATCH 2/2] fix(peft)!: make expert LoRA exact under suppressed tensor-parallel communication Complete the adapter arithmetic that explicit_expert_comm suppression leaves undone, using autograd-visible collectives on the groups each adapter already carries: reduce_from(copy_to(x)) over expert-TP for the row-parallel-input case (all-reduce on both passes), copy_to(x) for the column-parallel case (backward-only all-reduce restoring the suppressed dL/dz sum), and a differentiable last-dim all-gather (backward reduce-scatter-sum) replacing the grouped adapter's autograd-invisible bare all_gather in all three backends. On the output side, keep linear_out unsharded-gather-free and zero-embed the local shard at its rank offset, so the downstream sum (the dispatcher over expert-TP; SharedExpertMLP.post_forward_comm over dense TP) counts the delta exactly once with no scaling factor; MultiLoRALinear gets the same treatment on suppressed-comm row-parallel bases. Where no correct completion exists without new machinery, refuse loudly instead of training wrong: DoRA on expert or suppressed-comm TP-sharded linears, SharedOuterGroupedExpertAdapter at ETP>1, expert-adapter dropout at ETP>1 (per-rank RNG on dispatcher-replicated tokens), and LoRALinear.weight on grouped adapter classes. BREAKING CHANGE: the four configurations above now raise NotImplementedError where they previously ran with silently wrong numerics (or an accidental AttributeError). Checkpoint format, weight layout, and export paths are unchanged. The strict-xfail markers from the previous commit are deleted here; every demonstration now asserts for real. Verified on 2x H100 (--runxfail, TF32 off): CPU suite 13/13, distributed 11/11 on both ranks, refusal and wiring tests 26 passed, full peft regression 419 passed. At ETP=1 every new branch short-circuits at group size 1; peak memory is unchanged by construction (the embed buffer replaces the removed gather buffer). The fix design originates from Mohammad Taufeeque's earlier implementation of the same defect class. Co-authored-by: Taufeeque <9taufeeque9@gmail.com> Co-Authored-By: Claude Fable 5 Signed-off-by: Tigist Diriba --- src/megatron/bridge/peft/dora.py | 17 +- src/megatron/bridge/peft/lora_layers.py | 13 ++ src/megatron/bridge/peft/multi_lora_layers.py | 32 +++ src/megatron/bridge/peft/utils.py | 217 ++++++++++++++++-- tests/unit_tests/peft/test_dora.py | 16 +- tests/unit_tests/peft/test_expert_lora_etp.py | 50 +--- .../peft/test_expert_lora_etp_distributed.py | 42 +--- .../peft/test_lora_layers_weight_refusal.py | 67 ++++++ tests/unit_tests/peft/test_multi_lora_moe.py | 74 ++++++ tests/unit_tests/peft/test_utils.py | 4 + 10 files changed, 410 insertions(+), 122 deletions(-) create mode 100644 tests/unit_tests/peft/test_lora_layers_weight_refusal.py diff --git a/src/megatron/bridge/peft/dora.py b/src/megatron/bridge/peft/dora.py index 7812542a8b..319449da2f 100644 --- a/src/megatron/bridge/peft/dora.py +++ b/src/megatron/bridge/peft/dora.py @@ -21,7 +21,7 @@ from megatron.bridge.peft.base import PEFT from megatron.bridge.peft.dora_layers import DoRALinear, ParallelLinearDoRAAdapter from megatron.bridge.peft.module_matcher import ModuleMatcher -from megatron.bridge.peft.utils import get_adapter_attributes_from_linear +from megatron.bridge.peft.utils import get_adapter_attributes_from_linear, is_expert_linear logger = logging.getLogger(__name__) @@ -90,7 +90,22 @@ def transform(self, m: nn.Module, name: Optional[str] = None, prefix: Optional[s if (ans := self.match(m, name, prefix)) is not None: _, full_name = ans + if is_expert_linear(full_name): + raise NotImplementedError( + f"DoRA does not support MoE expert linears (matched {full_name}): the DoRA " + "adapter is built without is_expert, so its sharding and collectives do not " + "compose with expert parallelism (TP, ETP, or EP > 1). Exclude expert " + "modules from target_modules to proceed." + ) attrs = get_adapter_attributes_from_linear(m) + if attrs.disable_tensor_parallel_comm and attrs.base_linear_is_parallel: + raise NotImplementedError( + f"DoRA does not support TP-sharded linears whose own tensor-parallel " + f"communication is suppressed (matched {full_name}; e.g. shared experts " + "under moe_shared_expert_overlap): DoRA's per-rank magnitude norm does not " + "compose with the downstream TP reduction. Exclude the module from " + "target_modules. A replicated base (e.g. a duplicated TELinear) is fine." + ) logger.info(f"Adding DoRA to: {full_name}") adapter = ParallelLinearDoRAAdapter( attrs.in_features, diff --git a/src/megatron/bridge/peft/lora_layers.py b/src/megatron/bridge/peft/lora_layers.py index 9b0e00608d..96b09b81f1 100644 --- a/src/megatron/bridge/peft/lora_layers.py +++ b/src/megatron/bridge/peft/lora_layers.py @@ -35,6 +35,19 @@ class to provide a specific implementation of the forward method. @property def weight(self) -> torch.Tensor: """Return the effective base weight, including the LoRA delta when enabled.""" + from megatron.bridge.peft.utils import GroupedExpertLinearAdapter, SharedOuterGroupedExpertAdapter + + if isinstance(self.adapter, (GroupedExpertLinearAdapter, SharedOuterGroupedExpertAdapter)): + # Grouped bases expose weight0..N (no single 2D to_wrap.weight), and these + # adapters carry 3D per-expert weights with no `tp_group` attribute — the + # group-less merge below would silently merge only the local shard. The + # merge path for grouped adapters is the export-side materializer + # (models/conversion/peft_bridge.py), not this property. + raise NotImplementedError( + f"LoRALinear.weight is not supported for grouped expert adapters " + f"({type(self.adapter).__name__} on {self.adapter.base_linear_name}); " + "use the checkpoint/export merge path instead." + ) base_weight = self.to_wrap.weight if not self._adapter_enabled: return base_weight diff --git a/src/megatron/bridge/peft/multi_lora_layers.py b/src/megatron/bridge/peft/multi_lora_layers.py index 693028ddbe..75ba0c2d76 100644 --- a/src/megatron/bridge/peft/multi_lora_layers.py +++ b/src/megatron/bridge/peft/multi_lora_layers.py @@ -115,6 +115,18 @@ def __init__( # column-sharded adapter output must be gathered before the residual # add; a column-parallel base keeps the [tokens, out/tp] shard. self._gather_output = attrs.input_is_parallel or not attrs.base_linear_is_parallel + # Shared-expert fc2 under moe_shared_expert_overlap: the base's own TP + # collectives are suppressed (attrs.disable_tensor_parallel_comm) and + # SharedExpertMLP.post_forward_comm sums the full-width partials across + # the dense TP group. A gathered delta would be counted once per rank; + # zero-embedding this rank's shard instead gives each output element + # exactly one non-zero contributor, so the downstream sum counts it once. + # Mirrors ParallelLinearAdapter._shared_expert_row_parallel. + self._suppressed_comm_row_parallel = bool( + attrs.input_is_parallel and attrs.disable_tensor_parallel_comm and attrs.base_linear_is_parallel + ) + if self._suppressed_comm_row_parallel: + self._gather_output = False # ModuleList of ParallelLinearAdapters gives per-adapter optimizer state # isolation, clean checkpoint serialization, and bridge export compatibility. @@ -155,6 +167,21 @@ def __init__( "rank_values", torch.full((n_adapters,), dim, dtype=dtype, device=device), persistent=False ) + def _embed_row_parallel_shard(self, out: torch.Tensor) -> torch.Tensor: + """Zero-embed this rank's output shard at its dense-TP offset. + + Padding is differentiable and its adjoint is the matching slice, so the + downstream ``post_forward_comm`` sum reproduces the delta exactly once with + no ``1/tp`` factor and no gradient compensation. + """ + tp_size = parallel_state.get_tensor_model_parallel_world_size() + if tp_size <= 1: + return out + rank = parallel_state.get_tensor_model_parallel_rank() + shard_width = out.shape[-1] + left = rank * shard_width + return nn.functional.pad(out, (left, (tp_size - 1) * shard_width - left)) + def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: linear_output, bias, layernorm_output = self.base_linear_forward(x, *args, **kwargs) @@ -226,8 +253,13 @@ def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> Tuple[torch.Ten # Match the wrapped base linear's output layout: row-parallel base # produces a fully-summed [tokens, h_out] tensor (which we then SP # scatter); column-parallel base keeps the [tokens, h_out/tp] shard. + # A suppressed-comm row-parallel base (shared experts under overlap) + # emits a full-width partial that post_forward_comm sums across TP, so + # the delta must be contributed as a zero-embedded local shard instead. if self._gather_output: out = gather_from_tensor_model_parallel_region(out) + elif self._suppressed_comm_row_parallel: + out = self._embed_row_parallel_shard(out) if not self.disable_sequence_parallel_comm and self.input_is_parallel: if self.use_a2a: diff --git a/src/megatron/bridge/peft/utils.py b/src/megatron/bridge/peft/utils.py index 0aa0496a39..ad9d39d4b4 100644 --- a/src/megatron/bridge/peft/utils.py +++ b/src/megatron/bridge/peft/utils.py @@ -34,7 +34,10 @@ from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel import ColumnParallelLinear, RowParallelLinear, set_tensor_model_parallel_attributes from megatron.core.tensor_parallel.mappings import ( + all_gather_last_dim_from_tensor_parallel_region, + copy_to_tensor_model_parallel_region, gather_from_sequence_parallel_region, + reduce_from_tensor_model_parallel_region, scatter_to_sequence_parallel_region, ) from megatron.core.tensor_parallel.random import is_checkpointing @@ -1022,6 +1025,18 @@ def __init__( self.tp_group = _get_tensor_parallel_group(self.pg_collection, is_expert=is_expert) self.ep_group = _get_process_group(self.pg_collection, "ep") self.expert_dp_group = _get_process_group(self.pg_collection, "expt_dp") + if ( + is_expert + and dropout > 0.0 + and _process_group_size(self.tp_group, model_parallel_config.expert_tensor_parallel_size or 1) > 1 + ): + raise NotImplementedError( + f"Expert-adapter dropout is not supported at expert_tensor_parallel_size > 1 " + f"(got dropout={dropout} for {base_linear_name}): dropout draws per-rank RNG on " + "tokens the MoE dispatcher replicates across the expert tensor parallel group, so " + "the resulting delta is not expressible by any merged weight. Route dropout " + "through the expert-parallel RNG tracker or set dropout=0.0." + ) _sequence_parallel = model_parallel_config.sequence_parallel model_parallel_config.sequence_parallel = False # SP is irrelevant for the lora linear layer self.config = model_parallel_config @@ -1071,6 +1086,24 @@ def __init__( if not base_linear_is_parallel: lin_out_gather_output = True + # An expert linear whose input is already sharded (experts.linear_fc2) emits a + # full-width partial that the MoE token dispatcher sums across the expert-TP + # group. The adapter must contribute a partial term of that sum, not the full delta: keep linear_out's + # local shard and zero-embed it into full width in forward(), so each output + # element has exactly one non-zero contributor and the dispatcher's sum + # reconstructs B @ z exactly once. Gathering here instead would hand every + # rank the same full-width delta, and the sum would count it once per rank. + self._expert_row_parallel = bool(is_expert and input_is_parallel and base_linear_is_parallel) + # Shared-expert fc2 under moe_shared_expert_overlap is the same disease over + # the DENSE TP group: the base's own collectives are suppressed (which is what + # disable_tensor_parallel_comm reports) and SharedExpertMLP.post_forward_comm + # sums the full-width partials across TP. + self._shared_expert_row_parallel = bool( + not is_expert and disable_tensor_parallel_comm and input_is_parallel and base_linear_is_parallel + ) + if self._expert_row_parallel or self._shared_expert_row_parallel: + lin_out_gather_output = False + self.linear_out = ColumnParallelLinear( dim, out_features, @@ -1205,6 +1238,93 @@ def _get_init_fn(self, init_method: str) -> Callable[[torch.Tensor], torch.Tenso raise NotImplementedError("out_init_method should be zero, normal, kaiming or xavier") return init_fn + def _reduce_expert_low_rank_activation(self, x: torch.Tensor) -> torch.Tensor: + """Complete ``A @ h`` across the expert-tensor-parallel group in both directions. + + ``explicit_expert_comm`` (true for an ``is_expert`` linear when its TP group has + more than one rank or expert model parallelism is on) suppresses an expert + linear's own collectives because, for the base layers, the MoE token dispatcher + owns that communication. The adapter is not routed through the dispatcher the + same way, so two different completions are needed: + + - ``input_is_parallel`` (experts.linear_fc2): ``linear_in`` is row-parallel and + its output all-reduce is suppressed, so ``A_r @ h_r`` is a per-rank partial + that is never summed. ``copy_to`` is forward-identity with a backward + all-reduce; ``reduce_from`` is a forward all-reduce with backward identity. + Composing them gives an all-reduce on both passes, the adjoint pair for a + sum whose result every rank consumes differently. + - otherwise (experts.linear_fc1): ``linear_in`` is column-parallel and its + ``gather_output`` all-gather is not suppressed, so the forward is already + right — but the gather's adjoint is a split, and ``explicit_expert_comm`` + also forces ``allreduce_dgrad=False`` on the adapter's ``linear_out``, which + is what a dense column-parallel layer relies on to sum ``dL/dz`` across + ranks. ``copy_to``'s backward all-reduce restores exactly that sum. + + A bare ``torch.distributed`` collective would not work here: it is invisible + to autograd. The group must be passed explicitly — MCore's default resolves + the dense TP group, not this adapter's expert-TP group. + + Args: + x: Low-rank activation emitted by ``linear_in``. + + Returns: + The activation with expert-TP arithmetic completed; ``x`` unchanged for + non-expert adapters or a one-rank expert-TP group. + """ + if not self.is_expert: + return x + etp_size = _process_group_size(self.tp_group, self.config.expert_tensor_parallel_size or 1) + if etp_size <= 1: + return x + if self.tp_group is None: + raise ValueError( + f"{self.base_linear_name} requires initialized expert tensor parallel state " + f"when expert_tensor_parallel_size={etp_size}." + ) + if self.input_is_parallel: + return reduce_from_tensor_model_parallel_region( + copy_to_tensor_model_parallel_region(x, group=self.tp_group), group=self.tp_group + ) + return copy_to_tensor_model_parallel_region(x, group=self.tp_group) + + def _embed_row_parallel_shard(self, x: torch.Tensor) -> torch.Tensor: + """Place this rank's ``linear_out`` shard into a full-width buffer of zeros. + + The base emits a full-width partial and a downstream sum reduces those across + a group — the MoE token dispatcher over expert-TP for routed experts, and + ``SharedExpertMLP.post_forward_comm`` over the dense TP group for shared-expert + fc2 under ``moe_shared_expert_overlap`` — so the adapter must contribute a partial + term of that sum, not the full delta. Zero-embedding rather than gathering means each output element + has exactly one non-zero contributor, so the downstream sum reproduces + ``B @ z`` exactly once with no ``1/size`` factor. Padding is differentiable + and its adjoint is the matching slice, so ``dL/dB_r`` needs no compensation. + + Args: + x: This rank's ``linear_out`` output shard. + + Returns: + ``x`` zero-embedded at this rank's offset in the full output width, or + ``x`` unchanged when no downstream sum applies. + """ + if self._expert_row_parallel: + size = _process_group_size(self.tp_group, self.config.expert_tensor_parallel_size or 1) + elif self._shared_expert_row_parallel: + size = _process_group_size(self.tp_group, self.config.tensor_model_parallel_size or 1) + else: + return x + if size <= 1: + return x + if self.tp_group is None: + raise ValueError( + f"{self.base_linear_name} requires an initialized tensor parallel group to " + f"zero-embed its row-parallel LoRA shard (group size {size})." + ) + rank = _process_group_rank(self.tp_group) + shard_width = x.shape[-1] + left = rank * shard_width + right = (size - 1) * shard_width - left + return nn.functional.pad(x, (left, right)) + def forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: """Forward pass of the parallel linear adapter. @@ -1249,12 +1369,16 @@ def forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: # ColumnParallelLinear returns output and bias; adapters do not use the bias. x, _ = self.linear_in(x) + x = self._reduce_expert_low_rank_activation(x) + x = self.activation(x) if self.config.cpu_offloading and self.config.cpu_offloading_activations: x.activation_offloading = True x, _ = self.linear_out(x) + x = self._embed_row_parallel_shard(x) + if not self.disable_sequence_parallel_comm and self.input_is_parallel and not self.is_expert: # for attention_dense and linear_fc2 # layernorm after lora is impacted by sequence parallel, @@ -1852,6 +1976,14 @@ def __init__( self.expert_tp_group, model_parallel_config.expert_tensor_parallel_size or 1, ) + if dropout > 0.0 and expert_tp_size > 1: + raise NotImplementedError( + f"Expert-adapter dropout is not supported at expert_tensor_parallel_size > 1 " + f"(got dropout={dropout} for {base_linear_name}): dropout draws per-rank RNG on " + "tokens the MoE dispatcher replicates across the expert tensor parallel group, so " + "the resulting delta is not expressible by any merged weight. Route dropout " + "through the expert-parallel RNG tracker or set dropout=0.0." + ) tensor_parallel_size = _process_group_size( tensor_parallel_group, getattr(model_parallel_config, "tensor_model_parallel_size", 1) or 1, @@ -1940,25 +2072,55 @@ def _extract_expert_splits(self, args: Tuple, kwargs: Dict) -> List[int]: raise ValueError(f"Expert splits for {self.base_linear_name} must be non-negative, got {splits}") return splits - def _gather_along_last_dim(self, tensor: torch.Tensor) -> torch.Tensor: - """Gather a tensor across expert TP ranks by concatenating its last dimension.""" + def _require_expert_tp_group(self, expert_tp_size: int) -> None: + """Raise when expert-TP arithmetic is required but no group is available.""" - expert_tp_size = _process_group_size(self.expert_tp_group, self.config.expert_tensor_parallel_size or 1) - if expert_tp_size == 1: - return tensor - expert_tp_group = self.expert_tp_group - if expert_tp_group is None: + if self.expert_tp_group is None: raise ValueError( f"{self.base_linear_name} requires initialized expert tensor parallel state " f"when expert_tensor_parallel_size={expert_tp_size}." ) - gathered = [torch.empty_like(tensor) for _ in range(expert_tp_size)] - torch.distributed.all_gather( - gathered, - tensor, - group=expert_tp_group, - ) - return torch.cat(gathered, dim=-1) + + def _complete_low_rank_activation(self, hidden: torch.Tensor) -> torch.Tensor: + """Complete the low-rank activation across expert-TP ranks, visibly to autograd. + + - ``input_is_parallel`` (experts.linear_fc2): ``hidden`` is a per-rank partial + of ``A @ h`` because the base row-parallel reduce is suppressed under + ``explicit_expert_comm``. ``copy_to`` + ``reduce_from`` compose to an + all-reduce in both passes: forward sums the partials; backward sums + ``dL/dz`` across ranks that each consumed the full ``z`` through a + different ``B`` shard. + - otherwise (experts.linear_fc1): ``hidden`` is this rank's slice of the LoRA + rank axis; the all-gather mapping concatenates slices in forward and + reduce-scatters (sums) gradient slices in backward — the exact adjoint. + + A bare ``torch.distributed.all_gather`` into fresh buffers is invisible to + autograd and severs the graph, which is the defect this replaces. + """ + expert_tp_size = _process_group_size(self.expert_tp_group, self.config.expert_tensor_parallel_size or 1) + if expert_tp_size == 1: + return hidden + self._require_expert_tp_group(expert_tp_size) + if self.input_is_parallel: + return reduce_from_tensor_model_parallel_region( + copy_to_tensor_model_parallel_region(hidden, group=self.expert_tp_group), + group=self.expert_tp_group, + ) + return all_gather_last_dim_from_tensor_parallel_region(hidden, group=self.expert_tp_group) + + def _embed_expert_row_parallel_shard(self, expert_output: torch.Tensor) -> torch.Tensor: + """Zero-embed this rank's output shard so the dispatcher's expert-TP sum counts it once.""" + + if not self.input_is_parallel: + return expert_output + expert_tp_size = _process_group_size(self.expert_tp_group, self.config.expert_tensor_parallel_size or 1) + if expert_tp_size == 1: + return expert_output + self._require_expert_tp_group(expert_tp_size) + rank = _process_group_rank(self.expert_tp_group) + shard_width = expert_output.shape[-1] + left = rank * shard_width + return nn.functional.pad(expert_output, (left, (expert_tp_size - 1) * shard_width - left)) def _can_use_grouped_mm(self, x: torch.Tensor) -> bool: """Return whether the grouped GEMM fast path is supported for this input.""" @@ -2241,15 +2403,13 @@ def _forward_per_expert( expert_input.activation_offloading = True hidden = nn.functional.linear(expert_input, linear_in_weight[expert_idx]) - if not self.input_is_parallel: - hidden = self._gather_along_last_dim(hidden) + hidden = self._complete_low_rank_activation(hidden) hidden = self.activation(hidden) if self.config.cpu_offloading and self.config.cpu_offloading_activations: hidden.activation_offloading = True expert_output = nn.functional.linear(hidden, linear_out_weight[expert_idx]) - if self.input_is_parallel: - expert_output = self._gather_along_last_dim(expert_output) + expert_output = self._embed_expert_row_parallel_shard(expert_output) if self.dropout_position == "post": expert_output = self.dropout(expert_output) @@ -2323,9 +2483,12 @@ def forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: active_expert_indices=active_expert_tuple, offs=offs, ) - if not self.input_is_parallel: - hidden = self._gather_along_last_dim(hidden) + hidden = self._complete_low_rank_activation(hidden) hidden = self.activation(hidden) + # grouped_mm requires 16-byte-aligned strides; the completion above may hand + # back a freshly concatenated tensor, so normalize before tagging/projecting. + if not use_te_fp8 and not hidden.is_contiguous(): + hidden = hidden.contiguous() if self.config.cpu_offloading and self.config.cpu_offloading_activations: hidden.activation_offloading = True @@ -2339,8 +2502,7 @@ def forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: active_expert_indices=active_expert_tuple, offs=offs, ) - if self.input_is_parallel: - expert_output = self._gather_along_last_dim(expert_output) + expert_output = self._embed_expert_row_parallel_shard(expert_output) if self.dropout_position == "post": expert_output = self.dropout(expert_output) @@ -2584,6 +2746,17 @@ def __init__( model_parallel_config.perform_initialization = True self.config = model_parallel_config + expert_tp_size = int(model_parallel_config.expert_tensor_parallel_size or 1) + if expert_tp_size > 1: + raise NotImplementedError( + f"experts_shared_outer_loras requires expert_tensor_parallel_size=1 " + f"(got {expert_tp_size}) for {base_linear_name}: the per-expert packed " + "side is not expert-TP sharded, so its grouped GEMM cannot compose with " + "an ETP-sharded base, and the shared side's suppressed collectives have " + "no completion. Use share_expert_adapters=True (the default) or set " + "expert_tensor_parallel_size=1." + ) + # ``input_is_parallel`` selects fc1 (column-parallel base) vs fc2 # (row-parallel base). Mirrors :class:`ParallelLinearAdapter` and # :class:`GroupedExpertLinearAdapter`. diff --git a/tests/unit_tests/peft/test_dora.py b/tests/unit_tests/peft/test_dora.py index ccc4dfc0fb..c0e839c91d 100644 --- a/tests/unit_tests/peft/test_dora.py +++ b/tests/unit_tests/peft/test_dora.py @@ -415,15 +415,9 @@ def __init__(self, to_wrap, adapter): return _FakeDoRALinear - @pytest.mark.xfail( - strict=True, - reason="dora-guard: transform silently builds a dense-group, is_expert=False adapter on an " - "MoE expert linear — its sharding and collectives do not compose with expert " - "parallelism at TP, ETP, or EP > 1", - ) def test_dora_refuses_expert_linears(self): """DoRA must refuse expert linears loudly instead of mis-building an adapter.""" - # Demonstrates the missing guard fail-first; the strict marker is deleted by the fix change. + # Introduced fail-first with a strict xfail marker, removed when the guard landed. module = MockRowParallelLinear(16, 16) module.config = MockModelParallelConfig() dora = DoRA(dim=4, alpha=8) @@ -435,15 +429,9 @@ def test_dora_refuses_expert_linears(self): with pytest.raises(NotImplementedError, match="expert"): dora.transform(module, name="linear_fc2", prefix="decoder.layers.0.mlp.experts") - @pytest.mark.xfail( - strict=True, - reason="dora-guard: a TP-sharded base with suppressed tensor-parallel communication " - "(shared experts under moe_shared_expert_overlap) reaches DoRA unguarded — its " - "per-rank magnitude norm does not compose with the downstream TP reduction", - ) def test_dora_refuses_suppressed_comm_parallel_bases(self): """DoRA must refuse suppressed-comm TP-sharded bases (e.g. shared-expert overlap).""" - # Demonstrates the missing guard fail-first; the strict marker is deleted by the fix change. + # Introduced fail-first with a strict xfail marker, removed when the guard landed. module = MockRowParallelLinear(16, 16) module.config = MockModelParallelConfig() dora = DoRA(dim=4, alpha=8) diff --git a/tests/unit_tests/peft/test_expert_lora_etp.py b/tests/unit_tests/peft/test_expert_lora_etp.py index c73be0d215..1c233d1d59 100644 --- a/tests/unit_tests/peft/test_expert_lora_etp.py +++ b/tests/unit_tests/peft/test_expert_lora_etp.py @@ -14,10 +14,10 @@ """Fail-first wiring and exactness tests for expert LoRA at expert-TP > 1. -Each defect-demonstrating test carries ``@pytest.mark.xfail(strict=True, reason=": ...")`` -and fails against the current adapter code — the marker is the fail-first record. The fix -commit deletes the markers; ``strict=True`` turns the resulting XPASS into a suite failure, so -the fix cannot merge with any demonstration left unfixed. Behavior pins are plain tests. +Each defect-demonstrating test was introduced one commit earlier carrying +``@pytest.mark.xfail(strict=True)`` and failing against the pre-fix adapter code — the +fail-first record lives in that commit. The markers were deleted together with the fix, so +every demonstration now asserts for real. Behavior pins are plain tests throughout. The harness runs the real shipped adapters through the real MCore suppression logic on CPU: a one-rank gloo init plus a mock process group reporting ``size() == 2`` makes MCore's @@ -218,11 +218,6 @@ def _patch_fix_ops_if_present(stack, *, copy_to=None, reduce_from=None, ag_last_ True, True, False, - marks=pytest.mark.xfail( - strict=True, - reason="expert-fc2: linear_out keeps gather_output=True — the gathered delta is " - "summed once per rank by the dispatcher's expert-TP reduce", - ), id="expert-fc2", ), pytest.param( @@ -231,11 +226,6 @@ def _patch_fix_ops_if_present(stack, *, copy_to=None, reduce_from=None, ag_last_ True, True, False, - marks=pytest.mark.xfail( - strict=True, - reason="shared-overlap: linear_out keeps gather_output=True — the gathered " - "delta is summed TP times by SharedExpertMLP.post_forward_comm", - ), id="shared-overlap-fc2", ), pytest.param(False, True, True, False, True, id="dense-fc2-keeps-gather"), @@ -310,12 +300,6 @@ def _expert_fc2_fixtures(): return x, a, b, z_partials, out_slots -@pytest.mark.xfail( - strict=True, - reason="expert-fc2: A@h stays a per-rank partial (row-parallel reduce suppressed under " - "explicit_expert_comm) and the gathered delta is summed once per ETP rank — " - "combined == 2 * cat_s((x_s @ A_s^T) @ B_s^T), not x @ (B@A)^T", -) def test_expert_fc2_adapter_is_exact_under_the_dispatcher_etp_sum(): """Forward exactness of the default expert fc2 adapter under the dispatcher's ETP sum.""" @@ -383,11 +367,6 @@ def test_expert_fc1_forward_and_dL_dB_are_exact(): assert torch.allclose(adapter.linear_out.weight.grad, g[rank].t() @ z, atol=TOL) -@pytest.mark.xfail( - strict=True, - reason="expert-fc1-dgrad: allreduce_dgrad is forced False under explicit_expert_comm, so dL/dA_r keeps " - "only the own-rank term (g_r @ B_r) and loses every cross-rank term", -) def test_expert_fc1_dL_dA_recovers_cross_rank_terms(): x, a, b, g, z_slots = _expert_fc1_fixtures() dz_sum = sum(g[s] @ b[s * OUT_SHARD : (s + 1) * OUT_SHARD, :] for s in range(ETP)) @@ -462,11 +441,6 @@ def fake(gathered, tensor, group=None): return fake -@pytest.mark.xfail( - strict=True, - reason="grouped-autograd: _gather_along_last_dim is a bare torch.distributed.all_gather into empty_like " - "buffers — invisible to autograd, so dL/dA is None on the fc1 path", -) def test_grouped_fc1_gather_severs_dL_dA(): x, a, b = _grouped_fixtures() rank = 0 @@ -499,11 +473,6 @@ def test_grouped_fc1_gather_severs_dL_dA(): assert x.grad is not None -@pytest.mark.xfail( - strict=True, - reason="grouped-autograd: the fc2 path gathers after linear_out through the autograd-invisible " - "all_gather — the adapter output carries no graph at all (dL/dA = dL/dB = 0)", -) def test_grouped_fc2_output_requires_grad(): x, a, b = _grouped_fixtures() rank = 0 @@ -537,11 +506,6 @@ def test_grouped_fc2_output_requires_grad(): assert out.requires_grad -@pytest.mark.xfail( - strict=True, - reason="grouped-fc2: the adapter emits an unsummed-partial delta gathered identically on " - "every rank — the dispatcher's ETP sum yields 2 * cat_s((x_s A_s^T) B_s^T)", -) def test_grouped_fc2_is_exact_under_the_dispatcher_etp_sum(): x, a, b = _grouped_fixtures() outputs = [] @@ -602,12 +566,6 @@ def _build_shared_overlap_adapter(rank): ) -@pytest.mark.xfail( - strict=True, - reason="shared-overlap: the adapter's linear_in reduce completes z, linear_out gathers the full-width " - "delta on every rank, and SharedExpertMLP.post_forward_comm sums it TP times — " - "combined == 2 * (x @ (B@A)^T)", -) def test_shared_expert_fc2_delta_is_counted_once_by_post_forward_comm(): x = _seeded((TOKENS, IN), 0) a = _seeded((DIM, IN), 1) diff --git a/tests/unit_tests/peft/test_expert_lora_etp_distributed.py b/tests/unit_tests/peft/test_expert_lora_etp_distributed.py index 956ba92dc3..379e145783 100644 --- a/tests/unit_tests/peft/test_expert_lora_etp_distributed.py +++ b/tests/unit_tests/peft/test_expert_lora_etp_distributed.py @@ -24,9 +24,9 @@ same defect sites. One topology serves every defect: TP=2 gives the dense group for the shared-expert-overlap case (V5), ETP=2 the expert group for V1/V2/V4, world size 2, DP=1. -Each defect-demonstrating assertion carries ``@pytest.mark.xfail(strict=True)`` — the -fail-first record; the fix commit deletes the markers. All collectives run unconditionally -before any assert so a failing assert cannot desync the two ranks. +Each defect-demonstrating assertion was introduced one commit earlier as a strict xfail +(the fail-first record) and asserts for real now that the fix has landed. All collectives +run unconditionally before any assert so a failing assert cannot desync the two ranks. """ import os @@ -134,11 +134,6 @@ def _copy_expert_shards(adapter, a, b, rank, *, input_is_parallel): @pytest.mark.gpu -@pytest.mark.xfail( - strict=True, - reason="expert-fc2: A@h stays a per-rank partial and the gathered delta is summed once per rank " - "by the dispatcher's expert-TP reduce — real-collective form of the CPU demonstration", -) def test_default_fc2_adapter_delta_matches_merged_reference(pg_collection) -> None: device = torch.device("cuda", torch.cuda.current_device()) rank = parallel_state.get_expert_tensor_parallel_rank() @@ -221,11 +216,6 @@ def test_default_fc1_forward_and_dL_dB_are_exact(pg_collection) -> None: @pytest.mark.gpu -@pytest.mark.xfail( - strict=True, - reason="expert-fc1-dgrad: allreduce_dgrad is forced False under explicit_expert_comm — each rank's " - "dL/dA keeps only its own-rank term (~72% relative error measured on 2xH100)", -) def test_default_fc1_dL_dA_matches_merged_reference(pg_collection) -> None: _, _, _, _, dz_full, da_shards, _, x, _ = _run_fc1(pg_collection) for s in range(_ETP_SIZE): @@ -268,11 +258,6 @@ def _build_grouped(pg_collection, *, input_is_parallel): @pytest.mark.gpu -@pytest.mark.xfail( - strict=True, - reason="grouped-autograd: the real bare torch.distributed.all_gather severs autograd — grouped fc2 " - "adapter output carries no graph (dL/dA = dL/dB = 0, silently)", -) def test_grouped_fc2_output_requires_grad(pg_collection) -> None: adapter, device = _build_grouped(pg_collection, input_is_parallel=True) x = torch.randn(TOKENS, IN_SHARD, device=device) @@ -282,10 +267,6 @@ def test_grouped_fc2_output_requires_grad(pg_collection) -> None: @pytest.mark.gpu -@pytest.mark.xfail( - strict=True, - reason="grouped-autograd: grouped fc1 forward is exact but the gather severs the graph — dL/dA is None", -) def test_grouped_fc1_dL_dA_flows(pg_collection) -> None: adapter, device = _build_grouped(pg_collection, input_is_parallel=False) x = torch.randn(TOKENS, IN, device=device) @@ -306,12 +287,6 @@ def _per_expert_weights(device): @pytest.mark.gpu -@pytest.mark.xfail( - strict=True, - reason="grouped-fc2: the adapter emits an unsummed-partial delta gathered identically on " - "every rank — the real dispatcher-style expert-TP sum yields 2 * cat_s((x_s A_s^T) B_s^T), " - "not the merged reference", -) def test_grouped_fc2_delta_matches_merged_reference(pg_collection) -> None: """Real-collective form of the CPU grouped-fc2 wrong-value demonstration.""" adapter, device = _build_grouped(pg_collection, input_is_parallel=True) @@ -345,12 +320,6 @@ def test_grouped_fc2_delta_matches_merged_reference(pg_collection) -> None: @pytest.mark.gpu -@pytest.mark.xfail( - strict=True, - reason="multi-lora-overlap: the dense wrapper gathers its full-width delta on every " - "rank while the suppressed-comm base's partials are summed downstream across dense TP " - "— the delta is counted exactly TP times", -) def test_multi_lora_shared_expert_fc2_delta_counted_once(pg_collection) -> None: """Real-collective MultiLoRALinear on a base whose own TP comms are suppressed. @@ -456,11 +425,6 @@ def test_grouped_adapter_handles_empty_expert_split(pg_collection) -> None: @pytest.mark.gpu -@pytest.mark.xfail( - strict=True, - reason="shared-overlap: the gathered full-width delta is summed once per dense-TP rank by " - "post_forward_comm — combined comes out exactly 2x the merged reference", -) def test_shared_expert_fc2_delta_counted_once_by_tp_sum(pg_collection) -> None: device = torch.device("cuda", torch.cuda.current_device()) rank = parallel_state.get_tensor_model_parallel_rank() diff --git a/tests/unit_tests/peft/test_lora_layers_weight_refusal.py b/tests/unit_tests/peft/test_lora_layers_weight_refusal.py new file mode 100644 index 0000000000..53acae065b --- /dev/null +++ b/tests/unit_tests/peft/test_lora_layers_weight_refusal.py @@ -0,0 +1,67 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pins for LoRALinear.weight's grouped-adapter refusal. + +Grouped expert adapters have 3D per-expert weights, no single 2D ``to_wrap.weight`` to merge +into, and no ``tp_group`` attribute — before the guard, the property died on an accidental +``AttributeError`` (or worse, a group-less local-shard merge). The refusal makes that loud and +intentional; the export-side materializer remains the merge path for grouped adapters. +""" + +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import pytest +import torch +import torch.nn as nn + +from megatron.bridge.peft.lora_layers import LoRALinear +from megatron.bridge.peft.utils import GroupedExpertLinearAdapter + + +def _wrapper_with_adapter(adapter) -> LoRALinear: + wrapper = object.__new__(LoRALinear) # skip AdapterWrapper.__init__ plumbing + nn.Module.__init__(wrapper) + wrapper.to_wrap = SimpleNamespace(weight=torch.zeros(4, 4)) + wrapper.adapter = adapter + wrapper._adapter_enabled = True + return wrapper + + +def test_weight_property_refuses_grouped_expert_adapters(): + adapter = Mock(spec=GroupedExpertLinearAdapter) + adapter.base_linear_name = "decoder.layers.0.mlp.experts.linear_fc2" + wrapper = _wrapper_with_adapter(adapter) + + with pytest.raises(NotImplementedError, match="grouped expert adapters"): + _ = wrapper.weight + + +def test_weight_property_still_merges_plain_adapters(): + adapter = SimpleNamespace( + linear_in=SimpleNamespace(weight=torch.zeros(2, 4)), + linear_out=SimpleNamespace(weight=torch.zeros(4, 2)), + alpha=4, + dim=2, + tp_group=None, + ) + wrapper = _wrapper_with_adapter(adapter) + + with patch("megatron.bridge.peft.lora_layers.LoRAMerge") as mock_merge: + mock_merge.return_value.merge.return_value = torch.ones(4, 4) + merged = wrapper.weight + + mock_merge.return_value.merge.assert_called_once() + assert torch.equal(merged, torch.ones(4, 4)) diff --git a/tests/unit_tests/peft/test_multi_lora_moe.py b/tests/unit_tests/peft/test_multi_lora_moe.py index 6f5087936f..e8ccb7e80d 100644 --- a/tests/unit_tests/peft/test_multi_lora_moe.py +++ b/tests/unit_tests/peft/test_multi_lora_moe.py @@ -255,6 +255,80 @@ class _AllGatherLike: _co_permute_slot_ids(_AllGatherLike(), torch.zeros(4, dtype=torch.int32), num_local_experts=1) +# ====================================================================== +# CPU: dense wrapper partial-term output form (shared experts under overlap) +# ====================================================================== + + +def _dense_wrapper_with_attrs(*, input_is_parallel, disable_tensor_parallel_comm, base_linear_is_parallel): + from megatron.bridge.peft.multi_lora_layers import MultiLoRALinear + from megatron.bridge.peft.utils import AdapterAttributes + from tests.unit_tests.peft.test_utils import MockModelParallelConfig + + to_wrap = nn.Linear(8, 8) + to_wrap.config = MockModelParallelConfig() + attrs = AdapterAttributes( + input_is_parallel=input_is_parallel, + in_features=8, + out_features=8, + disable_tensor_parallel_comm=disable_tensor_parallel_comm, + disable_sequence_parallel_comm=True, + base_linear_is_parallel=base_linear_is_parallel, + ) + with patch( + "megatron.bridge.peft.multi_lora_layers.get_adapter_attributes_from_linear", + return_value=attrs, + ): + return MultiLoRALinear( + to_wrap=to_wrap, + n_adapters=2, + dim=4, + alpha=4, + full_name="decoder.layers.0.mlp.shared_experts.linear_fc2", + column_init_method="xavier", + row_init_method="zero", + ) + + +def test_suppressed_comm_row_parallel_wrapper_embeds_instead_of_gathering(): + """Shared-expert fc2 under overlap: the delta must be a zero-embedded local shard. + + post_forward_comm sums full-width partials across dense TP; a gathered delta would + be counted once per rank. Pins the predicate, the disabled gather, and the embed + values (each output element has exactly one non-zero contributor). + """ + wrapper = _dense_wrapper_with_attrs( + input_is_parallel=True, disable_tensor_parallel_comm=True, base_linear_is_parallel=True + ) + assert wrapper._suppressed_comm_row_parallel is True + assert wrapper._gather_output is False + + shard = torch.arange(1.0, 13.0).reshape(3, 4) + with ( + patch( + "megatron.bridge.peft.multi_lora_layers.parallel_state.get_tensor_model_parallel_world_size", + return_value=2, + ), + patch( + "megatron.bridge.peft.multi_lora_layers.parallel_state.get_tensor_model_parallel_rank", + return_value=1, + ), + ): + embedded = wrapper._embed_row_parallel_shard(shard) + assert embedded.shape == (3, 8) + assert torch.equal(embedded[:, :4], torch.zeros(3, 4)) + assert torch.equal(embedded[:, 4:], shard) + + +def test_dense_row_parallel_wrapper_still_gathers(): + """Scoping pin: an ordinary row-parallel base (comms not suppressed) keeps the gather.""" + wrapper = _dense_wrapper_with_attrs( + input_is_parallel=True, disable_tensor_parallel_comm=False, base_linear_is_parallel=True + ) + assert wrapper._suppressed_comm_row_parallel is False + assert wrapper._gather_output is True + + # ====================================================================== # CPU: hook installation # ====================================================================== diff --git a/tests/unit_tests/peft/test_utils.py b/tests/unit_tests/peft/test_utils.py index 49d97f0000..ab0299f84e 100644 --- a/tests/unit_tests/peft/test_utils.py +++ b/tests/unit_tests/peft/test_utils.py @@ -850,6 +850,10 @@ def test_parallel_linear_adapter_expert_mode(self, mock_row_linear, mock_col_lin base_linear_name="test", is_expert=True, model_parallel_config=mock_config, + # The expert-TP completion needs a resolvable group at ETP>1; the mock + # group's copy_to completion is a forward identity, so the padded-shape + # arithmetic under test is unchanged. + pg_collection=make_mock_pg_collection(etp_size=4), ) # Test with sequence length that needs padding (7 -> 8 when tensor_model_parallel_size=4)