From c37e4aa85131ed1d75f138032bf7257975cde284 Mon Sep 17 00:00:00 2001 From: ChengYM5685 Date: Tue, 10 Nov 2026 06:06:08 +0000 Subject: [PATCH 1/3] [MORI] Run MORI EPv2 across hosts (opt-in) MORI EPv2 required every expert-parallel rank in one CCO-LSA team, so EP across hosts fell back to EPv1, and VLLM_MORI_EP_V2=1 failed at startup. MORI's hip backend has an internode path: ranks in the same LSA team are reached over LSA, the others over GDA RDMA; it is selected by `gpu_per_node < world_size`. MoriEPv2All2AllManager now accepts equal LSA teams that tile the EP group and builds its ops with `gpu_per_node` and the hip backend, sizing their arena with MORI's `internode_regions`. The prepare/finalize is unchanged: the internode path dispatches unquantized BF16 and combines by gather, as EPv2 does on one host. Across hosts EPv1 stays the default; VLLM_MORI_EP_V2=1 selects EPv2 when the installed MoRI has the internode path. Signed-off-by: ChengYM5685 --- docs/serving/expert_parallel_deployment.md | 8 +-- .../moe/test_mori_epv2_prepare_finalize.py | 53 +++++++++++++++-- vllm/config/parallel.py | 12 +++- .../device_communicators/all2all.py | 59 +++++++++++++++++-- vllm/utils/import_utils.py | 11 ++++ 5 files changed, 126 insertions(+), 17 deletions(-) diff --git a/docs/serving/expert_parallel_deployment.md b/docs/serving/expert_parallel_deployment.md index 6cb110199d6d..292953f81f8f 100644 --- a/docs/serving/expert_parallel_deployment.md +++ b/docs/serving/expert_parallel_deployment.md @@ -112,8 +112,8 @@ vllm serve deepseek-ai/DeepSeek-V4-Pro \ | `VLLM_MORI_EP_V2` | Behavior | | ----------------- | -------- | -| unset | EPv2, falling back to EPv1 with a warning when MoRI is older than 1.2.4, the GPU is not gfx950 or gfx1250, EP spans hosts (except on gfx1250, whose UALink team can span hosts), `MORI_ENABLE_SDMA=1`, `--moe-backend aiter_mega_moe`, the model is not BF16, or MoE activations use per-tensor FP8 | -| `1` | Always EPv2. The checks above only log a warning; startup fails if the EP ranks are not in one CCO-LSA team | +| unset | EPv2, falling back to EPv1 with a warning when MoRI is older than 1.2.4, the GPU is not gfx950 or gfx1250, EP spans hosts (except on gfx1250, whose UALink team can span hosts; across hosts EPv2 is opt-in), `MORI_ENABLE_SDMA=1`, `--moe-backend aiter_mega_moe`, the model is not BF16, or MoE activations use per-tensor FP8 | +| `1` | Always EPv2. The checks above only log a warning. Across hosts EPv2 runs its internode path (see below); startup fails if the EP ranks do not split into equal CCO-LSA teams | | `0` | EPv1 | On EPv1, `mori` uses the IntraNode kernel within one node. Across nodes it uses InterNodeV1LL when the per-rank token budget (`--max-num-batched-tokens`) is at most `VLLM_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD` (default 256), and InterNodeV1 otherwise. `mori_high_throughput` and `mori_low_latency` always run EPv1 and pin the inter-node kernel to InterNodeV1 and InterNodeV1LL respectively. @@ -124,13 +124,13 @@ Choosing a MoRI backend: use EPv2 wherever it applies, and EPv1 for deployments | ---------- | ------- | | EP within one node (or one gfx1250 UALink fabric), BF16 model, AITER fused MoE | `mori` (runs EPv2) | | EP within one node, with experts that take activations quantized before dispatch (e.g. FP8 or INT8 Triton experts) | EPv2 rejects these experts at startup. Enable AITER fused MoE (`VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1`) so `mori` runs EPv2 with AITER experts; MoRI EPv1 (`VLLM_MORI_EP_V2=0`) also needs AITER fused MoE. To keep Triton experts without AITER, use a non-MoRI backend such as `allgather_reducescatter` | -| EP across nodes | `mori` (falls back to EPv1; picks InterNodeV1LL when the per-rank token budget is at most `VLLM_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD`, else InterNodeV1) | +| EP across nodes | `mori` (falls back to EPv1; picks InterNodeV1LL when the per-rank token budget is at most `VLLM_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD`, else InterNodeV1). `VLLM_MORI_EP_V2=1` runs EPv2's internode path instead | Within one node, all EPv1 options run the same IntraNode kernel. To pin the inter-node kernel, set the threshold: `0` always picks InterNodeV1, and a value at least the per-rank token budget always picks InterNodeV1LL. `mori_high_throughput` and `mori_low_latency` do the same pinning but are deprecated and will be removed in a future release; use `mori` instead. EPv2 limitations: -- All EP ranks must be in one MoRI CCO-LSA team, i.e. GPUs that can load and store each other's memory: one node, or one UALink fabric. Startup fails otherwise; there is no multi-node RDMA path yet. +- Within one MoRI CCO-LSA team (GPUs that can load and store each other's memory: one node, or one UALink fabric) EPv2 moves tokens through the shared window. Across nodes, with `VLLM_MORI_EP_V2=1`, it runs MoRI's internode path: ranks in the same team are reached as before and the others over GDA RDMA. The EP ranks must split into equal teams (e.g. 2 nodes x 8 GPUs), and MoRI must ship `mori.ops.dispatch_combine_v2.internode_regions`. - Activations are dispatched in BF16, so the model dtype must be BF16 and the experts must take unquantized inputs (AITER fused MoE, which quantizes after the all-to-all, or BF16 experts). Experts that expect activations quantized before dispatch, such as FP8 or INT8 Triton experts, are rejected at startup. Block FP8 and MXFP4 weights work; per-tensor FP8 activations (`--quantization fp8` without blocks, ModelOpt FP8) do not, since AITER cannot quantize the received rows per tensor. - AITER shared-expert fusion is not supported: set `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0`. - Install a MoRI build that ships `mori.ops.dispatch_combine_v2`. On NICs that limit a registered memory region to 2 GiB, the window registration fails once the dispatch arena outgrows that limit (DeepSeek-V4-Pro at EP8 already does at the default 16k batched tokens). diff --git a/tests/kernels/moe/test_mori_epv2_prepare_finalize.py b/tests/kernels/moe/test_mori_epv2_prepare_finalize.py index 577ac1a5da26..0f8919a97642 100644 --- a/tests/kernels/moe/test_mori_epv2_prepare_finalize.py +++ b/tests/kernels/moe/test_mori_epv2_prepare_finalize.py @@ -144,8 +144,16 @@ def test_padding_tokens_are_routed_to_local_experts(monkeypatch): assert op.sent_ids.tolist() == [[0, 7], [2, 3]] -@pytest.mark.parametrize("world_size,lsa_size", [(8, 8), (8, 4)]) -def test_manager_requires_one_lsa_team(monkeypatch, world_size, lsa_size): +@pytest.mark.parametrize( + "world_size,lsa_size,gpu_per_node", + [ + (8, 8, None), + # EP over two hosts: internode, with equal LSA teams. + (16, 8, 8), + (8, 3, "error"), + ], +) +def test_manager_lsa_teams(monkeypatch, world_size, lsa_size, gpu_per_node): from vllm.distributed.device_communicators import all2all from vllm.distributed.device_communicators.base_device_communicator import ( All2AllManagerBase, @@ -177,8 +185,8 @@ def test_manager_requires_one_lsa_team(monkeypatch, world_size, lsa_size): GDA_CONNECTION_NONE=0, ), ) - if lsa_size != world_size: - with pytest.raises(ValueError, match="one LSA team"): + if gpu_per_node == "error": + with pytest.raises(ValueError, match="equal LSA teams"): all2all.MoriEPv2All2AllManager(object()) else: cfg = dict( @@ -192,16 +200,34 @@ def test_manager_requires_one_lsa_team(monkeypatch, world_size, lsa_size): combine_dtype=None, schedule=None, ) + configs = [] + + def make_config(**kw): + config = SimpleNamespace(**(cfg | kw)) + config.is_internode = config.__dict__.get("gpu_per_node") is not None + configs.append(config) + return config + monkeypatch.setitem( sys.modules, "mori.ops.dispatch_combine_v2", SimpleNamespace( - EpDispatchCombineConfig=lambda **kw: SimpleNamespace(**(cfg | kw)), + EpDispatchCombineConfig=make_config, EpDispatchCombineOp=lambda config, comm: calls.op, ), ) + monkeypatch.setitem( + sys.modules, + "mori.ops.dispatch_combine_v2.internode_regions", + SimpleNamespace(internode_regions=lambda config: [("a", 1), ("b", 300)]), + ) manager = all2all.MoriEPv2All2AllManager(object()) assert manager.get_handle({}) is calls.op + (config,) = configs + assert config.__dict__.get("gpu_per_node") == gpu_per_node + assert config.__dict__.get("kernel_backend") == ( + "hip" if gpu_per_node else None + ) # The cache holds ops weakly; the manager keeps them alive itself. manager.handle_cache._cache.clear() manager.destroy() @@ -345,8 +371,17 @@ def _quark(qscheme): "parallel,model,moe_backend,env,ep_v2", [ ({}, {}, "aiter", {}, True), - # EP across hosts: DP over two nodes, even with ep_size <= 8. + # EP across hosts: DP over two nodes, even with ep_size <= 8. MORI's + # internode path is opt-in. ({"data_parallel_size": 16}, {}, "aiter", {}, False), + ({"data_parallel_size": 16}, {}, "aiter", {"VLLM_MORI_EP_V2": "1"}, True), + ( + {"data_parallel_size": 16, "internode_reason": "no internode path"}, + {}, + "aiter", + {}, + False, + ), ({"data_parallel_size_local": 4}, {}, "aiter", {}, False), # A DP replica (TP/PP) spanning nodes. ({"nnodes_within_dp": 2}, {}, "aiter", {}, False), @@ -392,9 +427,15 @@ def test_mori_picks_epv2_where_supported( sdma_picks_epv1 = parallel.pop("sdma_picks_epv1", True) validated_arch = parallel.pop("validated_arch", True) mori_reason = parallel.pop("mori_reason", None) + internode_reason = parallel.pop("internode_reason", None) monkeypatch.setattr( import_utils, "mori_epv2_unavailable_reason", lambda: mori_reason ) + monkeypatch.setattr( + import_utils, + "mori_epv2_internode_unavailable_reason", + lambda: internode_reason, + ) parallel_config = _parallel_config(**parallel) model_config = _model_config(**model) resolve_mori_all2all( diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index 7151625278c2..c702bc06d7bf 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -1403,7 +1403,17 @@ def _mori_epv2_unsupported_reason( # A guess before startup; MoriEPv2All2AllManager checks the actual # CCO-LSA team. if not lsa_spans_hosts and _ep_spans_hosts(parallel_config, ep_size): - return f"ep_size={ep_size} across hosts (EPv2 needs one host)" + from vllm.utils.import_utils import mori_epv2_internode_unavailable_reason + + internode_reason = mori_epv2_internode_unavailable_reason() + if internode_reason is not None: + return f"ep_size={ep_size} across hosts ({internode_reason})" + # The internode path is new: keep EPv1 across hosts unless asked for. + if not envs.VLLM_MORI_EP_V2: + return ( + f"ep_size={ep_size} across hosts by default (its internode path " + "is opt-in: set VLLM_MORI_EP_V2=1)" + ) sdma = os.environ.get("MORI_ENABLE_SDMA", "0").lower() in ("1", "true") if sdma and sdma_picks_epv1: return "MORI_ENABLE_SDMA=1 (SDMA is EPv1-only)" diff --git a/vllm/distributed/device_communicators/all2all.py b/vllm/distributed/device_communicators/all2all.py index 703ab206ded0..577afdd4db92 100644 --- a/vllm/distributed/device_communicators/all2all.py +++ b/vllm/distributed/device_communicators/all2all.py @@ -1079,6 +1079,18 @@ def _mori_epv2_arena_bytes(cfg) -> int: return cfg.effective_max_recv * per_recv_row + per_rank + 4 + 12 * 256 +def _mori_epv2_internode_arena_bytes(cfg) -> int: + """Exact symmetric arena of a MORI EPv2 internode (hip backend) op. + + MORI sizes the internode arena region by region (`internode_regions`) and + carves it out of the window with SymmArena, which aligns every region to + 256 B. + """ + from mori.ops.dispatch_combine_v2.internode_regions import internode_regions + + return sum(round_up(nbytes, 256) for _, nbytes in internode_regions(cfg)) + + # cco reserves each rank's window in 4 GiB steps. _MORI_EPV2_WINDOW_ALIGN = 4 << 30 @@ -1099,6 +1111,9 @@ def __init__(self, cpu_group): # arenas were charged to, so a dropped op is never charged twice. self._ops: list[Any] = [] self.handle_cache = Cache() + # GPUs per CCO-LSA team when the EP group spans several (internode), + # None when it is one team (intranode). + self._gpu_per_node: int | None = None # Open the first window now so a wrong topology fails before the # weights load; ops that need more get their own window later. comm = self._open_window(_MORI_EPV2_WINDOW_ALIGN).comm @@ -1109,15 +1124,31 @@ def __init__(self, cpu_group): lsa_size = device_comm.lsa_size device_comm.close() if lsa_size != self.world_size: - raise ValueError( - "MORI EPv2 requires all expert-parallel ranks in one LSA team, " - f"got LSA size {lsa_size} for {self.world_size} ranks" - ) + # EP spans hosts: MORI's internode path reaches ranks in the + # same LSA team over LSA and the others over GDA RDMA. It needs + # equal LSA teams that tile the EP group. + if lsa_size <= 0 or self.world_size % lsa_size: + raise ValueError( + "MORI EPv2 needs the expert-parallel ranks in one LSA " + "team or in equal LSA teams that tile them, got LSA size " + f"{lsa_size} for {self.world_size} ranks" + ) + self._gpu_per_node = lsa_size except Exception as e: e.add_note("MORI EPv2: probing the CCO-LSA team of the EP group failed") self.destroy() raise - logger.debug("MORI EPv2 all2all over a %d-rank CCO-LSA team", self.world_size) + if self._gpu_per_node is None: + logger.debug( + "MORI EPv2 all2all over a %d-rank CCO-LSA team", self.world_size + ) + else: + logger.info_once( + "MORI EPv2 internode all2all: %d ranks as %d LSA teams of %d", + self.world_size, + self.world_size // self._gpu_per_node, + self._gpu_per_node, + ) def _open_window(self, nbytes: int) -> _MoriEPv2Window: from mori.cco import Communicator @@ -1143,17 +1174,33 @@ def _make_handle(self, **kwargs): EpDispatchCombineOp, ) + internode = {} + if self._gpu_per_node is not None: + # Only MORI's hip backend has the internode path; it combines by + # gather and dispatches unquantized, as this prepare/finalize does. + internode = dict(gpu_per_node=self._gpu_per_node, kernel_backend="hip") config = EpDispatchCombineConfig( rank=self.rank, world_size=self.world_size, data_type=torch.bfloat16, combine_mode="gather", + **internode, **kwargs, ) # Each op carves its arena out of a cco window, which only reserves # address space, so open a big enough one instead of failing. Whole # GiB per op keeps MORI's allocation granularity inside the window. - arena = round_up(_mori_epv2_arena_bytes(config), 1 << 30) + if config.is_internode: + arena_bytes = _mori_epv2_internode_arena_bytes(config) + # The window is registered with the NIC as one memory region, so + # its size matters on NICs that limit a region's page entries. + logger.info_once( + "MORI EPv2 internode op arena: %.2f GiB per rank", + arena_bytes / (1 << 30), + ) + else: + arena_bytes = _mori_epv2_arena_bytes(config) + arena = round_up(arena_bytes, 1 << 30) window = next((w for w in self._windows if w.free_bytes >= arena), None) if window is None: window = self._open_window(round_up(arena, _MORI_EPV2_WINDOW_ALIGN)) diff --git a/vllm/utils/import_utils.py b/vllm/utils/import_utils.py index 83e56445f4cd..499a5f948564 100644 --- a/vllm/utils/import_utils.py +++ b/vllm/utils/import_utils.py @@ -660,6 +660,17 @@ def mori_epv2_unavailable_reason() -> str | None: return None +def mori_epv2_internode_unavailable_reason() -> str | None: + """Why the installed MoRI cannot run EPv2 across hosts, or None if it can. + + The internode path (LSA within a host, GDA RDMA across hosts) lives in + MoRI's hip backend, which sizes its arena with `internode_regions`. + """ + if not _has_module("mori.ops.dispatch_combine_v2.internode_regions"): + return "the installed MoRI has no EPv2 internode path" + return None + + def has_fbgemm_gpu() -> bool: """Whether the optional `fbgemm_gpu` package is available.""" return _has_module("fbgemm_gpu") From f15a31ceafa1868820afb26579675d7bb6df6426 Mon Sep 17 00:00:00 2001 From: ChengYM5685 Date: Tue, 10 Nov 2026 16:16:36 +0000 Subject: [PATCH 2/3] [MORI] Line up ranks before each cco window, size AsyncLL internode arenas Running MORI EPv2 across hosts with GLM-5.3 FP8 (DP16 + EP16) showed three issues in the manager: - An op whose arena outgrows the first window opens another one after weight loading, when hosts arrive tens of seconds apart; cco's P2P handle exchange then fails ("failed to connect to peer 0"). Barrier the EP group before Communicator.init. - An op that sets kernel_backend itself (AsyncLL) collided with the internode defaults; let the op's kwargs win. - An AsyncLL op across hosts was sized with internode_regions, whose send buffers are per node; take MORI's asyncll_regions as its hip backend does. Also drop the claim that a window only reserves address space: across hosts it takes device memory. Signed-off-by: ChengYM5685 --- .../moe/test_mori_epv2_prepare_finalize.py | 105 ++++++++++++++++++ .../device_communicators/all2all.py | 33 ++++-- 2 files changed, 128 insertions(+), 10 deletions(-) diff --git a/tests/kernels/moe/test_mori_epv2_prepare_finalize.py b/tests/kernels/moe/test_mori_epv2_prepare_finalize.py index 0f8919a97642..2a74cac5537c 100644 --- a/tests/kernels/moe/test_mori_epv2_prepare_finalize.py +++ b/tests/kernels/moe/test_mori_epv2_prepare_finalize.py @@ -169,6 +169,7 @@ def test_manager_lsa_teams(monkeypatch, world_size, lsa_size, gpu_per_node): monkeypatch.setattr( torch.distributed, "broadcast_object_list", lambda *a, **k: None ) + monkeypatch.setattr(torch.distributed, "barrier", lambda *a, **k: None) monkeypatch.setattr(all2all, "CudaRTLibrary", Mock) device_comm = Mock(lsa_size=lsa_size) calls = Mock() # records the order of op.close and comm.destroy @@ -238,6 +239,110 @@ def make_config(**kw): comm.destroy.assert_called_once() +def _internode_manager(monkeypatch, order, regions): + """A 16-rank MoriEPv2All2AllManager over two LSA teams of 8, MORI faked.""" + from vllm.distributed.device_communicators import all2all + from vllm.distributed.device_communicators.base_device_communicator import ( + All2AllManagerBase, + ) + + monkeypatch.setattr( + All2AllManagerBase, + "__init__", + lambda self, group: self.__dict__.update( + rank=0, world_size=16, cpu_group=group + ), + ) + monkeypatch.setattr( + torch.distributed, + "broadcast_object_list", + lambda *a, **k: order.append("broadcast"), + ) + monkeypatch.setattr( + torch.distributed, "barrier", lambda *a, **k: order.append("barrier") + ) + monkeypatch.setattr(all2all, "CudaRTLibrary", Mock) + communicator = Mock() + + def init(world_size, rank, uid, per_rank_vmm): + order.append(("init", per_rank_vmm)) + return Mock(create_dev_comm=Mock(return_value=Mock(lsa_size=8))) + + communicator.init.side_effect = init + monkeypatch.setitem( + sys.modules, + "mori.cco", + SimpleNamespace( + Communicator=communicator, + CCODevCommRequirements=SimpleNamespace, + GDA_CONNECTION_NONE=0, + ), + ) + configs = [] + + def make_config(**kw): + config = SimpleNamespace( + dispatch_dtype=None, + combine_dtype=None, + effective_max_recv=0, + schedule=None, + **kw, + ) + config.is_internode = True + configs.append(config) + return config + + monkeypatch.setitem( + sys.modules, + "mori.ops.dispatch_combine_v2", + SimpleNamespace( + EpDispatchCombineConfig=make_config, + EpDispatchCombineOp=lambda config, comm: Mock(), + ), + ) + monkeypatch.setitem( + sys.modules, "mori.ops.dispatch_combine_v2.internode_regions", regions + ) + return all2all.MoriEPv2All2AllManager(object()), configs + + +def test_ranks_line_up_before_each_cco_window(monkeypatch): + order: list = [] + regions = SimpleNamespace(internode_regions=lambda cfg: [("a", 5 << 30)]) + manager, _ = _internode_manager(monkeypatch, order, regions) + # The 5 GiB arena does not fit the first 4 GiB window: a second one opens, + # as it does after weight loading. + manager.get_handle({}) + inits = [i for i, step in enumerate(order) if isinstance(step, tuple)] + assert len(inits) == 2 + for i in inits: + assert order[i - 2 : i] == ["broadcast", "barrier"] + + +def test_internode_op_may_set_its_own_kernel_backend(monkeypatch): + regions = SimpleNamespace(internode_regions=lambda cfg: [("a", 1)]) + manager, configs = _internode_manager(monkeypatch, [], regions) + manager.get_handle({"kernel_backend": "hip"}) + (config,) = configs + assert config.kernel_backend == "hip" and config.gpu_per_node == 8 + + +def test_internode_asyncll_arena_uses_the_asyncll_regions(monkeypatch): + order: list = [] + regions = SimpleNamespace( + internode_regions=lambda cfg: [("a", 1)], + asyncll_regions=lambda cfg: [("a", 5 << 30)], + ) + manager, _ = _internode_manager(monkeypatch, order, regions) + manager.get_handle({"async_ll": True}) + # AsyncLL's per-rank send buffers outgrow the first window; the per-node + # internode table (1 byte here) would have fitted it and overrun. + assert [step for step in order if isinstance(step, tuple)][-1] == ( + "init", + 8 << 30, + ) + + def _unique_id_worker(rank, init_method): from vllm.distributed.device_communicators import all2all from vllm.distributed.device_communicators.base_device_communicator import ( diff --git a/vllm/distributed/device_communicators/all2all.py b/vllm/distributed/device_communicators/all2all.py index 577afdd4db92..912e7b1334b3 100644 --- a/vllm/distributed/device_communicators/all2all.py +++ b/vllm/distributed/device_communicators/all2all.py @@ -1082,13 +1082,20 @@ def _mori_epv2_arena_bytes(cfg) -> int: def _mori_epv2_internode_arena_bytes(cfg) -> int: """Exact symmetric arena of a MORI EPv2 internode (hip backend) op. - MORI sizes the internode arena region by region (`internode_regions`) and - carves it out of the window with SymmArena, which aligns every region to - 256 B. + MORI sizes the internode arena region by region and carves it out of the + window with SymmArena, which aligns every region to 256 B. Like the hip + backend, take the AsyncLL table for an AsyncLL op: its send buffers are + per rank rather than per node, so `internode_regions` would undersize it. """ - from mori.ops.dispatch_combine_v2.internode_regions import internode_regions + if getattr(cfg, "async_ll", False): + from mori.ops.dispatch_combine_v2.internode_regions import asyncll_regions - return sum(round_up(nbytes, 256) for _, nbytes in internode_regions(cfg)) + table = asyncll_regions(cfg) + else: + from mori.ops.dispatch_combine_v2.internode_regions import internode_regions + + table = internode_regions(cfg) + return sum(round_up(nbytes, 256) for _, nbytes in table) # cco reserves each rank's window in 4 GiB steps. @@ -1156,6 +1163,11 @@ def _open_window(self, nbytes: int) -> _MoriEPv2Window: uid = Communicator.get_unique_id() if self.rank == 0 else None values = [uid] dist.broadcast_object_list(values, group_src=0, group=self.cpu_group) + # Line the ranks up first: cco's init exchanges P2P handles with the + # node's other ranks, and a window opened late (after weight loading, + # where hosts finish tens of seconds apart) failed across hosts with + # "failed to connect to peer 0". + dist.barrier(group=self.cpu_group) comm = Communicator.init( self.world_size, self.rank, values[0], per_rank_vmm=nbytes ) @@ -1184,12 +1196,13 @@ def _make_handle(self, **kwargs): world_size=self.world_size, data_type=torch.bfloat16, combine_mode="gather", - **internode, - **kwargs, + # An op's own kwargs win, e.g. a kernel_backend it sets itself. + **(internode | kwargs), ) - # Each op carves its arena out of a cco window, which only reserves - # address space, so open a big enough one instead of failing. Whole - # GiB per op keeps MORI's allocation granularity inside the window. + # Each op carves its arena out of a cco window, so open a big enough + # one instead of failing. The window is backed by memory across hosts, + # so it is sized to what the ops need rather than generously. Whole GiB + # per op keeps MORI's allocation granularity inside the window. if config.is_internode: arena_bytes = _mori_epv2_internode_arena_bytes(config) # The window is registered with the NIC as one memory region, so From 86fb7d1a82ad96fa83c1fcac17790170ee47b260 Mon Sep 17 00:00:00 2001 From: ChengYM5685 Date: Wed, 11 Nov 2026 08:13:15 +0000 Subject: [PATCH 3/3] [MORI] Enforce the EPv2 internode opt-in in the all2all manager The selection before startup only guesses whether EP spans hosts: with external load balancing or Ray and ep_size <= 8 over two hosts, or on gfx1250 when the UALink team does not cover every rank, it picks EPv2 without VLLM_MORI_EP_V2=1 and the manager then ran the internode path by default. The manager, which sees the actual CCO-LSA team, now raises with the opt-in hint unless VLLM_MORI_EP_V2=1 is set, as main fails there. Drop mori_epv2_internode_unavailable_reason: every MoRI that has EPv2 (1.2.4 and later) ships internode_regions, so it could not fire. Signed-off-by: ChengYM5685 --- docs/serving/expert_parallel_deployment.md | 2 +- .../moe/test_mori_epv2_prepare_finalize.py | 39 +++++++++---------- vllm/config/parallel.py | 26 ++++++------- .../device_communicators/all2all.py | 12 +++++- vllm/utils/import_utils.py | 11 ------ 5 files changed, 41 insertions(+), 49 deletions(-) diff --git a/docs/serving/expert_parallel_deployment.md b/docs/serving/expert_parallel_deployment.md index 292953f81f8f..86b4c8c61247 100644 --- a/docs/serving/expert_parallel_deployment.md +++ b/docs/serving/expert_parallel_deployment.md @@ -130,7 +130,7 @@ Within one node, all EPv1 options run the same IntraNode kernel. To pin the inte EPv2 limitations: -- Within one MoRI CCO-LSA team (GPUs that can load and store each other's memory: one node, or one UALink fabric) EPv2 moves tokens through the shared window. Across nodes, with `VLLM_MORI_EP_V2=1`, it runs MoRI's internode path: ranks in the same team are reached as before and the others over GDA RDMA. The EP ranks must split into equal teams (e.g. 2 nodes x 8 GPUs), and MoRI must ship `mori.ops.dispatch_combine_v2.internode_regions`. +- Within one MoRI CCO-LSA team (GPUs that can load and store each other's memory: one node, or one UALink fabric) EPv2 moves tokens through the shared window. Across nodes, with `VLLM_MORI_EP_V2=1`, it runs MoRI's internode path: ranks in the same team are reached as before and the others over GDA RDMA. The EP ranks must split into equal teams (e.g. 2 nodes x 8 GPUs). Where vLLM cannot tell before startup that EP spans hosts (external load balancing, Ray, or a gfx1250 UALink team that does not cover every rank), startup fails unless `VLLM_MORI_EP_V2=1` is set. - Activations are dispatched in BF16, so the model dtype must be BF16 and the experts must take unquantized inputs (AITER fused MoE, which quantizes after the all-to-all, or BF16 experts). Experts that expect activations quantized before dispatch, such as FP8 or INT8 Triton experts, are rejected at startup. Block FP8 and MXFP4 weights work; per-tensor FP8 activations (`--quantization fp8` without blocks, ModelOpt FP8) do not, since AITER cannot quantize the received rows per tensor. - AITER shared-expert fusion is not supported: set `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0`. - Install a MoRI build that ships `mori.ops.dispatch_combine_v2`. On NICs that limit a registered memory region to 2 GiB, the window registration fails once the dispatch arena outgrows that limit (DeepSeek-V4-Pro at EP8 already does at the default 16k batched tokens). diff --git a/tests/kernels/moe/test_mori_epv2_prepare_finalize.py b/tests/kernels/moe/test_mori_epv2_prepare_finalize.py index 2a74cac5537c..d5cdd996833f 100644 --- a/tests/kernels/moe/test_mori_epv2_prepare_finalize.py +++ b/tests/kernels/moe/test_mori_epv2_prepare_finalize.py @@ -145,15 +145,17 @@ def test_padding_tokens_are_routed_to_local_experts(monkeypatch): @pytest.mark.parametrize( - "world_size,lsa_size,gpu_per_node", + "world_size,lsa_size,opt_in,gpu_per_node", [ - (8, 8, None), - # EP over two hosts: internode, with equal LSA teams. - (16, 8, 8), - (8, 3, "error"), + (8, 8, False, None), + # EP over two hosts: internode, with equal LSA teams, when opted in. + (16, 8, True, 8), + # Not opted in: fails as on main, whatever the guess before startup. + (16, 8, False, "opt-in"), + (8, 3, True, "error"), ], ) -def test_manager_lsa_teams(monkeypatch, world_size, lsa_size, gpu_per_node): +def test_manager_lsa_teams(monkeypatch, world_size, lsa_size, opt_in, gpu_per_node): from vllm.distributed.device_communicators import all2all from vllm.distributed.device_communicators.base_device_communicator import ( All2AllManagerBase, @@ -186,7 +188,13 @@ def test_manager_lsa_teams(monkeypatch, world_size, lsa_size, gpu_per_node): GDA_CONNECTION_NONE=0, ), ) - if gpu_per_node == "error": + monkeypatch.delenv("VLLM_MORI_EP_V2", raising=False) + if opt_in: + monkeypatch.setenv("VLLM_MORI_EP_V2", "1") + if gpu_per_node == "opt-in": + with pytest.raises(ValueError, match="opt-in: set VLLM_MORI_EP_V2=1"): + all2all.MoriEPv2All2AllManager(object()) + elif gpu_per_node == "error": with pytest.raises(ValueError, match="equal LSA teams"): all2all.MoriEPv2All2AllManager(object()) else: @@ -240,7 +248,8 @@ def make_config(**kw): def _internode_manager(monkeypatch, order, regions): - """A 16-rank MoriEPv2All2AllManager over two LSA teams of 8, MORI faked.""" + """A 16-rank MoriEPv2All2AllManager over two LSA teams of 8, MORI faked, + with the internode path opted in.""" from vllm.distributed.device_communicators import all2all from vllm.distributed.device_communicators.base_device_communicator import ( All2AllManagerBase, @@ -303,6 +312,7 @@ def make_config(**kw): monkeypatch.setitem( sys.modules, "mori.ops.dispatch_combine_v2.internode_regions", regions ) + monkeypatch.setenv("VLLM_MORI_EP_V2", "1") return all2all.MoriEPv2All2AllManager(object()), configs @@ -480,13 +490,6 @@ def _quark(qscheme): # internode path is opt-in. ({"data_parallel_size": 16}, {}, "aiter", {}, False), ({"data_parallel_size": 16}, {}, "aiter", {"VLLM_MORI_EP_V2": "1"}, True), - ( - {"data_parallel_size": 16, "internode_reason": "no internode path"}, - {}, - "aiter", - {}, - False, - ), ({"data_parallel_size_local": 4}, {}, "aiter", {}, False), # A DP replica (TP/PP) spanning nodes. ({"nnodes_within_dp": 2}, {}, "aiter", {}, False), @@ -532,15 +535,9 @@ def test_mori_picks_epv2_where_supported( sdma_picks_epv1 = parallel.pop("sdma_picks_epv1", True) validated_arch = parallel.pop("validated_arch", True) mori_reason = parallel.pop("mori_reason", None) - internode_reason = parallel.pop("internode_reason", None) monkeypatch.setattr( import_utils, "mori_epv2_unavailable_reason", lambda: mori_reason ) - monkeypatch.setattr( - import_utils, - "mori_epv2_internode_unavailable_reason", - lambda: internode_reason, - ) parallel_config = _parallel_config(**parallel) model_config = _model_config(**model) resolve_mori_all2all( diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index c702bc06d7bf..aaa4e033dad4 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -1400,20 +1400,18 @@ def _mori_epv2_unsupported_reason( * parallel_config.data_parallel_size * parallel_config.prefill_context_parallel_size ) - # A guess before startup; MoriEPv2All2AllManager checks the actual - # CCO-LSA team. - if not lsa_spans_hosts and _ep_spans_hosts(parallel_config, ep_size): - from vllm.utils.import_utils import mori_epv2_internode_unavailable_reason - - internode_reason = mori_epv2_internode_unavailable_reason() - if internode_reason is not None: - return f"ep_size={ep_size} across hosts ({internode_reason})" - # The internode path is new: keep EPv1 across hosts unless asked for. - if not envs.VLLM_MORI_EP_V2: - return ( - f"ep_size={ep_size} across hosts by default (its internode path " - "is opt-in: set VLLM_MORI_EP_V2=1)" - ) + # The internode path is new: keep EPv1 across hosts unless asked for. A + # guess before startup; MoriEPv2All2AllManager checks the actual CCO-LSA + # team and enforces the opt-in there too. + if ( + not envs.VLLM_MORI_EP_V2 + and not lsa_spans_hosts + and _ep_spans_hosts(parallel_config, ep_size) + ): + return ( + f"ep_size={ep_size} across hosts by default (its internode path " + "is opt-in: set VLLM_MORI_EP_V2=1)" + ) sdma = os.environ.get("MORI_ENABLE_SDMA", "0").lower() in ("1", "true") if sdma and sdma_picks_epv1: return "MORI_ENABLE_SDMA=1 (SDMA is EPv1-only)" diff --git a/vllm/distributed/device_communicators/all2all.py b/vllm/distributed/device_communicators/all2all.py index 912e7b1334b3..5a5ec4204f9b 100644 --- a/vllm/distributed/device_communicators/all2all.py +++ b/vllm/distributed/device_communicators/all2all.py @@ -1132,8 +1132,16 @@ def __init__(self, cpu_group): device_comm.close() if lsa_size != self.world_size: # EP spans hosts: MORI's internode path reaches ranks in the - # same LSA team over LSA and the others over GDA RDMA. It needs - # equal LSA teams that tile the EP group. + # same LSA team over LSA and the others over GDA RDMA. It is + # opt-in in every launch mode, whatever the guess before + # startup said, and needs equal LSA teams that tile the EP group. + if not envs.VLLM_MORI_EP_V2: + raise ValueError( + f"MORI EPv2 got LSA size {lsa_size} for {self.world_size} " + "expert-parallel ranks, so EP spans hosts; its internode " + "path is opt-in: set VLLM_MORI_EP_V2=1 to run it, or " + "VLLM_MORI_EP_V2=0 for MORI EPv1" + ) if lsa_size <= 0 or self.world_size % lsa_size: raise ValueError( "MORI EPv2 needs the expert-parallel ranks in one LSA " diff --git a/vllm/utils/import_utils.py b/vllm/utils/import_utils.py index 499a5f948564..83e56445f4cd 100644 --- a/vllm/utils/import_utils.py +++ b/vllm/utils/import_utils.py @@ -660,17 +660,6 @@ def mori_epv2_unavailable_reason() -> str | None: return None -def mori_epv2_internode_unavailable_reason() -> str | None: - """Why the installed MoRI cannot run EPv2 across hosts, or None if it can. - - The internode path (LSA within a host, GDA RDMA across hosts) lives in - MoRI's hip backend, which sizes its arena with `internode_regions`. - """ - if not _has_module("mori.ops.dispatch_combine_v2.internode_regions"): - return "the installed MoRI has no EPv2 internode path" - return None - - def has_fbgemm_gpu() -> bool: """Whether the optional `fbgemm_gpu` package is available.""" return _has_module("fbgemm_gpu")