From fbed210396a2561ff7da9dc522bbac46c4c4d30d Mon Sep 17 00:00:00 2001 From: Mehmet Aslan <286132399+maslan-coop26@users.noreply.github.com> Date: Wed, 21 Oct 2026 22:27:24 -0500 Subject: [PATCH 1/5] [ROCm][DSv4.1] Dispatch pre-quantized MXFP4 through MoRI for the A4W4 MoE With VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1 the DeepSeek-V4.1 experts quantize their activations to MXFP4 themselves, so MoRI dispatches BF16 and every rank a token is routed to quantizes it again on arrival. VLLM_MORI_A4W4_FP4_DISPATCH=1 takes the path #59596 added for mxfp4 w4a4: the owning rank quantizes once in prepare(), MoRI moves packed FP4 with e8m0 block scales (about a quarter of the BF16 bytes), and AITER takes the pre-quantized input, sorting only the scales. AiterExperts.workspace_shapes sizes the output at the unpacked hidden size for this input as well, as it already does for w4a4. Signed-off-by: Mehmet Aslan <286132399+maslan-coop26@users.noreply.github.com> --- vllm/envs.py | 7 +++++++ vllm/model_executor/layers/fused_moe/all2all_utils.py | 7 ++++++- .../layers/fused_moe/experts/rocm_aiter_moe.py | 8 ++++++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/vllm/envs.py b/vllm/envs.py index 01d0e61e0eee..0816ad6f1874 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -141,6 +141,7 @@ VLLM_ROCM_USE_AITER_MOE: bool = True VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0 VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4: bool | None = None + VLLM_MORI_A4W4_FP4_DISPATCH: bool = False VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto" VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_USE_AITER_MLA: bool = True @@ -1294,6 +1295,12 @@ def _resolve_rust_cli_path() -> str | None: "VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4": lambda: maybe_convert_bool( os.getenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4") ), + # With VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1 and MoRI: dispatch the MoE input + # as packed MXFP4 with e8m0 block scales, which the a4w4 experts take + # without quantizing again, instead of BF16 (a quarter of the bytes). + "VLLM_MORI_A4W4_FP4_DISPATCH": lambda: bool( + int(os.getenv("VLLM_MORI_A4W4_FP4_DISPATCH", "0")) + ), # MoE sorting dispatch policy for AITER fused MoE kernels. # 0 = auto (default): single-pass for small batches, multi-pass # for large batches diff --git a/vllm/model_executor/layers/fused_moe/all2all_utils.py b/vllm/model_executor/layers/fused_moe/all2all_utils.py index 875df4320c3e..4e61265fbb90 100644 --- a/vllm/model_executor/layers/fused_moe/all2all_utils.py +++ b/vllm/model_executor/layers/fused_moe/all2all_utils.py @@ -6,6 +6,7 @@ import torch +import vllm.envs as envs from vllm.config import get_current_vllm_config from vllm.distributed import ( GroupCoordinator, @@ -331,7 +332,11 @@ def maybe_make_prepare_finalize( # MXFP4 activations are dispatched as packed FP4 with their e8m0 # block scales, which the experts consume without re-quantizing. - use_fp4_dispatch = quant_config.use_mxfp4_w4a4 + # DeepSeek V4.1 a4w4 runs on a W4A16 quant config with MXFP4 + # activations; opt-in, it takes the same pre-quantized FP4 input. + use_fp4_dispatch = quant_config.use_mxfp4_w4a4 or ( + moe.use_mxfp4_w4a4_dsv4 and envs.VLLM_MORI_A4W4_FP4_DISPATCH + ) # Note: We may want to use FP8 dispatch just to reduce # data movement. use_fp8_dispatch = not use_fp4_dispatch and ( diff --git a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py index 3adf374dae79..fe712eb2a1b9 100644 --- a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py @@ -6,6 +6,7 @@ import torch +import vllm.envs as envs import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm._aiter_ops import rocm_aiter_ops from vllm.model_executor.layers.fused_moe.activation import MoEActivation @@ -552,8 +553,11 @@ def workspace_shapes( workspace2 = (0,) # MoRI dispatches MXFP4 activations packed two per byte, so K is # half the hidden size there. - packed_fp4_input = ( - self.moe_config.use_mori_kernels and self.quant_config.use_mxfp4_w4a4 + packed_fp4_input = self.moe_config.use_mori_kernels and ( + self.quant_config.use_mxfp4_w4a4 + or ( + self.moe_config.use_mxfp4_w4a4_dsv4 and envs.VLLM_MORI_A4W4_FP4_DISPATCH + ) ) output = (M, K * 2 if packed_fp4_input else K) return (workspace1, workspace2, output) From bb3b9f88801b717cbf1b91749d6f9de509910b64 Mon Sep 17 00:00:00 2001 From: Mehmet Aslan <286132399+maslan-coop26@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:32:42 -0500 Subject: [PATCH 2/5] [ROCm][DSv4.1] Test MoRI FP4 dispatch for the A4W4 MoE - test_rocm_aiter_moe.py: MoRI picks FP4 dispatch only for the DSv4.1 A4W4 MoE with VLLM_MORI_A4W4_FP4_DISPATCH=1, and the experts' output width doubles exactly when the input arrives as packed FP4. - test_mori_moe.py: an mxfp4_dsv4 scheme runs 2-rank MoRI with FP4 dispatch against AITER's non-EP A4W4 path, including a CUDA graph replay. Signed-off-by: Mehmet Aslan <286132399+maslan-coop26@users.noreply.github.com> --- tests/kernels/moe/test_mori_moe.py | 37 ++++++--- tests/kernels/moe/test_rocm_aiter_moe.py | 100 +++++++++++++++++++++++ 2 files changed, 126 insertions(+), 11 deletions(-) diff --git a/tests/kernels/moe/test_mori_moe.py b/tests/kernels/moe/test_mori_moe.py index 6cde84da3463..bfe8848013cd 100644 --- a/tests/kernels/moe/test_mori_moe.py +++ b/tests/kernels/moe/test_mori_moe.py @@ -40,13 +40,16 @@ ] -def _mxfp4_aiter_weights(w1, w2, start, end): - """MXFP4-quantize experts [start, end) into AITER's W4A4 kernel layout.""" +def _mxfp4_aiter_weights(w1, w2, start, end, dsv4=False): + """MXFP4-quantize experts [start, end) into AITER's W4A4 kernel layout, or + with dsv4 into DeepSeek V4.1's A4W4 one: a W4A16 quant config whose BF16 + input AITER quantizes to MXFP4, over SEPARATED gate/up weights.""" from aiter.ops.triton.quant import dynamic_mxfp4_quant from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( Mxfp4MoeBackend, convert_gpt_oss_weight_to_mxfp4_moe_kernel_format, + convert_weight_to_mxfp4_moe_kernel_format, make_mxfp4_moe_quant_config, ) @@ -55,11 +58,17 @@ def quantize(w): q, s = dynamic_mxfp4_quant(w.flatten(0, 1)) return q.view(*w.shape[:2], -1), s.view(*w.shape[:2], -1) - backend = Mxfp4MoeBackend.AITER_MXFP4_MXFP4 (q1, s1), (q2, s2) = quantize(w1), quantize(w2) - q1, q2, s1, s2, _, _ = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format( - backend, None, q1, q2, s1, s2 - ) + if dsv4: + backend = Mxfp4MoeBackend.AITER_MXFP4_BF16 + q1, q2, s1, s2, _, _ = convert_weight_to_mxfp4_moe_kernel_format( + backend, None, q1, q2, s1, s2, use_separated_a4w4=True + ) + else: + backend = Mxfp4MoeBackend.AITER_MXFP4_MXFP4 + q1, q2, s1, s2, _, _ = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format( + backend, None, q1, q2, s1, s2 + ) return q1, q2, make_mxfp4_moe_quant_config(backend, s1, s2) @@ -80,7 +89,10 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, scheme, graph): init_workspace_manager(pgi.device) set_random_seed(47) fp8 = scheme == "fp8_token_channel" - mxfp4 = scheme == "mxfp4" + # DeepSeek V4.1 A4W4: VLLM_MORI_A4W4_FP4_DISPATCH=1 dispatches its input + # as packed MXFP4, like the W4A4 path. + dsv4 = scheme == "mxfp4_dsv4" + mxfp4 = scheme == "mxfp4" or dsv4 dtype = current_platform.fp8_dtype() if fp8 else None (_, w1, s1, _), (_, w2, s2, _) = make_test_weights( 32, n, k, quant_dtype=dtype, per_out_ch_quant=fp8 @@ -112,11 +124,12 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, scheme, graph): max_num_tokens=256, ), moe_parallel_config=parallel, + use_mxfp4_w4a4_dsv4=dsv4, ) start, end = pgi.rank * 16, (pgi.rank + 1) * 16 if mxfp4: - local_w1, local_w2, quant = _mxfp4_aiter_weights(w1, w2, start, end) - ref_w1, ref_w2, ref_quant = _mxfp4_aiter_weights(w1, w2, 0, 32) + local_w1, local_w2, quant = _mxfp4_aiter_weights(w1, w2, start, end, dsv4) + ref_w1, ref_w2, ref_quant = _mxfp4_aiter_weights(w1, w2, 0, 32, dsv4) else: quant = FusedMoEQuantConfig.make( dtype, @@ -138,6 +151,7 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, scheme, graph): "bf16": AiterUnquantizedExperts, "fp8_token_channel": AiterFp8ChannelExperts, "mxfp4": AiterMxfp4Experts, + "mxfp4_dsv4": AiterMxfp4Experts, } experts_cls = experts_cls_by_scheme[scheme] experts = experts_cls(config, quant) @@ -225,10 +239,10 @@ def check(actual): not current_platform.is_rocm() or not has_mori(), reason="Requires ROCm MoRI" ) @pytest.mark.parametrize("m,n,k", MNKS) -@pytest.mark.parametrize("scheme", ["bf16", "fp8_token_channel", "mxfp4"]) +@pytest.mark.parametrize("scheme", ["bf16", "fp8_token_channel", "mxfp4", "mxfp4_dsv4"]) @multi_gpu_test(num_gpus=2) def test_mori_moe(m, n, k, scheme): - if scheme == "mxfp4": + if scheme in ("mxfp4", "mxfp4_dsv4"): from vllm.platforms.rocm import on_gfx950 if not on_gfx950(): @@ -245,5 +259,6 @@ def test_mori_moe(m, n, k, scheme): "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0", + "VLLM_MORI_A4W4_FP4_DISPATCH": "1" if scheme == "mxfp4_dsv4" else "0", } parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, scheme, m == 32) diff --git a/tests/kernels/moe/test_rocm_aiter_moe.py b/tests/kernels/moe/test_rocm_aiter_moe.py index de3be5539bce..caeffe889008 100644 --- a/tests/kernels/moe/test_rocm_aiter_moe.py +++ b/tests/kernels/moe/test_rocm_aiter_moe.py @@ -1593,3 +1593,103 @@ def test_aiter_moe_alignment_rule_holds_across_block_m(intermediate, num_tokens) intermediate % alignment == 0 ) assert _aiter_accepts_intermediate(padded, num_tokens) + + +class _MoriMoEConfig: + """FusedMoEConfig stand-in for a MoRI EP layer: unset backend flags are off.""" + + def __init__(self, use_mxfp4_w4a4_dsv4: bool): + import types + + self.moe_parallel_config = types.SimpleNamespace( + use_passthrough_all2all=False, use_all2all_kernels=True + ) + self.use_mori_kernels = True + self.use_mxfp4_w4a4_dsv4 = use_mxfp4_w4a4_dsv4 + self.hidden_dim = 256 + self.max_num_tokens = 8 + self.num_experts = 16 + self.experts_per_token = 2 + self.in_dtype = torch.bfloat16 + + def __getattr__(self, name: str) -> Any: + if name.startswith("use_"): + return False + raise AttributeError(name) + + +# The DeepSeek V4.1 a4w4 MoE opts in; without the flag, or on another quant +# path, MoRI keeps dispatching BF16. +_MORI_FP4_DISPATCH_CASES = [(True, "1", True), (True, "0", False), (False, "1", False)] + + +@pytest.mark.parametrize("dsv4, env_value, fp4", _MORI_FP4_DISPATCH_CASES) +def test_mori_fp4_dispatch_for_dsv4_a4w4( + dsv4: bool, env_value: str, fp4: bool, monkeypatch: pytest.MonkeyPatch +): + """VLLM_MORI_A4W4_FP4_DISPATCH=1 sends the a4w4 input through MoRI as packed + MXFP4 with one e8m0 scale per 32 elements; MoRI still sizes its buffers by + the unpacked hidden size.""" + import types + + import vllm.model_executor.layers.fused_moe.all2all_utils as all2all_utils + + handle_args: dict[str, Any] = {} + made: dict[str, Any] = {} + manager = types.SimpleNamespace( + rank=0, world_size=8, get_handle=lambda args: handle_args.update(args) + ) + monkeypatch.setattr( + all2all_utils, + "MoriPrepareAndFinalize", + lambda handle, **kwargs: made.update(kwargs), + raising=False, + ) + quant_config = types.SimpleNamespace( + use_mxfp4_w4a4=False, + is_per_act_token=False, + is_block_quantized=False, + is_per_tensor=False, + ) + monkeypatch.setenv("VLLM_MORI_A4W4_FP4_DISPATCH", env_value) + _reload_envs() + + all2all_utils.maybe_make_prepare_finalize( + _MoriMoEConfig(dsv4), quant_config, all2all_manager=manager + ) + + assert made["use_fp4_dispatch"] is fp4 + assert made["use_fp8_dispatch"] is False + assert handle_args["token_hidden_size"] == 256 + if fp4: + assert handle_args["quant_dtype"] == torch.float4_e2m1fn_x2 + assert handle_args["scale_dim"] == 256 // 32 + assert handle_args["scale_type_size"] == torch.float8_e8m0fnu.itemsize + else: + assert handle_args["quant_dtype"] == torch.bfloat16 + assert handle_args["scale_dim"] == 0 + + +@pytest.mark.parametrize("dsv4, env_value, packed", _MORI_FP4_DISPATCH_CASES) +def test_aiter_experts_output_width_with_packed_fp4_input( + dsv4: bool, env_value: str, packed: bool, monkeypatch: pytest.MonkeyPatch +): + """A packed MXFP4 input holds two values per byte, so the experts' output is + twice the input's K exactly when MoRI dispatched the a4w4 input as FP4.""" + import types + + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + AiterMxfp4Experts, + ) + + experts = object.__new__(AiterMxfp4Experts) + experts.moe_config = types.SimpleNamespace( + use_mori_kernels=True, use_mxfp4_w4a4_dsv4=dsv4 + ) + experts.quant_config = types.SimpleNamespace(use_mxfp4_w4a4=False) + monkeypatch.setenv("VLLM_MORI_A4W4_FP4_DISPATCH", env_value) + _reload_envs() + + _, _, output = experts.workspace_shapes(5, 512, 128, 2, 16, 2, None, None) + + assert output == (5, 256 if packed else 128) From 4ca3c01e4aec0a9e1a44596211ecbf1b1daed35a Mon Sep 17 00:00:00 2001 From: Mehmet Aslan <286132399+maslan-coop26@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:57:50 -0600 Subject: [PATCH 3/5] [ROCm][DSv4.1] Use use_mori_epv1_kernels in the MoRI FP4 dispatch tests Main split MoRI into EPv1 and EPv2 and renamed use_mori_kernels to use_mori_epv1_kernels. FP4 dispatch is on the EPv1 path, so the test stand-ins set the new name. Signed-off-by: Mehmet Aslan <286132399+maslan-coop26@users.noreply.github.com> --- tests/kernels/moe/test_rocm_aiter_moe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/kernels/moe/test_rocm_aiter_moe.py b/tests/kernels/moe/test_rocm_aiter_moe.py index e43194aca32c..5ce63121dd62 100644 --- a/tests/kernels/moe/test_rocm_aiter_moe.py +++ b/tests/kernels/moe/test_rocm_aiter_moe.py @@ -1678,7 +1678,7 @@ def test_aiter_moe_alignment_rule_holds_across_block_m(intermediate, num_tokens) class _MoriMoEConfig: - """FusedMoEConfig stand-in for a MoRI EP layer: unset backend flags are off.""" + """FusedMoEConfig stand-in for a MoRI EPv1 layer: unset backend flags are off.""" def __init__(self, use_mxfp4_w4a4_dsv4: bool): import types @@ -1686,7 +1686,7 @@ def __init__(self, use_mxfp4_w4a4_dsv4: bool): self.moe_parallel_config = types.SimpleNamespace( use_passthrough_all2all=False, use_all2all_kernels=True ) - self.use_mori_kernels = True + self.use_mori_epv1_kernels = True self.use_mxfp4_w4a4_dsv4 = use_mxfp4_w4a4_dsv4 self.hidden_dim = 256 self.max_num_tokens = 8 @@ -1766,7 +1766,7 @@ def test_aiter_experts_output_width_with_packed_fp4_input( experts = object.__new__(AiterMxfp4Experts) experts.moe_config = types.SimpleNamespace( - use_mori_kernels=True, use_mxfp4_w4a4_dsv4=dsv4 + use_mori_epv1_kernels=True, use_mxfp4_w4a4_dsv4=dsv4 ) experts.quant_config = types.SimpleNamespace(use_mxfp4_w4a4=False) monkeypatch.setenv("VLLM_MORI_A4W4_FP4_DISPATCH", env_value) From fbd8951a9f859976a137702b98a011273dc75dd5 Mon Sep 17 00:00:00 2001 From: Mehmet Aslan <286132399+maslan-coop26@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:11:15 -0600 Subject: [PATCH 4/5] [ROCm][DSv4.1] Fold the MoRI FP4 dispatch flag into VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4 The DeepSeek V4.1 a4w4 MoE now dispatches its input through MoRI EPv1 as packed MXFP4 whenever it runs (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1), with no second flag. The dispatch-side quant (per_1x32_mx_quant_hip) and the in-MoE quant it replaces (fused_dynamic_mx_quant_moe_sort) compute the same e8m0 scales, so this only moves where the FP4 values are produced. FusedMoEConfig.use_mori_fp4_dispatch() is now the one predicate for it, read by both the prepare/finalize selection and AiterExperts.workspace_shapes, so the two cannot disagree on whether the expert input arrives packed. MoRI EPv2 keeps dispatching BF16. Signed-off-by: Mehmet Aslan <286132399+maslan-coop26@users.noreply.github.com> --- tests/kernels/moe/test_mori_moe.py | 4 +- tests/kernels/moe/test_rocm_aiter_moe.py | 57 +++++++++++-------- vllm/envs.py | 11 +--- .../layers/fused_moe/all2all_utils.py | 7 +-- .../model_executor/layers/fused_moe/config.py | 14 +++++ .../fused_moe/experts/rocm_aiter_moe.py | 8 +-- 6 files changed, 54 insertions(+), 47 deletions(-) diff --git a/tests/kernels/moe/test_mori_moe.py b/tests/kernels/moe/test_mori_moe.py index ae108e37395f..e380d0032c52 100644 --- a/tests/kernels/moe/test_mori_moe.py +++ b/tests/kernels/moe/test_mori_moe.py @@ -91,8 +91,7 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, scheme, graph): init_workspace_manager(pgi.device) set_random_seed(47) fp8 = scheme == "fp8_token_channel" - # DeepSeek V4.1 A4W4: VLLM_MORI_A4W4_FP4_DISPATCH=1 dispatches its input - # as packed MXFP4, like the W4A4 path. + # DeepSeek V4.1 A4W4 dispatches its input as packed MXFP4, like W4A4. dsv4 = scheme == "mxfp4_dsv4" mxfp4 = scheme == "mxfp4" or dsv4 dtype = current_platform.fp8_dtype() if fp8 else None @@ -269,6 +268,5 @@ def test_mori_moe(m, n, k, scheme): "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0", - "VLLM_MORI_A4W4_FP4_DISPATCH": "1" if scheme == "mxfp4_dsv4" else "0", } parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, scheme, m == 32) diff --git a/tests/kernels/moe/test_rocm_aiter_moe.py b/tests/kernels/moe/test_rocm_aiter_moe.py index c1282d3454d1..e1a172188861 100644 --- a/tests/kernels/moe/test_rocm_aiter_moe.py +++ b/tests/kernels/moe/test_rocm_aiter_moe.py @@ -23,7 +23,7 @@ import importlib import math import warnings -from typing import Any, NamedTuple +from typing import Any, NamedTuple, cast from unittest import mock import pytest @@ -1892,15 +1892,15 @@ def test_aiter_moe_alignment_rule_holds_across_block_m(intermediate, num_tokens) class _MoriMoEConfig: - """FusedMoEConfig stand-in for a MoRI EPv1 layer: unset backend flags are off.""" + """FusedMoEConfig stand-in for a MoRI layer: unset backend flags are off.""" - def __init__(self, use_mxfp4_w4a4_dsv4: bool): + def __init__(self, use_mxfp4_w4a4_dsv4: bool, epv1: bool = True): import types self.moe_parallel_config = types.SimpleNamespace( use_passthrough_all2all=False, use_all2all_kernels=True ) - self.use_mori_epv1_kernels = True + self.use_mori_epv1_kernels = epv1 self.use_mxfp4_w4a4_dsv4 = use_mxfp4_w4a4_dsv4 self.hidden_dim = 256 self.max_num_tokens = 8 @@ -1908,24 +1908,36 @@ def __init__(self, use_mxfp4_w4a4_dsv4: bool): self.experts_per_token = 2 self.in_dtype = torch.bfloat16 + def use_mori_fp4_dispatch(self, quant_config: Any) -> bool: + from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig + + return FusedMoEConfig.use_mori_fp4_dispatch( + cast(FusedMoEConfig, self), quant_config + ) + def __getattr__(self, name: str) -> Any: if name.startswith("use_"): return False raise AttributeError(name) -# The DeepSeek V4.1 a4w4 MoE opts in; without the flag, or on another quant -# path, MoRI keeps dispatching BF16. -_MORI_FP4_DISPATCH_CASES = [(True, "1", True), (True, "0", False), (False, "1", False)] +# (use_mxfp4_w4a4_dsv4, use_mxfp4_w4a16, fp4): the DeepSeek V4.1 a4w4 MoE runs on +# a W4A16 quant config and dispatches FP4; the plain W4A16 path, or the a4w4 +# flag on another quant path, keeps dispatching BF16. +_MORI_FP4_DISPATCH_CASES = [ + (True, True, True), + (False, True, False), + (True, False, False), +] -@pytest.mark.parametrize("dsv4, env_value, fp4", _MORI_FP4_DISPATCH_CASES) +@pytest.mark.parametrize("dsv4, w4a16, fp4", _MORI_FP4_DISPATCH_CASES) def test_mori_fp4_dispatch_for_dsv4_a4w4( - dsv4: bool, env_value: str, fp4: bool, monkeypatch: pytest.MonkeyPatch + dsv4: bool, w4a16: bool, fp4: bool, monkeypatch: pytest.MonkeyPatch ): - """VLLM_MORI_A4W4_FP4_DISPATCH=1 sends the a4w4 input through MoRI as packed - MXFP4 with one e8m0 scale per 32 elements; MoRI still sizes its buffers by - the unpacked hidden size.""" + """The DeepSeek V4.1 a4w4 MoE (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1) sends its + input through MoRI EPv1 as packed MXFP4 with one e8m0 scale per 32 elements; + MoRI still sizes its buffers by the unpacked hidden size.""" import types import vllm.model_executor.layers.fused_moe.all2all_utils as all2all_utils @@ -1943,12 +1955,11 @@ def test_mori_fp4_dispatch_for_dsv4_a4w4( ) quant_config = types.SimpleNamespace( use_mxfp4_w4a4=False, + use_mxfp4_w4a16=w4a16, is_per_act_token=False, is_block_quantized=False, is_per_tensor=False, ) - monkeypatch.setenv("VLLM_MORI_A4W4_FP4_DISPATCH", env_value) - _reload_envs() all2all_utils.maybe_make_prepare_finalize( _MoriMoEConfig(dsv4), quant_config, all2all_manager=manager @@ -1966,12 +1977,14 @@ def test_mori_fp4_dispatch_for_dsv4_a4w4( assert handle_args["scale_dim"] == 0 -@pytest.mark.parametrize("dsv4, env_value, packed", _MORI_FP4_DISPATCH_CASES) +@pytest.mark.parametrize("epv1", [True, False]) +@pytest.mark.parametrize("dsv4, w4a16, fp4", _MORI_FP4_DISPATCH_CASES) def test_aiter_experts_output_width_with_packed_fp4_input( - dsv4: bool, env_value: str, packed: bool, monkeypatch: pytest.MonkeyPatch + dsv4: bool, w4a16: bool, fp4: bool, epv1: bool ): """A packed MXFP4 input holds two values per byte, so the experts' output is - twice the input's K exactly when MoRI dispatched the a4w4 input as FP4.""" + twice the input's K exactly when MoRI EPv1 dispatched the a4w4 input as FP4. + MoRI EPv2 dispatches it as BF16.""" import types from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( @@ -1979,13 +1992,11 @@ def test_aiter_experts_output_width_with_packed_fp4_input( ) experts = object.__new__(AiterMxfp4Experts) - experts.moe_config = types.SimpleNamespace( - use_mori_epv1_kernels=True, use_mxfp4_w4a4_dsv4=dsv4 + experts.moe_config = _MoriMoEConfig(dsv4, epv1=epv1) + experts.quant_config = types.SimpleNamespace( + use_mxfp4_w4a4=False, use_mxfp4_w4a16=w4a16 ) - experts.quant_config = types.SimpleNamespace(use_mxfp4_w4a4=False) - monkeypatch.setenv("VLLM_MORI_A4W4_FP4_DISPATCH", env_value) - _reload_envs() _, _, output = experts.workspace_shapes(5, 512, 128, 2, 16, 2, None, None) - assert output == (5, 256 if packed else 128) + assert output == (5, 256 if fp4 and epv1 else 128) diff --git a/vllm/envs.py b/vllm/envs.py index 82ac73556535..5c17c3c89ab9 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -148,7 +148,6 @@ VLLM_ROCM_USE_AITER_MOE: bool = True VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0 VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4: bool | None = None - VLLM_MORI_A4W4_FP4_DISPATCH: bool = False VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto" VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_USE_AITER_MLA: bool = True @@ -1376,16 +1375,12 @@ def _resolve_rust_cli_path() -> str | None: # Opt-in switch for a4w4 (FP4 activation) MoE on DeepSeek V4.1, AITER # MXFP4 backend. Default is a8w4 (FP8); set to "1" to enable a4w4 # ("true" is not accepted -- only "0"/"1"). Raises if set for other - # models. + # models. Under MoRI EPv1, a4w4 also dispatches the MoE input as packed + # MXFP4 with e8m0 block scales, which the experts take without + # quantizing again. "VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4": lambda: maybe_convert_bool( os.getenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4") ), - # With VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1 and MoRI: dispatch the MoE input - # as packed MXFP4 with e8m0 block scales, which the a4w4 experts take - # without quantizing again, instead of BF16 (a quarter of the bytes). - "VLLM_MORI_A4W4_FP4_DISPATCH": lambda: bool( - int(os.getenv("VLLM_MORI_A4W4_FP4_DISPATCH", "0")) - ), # MoE sorting dispatch policy for AITER fused MoE kernels. # 0 = auto (default): single-pass for small batches, multi-pass # for large batches diff --git a/vllm/model_executor/layers/fused_moe/all2all_utils.py b/vllm/model_executor/layers/fused_moe/all2all_utils.py index e3192f6f2ca8..59df2b2d6bec 100644 --- a/vllm/model_executor/layers/fused_moe/all2all_utils.py +++ b/vllm/model_executor/layers/fused_moe/all2all_utils.py @@ -6,7 +6,6 @@ import torch -import vllm.envs as envs from vllm.config import get_current_vllm_config from vllm.distributed import ( GroupCoordinator, @@ -366,11 +365,7 @@ def maybe_make_prepare_finalize( # MXFP4 activations are dispatched as packed FP4 with their e8m0 # block scales, which the experts consume without re-quantizing. - # DeepSeek V4.1 a4w4 runs on a W4A16 quant config with MXFP4 - # activations; opt-in, it takes the same pre-quantized FP4 input. - use_fp4_dispatch = quant_config.use_mxfp4_w4a4 or ( - moe.use_mxfp4_w4a4_dsv4 and envs.VLLM_MORI_A4W4_FP4_DISPATCH - ) + use_fp4_dispatch = moe.use_mori_fp4_dispatch(quant_config) # Note: We may want to use FP8 dispatch just to reduce # data movement. use_fp8_dispatch = not use_fp4_dispatch and ( diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index 3b4c53b91574..2286251abb7e 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -1656,6 +1656,20 @@ def use_mori_epv1_kernels(self): def use_mori_epv2_kernels(self): return self.moe_parallel_config.use_mori_epv2_kernels + def use_mori_fp4_dispatch(self, quant_config: FusedMoEQuantConfig) -> bool: + """Whether MoRI EPv1 dispatches the MoE input as packed MXFP4 with e8m0 + block scales, which the experts take without quantizing again. + + Covers MXFP4 W4A4 checkpoints and the DeepSeek V4.1 a4w4 MoE, whose W4A16 + quant config runs MXFP4 activations in AITER. The prepare/finalize + selection and AiterExperts.workspace_shapes both read this; if they + disagreed, the expert output would be allocated at half width. + """ + return self.use_mori_epv1_kernels and ( + quant_config.use_mxfp4_w4a4 + or (quant_config.use_mxfp4_w4a16 and self.use_mxfp4_w4a4_dsv4) + ) + @property def use_fi_nvl_two_sided_kernels(self): return self.moe_parallel_config.use_fi_nvl_two_sided_kernels diff --git a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py index bd7ee7565c31..105226fa6c41 100644 --- a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py @@ -6,7 +6,6 @@ import torch -import vllm.envs as envs import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm._aiter_ops import rocm_aiter_ops from vllm.model_executor.layers.fused_moe.activation import MoEActivation @@ -570,12 +569,7 @@ def workspace_shapes( workspace2 = (0,) # MoRI dispatches MXFP4 activations packed two per byte, so K is # half the hidden size there. - packed_fp4_input = self.moe_config.use_mori_epv1_kernels and ( - self.quant_config.use_mxfp4_w4a4 - or ( - self.moe_config.use_mxfp4_w4a4_dsv4 and envs.VLLM_MORI_A4W4_FP4_DISPATCH - ) - ) + packed_fp4_input = self.moe_config.use_mori_fp4_dispatch(self.quant_config) output = (M, K * 2 if packed_fp4_input else K) return (workspace1, workspace2, output) From 5cd0a05f20d4ee45194b0851559af2a4c28ab431 Mon Sep 17 00:00:00 2001 From: Mehmet Aslan <286132399+maslan-coop26@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:55:44 -0600 Subject: [PATCH 5/5] [ROCm][DSv4.1] Size an idle EP rank's MoE output at the hidden width under FP4 dispatch An EP rank that receives no tokens builds its empty output from the dispatched width through output_hidden_dim, and MoRI's packed MXFP4 dispatch halves that width. AiterExperts now maps it back to the hidden size. MoRI hands every rank its full receive buffer today, so the path is not reached yet. Signed-off-by: Mehmet Aslan <286132399+maslan-coop26@users.noreply.github.com> --- tests/kernels/moe/test_rocm_aiter_moe.py | 24 +++++++++++++++++++ .../fused_moe/experts/rocm_aiter_moe.py | 11 +++++++++ 2 files changed, 35 insertions(+) diff --git a/tests/kernels/moe/test_rocm_aiter_moe.py b/tests/kernels/moe/test_rocm_aiter_moe.py index e1a172188861..391c2b939da7 100644 --- a/tests/kernels/moe/test_rocm_aiter_moe.py +++ b/tests/kernels/moe/test_rocm_aiter_moe.py @@ -2000,3 +2000,27 @@ def test_aiter_experts_output_width_with_packed_fp4_input( _, _, output = experts.workspace_shapes(5, 512, 128, 2, 16, 2, None, None) assert output == (5, 256 if fp4 and epv1 else 128) + + +@pytest.mark.parametrize("epv1", [True, False]) +@pytest.mark.parametrize("dsv4, w4a16, fp4", _MORI_FP4_DISPATCH_CASES) +def test_aiter_experts_output_hidden_dim_with_packed_fp4_input( + dsv4: bool, w4a16: bool, fp4: bool, epv1: bool +): + """An EP rank that receives no tokens sizes its empty output from the + dispatched width, which packed MXFP4 halves; it maps back to the hidden + size. The layer's unpacked hidden states keep their width.""" + import types + + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + AiterMxfp4Experts, + ) + + experts = object.__new__(AiterMxfp4Experts) + experts.moe_config = _MoriMoEConfig(dsv4, epv1=epv1) + experts.quant_config = types.SimpleNamespace( + use_mxfp4_w4a4=False, use_mxfp4_w4a16=w4a16 + ) + + assert experts.output_hidden_dim(128) == (256 if fp4 and epv1 else 128) + assert experts.output_hidden_dim(256) == 256 diff --git a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py index 105226fa6c41..78fdfa815445 100644 --- a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py @@ -573,6 +573,17 @@ def workspace_shapes( output = (M, K * 2 if packed_fp4_input else K) return (workspace1, workspace2, output) + def output_hidden_dim(self, hidden_dim: int) -> int: + # An EP rank that receives no tokens sizes its empty output from the + # dispatched width, which packed MXFP4 halves; the layer's own + # (unpacked) hidden states keep their width. + if ( + self.moe_config.use_mori_fp4_dispatch(self.quant_config) + and hidden_dim * 2 == self.moe_config.hidden_dim + ): + return hidden_dim * 2 + return hidden_dim + def apply( self, output: torch.Tensor,