Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 24 additions & 11 deletions tests/kernels/moe/test_mori_moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,16 @@
]


def _mxfp4_aiter_weights(w1, w2, start, end):
"""MXFP4-quantize experts [start, end) into AITER's W4A4 kernel layout."""
def _mxfp4_aiter_weights(w1, w2, start, end, dsv4=False):
"""MXFP4-quantize experts [start, end) into AITER's W4A4 kernel layout, or
with dsv4 into DeepSeek V4.1's A4W4 one: a W4A16 quant config whose BF16
input AITER quantizes to MXFP4, over SEPARATED gate/up weights."""
from aiter.ops.triton.quant import dynamic_mxfp4_quant

from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_quant_config,
)

Expand All @@ -55,11 +58,17 @@ def quantize(w):
q, s = dynamic_mxfp4_quant(w.flatten(0, 1))
return q.view(*w.shape[:2], -1), s.view(*w.shape[:2], -1)

backend = Mxfp4MoeBackend.AITER_MXFP4_MXFP4
(q1, s1), (q2, s2) = quantize(w1), quantize(w2)
q1, q2, s1, s2, _, _ = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
backend, None, q1, q2, s1, s2
)
if dsv4:
backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
q1, q2, s1, s2, _, _ = convert_weight_to_mxfp4_moe_kernel_format(
backend, None, q1, q2, s1, s2, use_separated_a4w4=True
)
else:
backend = Mxfp4MoeBackend.AITER_MXFP4_MXFP4
q1, q2, s1, s2, _, _ = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
backend, None, q1, q2, s1, s2
)
return q1, q2, make_mxfp4_moe_quant_config(backend, s1, s2)


Expand All @@ -82,7 +91,9 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, scheme, graph):
init_workspace_manager(pgi.device)
set_random_seed(47)
fp8 = scheme == "fp8_token_channel"
mxfp4 = scheme == "mxfp4"
# DeepSeek V4.1 A4W4 dispatches its input as packed MXFP4, like W4A4.
dsv4 = scheme == "mxfp4_dsv4"
mxfp4 = scheme == "mxfp4" or dsv4
dtype = current_platform.fp8_dtype() if fp8 else None
(_, w1, s1, _), (_, w2, s2, _) = make_test_weights(
32, n, k, quant_dtype=dtype, per_out_ch_quant=fp8
Expand Down Expand Up @@ -114,11 +125,12 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, scheme, graph):
max_num_tokens=256,
),
moe_parallel_config=parallel,
use_mxfp4_w4a4_dsv4=dsv4,
)
start, end = pgi.rank * 16, (pgi.rank + 1) * 16
if mxfp4:
local_w1, local_w2, quant = _mxfp4_aiter_weights(w1, w2, start, end)
ref_w1, ref_w2, ref_quant = _mxfp4_aiter_weights(w1, w2, 0, 32)
local_w1, local_w2, quant = _mxfp4_aiter_weights(w1, w2, start, end, dsv4)
ref_w1, ref_w2, ref_quant = _mxfp4_aiter_weights(w1, w2, 0, 32, dsv4)
else:
quant = FusedMoEQuantConfig.make(
dtype,
Expand All @@ -140,6 +152,7 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, scheme, graph):
"bf16": AiterUnquantizedExperts,
"fp8_token_channel": AiterFp8ChannelExperts,
"mxfp4": AiterMxfp4Experts,
"mxfp4_dsv4": AiterMxfp4Experts,
}
experts_cls = experts_cls_by_scheme[scheme]
experts = experts_cls(config, quant)
Expand Down Expand Up @@ -235,10 +248,10 @@ def check(actual):
not current_platform.is_rocm() or not has_mori(), reason="Requires ROCm MoRI"
)
@pytest.mark.parametrize("m,n,k", MNKS)
@pytest.mark.parametrize("scheme", ["bf16", "fp8_token_channel", "mxfp4"])
@pytest.mark.parametrize("scheme", ["bf16", "fp8_token_channel", "mxfp4", "mxfp4_dsv4"])
@multi_gpu_test(num_gpus=2)
def test_mori_moe(m, n, k, scheme):
if scheme == "mxfp4":
if scheme in ("mxfp4", "mxfp4_dsv4"):
from vllm.platforms.rocm import on_gfx950

if not on_gfx950():
Expand Down
137 changes: 136 additions & 1 deletion tests/kernels/moe/test_rocm_aiter_moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import importlib
import math
import warnings
from typing import Any, NamedTuple
from typing import Any, NamedTuple, cast
from unittest import mock

import pytest
Expand Down Expand Up @@ -1889,3 +1889,138 @@ def test_aiter_moe_alignment_rule_holds_across_block_m(intermediate, num_tokens)
intermediate % alignment == 0
)
assert _aiter_accepts_intermediate(padded, num_tokens)


class _MoriMoEConfig:
"""FusedMoEConfig stand-in for a MoRI layer: unset backend flags are off."""

def __init__(self, use_mxfp4_w4a4_dsv4: bool, epv1: bool = True):
import types

self.moe_parallel_config = types.SimpleNamespace(
use_passthrough_all2all=False, use_all2all_kernels=True
)
self.use_mori_epv1_kernels = epv1
self.use_mxfp4_w4a4_dsv4 = use_mxfp4_w4a4_dsv4
self.hidden_dim = 256
self.max_num_tokens = 8
self.num_experts = 16
self.experts_per_token = 2
self.in_dtype = torch.bfloat16

def use_mori_fp4_dispatch(self, quant_config: Any) -> bool:
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig

return FusedMoEConfig.use_mori_fp4_dispatch(
cast(FusedMoEConfig, self), quant_config
)

def __getattr__(self, name: str) -> Any:
if name.startswith("use_"):
return False
raise AttributeError(name)


# (use_mxfp4_w4a4_dsv4, use_mxfp4_w4a16, fp4): the DeepSeek V4.1 a4w4 MoE runs on
# a W4A16 quant config and dispatches FP4; the plain W4A16 path, or the a4w4
# flag on another quant path, keeps dispatching BF16.
_MORI_FP4_DISPATCH_CASES = [
(True, True, True),
(False, True, False),
(True, False, False),
]


@pytest.mark.parametrize("dsv4, w4a16, fp4", _MORI_FP4_DISPATCH_CASES)
def test_mori_fp4_dispatch_for_dsv4_a4w4(
dsv4: bool, w4a16: bool, fp4: bool, monkeypatch: pytest.MonkeyPatch
):
"""The DeepSeek V4.1 a4w4 MoE (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1) sends its
input through MoRI EPv1 as packed MXFP4 with one e8m0 scale per 32 elements;
MoRI still sizes its buffers by the unpacked hidden size."""
import types

import vllm.model_executor.layers.fused_moe.all2all_utils as all2all_utils

handle_args: dict[str, Any] = {}
made: dict[str, Any] = {}
manager = types.SimpleNamespace(
rank=0, world_size=8, get_handle=lambda args: handle_args.update(args)
)
monkeypatch.setattr(
all2all_utils,
"MoriPrepareAndFinalize",
lambda handle, **kwargs: made.update(kwargs),
raising=False,
)
quant_config = types.SimpleNamespace(
use_mxfp4_w4a4=False,
use_mxfp4_w4a16=w4a16,
is_per_act_token=False,
is_block_quantized=False,
is_per_tensor=False,
)

all2all_utils.maybe_make_prepare_finalize(
_MoriMoEConfig(dsv4), quant_config, all2all_manager=manager
)

assert made["use_fp4_dispatch"] is fp4
assert made["use_fp8_dispatch"] is False
assert handle_args["token_hidden_size"] == 256
if fp4:
assert handle_args["quant_dtype"] == torch.float4_e2m1fn_x2
assert handle_args["scale_dim"] == 256 // 32
assert handle_args["scale_type_size"] == torch.float8_e8m0fnu.itemsize
else:
assert handle_args["quant_dtype"] == torch.bfloat16
assert handle_args["scale_dim"] == 0


@pytest.mark.parametrize("epv1", [True, False])
@pytest.mark.parametrize("dsv4, w4a16, fp4", _MORI_FP4_DISPATCH_CASES)
def test_aiter_experts_output_width_with_packed_fp4_input(
dsv4: bool, w4a16: bool, fp4: bool, epv1: bool
):
"""A packed MXFP4 input holds two values per byte, so the experts' output is
twice the input's K exactly when MoRI EPv1 dispatched the a4w4 input as FP4.
MoRI EPv2 dispatches it as BF16."""
import types

from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterMxfp4Experts,
)

experts = object.__new__(AiterMxfp4Experts)
experts.moe_config = _MoriMoEConfig(dsv4, epv1=epv1)
experts.quant_config = types.SimpleNamespace(
use_mxfp4_w4a4=False, use_mxfp4_w4a16=w4a16
)

_, _, output = experts.workspace_shapes(5, 512, 128, 2, 16, 2, None, None)

assert output == (5, 256 if fp4 and epv1 else 128)


@pytest.mark.parametrize("epv1", [True, False])
@pytest.mark.parametrize("dsv4, w4a16, fp4", _MORI_FP4_DISPATCH_CASES)
def test_aiter_experts_output_hidden_dim_with_packed_fp4_input(
dsv4: bool, w4a16: bool, fp4: bool, epv1: bool
):
"""An EP rank that receives no tokens sizes its empty output from the
dispatched width, which packed MXFP4 halves; it maps back to the hidden
size. The layer's unpacked hidden states keep their width."""
import types

from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterMxfp4Experts,
)

experts = object.__new__(AiterMxfp4Experts)
experts.moe_config = _MoriMoEConfig(dsv4, epv1=epv1)
experts.quant_config = types.SimpleNamespace(
use_mxfp4_w4a4=False, use_mxfp4_w4a16=w4a16
)

assert experts.output_hidden_dim(128) == (256 if fp4 and epv1 else 128)
assert experts.output_hidden_dim(256) == 256
4 changes: 3 additions & 1 deletion vllm/envs.py
Original file line number Diff line number Diff line change
Expand Up @@ -1402,7 +1402,9 @@ def _resolve_rust_cli_path() -> str | None:
# Opt-in switch for a4w4 (FP4 activation) MoE on DeepSeek V4.1, AITER
# MXFP4 backend. Default is a8w4 (FP8); set to "1" to enable a4w4
# ("true" is not accepted -- only "0"/"1"). Raises if set for other
# models.
# models. Under MoRI EPv1, a4w4 also dispatches the MoE input as packed
# MXFP4 with e8m0 block scales, which the experts take without
# quantizing again.
"VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4": lambda: maybe_convert_bool(
os.getenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4")
),
Expand Down
2 changes: 1 addition & 1 deletion vllm/model_executor/layers/fused_moe/all2all_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -361,7 +361,7 @@ def maybe_make_prepare_finalize(

# MXFP4 activations are dispatched as packed FP4 with their e8m0
# block scales, which the experts consume without re-quantizing.
use_fp4_dispatch = quant_config.use_mxfp4_w4a4
use_fp4_dispatch = moe.use_mori_fp4_dispatch(quant_config)
# Note: We may want to use FP8 dispatch just to reduce
# data movement.
use_fp8_dispatch = not use_fp4_dispatch and (
Expand Down
14 changes: 14 additions & 0 deletions vllm/model_executor/layers/fused_moe/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -1656,6 +1656,20 @@ def use_mori_epv1_kernels(self):
def use_mori_epv2_kernels(self):
return self.moe_parallel_config.use_mori_epv2_kernels

def use_mori_fp4_dispatch(self, quant_config: FusedMoEQuantConfig) -> bool:
"""Whether MoRI EPv1 dispatches the MoE input as packed MXFP4 with e8m0
block scales, which the experts take without quantizing again.

Covers MXFP4 W4A4 checkpoints and the DeepSeek V4.1 a4w4 MoE, whose W4A16
quant config runs MXFP4 activations in AITER. The prepare/finalize
selection and AiterExperts.workspace_shapes both read this; if they
disagreed, the expert output would be allocated at half width.
"""
return self.use_mori_epv1_kernels and (
quant_config.use_mxfp4_w4a4
or (quant_config.use_mxfp4_w4a16 and self.use_mxfp4_w4a4_dsv4)
)

@property
def use_fi_nvl_two_sided_kernels(self):
return self.moe_parallel_config.use_fi_nvl_two_sided_kernels
Expand Down
15 changes: 12 additions & 3 deletions vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -569,12 +569,21 @@ def workspace_shapes(
workspace2 = (0,)
# MoRI dispatches MXFP4 activations packed two per byte, so K is
# half the hidden size there.
packed_fp4_input = (
self.moe_config.use_mori_epv1_kernels and self.quant_config.use_mxfp4_w4a4
)
packed_fp4_input = self.moe_config.use_mori_fp4_dispatch(self.quant_config)
output = (M, K * 2 if packed_fp4_input else K)
return (workspace1, workspace2, output)

def output_hidden_dim(self, hidden_dim: int) -> int:
# An EP rank that receives no tokens sizes its empty output from the
# dispatched width, which packed MXFP4 halves; the layer's own
# (unpacked) hidden states keep their width.
if (
self.moe_config.use_mori_fp4_dispatch(self.quant_config)
and hidden_dim * 2 == self.moe_config.hidden_dim
):
return hidden_dim * 2
return hidden_dim

def apply(
self,
output: torch.Tensor,
Expand Down
Loading