Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 154 additions & 0 deletions tests/compile/passes/test_fuse_mla_dual_rms_norm.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit test for the MLADualRMSNormFusionPass.

The pass fuses paired q/kv RMS norms in MLA attention into a single
fused_mla_dual_rms_norm op backed by AITer's fused_qk_rmsnorm kernel.
"""

import pytest
import torch

import vllm.config
from tests.compile.backend import TestBackend
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
)
from vllm.model_executor.layers.layernorm import RMSNorm

# MLA attention geometry for DeepSeek-V3 / Kimi-K2
Q_DIM = 1536
KV_C_DIM = 512
K_PE_DIM = 64
EPS = 1e-6


class MLADualRMSNormTestModel(torch.nn.Module):
"""
Minimal model reproducing the MLA dual RMS norm pattern:
linear -> split([q_dim, kv_dim])
+-- q_c (getitem 0) -> rms_norm(q_w, eps) -> linear
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
+-- kv_c (getitem 0) -> rms_norm(kv_w, eps)
+-- k_pe
"""

def __init__(
self,
hidden_size: int,
q_dim: int = Q_DIM,
kv_c_dim: int = KV_C_DIM,
k_pe_dim: int = K_PE_DIM,
eps: float = EPS,
):
super().__init__()
self.q_dim = q_dim
self.kv_dim = kv_c_dim + k_pe_dim
self.kv_c_dim = kv_c_dim
self.k_pe_dim = k_pe_dim

self.proj = torch.nn.Linear(
hidden_size, q_dim + self.kv_dim, bias=False
)
self.q_norm = RMSNorm(q_dim, eps=eps)
self.kv_norm = RMSNorm(kv_c_dim, eps=eps)
self.q_b_proj = torch.nn.Linear(q_dim, hidden_size, bias=False)

def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
# Avoid graph input being a direct arg to a matched pattern node
x = torch.relu(x)

projected = self.proj(x)

q_c, kv_lora = projected.split(
[self.q_dim, self.kv_dim], dim=-1
)
kv_c, k_pe = kv_lora.split(
[self.kv_c_dim, self.k_pe_dim], dim=-1
)

q_normed = self.q_norm(q_c)
kv_normed = self.kv_norm(kv_c)

q_out = self.q_b_proj(q_normed)
return q_out, kv_normed, k_pe

def ops_in_model_before(self):
return [torch.ops.vllm_ir.rms_norm.default]

def ops_in_model_after(self):
return [torch.ops.vllm.fused_mla_dual_rms_norm.default]


@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [7168])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
def test_fuse_mla_dual_rms_norm(
dtype: torch.dtype,
hidden_size: int,
monkeypatch: pytest.MonkeyPatch,
):
torch._dynamo.reset()

vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm"],
pass_config=PassConfig(
fuse_mla_dual_rms_norm=True,
eliminate_noops=True,
),
),
)

with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
MLADualRMSNormFusionPass,
)

torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(42)

m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()

fusion_pass = MLADualRMSNormFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
model = MLADualRMSNormTestModel(hidden_size)

x = torch.randn(1, hidden_size)
torch._dynamo.mark_dynamic(x, 0)

outputs_unfused = model(x)

model_fused = torch.compile(model, backend=backend)
outputs_fused = model_fused(x)

torch.testing.assert_close(
outputs_unfused, outputs_fused, atol=1e-2, rtol=1e-2
)

assert fusion_pass.matched_count == 1, (
f"Expected 1 fused pair, got {fusion_pass.matched_count}"
)

backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
42 changes: 42 additions & 0 deletions vllm/_aiter_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -904,6 +904,37 @@ def _rocm_aiter_triton_add_rmsnorm_pad_fake(
return out, residual_out


def _fused_mla_dual_rms_norm_impl(
x1: torch.Tensor,
x1_weight: torch.Tensor,
x2: torch.Tensor,
x2_weight: torch.Tensor,
x1_epsilon: float,
x2_epsilon: float,
) -> tuple[torch.Tensor, torch.Tensor]:
from aiter.ops.fused_qk_norm_rope_cache_quant import fused_qk_rmsnorm

return fused_qk_rmsnorm(
q=x1,
q_weight=x1_weight,
q_eps=x1_epsilon,
k=x2,
k_weight=x2_weight,
k_eps=x2_epsilon,
)


def _fused_mla_dual_rms_norm_fake(
x1: torch.Tensor,
x1_weight: torch.Tensor,
x2: torch.Tensor,
x2_weight: torch.Tensor,
x1_epsilon: float,
x2_epsilon: float,
) -> tuple[torch.Tensor, torch.Tensor]:
return (torch.empty_like(x1), torch.empty_like(x2))


def _rocm_aiter_gemm_a8wfp4_impl(
x: torch.Tensor,
w: torch.Tensor,
Expand Down Expand Up @@ -1429,6 +1460,13 @@ def register_ops_once() -> None:
fake_impl=_triton_rotary_embedding_fake,
)

direct_register_custom_op(
op_name="fused_mla_dual_rms_norm",
op_func=_fused_mla_dual_rms_norm_impl,
mutates_args=[],
fake_impl=_fused_mla_dual_rms_norm_fake,
)

_OPS_REGISTERED = True

@staticmethod
Expand Down Expand Up @@ -1475,6 +1513,10 @@ def get_triton_add_rmsnorm_pad_op() -> OpOverload:
def get_triton_rotary_embedding_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default

@staticmethod
def get_fused_mla_dual_rms_norm_op() -> OpOverload:
return torch.ops.vllm.fused_mla_dual_rms_norm.default

@staticmethod
def rms_norm(
x: torch.Tensor, weight: torch.Tensor, variance_epsilon: float
Expand Down
135 changes: 135 additions & 0 deletions vllm/compilation/passes/fusion/rocm_aiter_fusion.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from torch import fx
from torch._inductor.pattern_matcher import PatternMatcherPass

import vllm.ir.ops
import vllm.model_executor.layers.quantization.utils.fp8_utils # noqa: F401
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
Expand Down Expand Up @@ -512,3 +513,137 @@ def __call__(self, graph: torch.fx.Graph) -> None:

def uuid(self) -> str:
return VllmInductorPass.hash_source(self, AddAiterRMSNormPadPattern)


# ---------------------------------------------------------------------------
Comment thread
rbrugaro-amd marked this conversation as resolved.
Resolved
# MLA Dual RMS Norm Fusion (DeepSeek-V3 / Kimi-K2)
#
# Fuses paired q_a_layernorm + kv_a_layernorm in MLA attention into
# aiter's fused_qk_rmsnorm HIP kernel.
#
# Target FX-graph pattern (unfused, vllm_ir stage):
# gemm -> split_with_sizes([q_dim, kv_dim])
# +-- q_c -> vllm_ir.rms_norm(q_c, q_w, eps)
# +-- kv_lora -> split_with_sizes([kv_c_dim, k_pe_dim])
# +-- kv_c -> vllm_ir.rms_norm(kv_c, kv_w, eps)
# +-- k_pe
# ---------------------------------------------------------------------------

_MLA_DEVICE = torch.device("cuda")


def _mla_empty_bf16(*shape: int) -> torch.Tensor:
Comment thread
rbrugaro-amd marked this conversation as resolved.
Resolved
return torch.empty(*shape, dtype=torch.bfloat16, device=_MLA_DEVICE)


# DeepSeek-V3 / Kimi-K2 MLA geometry

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You don't need exact shapes for your dummy inputs, pattern matching should be shape agnostic (except for the dim0 batch dimension)

_MLA_Q_DIM = 1536
_MLA_KV_C_DIM = 512
_MLA_K_PE_DIM = 64


class MLADualRMSNormPattern:
"""
Match the paired q/kv RMS norm topology in MLA attention and replace
with a single fused_mla_dual_rms_norm call.

The pattern covers the connected subgraph rooted at the first
split_with_sizes (which produces q_c and kv_lora), through the
two rms_norm calls, and the k_pe passthrough.
"""

def __init__(
self,
q_dim: int,
kv_c_dim: int,
k_pe_dim: int,
epsilon: float,
) -> None:
self.q_dim = q_dim
self.kv_dim = kv_c_dim + k_pe_dim
self.kv_c_dim = kv_c_dim
self.k_pe_dim = k_pe_dim
self.epsilon = epsilon

def get_inputs(self) -> list[torch.Tensor]:
T = 5
projected = _mla_empty_bf16(T, self.q_dim + self.kv_dim)
q_weight = _mla_empty_bf16(self.q_dim)
kv_weight = _mla_empty_bf16(self.kv_c_dim)
return [projected, q_weight, kv_weight]

def register(self, pm_pass: PatternMatcherPass) -> None:
q_dim = self.q_dim
kv_dim = self.kv_dim
kv_c_dim = self.kv_c_dim
k_pe_dim = self.k_pe_dim
eps = self.epsilon

def pattern(
projected: torch.Tensor,
q_weight: torch.Tensor,
kv_weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
q_c, kv_lora = projected.split([q_dim, kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([kv_c_dim, k_pe_dim], dim=-1)
q_normed = vllm.ir.ops.rms_norm(q_c, q_weight, eps)
kv_normed = vllm.ir.ops.rms_norm(kv_c, kv_weight, eps)
return q_normed, kv_normed, k_pe

def replacement(
projected: torch.Tensor,
q_weight: torch.Tensor,
kv_weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
q_c, kv_lora = projected.split([q_dim, kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([kv_c_dim, k_pe_dim], dim=-1)
result = torch.ops.vllm.fused_mla_dual_rms_norm(
q_c, q_weight, kv_c, kv_weight, eps, eps,
)
return result[0], result[1], k_pe

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit:

Suggested change
result = torch.ops.vllm.fused_mla_dual_rms_norm(
q_c, q_weight, kv_c, kv_weight, eps, eps,
)
return result[0], result[1], k_pe
q_normed, kv_normed = torch.ops.vllm.fused_mla_dual_rms_norm(
q_c, q_weight, kv_c, kv_weight, eps, eps,
)
return q_normed, kv_normed, k_pe


pm.register_replacement(
pattern,
replacement,
self.get_inputs(),
pm.fwd_only,
pm_pass,
)


class MLADualRMSNormFusionPass(VllmPatternMatcherPass):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's use the new VllmFusionPatternMatcherPass please - will make it cleaner!

"""
Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
MLA attention into ``fused_mla_dual_rms_norm`` backed by aiter's
``fused_qk_rmsnorm`` HIP kernel.
"""

@enable_fake_mode
def __init__(self, config: VllmConfig) -> None:
super().__init__(config)

self.patterns: PatternMatcherPass = PatternMatcherPass(
pass_name="mla_dual_rms_norm_fusion_pass"
)

for epsilon in [1e-5, 1e-6]:
MLADualRMSNormPattern(
q_dim=_MLA_Q_DIM,
kv_c_dim=_MLA_KV_C_DIM,
k_pe_dim=_MLA_K_PE_DIM,
epsilon=epsilon,
).register(self.patterns)

self.dump_patterns(config, self.patterns)

@VllmInductorPass.time_and_log
def __call__(self, graph: fx.Graph) -> None:
self.matched_count = self.patterns.apply(graph)
if self.matched_count > 0:
logger.info(
"MLADualRMSNormFusionPass: fused %d q/kv norm pair(s)",
self.matched_count,
)

def uuid(self) -> str:
return VllmInductorPass.hash_source(self, MLADualRMSNormPattern)
5 changes: 5 additions & 0 deletions vllm/compilation/passes/pass_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

if rocm_aiter_ops.is_enabled():
from .fusion.rocm_aiter_fusion import (
MLADualRMSNormFusionPass,
RocmAiterRMSNormQuantFusionPass,
RocmAiterSiluMulFp8GroupQuantFusionPass,
RocmAiterTritonAddRMSNormPadFusionPass,
Expand Down Expand Up @@ -155,6 +156,10 @@ def configure(self, config: VllmConfig) -> None:
if self.pass_config.fuse_act_padding and rocm_aiter_ops.is_enabled():
self.passes += [RocmAiterTritonAddRMSNormPadFusionPass(config)]

if (self.pass_config.fuse_mla_dual_rms_norm
and rocm_aiter_ops.is_enabled()):
self.passes += [MLADualRMSNormFusionPass(config)]

if self.pass_config.fuse_rope_kvcache:
self.passes += [SplitCoalescingPass(config)]
self.passes += [ScatterSplitReplacementPass(config)]
Expand Down
Loading
Loading