Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions python/sglang/srt/environ.py
Original file line number Diff line number Diff line change
Expand Up @@ -918,6 +918,8 @@ class Envs:
SGLANG_ROCM_FUSED_DECODE_MLA = EnvBool(False)
SGLANG_ROCM_DISABLE_LINEARQUANT = EnvBool(False)
USE_ROCM_AITER_ROPE_BACKEND = EnvStr("0")
SGLANG_ROCM_USE_AITER_LINEAR_SHUFFLE = EnvBool(False)
SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB = EnvBool(False)
# Enable dual-stream MoE (shared experts vs routed experts) on the
# ROCm/AITER path. Requires GPU_MAX_HW_QUEUES>=5 to avoid HW-queue serialization.
SGLANG_ROCM_USE_MULTI_STREAM = EnvBool(False)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from sglang.srt.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsLinearScheme,
)
from sglang.srt.layers.quantization.compressed_tensors.utils import AiterHipblaslt
from sglang.srt.layers.quantization.fp8_utils import (
apply_fp8_linear,
apply_fp8_ptpc_linear,
Expand Down Expand Up @@ -178,11 +179,30 @@ def process_weights_after_loading(self, layer) -> None:
weight_scale = layer.weight_scale.data

if _use_aiter:
layout = (16, 16)
# keep the weight as (N, K)
layer.weight = Parameter(
shuffle_weight(weight, (16, 16)), requires_grad=False
shuffle_weight(weight, layout), requires_grad=False
)
# FP8HIPB need to keep the weight as (K, N)
if get_bool_env_var("SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB"):
AiterHipblaslt._initialize_hipblaslt()
if AiterHipblaslt.can_shuffle(
weight.shape[0], weight.shape[1], layout
):
layer.weight = Parameter(
shuffle_weight(weight, layout).t().data,
requires_grad=False,
)
# Tag the layout on the parameter so apply time dispatches
# to hipb_mm only for weights actually stored as (K, N).
# Shapes that fail can_shuffle keep the (N, K) aiter
# layout and the untransposed scale, and fall through to
# gemm_a8w8_bpreshuffle.
layer.weight.hipb_swizzled = True
weight_scale = weight_scale.t()
else:
# keep the weight as (K, N)
layer.weight = Parameter(weight.t(), requires_grad=False)

# required by torch.compile to be torch.nn.Parameter
Expand Down Expand Up @@ -236,6 +256,14 @@ def apply_weights(
# x = (fp8_input, per_tensor_input_scale[, orig_dtype]).
# apply_fp8_linear detects the fp8 dtype and skips re-quantizing;
# orig_dtype (when present) sets the GEMM output dtype.
if getattr(layer.weight, "hipb_swizzled", False):
return apply_fp8_ptpc_linear(
input=x,
weight=layer.weight,
weight_scale=layer.weight_scale,
input_scale=layer.input_scale,
bias=bias,
)
qx, x_scale = x[0], x[1]
out_dtype = x[2] if len(x) > 2 else None
return apply_fp8_linear(
Expand Down
98 changes: 98 additions & 0 deletions python/sglang/srt/layers/quantization/compressed_tensors/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,14 @@
from types import MappingProxyType
from typing import Iterable, List, Mapping, Optional

import torch
from compressed_tensors import CompressionFormat
from torch.nn import Module

from sglang.srt.utils import is_hip

_is_hip = is_hip()


def is_activation_quantization_format(format: str) -> bool:
_ACTIVATION_QUANTIZATION_FORMATS = [
Expand Down Expand Up @@ -232,3 +237,96 @@ def _match_fused_layer(
unfused_matches.append(None)

return unfused_matches[0] if all(unfused_matches) else None


class AiterHipblaslt:

@sammysun0711 sammysun0711 Nov 6, 2026 •

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we need cherry pick this PR as is? I see unconditionally call hipblaslt will trigger the decode perf regression.

Can we consider choose aiter.tgemm instead
Example: sammysun0711@ca9c322

Current this example only cover FP8 linear in LLM case, should add BF16 linear in VIT part for compute bound case.

For decode case memory bound case, we can either

cc @apinge, @yicyang

I create a follow up issue: #88

"""
AiterHipblaslt is a class that initializes the hipblaslt extension for aiter.
It also provides a static method to swizzle the unquantized gemm.
"""

_HIPBLASLT_INITIALIZED = False

@classmethod
def _initialize_hipblaslt(cls) -> None:
if not _is_hip:
return
if cls._HIPBLASLT_INITIALIZED:
return
from aiter import hipb_create_extension

hipb_create_extension()
cls._HIPBLASLT_INITIALIZED = True

@staticmethod
def can_shuffle(n: int, k: int, layout: tuple[int, int]) -> bool:
IN, IK = layout
BK = IK * 2
return (n % IN == 0) and (k % BK == 0)

@staticmethod
def hip_bpreshuffle_gemm(
input: torch.Tensor, # [M, K]
weight: torch.Tensor, # [K, N]
bias: torch.Tensor | None = None,
out_dtype: torch.dtype | None = None,
scale_a: torch.Tensor | None = None,
scale_b: torch.Tensor | None = None,
) -> torch.Tensor:
if out_dtype is None:
out_dtype = torch.bfloat16

assert out_dtype == torch.bfloat16, (
f"hip_bpreshuffle_gemm only supports bfloat16 output dtype"
f", you have passed in {out_dtype}"
)
if input.dim() >= 3:
inp_view = input.view(-1, input.size(-1))
batched = True
else:
inp_view = input
batched = False

from aiter import hipb_mm

output = hipb_mm(
inp_view,
weight,
solution_index=-1,
bias=bias,
out_dtype=out_dtype,
scaleA=scale_a,
scaleB=scale_b,
scaleOut=None,
bpreshuffle=True,
)

if batched:
output = output.view(*input.shape[:-1], weight.shape[1])

return output


def rocm_aiter_swizzle_hipb_unquantized_gemm(
x: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None = None,
):
output = AiterHipblaslt.hip_bpreshuffle_gemm(x, weight, bias=bias)
return output


def rocm_aiter_swizzle_hipb_fp8_gemm(
input: torch.Tensor,
weight: torch.Tensor,
scale_a: torch.Tensor | None = None,
scale_b: torch.Tensor | None = None,
bias: torch.Tensor | None = None,
out_dtype: torch.dtype | None = None,
):
output = AiterHipblaslt.hip_bpreshuffle_gemm(
input, weight, out_dtype=out_dtype, scale_a=scale_a, scale_b=scale_b
)
if bias is not None:
output = output + bias
return output
74 changes: 58 additions & 16 deletions python/sglang/srt/layers/quantization/fp8_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
from sglang.srt.environ import envs
from sglang.srt.hardware_backend.npu.utils import is_npu_arch35
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.quantization.compressed_tensors.utils import (
rocm_aiter_swizzle_hipb_fp8_gemm,
)
from sglang.srt.layers.quantization.mxfp4_tensor import MXFP4QuantizeUtil
from sglang.srt.runtime_context import (
get_exec,
Expand Down Expand Up @@ -2355,36 +2358,75 @@ def apply_fp8_ptpc_linear(
compressed_tensor_quant: bool = False,
) -> torch.Tensor:
"""FP8 per-token per-channel linear. Only used with the aiter (ROCm) backend."""
# Set at load time by CompressedTensorsW8A8Fp8.process_weights_after_loading when
# SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB is on AND the shape was swizzleable, in
# which case `weight` is stored (K, N) and `weight_scale` is transposed to match.
# Reading the tag rather than the env var keeps shapes that failed can_shuffle --
# which keep the (N, K) aiter layout -- on the gemm_a8w8_bpreshuffle path.
hipb_swizzled = getattr(weight, "hipb_swizzled", False)

# Handle pre-quantized (fp8_tensor, scale) tuple from fused RMSNorm+Quant
if isinstance(input, tuple):
q_input, x_scale = input
q_input, x_scale = input[0], input[1]
leading_shape = q_input.shape[:-1]
out_dtype = input[2] if len(input) > 2 else torch.bfloat16
q_input = q_input.view(-1, q_input.shape[-1])
output_shape = [*q_input.shape[:-1], weight.shape[0]]
output = aiter.gemm_a8w8_bpreshuffle(
q_input, weight, x_scale, weight_scale, None, torch.bfloat16
)
if hipb_swizzled:
# weight is transposed (K, N)
output_shape = [*leading_shape, weight.shape[1]]
output = rocm_aiter_swizzle_hipb_fp8_gemm(
q_input,
weight,
x_scale,
weight_scale,
None,
out_dtype=out_dtype,
)
else:
# weight is in (N, K)
output_shape = [*leading_shape, weight.shape[0]]
output = aiter.gemm_a8w8_bpreshuffle(
q_input, weight, x_scale, weight_scale, None, out_dtype
)
if bias is not None:
output = output + bias
return output.view(*output_shape)

# View input as 2D matrix for fp8 methods
input_2d = input.view(-1, input.shape[-1])

# weight is transposed (K, N)
output_shape = [*input.shape[:-1], weight.shape[1]]

q_input, x_scale = aiter.per_token_quant_hip(input_2d, quant_dtype=aiter.dtypes.fp8)

per_tensor_weights = (weight_scale.numel() == 1) and weight_scale.dim() < 2
per_tensor_activations = (x_scale.numel() == 1) and x_scale.dim() < 2
if hipb_swizzled:
# weight is transposed (K, N)
output_shape = [*input.shape[:-1], weight.shape[1]]
output = rocm_aiter_swizzle_hipb_fp8_gemm(
q_input,
weight,
x_scale,
weight_scale,
None,
out_dtype=torch.bfloat16 if input_scale is not None else input.dtype,
)
else:
# weight is transposed (K, N)
output_shape = [*input.shape[:-1], weight.shape[1]]

if not (per_tensor_weights and per_tensor_activations):
# weight is in (N, K)
output_shape = [*input.shape[:-1], weight.shape[0]]
per_tensor_weights = (weight_scale.numel() == 1) and weight_scale.dim() < 2
per_tensor_activations = (x_scale.numel() == 1) and x_scale.dim() < 2

output = aiter.gemm_a8w8_bpreshuffle(
q_input, weight, x_scale, weight_scale, None, input.dtype
)
if not (per_tensor_weights and per_tensor_activations):
# weight is in (N, K)
output_shape = [*input.shape[:-1], weight.shape[0]]

output = aiter.gemm_a8w8_bpreshuffle(
q_input,
weight,
x_scale,
weight_scale,
None,
torch.bfloat16 if input_scale is not None else input.dtype,
)
if bias is not None:
output = output + bias
return output.view(*output_shape)
Expand Down
29 changes: 29 additions & 0 deletions python/sglang/srt/layers/quantization/unquant.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,10 @@
from sglang.srt.hardware_backend.npu.quantization.moe_methods import (
NPUUnquantMoEMethod,
)
from sglang.srt.layers.quantization.compressed_tensors.utils import (
AiterHipblaslt,
rocm_aiter_swizzle_hipb_unquantized_gemm,
)

_is_cpu_amx_available = cpu_has_amx_support()
_is_cuda = is_cuda()
Expand Down Expand Up @@ -458,6 +462,23 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if _is_cpu and _is_cpu_amx_available:
_amx_process_weight_after_loading(layer, ["weight"])

if _use_aiter and get_bool_env_var("SGLANG_ROCM_USE_AITER_LINEAR_SHUFFLE"):
AiterHipblaslt._initialize_hipblaslt()
layout = (16, 16)
weight = layer.weight
if AiterHipblaslt.can_shuffle(weight.shape[0], weight.shape[1], layout):
shuffled_weight = shuffle_weight(weight, layout).t()
trans = True
else:
shuffled_weight = weight
trans = False

layer.weight = Parameter(shuffled_weight.data, requires_grad=False)
# Tag the parameter rather than self: a quant-method instance is not
# guaranteed to be per-layer, and apply() must not misread another
# layer's layout.
layer.weight.aiter_trans_weight = trans

def apply(
self,
layer: torch.nn.Module,
Expand All @@ -478,6 +499,11 @@ def apply(
output = output.view(x_shapes[0], x_shapes[1], -1)
return output

elif getattr(layer.weight, "aiter_trans_weight", False):
# Must precede the tgemm.mm branch below, which would otherwise
# swallow every aiter case and leave this path unreachable.
return rocm_aiter_swizzle_hipb_unquantized_gemm(x, layer.weight, bias)

elif _use_aiter and type(layer.weight.data) is torch.Tensor:
return tgemm.mm(x, layer.weight, bias, otype=x.dtype)

Expand Down Expand Up @@ -527,6 +553,9 @@ def apply_into(
bias: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Run an inference-only BF16 linear into caller-owned storage."""
if getattr(layer.weight, "aiter_trans_weight", False):
output.copy_(self.apply(layer, x, bias))
return output
if (
_enable_bf16_splitk_gemm
and bias is None
Expand Down
10 changes: 9 additions & 1 deletion python/sglang/srt/speculative/dflash_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -852,8 +852,16 @@ def can_dflash_slice_qkv_weight(qkv_proj: Any) -> Tuple[bool, str]:
"quantized qkv_proj is not supported for this path "
f"(quant_method={type(quant_method).__name__})",
)
if not hasattr(qkv_proj, "weight"):
weight = getattr(qkv_proj, "weight", None)
if weight is None:
return False, "qkv weight tensor is missing"
# UnquantizedLinearMethod may store an AITER-swizzled [K, N] matrix.
# Both KV-only paths read rows directly, so such weights must instead
# go through the linear layer's layout-aware dispatch.
if getattr(weight, "aiter_trans_weight", False):
return False, "AITER-swizzled qkv weight requires the linear dispatch"
if weight.ndim != 2 or weight.dtype not in _DENSE_HEAD_DTYPES:
return False, "qkv weight is not a dense floating-point matrix"
return True, ""


Expand Down
Loading
Loading