Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 91 additions & 0 deletions python/sglang/kernels/ops/gemm/swizzled_bf16_gemv.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
import torch
import triton
import triton.language as tl


@triton.jit
def _swizzled_gemv(
X,
W,
Y,
M: tl.constexpr,
N: tl.constexpr,
K: tl.constexpr,
BN: tl.constexpr,
BK: tl.constexpr,
STRIDE_Y_M: tl.constexpr,
STRIDE_Y_N: tl.constexpr,
):
row = tl.program_id(0)
cols = tl.program_id(1) * BN + tl.arange(0, BN)
ks = tl.arange(0, BK)
acc = tl.full((BN, BK), 0, tl.float32)
for kb in range(tl.cdiv(K, BK)):
kk = kb * BK + ks
x = tl.load(X + row * K + kk, kk < K, other=0).to(tl.float32)
offset = (
(cols[:, None] // 16) * (K * 16)
+ (kk[None, :] // 32) * 512
+ ((kk[None, :] % 32) // 8) * 128
+ (cols[:, None] % 16) * 8
+ kk[None, :] % 8
)
w = tl.load(W + offset, (cols[:, None] < N) & (kk[None, :] < K), other=0).to(
tl.float32
)
acc += x[None, :] * w
out = tl.sum(acc, axis=1)
tl.store(Y + row * STRIDE_Y_M + cols * STRIDE_Y_N, out, cols < N)


def can_use_swizzled_gemv(x, weight, bias=None):
return (
bias is None
and x.is_cuda
and x.ndim == 2
and x.shape[1] == 4096
and x.dtype == weight.dtype == torch.bfloat16
and x.device == weight.device
and x.is_contiguous()
and not x.requires_grad
and not weight.requires_grad
and weight.shape == (4096, 16)
and weight.stride() == (1, 4096)
and 0 < x.shape[0] <= 32
)


def can_write_swizzled_gemv_output(x, weight, output):
return (
output.ndim == 2
and output.shape == (x.shape[0], weight.shape[1])
and output.dtype == torch.bfloat16
and output.device == x.device
and output.stride(1) > 0
and output.stride(0) >= output.shape[1] * output.stride(1)
and output.untyped_storage().data_ptr()
not in (x.untyped_storage().data_ptr(), weight.untyped_storage().data_ptr())
)


def swizzled_gemv(x, w, bn=4, bk=1024, num_warps=4, output=None):
m, k = x.shape
n = w.shape[1]
if output is None:
output = torch.empty(m, n, device=x.device, dtype=torch.bfloat16)
elif not can_write_swizzled_gemv_output(x, w, output):
raise ValueError("invalid or overlapping swizzled GEMV output buffer")
_swizzled_gemv[(m, triton.cdiv(n, bn))](
x,
w,
output,
m,
n,
k,
bn,
bk,
output.stride(0),
output.stride(1),
num_warps=num_warps,
)
return output
4 changes: 4 additions & 0 deletions python/sglang/srt/environ.py
Original file line number Diff line number Diff line change
Expand Up @@ -918,6 +918,10 @@ class Envs:
SGLANG_ROCM_FUSED_DECODE_MLA = EnvBool(False)
SGLANG_ROCM_DISABLE_LINEARQUANT = EnvBool(False)
USE_ROCM_AITER_ROPE_BACKEND = EnvStr("0")
SGLANG_ROCM_USE_AITER_LINEAR_SHUFFLE = EnvBool(False)
SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB = EnvBool(False)
SGLANG_ROCM_HIPB_DECODE_GEMV = EnvBool(False)
SGLANG_ROCM_HIPB_TUNED_SOLUTIONS = EnvStr("")
# Enable dual-stream MoE (shared experts vs routed experts) on the
# ROCm/AITER path. Requires GPU_MAX_HW_QUEUES>=5 to avoid HW-queue serialization.
SGLANG_ROCM_USE_MULTI_STREAM = EnvBool(False)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from sglang.srt.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsLinearScheme,
)
from sglang.srt.layers.quantization.compressed_tensors.utils import AiterHipblaslt
from sglang.srt.layers.quantization.fp8_utils import (
apply_fp8_linear,
apply_fp8_ptpc_linear,
Expand Down Expand Up @@ -178,11 +179,30 @@ def process_weights_after_loading(self, layer) -> None:
weight_scale = layer.weight_scale.data

if _use_aiter:
layout = (16, 16)
# keep the weight as (N, K)
layer.weight = Parameter(
shuffle_weight(weight, (16, 16)), requires_grad=False
shuffle_weight(weight, layout), requires_grad=False
)
# FP8HIPB need to keep the weight as (K, N)
if get_bool_env_var("SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB"):
AiterHipblaslt._initialize_hipblaslt()
if AiterHipblaslt.can_shuffle(
weight.shape[0], weight.shape[1], layout
):
layer.weight = Parameter(
shuffle_weight(weight, layout).t().data,
requires_grad=False,
)
# Tag the layout on the parameter so apply time dispatches
# to hipb_mm only for weights actually stored as (K, N).
# Shapes that fail can_shuffle keep the (N, K) aiter
# layout and the untransposed scale, and fall through to
# gemm_a8w8_bpreshuffle.
layer.weight.hipb_swizzled = True
weight_scale = weight_scale.t()
else:
# keep the weight as (K, N)
layer.weight = Parameter(weight.t(), requires_grad=False)

# required by torch.compile to be torch.nn.Parameter
Expand Down Expand Up @@ -236,6 +256,14 @@ def apply_weights(
# x = (fp8_input, per_tensor_input_scale[, orig_dtype]).
# apply_fp8_linear detects the fp8 dtype and skips re-quantizing;
# orig_dtype (when present) sets the GEMM output dtype.
if getattr(layer.weight, "hipb_swizzled", False):
return apply_fp8_ptpc_linear(
input=x,
weight=layer.weight,
weight_scale=layer.weight_scale,
input_scale=layer.input_scale,
bias=bias,
)
qx, x_scale = x[0], x[1]
out_dtype = x[2] if len(x) > 2 else None
return apply_fp8_linear(
Expand Down
122 changes: 122 additions & 0 deletions python/sglang/srt/layers/quantization/compressed_tensors/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import json
import re
from functools import lru_cache
from types import MappingProxyType
from typing import Iterable, List, Mapping, Optional

import torch
from compressed_tensors import CompressionFormat
from torch.nn import Module

from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_forward
from sglang.srt.utils import is_hip

_is_hip = is_hip()


def is_activation_quantization_format(format: str) -> bool:
_ACTIVATION_QUANTIZATION_FORMATS = [
Expand Down Expand Up @@ -232,3 +241,116 @@ def _match_fused_layer(
unfused_matches.append(None)

return unfused_matches[0] if all(unfused_matches) else None


@lru_cache(maxsize=1)
def _decode_solutions():
path = envs.SGLANG_ROCM_HIPB_TUNED_SOLUTIONS.get()
if not path:
return {}
with open(path) as source:
return json.load(source)["solutions"]


class AiterHipblaslt:
"""
AiterHipblaslt is a class that initializes the hipblaslt extension for aiter.
It also provides a static method to swizzle the unquantized gemm.
"""

_HIPBLASLT_INITIALIZED = False

@classmethod
def _initialize_hipblaslt(cls) -> None:
if not _is_hip:
return
if cls._HIPBLASLT_INITIALIZED:
return
from aiter import hipb_create_extension

hipb_create_extension()
cls._HIPBLASLT_INITIALIZED = True

@staticmethod
def can_shuffle(n: int, k: int, layout: tuple[int, int]) -> bool:
IN, IK = layout
BK = IK * 2
return (n % IN == 0) and (k % BK == 0)

@staticmethod
def hip_bpreshuffle_gemm(
input: torch.Tensor, # [M, K]
weight: torch.Tensor, # [K, N]
bias: torch.Tensor | None = None,
out_dtype: torch.dtype | None = None,
scale_a: torch.Tensor | None = None,
scale_b: torch.Tensor | None = None,
) -> torch.Tensor:
if out_dtype is None:
out_dtype = torch.bfloat16

assert out_dtype == torch.bfloat16, (
f"hip_bpreshuffle_gemm only supports bfloat16 output dtype"
f", you have passed in {out_dtype}"
)
if input.dim() >= 3:
inp_view = input.view(-1, input.size(-1))
batched = True
else:
inp_view = input
batched = False

from aiter import hipb_mm

solution_index = -1
if (
envs.SGLANG_ROCM_HIPB_DECODE_GEMV.get()
and get_forward().hipb_linear_decode
and inp_view.dtype == torch.bfloat16
and bias is None
and scale_a is None
and scale_b is None
):
key = f"{inp_view.shape[0]},{weight.shape[1]},{inp_view.shape[1]}"
solution_index = _decode_solutions().get(key, -1)
output = hipb_mm(
inp_view,
weight,
solution_index=solution_index,
bias=bias,
out_dtype=out_dtype,
scaleA=scale_a,
scaleB=scale_b,
scaleOut=None,
bpreshuffle=True,
)

if batched:
output = output.view(*input.shape[:-1], weight.shape[1])

return output


def rocm_aiter_swizzle_hipb_unquantized_gemm(
x: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None = None,
):
output = AiterHipblaslt.hip_bpreshuffle_gemm(x, weight, bias=bias)
return output


def rocm_aiter_swizzle_hipb_fp8_gemm(
input: torch.Tensor,
weight: torch.Tensor,
scale_a: torch.Tensor | None = None,
scale_b: torch.Tensor | None = None,
bias: torch.Tensor | None = None,
out_dtype: torch.dtype | None = None,
):
output = AiterHipblaslt.hip_bpreshuffle_gemm(
input, weight, out_dtype=out_dtype, scale_a=scale_a, scale_b=scale_b
)
if bias is not None:
output = output + bias
return output
74 changes: 58 additions & 16 deletions python/sglang/srt/layers/quantization/fp8_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
from sglang.srt.environ import envs
from sglang.srt.hardware_backend.npu.utils import is_npu_arch35
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.quantization.compressed_tensors.utils import (
rocm_aiter_swizzle_hipb_fp8_gemm,
)
from sglang.srt.layers.quantization.mxfp4_tensor import MXFP4QuantizeUtil
from sglang.srt.runtime_context import (
get_exec,
Expand Down Expand Up @@ -2355,36 +2358,75 @@ def apply_fp8_ptpc_linear(
compressed_tensor_quant: bool = False,
) -> torch.Tensor:
"""FP8 per-token per-channel linear. Only used with the aiter (ROCm) backend."""
# Set at load time by CompressedTensorsW8A8Fp8.process_weights_after_loading when
# SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB is on AND the shape was swizzleable, in
# which case `weight` is stored (K, N) and `weight_scale` is transposed to match.
# Reading the tag rather than the env var keeps shapes that failed can_shuffle --
# which keep the (N, K) aiter layout -- on the gemm_a8w8_bpreshuffle path.
hipb_swizzled = getattr(weight, "hipb_swizzled", False)

# Handle pre-quantized (fp8_tensor, scale) tuple from fused RMSNorm+Quant
if isinstance(input, tuple):
q_input, x_scale = input
q_input, x_scale = input[0], input[1]
leading_shape = q_input.shape[:-1]
out_dtype = input[2] if len(input) > 2 else torch.bfloat16
q_input = q_input.view(-1, q_input.shape[-1])
output_shape = [*q_input.shape[:-1], weight.shape[0]]
output = aiter.gemm_a8w8_bpreshuffle(
q_input, weight, x_scale, weight_scale, None, torch.bfloat16
)
if hipb_swizzled:
# weight is transposed (K, N)
output_shape = [*leading_shape, weight.shape[1]]
output = rocm_aiter_swizzle_hipb_fp8_gemm(
q_input,
weight,
x_scale,
weight_scale,
None,
out_dtype=out_dtype,
)
else:
# weight is in (N, K)
output_shape = [*leading_shape, weight.shape[0]]
output = aiter.gemm_a8w8_bpreshuffle(
q_input, weight, x_scale, weight_scale, None, out_dtype
)
if bias is not None:
output = output + bias
return output.view(*output_shape)

# View input as 2D matrix for fp8 methods
input_2d = input.view(-1, input.shape[-1])

# weight is transposed (K, N)
output_shape = [*input.shape[:-1], weight.shape[1]]

q_input, x_scale = aiter.per_token_quant_hip(input_2d, quant_dtype=aiter.dtypes.fp8)

per_tensor_weights = (weight_scale.numel() == 1) and weight_scale.dim() < 2
per_tensor_activations = (x_scale.numel() == 1) and x_scale.dim() < 2
if hipb_swizzled:
# weight is transposed (K, N)
output_shape = [*input.shape[:-1], weight.shape[1]]
output = rocm_aiter_swizzle_hipb_fp8_gemm(
q_input,
weight,
x_scale,
weight_scale,
None,
out_dtype=torch.bfloat16 if input_scale is not None else input.dtype,
)
else:
# weight is transposed (K, N)
output_shape = [*input.shape[:-1], weight.shape[1]]

if not (per_tensor_weights and per_tensor_activations):
# weight is in (N, K)
output_shape = [*input.shape[:-1], weight.shape[0]]
per_tensor_weights = (weight_scale.numel() == 1) and weight_scale.dim() < 2
per_tensor_activations = (x_scale.numel() == 1) and x_scale.dim() < 2

output = aiter.gemm_a8w8_bpreshuffle(
q_input, weight, x_scale, weight_scale, None, input.dtype
)
if not (per_tensor_weights and per_tensor_activations):
# weight is in (N, K)
output_shape = [*input.shape[:-1], weight.shape[0]]

output = aiter.gemm_a8w8_bpreshuffle(
q_input,
weight,
x_scale,
weight_scale,
None,
torch.bfloat16 if input_scale is not None else input.dtype,
)
if bias is not None:
output = output + bias
return output.view(*output_shape)
Expand Down
Loading
Loading