From da8d30a64374a4789d8a5f9b3fb3eeca50c27b83 Mon Sep 17 00:00:00 2001 From: wenfei qi Date: Mon, 5 Oct 2026 20:16:46 -0700 Subject: [PATCH 1/3] [Bugfix][Quantization] Fix Humming FP8 linear for per-tensor/per-channel weights Fp8LinearMethod and the online per-tensor/per-channel FP8 methods hand the kernel a (K, N) weight but drop its dim tags, so Humming treats it as (N, K) and fails to view it as int32. Retag the weight as CompressedTensorsW8A8Fp8 already does. In batch-invariant mode, call Humming's own apply instead of dequantizing a weight it has repacked. On Ampere, Humming is the only FP8 linear kernel in batch-invariant mode, so these models could not start with VLLM_BATCH_INVARIANT=1. Fixes #60162 Co-Authored-By: Claude Opus 5.5 Signed-off-by: wenfei qi --- tests/quantization/test_fp8.py | 9 +++++---- .../model_executor/layers/quantization/fp8.py | 11 ++++++++++- .../layers/quantization/online/fp8.py | 19 +++++++++++++++++-- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 1cb53f223929..b0ccb5f1d247 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -579,14 +579,15 @@ def test_static_fp8_moe_input_scales_remain_scalar() -> None: ) @pytest.mark.parametrize("model_id", MODELS) @pytest.mark.parametrize( - "force_marlin", [True, False] if current_platform.is_cuda() else [False] + "linear_backend", + ["auto", "marlin", "humming"] if current_platform.is_cuda() else ["auto"], ) @pytest.mark.parametrize( "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False] ) def test_model_load_and_run( model_id: str, - force_marlin: bool, + linear_backend: str, use_rocm_aiter: bool, monkeypatch, dist_init, @@ -596,8 +597,8 @@ def test_model_load_and_run( monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") kernel_config = KernelConfig( - linear_backend="marlin" if force_marlin else "auto", - moe_backend="marlin" if force_marlin else "auto", + linear_backend=linear_backend, + moe_backend="marlin" if linear_backend == "marlin" else "auto", ) model, vllm_config = load_model_without_vllm_runner( model_id, diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 1c1d18614ecd..cddf7e07bb43 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -15,6 +15,9 @@ CutlassFP8ScaledMMLinearKernel, MarlinFP8ScaledMMLinearKernel, ) +from vllm.model_executor.kernels.linear.scaled_mm.humming import ( + HummingFP8ScaledMMLinearKernel, +) from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEMethodBase, @@ -406,6 +409,9 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: # Update layer with new values. replace_parameter(layer, "weight", weight.data) + # The helper returns (K, N); retag so layout-aware kernels see it. + layer.weight.input_dim = 0 + layer.weight.output_dim = 1 replace_parameter(layer, "weight_scale", weight_scale.data) if input_scale is not None: @@ -432,7 +438,10 @@ def apply( bias, ) else: - if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel): + if isinstance( + self.fp8_linear, + (CutlassFP8ScaledMMLinearKernel, HummingFP8ScaledMMLinearKernel), + ): return self.fp8_linear.apply_weights(layer, x, bias) # per-tensor/channel: dequant to BF16 and run GEMM diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index ed018c81bf0a..4bcea38ebf71 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -22,6 +22,9 @@ CutlassFP8ScaledMMLinearKernel, MarlinFP8ScaledMMLinearKernel, ) +from vllm.model_executor.kernels.linear.scaled_mm.humming import ( + HummingFP8ScaledMMLinearKernel, +) from vllm.model_executor.layers.fused_moe import RoutedExperts from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( @@ -220,6 +223,9 @@ def process_weights_after_loading(self, layer: Module) -> None: # Update layer with new values. replace_parameter(layer, "weight", qweight.t().data) + # Transposed to (K, N); retag so layout-aware kernels see it. + layer.weight.input_dim = 0 + layer.weight.output_dim = 1 replace_parameter(layer, "weight_scale", weight_scale.data) if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"): @@ -237,7 +243,10 @@ def apply( ) -> torch.Tensor: # if batch invariant mode is enabled, use BF16 dequant if envs.VLLM_BATCH_INVARIANT: - if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel): + if isinstance( + self.fp8_linear, + (CutlassFP8ScaledMMLinearKernel, HummingFP8ScaledMMLinearKernel), + ): return self.fp8_linear.apply_weights(layer, x, bias) weight_fp8 = layer.weight.to(torch.bfloat16) @@ -403,6 +412,9 @@ def process_weights_after_loading(self, layer: Module) -> None: qweight = _fp8_quant_per_channel(layer.weight, weight_scale) replace_parameter(layer, "weight", qweight.t()) + # Transposed to (K, N); retag so layout-aware kernels see it. + layer.weight.input_dim = 0 + layer.weight.output_dim = 1 replace_parameter(layer, "weight_scale", weight_scale) self.fp8_linear.process_weights_after_loading(layer) @@ -419,7 +431,10 @@ def apply( # if batch invariant mode is enabled dequant if ( envs.VLLM_BATCH_INVARIANT - and not isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel) + and not isinstance( + self.fp8_linear, + (CutlassFP8ScaledMMLinearKernel, HummingFP8ScaledMMLinearKernel), + ) and not isinstance(x, QuantizedActivation) ): weight_dequant = ( From 970068ec5bf9749a3ec60914c22546a5fd44c39c Mon Sep 17 00:00:00 2001 From: wenfei qi Date: Mon, 26 Oct 2026 14:17:45 -0700 Subject: [PATCH 2/3] Reject Humming FP8 for online PTPC, like Marlin Humming's FP8 scaled-mm kernel runs W8A16, so PTPC online quant on it would drop the per-token FP8 activation quantization it asks for. Raise the same error as for Marlin and drop the now-unused weight retag in the PTPC method. Co-Authored-By: Claude Opus 5.5 Signed-off-by: wenfei qi --- .../layers/quantization/online/fp8.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index 38e0f26b414a..0f137710428a 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -22,6 +22,9 @@ from vllm.model_executor.kernels.linear.scaled_mm import ( MarlinFP8ScaledMMLinearKernel, ) +from vllm.model_executor.kernels.linear.scaled_mm.humming import ( + HummingFP8ScaledMMLinearKernel, +) from vllm.model_executor.layers.fused_moe import RoutedExperts from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( @@ -418,13 +421,17 @@ def create_weights( out_dtype=self.out_dtype, module_name=self.__class__.__name__, ) - # PTPC requires per-token activation FP8; MarlinFP8 is W8A16 and - # would silently produce a weight-only fp8 model. - if isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel): + # PTPC requires per-token activation FP8; Marlin and Humming FP8 are + # W8A16 and would silently produce a weight-only fp8 model. + if isinstance( + self.fp8_linear, + (MarlinFP8ScaledMMLinearKernel, HummingFP8ScaledMMLinearKernel), + ): raise ValueError( "FP8 PTPC online quant requires a kernel that honors " - "per-token activation quantization; MarlinFP8 is W8A16 " - "weight-only. Requires SM89+ for Cutlass FP8 or ROCm MI3xx " + "per-token activation quantization; " + f"{type(self.fp8_linear).__name__} is W8A16 weight-only. " + "Requires SM89+ for Cutlass FP8 or ROCm MI3xx " "for rowwise scaled_mm." ) @@ -442,9 +449,6 @@ def process_weights_after_loading(self, layer: Module) -> None: qweight = _fp8_quant_per_channel(weight, weight_scale) replace_parameter(layer, "weight", qweight.t()) - # Transposed to (K, N); retag so layout-aware kernels see it. - layer.weight.input_dim = 0 - layer.weight.output_dim = 1 replace_parameter(layer, "weight_scale", weight_scale) self.fp8_linear.process_weights_after_loading(layer) From 606a1e6fa54e094ac0f810d75e3192a31d0b8f6e Mon Sep 17 00:00:00 2001 From: wenfei qi Date: Mon, 26 Oct 2026 14:35:11 -0700 Subject: [PATCH 3/3] Tag the (K, N) weight layout with processed_weight_dims After #62326, replace_parameter keeps the ModelWeightParameter, whose input_dim/output_dim are read-only. Use the layer attribute that the other FP8 methods and Humming now use. Co-Authored-By: Claude Opus 5.5 Signed-off-by: wenfei qi --- vllm/model_executor/layers/quantization/fp8.py | 5 ++--- vllm/model_executor/layers/quantization/online/fp8.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index dca16833c8f1..e63efb792fbe 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -406,9 +406,8 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: # Update layer with new values. replace_parameter(layer, "weight", weight.data) - # The helper returns (K, N); retag so layout-aware kernels see it. - layer.weight.input_dim = 0 - layer.weight.output_dim = 1 + # The helper returns (K, N); the param keeps its checkpoint dims. + layer.processed_weight_dims = (0, 1) replace_parameter(layer, "weight_scale", weight_scale.data) if input_scale is not None: diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index 0f137710428a..01bf5f87033a 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -252,9 +252,8 @@ def process_weights_after_loading(self, layer: Module) -> None: # Update layer with new values. replace_parameter(layer, "weight", qweight.t().data) - # Transposed to (K, N); retag so layout-aware kernels see it. - layer.weight.input_dim = 0 - layer.weight.output_dim = 1 + # Transposed to (K, N); the param keeps its checkpoint dims. + layer.processed_weight_dims = (0, 1) replace_parameter(layer, "weight_scale", weight_scale.data) if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"):