diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 0b59a068cafa..0638c752c512 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -659,14 +659,15 @@ def test_static_fp8_moe_input_scales_remain_scalar() -> None: ) @pytest.mark.parametrize("model_id", MODELS) @pytest.mark.parametrize( - "force_marlin", [True, False] if current_platform.is_cuda() else [False] + "linear_backend", + ["auto", "marlin", "humming"] if current_platform.is_cuda() else ["auto"], ) @pytest.mark.parametrize( "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False] ) def test_model_load_and_run( model_id: str, - force_marlin: bool, + linear_backend: str, use_rocm_aiter: bool, monkeypatch, dist_init, @@ -676,8 +677,8 @@ def test_model_load_and_run( monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") kernel_config = KernelConfig( - linear_backend="marlin" if force_marlin else "auto", - moe_backend="marlin" if force_marlin else "auto", + linear_backend=linear_backend, + moe_backend="marlin" if linear_backend == "marlin" else "auto", ) model, vllm_config = load_model_without_vllm_runner( model_id, diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index b4d7bd3ef669..8301cbe481a3 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -836,6 +836,9 @@ def fp8_scaled_mm_is_batch_invariant( kernel: FP8ScaledMMLinearKernel | Fp8BlockScaledMMLinearKernel, ) -> bool: """Whether a row's output from the kernel is independent of the other rows.""" + # Weight-only here, and Humming honors VLLM_BATCH_INVARIANT itself. + if isinstance(kernel, HummingFP8ScaledMMLinearKernel): + return True if not isinstance( kernel, (CutlassFP8ScaledMMLinearKernel, TritonFP8ScaledMMLinearKernel) ): diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 0c4fe9889268..e63efb792fbe 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -406,6 +406,8 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: # Update layer with new values. replace_parameter(layer, "weight", weight.data) + # The helper returns (K, N); the param keeps its checkpoint dims. + layer.processed_weight_dims = (0, 1) replace_parameter(layer, "weight_scale", weight_scale.data) if input_scale is not None: diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index f4a327192108..01bf5f87033a 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -22,6 +22,9 @@ from vllm.model_executor.kernels.linear.scaled_mm import ( MarlinFP8ScaledMMLinearKernel, ) +from vllm.model_executor.kernels.linear.scaled_mm.humming import ( + HummingFP8ScaledMMLinearKernel, +) from vllm.model_executor.layers.fused_moe import RoutedExperts from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( @@ -249,6 +252,8 @@ def process_weights_after_loading(self, layer: Module) -> None: # Update layer with new values. replace_parameter(layer, "weight", qweight.t().data) + # Transposed to (K, N); the param keeps its checkpoint dims. + layer.processed_weight_dims = (0, 1) replace_parameter(layer, "weight_scale", weight_scale.data) if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"): @@ -415,13 +420,17 @@ def create_weights( out_dtype=self.out_dtype, module_name=self.__class__.__name__, ) - # PTPC requires per-token activation FP8; MarlinFP8 is W8A16 and - # would silently produce a weight-only fp8 model. - if isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel): + # PTPC requires per-token activation FP8; Marlin and Humming FP8 are + # W8A16 and would silently produce a weight-only fp8 model. + if isinstance( + self.fp8_linear, + (MarlinFP8ScaledMMLinearKernel, HummingFP8ScaledMMLinearKernel), + ): raise ValueError( "FP8 PTPC online quant requires a kernel that honors " - "per-token activation quantization; MarlinFP8 is W8A16 " - "weight-only. Requires SM89+ for Cutlass FP8 or ROCm MI3xx " + "per-token activation quantization; " + f"{type(self.fp8_linear).__name__} is W8A16 weight-only. " + "Requires SM89+ for Cutlass FP8 or ROCm MI3xx " "for rowwise scaled_mm." )