Repository navigation
[Model][Qwen4Exp] Run routed experts on DeepGEMM MegaMoE (MXFP8 or MXFP4) - #64315
Conversation
|
Documentation preview: https://vllm--64315.org.readthedocs.build/en/64315/ |
|
This pull request has merge conflicts that must be resolved before it can be |
471486d to
5ed2362
Compare
| def flush_mega_moe_requantize(model: nn.Module) -> None: | ||
| """Requantize the complete expert shards buffered by ``load_weights``. | ||
|
|
||
| Half pairs stay buffered: a chunked weight update may pass an expert's | ||
| weight and its scale in different calls. | ||
| """ | ||
| for module in model.modules(): | ||
| if isinstance(module, Qwen4ExpMegaMoEExperts): | ||
| module.flush_requantize() | ||
|
|
||
|
|
||
| class _FinishRequantize(QuantizeMethodBase): | ||
| """Flushes the experts; fails on shards missing their weight or scale.""" | ||
|
|
||
| requires_device_loading = False | ||
| supports_pre_processed_weights = True | ||
|
|
||
| def create_weights(self, layer: nn.Module, *args, **kwargs) -> None: | ||
| raise NotImplementedError("MegaMoE experts allocate weights at construction") | ||
|
|
||
| def apply(self, layer: nn.Module, *args, **kwargs) -> torch.Tensor: | ||
| raise NotImplementedError("MegaMoE experts run through their forward") | ||
|
|
||
| def process_weights_after_loading(self, layer: nn.Module) -> None: | ||
| assert isinstance(layer, Qwen4ExpMegaMoEExperts) | ||
| layer.flush_requantize(strict=True) | ||
|
|
||
|
|
||
| class Qwen4ExpMegaMoEExperts(DeepGemmMegaMoEExperts): | ||
| """MegaMoE experts loaded from per-expert block-FP8 checkpoint tensors. | ||
|
|
||
| Each EP rank buffers the complete (weight, scale) pairs of its own experts | ||
| and requantizes them per projection in a few batched calls. | ||
| """ | ||
|
|
||
| def __init__(self, vllm_config: VllmConfig, **kwargs) -> None: | ||
| super().__init__(vllm_config, activation=MegaMoEActivation(), **kwargs) | ||
| self.quant_method = _FinishRequantize() | ||
| # Parameters may sit on meta (weight-cache IPC loading, layerwise | ||
| # reload), so stage and requantize on the current GPU. | ||
| self._requantize_device = torch.device( | ||
| current_platform.device_type, torch.accelerator.current_device_index() | ||
| ) | ||
| self._pending_shards: dict[tuple[int, str], dict[str, torch.Tensor]] = {} | ||
| # Complete (expert, weight, block scale) shards per shard id. | ||
| self._ready_shards: dict[str, list[tuple[int, torch.Tensor, torch.Tensor]]] = {} | ||
| # Logical experts this rank holds (redundant replicas share one). | ||
| self._num_owned_experts = len( | ||
| { | ||
| p % self.num_logical_experts | ||
| for p in range(self.experts_start_idx, self.experts_end_idx) | ||
| } | ||
| ) | ||
|
|
||
| def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: | ||
| """Load ``{expert}.{gate,up,down}_proj.{weight,weight_scale_inv}``.""" | ||
| loaded: set[str] = set() | ||
| for name, tensor in weights: | ||
| expert, proj, kind = name.split(".") | ||
| expert_id, shard_id = int(expert), _CHECKPOINT_SHARDS[proj] | ||
| prefix = "w2" if shard_id == "w2" else "w13" | ||
| loaded.update((f"{prefix}_weight", f"{prefix}_weight_scale")) | ||
| if not self._map_global_expert_id(expert_id): | ||
| continue | ||
| pending = self._pending_shards.setdefault((expert_id, shard_id), {}) | ||
| # Stage on the GPU as it arrives: no CPU-side torch ops, which stall | ||
| # when several engines load on one host. | ||
| pending[kind] = tensor.to(self._requantize_device) | ||
| if len(pending) < 2: | ||
| continue | ||
| del self._pending_shards[(expert_id, shard_id)] | ||
| ready = self._ready_shards.setdefault(shard_id, []) | ||
| ready.append((expert_id, pending["weight"], pending["weight_scale_inv"])) | ||
| if len(ready) == self._num_owned_experts: | ||
| self._requantize_ready(shard_id) | ||
| return loaded | ||
|
|
||
| def flush_requantize(self, strict: bool = False) -> None: | ||
| for shard_id in list(self._ready_shards): | ||
| self._requantize_ready(shard_id) | ||
| if strict and self._pending_shards: | ||
| raise ValueError( | ||
| f"{self.prefix}: expert shards without both weight and scale: " | ||
| f"{sorted(self._pending_shards)[:4]}" | ||
| ) | ||
|
|
||
| def _requantize_ready(self, shard_id: str) -> None: | ||
| ready = self._ready_shards.pop(shard_id) | ||
| rows = ready[0][1].shape[0] | ||
| if any(w.shape != ready[0][1].shape for _, w, _ in ready): | ||
| raise ValueError(f"{self.prefix}: {shard_id} shards differ in shape.") | ||
| prefix = "w2" if shard_id == "w2" else "w13" | ||
| chunk = max(1, _REQUANTIZE_CHUNK_ELEMENTS // ready[0][1].numel()) | ||
| for start in range(0, len(ready), chunk): | ||
| batch = ready[start : start + chunk] | ||
| weight, scale = requantize_fp8_block( | ||
| torch.cat([w for _, w, _ in batch]), | ||
| torch.cat([s for _, _, s in batch]), | ||
| self.weight_format, | ||
| ) | ||
| for i, (expert_id, _, _) in enumerate(batch): | ||
| for suffix, value in (("weight", weight), ("weight_scale", scale)): | ||
| param_name = f"{prefix}_{suffix}" | ||
| # The parameter's own loader: layerwise reload wraps it. | ||
| param = getattr(self, param_name) | ||
| param.weight_loader( | ||
| param, | ||
| value[i * rows : (i + 1) * rows], | ||
| param_name, | ||
| shard_id=shard_id, | ||
| expert_id=expert_id, | ||
| ) |
There was a problem hiding this comment.
I want to land this PR soon so I stick to the existing per-model DeepGEMM integration. We can do a refactor for all models in a future PR.
There was a problem hiding this comment.
Yes, I realized MegaMOE on cuda side does not use this online quantization infra. It's fine
| ) | ||
|
|
||
|
|
||
| class Qwen4ExpMegaMoeBlock(nn.Module): |
There was a problem hiding this comment.
Any chance you could use oracle + experts implementation (similar to #63229) if this can be re-used for other models?
There was a problem hiding this comment.
To be done later - existing code in vllm does not use that
Lines 163 to 166 in 28dd9c3
|
This pull request has merge conflicts that must be resolved before it can be |
5ed2362 to
dbbda04
Compare
dbbda04 to
55796ae
Compare
…FP4)
With --kernel-config '{"moe_backend":"deep_gemm_mega_moe"}' and expert
parallelism, Qwen4Exp's routed experts run as one DeepGEMM MegaMoE launch
(fused EP dispatch, expert GEMMs and combine) instead of the FusedMoE path.
The official checkpoint stores experts as 128x128 block FP8, which MegaMoE
does not take. Each EP rank requantizes the experts it owns as they load,
to MXFP8 (1x32 UE8M0 scales) by default, or to MXFP4 with
--quantization-config '{"moe": "mxfp4"}'; ignore/targets patterns of the
online quantization config pick formats per layer, and requantized layers
show in the online quantization summary. The MTP drafter's config does not
carry --quantization-config, so its experts stay MXFP8.
Requantization rounds as DeepGEMM's own casts, runs on the GPU and is
batched over a projection's local experts. Shards are written through each
parameter's weight_loader, so layerwise reload and chunked weight updates
(which may split an expert's weight from its scale) work; a shard still
missing its pair after loading fails in process_weights_after_loading.
The sigmoid-gated shared expert stays unfused and runs on an auxiliary
stream next to the router (VLLM_DISABLE_SHARED_EXPERTS_STREAM turns it
off). TP>1 is not supported yet (use DP+EP). The DeepGEMM MegaMoE warmup
covers Qwen4Exp blocks and keys its cache by the routed-expert format.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Bowen Wang <bowen@inferact.ai>
…ntization The Qwen4Exp MegaMoE loader accepted only 128x128 block-FP8 checkpoints. It now reads the routed experts in every format MegaMoE runs, and requantizes when a format is forced: - BF16 (the official checkpoint's fused gate_up_proj/down_proj, or per-expert tensors), MXFP8 (ModelOpt) and MXFP4 (Quark) experts load straight into the kernel layout. BF16 runs the bf16_mega_moe kernel. - --quantization mxfp8|mxfp4 or --quantization-config moe/targets force a format; the experts are then dequantized from their checkpoint format and requantized with DeepGEMM's casts, as before for block FP8. - Block-FP8 experts, which MegaMoE cannot run, still default to MXFP8, now with a warning that names the flag to choose the format. - Checkpoint formats are read with get_checkpoint_quant_keys; unexpected per-expert tensors fail with an explicit error. Tests cover every (checkpoint format, MegaMoE format) pair: loading into the kernel layout, requantization, layerwise reload, and the kernel against a dequantized reference. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
- Dequantize MX checkpoints in torch; only the cast needs DeepGEMM. - Drop the BF16 kernel probe, a dead online target and the docs changes. - Run the tests in the existing B200 Qwen4 Exp kernels job. - Test one representative case per path instead of every format pair. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
…fter_loading Drop the module-level flush called from Qwen4ExpModel/Qwen4ExpMTP load_weights: every load path (initial, layerwise and direct reload) already runs the experts' process_weights_after_loading, which now requantizes whatever is still buffered, as Step5's MegaMoE does. Trim the tests: the load test checks requantization against DeepGEMM's own casts, which subsumes the requantize test; the kernels are covered by tests/kernels/moe/test_deep_gemm_mega_moe.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Requantize each expert shard as soon as its weight and scale have both arrived instead of batching a projection's owned experts. Staging stays at one shard, which removes the batching, the chunk bound and the deferred requantization; process_weights_after_loading only checks that no shard is missing its weight or scale. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Drop the module-level E2M1 table, which a model built under a device or dtype context could create on the GPU at import. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
…ntize into requantize The deep_gemm module re-exports its utils, so the import_module detour is not needed. Dequantization has a single caller; merge it into a private _requantize and clarify the _CheckLoaded and MTP docstrings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
827d441 to
d5be2a5
Compare
|
/ci run |
|
✅ Triggered Buildkite CI #101001 for commit |
- Overlap the shared expert with routing through maybe_execute_in_parallel, which also falls back to one stream under breakable CUDA graph capture. - Report only the expert parameters a load actually wrote, so the loader's missing-parameter check still catches a checkpoint without scales. - Test a targets override in format selection. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
|
/ci cancel |
|
/ci run |
|
✅ Requested cancellation of 1 CI build for |
|
✅ Triggered Buildkite CI #101034 for commit ℹ️ This PR is 7 commits behind upstream |
|
/ci retry |
|
✅ Queued 1 failed job(s) for retry in Buildkite CI #101034. |
Note
This description was edited by Claude (an AI assistant) with authorization from Thien Tran (gau-nernst). The previous version is in the edit history.
Overview
Runs Qwen4Exp (Qwen3.8-Flash-Next) routed experts on the DeepGEMM MegaMoE backend (
--kernel-config '{"moe_backend":"deep_gemm_mega_moe"}', built onDeepGemmMegaMoEExpertsfrom #61538). Expert parallelism is required.The routed experts load in the checkpoint's format when MegaMoE can run it, and are requantized otherwise:
--quantization/--quantization-configQwen/Qwen3.8-Flash-Next, fusedgate_up_proj/down_proj)mxfp8ormxfp4Qwen/Qwen3.8-Flash-Next-FP8)mxfp8ormxfp4mxfp4mxfp8Claims
deep_gemm_mega_moewith DP + EP on Blackwell, from the official BF16 and FP8 checkpoints. MXFP8 and MXFP4 checkpoints load directly; no public one exists yet, so that path is covered by unit tests only (format detection is tested for ModelOpt MXFP8; Quark MXFP4 goes through vLLM's existing Quark config lookup).Validation
Unit tests
pytest tests/models/qwen4_exp/test_mega_moe.py: 10 passed on GB200. They run in the existingqwen4-exp-hc-kernels-b200Buildkite step. The tests cover:ignore,targetsand forced MXFP8/MXFP4;DeepGemmMegaMoEExperts.weight_loaderfed tensors requantized with DeepGEMM's own casts;load_weightscalls, and the post-load error on an incomplete pair (requantized formats);The MegaMoE kernels themselves are covered by
tests/kernels/moe/test_deep_gemm_mega_moe.py. CI does not runQwen4ExpMegaMoeBlock's forward or the decoder's backend selection; the GSM8K runs below cover them end to end.GSM8K
4x GB200 node, DP4 + EP, MTP3 speculative decoding, 5-shot, greedy, thinking off, full test set (1319 questions), one run per arm:
Four runs of the FP8 MegaMoE arm on different revisions of this branch scored 96.59-96.89 flexible. The requantized weights are identical across those revisions, so this range is run-to-run noise.
Reasoning evals and throughput (FP8 checkpoint, B300)
These were measured on an earlier revision of this PR. The FP8 -> MXFP8/MXFP4 requantization is numerically unchanged since then: same DeepGEMM casts, applied row by row.
Reasoning evals with thinking on,
temperature=1.0, top_p=0.95, top_k=20,max_tokens=98304, MTP3. Stock FP8 is TP4 + EP on the default MoE backend; MegaMoE is DP4 + EP. Rule: paired difference vs stock with a bootstrap 95% CI; non-inferior if the lower bound is >= -2 pt (AIME) or -1 pt (GPQA-D, MMMU-Pro).vllm bench serve --dataset-name random --ignore-eos,--max-num-seqs 256 --gpu-memory-utilization 0.9 --no-enable-prefix-caching, no speculative decoding; each point run twice (runs differed by <= 3%).Weight loading
FP8 checkpoint -> MXFP8 on GB200:
Loading weights took48 s per rank at DP4 + EP, and 148 s at TP1. TP1 is the worst case: one rank requantizes all 512 experts of 48 layers, one expert projection at a time. Batching a projection's experts loaded in 120 s at TP1, but needed a chunking bound and a deferred flush; the simpler per-expert path was preferred.Details
Format selection. There is no new flag.
mega_moe_expert_formatsreads the checkpoint's expert format (Fp8Config, orget_checkpoint_quant_keysfor ModelOpt and Quark), then applies the online quantization config per layer.moeortargetsforces MXFP8 or MXFP4. Layers matched byignorekeep the default. Requantized layers show in the online quantization summary. The MTP drafter's config does not carry the online config, so its experts take the default format.Loading. Experts in a MegaMoE format are written straight into the kernel layout through each parameter's
weight_loader, as for DeepSeek V4, Kimi K3 and MiniMax M3. Other experts are requantized one expert projection at a time, as soon as its weight and scale have both arrived: dequantize to FP32, then DeepGEMM'sper_token_cast_to_fp8/per_token_cast_to_fp4with UE8M0 scales over 1x32 groups. The results go through the sameweight_loader, so layerwise reload (reload_weights, NCCL/IPC weight transfer) can buffer and replay them. A chunked update may split an expert's weight from its scale; the half pair waits for the nextload_weightscall, andprocess_weights_after_loadingfails on any pair still incomplete.Shared expert. Qwen4Exp gates the shared expert per token with
sigmoid(shared_expert_gate(x)), which MegaMoE's fused shared expert cannot apply, so the shared expert stays unfused. It runs on an auxiliary stream while the router and top-k run (maybe_execute_in_parallel, which runs both on one stream under breakable CUDA graph capture), and joins before the MegaMoE launch, which needs every SM.VLLM_DISABLE_SHARED_EXPERTS_STREAMturns the overlap off.Other changes. The DeepGEMM MegaMoE warmup covers Qwen4Exp blocks, and its cache key includes the routed-expert format.
Docs. None, as for the other DeepGEMM MegaMoE integrations (GLM-5.3-Flash, Kimi K3, MiniMax M3, Step5): the docs do not cover the
deep_gemm_mega_moebackend for any model.Limitations.
--expert-placement-strategyis rejected.nvfp4_mega_moekernel issuesceil(640 / 256) = 3TMA stages per L2 task while the MMA warp consumesfloor = 2. Large launches hang and small ones return wrong output.DeepGemmMegaMoEExpertsbut was not exercised here.Failed to load weights), and the unpaired tensors stay buffered. For natively loaded MXFP8/MXFP4 experts, a missing scale is caught only when no scale of that parameter arrives at all, as for the other MegaMoE models.Pull Request Checklist
I used vLLM's
/pr-checklistskill. (Mandatory for agents, optional for humans).AI assistance was used during the creation of this PR.
Design Fit: Minimizes impact on core components, reuses existing functionality, and justifies added complexity.
Testing and Validation: Validates the change and ensures any added tests are meaningful and reliable, with CI coverage or documented CI resource constraints and validation performed outside CI.
Code Quality and Style: Keeps code and comments clear and concise, and updates relevant documentation and examples.
Pull Request Contents: Includes a brief summary and relevant links, supports claims with evidence, explains root causes and implementation trade-offs, and follows the contributing guide.
🤖 Generated with Claude Code