Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions python/sglang/srt/arg_groups/fields/spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,15 +166,15 @@ class Spec(msgspec.Struct):
speculative_moe_runner_backend: A[
Optional[str],
Arg(
help="Choose the runner backend for MoE in speculative decoding.",
help="Choose the runner backend for MoE in speculative decoding. Defaults to --moe-runner-backend, except megamoe defaults the draft to auto (MTP/NextN draft layers are unquantized, which megamoe's global-expert-id dispatch doesn't support).",
choices=MOE_RUNNER_BACKEND_CHOICES,
resolvable=True,
),
] = None
speculative_moe_a2a_backend: A[
Optional[str],
Arg(
help="Choose the backend for MoE A2A in speculative decoding",
help="Choose the backend for MoE A2A in speculative decoding. Defaults to --moe-a2a-backend, except megamoe defaults the draft to none (MTP/NextN draft layers are unquantized, which megamoe's global-expert-id dispatch doesn't support).",
choices=[
"none",
"deepep",
Expand Down
14 changes: 12 additions & 2 deletions python/sglang/srt/arg_groups/overrides.py
Original file line number Diff line number Diff line change
Expand Up @@ -1560,9 +1560,19 @@ def _pipeline_parallel_overlap_disable(view: Any) -> dict:
def _speculative_moe_runner_default(view: Any) -> dict:
"""Default the speculative (draft) MoE runner backend to the resolved
target-model backend. Invoked at the head of the speculative-decoding
hook, after the MoE kernel chain has resolved."""
hook, after the MoE kernel chain has resolved.

megamoe's global-expert-id dispatch assumes a quantized layer; MTP/NextN
drafts never are, so inheriting it here mismatches their actual (Triton)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: modify the comment so that they are not so specific to issue 40623. Make it more general.

For example:

 MTP/NextN
    drafts never are, so inheriting it here mismatches their actual (Triton)

"drafts never are" is too strong. Some MTPs are indeed quantized.

"actual (Triton)" is too strong. Triton is not always the actual runner.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reworded. It no longer cites the issue, and it says drafts are 'typically' unquantized. It also no longer names Triton as the runner.

runner and corrupts expert-id dispatch (issue #40623;
_deepseek_spec_moe_resolution above carries the same fact for the
HIP+DeepSeek-fp4 case). Fall back to auto instead.
"""
if view.speculative_moe_runner_backend is None:
return {"speculative_moe_runner_backend": view.moe_runner_backend}
default = view.moe_runner_backend
if default == "flashinfer_megamoe":

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what about deepgemm megamoe?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DeepGEMM megamoe is selected via the a2a backend (megamoe), not through the global-expert-id dispatch path that flashinfer_megamoe uses. It also falls back per layer when its weights aren't prepared, so a draft can inherit it safely. I left it unchanged and added a comment saying so.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, what happens if the draft model's moe is also quantized? Should we allow MegaMoE for those cases by default for draft?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can't tell at this point whether the draft is quantized, so the default is flashinfer_trtllm for the runner and none for the a2a backend. A quantized draft can opt in with --speculative-moe-runner-backend and --speculative-moe-a2a-backend. This is documented in the comments and help text.

default = "auto"
Comment thread
wenscarl marked this conversation as resolved.
Resolved
return {"speculative_moe_runner_backend": default}
return {}


Expand Down
10 changes: 5 additions & 5 deletions python/sglang/srt/layers/moe/flashinfer_megamoe.py
Original file line number Diff line number Diff line change
Expand Up @@ -216,11 +216,11 @@ def _validate_nvfp4_fc1_alpha(layer: FusedMoE) -> None:
return
gate_alpha = _local_expert_vector(layer.g1_alphas, layer.num_local_experts)
up_alpha = _local_expert_vector(layer.g1_alphas_up, layer.num_local_experts)
if not torch.allclose(gate_alpha, up_alpha):
raise ValueError(
"FlashInfer NVFP4 MegaMOE requires matching gate/up FC1 alpha "
"values because the kernel accepts one alpha per expert."
)
if not torch.allclose(gate_alpha, up_alpha):
Comment thread
wenscarl marked this conversation as resolved.
Resolved
raise ValueError(
"FlashInfer NVFP4 MegaMOE requires matching gate/up FC1 alpha "
"values because the kernel accepts one alpha per expert."
)


def _bind_transformed_weights(
Expand Down
18 changes: 16 additions & 2 deletions python/sglang/srt/layers/moe/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -440,6 +440,20 @@ def get_deepep_v2_fp8_scale_format() -> DeepEPv2Fp8ScaleFormat:
)


def _default_speculative_a2a_backend(target_a2a_backend: MoeA2ABackend) -> MoeA2ABackend:
if target_a2a_backend.is_flashinfer_megamoe():

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what about deepgemm megamoe?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same above.

Comment thread
wenscarl marked this conversation as resolved.
Resolved
return MoeA2ABackend.NONE
return target_a2a_backend


def _default_speculative_moe_runner_backend(
target_runner_backend: MoeRunnerBackendLike,
) -> MoeRunnerBackendLike:
if target_runner_backend.is_flashinfer_megamoe():

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same here: what about deepgemm megamoe and what if the draft moe is actually quantized?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same above.

return MoeRunnerBackend.FLASHINFER_TRTLLM
return target_runner_backend


def initialize_moe_config():
"""Seed the MoE runtime flags from the published configuration.

Expand All @@ -457,12 +471,12 @@ def initialize_moe_config():
moe.speculative_runner_backend = (
resolve_moe_runner_backend(spec.speculative_moe_runner_backend)
if spec.speculative_moe_runner_backend is not None
else moe.runner_backend
else _default_speculative_moe_runner_backend(moe.runner_backend)
)
moe.speculative_a2a_backend = (
MoeA2ABackend(spec.speculative_moe_a2a_backend)
if spec.speculative_moe_a2a_backend is not None
else moe.a2a_backend
else _default_speculative_a2a_backend(moe.a2a_backend)
)
moe.deepep_mode = DeepEPMode(exec_moe.deepep_mode)
moe.deepep_config = exec_moe.deepep_config or ""
Expand Down
13 changes: 12 additions & 1 deletion test/registered/unit/test_model_overrides.py
Original file line number Diff line number Diff line change
Expand Up @@ -2217,6 +2217,17 @@ def test_speculative_moe_runner_default_pass(self):
),
{"speculative_moe_runner_backend": "triton"},
)
self.assertEqual(
_speculative_moe_runner_default(
ResolvedView(
SimpleNamespace(
speculative_moe_runner_backend=None,
moe_runner_backend="flashinfer_megamoe",
)
)
),
{"speculative_moe_runner_backend": "flashinfer_trtllm"},
)
# user-set draft backend survives
self.assertEqual(
_speculative_moe_runner_default(
Expand Down Expand Up @@ -2635,7 +2646,7 @@ def _view(**kw):
self.assertEqual(
_deepseek_spec_moe_resolution(
_view(
speculative_moe_runner_backend="triton",
speculative_moe_runner_backend="flashinfer_trtllm",
speculative_moe_a2a_backend="none",
)
),
Expand Down
Loading