Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions python/sglang/srt/arg_groups/deepseek_v4_hook.py
Original file line number Diff line number Diff line change
Expand Up @@ -289,23 +289,41 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
read_ragged_verify_mode,
)

# CP size may still be implicit here; validate_deepseek_v4_cp resolves it.
supported_prefill_cp = (
cfg.disaggregation_mode == "prefill"
and cfg.enable_prefill_cp
and cfg.cp_strategy == "interleave"
)
if (
read_ragged_verify_mode() is not RaggedVerifyMode.STATIC
or cfg.disaggregation_transfer_backend != "mooncake"
or cfg.enable_prefill_cp
or cfg.attn_cp_size != 1
or (
(cfg.enable_prefill_cp or cfg.attn_cp_size != 1)
and not supported_prefill_cp
)
or cfg.dcp_size != 1
):
raise ValueError(
"DeepSeek-V4.1 DSpark PD requires static verify, Mooncake, "
"and CP=1 on both servers. DP attention is supported when "
"DCP=1, and CP=1 on decode. Prefill CP requires "
"--enable-prefill-cp with the interleave strategy. "
"DP attention is supported when "
"both servers use the same block size and target/draft KV layout."
)

prefill_graph = cfg.cuda_graph_config.prefill
if prefill_graph.backend != Backend.DISABLED and prefill_graph.max_seq_len is None:
# The captured low-ratio indexer scores a static context width; 16k
# keeps it inside the candidate window at under 1 ms per layer.
cp_breakable_prefill = (
cfg.enable_prefill_cp
and cfg.cp_strategy == "interleave"
and cfg.tp_size > 1
and prefill_graph.backend == Backend.BREAKABLE
)
if (
prefill_graph.backend != Backend.DISABLED
and prefill_graph.max_seq_len is None
and not cp_breakable_prefill
):
declare_resolution(
server_args,
"validate_deepseek_v41_features",
Expand Down
6 changes: 4 additions & 2 deletions python/sglang/srt/arg_groups/model_hook.py
Original file line number Diff line number Diff line change
Expand Up @@ -1071,12 +1071,14 @@ def handle_language_model_only(server_args: Any):
):
if flag:
raise ValueError(f"--language-model-only cannot be combined with {name}")
if cfg.disaggregation_mode != "null":
hf_config = model_config_of(server_args).hf_config
# V4.1 text-only workers use the standard PD KV transfer path.
if cfg.disaggregation_mode != "null" and hf_config.model_type != "deepseek_v41":
raise ValueError(
"--language-model-only is incompatible with --disaggregation-mode "
"prefill/decode"
)
architectures = model_config_of(server_args).hf_config.architectures
architectures = hf_config.architectures
if not any(
a in server_args.LANGUAGE_MODEL_ONLY_ARCHITECTURES for a in architectures
):
Expand Down
21 changes: 12 additions & 9 deletions python/sglang/srt/disaggregation/common/conn.py
Original file line number Diff line number Diff line change
Expand Up @@ -1210,17 +1210,20 @@ def try_ensure_parallel_info(self, bootstrap_addr: str) -> bool:
"enable DSpark with the same block size and target/draft KV "
"layout. Upgrade both servers together."
)
if info.attn_cp_size != 1 or self.attn_cp_size != 1:
raise RuntimeError(
"DeepSeek-V4.1 DP-only DSpark PD requires CP=1 on both servers"
)
non_cp_mla_layout = info.attn_cp_size == self.attn_cp_size == 1 and (
self.is_mla_backend or self.is_hybrid_mla_backend

decode_tp_size_aligns_prefill_world_size = (
info.attn_cp_size > 1
and (self.is_mla_backend or self.is_hybrid_mla_backend)
and self.attn_cp_size == 1
and info.attn_tp_size * info.attn_cp_size == self.attn_tp_size
)
if info.attn_tp_size != self.attn_tp_size and not non_cp_mla_layout:
if (
info.attn_tp_size != self.attn_tp_size
and not decode_tp_size_aligns_prefill_world_size
):
raise RuntimeError(
"DeepSeek-V4.1 DSpark PD requires matching attention TP "
"unless both servers use CP=1 with an MLA KV layout"
"DeepSeek-V4.1 DSpark PD requires the same TP size on both "
"servers (including prefill CP ranks for MLA)"
)

if self.dcp_size > 1:
Expand Down
8 changes: 7 additions & 1 deletion python/sglang/srt/layers/attention/deepseek_v4_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -1526,7 +1526,13 @@ def _low_ratio_prefill_indexer_metadata(

@property
def low_ratio_prefill_graph(self) -> bool:
return bool(self.low_ratios) and has_dense_fp4_indexer() and is_sm100_or_newer()
"""Whether ratio-1/2 sources use captured projections and indexer metadata."""
return (
bool(self.low_ratios)
and has_dense_fp4_indexer()
and is_sm100_or_newer()
and get_parallel().attn_cp_size == 1
)

def can_run_prefill_cuda_graph(self, forward_batch: ForwardBatch) -> bool:
max_seq_len = _prefill_graph_max_seq_len()
Expand Down
1 change: 1 addition & 0 deletions python/sglang/srt/model_executor/runner/eager_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,7 @@ def _execute_extend_cp(
model_kwargs = {}
if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None:
model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors

with cp_shard_model_inputs(
input_embeds,
positions,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1404,9 +1404,9 @@ def can_run_graph(self, forward_batch: ForwardBatch) -> bool:
batch_max_context_len=batch_max_context_len,
):
return False
if getattr(self, "enable_cp_bcg_capture", False) and is_cp_active(
forward_batch
):
if getattr(self, "enable_cp_bcg_capture", False):
if not is_cp_active(forward_batch):
return False
assert self.prefill_cp_bcg_input is not None
if (
self.prefill_cp_bcg_input.select_replay_bucket_for_batch(
Expand Down
46 changes: 37 additions & 9 deletions python/sglang/srt/models/deepseek_v4.py
Original file line number Diff line number Diff line change
Expand Up @@ -2724,7 +2724,8 @@ def __init__(
is_nextn=is_nextn,
is_deepseek_v4=True,
vl_correction_bias=config.model_type == "deepseek_v41"
and config.vision_n_layers > 0,
and config.vision_n_layers > 0
and not getattr(config, "language_model_only", False),
)

self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
Expand Down Expand Up @@ -3512,7 +3513,14 @@ def _run_moe_ffn_dp_sync(
)

if _use_cp and get_moe_a2a_backend().is_none():
hidden_states = attn_cp_interleave_reduce_scatter(hidden_states)
if self.config.model_type == "deepseek_v41":
Comment thread
BJWang-ant marked this conversation as resolved.
Resolved
parallel = get_parallel()
hidden_states = parallel.tp_group.all_reduce(hidden_states)
hidden_states = hidden_states.tensor_split(parallel.attn_cp_size)[
parallel.attn_cp_rank
].contiguous()
else:
hidden_states = attn_cp_interleave_reduce_scatter(hidden_states)
elif _use_tp_moe_gather:
hidden_states, global_hidden_states = (
get_local_dp_buffer(get_parallel().tp_group),
Expand Down Expand Up @@ -4465,7 +4473,11 @@ def __init__(
self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config)
self.determine_num_fused_shared_experts()
self.vision = None
if config.model_type == "deepseek_v41" and config.vision_n_layers > 0:
if (
config.model_type == "deepseek_v41"
and config.vision_n_layers > 0
and not getattr(config, "language_model_only", False)
):
if (
get_parallel().attn_cp_size != 1
or get_parallel().pp_group.world_size != 1
Expand Down Expand Up @@ -4683,16 +4695,19 @@ def determine_num_fused_shared_experts(self):
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
)

def forward(
def prepare_language_model_inputs(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: Optional[torch.Tensor] = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> torch.Tensor:
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
"""Prepare full-sequence image embeddings and model IDs before CP splits.

Scheduler hash IDs stay intact for multimodal cache keys; the language
model uses image_token_id for Engram masking and visual MoE routing.
"""
if (
self.vision is not None
getattr(self, "vision", None) is not None
and not forward_batch.forward_mode.is_decode()
and not forward_batch.forward_mode.is_target_verify()
and forward_batch.mm_inputs is not None
Expand All @@ -4701,7 +4716,7 @@ def forward(
if input_embeds is not None:
raise ValueError("Cannot combine input_embeds and image inputs")
input_embeds = self._prepare_mm_embeddings(input_ids, forward_batch)
if self.vision is not None and not (
if getattr(self, "vision", None) is not None and not (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
):
Expand All @@ -4711,6 +4726,19 @@ def forward(
input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
)

return input_ids, input_embeds

def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: Optional[torch.Tensor] = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> torch.Tensor:
input_ids, input_embeds = self.prepare_language_model_inputs(
input_ids, forward_batch, input_embeds
)
with get_attn_tp_context().maybe_input_scattered(forward_batch):
hidden_states = self.model.forward(
input_ids, positions, forward_batch, input_embeds, pp_proxy_tensors
Expand Down
1 change: 1 addition & 0 deletions python/sglang/srt/server_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,7 @@ def resolved_dict(self) -> dict[str, Any]:
return resolved

LANGUAGE_MODEL_ONLY_ARCHITECTURES = (
"DeepseekV4ForCausalLM",
Comment thread
BJWang-ant marked this conversation as resolved.
Resolved
Comment thread
Fridge003 marked this conversation as resolved.
Resolved
"MuseGlimmerForConditionalGeneration",
"Cosmos3ForConditionalGeneration",
"Cosmos3EdgeForConditionalGeneration",
Expand Down
127 changes: 127 additions & 0 deletions test/registered/e2e/models/test_deepseek_v41_pd_cp_dspark.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
"""V4.1 prefill CP + PD disaggregation + DSpark on eight B300 GPUs.

GPU 0-3: TP4/CP4 prefill. GPU 4-7: TP4 decode. Both workers use the
DeepSeek-V4.1-Flash checkpoint's bundled DSpark draft and Mooncake transfer.
"""

import unittest

import requests

from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.disaggregation_fixture import (
PDDisaggregationServerBase,
)
from sglang.test.test_utils import try_cached_model

# Initial estimate; update after the first B300 CI run.
register_cuda_ci(est_time=1800, stage="extra-b", runner_config="8-gpu-b300")

MODEL = "deepseek-ai/DeepSeek-V4.1-Flash"

COMMON_ARGS = [
"--language-model-only",
"--attention-backend",
"dsv4",
"--moe-a2a-backend",
"none",
"--moe-runner-backend",
"flashinfer_mxfp4",
"--ep-size",
"4",
"--speculative-algorithm",
"DSPARK",
"--speculative-dspark-block-size",
"5",
"--mem-fraction-static",
"0.8",
"--max-running-requests",
"32",
"--cuda-graph-max-bs-decode",
"32",
"--cuda-graph-backend-prefill",
"disabled",
"--context-length",
"16384",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 6}',
"--random-seed",
"0",
]


class TestDeepseekV41PDCPDSpark(PDDisaggregationServerBase, GSM8KMixin):
model = MODEL
prefill_tp_size = 4
decode_tp_size = 4
decode_base_gpu_id = 4

gsm8k_score_threshold = 0.85
gsm8k_num_examples = 200
gsm8k_num_threads = 32
gsm8k_num_shots = 8

extra_prefill_args = COMMON_ARGS + [
"--enable-prefill-cp",
"--attn-cp-size",
"4",
"--cp-strategy",
"interleave",
# Small chunks exercise continuation as well as the P-to-D handoff.
"--chunked-prefill-size",
"1024",
"--max-prefill-tokens",
"4096",
]
extra_decode_args = COMMON_ARGS
extra_prefill_env = {"SGLANG_RAGGED_VERIFY_MODE": "static"}
extra_decode_env = {"SGLANG_RAGGED_VERIFY_MODE": "static"}

@classmethod
def setUpClass(cls):
super().setUpClass()
cls.model = try_cached_model(MODEL)
# V4.1's target/draft KV transfer requires Mooncake on both sides.
cls.transfer_backend = ["--disaggregation-transfer-backend", "mooncake"]
cls.launch_all()

def _server_info(self, url):
response = requests.get(url + "/server_info", timeout=30)
response.raise_for_status()
return response.json()

def test_gsm8k(self):
# Inspect workers directly: the router does not expose both configs.
for url, role, cp_size in (
(self.prefill_url, "prefill", 4),
(self.decode_url, "decode", 1),
):
info = self._server_info(url)
self.assertEqual(info["disaggregation_mode"], role)
self.assertEqual(info["disaggregation_transfer_backend"], "mooncake")
self.assertEqual(info["tp_size"], 4)
self.assertEqual(info["attn_cp_size"], cp_size)
self.assertEqual(info["enable_prefill_cp"], role == "prefill")
self.assertEqual(info["speculative_algorithm"], "DSPARK")
self.assertEqual(info["speculative_num_draft_tokens"], 6)
self.assertTrue(info["language_model_only"])

# Completion evaluation avoids V4.1 chat-template thinking mode.
# base_url is the PD router, so every request exercises both workers.
super().test_gsm8k()

# Require evidence that DSpark actually accepted draft tokens. Query D
# directly rather than letting the optional mixin check skip a missing
# metric from the router or the prefill worker.
states = self._server_info(self.decode_url)["internal_states"]
self.assertTrue(states, "Decode worker returned no scheduler state")
for state in states:
self.assertIn("avg_spec_accept_length", state)
accept_length = state["avg_spec_accept_length"]
print(f"V4.1 PD+CP DSpark accept length: {accept_length:.4f}")
self.assertGreater(accept_length, 1.0)


if __name__ == "__main__":
unittest.main()
Loading