Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions python/sglang/srt/arg_groups/deepseek_v4_hook.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,15 +343,21 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
"the prefill CUDA graph",
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
),
# DP attention + TP MoE re-sizes the MoE gather to every rank's tail
# (deepseek_v4 _late_layer_dp_counts, ROCm only); other DP layouts do not.
# Late-layer communication uses one row count per attention-DP rank.
# Attention TP/CP shards need a separate tail padding layout.
(
"DP attention other than TP MoE with attention TP 1 on ROCm",
"DP attention other than attention TP 1 with supported MoE backends",
attn_dp_enabled_of(cfg)
and not (
is_hip()
and cfg.moe_a2a_backend == "none"
and cfg.attn_dp_size == cfg.tp_size
cfg.attn_dp_size == cfg.tp_size
and cfg.attn_cp_size == 1
and (
(is_hip() and cfg.moe_a2a_backend == "none")
Comment thread
kpham-sgl marked this conversation as resolved.
Resolved
or (
get_platform().is_cuda
and cfg.moe_a2a_backend in ("none", "megamoe")
)
)
),
),
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -713,6 +713,16 @@ def _max_dp_batch_size(forward_batch: ForwardBatch) -> int:
return max(request_counts)

def can_run_graph(self, forward_batch: ForwardBatch):
# A prefill peer changes the late-layer DP geometry eagerly. Every target
# rank must execute that boundary; a decode replay would skip it.
if (
get_exec().features.enable_decoder_swa_bounded_replay
and self.require_mlp_sync
and not self.model_runner.is_draft_worker
and forward_batch.is_extend_in_batch
):
return False
Comment thread
YAMY1234 marked this conversation as resolved.
Resolved

# Disable for token embedding overrides (dynamic per-request)
if forward_batch.replace_embeds is not None:
return False
Expand Down
92 changes: 92 additions & 0 deletions python/sglang/srt/model_executor/swa_replay_dp.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
from __future__ import annotations

from typing import TYPE_CHECKING, List, Optional

import torch

from sglang.srt.layers.dp_attention import DpPaddingMode, set_dp_buffer_len_from_batch
from sglang.srt.runtime_context import get_parallel

if TYPE_CHECKING:
from sglang.srt.model_executor.forward_batch_info import ForwardBatch


def late_layer_dp_counts(
forward_batch: ForwardBatch, local_late_rows: int
) -> Optional[List[int]]:
# Bounded replay shrinks an extending rank's late layers to its tail, but the
# TP-MoE gather / combine there spans every DP rank, so each rank needs all
# ranks' late-layer row counts. Eager forwards only: every rank takes this
# branch together, and graph replays never reach the late-layer switch.
early = forward_batch.global_num_tokens_padded_cpu
if early is None:
early = forward_batch.global_num_tokens_cpu
if early is None or torch.cuda.is_current_stream_capturing():
return None
device = forward_batch.global_num_tokens_gpu.device
local = torch.tensor([local_late_rows], dtype=torch.int64, device=device)
counts = torch.empty(len(early), dtype=torch.int64, device=device)
get_parallel().tp_group.all_gather_into_tensor(counts, local)
Comment thread
kpham-sgl marked this conversation as resolved.
Resolved
counts = counts.tolist()
Comment thread
YAMY1234 marked this conversation as resolved.
Resolved
return None if counts == list(early) else counts


def enter_late_layer_dp(forward_batch: ForwardBatch, counts: List[int]) -> tuple:
fb = forward_batch
saved = (
fb.global_num_tokens_cpu,
fb.global_num_tokens_gpu,
fb.global_num_tokens_padded_cpu,
fb.global_num_tokens_for_logprob_cpu,
fb.global_num_tokens_for_logprob_gpu,
fb.global_dp_buffer_len,
fb.dp_padding_mode,
fb.dp_local_start_pos,
fb.dp_local_num_tokens,
fb.global_num_token_non_padded,
fb.global_num_token_non_padded_cpu,
fb.num_token_non_padded,
)
fb.global_num_tokens_cpu = counts
fb.global_num_tokens_gpu = torch.tensor(
counts, dtype=torch.int64, device=fb.global_num_tokens_gpu.device
)
fb.global_num_tokens_padded_cpu = counts
# The MoE gatherv reads its sizes from the logprob counts when present.
fb.global_num_tokens_for_logprob_cpu = counts
fb.global_num_tokens_for_logprob_gpu = fb.global_num_tokens_gpu
fb.global_dp_buffer_len = sum(counts)
# Tail rows are real rows only, so the late layers gather variable lengths.
fb.dp_padding_mode = DpPaddingMode.SUM_LEN
fb.dp_local_start_pos = fb.dp_local_num_tokens = None
# With attention TP=1 the invariant scalar also describes local MoE rows.
# Preserve an idle peer's zero bound even when it carries fabricated rows.
local_rows = counts[get_parallel().attn_dp_rank]
real_rows = min(local_rows, fb.global_num_token_non_padded_cpu)
fb.global_num_token_non_padded_cpu = real_rows
if fb.global_num_token_non_padded is not None:
fb.global_num_token_non_padded = fb.global_num_token_non_padded.new_tensor(
real_rows
)
fb.num_token_non_padded = fb.global_num_token_non_padded
set_dp_buffer_len_from_batch(fb)
return saved


def exit_late_layer_dp(forward_batch: ForwardBatch, saved: tuple) -> None:
fb = forward_batch
(
fb.global_num_tokens_cpu,
fb.global_num_tokens_gpu,
fb.global_num_tokens_padded_cpu,
fb.global_num_tokens_for_logprob_cpu,
fb.global_num_tokens_for_logprob_gpu,
fb.global_dp_buffer_len,
fb.dp_padding_mode,
fb.dp_local_start_pos,
fb.dp_local_num_tokens,
fb.global_num_token_non_padded,
fb.global_num_token_non_padded_cpu,
fb.num_token_non_padded,
) = saved
set_dp_buffer_len_from_batch(fb)
101 changes: 23 additions & 78 deletions python/sglang/srt/models/deepseek_v4.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@
)
from sglang.srt.layers.deep_gemm_wrapper.configurer import DEEPGEMM_SCALE_UE8M0
from sglang.srt.layers.dp_attention import (
DpPaddingMode,
_tbo_event,
attn_tp_all_gather,
attn_tp_all_gather_into_tensor,
Expand All @@ -96,7 +95,6 @@
is_allocation_symmetric,
is_dp_attention_enabled,
is_dp_gatherv_active,
set_dp_buffer_len_from_batch,
)
from sglang.srt.layers.engram import Engram, EngramHasher, EngramLayout
from sglang.srt.layers.layer_boundary import get_attn_tp_context
Expand Down Expand Up @@ -155,6 +153,11 @@
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
get_tc_piecewise_forward_context,
)
from sglang.srt.model_executor.swa_replay_dp import (
enter_late_layer_dp,
exit_late_layer_dp,
late_layer_dp_counts,
)
from sglang.srt.model_loader.utils import maybe_executor_submit, should_async_load
from sglang.srt.model_loader.weight_utils import (
RUNAI_STREAMER_TENSOR_ATTR,
Expand Down Expand Up @@ -3863,68 +3866,6 @@ def _mask_dp_pad_rows(local_hidden_states, forward_batch) -> None:
)


def _late_layer_dp_counts(
forward_batch: ForwardBatch, local_late_rows: int
) -> Optional[List[int]]:
# Bounded replay shrinks an extending rank's late layers to its tail, but the
# TP-MoE gather / combine there spans every DP rank, so each rank needs all
# ranks' late-layer row counts. Eager forwards only: every rank takes this
# branch together, and graph replays never reach the late-layer switch.
early = forward_batch.global_num_tokens_padded_cpu
if early is None:
early = forward_batch.global_num_tokens_cpu
if early is None or torch.cuda.is_current_stream_capturing():
return None
device = forward_batch.global_num_tokens_gpu.device
local = torch.tensor([local_late_rows], dtype=torch.int64, device=device)
counts = torch.empty(len(early), dtype=torch.int64, device=device)
get_parallel().tp_group.all_gather_into_tensor(counts, local)
counts = counts.tolist()
return None if counts == list(early) else counts


def _enter_late_layer_dp(forward_batch: ForwardBatch, counts: List[int]) -> tuple:
fb = forward_batch
saved = (
fb.global_num_tokens_cpu,
fb.global_num_tokens_gpu,
fb.global_num_tokens_padded_cpu,
fb.global_num_tokens_for_logprob_cpu,
fb.global_dp_buffer_len,
fb.dp_padding_mode,
fb.dp_local_start_pos,
fb.dp_local_num_tokens,
)
fb.global_num_tokens_cpu = counts
fb.global_num_tokens_gpu = torch.tensor(
counts, dtype=torch.int64, device=fb.global_num_tokens_gpu.device
)
fb.global_num_tokens_padded_cpu = counts
# The MoE gatherv reads its sizes from the logprob counts when present.
fb.global_num_tokens_for_logprob_cpu = counts
fb.global_dp_buffer_len = sum(counts)
# Tail rows are real rows only, so the late layers gather variable lengths.
fb.dp_padding_mode = DpPaddingMode.SUM_LEN
fb.dp_local_start_pos = fb.dp_local_num_tokens = None
set_dp_buffer_len_from_batch(fb)
return saved


def _exit_late_layer_dp(forward_batch: ForwardBatch, saved: tuple) -> None:
fb = forward_batch
(
fb.global_num_tokens_cpu,
fb.global_num_tokens_gpu,
fb.global_num_tokens_padded_cpu,
fb.global_num_tokens_for_logprob_cpu,
fb.global_dp_buffer_len,
fb.dp_padding_mode,
fb.dp_local_start_pos,
fb.dp_local_num_tokens,
) = saved
set_dp_buffer_len_from_batch(fb)


def _scatter_tail_rows(
tail: LateLayerTail, rows: torch.Tensor, num_tokens: int
) -> torch.Tensor:
Expand Down Expand Up @@ -4178,12 +4119,11 @@ def _forward_layers_hc_pre_from_prev(
late_dp_counts = None
saved_dp = None
if (
_is_hip
and self.late_layer_start is not None
self.late_layer_start is not None
and get_parallel().attn_dp_size > 1
and get_moe_a2a_backend().is_none()
and (forward_batch.is_extend_in_batch or _is_hip)
):
late_dp_counts = _late_layer_dp_counts(
late_dp_counts = late_layer_dp_counts(
forward_batch,
(
tail.token_indices.shape[0]
Expand All @@ -4202,18 +4142,24 @@ def _forward_layers_hc_pre_from_prev(
# Decode reaches back at most SWA_WINDOW positions.
saved_full = attn_backend.enter_late_layer_tail(forward_batch)
state = state.take_rows(tail.rows)
input_ids, input_ids_global = (
tail.rows(input_ids),
tail.rows(input_ids_global),
)
input_ids = tail.rows(input_ids)
if not (
get_parallel().attn_dp_size > 1 and get_moe_a2a_backend().is_none()
):
input_ids_global = tail.rows(input_ids_global)
positions = tail.positions
if hash_ids is not None:
hash_ids = tail.rows(hash_ids)
# late_dp_counts is only not None on hip
if late_dp_counts is not None and i == self.late_layer_start:
saved_dp = _enter_late_layer_dp(forward_batch, late_dp_counts)
# Gathered over the full extend; no late layer is hash-routed.
input_ids_global = None
saved_dp = enter_late_layer_dp(forward_batch, late_dp_counts)
if get_moe_a2a_backend().is_none():
# Full-prefill IDs are not indexed by this rank's tail rows.
# Re-gather the compact IDs in the late-layer MoE layout.
gathered_ids = input_ids.new_empty((sum(late_dp_counts), 1))
dp_gather_replicate(
gathered_ids, input_ids[:, None].clone(), forward_batch
)
input_ids_global = gathered_ids.squeeze(-1)
engram = self.layers[i].engram
if engram is not None:
before_engram = state.residual
Expand Down Expand Up @@ -4270,9 +4216,8 @@ def _forward_layers_hc_pre_from_prev(
mega_mhc_seam_open=i + 1 != self.late_layer_start,
)
state = state.materialized(self.layers[self.end_layer - 1].hc_cfg)
# saved_dp is only not None on hip
if saved_dp is not None:
_exit_late_layer_dp(forward_batch, saved_dp)
exit_late_layer_dp(forward_batch, saved_dp)
if saved_full is not None:
attn_backend.exit_late_layer_tail(saved_full, forward_batch)
return state.residual, state.pre, tail
Expand Down
Loading
Loading