Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/docs/advanced_features/dcp.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ For context partition `r`, the kernel returns locally normalized output `o_r` an

### Query-projection replication

Normally each layer all-gathers its absorbed and rotary query components. `--dcp-replicate-q-proj` gathers the query-projection and `w_kc` weights once at startup and computes the full DCP-group query locally. The trade is more weight memory and GEMM work for one fewer collective per layer. Only unquantized BF16/FP16 weights use this path; other layers fall back to the query all-gather.
Normally each layer all-gathers its absorbed and rotary query components. `--dcp-replicate-q-proj` gathers the query-projection and `w_kc` weights once at startup and computes the full DCP-group query locally. The trade is more weight memory and GEMM work for one fewer collective per layer. `w_kc` must be BF16/FP16, and the query projection may be unquantized BF16/FP16 or 128x128 block-FP8 (gathered as FP8 weight plus block scales, then post-processed like the per-rank shard so it runs the same GEMM); other layers fall back to the query all-gather.

### Communication backends

Expand Down
82 changes: 63 additions & 19 deletions python/sglang/srt/model_executor/model_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -1124,7 +1124,7 @@ def init_attention_backends(self):
def _prepare_replicated_q_proj(self) -> None:
# --dcp-replicate-q-proj: gather each rank's attn_tp head-shard of
# q_b_proj / w_kc into full-head buffers once here (pre-capture) so the
# MLA decode path can skip the per-layer Q all-gather. bf16/fp16 only.
# MLA decode path can skip the per-layer Q all-gather.
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA

Expand All @@ -1133,28 +1133,20 @@ def _prepare_replicated_q_proj(self) -> None:
return
n_prepared = 0
for m in self.model.modules():
if not isinstance(m, DeepseekV2AttentionMLA):
continue
if m.w_kc is None:
if not isinstance(m, DeepseekV2AttentionMLA) or m.w_kc is None:
continue
qp = m.q_b_proj if m.has_q_b_proj else m.q_proj
# q-replicate only supports the unquantized bf16/fp16 absorb path;
# quantized q-proj (packed weights) and non-16-bit w_kc keep the
# per-layer Q all-gather.
if (
m.w_kc.dtype not in (torch.bfloat16, torch.float16)
or not isinstance(qp.quant_method, UnquantizedLinearMethod)
or qp.weight.dtype not in (torch.bfloat16, torch.float16)
):
logger.warning(
"dcp_replicate_q_proj: skipping quantized q-proj/w_kc "
"(bf16/fp16 only); this layer keeps the Q all-gather."
)
if m.w_kc.dtype not in (torch.bfloat16, torch.float16):
logger.warning("dcp_replicate_q_proj: skipping non-16-bit w_kc")
continue
if isinstance(qp.quant_method, UnquantizedLinearMethod):
m.q_b_proj_qrep_weight = dcp_group.all_gather(qp.weight.data, dim=0)
elif _is_block_fp8(qp):
m.q_b_proj_qrep = _gather_block_fp8_linear(qp, dcp_group)
else:
logger.warning("dcp_replicate_q_proj: skipping unsupported q-proj")
continue
m.w_kc_qrep = dcp_group.all_gather(m.w_kc.contiguous(), dim=0)
m.q_b_proj_qrep_weight = dcp_group.all_gather(
qp.weight.data.contiguous(), dim=0
)
n_prepared += 1
logger.info(
"dcp_replicate_q_proj: prepared full-head Q weights for %d MLA layers",
Expand Down Expand Up @@ -2604,3 +2596,55 @@ def update_model_fields(
load_format=load_format,
)
self.load_config = load_config


def _is_block_fp8(layer: torch.nn.Module) -> bool:
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod

method = layer.quant_method
return (
isinstance(method, Fp8LinearMethod)
and not method.use_mxfp8
and not method.use_marlin
and method.weight_block_size == [128, 128]
and layer.weight.dtype == torch.float8_e4m3fn
# An aiter-preshuffled weight is no longer row-major, so it cannot be
# gathered along the output dim.
and not getattr(layer, "aiter_bpreshuffled", False)
)


def _gather_block_fp8_linear(layer: torch.nn.Module, group) -> torch.nn.Module:
"""Gather a column-sharded block-FP8 linear and re-run its post-load step."""
from sglang.srt.layers.quantization.fp8_utils import (
inverse_transform_scale_ue8m0,
transform_scale_ue8m0,
)

scale = layer.weight_scale_inv.data
packed_ue8m0 = scale.dtype == torch.int32 # DeepGEMM layout
if packed_ue8m0:
scale = inverse_transform_scale_ue8m0(scale, mn=layer.weight.shape[0])
replica = torch.nn.Module()
replica.weight = torch.nn.Parameter(
group.all_gather(layer.weight.data, dim=0), requires_grad=False
)
scale = group.all_gather(scale, dim=0)
if packed_ue8m0:
# The shards' weights are already requantized against these UE8M0
# scales. Re-pack instead of letting the post-load step requantize
# again: that pass is not idempotent (a block whose amax sits at a
# power-of-two boundary halves its scale), so the replica would not
# match the shards bit for bit.
scale = transform_scale_ue8m0(scale, mn=replica.weight.shape[0])
replica.weight_scale_inv = torch.nn.Parameter(scale, requires_grad=False)
replica.weight_scale_inv.format_ue8m0 = packed_ue8m0
# The attributes Fp8LinearMethod's post-load step reads off a LinearBase,
# copied so the replica picks the same kernel (UE8M0 requant, MXFP8 view).
replica.orig_dtype = getattr(layer, "orig_dtype", None)
replica.deepgemm_mxfp8_input = getattr(layer, "deepgemm_mxfp8_input", False)
replica.keep_plain_weight_layout = getattr(layer, "keep_plain_weight_layout", False)
replica.prefix = f"{getattr(layer, 'prefix', 'q_b_proj')}.dcp_replica"
replica.quant_method = layer.quant_method
layer.quant_method.process_weights_after_loading(replica)
return replica
Original file line number Diff line number Diff line change
Expand Up @@ -314,12 +314,11 @@ def forward_absorb_prepare(
and self._can_fuse_bmm_into_attention(forward_batch)
)
# --dcp-replicate-q-proj: project full-head Q locally from pre-gathered
# weights and skip the per-layer Q all-gather (bf16 decode absorb only).
# weights and skip the per-layer Q all-gather.
q_replicate_active = (
self.is_dcp_q_replicate_phase(forward_batch)
and not self.use_deep_gemm_bmm
and self.w_kc_qrep is not None
and self.q_b_proj_qrep_weight is not None
)
if q_replicate_active:
# force standard absorb so the full-head w_kc bmm runs
Expand Down Expand Up @@ -391,11 +390,7 @@ def forward_absorb_prepare(
k_nope = k_nope.unsqueeze(1)
if q_replicate_active:
# full-head Q from the gathered weight (skips Q all-gather)
q = torch.nn.functional.linear(q, self.q_b_proj_qrep_weight).view(
-1,
self.num_local_heads * self.dcp_size,
self.qk_head_dim,
)
q = self.q_b_proj_replicated_forward(q)
else:
q = self.q_b_proj_forward(q)

Expand Down Expand Up @@ -461,13 +456,7 @@ def forward_absorb_prepare(
)
else:
if q_replicate_active:
q = torch.nn.functional.linear(
hidden_states, self.q_b_proj_qrep_weight
).view(
-1,
self.num_local_heads * self.dcp_size,
self.qk_head_dim,
)
q = self.q_b_proj_replicated_forward(hidden_states)
else:
q = self.q_proj(hidden_states)[0].view(
-1, self.num_local_heads, self.qk_head_dim
Expand All @@ -491,10 +480,8 @@ def forward_absorb_prepare(
)
if q_replicate_active:
# full-head absorb with the pre-gathered w_kc (q_nope already full-head)
q_nope_out = (
torch.bmm(q_nope.transpose(0, 1), self.w_kc_qrep)
.transpose(0, 1)
.contiguous()
q_nope_out = torch.bmm(q_nope.transpose(0, 1), self.w_kc_qrep).transpose(
0, 1
)
elif getattr(self, "_kimi_split_gguf_kv_b", False):
from sglang.srt.layers.quantization.gguf import fused_mul_mat_gguf
Expand Down
8 changes: 8 additions & 0 deletions python/sglang/srt/models/deepseek_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -2224,6 +2224,7 @@ def __init__(
# pre-CUDA-graph-capture by the model runner; None unless replicate is on.
self.w_kc_qrep = None
self.q_b_proj_qrep_weight = None
self.q_b_proj_qrep = None

self.w_scale_k = None
self.w_scale_v = None
Expand Down Expand Up @@ -2547,6 +2548,13 @@ def prepare_qkv_latent(
)
return self.fused_qkv_a_proj_with_mqa(hidden_states)[0]

def q_b_proj_replicated_forward(self, x: torch.Tensor) -> torch.Tensor:
if self.q_b_proj_qrep is not None:
q = self.q_b_proj_qrep.quant_method.apply(self.q_b_proj_qrep, x)
else:
q = torch.nn.functional.linear(x, self.q_b_proj_qrep_weight)
return q.view(-1, self.num_local_heads * self.dcp_size, self.qk_head_dim)

def q_b_proj_forward(self, q_lora: torch.Tensor) -> torch.Tensor:
if self._use_min_latency_q_b_gemm is None:
self._use_min_latency_q_b_gemm = (
Expand Down
Loading