From 6b232c19b80f0200acb2f15c11ab72b0606fc967 Mon Sep 17 00:00:00 2001 From: alichen Date: Fri, 10 Jul 2026 04:00:49 -0700 Subject: [PATCH 01/28] Fix PP proxy buffer sizing and refresh for speculative verify Four related fixes to the pipeline-parallel proxy tensor plumbing that only manifest when PP is combined with speculative decoding (multi-token verify); plain decode (num_tokens_per_bs == 1) is bit-identical: 1. _allocate_decode_buffers / DecodeInputBuffers.create: hidden_states and residual proxy buffers were sized (max_bs, hidden) while every other token-axis buffer (input_ids, positions, and topk_indices in the same dict) uses max_num_token = max_bs * num_tokens_per_bs. Under TARGET_VERIFY the [:num_tokens] slice silently returned fewer rows, crashing rotary with mismatched query/positions during warmup. 2. DecodeCudaGraphRunner.load_batch: the pre-planned early-return path (taken when eagle_prepare_for_verify already ran load_batch) copied input_ids and positions but never refreshed the pp_proxy_tensors input buffers, so the last PP stage replayed verify graphs against stale hidden states. 3. DecodeCudaGraphRunner.execute: the PPProxyTensors output was sliced with [:self.bs] (request rows) instead of [:self.bs * self.num_tokens_per_bs] (token rows). With verify (3 tokens/request) only the first bs rows were forwarded downstream, corrupting every request after the first in a microbatch while single-request runs looked healthy. 4. cuda_graph_buffer_registry: the pp-proxy slot source indexed ppx.tensors[key] unconditionally; an entry can legitimately be absent (e.g. topk_indices when a DSA model runs a dense attention backend). Use .get() so the established None-skips-copy contract applies. Co-Authored-By: Claude Fable 5 --- .../cuda_graph_buffer_registry.py | 5 +++- .../srt/model_executor/runner/base_runner.py | 9 ++++++-- .../runner/decode_cuda_graph_runner.py | 23 ++++++++++++++++++- .../model_executor/runner_utils/buffers.py | 7 ++++-- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py index a90b9bef9527..a4dde273af0f 100644 --- a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py +++ b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py @@ -742,7 +742,10 @@ def _fn(fb, _ctx): def _pp_source(key): def _fn(_fb, ctx): ppx = ctx.pp_proxy_tensors - return None if ppx is None else ppx.tensors[key] + # .get(): a proxy entry can be absent (e.g. topk_indices + # when a DSA model runs a dense attention backend); + # returning None skips the copy for that slot. + return None if ppx is None else ppx.tensors.get(key) return _fn diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index 56420661a9d7..ee0e2266df64 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -110,12 +110,17 @@ def _allocate_decode_buffers( # mHC (e.g. DSV4) flattens residual into hidden_states (size = hc_hidden_size). is_mhc = hc_hidden_size is not None hs = hc_hidden_size if is_mhc else hidden_size + # Sized in tokens, not requests: under speculative decoding the + # verify forward carries num_tokens_per_bs tokens per request and + # _dummy_run slices these buffers to num_tokens (same as + # topk_indices below). Identical for plain decode where + # num_tokens_per_bs == 1. pp_proxy_tensors = { - "hidden_states": torch.zeros((max_bs, hs), dtype=dtype), + "hidden_states": torch.zeros((max_num_token, hs), dtype=dtype), } if not is_mhc: pp_proxy_tensors["residual"] = torch.zeros( - (max_bs, hidden_size), dtype=dtype + (max_num_token, hidden_size), dtype=dtype ) if pp_proxy_topk_size is not None: pp_proxy_tensors["topk_indices"] = torch.zeros( diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index 83a1b4ac5425..5b7dabdb727a 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -912,6 +912,19 @@ def load_batch( # In speculative decoding, these two fields are still needed. self.buffers.input_ids[: self.raw_num_token].copy_(forward_batch.input_ids) self.buffers.positions[: self.raw_num_token].copy_(forward_batch.positions) + if ( + pp_proxy_tensors is not None + and self.buffers.pp_proxy_tensors is not None + ): + # PP + spec verify: the pre-planned load ran without the proxy + # (eagle_prepare_for_verify has no access to it), so the + # graph's proxy input buffers must be refreshed here -- the + # captured graph reads these rows (mirrors fill_from's + # side-slot copy). + for k, v in pp_proxy_tensors.tensors.items(): + buf = self.buffers.pp_proxy_tensors.get(k) + if buf is not None: # skip markers like __msg_type__ + buf[: v.shape[0]].copy_(v) if ( self.model_runner.spec_algorithm.is_dflash() and self.model_runner.is_draft_worker @@ -1056,7 +1069,15 @@ def execute( ) else: assert isinstance(output, PPProxyTensors) - return PPProxyTensors({k: v[: self.bs] for k, v in output.tensors.items()}) + # Slice in token rows, not request rows: under speculative verify + # each request carries num_tokens_per_bs tokens (identical for + # plain decode, where num_tokens_per_bs == 1). + return PPProxyTensors( + { + k: v[: self.bs * self.num_tokens_per_bs] + for k, v in output.tensors.items() + } + ) def get_spec_info(self, num_tokens: int): spec_info = None diff --git a/python/sglang/srt/model_executor/runner_utils/buffers.py b/python/sglang/srt/model_executor/runner_utils/buffers.py index 00d32a24945c..78b0c4ce408e 100644 --- a/python/sglang/srt/model_executor/runner_utils/buffers.py +++ b/python/sglang/srt/model_executor/runner_utils/buffers.py @@ -132,12 +132,15 @@ def create( if pp_size > 1: is_mhc = hc_hidden_size is not None hs = hc_hidden_size if is_mhc else hidden_size + # Sized in tokens, not requests: under speculative decoding + # the verify forward carries num_tokens_per_bs tokens per + # request (same as topk_indices below). pp_proxy_tensors = { - "hidden_states": torch.zeros((max_bs, hs), dtype=dtype), + "hidden_states": torch.zeros((max_num_token, hs), dtype=dtype), } if not is_mhc: pp_proxy_tensors["residual"] = torch.zeros( - (max_bs, hidden_size), dtype=dtype + (max_num_token, hidden_size), dtype=dtype ) if pp_proxy_topk_size is not None: pp_proxy_tensors["topk_indices"] = torch.zeros( From c79d133b3a655ea3797618bb2f4aa1c9b89608d3 Mon Sep 17 00:00:00 2001 From: alichen Date: Fri, 10 Jul 2026 04:04:09 -0700 Subject: [PATCH 02/28] [RFC] Pipeline parallelism x speculative decoding (EAGLE/MTP) compatibility Enables PP + spec-v2 (EAGLE/MTP, topk=1 chains) behind SGLANG_ALLOW_PP_SPEC=1. Default behavior is unchanged; the existing PP+spec assert stays in place unless the flag is set. Architecture (mirrors vLLM #16568's drafter-on-last-stage): - The draft worker exists only on the last PP stage (needs final hidden states and lm_head). Non-last stages run the verify-shaped target chunk through the plain TpModelWorker, which is already PP-aware. - The last stage drafts at the TAIL of its iteration (verify -> draft_extend -> draft-for-next-round): round r's verify tokens must exist before stage 0 runs its half of round r, so drafting cannot stay at the head of the iteration as in the single-stage flow. - Spec state rides the existing last->first output-dict relay (which every stage already forwards): accept_lens, new_seq_lens, bonus token, and the tail-drafted chain for the next round. No new communication channel. - Chains are stored per-request (rid-keyed) and every stage rebuilds an identical EagleVerifyInput at compose time via build_tree_kernel_efficient with the topk=1 chain constants -- no draft model needed off the last stage, and the design is robust to microbatch recomposition (finish/retract/merge) between rounds. - Batch results are processed against a launch-time ScheduleBatch.copy() snapshot (PPBatchMetadata.fwd_batch); the live microbatch object can be merged/filtered in place before its relayed result arrives. seq_lens is scatter-updated per rid. - KV bookkeeping needs no new plumbing: kv_committed_len advances in the shared batch-result processor from the relayed CPU accept_lens, and the verify slots stay within eagle_prepare_for_decode's 2x reserve (same mechanism the overlap path relies on). Validated on 8x RTX PRO 6000 (SM120, PCIe), GLM-5.2-NVFP4, TP4xPP2 + EAGLE(2 steps / topk 1 / 3 draft tokens): - greedy decode identical to non-spec; GSM8K 20q 0.900 (= TP8 baseline) - concurrent/staggered workloads correct (relay verified token-exact) Known limitations (hence RFC): - topk=1 chains only; tree speculation needs rebuild-logic extensions - non-overlap schedule only (PP itself requires that today) - draft-model weight loading on the last stage must not apply PP filtering (the NextN/MTP embed_tokens is shared with the target model and lives in the first-stage partition; loading the draft with the target's pp_rank leaves its embedding randomly initialized and collapses acceptance length) -- follow-up in progress - depends on the PP proxy buffer fixes in the companion PR Co-Authored-By: Claude Fable 5 --- python/sglang/srt/managers/scheduler.py | 113 +++++++-- .../batch_result_processor.py | 9 +- .../sglang/srt/managers/scheduler_pp_mixin.py | 228 +++++++++++++++++- python/sglang/srt/managers/tp_worker.py | 23 +- python/sglang/srt/managers/utils.py | 4 + .../sglang/srt/model_executor/model_runner.py | 34 ++- .../model_runner_kv_cache_mixin.py | 8 +- python/sglang/srt/server_args.py | 10 +- .../sglang/srt/speculative/eagle_worker_v2.py | 68 +++++- 9 files changed, 437 insertions(+), 60 deletions(-) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index d464e74af250..515430ad6b64 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -771,6 +771,17 @@ def maybe_init_draft_worker(self): self.draft_worker = None self.external_corpus_manager = None return + import os as _os + if ( + _os.environ.get("SGLANG_ALLOW_PP_SPEC") + and self.server_args.pp_size > 1 + and self.ps.pp_rank != self.server_args.pp_size - 1 + ): + # PP+spec: the draft model (MTP layer) needs final hidden states and + # the lm_head, both of which live on the last PP stage only. + self.draft_worker = None + self.external_corpus_manager = None + return # Launch a draft worker for speculative decoding draft_worker_kwargs = dict( @@ -858,7 +869,9 @@ def init_model_worker(self): model_runner.post_capture_resize_kv_pool() # Dispatch the model worker - if self.spec_algorithm.is_none(): + if self.spec_algorithm.is_none() or self.draft_worker is None: + # PP+spec: non-last stages have no draft worker; they run the + # verify-shaped target forward through the plain tp_worker. self.model_worker = self.tp_worker else: self.model_worker = self.draft_worker @@ -1126,7 +1139,7 @@ def init_disaggregation(self): server_args=self.server_args, ) - if self.spec_algorithm.carries_draft_hidden_states(): + if self.spec_algorithm.carries_draft_hidden_states() and self.draft_worker is not None: # `draft_runner` aliases `draft_runner_list[0]` in the multi-layer # worker, so a single accessor covers both shapes. draft_runner = self.draft_worker.draft_worker.draft_runner @@ -3322,27 +3335,83 @@ def run_batch( self._relay_forward_payload(batch.req_pool_indices, batch_result) batch.input_ids = None elif not batch.spec_algorithm.is_none(): - # Non-overlap: drive the V2 worker synchronously (no - # future_map relay / on_publish). - resolve_forward_inputs(batch, self.future_map) - with self._forward_isolation(batch, overlap=False): - batch_result = self.model_worker.forward_batch_generation(batch) - # The isolation restore reverted the worker's in-forward SB edits; - # re-apply what must carry to the next iter. - batch.spec_info = batch_result.next_draft_input - if batch_result.new_seq_lens is not None: - batch.seq_lens = batch_result.new_seq_lens - if batch.seq_lens_cpu is not None: - batch.seq_lens_cpu = batch_result.new_seq_lens.to("cpu") - batch.seq_lens_sum = int(batch.seq_lens_cpu.sum()) - batch.input_ids = None # rebuilt next iter from draft_token - self.update_cache_from_scheduler(batch, batch_result) - # Sync D2H so the result processor can read CPU tensors. - batch_result.copy_done = self.device_module.Event() - batch_result.copy_to_cpu( - return_logprob=batch.return_logprob, - return_hidden_states=batch.return_hidden_states, + is_verify_round = self.server_args.pp_size > 1 and not ( + batch.forward_mode.is_extend() or batch.is_extend_in_batch ) + if is_verify_round: + # PP+spec decode: every stage rebuilds the same verify + # input from relayed per-req state (draft lives on the + # last stage only). + self._pp_spec_rebuild_verify_input(batch) + if not self.pp_group.is_last_rank: + # PP+spec: non-last stages run only their model chunk on + # the verify-shaped batch; sampling, accept and draft all + # live on the last stage. The plain tp_worker path already + # returns pp_hidden_states_proxy_tensors for relay. + resolve_forward_inputs(batch, self.future_map) + if is_verify_round: + from sglang.srt.speculative.eagle_utils import ( + eagle_prepare_for_verify, + ) + + # Isolation is load-bearing: eagle_prepare_for_verify + # mutates SB fields (forward_mode -> TARGET_VERIFY, + # input_ids, out_cache_loc); without the restore the + # next get_next_batch_to_run treats this decode batch + # as extend and re-merges it (duplicate reqs). + with self._forward_isolation(batch, overlap=False): + verify_forward_batch, can_run_cuda_graph = ( + eagle_prepare_for_verify( + batch.spec_info, + self.req_to_token_pool, + batch, + self.tp_worker, + ) + ) + batch_result = self.tp_worker.forward_batch_generation( + batch=None, + forward_batch=verify_forward_batch, + pp_proxy_tensors=pp_proxy_tensors, + is_verify=True, + ) + batch_result.can_run_cuda_graph = can_run_cuda_graph + else: + batch_result = self.model_worker.forward_batch_generation( + batch, pp_proxy_tensors=pp_proxy_tensors + ) + batch.input_ids = None # rebuilt next iter from draft_token + # The verify input is per-round; between iterations + # spec_info must be merge/filter-safe (None on stages + # without a draft worker). + batch.spec_info = None + else: + # Non-overlap: drive the V2 worker synchronously (no + # future_map relay / on_publish). + resolve_forward_inputs(batch, self.future_map) + with self._forward_isolation(batch, overlap=False): + batch_result = self.model_worker.forward_batch_generation( + batch, pp_proxy_tensors=pp_proxy_tensors + ) + # The isolation restore reverted the worker's in-forward SB edits; + # re-apply what must carry to the next iter. + batch.spec_info = batch_result.next_draft_input + if batch_result.new_seq_lens is not None: + batch.seq_lens = batch_result.new_seq_lens + if batch.seq_lens_cpu is not None: + batch.seq_lens_cpu = batch_result.new_seq_lens.to("cpu") + batch.seq_lens_sum = int(batch.seq_lens_cpu.sum()) + batch.input_ids = None # rebuilt next iter from draft_token + self.update_cache_from_scheduler(batch, batch_result) + if self.server_args.pp_size == 1: + # Sync D2H so the result processor can read CPU tensors. + # Under PP this local result is only relayed (GPU send); + # processing consumes the round-tripped copy, which + # _pp_prep_batch_result already lands on CPU. + batch_result.copy_done = self.device_module.Event() + batch_result.copy_to_cpu( + return_logprob=batch.return_logprob, + return_hidden_states=batch.return_hidden_states, + ) else: kwargs = ( {"pp_proxy_tensors": pp_proxy_tensors} diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index 8d0888064f16..a8e608cb7696 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -545,9 +545,12 @@ def _resolve_spec_v2_tokens( # Feed the adaptive controller now that accept_lens is on CPU, # instead of doing a synchronous GPU→CPU copy in the worker hot path. # BaseSpecWorker provides a no-op default for non-adaptive workers. - self.model_worker.on_verify_complete_cpu( - result.num_correct_drafts_per_req_cpu, batch_size=len(batch.reqs) - ) + # PP+spec non-last stages process relayed results through the plain + # TpModelWorker, which has no adaptive controller. + if hasattr(self.model_worker, "on_verify_complete_cpu"): + self.model_worker.on_verify_complete_cpu( + result.num_correct_drafts_per_req_cpu, batch_size=len(batch.reqs) + ) predict_tokens = [] # In adaptive spec-v2, the worker state may already have switched when this diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 2cb2c7bf0c4c..37d6c86be4c5 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -61,6 +61,11 @@ def _pp_can_skip_output_comm(batch: ScheduleBatch) -> bool: @dataclass class PPBatchMetadata: can_run_cuda_graph: bool + # PP+spec: forward-time snapshot of the microbatch (ScheduleBatch.copy()). + # The live mb object can be merged/filtered in place before its relayed + # result arrives, so relayed tensors must be applied against the + # composition that actually ran the forward. + fwd_batch: Optional[ScheduleBatch] = None class SchedulerPPMixin: @@ -147,9 +152,15 @@ def event_loop_pp(self: Scheduler): ) if self.mbs[next_mb_id] is not None: d2h_event.synchronize() + process_target = self.mbs[next_mb_id] + next_md = self.mb_metadata[next_mb_id] + if next_md is not None and next_md.fwd_batch is not None: + # PP+spec: process against the forward-time snapshot; + # the live mb may have been recomposed since launch. + process_target = next_md.fwd_batch with torch.profiler.record_function("process_batch_result"): self._pp_process_batch_result( - self.mbs[next_mb_id], + process_target, next_batch_result, ) self.last_mbs[next_mb_id] = self.mbs[next_mb_id] @@ -998,6 +1009,17 @@ def _pp_prepare_tensor_dict( "next_token_ids": result.next_token_ids, } + if not batch.spec_algorithm.is_none() and result.accept_lens is not None: + # PP+spec verify round: earlier stages need the accept results to + # mirror seq_lens/KV bookkeeping and the bonus token to root the + # next round's verify chain. + tensor_dict["spec_accept_lens"] = result.accept_lens + tensor_dict["spec_new_seq_lens"] = result.new_seq_lens + tensor_dict["spec_bonus_tokens"] = result.next_draft_input.bonus_tokens + if result.next_verify_chain is not None: + # Tail-drafted chain for the next verify round (root = bonus). + tensor_dict["spec_next_chain"] = result.next_verify_chain + if batch.return_logprob: logprob_dict = get_logprob_dict_from_result(result) tensor_dict = { @@ -1129,13 +1151,75 @@ def _pp_prep_batch_result( extend_input_len_per_req, extend_logprob_start_len_per_req, ) = get_logprob_from_pp_outputs(pp_outputs) + is_spec = not batch.spec_algorithm.is_none() + if is_spec and "spec_accept_lens" in pp_outputs.tensors: + # PP+spec verify round. Relayed tensors align with the composition + # that ran the forward (mb_metadata.fwd_batch snapshot); the live + # mb object may have been merged/filtered in place since. Mirror + # the seq state per-rid onto the live batch, stash the bonus + # tokens that root the next verify chain, and rebuild a result the + # spec output processor can consume (CPU tensors). + fwd_batch = mb_metadata.fwd_batch if mb_metadata.fwd_batch else batch + new_seq_lens = pp_outputs["spec_new_seq_lens"] + fwd_rids = [req.rid for req in fwd_batch.reqs] + live_rids = [req.rid for req in batch.reqs] + if live_rids == fwd_rids: + batch.seq_lens = new_seq_lens + if batch.seq_lens_cpu is not None: + batch.seq_lens_cpu = new_seq_lens.to("cpu") + batch.seq_lens_sum = int(batch.seq_lens_cpu.sum()) + else: + row_by_rid = {rid: i for i, rid in enumerate(fwd_rids)} + new_cpu = new_seq_lens.tolist() + cur_cpu = batch.seq_lens.tolist() + merged = [ + new_cpu[row_by_rid[rid]] if rid in row_by_rid else cur_cpu[j] + for j, rid in enumerate(live_rids) + ] + batch.seq_lens = torch.tensor( + merged, dtype=batch.seq_lens.dtype, device=batch.seq_lens.device + ) + if batch.seq_lens_cpu is not None: + batch.seq_lens_cpu = torch.tensor( + merged, dtype=batch.seq_lens_cpu.dtype + ) + batch.seq_lens_sum = int(sum(merged)) + self._pp_spec_store_bonus( + fwd_batch, + pp_outputs["spec_bonus_tokens"], + chain_tokens=( + pp_outputs["spec_next_chain"] + if "spec_next_chain" in pp_outputs.tensors + else None + ), + ) + output_result = GenerationBatchResult( + logits_output=logits_output, + pp_hidden_states_proxy_tensors=None, + next_token_ids=pp_outputs["next_token_ids"].cpu(), + accept_lens=pp_outputs["spec_accept_lens"].cpu(), + new_seq_lens=new_seq_lens, + speculative_num_draft_tokens=self.server_args.speculative_num_draft_tokens, + extend_input_len_per_req=extend_input_len_per_req, + extend_logprob_start_len_per_req=extend_logprob_start_len_per_req, + can_run_cuda_graph=mb_metadata.can_run_cuda_graph, + ) + return output_result + batch.input_ids = pp_outputs["next_token_ids"].to(torch.int64) - # PP rank 0 also relays into output_tokens_buf so the next iter's - # resolve_forward_inputs finds these tokens for the decode portion - # of mixed-chunk batches (which gather via mix_running_indices). - self.future_map.stash( - batch.req_pool_indices, RelayPayload(bonus_tokens=batch.input_ids) - ) + if is_spec: + # Spec prefill round: the sampled first token roots round 1's chain. + fwd_batch = mb_metadata.fwd_batch if mb_metadata.fwd_batch else batch + self._pp_spec_store_bonus(fwd_batch, batch.input_ids) + else: + # PP rank 0 also relays into output_tokens_buf so the next iter's + # resolve_forward_inputs finds these tokens for the decode portion + # of mixed-chunk batches (which gather via mix_running_indices). + # (Spec FutureMap stores draft-input payloads, not bare bonus + # tokens, so the stash is non-spec only.) + self.future_map.stash( + batch.req_pool_indices, RelayPayload(bonus_tokens=batch.input_ids) + ) output_result = GenerationBatchResult( logits_output=logits_output, pp_hidden_states_proxy_tensors=None, @@ -1151,6 +1235,131 @@ def _pp_process_batch_result( ): self.process_batch_result(batch, output_result) + def _pp_spec_store_bonus( + self: Scheduler, + batch: ScheduleBatch, + bonus_tokens: torch.Tensor, + chain_tokens: Optional[torch.Tensor] = None, + ) -> None: + """Stash the per-request chain row (root = bonus token) that seeds the + next verify round. + + chain_tokens is the last stage's tail-drafted chain (flat bs*dtn); + without it (prefill rounds) a degenerate row [bonus, 0, ...] is stored + — the zero drafts just get rejected, costing acceptance not + correctness. Keyed by rid (not batch position): the microbatch + composition can change between rounds (finish / retract / merge).""" + if not hasattr(self, "_pp_spec_chain_by_rid"): + self._pp_spec_chain_by_rid = {} + num_draft_tokens = self.server_args.speculative_num_draft_tokens + if chain_tokens is not None: + rows = chain_tokens.to(torch.int64).reshape( + len(batch.reqs), num_draft_tokens + ) + else: + rows = torch.zeros( + (len(batch.reqs), num_draft_tokens), + dtype=torch.int64, + device=bonus_tokens.device, + ) + rows[:, 0] = bonus_tokens.to(torch.int64) + for i, req in enumerate(batch.reqs): + if req.finished(): + self._pp_spec_chain_by_rid.pop(req.rid, None) + else: + self._pp_spec_chain_by_rid[req.rid] = rows[i] + + def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None: + """Rebuild batch.spec_info (EagleVerifyInput) from relayed per-request + state, without a draft model. + + Phase 1 (correctness bring-up): degenerate topk=1 chains — the root is + the relayed bonus token (always accepted by the verify kernel), padded + with zero draft tokens that simply get rejected. Verify is sound for + arbitrary proposals, so this costs acceptance rate, not correctness. + Phase 2 relays the last stage's tail-drafted real chains instead.""" + from sglang.srt.speculative.eagle_info import EagleVerifyInput + from sglang.srt.speculative.eagle_utils import ( + TreeMaskMode, + build_tree_kernel_efficient, + ) + + sa = self.server_args + steps = sa.speculative_num_steps + num_draft_tokens = sa.speculative_num_draft_tokens + bs = batch.batch_size() + device = self.device + + if batch.forward_mode.is_idle() or bs == 0: + batch.spec_info = EagleVerifyInput.create_idle_input( + topk=sa.speculative_eagle_topk, + spec_steps=steps, + num_verify_tokens=num_draft_tokens, + ) + return + + chain_rows = torch.stack( + [self._pp_spec_chain_by_rid[req.rid] for req in batch.reqs] + ).to(device=device, dtype=torch.int64) + bonus_tokens = chain_rows[:, 0].contiguous() + draft_tokens = chain_rows[:, 1:].contiguous() + # topk=1 chain constants (mirrors EagleDraftWorker._rebuild_topk1_chain_buffers) + parent_width = steps if steps > 1 else 0 + parent_list = torch.arange( + -1, parent_width - 1, dtype=torch.long, device=device + ).repeat(bs, 1) + top_scores_index = torch.arange( + steps, dtype=torch.long, device=device + ).repeat(bs, 1) + + attn_backend = self.tp_worker.model_runner.attn_backend + tree_mask_buf, position_buf = ( + attn_backend.get_verify_buffers_to_fill_after_draft() + ) + seq_lens_sum = batch.seq_lens_sum + if seq_lens_sum is None: + if tree_mask_buf is None: + seq_lens_sum = bs * attn_backend.max_context_len + else: + seq_lens_sum = 0 # preallocated buf -> kernel ignores it + + ( + tree_mask, + positions, + retrieve_index, + retrieve_next_token, + retrieve_next_sibling, + flat_draft_tokens, + ) = build_tree_kernel_efficient( + bonus_tokens, + parent_list, + top_scores_index, + draft_tokens, + batch.seq_lens, + seq_lens_sum, + sa.speculative_eagle_topk, + steps, + num_draft_tokens, + TreeMaskMode.FULL_MASK, + tree_mask_buf, + position_buf, + ) + batch.spec_info = EagleVerifyInput( + draft_token=flat_draft_tokens, + custom_mask=tree_mask, + positions=positions, + retrieve_index=retrieve_index, + retrieve_next_token=retrieve_next_token, + retrieve_next_sibling=retrieve_next_sibling, + retrieve_cum_len=None, + spec_steps=steps, + topk=sa.speculative_eagle_topk, + draft_token_num=num_draft_tokens, + capture_hidden_mode=None, + seq_lens_sum=batch.seq_lens_sum, + seq_lens_cpu=batch.seq_lens_cpu, + ) + def _pp_send_output_to_next_stage( self: Scheduler, next_first_rank_mb_id: int, @@ -1280,6 +1489,11 @@ def _pp_launch_batch( ) mb_metadata[mb_id] = PPBatchMetadata( can_run_cuda_graph=result.can_run_cuda_graph, + fwd_batch=( + self.cur_batch.copy() + if not self.cur_batch.spec_algorithm.is_none() + else None + ), ) event = self.device_module.Event() event.record(self.device_module.current_stream()) diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index 3db49daf82c0..370b090bb831 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -307,12 +307,23 @@ def __init__( self.world_group = get_world_group() # Sync random seed across TP workers - self.random_seed = broadcast_pyobj( - [server_args.random_seed], - self.tp_size * self.pp_rank + tp_rank, - self.world_group.cpu_group, - src=self.world_group.ranks[0], - )[0] + import os as _os + if ( + _os.environ.get("SGLANG_ALLOW_PP_SPEC") + and is_draft_worker + and server_args.pp_size > 1 + ): + # PP+spec: the draft worker exists only on the last PP stage, so a + # world-group broadcast here would deadlock (first-stage ranks never + # join). The seed was already synced during target worker init. + self.random_seed = server_args.random_seed + else: + self.random_seed = broadcast_pyobj( + [server_args.random_seed], + self.tp_size * self.pp_rank + tp_rank, + self.world_group.cpu_group, + src=self.world_group.ranks[0], + )[0] set_random_seed(self.random_seed) self.enable_overlap = not server_args.disable_overlap_schedule diff --git a/python/sglang/srt/managers/utils.py b/python/sglang/srt/managers/utils.py index 87fbb35ded8d..cd2595bf6ec5 100644 --- a/python/sglang/srt/managers/utils.py +++ b/python/sglang/srt/managers/utils.py @@ -69,6 +69,10 @@ class GenerationBatchResult: # relay path: forward stream -> next step forward next_draft_input: Optional[EagleDraftInput] = None + # PP+spec: tail-drafted chain tokens (flat bs*num_draft_tokens, root = + # bonus) for the NEXT verify round, relayed last stage -> all stages. + next_verify_chain: Optional[torch.Tensor] = None + # Refs the worker wants scheduler to keep alive for the same 2-iter window # as batch_record_buf. Used for cross-stream tensor lifetime (e.g. a spec # V2 verify ForwardBatch whose tensors must outlive mid-iter SB rebinds). diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index ca641d1fe806..9943d9c3628c 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -603,9 +603,11 @@ def __init__( ) if self.pp_size > 1: - assert ( - self.support_pp - ), "Pipeline Parallel is not compatible with this model." + import os as _os + if not (_os.environ.get("SGLANG_ALLOW_PP_SPEC") and self.is_draft_worker): + assert ( + self.support_pp + ), "Pipeline Parallel is not compatible with this model." # For weight updates self._model_update_group = {} @@ -767,14 +769,16 @@ def initialize(self): if loop_num > 1: self.num_effective_layers = self.num_effective_layers * loop_num - assert ( - (not model_has_mtp_layers) - or (self.spec_algorithm.is_none()) - or ( - (not self.spec_algorithm.is_none()) - and (self.num_effective_layers == model_num_layers) - ) - ), "PP is not compatible with MTP models." + import os as _os + if not _os.environ.get("SGLANG_ALLOW_PP_SPEC"): + assert ( + (not model_has_mtp_layers) + or (self.spec_algorithm.is_none()) + or ( + (not self.spec_algorithm.is_none()) + and (self.num_effective_layers == model_num_layers) + ) + ), "PP is not compatible with MTP models." # Apply torchao quantization torchao_applied = getattr(self.model, "torchao_applied", False) @@ -1282,10 +1286,16 @@ def init_torch_distributed(self): f"(tp_size={self.tp_size}, pp_size={self.pp_size}, ep_size={self.moe_ep_size})" ) + _pp_spec_draft_local = ( + __import__("os").environ.get("SGLANG_ALLOW_PP_SPEC") + and getattr(self, "is_draft_worker", False) + and self.pp_size > 1 + ) pre_model_load_memory = get_available_gpu_memory( self.device, self.gpu_id, - distributed=get_world_group().world_size > 1, + distributed=(get_world_group().world_size > 1) + and not _pp_spec_draft_local, cpu_group=get_world_group().cpu_group, ) self.tp_group = get_tp_group() diff --git a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py index 9ae58cede93e..9a711cd73ab0 100644 --- a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py +++ b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py @@ -105,10 +105,16 @@ def _profile_available_bytes(self: ModelRunner, pre_model_load_memory: int) -> i # KV pool budget = currently-free GPU memory minus the non-static runtime # slack (pre_model_load_memory * (1 - mem_fraction_static)). Whatever is # already resident (model weights, etc.) is thus charged against it. + _pp_spec_draft_local = ( + __import__("os").environ.get("SGLANG_ALLOW_PP_SPEC") + and getattr(self, "is_draft_worker", False) + and self.pp_size > 1 + ) available_gpu_memory = get_available_gpu_memory( self.device, self.gpu_id, - distributed=get_world_group().world_size > 1, + distributed=(get_world_group().world_size > 1) + and not _pp_spec_draft_local, cpu_group=get_world_group().cpu_group, ) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 3461b8aebdb2..31fde4ca2e46 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -6921,9 +6921,13 @@ def check_server_args(self): ) if self.pp_size > 1: - assert ( - self.disable_overlap_schedule and self.speculative_algorithm is None - ), "Pipeline parallelism is not compatible with overlap schedule, speculative decoding" + import os as _os + if not _os.environ.get("SGLANG_ALLOW_PP_SPEC"): + assert ( + self.disable_overlap_schedule and self.speculative_algorithm is None + ), "Pipeline parallelism is not compatible with overlap schedule, speculative decoding" + else: + assert self.disable_overlap_schedule assert not ( self.dp_size > 1 and self.nnodes != 1 and not self.enable_dp_attention diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 624dedf1b43c..e225727c9d3c 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -40,7 +40,11 @@ Phase, check_cuda_graph_backend, ) -from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode, ForwardBatch +from sglang.srt.model_executor.forward_batch_info import ( + CaptureHiddenMode, + ForwardBatch, + ForwardMode, +) from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.model_executor.runner import ( DecodeCudaGraphRunner, @@ -333,6 +337,15 @@ def init_token_map(self): self.hot_token_id = None def init_lm_head(self): + import os as _os + if _os.environ.get("SGLANG_ALLOW_PP_SPEC") and self.server_args.pp_size > 1: + # PP+spec: the target's embedding lives on the first PP stage + # (PPMissingLayer here on the last stage), so the draft keeps its + # own embedding weights and shares only the target's lm_head. + embed = self.draft_runner.model.model.embed_tokens.weight + head = self.target_worker.model_runner.model.lm_head.weight + self.draft_runner.model.set_embed_and_head(embed, head) + return embed, head = self.target_worker.model_runner.model.get_embed_and_head() target_lm_head = getattr(self.target_worker.model_runner.model, "lm_head", None) @@ -1175,7 +1188,9 @@ def clear_cache_pool(self): # allocator and kv cache pool are shared with target worker, which are cleared in scheduler pass - def forward_batch_generation(self, batch: ScheduleBatch, on_publish=None): + def forward_batch_generation( + self, batch: ScheduleBatch, on_publish=None, pp_proxy_tensors=None + ): if batch.forward_mode.is_extend() or batch.is_extend_in_batch: # Target prefill target_capture_mode = ( @@ -1184,7 +1199,9 @@ def forward_batch_generation(self, batch: ScheduleBatch, on_publish=None): else CaptureHiddenMode.FULL ) batch.capture_hidden_mode = target_capture_mode - batch_output = self.target_worker.forward_batch_generation(batch) + batch_output = self.target_worker.forward_batch_generation( + batch, pp_proxy_tensors=pp_proxy_tensors + ) # Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens. # Extend processed L prompt tokens; next verify iter expects same L. @@ -1231,7 +1248,12 @@ def forward_batch_generation(self, batch: ScheduleBatch, on_publish=None): capture_hidden_mode=capture_mode, vocab_size=self.target_worker.model_config.vocab_size, ) - if self.speculative_num_steps == 0: + if batch.spec_info is not None and batch.spec_info.is_verify_input(): + # PP+spec: the scheduler pre-built this round's verify input + # from relayed per-req chains — it must match what earlier + # stages already ran, so do not re-draft here. + verify_input = batch.spec_info + elif self.speculative_num_steps == 0: # Drafting disabled (high batch size). _draft_extend below still # runs, keeping draft KV warm for when the batch shrinks. verify_input = self._build_trivial_verify_input(batch) @@ -1247,7 +1269,7 @@ def forward_batch_generation(self, batch: ScheduleBatch, on_publish=None): verify_input: EagleVerifyInput = self.draft_worker.draft(batch) assert verify_input.is_verify_input() batch.spec_info = verify_input - batch_output = self.verify(batch) + batch_output = self.verify(batch, pp_proxy_tensors=pp_proxy_tensors) # Publish before draft_extend so the fence is at verify-end. if on_publish is not None: on_publish(batch_output.new_seq_lens) @@ -1267,6 +1289,39 @@ def forward_batch_generation(self, batch: ScheduleBatch, on_publish=None): ): self.draft_worker._draft_extend_for_decode(batch, batch_output) + if ( + self.server_args.pp_size > 1 + and not batch.forward_mode.is_idle() + and self.speculative_num_steps > 0 + ): + # PP tail-draft: draft the NEXT round's chain now — earlier + # stages must have the tokens before running their half of the + # next verify forward, so drafting cannot wait for the next + # iteration. Mimic the head-of-iteration state draft() expects; + # the scheduler's forward isolation reverts these SB edits, and + # the chain rides out on batch_output. + batch.spec_info = batch_output.next_draft_input + batch.seq_lens = batch_output.new_seq_lens + batch.forward_mode = ForwardMode.DECODE + # eagle_prepare_for_verify left the verify tokens here; the + # head-of-iteration draft always sees None (the scheduler + # clears it), so mirror that state. + batch.input_ids = None + # Attention metadata planning reads the CPU copies; one D2H + # per round (TODO: async or upper-bound estimate). + batch.seq_lens_cpu = batch_output.new_seq_lens.to("cpu") + batch.seq_lens_sum = int(batch.seq_lens_cpu.sum()) + with ( + self.draft_worker.draft_tp_context( + self.draft_worker.draft_runner.tp_group + ), + speculative_moe_backend_context(), + speculative_moe_a2a_backend_context(), + spec_stage_span("draft"), + ): + next_verify_input = self.draft_worker.draft(batch) + batch_output.next_verify_chain = next_verify_input.draft_token + return batch_output def _build_trivial_verify_input(self, batch: ScheduleBatch) -> EagleVerifyInput: @@ -1550,7 +1605,7 @@ def _override_worker_state( ) dw._rebuild_topk1_chain_buffers() - def verify(self, batch: ScheduleBatch): + def verify(self, batch: ScheduleBatch, pp_proxy_tensors=None): fwd_stream = torch.get_device_module(self.device).current_stream() verify_input: EagleVerifyInput = batch.spec_info record_stream_for_v2_verify(batch, verify_input, fwd_stream) @@ -1618,6 +1673,7 @@ def verify(self, batch: ScheduleBatch): batch=None, forward_batch=verify_forward_batch, is_verify=True, + pp_proxy_tensors=pp_proxy_tensors, ) logits_output = forward_batch_output.logits_output From 682d323a19333d94a9e75f02da651c42e3666131 Mon Sep 17 00:00:00 2001 From: alichen Date: Mon, 13 Jul 2026 02:24:13 -0700 Subject: [PATCH 03/28] Fix lint: apply black/isort formatting Co-Authored-By: Claude Fable 5 --- python/sglang/srt/managers/scheduler.py | 6 +++++- python/sglang/srt/managers/scheduler_pp_mixin.py | 6 +++--- python/sglang/srt/managers/tp_worker.py | 1 + python/sglang/srt/model_executor/model_runner.py | 5 +++-- .../srt/model_executor/model_runner_kv_cache_mixin.py | 3 +-- python/sglang/srt/server_args.py | 1 + python/sglang/srt/speculative/eagle_worker_v2.py | 1 + 7 files changed, 15 insertions(+), 8 deletions(-) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 515430ad6b64..9fe10f09c3a1 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -772,6 +772,7 @@ def maybe_init_draft_worker(self): self.external_corpus_manager = None return import os as _os + if ( _os.environ.get("SGLANG_ALLOW_PP_SPEC") and self.server_args.pp_size > 1 @@ -1139,7 +1140,10 @@ def init_disaggregation(self): server_args=self.server_args, ) - if self.spec_algorithm.carries_draft_hidden_states() and self.draft_worker is not None: + if ( + self.spec_algorithm.carries_draft_hidden_states() + and self.draft_worker is not None + ): # `draft_runner` aliases `draft_runner_list[0]` in the multi-layer # worker, so a single accessor covers both shapes. draft_runner = self.draft_worker.draft_worker.draft_runner diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 37d6c86be4c5..7dca497b924d 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -1308,9 +1308,9 @@ def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None parent_list = torch.arange( -1, parent_width - 1, dtype=torch.long, device=device ).repeat(bs, 1) - top_scores_index = torch.arange( - steps, dtype=torch.long, device=device - ).repeat(bs, 1) + top_scores_index = torch.arange(steps, dtype=torch.long, device=device).repeat( + bs, 1 + ) attn_backend = self.tp_worker.model_runner.attn_backend tree_mask_buf, position_buf = ( diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index 370b090bb831..b4dd8fb26bcf 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -308,6 +308,7 @@ def __init__( # Sync random seed across TP workers import os as _os + if ( _os.environ.get("SGLANG_ALLOW_PP_SPEC") and is_draft_worker diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 9943d9c3628c..ddca90eaa113 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -604,6 +604,7 @@ def __init__( if self.pp_size > 1: import os as _os + if not (_os.environ.get("SGLANG_ALLOW_PP_SPEC") and self.is_draft_worker): assert ( self.support_pp @@ -770,6 +771,7 @@ def initialize(self): self.num_effective_layers = self.num_effective_layers * loop_num import os as _os + if not _os.environ.get("SGLANG_ALLOW_PP_SPEC"): assert ( (not model_has_mtp_layers) @@ -1294,8 +1296,7 @@ def init_torch_distributed(self): pre_model_load_memory = get_available_gpu_memory( self.device, self.gpu_id, - distributed=(get_world_group().world_size > 1) - and not _pp_spec_draft_local, + distributed=(get_world_group().world_size > 1) and not _pp_spec_draft_local, cpu_group=get_world_group().cpu_group, ) self.tp_group = get_tp_group() diff --git a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py index 9a711cd73ab0..63ee84432bfc 100644 --- a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py +++ b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py @@ -113,8 +113,7 @@ def _profile_available_bytes(self: ModelRunner, pre_model_load_memory: int) -> i available_gpu_memory = get_available_gpu_memory( self.device, self.gpu_id, - distributed=(get_world_group().world_size > 1) - and not _pp_spec_draft_local, + distributed=(get_world_group().world_size > 1) and not _pp_spec_draft_local, cpu_group=get_world_group().cpu_group, ) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 31fde4ca2e46..74623c334032 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -6922,6 +6922,7 @@ def check_server_args(self): if self.pp_size > 1: import os as _os + if not _os.environ.get("SGLANG_ALLOW_PP_SPEC"): assert ( self.disable_overlap_schedule and self.speculative_algorithm is None diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index e225727c9d3c..70f8530fc922 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -338,6 +338,7 @@ def init_token_map(self): def init_lm_head(self): import os as _os + if _os.environ.get("SGLANG_ALLOW_PP_SPEC") and self.server_args.pp_size > 1: # PP+spec: the target's embedding lives on the first PP stage # (PPMissingLayer here on the last stage), so the draft keeps its From 5718cc602ba19575a2667de7e6392fdc429a0fd7 Mon Sep 17 00:00:00 2001 From: alichen Date: Mon, 13 Jul 2026 03:22:36 -0700 Subject: [PATCH 04/28] Fix PP+spec draft embedding loading; accept_length recovers to parity NextN/MTP draft layers carry no embedding of their own in the checkpoint, and under PP the target's embed_tokens lives on the first stage (PPMissingLayer on the last stage where the draft runs), so the draft embedding stayed randomly initialized and accept_length collapsed to ~1 (correct output, zero speculative gain). Load the target's embed_tokens from the checkpoint safetensors directly for the draft, routed through VocabParallelEmbedding.weight_loader for TP sharding. Skipped under --load-format dummy. Also fix a porting slip in _pp_launch_batch: self.cur_batch -> the cur_batch local (the attribute does not exist on Scheduler in the PP event loop). Validated on 8x RTX 6000D, GLM-5.2-NVFP4, TP4xPP2 + EAGLE(2s/topk1/3d): accept_length 2.71-2.91 during GSM8K and serving bench (TP8+MTP baseline on the same host: 2.60-3.00; before the fix: ~1.07), GSM8K 20q 0.900, bs=1 median TPOT 13.5ms vs 25.5ms without MTP. Co-Authored-By: Claude Fable 5 --- .../sglang/srt/managers/scheduler_pp_mixin.py | 4 +- .../sglang/srt/speculative/eagle_worker_v2.py | 39 ++++++++++++++++++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 7dca497b924d..80377506c3c3 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -1490,8 +1490,8 @@ def _pp_launch_batch( mb_metadata[mb_id] = PPBatchMetadata( can_run_cuda_graph=result.can_run_cuda_graph, fwd_batch=( - self.cur_batch.copy() - if not self.cur_batch.spec_algorithm.is_none() + cur_batch.copy() + if not cur_batch.spec_algorithm.is_none() else None ), ) diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 70f8530fc922..eaaef1e49e9f 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -137,6 +137,32 @@ def _get_plan_stream( return None, contextlib.nullcontext() +def _load_checkpoint_tensor(model_path: str, tensor_name: str) -> torch.Tensor: + """Load one tensor from a local safetensors checkpoint (PP+spec path).""" + import glob + import json + import os + + from safetensors import safe_open + + index_path = os.path.join(model_path, "model.safetensors.index.json") + if os.path.exists(index_path): + with open(index_path) as f: + shard_files = [json.load(f)["weight_map"][tensor_name]] + else: + shard_files = sorted( + os.path.basename(p) + for p in glob.glob(os.path.join(model_path, "*.safetensors")) + ) + for shard_file in shard_files: + with safe_open( + os.path.join(model_path, shard_file), framework="pt", device="cpu" + ) as f: + if tensor_name in f.keys(): + return f.get_tensor(tensor_name) + raise ValueError(f"{tensor_name} not found in checkpoint at {model_path}") + + class EagleDraftWorker(EagleDraftWorkerBase): def __init__( self, @@ -341,9 +367,18 @@ def init_lm_head(self): if _os.environ.get("SGLANG_ALLOW_PP_SPEC") and self.server_args.pp_size > 1: # PP+spec: the target's embedding lives on the first PP stage - # (PPMissingLayer here on the last stage), so the draft keeps its - # own embedding weights and shares only the target's lm_head. + # (PPMissingLayer here on the last stage) and NextN/MTP layers + # carry no embedding of their own in the checkpoint, so the + # draft's embedding must be loaded from the checkpoint directly + # — otherwise it stays randomly initialized and accept_length + # collapses to ~1. embed = self.draft_runner.model.model.embed_tokens.weight + if self.server_args.load_format != "dummy": + loaded_embed = _load_checkpoint_tensor( + model_path=self.draft_runner.model_config.model_path, + tensor_name="model.embed_tokens.weight", + ) + embed.weight_loader(embed, loaded_embed) head = self.target_worker.model_runner.model.lm_head.weight self.draft_runner.model.set_embed_and_head(embed, head) return From 46fee6c421778826adda09759924b97c2edbae87 Mon Sep 17 00:00:00 2001 From: alichen Date: Mon, 13 Jul 2026 07:16:16 -0700 Subject: [PATCH 05/28] Generalize PP+spec beyond GLM: portable embed name and context-len bound Validated the PP+spec path on a second model family (DeepSeek-V4-Flash, DeepseekV4ForCausalLM) and fixed the two portability gaps it surfaced: - The draft embedding checkpoint tensor is looked up from a candidate list: DeepSeek-V4 checkpoints store it as embed.weight, not the GLM/DeepSeek-V3 style model.embed_tokens.weight. - The verify-input rebuild fallback bound now uses model_config.context_len instead of attn_backend.max_context_len, which only some backends expose (DeepseekV4AttnBackend does not; on FlashInfer backends the two values are identical). Validation on 4x RTX 6000D, DeepSeek-V4-Flash (FP8), TP2xPP2 + EAGLE(2s/topk1/3d): accept_length 2.44-2.72 sustained under GSM8K and serving load, GSM8K 20q 0.95-1.00, greedy output correct; bench at concurrency 1/4/8/16 completes end to end. Co-Authored-By: Claude Fable 5 --- .../sglang/srt/managers/scheduler_pp_mixin.py | 4 ++- .../sglang/srt/speculative/eagle_worker_v2.py | 33 +++++++++++-------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 80377506c3c3..8ebbba11a554 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -1319,7 +1319,9 @@ def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None seq_lens_sum = batch.seq_lens_sum if seq_lens_sum is None: if tree_mask_buf is None: - seq_lens_sum = bs * attn_backend.max_context_len + # Conservative upper bound; backend-agnostic (not every + # attention backend exposes max_context_len). + seq_lens_sum = bs * self.tp_worker.model_runner.model_config.context_len else: seq_lens_sum = 0 # preallocated buf -> kernel ignores it diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index eaaef1e49e9f..bc4b36c6a8f1 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -137,7 +137,12 @@ def _get_plan_stream( return None, contextlib.nullcontext() -def _load_checkpoint_tensor(model_path: str, tensor_name: str) -> torch.Tensor: +# Embedding weight names across checkpoint layouts (GLM/DeepSeek-V3 style +# vs DeepSeek-V4 style). +_EMBED_TENSOR_NAMES = ("model.embed_tokens.weight", "embed.weight") + + +def _load_checkpoint_tensor(model_path: str, tensor_names: tuple) -> torch.Tensor: """Load one tensor from a local safetensors checkpoint (PP+spec path).""" import glob import json @@ -148,19 +153,19 @@ def _load_checkpoint_tensor(model_path: str, tensor_name: str) -> torch.Tensor: index_path = os.path.join(model_path, "model.safetensors.index.json") if os.path.exists(index_path): with open(index_path) as f: - shard_files = [json.load(f)["weight_map"][tensor_name]] + weight_map = json.load(f)["weight_map"] + for tensor_name in tensor_names: + if tensor_name in weight_map: + shard_path = os.path.join(model_path, weight_map[tensor_name]) + with safe_open(shard_path, framework="pt", device="cpu") as f: + return f.get_tensor(tensor_name) else: - shard_files = sorted( - os.path.basename(p) - for p in glob.glob(os.path.join(model_path, "*.safetensors")) - ) - for shard_file in shard_files: - with safe_open( - os.path.join(model_path, shard_file), framework="pt", device="cpu" - ) as f: - if tensor_name in f.keys(): - return f.get_tensor(tensor_name) - raise ValueError(f"{tensor_name} not found in checkpoint at {model_path}") + for shard_path in sorted(glob.glob(os.path.join(model_path, "*.safetensors"))): + with safe_open(shard_path, framework="pt", device="cpu") as f: + for tensor_name in tensor_names: + if tensor_name in f.keys(): + return f.get_tensor(tensor_name) + raise ValueError(f"none of {tensor_names} found in checkpoint at {model_path}") class EagleDraftWorker(EagleDraftWorkerBase): @@ -376,7 +381,7 @@ def init_lm_head(self): if self.server_args.load_format != "dummy": loaded_embed = _load_checkpoint_tensor( model_path=self.draft_runner.model_config.model_path, - tensor_name="model.embed_tokens.weight", + tensor_names=_EMBED_TENSOR_NAMES, ) embed.weight_loader(embed, loaded_embed) head = self.target_worker.model_runner.model.lm_head.weight From 3686cd9040216eac726fdd307367f5568cf4b98f Mon Sep 17 00:00:00 2001 From: alichen Date: Mon, 13 Jul 2026 10:12:30 -0700 Subject: [PATCH 06/28] Address self-review: harden the PP+spec gate and fix ungated regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes from an adversarial review pass over the branch diff: Ungated default-path fixes (gate off must be behavior-identical): - Non-overlap spec path passed pp_proxy_tensors= to every spec worker; only the V2 EAGLE worker accepts it (NGRAM/DFLASH/etc. would crash). Pass it only when pp_size > 1, mirroring the non-spec branch. - on_verify_complete_cpu: replace the hasattr guard with a no-op hook on BaseTpWorker (repo convention: no defensive hasattr). Gate hardening: - Register SGLANG_ENABLE_PP_SPEC as a typed EnvBool in environ.py (renamed from SGLANG_ALLOW_PP_SPEC per naming conventions) and use envs.*.get() at all sites — raw string truthiness treated =0/=false as enabled. Removes the scattered function-local os imports. - Assert PP+spec is incompatible with --speculative-adaptive: the relay slices results with the configured num_draft_tokens, which adaptive spec changes at runtime. - init_lm_head PP branch fails fast on --speculative-token-map and EAGLE3 drafts (their head wiring is skipped by this branch). - _load_checkpoint_tensor asserts the model path is a local directory (HF hub ids resolve into the loader's cache, not model_path). PP+spec path fixes: - _pp_spec_chain_by_rid leaked one GPU chain row per completed request (stored before finish flags are set, and finished reqs never re-enter a batch). Initialize it in init_pp_loop_state and drop finished rids after process_batch_result. - Draft-worker seed sync: broadcast within the stage's TP group (exactly the ranks holding a draft worker) instead of adopting the node-local server_args seed, which diverges across nodes when --random-seed is unset. The caller's global rank comes from tp_group.ranks (the draft worker is constructed with pp_rank=0). - Deduplicate the draft-local memory-profiling predicate into ModelRunner.pp_spec_draft_local computed once in __init__. Validated on 8x RTX 6000D: GLM-5.2 TP4xPP2+MTP and DSv4-Flash TP2xPP2+MTP dummy smokes pass with the gate on; DSv4 TP4+MTP with --disable-overlap-schedule passes with the gate off (the ungated regression case); GLM-5.2 TP4xPP2+MTP real weights: GSM8K 20q 0.900, accept_length 2.67-2.80 (unchanged from before this commit). Co-Authored-By: Claude Fable 5 --- python/sglang/srt/environ.py | 4 ++++ python/sglang/srt/managers/scheduler.py | 14 +++++++---- .../batch_result_processor.py | 13 +++++------ .../sglang/srt/managers/scheduler_pp_mixin.py | 10 ++++++-- python/sglang/srt/managers/tp_worker.py | 23 +++++++++++++++---- .../sglang/srt/model_executor/model_runner.py | 23 ++++++++++--------- .../model_runner_kv_cache_mixin.py | 8 ++----- python/sglang/srt/server_args.py | 9 ++++++-- .../sglang/srt/speculative/eagle_worker_v2.py | 13 +++++++++-- 9 files changed, 79 insertions(+), 38 deletions(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 3a724e7d6766..68e4c72fd9f6 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -691,6 +691,10 @@ class Envs: SGLANG_ENABLE_OVERLAP_PLAN_STREAM = EnvBool(False) # Spec Config + # Experimental: allow pipeline parallelism x speculative decoding + # (EAGLE/MTP). Off by default; see the PP+spec RFC for constraints + # (topk=1 chains, non-overlap schedule). + SGLANG_ENABLE_PP_SPEC = EnvBool(False) SGLANG_SPEC_ENABLE_STRICT_FILTER_CHECK = EnvBool(True) # Skip draft_extend while adaptive spec is at steps=0 (drafting disabled). # Saves the per-step draft forward, but the draft KV goes stale: an upshift diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 9fe10f09c3a1..b229b44f2e78 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -771,10 +771,9 @@ def maybe_init_draft_worker(self): self.draft_worker = None self.external_corpus_manager = None return - import os as _os if ( - _os.environ.get("SGLANG_ALLOW_PP_SPEC") + envs.SGLANG_ENABLE_PP_SPEC.get() and self.server_args.pp_size > 1 and self.ps.pp_rank != self.server_args.pp_size - 1 ): @@ -3390,11 +3389,18 @@ def run_batch( batch.spec_info = None else: # Non-overlap: drive the V2 worker synchronously (no - # future_map relay / on_publish). + # future_map relay / on_publish). Only the PP+spec worker + # takes pp_proxy_tensors; other spec workers keep their + # signature (and pp_size == 1 has nothing to relay). + kwargs = ( + {"pp_proxy_tensors": pp_proxy_tensors} + if self.server_args.pp_size > 1 + else {} + ) resolve_forward_inputs(batch, self.future_map) with self._forward_isolation(batch, overlap=False): batch_result = self.model_worker.forward_batch_generation( - batch, pp_proxy_tensors=pp_proxy_tensors + batch, **kwargs ) # The isolation restore reverted the worker's in-forward SB edits; # re-apply what must carry to the next iter. diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index a8e608cb7696..525b52f19b6c 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -544,13 +544,12 @@ def _resolve_spec_v2_tokens( # Feed the adaptive controller now that accept_lens is on CPU, # instead of doing a synchronous GPU→CPU copy in the worker hot path. - # BaseSpecWorker provides a no-op default for non-adaptive workers. - # PP+spec non-last stages process relayed results through the plain - # TpModelWorker, which has no adaptive controller. - if hasattr(self.model_worker, "on_verify_complete_cpu"): - self.model_worker.on_verify_complete_cpu( - result.num_correct_drafts_per_req_cpu, batch_size=len(batch.reqs) - ) + # BaseSpecWorker and BaseTpWorker both provide a no-op default + # (PP+spec non-last stages process relayed results through the + # plain worker, which has no adaptive controller). + self.model_worker.on_verify_complete_cpu( + result.num_correct_drafts_per_req_cpu, batch_size=len(batch.reqs) + ) predict_tokens = [] # In adaptive spec-v2, the worker state may already have switched when this diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 8ebbba11a554..4e6abdd2079b 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -581,6 +581,8 @@ def init_pp_loop_state(self: Scheduler): self.mb_metadata: List[Optional[PPBatchMetadata]] = [None] * self.pp_loop_size self.pp_outputs: Optional[PPProxyTensors] = None self.last_rank_comm_queue: deque[Tuple[torch.Event, PPProxyTensors]] = deque() + # PP+spec: per-rid chain rows seeding the next verify round. + self._pp_spec_chain_by_rid: Dict[str, torch.Tensor] = {} self.send_req_work = [] self.send_proxy_work = [] @@ -1234,6 +1236,12 @@ def _pp_process_batch_result( self: Scheduler, batch: ScheduleBatch, output_result: GenerationBatchResult ): self.process_batch_result(batch, output_result) + # Chains are stored before finish flags are set (at result prep), so + # finished requests must be dropped here or the dict grows per request. + if self._pp_spec_chain_by_rid: + for req in batch.reqs: + if req.finished(): + self._pp_spec_chain_by_rid.pop(req.rid, None) def _pp_spec_store_bonus( self: Scheduler, @@ -1249,8 +1257,6 @@ def _pp_spec_store_bonus( — the zero drafts just get rejected, costing acceptance not correctness. Keyed by rid (not batch position): the microbatch composition can change between rounds (finish / retract / merge).""" - if not hasattr(self, "_pp_spec_chain_by_rid"): - self._pp_spec_chain_by_rid = {} num_draft_tokens = self.server_args.speculative_num_draft_tokens if chain_tokens is not None: rows = chain_tokens.to(torch.int64).reshape( diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index b4dd8fb26bcf..6df060e0eeea 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -22,6 +22,7 @@ import torch from sglang.srt.distributed import get_pp_group, get_world_group +from sglang.srt.environ import envs from sglang.srt.managers.io_struct import ( DestroyWeightsUpdateGroupReqInput, GetWeightsByNameReqInput, @@ -70,6 +71,12 @@ def forward_batch_generation(self, forward_batch: ForwardBatch): def model_runner(self) -> ModelRunner: pass + def on_verify_complete_cpu( + self, num_correct_drafts_per_req: list[int], batch_size: int = 0 + ) -> None: + """No-op mirror of BaseSpecWorker's hook: PP+spec non-last stages + process relayed spec results through a plain worker.""" + @property def war_fastpath_runner(self): # The runner that runs the step's LAST shared-buffer-reading phase -- @@ -307,17 +314,25 @@ def __init__( self.world_group = get_world_group() # Sync random seed across TP workers - import os as _os if ( - _os.environ.get("SGLANG_ALLOW_PP_SPEC") + envs.SGLANG_ENABLE_PP_SPEC.get() and is_draft_worker and server_args.pp_size > 1 ): # PP+spec: the draft worker exists only on the last PP stage, so a # world-group broadcast here would deadlock (first-stage ranks never - # join). The seed was already synced during target worker init. - self.random_seed = server_args.random_seed + # join). Sync within the stage's TP group instead — that is exactly + # the set of ranks holding a draft worker. The draft worker is + # constructed with pp_rank=0, so derive the caller's global rank + # from the TP group rather than tp_size * pp_rank + tp_rank. + tp_group = self.model_runner.tp_group + self.random_seed = broadcast_pyobj( + [server_args.random_seed], + tp_group.ranks[tp_rank], + tp_group.cpu_group, + src=tp_group.ranks[0], + )[0] else: self.random_seed = broadcast_pyobj( [server_args.random_seed], diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index ddca90eaa113..d0c657ba2d4d 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -380,6 +380,14 @@ def __init__( self.dist_port = nccl_port self.server_args = server_args self.is_draft_worker = is_draft_worker + # PP+spec: the draft runner exists only on the last PP stage, so its + # memory profiling must stay rank-local (world-group reduces would + # deadlock against stages without a draft worker). + self.pp_spec_draft_local = ( + envs.SGLANG_ENABLE_PP_SPEC.get() + and is_draft_worker + and server_args.pp_size > 1 + ) self.is_generation = model_config.is_generation self.device_timer = None self.is_multimodal = model_config.is_multimodal @@ -603,9 +611,8 @@ def __init__( ) if self.pp_size > 1: - import os as _os - if not (_os.environ.get("SGLANG_ALLOW_PP_SPEC") and self.is_draft_worker): + if not (envs.SGLANG_ENABLE_PP_SPEC.get() and self.is_draft_worker): assert ( self.support_pp ), "Pipeline Parallel is not compatible with this model." @@ -770,9 +777,7 @@ def initialize(self): if loop_num > 1: self.num_effective_layers = self.num_effective_layers * loop_num - import os as _os - - if not _os.environ.get("SGLANG_ALLOW_PP_SPEC"): + if not envs.SGLANG_ENABLE_PP_SPEC.get(): assert ( (not model_has_mtp_layers) or (self.spec_algorithm.is_none()) @@ -1288,15 +1293,11 @@ def init_torch_distributed(self): f"(tp_size={self.tp_size}, pp_size={self.pp_size}, ep_size={self.moe_ep_size})" ) - _pp_spec_draft_local = ( - __import__("os").environ.get("SGLANG_ALLOW_PP_SPEC") - and getattr(self, "is_draft_worker", False) - and self.pp_size > 1 - ) pre_model_load_memory = get_available_gpu_memory( self.device, self.gpu_id, - distributed=(get_world_group().world_size > 1) and not _pp_spec_draft_local, + distributed=(get_world_group().world_size > 1) + and not self.pp_spec_draft_local, cpu_group=get_world_group().cpu_group, ) self.tp_group = get_tp_group() diff --git a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py index 63ee84432bfc..be7db7a7a618 100644 --- a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py +++ b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py @@ -105,15 +105,11 @@ def _profile_available_bytes(self: ModelRunner, pre_model_load_memory: int) -> i # KV pool budget = currently-free GPU memory minus the non-static runtime # slack (pre_model_load_memory * (1 - mem_fraction_static)). Whatever is # already resident (model weights, etc.) is thus charged against it. - _pp_spec_draft_local = ( - __import__("os").environ.get("SGLANG_ALLOW_PP_SPEC") - and getattr(self, "is_draft_worker", False) - and self.pp_size > 1 - ) available_gpu_memory = get_available_gpu_memory( self.device, self.gpu_id, - distributed=(get_world_group().world_size > 1) and not _pp_spec_draft_local, + distributed=(get_world_group().world_size > 1) + and not self.pp_spec_draft_local, cpu_group=get_world_group().cpu_group, ) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 74623c334032..eca1ba461dbe 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -6921,14 +6921,19 @@ def check_server_args(self): ) if self.pp_size > 1: - import os as _os - if not _os.environ.get("SGLANG_ALLOW_PP_SPEC"): + if not envs.SGLANG_ENABLE_PP_SPEC.get(): assert ( self.disable_overlap_schedule and self.speculative_algorithm is None ), "Pipeline parallelism is not compatible with overlap schedule, speculative decoding" else: assert self.disable_overlap_schedule + # The PP relay slices spec results with the configured + # num_draft_tokens; adaptive spec changes it at runtime. + assert not self.speculative_adaptive, ( + "SGLANG_ENABLE_PP_SPEC is not compatible with " + "--speculative-adaptive" + ) assert not ( self.dp_size > 1 and self.nnodes != 1 and not self.enable_dp_attention diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index bc4b36c6a8f1..e8e6b6f8bf05 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -150,6 +150,12 @@ def _load_checkpoint_tensor(model_path: str, tensor_names: tuple) -> torch.Tenso from safetensors import safe_open + # The standard loader may have resolved a HF hub id into its own cache; + # this simplified reader only handles local checkpoint directories. + assert os.path.isdir(model_path), ( + "PP+spec draft embedding loading requires --model-path to be a local " + f"checkpoint directory, got {model_path!r}" + ) index_path = os.path.join(model_path, "model.safetensors.index.json") if os.path.exists(index_path): with open(index_path) as f: @@ -368,9 +374,12 @@ def init_token_map(self): self.hot_token_id = None def init_lm_head(self): - import os as _os - if _os.environ.get("SGLANG_ALLOW_PP_SPEC") and self.server_args.pp_size > 1: + if envs.SGLANG_ENABLE_PP_SPEC.get() and self.server_args.pp_size > 1: + # This branch skips the hot-token-map / EAGLE3 head wiring below. + assert self.hot_token_id is None and not ( + self.speculative_algorithm.is_eagle3() + ), "PP+spec does not support --speculative-token-map or EAGLE3 drafts yet" # PP+spec: the target's embedding lives on the first PP stage # (PPMissingLayer here on the last stage) and NextN/MTP layers # carry no embedding of their own in the checkpoint, so the From 32e3751ef9efd220e3eb90ef13074ea62b6128f4 Mon Sep 17 00:00:00 2001 From: alichen Date: Mon, 13 Jul 2026 19:10:05 -0700 Subject: [PATCH 07/28] Revert batch_result_processor.py to upstream (comment-only net diff) Co-Authored-By: Claude Fable 5 --- .../managers/scheduler_components/batch_result_processor.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index 525b52f19b6c..8d0888064f16 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -544,9 +544,7 @@ def _resolve_spec_v2_tokens( # Feed the adaptive controller now that accept_lens is on CPU, # instead of doing a synchronous GPU→CPU copy in the worker hot path. - # BaseSpecWorker and BaseTpWorker both provide a no-op default - # (PP+spec non-last stages process relayed results through the - # plain worker, which has no adaptive controller). + # BaseSpecWorker provides a no-op default for non-adaptive workers. self.model_worker.on_verify_complete_cpu( result.num_correct_drafts_per_req_cpu, batch_size=len(batch.reqs) ) From 5b51ecb469281a2bc5415149d4c37f314f6cf1ee Mon Sep 17 00:00:00 2001 From: alichen Date: Mon, 13 Jul 2026 22:58:28 -0700 Subject: [PATCH 08/28] Review polish: assert message, explicit None checks, stray blank lines Zero-semantic cleanups from the file-by-file pass: give the gated non-overlap assert a message, make the fwd_batch fallback an explicit None check, and drop blank lines left behind by the os-import removal. Co-Authored-By: Claude Fable 5 --- python/sglang/srt/managers/scheduler_pp_mixin.py | 8 ++++++-- python/sglang/srt/managers/tp_worker.py | 1 - python/sglang/srt/model_executor/model_runner.py | 1 - python/sglang/srt/server_args.py | 5 +++-- python/sglang/srt/speculative/eagle_worker_v2.py | 1 - 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 4e6abdd2079b..4c224607aaf1 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -1161,7 +1161,9 @@ def _pp_prep_batch_result( # the seq state per-rid onto the live batch, stash the bonus # tokens that root the next verify chain, and rebuild a result the # spec output processor can consume (CPU tensors). - fwd_batch = mb_metadata.fwd_batch if mb_metadata.fwd_batch else batch + fwd_batch = ( + mb_metadata.fwd_batch if mb_metadata.fwd_batch is not None else batch + ) new_seq_lens = pp_outputs["spec_new_seq_lens"] fwd_rids = [req.rid for req in fwd_batch.reqs] live_rids = [req.rid for req in batch.reqs] @@ -1211,7 +1213,9 @@ def _pp_prep_batch_result( batch.input_ids = pp_outputs["next_token_ids"].to(torch.int64) if is_spec: # Spec prefill round: the sampled first token roots round 1's chain. - fwd_batch = mb_metadata.fwd_batch if mb_metadata.fwd_batch else batch + fwd_batch = ( + mb_metadata.fwd_batch if mb_metadata.fwd_batch is not None else batch + ) self._pp_spec_store_bonus(fwd_batch, batch.input_ids) else: # PP rank 0 also relays into output_tokens_buf so the next iter's diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index cc4d66a0a05c..ec6847385446 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -314,7 +314,6 @@ def __init__( self.world_group = get_world_group() # Sync random seed across TP workers - if ( envs.SGLANG_ENABLE_PP_SPEC.get() and is_draft_worker diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 2d50125fc23c..392253c3adc0 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -631,7 +631,6 @@ def __init__( ) if self.pp_size > 1: - if not (envs.SGLANG_ENABLE_PP_SPEC.get() and self.is_draft_worker): assert ( self.support_pp diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index c4f2393df648..93e333e06372 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -7046,13 +7046,14 @@ def check_server_args(self): ) if self.pp_size > 1: - if not envs.SGLANG_ENABLE_PP_SPEC.get(): assert ( self.disable_overlap_schedule and self.speculative_algorithm is None ), "Pipeline parallelism is not compatible with overlap schedule, speculative decoding" else: - assert self.disable_overlap_schedule + assert ( + self.disable_overlap_schedule + ), "SGLANG_ENABLE_PP_SPEC requires --disable-overlap-schedule" # The PP relay slices spec results with the configured # num_draft_tokens; adaptive spec changes it at runtime. assert not self.speculative_adaptive, ( diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 18cf6140f093..3aad8b65ed9a 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -364,7 +364,6 @@ def init_token_map(self): self.hot_token_id = None def init_lm_head(self): - if envs.SGLANG_ENABLE_PP_SPEC.get() and self.server_args.pp_size > 1: # This branch skips the hot-token-map / EAGLE3 head wiring below. assert self.hot_token_id is None and not ( From 635cf591bd18699eb0ea5729ff7984c4a58180ac Mon Sep 17 00:00:00 2001 From: alichen Date: Thu, 16 Jul 2026 10:21:45 -0700 Subject: [PATCH 09/28] Own the stored chain row instead of a slice view into the relay tensor rows[i] kept the whole rows storage alive per entry, and when chain_tokens is already int64 the .to() is a no-op view of the relay buffer -- a later relay reusing that buffer would corrupt stored chains (the row root is force-accepted by verify). Addresses the gemini-code-assist finding on _pp_spec_store_bonus. Co-Authored-By: Claude Fable 5 --- python/sglang/srt/managers/scheduler_pp_mixin.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 10d1dd6850e1..5828ef34c473 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -1284,7 +1284,12 @@ def _pp_spec_store_bonus( if req.finished(): self._pp_spec_chain_by_rid.pop(req.rid, None) else: - self._pp_spec_chain_by_rid[req.rid] = rows[i] + # clone(): a slice view would pin the whole rows storage for + # the entry's lifetime, and when chain_tokens is already + # int64 the .to() above is a no-op view of the relay buffer + # — a later relay reusing that buffer would corrupt stored + # chains (the row root is force-accepted by verify). + self._pp_spec_chain_by_rid[req.rid] = rows[i].clone() def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None: """Rebuild batch.spec_info (EagleVerifyInput) from relayed per-request From aa737e62beb87b278e97d85f21b07e0c3efa974b Mon Sep 17 00:00:00 2001 From: alichen Date: Thu, 16 Jul 2026 20:34:31 -0700 Subject: [PATCH 10/28] Lint: black formatting for bootstrap.py Co-Authored-By: Claude Fable 5 --- python/sglang/srt/distributed/bootstrap.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/sglang/srt/distributed/bootstrap.py b/python/sglang/srt/distributed/bootstrap.py index 3e6c8379da4c..1b6a61da1b8d 100644 --- a/python/sglang/srt/distributed/bootstrap.py +++ b/python/sglang/srt/distributed/bootstrap.py @@ -117,9 +117,7 @@ def init_torch_distributed( # profiling must stay rank-local (a world-group reduce would deadlock # against stages without a draft worker). pp_spec_draft_local = ( - envs.SGLANG_ENABLE_PP_SPEC.get() - and is_draft_worker - and server_args.pp_size > 1 + envs.SGLANG_ENABLE_PP_SPEC.get() and is_draft_worker and server_args.pp_size > 1 ) pre_model_load_memory = get_available_gpu_memory( device, From 598fa411c609c9bf0f6fa810b7604ec232e89ea5 Mon Sep 17 00:00:00 2001 From: alichen Date: Sun, 26 Jul 2026 23:05:07 -0700 Subject: [PATCH 11/28] Fix stale attribute after merge: num_tokens_per_req -> captured_req_width The PP proxy output slicing in DecodeCudaGraphRunner.execute still read `self.num_tokens_per_req`, which upstream renamed to `self.captured_req_width`. Auto-merge kept the old name, so any PP + speculative decode step raised AttributeError at runtime (caught on 8x RTX 6000D with TP4xPP2 + MTP). Co-Authored-By: Claude Opus 5 (1M context) --- .../srt/model_executor/runner/decode_cuda_graph_runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index 5940f3302a10..8d705009dd03 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -1302,11 +1302,11 @@ def execute( else: assert isinstance(output, PPProxyTensors) # Slice in token rows, not request rows: under speculative verify - # each request carries num_tokens_per_req tokens (identical for - # plain decode, where num_tokens_per_req == 1). + # each request carries captured_req_width tokens (identical for + # plain decode, where captured_req_width == 1). return PPProxyTensors( { - k: v[: self.bs * self.num_tokens_per_req] + k: v[: self.bs * self.captured_req_width] for k, v in output.tensors.items() } ) From 06285ecd65c37663bb2ced1070eb3f7a02513510 Mon Sep 17 00:00:00 2001 From: alichen Date: Sun, 16 Aug 2026 22:42:07 -0700 Subject: [PATCH 12/28] Adapt the PP+spec verify rebuild to upstream's verify-mask API Two upstream API changes silently auto-merged into calls that would only fail at runtime, on the PP+spec path: - build_tree_kernel_efficient dropped its position-buffer parameter (it always allocates `positions` itself) and added `fill_prefix_mask` in that slot, so the rebuild passed a tensor where a bool is expected. - The buffer/mode selection moved from attn_backend.get_verify_buffers_to_fill_after_draft() to the attn_backend.verify_mask struct (mode / is_read / buffer / fits). Mirror build_eagle_verify_input's selection so every PP stage builds the same mask as the last stage's draft tail, and keep the QLEN_ONLY case out of the seq_lens_sum upper bound. - EagleVerifyInput.create_idle_input now requires `device`. Also black-format the widened forward_batch_generation signature. Co-Authored-By: Claude Opus 5 (1M context) --- .../sglang/srt/managers/scheduler_pp_mixin.py | 25 ++++++++++++------- .../sglang/srt/speculative/eagle_worker_v2.py | 6 ++++- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 928f954e9eb3..f2017d6f1116 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -1319,6 +1319,7 @@ def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None from sglang.srt.speculative.eagle_utils import ( TreeMaskMode, build_tree_kernel_efficient, + default_tree_mask_mode, ) sa = self.server_args @@ -1332,6 +1333,7 @@ def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None topk=sa.speculative_eagle_topk, spec_steps=steps, num_verify_tokens=num_draft_tokens, + device=device, ) return @@ -1349,18 +1351,23 @@ def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None bs, 1 ) - attn_backend = self.tp_worker.model_runner.attn_backend - tree_mask_buf, position_buf = ( - attn_backend.get_verify_buffers_to_fill_after_draft() - ) + # Mask selection mirrors the last stage's draft() tail + # (build_eagle_verify_input) so every stage builds the same mask. + verify_mask = self.tp_worker.model_runner.attn_backend.verify_mask + if verify_mask is None: + tree_mask_buf, mask_mode, fill_mask = None, default_tree_mask_mode(), True + else: + mask_mode, fill_mask = verify_mask.mode, verify_mask.is_read + tree_mask_buf = verify_mask.buffer if verify_mask.fits(bs) else None + seq_lens_sum = batch.seq_lens_sum if seq_lens_sum is None: - if tree_mask_buf is None: + if tree_mask_buf is not None or mask_mode == TreeMaskMode.QLEN_ONLY: + seq_lens_sum = 0 # preallocated / bs-sized -> kernel ignores it + else: # Conservative upper bound; backend-agnostic (not every # attention backend exposes max_context_len). seq_lens_sum = bs * self.tp_worker.model_runner.model_config.context_len - else: - seq_lens_sum = 0 # preallocated buf -> kernel ignores it ( tree_mask, @@ -1379,9 +1386,9 @@ def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None sa.speculative_eagle_topk, steps, num_draft_tokens, - TreeMaskMode.FULL_MASK, + mask_mode, tree_mask_buf, - position_buf, + fill_prefix_mask=fill_mask, ) batch.spec_info = EagleVerifyInput( draft_token=flat_draft_tokens, diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index b45ad0b42695..d07b3d853b91 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -1165,7 +1165,11 @@ def init_cuda_graphs(self): ) def forward_batch_generation( - self, batch: ScheduleBatch, on_publish=None, pp_proxy_tensors=None, grammar_barrier=None + self, + batch: ScheduleBatch, + on_publish=None, + pp_proxy_tensors=None, + grammar_barrier=None, ): if batch.forward_mode.is_extend() or batch.is_extend_in_batch: # Target prefill From ffd716a79eec68032b004e2cf56e86c85f088418 Mon Sep 17 00:00:00 2001 From: alichen Date: Mon, 17 Aug 2026 00:05:39 -0700 Subject: [PATCH 13/28] Harden the PP+spec relay: ring ordering, chunked prefill, batch invariants Three gaps found while comparing this branch against the other in-flight PP x spec efforts (#31139, #33863); all are confined to the gated path. - PP ring send/recv ordering. The relay carries several extra GPU tensors under spec (accept_lens, new_seq_lens, bonus tokens, next chain). Device P2P work stays ordered on the stream, so enqueueing that many sends before any recv on every rank can form a ring wait even on CUDA, where the existing comment assumes send-first is always safe. Pair adjacent stages by pp_rank parity when the spec relay is active, which makes rank 1 post its recv first and breaks the cycle for any pp_size > 1. Non-spec PP keeps the existing send-first behavior. Credit to #31139 for finding this. - Chunked prefill middle chunks. Only the chunk that finishes the prompt samples a real token; a middle chunk's relayed next_token_ids is a placeholder that no consumer reads. Storing it seeded the next verify round with a token the model never emitted. - Batch composition invariants. Every stage rebuilds the same verify input from relayed per-request state, so all stages must see the same batch. Assert against --enable-mixed-chunk (splices decode requests into an extend batch) and --enable-dp-attention (partitions the batch per DP rank) rather than silently mis-rebuilding. Co-Authored-By: Claude Opus 5 (1M context) --- .../sglang/srt/managers/scheduler_pp_mixin.py | 29 +++++++++++++++---- python/sglang/srt/server_args.py | 12 ++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index f2017d6f1116..5e2916e4bee0 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -586,6 +586,11 @@ def init_pp_loop_state(self: Scheduler): self.last_rank_comm_queue: deque[Tuple[torch.Event, PPProxyTensors]] = deque() # PP+spec: per-rid chain rows seeding the next verify round. self._pp_spec_chain_by_rid: Dict[str, torch.Tensor] = {} + self._pp_spec_relay = ( + envs.SGLANG_ENABLE_PP_SPEC.get() + and self.ps.pp_size > 1 + and not self.spec_algorithm.is_none() + ) self.send_req_work = [] self.send_proxy_work = [] @@ -1234,10 +1239,17 @@ def _pp_prep_batch_result( next_token_ids = pp_outputs["next_token_ids"].to(torch.int64) if is_spec: # Spec prefill round: the sampled first token roots round 1's chain. - fwd_batch = ( - mb_metadata.fwd_batch if mb_metadata.fwd_batch is not None else batch - ) - self._pp_spec_store_bonus(fwd_batch, next_token_ids) + # Only the chunk that finishes the prompt samples a real token; a + # middle chunk's next_token_ids is a placeholder no consumer reads, + # and storing it would seed the next verify round with a token the + # model never emitted. + if batch.contains_last_prefill_chunk: + fwd_batch = ( + mb_metadata.fwd_batch + if mb_metadata.fwd_batch is not None + else batch + ) + self._pp_spec_store_bonus(fwd_batch, next_token_ids) else: # PP rank 0 also relays into output_tokens_buf so the next iter's # resolve_forward_inputs finds these tokens for the decode portion @@ -1471,7 +1483,14 @@ def _pp_send_recv_and_preprocess_output_tensors( # CUDA: send first # XPU: even ranks send first, odd ranks recv first. - send_first = (not is_xpu()) or ((self.ps.pp_rank % 2) == 0) + # PP+spec also pairs by parity: its relay carries several extra GPU + # tensors (accept_lens, new_seq_lens, bonus tokens, next chain), and + # device-side P2P stays ordered on the stream, so enqueueing that many + # sends before any recv can form the same ring wait on CUDA. Parity + # makes rank 1 post its recv first, which breaks the cycle for any + # pp_size > 1. + needs_pairing = is_xpu() or self._pp_spec_relay + send_first = (not needs_pairing) or ((self.ps.pp_rank % 2) == 0) def _do_send(): return self._pp_send_output_to_next_stage( diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index ab5ac2d21e0a..fe6655948bcb 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -9097,6 +9097,18 @@ def check_server_args(self): "SGLANG_ENABLE_PP_SPEC is not compatible with " "--speculative-adaptive" ) + # Every stage rebuilds the same verify input from the relayed + # per-request state, so all stages must see the same batch + # composition. Mixed-chunk splices decode requests into an + # extend batch (whose relayed next_token_ids are prefill + # placeholders), and DP attention partitions the batch per DP + # rank; both break that invariant. + assert ( + not self.enable_mixed_chunk + ), "SGLANG_ENABLE_PP_SPEC is not compatible with --enable-mixed-chunk" + assert ( + not self.enable_dp_attention + ), "SGLANG_ENABLE_PP_SPEC is not compatible with --enable-dp-attention" assert self.min_free_slots_delay is None, ( "--min-free-slots-delay is not supported with pipeline " "parallelism: allocatable slots per microbatch are bounded by " From 090c391920b3a826b737bd2a679649eab83804b6 Mon Sep 17 00:00:00 2001 From: alichen Date: Mon, 17 Aug 2026 00:12:25 -0700 Subject: [PATCH 14/28] Relay the drafted tree topology so PP+spec is not limited to topk=1 chains The rebuild on each stage synthesized parent_list / top_scores_index as arange constants, which is only the tree the topk=1 chain fast path produces. Once topk > 1 the shape is data-dependent, so the other stages cannot re-derive what the last stage drafted. Carry it instead: draft() optionally returns the topology it built the verify input from, the tail draft clones it out of the cuda-graph-owned buffers, and it rides the relay next to the tokens (parent_list and top_scores_index have different widths, so they stay separate tensors). A request that has not been drafted for yet -- its first decode after prefill -- stores no topology; the rebuild substitutes chain constants at the width the drafted rows use, keeping the stack rectangular. Those rows carry zero drafts, which the verify kernel rejects whatever tree shape they are hung on, so only their acceptance rate is affected. Co-Authored-By: Claude Opus 5 (1M context) --- .../sglang/srt/managers/scheduler_pp_mixin.py | 134 +++++++++++++----- python/sglang/srt/managers/utils.py | 6 +- .../sglang/srt/speculative/eagle_worker_v2.py | 20 ++- 3 files changed, 124 insertions(+), 36 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 5e2916e4bee0..ae8f6be57ea1 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -8,6 +8,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +import msgspec import numpy as np import torch import torch.distributed @@ -59,6 +60,19 @@ def _pp_can_skip_output_comm(batch: ScheduleBatch) -> bool: ) +class _PPSpecTreeRow(msgspec.Struct): + """One request's relayed draft tree: the tokens plus the topology they + were arranged by. Held per rid between PP+spec rounds.""" + + tokens: torch.Tensor # [num_draft_tokens], index 0 is the bonus token + # parent_list / top_scores_index rows. None until the request has been + # drafted for (its first decode after prefill carries zero drafts, which + # are rejected whatever tree shape they are hung on); the rebuild then + # substitutes chain constants at the width the drafted rows use. + parents: Optional[torch.Tensor] + top_scores: Optional[torch.Tensor] + + @dataclass class PPBatchMetadata: can_run_cuda_graph: bool @@ -1046,8 +1060,13 @@ def _pp_prepare_tensor_dict( tensor_dict["spec_new_seq_lens"] = result.new_seq_lens tensor_dict["spec_bonus_tokens"] = result.next_draft_input.bonus_tokens if result.next_verify_chain is not None: - # Tail-drafted chain for the next verify round (root = bonus). + # Tail-drafted tree for the next verify round (root = bonus), + # with the topology its tokens were arranged by. tensor_dict["spec_next_chain"] = result.next_verify_chain + tensor_dict["spec_next_parents"] = result.next_verify_parent_list + tensor_dict["spec_next_top_scores"] = ( + result.next_verify_top_scores_index + ) if batch.return_logprob: logprob_dict = get_logprob_dict_from_result(result) @@ -1222,6 +1241,16 @@ def _pp_prep_batch_result( if "spec_next_chain" in pp_outputs.tensors else None ), + parent_list=( + pp_outputs["spec_next_parents"] + if "spec_next_parents" in pp_outputs.tensors + else None + ), + top_scores_index=( + pp_outputs["spec_next_top_scores"] + if "spec_next_top_scores" in pp_outputs.tensors + else None + ), ) output_result = GenerationBatchResult( logits_output=logits_output, @@ -1286,23 +1315,25 @@ def _pp_spec_store_bonus( batch: ScheduleBatch, bonus_tokens: torch.Tensor, chain_tokens: Optional[torch.Tensor] = None, + parent_list: Optional[torch.Tensor] = None, + top_scores_index: Optional[torch.Tensor] = None, ) -> None: - """Stash the per-request chain row (root = bonus token) that seeds the - next verify round. - - chain_tokens is the last stage's tail-drafted chain (flat bs*dtn); - without it (prefill rounds) a degenerate row [bonus, 0, ...] is stored - — the zero drafts just get rejected, costing acceptance not - correctness. Keyed by rid (not batch position): the microbatch - composition can change between rounds (finish / retract / merge).""" + """Stash the per-request tree row (root = bonus token) that seeds the + next verify round, together with the topology it was arranged by. + + chain_tokens is the last stage's tail-drafted tree (flat bs*dtn) and + parent_list / top_scores_index its shape; without them (prefill rounds) + a degenerate row [bonus, 0, ...] over a chain topology is stored — the + zero drafts just get rejected, costing acceptance not correctness. + Keyed by rid (not batch position): the microbatch composition can + change between rounds (finish / retract / merge).""" num_draft_tokens = self.server_args.speculative_num_draft_tokens + bs = len(batch.reqs) if chain_tokens is not None: - rows = chain_tokens.to(torch.int64).reshape( - len(batch.reqs), num_draft_tokens - ) + rows = chain_tokens.to(torch.int64).reshape(bs, num_draft_tokens) else: rows = torch.zeros( - (len(batch.reqs), num_draft_tokens), + (bs, num_draft_tokens), dtype=torch.int64, device=bonus_tokens.device, ) @@ -1315,18 +1346,43 @@ def _pp_spec_store_bonus( # the entry's lifetime, and when chain_tokens is already # int64 the .to() above is a no-op view of the relay buffer # — a later relay reusing that buffer would corrupt stored - # chains (the row root is force-accepted by verify). - self._pp_spec_chain_by_rid[req.rid] = rows[i].clone() + # rows (the row root is force-accepted by verify). + self._pp_spec_chain_by_rid[req.rid] = _PPSpecTreeRow( + tokens=rows[i].clone(), + parents=None if parent_list is None else parent_list[i].clone(), + top_scores=( + None + if top_scores_index is None + else top_scores_index[i].clone() + ), + ) + + def _pp_spec_chain_topology( + self: Scheduler, bs: int, device: str + ) -> Tuple[torch.Tensor, torch.Tensor]: + """topk=1 chain constants, the shape a prefill round's degenerate row + implies. Mirrors _rebuild_topk1_chain_buffers: a single-step chain has + no parent entries, and both widths key off num_steps.""" + num_steps = self.server_args.speculative_num_steps + parent_width = num_steps if num_steps > 1 else 0 + parent_list = torch.arange( + -1, parent_width - 1, dtype=torch.long, device=device + ).repeat(bs, 1) + top_scores_index = torch.arange( + num_steps, dtype=torch.long, device=device + ).repeat(bs, 1) + return parent_list, top_scores_index def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None: """Rebuild batch.spec_info (EagleVerifyInput) from relayed per-request state, without a draft model. - Phase 1 (correctness bring-up): degenerate topk=1 chains — the root is - the relayed bonus token (always accepted by the verify kernel), padded - with zero draft tokens that simply get rejected. Verify is sound for - arbitrary proposals, so this costs acceptance rate, not correctness. - Phase 2 relays the last stage's tail-drafted real chains instead.""" + Runs on every stage, including the last one, so all stages verify the + exact same tree. The tokens and the topology both come from the last + stage's tail draft; a request that has not been drafted for yet (its + first decode after prefill) carries a degenerate row -- bonus token + plus zero drafts over a chain topology -- whose drafts simply get + rejected, costing acceptance rate, not correctness.""" from sglang.srt.speculative.eagle_info import EagleVerifyInput from sglang.srt.speculative.eagle_utils import ( TreeMaskMode, @@ -1349,19 +1405,33 @@ def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None ) return - chain_rows = torch.stack( - [self._pp_spec_chain_by_rid[req.rid] for req in batch.reqs] - ).to(device=device, dtype=torch.int64) - bonus_tokens = chain_rows[:, 0].contiguous() - draft_tokens = chain_rows[:, 1:].contiguous() - # topk=1 chain constants (mirrors EagleDraftWorker._rebuild_topk1_chain_buffers) - parent_width = steps if steps > 1 else 0 - parent_list = torch.arange( - -1, parent_width - 1, dtype=torch.long, device=device - ).repeat(bs, 1) - top_scores_index = torch.arange(steps, dtype=torch.long, device=device).repeat( - bs, 1 + rows = [self._pp_spec_chain_by_rid[req.rid] for req in batch.reqs] + tree_rows = torch.stack([r.tokens for r in rows]).to( + device=device, dtype=torch.int64 ) + bonus_tokens = tree_rows[:, 0].contiguous() + draft_tokens = tree_rows[:, 1:].contiguous() + # Topology as drafted on the last stage. It is data-dependent once + # topk > 1, so it rides the relay rather than being re-derived here. + # Not-yet-drafted rows borrow the drafted rows' widths so the stack is + # rectangular; their drafts are zeros and get rejected regardless of + # the shape they hang on. + drafted = next((r for r in rows if r.parents is not None), None) + if drafted is None: + parent_list, top_scores_index = self._pp_spec_chain_topology(bs, device) + else: + chain_parents = torch.arange( + -1, drafted.parents.shape[0] - 1, dtype=torch.long, device=device + ) + chain_scores = torch.arange( + drafted.top_scores.shape[0], dtype=torch.long, device=device + ) + parent_list = torch.stack( + [chain_parents if r.parents is None else r.parents for r in rows] + ).to(device=device, dtype=torch.long) + top_scores_index = torch.stack( + [chain_scores if r.top_scores is None else r.top_scores for r in rows] + ).to(device=device, dtype=torch.long) # Mask selection mirrors the last stage's draft() tail # (build_eagle_verify_input) so every stage builds the same mask. diff --git a/python/sglang/srt/managers/utils.py b/python/sglang/srt/managers/utils.py index 80d5a0ae6926..b69c47b03484 100644 --- a/python/sglang/srt/managers/utils.py +++ b/python/sglang/srt/managers/utils.py @@ -93,8 +93,12 @@ class GenerationBatchResult: next_draft_input: Optional[EagleDraftInput] = None # PP+spec: tail-drafted chain tokens (flat bs*num_draft_tokens, root = - # bonus) for the NEXT verify round, relayed last stage -> all stages. + # bonus) for the NEXT verify round, relayed last stage -> all stages, + # with the tree topology the tokens were arranged by (parent_list and + # top_scores_index have different widths, so they stay separate). next_verify_chain: Optional[torch.Tensor] = None + next_verify_parent_list: Optional[torch.Tensor] = None + next_verify_top_scores_index: Optional[torch.Tensor] = None # Refs the worker wants scheduler to keep alive for the same 2-iter window # as batch_record_buf. Used for cross-stream tensor lifetime (e.g. a spec diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index d07b3d853b91..923afbbe78c9 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -553,7 +553,7 @@ def _capture_cuda_graphs(self): f"avail mem={after_mem:.2f} GB.", ) - def draft(self, batch: ScheduleBatch): + def draft(self, batch: ScheduleBatch, *, with_topology: bool = False): draft_input: EagleDraftInput = batch.spec_info forward_batch, can_run_decode_cuda_graph = prepare_for_draft( draft_input, @@ -601,7 +601,7 @@ def draft(self, batch: ScheduleBatch): self.draft_forward(forward_batch) ) - return build_eagle_verify_input( + verify_input = build_eagle_verify_input( batch, draft_input, parent_list, @@ -615,6 +615,12 @@ def draft(self, batch: ScheduleBatch): tree_mask_mode=self.tree_mask_mode, device=self.device, ) + if with_topology: + # PP+spec relays the tree so every stage rebuilds the same verify + # input; the mask build needs the topology this one was built from. + # Returned rather than stashed on self so the caller owns lifetime. + return verify_input, parent_list, top_scores_index + return verify_input def draft_forward(self, forward_batch: ForwardBatch): # Parse args @@ -1304,8 +1310,16 @@ def forward_batch_generation( speculative_moe_a2a_backend_context(), spec_stage_span("draft"), ): - next_verify_input = self.draft_worker.draft(batch) + next_verify_input, parent_list, top_scores_index = ( + self.draft_worker.draft(batch, with_topology=True) + ) batch_output.next_verify_chain = next_verify_input.draft_token + # The tree shape is data-dependent once topk > 1, so the other + # stages cannot re-derive it; relay it alongside the tokens. + # clone(): both come out of cuda-graph-owned buffers under + # decode replay and would be overwritten before the relay. + batch_output.next_verify_parent_list = parent_list.clone() + batch_output.next_verify_top_scores_index = top_scores_index.clone() return batch_output From dc440b9496caa18273ea3065ab92d33c33401879 Mon Sep 17 00:00:00 2001 From: alichen Date: Mon, 17 Aug 2026 00:31:15 -0700 Subject: [PATCH 15/28] Add PP x spec tests; drop the mixed-chunk assert as unreachable test/registered/pp/test_pp_spec.py: - TestPPSpecConsistency runs GSM8K under plain spec and under PP2+spec and requires the PP run to match on both accuracy and accept length. Accuracy alone cannot distinguish a working relay from a broken one: if the relayed tree is lost, the bonus token is still force-accepted and the rest is rejected, so the output stays correct and only acceptance collapses. Both a topk=1 chain and a topk=2 tree are covered, since the relayed topology is constant for the former and data-dependent for the latter. - TestPPSpecGate pins that the gate is off by default (the existing PP+spec ban still fires) and that the combinations the relay cannot reproduce identically on every stage are rejected. It resolves argument state only, so it stays off the GPU. Also drop the --enable-mixed-chunk assert added in the previous commit: the EAGLE argument hook already forces enable_mixed_chunk = False for every EAGLE config, so it could never fire. The --enable-dp-attention assert is load-bearing (verified: nothing upstream rejects that combination first) and stays. Co-Authored-By: Claude Opus 5 (1M context) --- python/sglang/srt/server_args.py | 12 +- test/registered/pp/test_pp_spec.py | 174 +++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+), 8 deletions(-) create mode 100644 test/registered/pp/test_pp_spec.py diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index fe6655948bcb..687815f02ffb 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -9098,14 +9098,10 @@ def check_server_args(self): "--speculative-adaptive" ) # Every stage rebuilds the same verify input from the relayed - # per-request state, so all stages must see the same batch - # composition. Mixed-chunk splices decode requests into an - # extend batch (whose relayed next_token_ids are prefill - # placeholders), and DP attention partitions the batch per DP - # rank; both break that invariant. - assert ( - not self.enable_mixed_chunk - ), "SGLANG_ENABLE_PP_SPEC is not compatible with --enable-mixed-chunk" + # per-request state, so all stages must see the same batch. + # DP attention partitions it per DP rank. (Mixed chunk would + # break the same invariant, but the EAGLE arg hook already + # forces it off.) assert ( not self.enable_dp_attention ), "SGLANG_ENABLE_PP_SPEC is not compatible with --enable-dp-attention" diff --git a/test/registered/pp/test_pp_spec.py b/test/registered/pp/test_pp_spec.py new file mode 100644 index 000000000000..20f5dc39f8fa --- /dev/null +++ b/test/registered/pp/test_pp_spec.py @@ -0,0 +1,174 @@ +""" +Usage: +python3 -m unittest test_pp_spec.TestPPSpecConsistency.test_pp_matches_non_pp +""" + +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_DRAFT_MODEL_EAGLE, + DEFAULT_TARGET_MODEL_EAGLE, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_cuda_ci(est_time=900, stage="extra-b", runner_config="4-gpu-h100") + +# topk=1 chains and a topk=2 tree: the relayed topology is constant for the +# former and data-dependent for the latter, so both shapes are covered. +SPEC_SHAPES = { + "chain": ("2", "1", "3"), # num_steps, eagle_topk, num_draft_tokens + "tree": ("2", "2", "4"), +} + + +class TestPPSpecConsistency(CustomTestCase): + """PP x speculative decoding must match plain speculative decoding. + + Every PP stage rebuilds the verify tree from state relayed by the last + stage, so a mis-sized proxy buffer or a mis-rebuilt tree shows up as an + accuracy drop or as speculation that never gets accepted -- not as a + crash. Pin both. + """ + + def _run(self, pp_size: int, shape: str): + num_steps, topk, num_draft_tokens = SPEC_SHAPES[shape] + other_args = [ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_DRAFT_MODEL_EAGLE, + "--speculative-num-steps", + num_steps, + "--speculative-eagle-topk", + topk, + "--speculative-num-draft-tokens", + num_draft_tokens, + "--mem-fraction-static", + "0.7", + ] + if pp_size > 1: + other_args += ["--pp-size", str(pp_size), "--disable-overlap-schedule"] + + process = popen_launch_server( + DEFAULT_TARGET_MODEL_EAGLE, + DEFAULT_URL_FOR_TEST, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env={**os.environ, "SGLANG_ENABLE_PP_SPEC": "1"}, + ) + try: + metrics = run_eval( + SimpleNamespace( + base_url=DEFAULT_URL_FOR_TEST, + model=DEFAULT_TARGET_MODEL_EAGLE, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=256, + num_threads=32, + ) + ) + server_info = requests.get(f"{DEFAULT_URL_FOR_TEST}/get_server_info") + accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + finally: + kill_process_tree(process.pid) + return metrics["score"], accept_length + + def _assert_matches(self, shape: str): + base_score, base_accept = self._run(pp_size=1, shape=shape) + pp_score, pp_accept = self._run(pp_size=2, shape=shape) + print( + f"[PP spec {shape}] no-PP: score={base_score:.4f} accept={base_accept:.2f}" + f" | PP2: score={pp_score:.4f} accept={pp_accept:.2f}" + ) + + self.assertGreaterEqual( + pp_score, + base_score - 0.02, + msg=( + f"PP+spec accuracy dropped more than 2% against plain spec. " + f"no-PP: {base_score:.2%}, PP2: {pp_score:.2%}" + ), + ) + # A relay that loses the drafted tree still produces correct output -- + # the bonus token is force-accepted and the rest is rejected -- so + # accuracy alone cannot tell drafting from a no-op. Acceptance can. + self.assertGreaterEqual( + pp_accept, + base_accept - 0.2, + msg=( + f"PP+spec accept length collapsed against plain spec. " + f"no-PP: {base_accept:.2f}, PP2: {pp_accept:.2f}" + ), + ) + + def test_pp_matches_non_pp(self): + self._assert_matches("chain") + + def test_pp_matches_non_pp_tree(self): + self._assert_matches("tree") + + +class TestPPSpecGate(CustomTestCase): + """The gate is off by default, and the combinations the relay cannot + reproduce identically on every stage are rejected rather than silently + mis-rebuilt.""" + + def _server_args(self, **overrides): + from sglang.srt.server_args import ServerArgs + from sglang.test.mock_model.utils import MOCK_MODEL_PATH + + # Argument validation only reads the config, so the small mock model + # keeps these cases off the GPU entirely. + args = dict( + model_path=MOCK_MODEL_PATH, + pp_size=2, + disable_overlap_schedule=True, + speculative_algorithm="EAGLE", + speculative_draft_model_path=MOCK_MODEL_PATH, + speculative_num_steps=2, + speculative_eagle_topk=1, + speculative_num_draft_tokens=3, + ) + args.update(overrides) + return ServerArgs(**args) + + def test_gate_off_keeps_the_ban(self): + os.environ.pop("SGLANG_ENABLE_PP_SPEC", None) + with self.assertRaises(AssertionError): + self._server_args().check_server_args() + + def test_gate_on_rejects_unsupported_combinations(self): + os.environ["SGLANG_ENABLE_PP_SPEC"] = "1" + try: + self._server_args().check_server_args() + # DP attention partitions the batch per DP rank, so the stages + # would no longer rebuild the same verify tree. + with self.assertRaises(AssertionError): + self._server_args( + tp_size=2, dp_size=2, enable_dp_attention=True + ).check_server_args() + # Adaptive spec changes num_draft_tokens at runtime, which the + # relay slices results with. + with self.assertRaises(AssertionError): + self._server_args( + speculative_adaptive=True, speculative_num_steps=3 + ).check_server_args() + finally: + os.environ.pop("SGLANG_ENABLE_PP_SPEC", None) + + +if __name__ == "__main__": + unittest.main() From d41e09703e10f24b07d6c5694b52bddb7ef4b426 Mon Sep 17 00:00:00 2001 From: alichen Date: Mon, 17 Aug 2026 01:53:47 -0700 Subject: [PATCH 16/28] Carry the PP relay state as a SpecInput instead of a scheduler side table The tree the last stage drafts was held in a rid-keyed dict on the scheduler, with a per-launch ScheduleBatch.copy() to reconcile it against a microbatch that had been recomposed since. That works, but it is machinery this PR owns alone: a second speculative algorithm would have to reimplement it, and the state is invisible to the batch it describes. Make it a first-class PPSpecRelayInput(SpecInput) that rides on ScheduleBatch.spec_info and implements the filter / merge hooks, so the existing recomposition path carries it: a request that finishes, is retracted, or merges in from a just-finished prefill takes its row along. The rows keep the rids they were labelled with, since the relayed tensors are sized by the composition that ran the forward, which the live batch can have moved past by the time the result comes back around the ring; adopt() folds them in by rid, and the rebuild relabels if anything desynced the two. The type is deliberately the algorithm-agnostic half -- proposed tokens plus the topology they were arranged by -- so another algorithm supplies only its own rebuild. Validated on 8x RTX PRO 6000 (SM120), GLM-5.2-NVFP4, TP4xPP2 + EAGLE (2 steps / topk 1 / 3 draft tokens), GSM8K 8-shot greedy, 200 questions per run: rid-keyed dict: 0.940 / 0.945 / 0.955 accept 2.37-2.92 this commit: 0.940 / 0.965 / 0.945 / 0.935 accept 2.57-2.85 Note the spread: the same build varies 0.940-0.965 run to run here, because batch composition follows arrival timing and the reductions follow batch composition. Single runs cannot resolve a change of this size, and greedy output is not token-reproducible on this stack either, so this is measured as "no detectable regression", not as bit-identical behavior. Co-Authored-By: Claude Opus 5 (1M context) --- python/sglang/srt/managers/scheduler.py | 21 +- .../sglang/srt/managers/scheduler_pp_mixin.py | 184 +++++++----------- .../sglang/srt/speculative/pp_spec_relay.py | 178 +++++++++++++++++ python/sglang/srt/speculative/spec_info.py | 4 + 4 files changed, 270 insertions(+), 117 deletions(-) create mode 100644 python/sglang/srt/speculative/pp_spec_relay.py diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index c59ba4c9dfc6..260d581a8314 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -3777,6 +3777,10 @@ def run_batch( is_verify_round = self.server_args.pp_size > 1 and not ( batch.forward_mode.is_extend() or batch.is_extend_in_batch ) + # The relayed tree is what the requests carry between rounds; + # the rebuild below swaps it for this round's verify input, so + # hold on to it and put it back once the forward is done. + relay_input = batch.spec_info if is_verify_round else None if is_verify_round: # PP+spec decode: every stage rebuilds the same verify # input from relayed per-req state (draft lives on the @@ -3820,9 +3824,9 @@ def run_batch( ) batch.input_ids = None # rebuilt next iter from draft_token # The verify input is per-round; between iterations - # spec_info must be merge/filter-safe (None on stages - # without a draft worker). - batch.spec_info = None + # spec_info carries the relayed tree, which is + # merge/filter-safe. + batch.spec_info = relay_input else: # Non-overlap: drive the V2 worker synchronously (no # future_map relay / on_publish). Only the PP+spec worker @@ -3839,8 +3843,15 @@ def run_batch( batch, **kwargs ) # The isolation restore reverted the worker's in-forward SB edits; - # re-apply what must carry to the next iter. - batch.spec_info = batch_result.next_draft_input + # re-apply what must carry to the next iter. Under PP the + # tail draft already consumed the draft input in-round, and + # the next round's tree comes from the relay, so the last + # stage carries the same relayed tree as the others. + batch.spec_info = ( + relay_input + if is_verify_round + else batch_result.next_draft_input + ) if batch_result.new_seq_lens is not None: batch.seq_lens = batch_result.new_seq_lens if batch.seq_lens_cpu is not None: diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index ae8f6be57ea1..056c5ec96c17 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -8,7 +8,6 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Dict, List, Optional, Tuple -import msgspec import numpy as np import torch import torch.distributed @@ -60,19 +59,6 @@ def _pp_can_skip_output_comm(batch: ScheduleBatch) -> bool: ) -class _PPSpecTreeRow(msgspec.Struct): - """One request's relayed draft tree: the tokens plus the topology they - were arranged by. Held per rid between PP+spec rounds.""" - - tokens: torch.Tensor # [num_draft_tokens], index 0 is the bonus token - # parent_list / top_scores_index rows. None until the request has been - # drafted for (its first decode after prefill carries zero drafts, which - # are rejected whatever tree shape they are hung on); the rebuild then - # substitutes chain constants at the width the drafted rows use. - parents: Optional[torch.Tensor] - top_scores: Optional[torch.Tensor] - - @dataclass class PPBatchMetadata: can_run_cuda_graph: bool @@ -598,8 +584,6 @@ def init_pp_loop_state(self: Scheduler): self.mb_metadata: List[Optional[PPBatchMetadata]] = [None] * self.pp_loop_size self.pp_outputs: Optional[PPProxyTensors] = None self.last_rank_comm_queue: deque[Tuple[torch.Event, PPProxyTensors]] = deque() - # PP+spec: per-rid chain rows seeding the next verify round. - self._pp_spec_chain_by_rid: Dict[str, torch.Tensor] = {} self._pp_spec_relay = ( envs.SGLANG_ENABLE_PP_SPEC.get() and self.ps.pp_size > 1 @@ -1233,25 +1217,7 @@ def _pp_prep_batch_result( merged, dtype=batch.seq_lens_cpu.dtype ) batch.seq_lens_sum = int(sum(merged)) - self._pp_spec_store_bonus( - fwd_batch, - pp_outputs["spec_bonus_tokens"], - chain_tokens=( - pp_outputs["spec_next_chain"] - if "spec_next_chain" in pp_outputs.tensors - else None - ), - parent_list=( - pp_outputs["spec_next_parents"] - if "spec_next_parents" in pp_outputs.tensors - else None - ), - top_scores_index=( - pp_outputs["spec_next_top_scores"] - if "spec_next_top_scores" in pp_outputs.tensors - else None - ), - ) + self._pp_spec_adopt_relayed_tree(batch, fwd_rids, pp_outputs) output_result = GenerationBatchResult( logits_output=logits_output, pp_hidden_states_proxy_tensors=None, @@ -1273,12 +1239,23 @@ def _pp_prep_batch_result( # and storing it would seed the next verify round with a token the # model never emitted. if batch.contains_last_prefill_chunk: + from sglang.srt.speculative.pp_spec_relay import PPSpecRelayInput + + # The request leaves prefill proposing nothing; the last stage + # drafts a real tree for it at the end of its first verify. fwd_batch = ( mb_metadata.fwd_batch if mb_metadata.fwd_batch is not None else batch ) - self._pp_spec_store_bonus(fwd_batch, next_token_ids) + self._pp_spec_set_relay( + batch, + PPSpecRelayInput.degenerate( + rids=[req.rid for req in fwd_batch.reqs], + bonus_tokens=next_token_ids, + num_draft_tokens=self.server_args.speculative_num_draft_tokens, + ), + ) else: # PP rank 0 also relays into output_tokens_buf so the next iter's # resolve_forward_inputs finds these tokens for the decode portion @@ -1303,66 +1280,54 @@ def _pp_process_batch_result( self: Scheduler, batch: ScheduleBatch, output_result: GenerationBatchResult ): self.process_batch_result(batch, output_result) - # Chains are stored before finish flags are set (at result prep), so - # finished requests must be dropped here or the dict grows per request. - if self._pp_spec_chain_by_rid: - for req in batch.reqs: - if req.finished(): - self._pp_spec_chain_by_rid.pop(req.rid, None) - - def _pp_spec_store_bonus( + + def _pp_spec_adopt_relayed_tree( self: Scheduler, batch: ScheduleBatch, - bonus_tokens: torch.Tensor, - chain_tokens: Optional[torch.Tensor] = None, - parent_list: Optional[torch.Tensor] = None, - top_scores_index: Optional[torch.Tensor] = None, + fwd_rids: List[str], + pp_outputs: PPProxyTensors, ) -> None: - """Stash the per-request tree row (root = bonus token) that seeds the - next verify round, together with the topology it was arranged by. - - chain_tokens is the last stage's tail-drafted tree (flat bs*dtn) and - parent_list / top_scores_index its shape; without them (prefill rounds) - a degenerate row [bonus, 0, ...] over a chain topology is stored — the - zero drafts just get rejected, costing acceptance not correctness. - Keyed by rid (not batch position): the microbatch composition can - change between rounds (finish / retract / merge).""" + """Fold the tree the last stage drafted into the live batch. + + The relayed rows are labelled with the composition that ran the + forward; the live microbatch may have been recomposed since, so they + are folded in by rid rather than by position.""" + from sglang.srt.speculative.pp_spec_relay import PPSpecRelayInput + + chain = pp_outputs.tensors.get("spec_next_chain") + if chain is None: + # The last stage skipped drafting this round (idle / zero steps); + # the requests keep the tree they already carry. + return num_draft_tokens = self.server_args.speculative_num_draft_tokens - bs = len(batch.reqs) - if chain_tokens is not None: - rows = chain_tokens.to(torch.int64).reshape(bs, num_draft_tokens) - else: - rows = torch.zeros( - (bs, num_draft_tokens), - dtype=torch.int64, - device=bonus_tokens.device, - ) - rows[:, 0] = bonus_tokens.to(torch.int64) - for i, req in enumerate(batch.reqs): - if req.finished(): - self._pp_spec_chain_by_rid.pop(req.rid, None) - else: - # clone(): a slice view would pin the whole rows storage for - # the entry's lifetime, and when chain_tokens is already - # int64 the .to() above is a no-op view of the relay buffer - # — a later relay reusing that buffer would corrupt stored - # rows (the row root is force-accepted by verify). - self._pp_spec_chain_by_rid[req.rid] = _PPSpecTreeRow( - tokens=rows[i].clone(), - parents=None if parent_list is None else parent_list[i].clone(), - top_scores=( - None - if top_scores_index is None - else top_scores_index[i].clone() - ), - ) + relayed = PPSpecRelayInput( + rids=fwd_rids, + tokens=chain.to(torch.int64).reshape(len(fwd_rids), num_draft_tokens), + parents=pp_outputs.tensors.get("spec_next_parents"), + top_scores=pp_outputs.tensors.get("spec_next_top_scores"), + ) + self._pp_spec_set_relay(batch, relayed) + + def _pp_spec_set_relay(self: Scheduler, batch: ScheduleBatch, relayed) -> None: + """Attach rows labelled with the forward-time composition to the live + batch: fold them into what the requests already carry, or relabel them + into the live order when the batch carries nothing yet.""" + from sglang.srt.speculative.pp_spec_relay import PPSpecRelayInput + + if isinstance(batch.spec_info, PPSpecRelayInput): + batch.spec_info.adopt(relayed) + return + live_rids = [req.rid for req in batch.reqs] + batch.spec_info = ( + relayed if live_rids == relayed.rids else relayed.reindex(live_rids) + ) def _pp_spec_chain_topology( self: Scheduler, bs: int, device: str ) -> Tuple[torch.Tensor, torch.Tensor]: - """topk=1 chain constants, the shape a prefill round's degenerate row - implies. Mirrors _rebuild_topk1_chain_buffers: a single-step chain has - no parent entries, and both widths key off num_steps.""" + """topk=1 chain constants, the shape a not-yet-drafted row implies. + Mirrors _rebuild_topk1_chain_buffers: a single-step chain has no parent + entries, and both widths key off num_steps.""" num_steps = self.server_args.speculative_num_steps parent_width = num_steps if num_steps > 1 else 0 parent_list = torch.arange( @@ -1389,6 +1354,7 @@ def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None build_tree_kernel_efficient, default_tree_mask_mode, ) + from sglang.srt.speculative.pp_spec_relay import PPSpecRelayInput sa = self.server_args steps = sa.speculative_num_steps @@ -1405,33 +1371,27 @@ def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None ) return - rows = [self._pp_spec_chain_by_rid[req.rid] for req in batch.reqs] - tree_rows = torch.stack([r.tokens for r in rows]).to( - device=device, dtype=torch.int64 - ) + relay: PPSpecRelayInput = batch.spec_info + # The rows track the batch through filter / merge, but a recomposition + # that bypasses those hooks would leave them labelled for a different + # order, and the rebuild reads them positionally. Relabel rather than + # hand the verify kernel another request's bonus token. + live_rids = [req.rid for req in batch.reqs] + if relay.rids != live_rids: + relay = relay.reindex(live_rids) + batch.spec_info = relay + tree_rows = relay.tokens.to(device=device, dtype=torch.int64) bonus_tokens = tree_rows[:, 0].contiguous() draft_tokens = tree_rows[:, 1:].contiguous() # Topology as drafted on the last stage. It is data-dependent once - # topk > 1, so it rides the relay rather than being re-derived here. - # Not-yet-drafted rows borrow the drafted rows' widths so the stack is - # rectangular; their drafts are zeros and get rejected regardless of - # the shape they hang on. - drafted = next((r for r in rows if r.parents is not None), None) - if drafted is None: - parent_list, top_scores_index = self._pp_spec_chain_topology(bs, device) - else: - chain_parents = torch.arange( - -1, drafted.parents.shape[0] - 1, dtype=torch.long, device=device - ) - chain_scores = torch.arange( - drafted.top_scores.shape[0], dtype=torch.long, device=device - ) - parent_list = torch.stack( - [chain_parents if r.parents is None else r.parents for r in rows] - ).to(device=device, dtype=torch.long) - top_scores_index = torch.stack( - [chain_scores if r.top_scores is None else r.top_scores for r in rows] - ).to(device=device, dtype=torch.long) + # topk > 1, so it rides the relay rather than being re-derived here; + # requests that have not been drafted for yet fall back to the chain + # constants, and their zero drafts get rejected either way. + parent_list, top_scores_index = relay.topology( + fallback=lambda: self._pp_spec_chain_topology(bs, device) + ) + parent_list = parent_list.to(device=device, dtype=torch.long) + top_scores_index = top_scores_index.to(device=device, dtype=torch.long) # Mask selection mirrors the last stage's draft() tail # (build_eagle_verify_input) so every stage builds the same mask. diff --git a/python/sglang/srt/speculative/pp_spec_relay.py b/python/sglang/srt/speculative/pp_spec_relay.py new file mode 100644 index 000000000000..ffc7fea9a970 --- /dev/null +++ b/python/sglang/srt/speculative/pp_spec_relay.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +from typing import List, Optional + +import torch + +from sglang.srt.speculative.spec_info import SpecInput, SpecInputType + + +class PPSpecRelayInput(SpecInput): + """The draft tree the last PP stage produced, carried by the requests it + belongs to so every stage can rebuild the same verify input. + + Under PP the draft model lives on the last stage only, so the other stages + receive the tree over the output relay instead of drafting it. It has to + survive between rounds and across microbatch recomposition, which is why + it rides on ``ScheduleBatch.spec_info`` and implements the filter / merge + hooks rather than living in a side table: a request that finishes, gets + retracted, or is merged in from a just-finished prefill carries its own + row along with it. + + Rows are per request and aligned with ``batch.reqs``. ``rids`` is kept + alongside them because the relayed tensors are sized by the composition + that ran the forward, which can differ from the live batch by the time + the result comes back around the ring. + + This is the algorithm-agnostic half of the PP relay: the tokens plus the + topology they were arranged by. A speculative algorithm whose proposal is + not an EAGLE-style tree can subclass it (or mirror it) and only has to + supply its own rebuild. + """ + + def __init__( + self, + rids: List[str], + tokens: torch.Tensor, + parents: Optional[torch.Tensor] = None, + top_scores: Optional[torch.Tensor] = None, + ): + super().__init__(SpecInputType.PP_SPEC_RELAY) + # [bs, num_draft_tokens], column 0 is the bonus token + self.rids = rids + self.tokens = tokens + # parent_list / top_scores_index, [bs, *]. None until the request has + # been drafted for: its first decode after prefill carries zero drafts, + # which are rejected whatever tree shape they hang on. + self.parents = parents + self.top_scores = top_scores + + def __repr__(self) -> str: + return ( + f"PPSpecRelayInput(bs={len(self.rids)}, " + f"drafted={self.parents is not None})" + ) + + @classmethod + def degenerate( + cls, rids: List[str], bonus_tokens: torch.Tensor, num_draft_tokens: int + ) -> PPSpecRelayInput: + """A tree that proposes nothing: just the sampled token, padded with + zeros. What a request carries out of prefill, before the last stage + has drafted for it.""" + tokens = torch.zeros( + (len(rids), num_draft_tokens), + dtype=torch.int64, + device=bonus_tokens.device, + ) + tokens[:, 0] = bonus_tokens.to(torch.int64) + return cls(rids=list(rids), tokens=tokens) + + def filter_batch( + self, new_indices: torch.Tensor, new_indices_cpu: Optional[List[int]] = None + ) -> None: + keep = new_indices_cpu if new_indices_cpu is not None else new_indices.tolist() + self.rids = [self.rids[i] for i in keep] + self.tokens = self.tokens[new_indices] + if self.parents is not None: + self.parents = self.parents[new_indices] + self.top_scores = self.top_scores[new_indices] + + def merge_batch(self, other: PPSpecRelayInput) -> None: + if not other.rids: + return + if not self.rids: + self.rids, self.tokens = list(other.rids), other.tokens + self.parents, self.top_scores = other.parents, other.top_scores + return + self.rids = self.rids + list(other.rids) + self.tokens = torch.cat([self.tokens, other.tokens]) + # A batch merging in from prefill has no topology yet; give it the + # other side's widths so the rows stay stackable. + left, right = self._widths(), other._widths() + widths = left if left is not None else right + if widths is None: + self.parents = self.top_scores = None + return + self.parents = torch.cat( + [self._parents_or_chain(widths), other._parents_or_chain(widths)] + ) + self.top_scores = torch.cat( + [self._top_scores_or_chain(widths), other._top_scores_or_chain(widths)] + ) + + def adopt(self, relayed: PPSpecRelayInput) -> None: + """Take the relayed rows for the requests they cover, in this input's + order, keeping the current row for any request the relay does not + mention (one merged in after the forward was launched).""" + by_rid = {rid: i for i, rid in enumerate(relayed.rids)} + rows = [by_rid.get(rid) for rid in self.rids] + if all(r is None for r in rows): + return + take = torch.tensor( + [r if r is not None else 0 for r in rows], + dtype=torch.long, + device=relayed.tokens.device, + ) + keep = torch.tensor( + [r is None for r in rows], dtype=torch.bool, device=self.tokens.device + ) + relayed_tokens = relayed.tokens.to(self.tokens.device)[take] + self.tokens = torch.where(keep.unsqueeze(1), self.tokens, relayed_tokens) + if relayed.parents is None: + return + widths = relayed._widths() + self.parents = torch.where( + keep.unsqueeze(1), + self._parents_or_chain(widths), + relayed.parents.to(self.tokens.device)[take], + ) + self.top_scores = torch.where( + keep.unsqueeze(1), + self._top_scores_or_chain(widths), + relayed.top_scores.to(self.tokens.device)[take], + ) + + def reindex(self, rids: List[str]) -> PPSpecRelayInput: + """This input's rows in another composition's order. Every rid must be + covered -- the caller is relabelling the same requests, not adding.""" + by_rid = {rid: i for i, rid in enumerate(self.rids)} + take = torch.tensor( + [by_rid[rid] for rid in rids], dtype=torch.long, device=self.tokens.device + ) + return PPSpecRelayInput( + rids=list(rids), + tokens=self.tokens[take], + parents=None if self.parents is None else self.parents[take], + top_scores=None if self.top_scores is None else self.top_scores[take], + ) + + def topology(self, *, fallback): + """The rows' tree shape, as a rectangular pair. ``fallback`` supplies + chain constants for the case where no request has been drafted for + yet, since their width comes from the spec config, not from a row.""" + widths = self._widths() + if widths is None: + return fallback() + return self._parents_or_chain(widths), self._top_scores_or_chain(widths) + + def _widths(self): + if self.parents is None: + return None + return self.parents.shape[1], self.top_scores.shape[1] + + def _parents_or_chain(self, widths) -> torch.Tensor: + if self.parents is not None: + return self.parents + width = widths[0] + return torch.arange( + -1, width - 1, dtype=torch.long, device=self.tokens.device + ).repeat(len(self.rids), 1) + + def _top_scores_or_chain(self, widths) -> torch.Tensor: + if self.top_scores is not None: + return self.top_scores + width = widths[1] + return torch.arange(width, dtype=torch.long, device=self.tokens.device).repeat( + len(self.rids), 1 + ) diff --git a/python/sglang/srt/speculative/spec_info.py b/python/sglang/srt/speculative/spec_info.py index e018ee4c09e8..d4c58943444b 100644 --- a/python/sglang/srt/speculative/spec_info.py +++ b/python/sglang/srt/speculative/spec_info.py @@ -327,6 +327,10 @@ class SpecInputType(IntEnum): DFLASH_DRAFT = auto() DFLASH_VERIFY = auto() NGRAM_VERIFY = auto() + # Carried between rounds under PP: the tree the last stage drafted, which + # every stage rebuilds its verify input from. Neither a draft nor a verify + # input -- no forward ever runs on it. + PP_SPEC_RELAY = auto() class SpecInput(ABC): From 6a0ec0edcb7422b751486eb5be91ddca4ad62509 Mon Sep 17 00:00:00 2001 From: alichen Date: Mon, 31 Aug 2026 03:16:23 -0700 Subject: [PATCH 17/28] Lint: isort/black over the merge --- python/sglang/srt/arg_groups/validation_hook.py | 5 ++--- python/sglang/srt/managers/scheduler_pp_mixin.py | 8 ++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/arg_groups/validation_hook.py b/python/sglang/srt/arg_groups/validation_hook.py index 89217b85e23e..b07c206cfeef 100644 --- a/python/sglang/srt/arg_groups/validation_hook.py +++ b/python/sglang/srt/arg_groups/validation_hook.py @@ -14,10 +14,10 @@ resolving_view, run_post_process_pass, ) -from sglang.srt.environ import envs from sglang.srt.distributed.device_communicators.mooncake_transfer_engine import ( parse_ib_device_config, ) +from sglang.srt.environ import envs from sglang.srt.runtime_context import get_platform from sglang.srt.utils.common import torch_release from sglang.srt.utils.runai_utils import is_runai_obj_uri @@ -61,8 +61,7 @@ def check_server_args(server_args: Any): # The PP relay slices spec results with the configured # num_draft_tokens; adaptive spec changes it at runtime. assert not cfg.speculative_adaptive, ( - "SGLANG_ENABLE_PP_SPEC is not compatible with " - "--speculative-adaptive" + "SGLANG_ENABLE_PP_SPEC is not compatible with " "--speculative-adaptive" ) # Every stage rebuilds the same verify input from the relayed # per-request state, so all stages must see the same batch. diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 91a809a00150..3c8d9e840876 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -1310,9 +1310,13 @@ def _pp_prep_batch_result( batch.req_pool_indices, RelayPayload( bonus_tokens=next_token_ids, - topk_p=None if next_draft_input is None else next_draft_input.topk_p, + topk_p=( + None if next_draft_input is None else next_draft_input.topk_p + ), topk_index=( - None if next_draft_input is None else next_draft_input.topk_index + None + if next_draft_input is None + else next_draft_input.topk_index ), hidden_states=( None From 08ae55cbfe888f48a660d37b707a6380f8d68806 Mon Sep 17 00:00:00 2001 From: alichen Date: Mon, 31 Aug 2026 22:05:56 -0700 Subject: [PATCH 18/28] Gate the PP+spec relay reads mock-safely test_generation_auxiliary_output.py drives _pp_prepare_tensor_dict and _pp_prep_batch_result with object.__new__(Scheduler) and SimpleNamespace batches that carry only the fields those functions read upstream. The gated branches added batch.spec_algorithm / self._pp_spec_relay reads, widening the functions' implicit input surface and failing five of those tests with AttributeError. Give the mixin a class-level `_pp_spec_relay = False` default -- a scheduler that never ran init_pp_loop_state (plain TP, test doubles) reads False and takes the upstream path -- and gate on it plus result/relay-side signals (accept_lens, the spec_* relay keys) instead of re-deriving spec-ness from the batch. No behavior change when the gate is on: init_pp_loop_state already computed the flag from the same inputs. Co-Authored-By: Claude Opus 5 (1M context) --- python/sglang/srt/managers/scheduler_pp_mixin.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 3c8d9e840876..2eccab94b1fe 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -75,6 +75,11 @@ class PPBatchMetadata: class SchedulerPPMixin: + # Gated PP+spec relay flow. Class-level default so schedulers that never + # ran init_pp_loop_state (plain TP, unit-test doubles) read False; + # init_pp_loop_state overwrites it per instance. + _pp_spec_relay: bool = False + @DynamicGradMode() def event_loop_pp(self: Scheduler): """ @@ -1041,7 +1046,7 @@ def _pp_prepare_tensor_dict( "next_token_ids": result.next_token_ids, } - if not batch.spec_algorithm.is_none() and result.accept_lens is not None: + if self._pp_spec_relay and result.accept_lens is not None: # PP+spec verify round: earlier stages need the accept results to # mirror seq_lens/KV bookkeeping and the bonus token to root the # next round's verify chain. @@ -1212,8 +1217,7 @@ def _pp_prep_batch_result( logits_output = LogitsProcessorOutput(next_token_logits=None) logits_output.auxiliary_device_output = auxiliary_output - is_spec = not batch.spec_algorithm.is_none() - if is_spec and "spec_accept_lens" in pp_outputs.tensors: + if self._pp_spec_relay and "spec_accept_lens" in pp_outputs.tensors: # PP+spec verify round. Relayed tensors align with the composition # that ran the forward (mb_metadata.fwd_batch snapshot); the live # mb object may have been merged/filtered in place since. Mirror @@ -1279,7 +1283,7 @@ def _pp_prep_batch_result( ) batch.spec_info = next_draft_input - if is_spec and self._pp_spec_relay: + if self._pp_spec_relay: # Gated single-instance PP+spec: the sampled first token roots # round 1's tree. Only the chunk that finishes the prompt samples # a real token; a middle chunk's next_token_ids is a placeholder From 26840318dc45221e0b35754bbac0da4ec8a46f69 Mon Sep 17 00:00:00 2001 From: alichen Date: Tue, 1 Sep 2026 02:39:45 -0700 Subject: [PATCH 19/28] Address review: config reads, algorithm whitelist, PD gate, embed loader, steps=0 Five fixes from @YAMY1234's review, plus the supplied-instance ratchet the same reads were failing in CI. - Config reads go through the resolved bags. pp_size comes from self.ps (scheduler) or get_parallel() (workers -- the draft worker's ps is replace(ps, pp_size=1), so its own view would defeat the gate), load_format from get_model(), and the speculative_* reads in scheduler_pp_mixin from get_spec(). Clears the two base-a ratchet failures. - The gate now whitelists what the relay actually supports: single-layer EAGLE/MTP only. NGRAM, FROZEN_KV_MTP, multi-layer EAGLE, and custom algorithms were admitted before, and every stage would have rebuilt an EAGLE-shaped verify input for them. - PD disaggregation is rejected under the gate: the gated flow replaces the future_map RelayPayload relay, and PPSpecRelayInput does not carry the topk_p / topk_index / hidden states PD prefill needs. - The draft embedding loads from the *target* checkpoint through the standard weight loader (DefaultModelLoader._prepare_weights + weights iterators). The old helper read the draft checkpoint -- GLM/DSv4 MTP hid this because draft and target share one path -- and required a local safetensors directory, which a hub id with a .bin checkpoint fails. - speculative_num_steps == 0 no longer freezes the verify root: the last stage emits no tail-drafted tree in that mode, so the relay now falls back to a root-only row built from the relayed bonus token. Keeping the old row re-verified a token that was already accepted. The gate test gains cases for the whitelist and the PD rejection, and resolves args via resolve_once() so check_server_args sees the resolution-filled fields. Validated on 8x RTX PRO 6000 (SM120), GLM-5.2-NVFP4, TP4xPP2: - EAGLE 2s/topk1/3dt: GSM8K-100 0.900, accept 2.62-2.98 - steps=0 (the previously-broken mode): GSM8K-100 0.950, accept 1.00 - bench ISL8K bs1/8/32 TPOT 9.96/30.98/97.01 ms -- within noise of the pre-fix build (9.91/30.99/96.9) - ratchet 2 passed, auxiliary-output 27 passed, gate tests 2 passed Co-Authored-By: Claude Opus 5 (1M context) --- .../sglang/srt/arg_groups/validation_hook.py | 15 +++++ python/sglang/srt/managers/scheduler.py | 4 +- .../sglang/srt/managers/scheduler_pp_mixin.py | 47 ++++++++------ python/sglang/srt/managers/tp_worker.py | 3 +- .../sglang/srt/speculative/eagle_worker_v2.py | 61 +++++++++---------- test/registered/pp/test_pp_spec.py | 16 ++++- 6 files changed, 90 insertions(+), 56 deletions(-) diff --git a/python/sglang/srt/arg_groups/validation_hook.py b/python/sglang/srt/arg_groups/validation_hook.py index b07c206cfeef..df2d23119a99 100644 --- a/python/sglang/srt/arg_groups/validation_hook.py +++ b/python/sglang/srt/arg_groups/validation_hook.py @@ -58,6 +58,21 @@ def check_server_args(server_args: Any): assert ( cfg.disable_overlap_schedule ), "SGLANG_ENABLE_PP_SPEC requires --disable-overlap-schedule" + # The relay carries an EAGLE-shaped tree and only EAGLEWorkerV2 + # tail-drafts; every other algorithm would be mis-rebuilt. + assert ( + cfg.speculative_algorithm == "EAGLE" + and not cfg.enable_multi_layer_eagle + ), ( + "SGLANG_ENABLE_PP_SPEC supports single-layer EAGLE/MTP only, " + f"got {cfg.speculative_algorithm}" + ) + # PD prefill relays topk_p / topk_index / hidden states through + # RelayPayload; the gated flow replaces that relay with its own + # and does not carry those fields. + assert cfg.disaggregation_mode == "null", ( + "SGLANG_ENABLE_PP_SPEC is not compatible with " "--disaggregation-mode" + ) # The PP relay slices spec results with the configured # num_draft_tokens; adaptive spec changes it at runtime. assert not cfg.speculative_adaptive, ( diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index f65e96963925..8019b202531b 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -967,8 +967,8 @@ def maybe_init_draft_worker(self): if ( envs.SGLANG_ENABLE_PP_SPEC.get() - and self.server_args.pp_size > 1 - and self.ps.pp_rank != self.server_args.pp_size - 1 + and self.ps.pp_size > 1 + and self.ps.pp_rank != self.ps.pp_size - 1 ): # PP+spec: the draft model (MTP layer) needs final hidden states and # the lm_head, both of which live on the last PP stage only. diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 2eccab94b1fe..b45205d193ab 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -37,7 +37,7 @@ PPProxyTensors, ) from sglang.srt.observability.req_time_stats import set_time_batch -from sglang.srt.runtime_context import get_disagg, get_parallel +from sglang.srt.runtime_context import get_disagg, get_parallel, get_spec from sglang.srt.sampling.sampling_observer_pp import ( add_auxiliary_output_to_pp_tensors, pop_auxiliary_output_from_pp_tensors, @@ -1258,7 +1258,7 @@ def _pp_prep_batch_result( next_token_ids=pp_outputs["next_token_ids"].cpu(), accept_lens=pp_outputs["spec_accept_lens"].cpu(), new_seq_lens=new_seq_lens, - speculative_num_draft_tokens=self.server_args.speculative_num_draft_tokens, + speculative_num_draft_tokens=get_spec().speculative_num_draft_tokens, extend_input_len_per_req=extend_input_len_per_req, extend_logprob_start_len_per_req=extend_logprob_start_len_per_req, can_run_cuda_graph=mb_metadata.can_run_cuda_graph, @@ -1303,7 +1303,7 @@ def _pp_prep_batch_result( PPSpecRelayInput.degenerate( rids=[req.rid for req in fwd_batch.reqs], bonus_tokens=next_token_ids, - num_draft_tokens=self.server_args.speculative_num_draft_tokens, + num_draft_tokens=get_spec().speculative_num_draft_tokens, ), ) else: @@ -1360,18 +1360,25 @@ def _pp_spec_adopt_relayed_tree( are folded in by rid rather than by position.""" from sglang.srt.speculative.pp_spec_relay import PPSpecRelayInput + num_draft_tokens = get_spec().speculative_num_draft_tokens chain = pp_outputs.tensors.get("spec_next_chain") if chain is None: - # The last stage skipped drafting this round (idle / zero steps); - # the requests keep the tree they already carry. - return - num_draft_tokens = self.server_args.speculative_num_draft_tokens - relayed = PPSpecRelayInput( - rids=fwd_rids, - tokens=chain.to(torch.int64).reshape(len(fwd_rids), num_draft_tokens), - parents=pp_outputs.tensors.get("spec_next_parents"), - top_scores=pp_outputs.tensors.get("spec_next_top_scores"), - ) + # The last stage verified but skipped drafting (num_steps == 0 + # keeps draft KV warm without proposing). The bonus token it + # sampled must still become the next round's root: keeping the + # old row would re-verify a token that was already accepted. + relayed = PPSpecRelayInput.degenerate( + rids=fwd_rids, + bonus_tokens=pp_outputs["spec_bonus_tokens"], + num_draft_tokens=num_draft_tokens, + ) + else: + relayed = PPSpecRelayInput( + rids=fwd_rids, + tokens=chain.to(torch.int64).reshape(len(fwd_rids), num_draft_tokens), + parents=pp_outputs.tensors.get("spec_next_parents"), + top_scores=pp_outputs.tensors.get("spec_next_top_scores"), + ) self._pp_spec_set_relay(batch, relayed) def _pp_spec_set_relay(self: Scheduler, batch: ScheduleBatch, relayed) -> None: @@ -1394,7 +1401,7 @@ def _pp_spec_chain_topology( """topk=1 chain constants, the shape a not-yet-drafted row implies. Mirrors _rebuild_topk1_chain_buffers: a single-step chain has no parent entries, and both widths key off num_steps.""" - num_steps = self.server_args.speculative_num_steps + num_steps = get_spec().speculative_num_steps parent_width = num_steps if num_steps > 1 else 0 parent_list = torch.arange( -1, parent_width - 1, dtype=torch.long, device=device @@ -1422,15 +1429,15 @@ def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None ) from sglang.srt.speculative.pp_spec_relay import PPSpecRelayInput - sa = self.server_args - steps = sa.speculative_num_steps - num_draft_tokens = sa.speculative_num_draft_tokens + spec = get_spec() + steps = spec.speculative_num_steps + num_draft_tokens = spec.speculative_num_draft_tokens bs = batch.batch_size() device = self.device if batch.forward_mode.is_idle() or bs == 0: batch.spec_info = EagleVerifyInput.create_idle_input( - topk=sa.speculative_eagle_topk, + topk=spec.speculative_eagle_topk, spec_steps=steps, num_verify_tokens=num_draft_tokens, device=device, @@ -1491,7 +1498,7 @@ def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None draft_tokens, batch.seq_lens, seq_lens_sum, - sa.speculative_eagle_topk, + spec.speculative_eagle_topk, steps, num_draft_tokens, mask_mode, @@ -1507,7 +1514,7 @@ def _pp_spec_rebuild_verify_input(self: Scheduler, batch: ScheduleBatch) -> None retrieve_next_sibling=retrieve_next_sibling, retrieve_cum_len=None, spec_steps=steps, - topk=sa.speculative_eagle_topk, + topk=spec.speculative_eagle_topk, draft_token_num=num_draft_tokens, capture_hidden_mode=None, seq_lens_sum=batch.seq_lens_sum, diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index ad38d88183b3..124a252b34a9 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -57,6 +57,7 @@ get_device, get_exec, get_model, + get_parallel, get_schedule, get_serving, get_spec, @@ -400,7 +401,7 @@ def __init__( elif ( envs.SGLANG_ENABLE_PP_SPEC.get() and is_draft_worker - and server_args.pp_size > 1 + and get_parallel().pp_size > 1 ): # PP+spec: the draft worker exists only on the last PP stage, so a # world-group broadcast here would deadlock (first-stage ranks never diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 6b6e32239947..42506813b4e9 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -138,35 +138,30 @@ _EMBED_TENSOR_NAMES = ("model.embed_tokens.weight", "embed.weight") -def _load_checkpoint_tensor(model_path: str, tensor_names: tuple) -> torch.Tensor: - """Load one tensor from a local safetensors checkpoint (PP+spec path).""" - import glob - import json - import os - - from safetensors import safe_open - - # The standard loader may have resolved a HF hub id into its own cache; - # this simplified reader only handles local checkpoint directories. - assert os.path.isdir(model_path), ( - "PP+spec draft embedding loading requires --model-path to be a local " - f"checkpoint directory, got {model_path!r}" +def _load_checkpoint_tensor( + model_path: str, revision, tensor_names: tuple +) -> torch.Tensor: + """Load one tensor from a checkpoint via the standard weight loader.""" + from sglang.srt.configs.load_config import LoadConfig + from sglang.srt.model_loader.loader import DefaultModelLoader + from sglang.srt.model_loader.weight_utils import ( + pt_weights_iterator, + safetensors_weights_iterator, ) - index_path = os.path.join(model_path, "model.safetensors.index.json") - if os.path.exists(index_path): - with open(index_path) as f: - weight_map = json.load(f)["weight_map"] - for tensor_name in tensor_names: - if tensor_name in weight_map: - shard_path = os.path.join(model_path, weight_map[tensor_name]) - with safe_open(shard_path, framework="pt", device="cpu") as f: - return f.get_tensor(tensor_name) - else: - for shard_path in sorted(glob.glob(os.path.join(model_path, "*.safetensors"))): - with safe_open(shard_path, framework="pt", device="cpu") as f: - for tensor_name in tensor_names: - if tensor_name in f.keys(): - return f.get_tensor(tensor_name) + + # Resolves hub ids into the local cache and picks the weight format, the + # same way the model itself was loaded. + _, weight_files, use_safetensors = DefaultModelLoader( + LoadConfig() + )._prepare_weights(model_path, revision, fall_back_to_pt=True) + iterator = ( + safetensors_weights_iterator(weight_files) + if use_safetensors + else pt_weights_iterator(weight_files) + ) + for name, tensor in iterator: + if name in tensor_names: + return tensor raise ValueError(f"none of {tensor_names} found in checkpoint at {model_path}") @@ -325,7 +320,7 @@ def init_token_map(self): def init_lm_head(self): from sglang.srt.lora.layers import unwrap_lora_layer - if envs.SGLANG_ENABLE_PP_SPEC.get() and self.server_args.pp_size > 1: + if envs.SGLANG_ENABLE_PP_SPEC.get() and get_parallel().pp_size > 1: # This branch skips the hot-token-map / EAGLE3 head wiring below. assert self.hot_token_id is None and not ( self.speculative_algorithm.is_eagle3() @@ -337,9 +332,11 @@ def init_lm_head(self): # — otherwise it stays randomly initialized and accept_length # collapses to ~1. embed = self.draft_runner.model.model.embed_tokens.weight - if self.server_args.load_format != "dummy": + if get_model().load_format != "dummy": + target_config = self.target_worker.model_runner.model_config loaded_embed = _load_checkpoint_tensor( - model_path=self.draft_runner.model_config.model_path, + model_path=target_config.model_path, + revision=target_config.revision, tensor_names=_EMBED_TENSOR_NAMES, ) embed.weight_loader(embed, loaded_embed) @@ -1345,7 +1342,7 @@ def forward_batch_generation( self.draft_worker._draft_extend_for_decode(batch, batch_output) if ( - self.server_args.pp_size > 1 + get_parallel().pp_size > 1 and not batch.forward_mode.is_idle() and self.speculative_num_steps > 0 ): diff --git a/test/registered/pp/test_pp_spec.py b/test/registered/pp/test_pp_spec.py index 20f5dc39f8fa..264a6504b702 100644 --- a/test/registered/pp/test_pp_spec.py +++ b/test/registered/pp/test_pp_spec.py @@ -143,7 +143,11 @@ def _server_args(self, **overrides): speculative_num_draft_tokens=3, ) args.update(overrides) - return ServerArgs(**args) + server_args = ServerArgs(**args) + # check_server_args reads resolution-filled fields (served_model_name, + # chunked_prefill_size); the bare constructor leaves them None. + server_args.resolve_once() + return server_args def test_gate_off_keeps_the_ban(self): os.environ.pop("SGLANG_ENABLE_PP_SPEC", None) @@ -166,6 +170,16 @@ def test_gate_on_rejects_unsupported_combinations(self): self._server_args( speculative_adaptive=True, speculative_num_steps=3 ).check_server_args() + # The relay carries an EAGLE-shaped tree; other algorithms would + # be mis-rebuilt on the non-last stages. + with self.assertRaises(AssertionError): + self._server_args( + speculative_algorithm="NGRAM", speculative_draft_model_path=None + ).check_server_args() + # PD prefill needs the RelayPayload draft fields the gated flow + # does not carry. + with self.assertRaises(AssertionError): + self._server_args(disaggregation_mode="prefill").check_server_args() finally: os.environ.pop("SGLANG_ENABLE_PP_SPEC", None) From 8d78574dfea65df6e0d74769baf108482e411600 Mon Sep 17 00:00:00 2001 From: alichen Date: Tue, 1 Sep 2026 19:27:41 -0700 Subject: [PATCH 20/28] Reuse the target runner's load config for the draft embedding read A bare LoadConfig() dropped --download-dir, ignore patterns, and the selected load format, so an offline host or a custom cache would re-download or fail. The helper now takes the target runner's own load_config, and load formats it cannot re-open (streaming / cache-transport: sharded_state, remote_instance, ipc_cache, ...) are rejected with an explicit error instead of failing downstream; dummy is already skipped by the caller. Validated on real weights on both families the helper serves: - GLM-5.2-NVFP4 TP4xPP2: GSM8K-100 0.960, accept 2.64-2.77 - DeepSeek-V4-Flash FP8 TP2xPP2 (the embed.weight name branch): GSM8K-100 0.980, accept 2.54-2.69, matching the July baseline 2.44-2.72 Co-Authored-By: Claude Opus 5 (1M context) --- .../sglang/srt/speculative/eagle_worker_v2.py | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 42506813b4e9..87a58a5fb0a4 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -139,21 +139,38 @@ def _load_checkpoint_tensor( - model_path: str, revision, tensor_names: tuple + model_path: str, revision, tensor_names: tuple, load_config ) -> torch.Tensor: """Load one tensor from a checkpoint via the standard weight loader.""" - from sglang.srt.configs.load_config import LoadConfig + from sglang.srt.configs.load_config import LoadFormat from sglang.srt.model_loader.loader import DefaultModelLoader from sglang.srt.model_loader.weight_utils import ( pt_weights_iterator, safetensors_weights_iterator, ) - # Resolves hub ids into the local cache and picks the weight format, the - # same way the model itself was loaded. - _, weight_files, use_safetensors = DefaultModelLoader( - LoadConfig() - )._prepare_weights(model_path, revision, fall_back_to_pt=True) + # Streaming and cache-transport formats have no weight files this helper + # could reopen; the dummy format is already skipped by the caller. + reopenable = ( + LoadFormat.AUTO, + LoadFormat.SAFETENSORS, + LoadFormat.FASTSAFETENSORS, + LoadFormat.MISTRAL, + LoadFormat.PT, + LoadFormat.NPCACHE, + ) + if load_config.load_format not in reopenable: + raise ValueError( + "PP+spec draft embedding loading cannot re-open weights under " + f"load format {load_config.load_format!r}; use a disk-backed " + "load format or disable SGLANG_ENABLE_PP_SPEC" + ) + # The target's own load config keeps --download-dir, ignore patterns and + # the selected format, so hub ids resolve into the same cache the model + # was loaded from instead of a fresh default-location download. + _, weight_files, use_safetensors = DefaultModelLoader(load_config)._prepare_weights( + model_path, revision, fall_back_to_pt=True + ) iterator = ( safetensors_weights_iterator(weight_files) if use_safetensors @@ -333,11 +350,12 @@ def init_lm_head(self): # collapses to ~1. embed = self.draft_runner.model.model.embed_tokens.weight if get_model().load_format != "dummy": - target_config = self.target_worker.model_runner.model_config + target_runner = self.target_worker.model_runner loaded_embed = _load_checkpoint_tensor( - model_path=target_config.model_path, - revision=target_config.revision, + model_path=target_runner.model_config.model_path, + revision=target_runner.model_config.revision, tensor_names=_EMBED_TENSOR_NAMES, + load_config=target_runner.load_config, ) embed.weight_loader(embed, loaded_embed) head = self.target_worker.model_runner.model.lm_head.weight From 506d64773f1554d64dd40413b478e2c5a25f0aa4 Mon Sep 17 00:00:00 2001 From: YAMY1234 <74099316+YAMY1234@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:07:53 -0700 Subject: [PATCH 21/28] style: format PP speculative decoding changes --- .../sglang/srt/arg_groups/validation_hook.py | 22 +++++++++---------- .../sglang/srt/model_executor/model_runner.py | 6 ++--- .../sglang/srt/speculative/pp_spec_relay.py | 3 +-- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/python/sglang/srt/arg_groups/validation_hook.py b/python/sglang/srt/arg_groups/validation_hook.py index 5ae0b18d38ec..cb813c2ee5f2 100644 --- a/python/sglang/srt/arg_groups/validation_hook.py +++ b/python/sglang/srt/arg_groups/validation_hook.py @@ -51,13 +51,13 @@ def check_server_args(server_args: Any): if cfg.pp_size > 1: if not envs.SGLANG_ENABLE_PP_SPEC.get(): - assert ( - cfg.disable_overlap_schedule and cfg.speculative_algorithm is None - ), "Pipeline parallelism is not compatible with overlap schedule, speculative decoding" + assert cfg.disable_overlap_schedule and cfg.speculative_algorithm is None, ( + "Pipeline parallelism is not compatible with overlap schedule, speculative decoding" + ) else: - assert ( - cfg.disable_overlap_schedule - ), "SGLANG_ENABLE_PP_SPEC requires --disable-overlap-schedule" + assert cfg.disable_overlap_schedule, ( + "SGLANG_ENABLE_PP_SPEC requires --disable-overlap-schedule" + ) # The relay carries an EAGLE-shaped tree and only EAGLEWorkerV2 # tail-drafts; every other algorithm would be mis-rebuilt. assert ( @@ -71,19 +71,19 @@ def check_server_args(server_args: Any): # RelayPayload; the gated flow replaces that relay with its own # and does not carry those fields. assert cfg.disaggregation_mode == "null", ( - "SGLANG_ENABLE_PP_SPEC is not compatible with " "--disaggregation-mode" + "SGLANG_ENABLE_PP_SPEC is not compatible with --disaggregation-mode" ) # The PP relay slices spec results with the configured # num_draft_tokens; adaptive spec changes it at runtime. assert not cfg.speculative_adaptive, ( - "SGLANG_ENABLE_PP_SPEC is not compatible with " "--speculative-adaptive" + "SGLANG_ENABLE_PP_SPEC is not compatible with --speculative-adaptive" ) # Every stage rebuilds the same verify input from the relayed # per-request state, so all stages must see the same batch. # DP attention partitions it per DP rank. - assert ( - not cfg.enable_dp_attention - ), "SGLANG_ENABLE_PP_SPEC is not compatible with --enable-dp-attention" + assert not cfg.enable_dp_attention, ( + "SGLANG_ENABLE_PP_SPEC is not compatible with --enable-dp-attention" + ) assert cfg.min_free_slots_delay is None, ( "--min-free-slots-delay is not supported with pipeline " "parallelism: allocatable slots per microbatch are bounded by " diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 93157750cd30..b72432904386 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -481,9 +481,9 @@ def __init__( if self.ps.pp_size > 1: if not (envs.SGLANG_ENABLE_PP_SPEC.get() and self.is_draft_worker): - assert ( - self.support_pp - ), "Pipeline Parallel is not compatible with this model." + assert self.support_pp, ( + "Pipeline Parallel is not compatible with this model." + ) # For weight updates self.init_weight_updater() diff --git a/python/sglang/srt/speculative/pp_spec_relay.py b/python/sglang/srt/speculative/pp_spec_relay.py index ffc7fea9a970..37d1c0465b2c 100644 --- a/python/sglang/srt/speculative/pp_spec_relay.py +++ b/python/sglang/srt/speculative/pp_spec_relay.py @@ -49,8 +49,7 @@ def __init__( def __repr__(self) -> str: return ( - f"PPSpecRelayInput(bs={len(self.rids)}, " - f"drafted={self.parents is not None})" + f"PPSpecRelayInput(bs={len(self.rids)}, drafted={self.parents is not None})" ) @classmethod From ac3eae06d5f6a431e0515c12c6270932254bbe24 Mon Sep 17 00:00:00 2001 From: alichen Date: Thu, 3 Sep 2026 01:41:46 -0700 Subject: [PATCH 22/28] Find the draft input embedding by type instead of attribute path The embedding read hardcoded model.model.embed_tokens, which misses draft families that hang the embedding elsewhere (Bailing MTP word_embeddings, Mistral consolidated tok_embeddings). Scan for the one VocabParallelEmbedding that is not the ParallelLMHead, extend the checkpoint-name table with both spellings (verified against the published Ling-2.0 and Mistral-Large-3 indexes), and pin the scan on real Bailing/Mistral draft classes in a unit test. Validated on 8x RTX PRO 6000: GLM-5.2 TP4xPP2+MTP GSM8K-100 0.950 / accept 2.78-2.84; DeepSeek-V4-Flash TP2xPP2+MTP 0.960 / accept 2.55-2.70. Co-Authored-By: Claude Fable 5 --- .../sglang/srt/speculative/eagle_worker_v2.py | 38 +++++- test/registered/pp/test_pp_spec_embed_scan.py | 120 ++++++++++++++++++ 2 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 test/registered/pp/test_pp_spec_embed_scan.py diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 87a58a5fb0a4..7b0119fa80f8 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -133,9 +133,39 @@ logger = logging.getLogger(__name__) -# Embedding weight names across checkpoint layouts (GLM/DeepSeek-V3 style -# vs DeepSeek-V4 style). -_EMBED_TENSOR_NAMES = ("model.embed_tokens.weight", "embed.weight") +# Checkpoint spellings of the input embedding across the model families the +# PP+spec gate admits (GLM/DeepSeek NextN, Bailing MTP, Mistral-style drafts). +_EMBED_TENSOR_NAMES = ( + "model.embed_tokens.weight", + "embed.weight", + "model.word_embeddings.weight", + "tok_embeddings.weight", +) + + +def _find_draft_input_embedding(model) -> "torch.nn.Module": + """The draft's input embedding, found by type rather than attribute path. + + Draft models hang it under different names (embed_tokens, word_embeddings, + embed, tok_embeddings), but it is always the one VocabParallelEmbedding + that is not the ParallelLMHead.""" + from sglang.srt.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, + ) + + found = [ + (name, module) + for name, module in model.named_modules() + if isinstance(module, VocabParallelEmbedding) + and not isinstance(module, ParallelLMHead) + ] + if len(found) != 1: + raise ValueError( + "PP+spec needs exactly one input embedding on the draft model, " + f"found {[name for name, _ in found]!r}" + ) + return found[0][1] def _load_checkpoint_tensor( @@ -348,7 +378,7 @@ def init_lm_head(self): # draft's embedding must be loaded from the checkpoint directly # — otherwise it stays randomly initialized and accept_length # collapses to ~1. - embed = self.draft_runner.model.model.embed_tokens.weight + embed = _find_draft_input_embedding(self.draft_runner.model).weight if get_model().load_format != "dummy": target_runner = self.target_worker.model_runner loaded_embed = _load_checkpoint_tensor( diff --git a/test/registered/pp/test_pp_spec_embed_scan.py b/test/registered/pp/test_pp_spec_embed_scan.py new file mode 100644 index 000000000000..5c8cd9c18bc9 --- /dev/null +++ b/test/registered/pp/test_pp_spec_embed_scan.py @@ -0,0 +1,120 @@ +""" +Usage: +python3 -m unittest test_pp_spec_embed_scan.TestDraftEmbedScan +""" + +import os +import unittest + +import torch +from transformers import MistralConfig, PretrainedConfig + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase + +register_cuda_ci(est_time=60, stage="base-b", runner_config="1-gpu-small") + +# Shrunk from inclusionAI/Ling-mini-2.0; the field names are the real +# checkpoint's, only the sizes are cut down for test speed. +_BAILING_CONFIG = { + "architectures": ["BailingMoeV2ForCausalLM"], + "model_type": "bailing_moe", + "num_hidden_layers": 1, + "num_nextn_predict_layers": 1, + "hidden_size": 256, + "intermediate_size": 512, + "moe_intermediate_size": 128, + "moe_shared_expert_intermediate_size": 128, + "num_shared_experts": 1, + "num_experts": 16, + "num_experts_per_tok": 4, + "n_group": 4, + "topk_group": 2, + "norm_topk_prob": True, + "moe_router_enable_expert_bias": True, + "routed_scaling_factor": 2.5, + "score_function": "sigmoid", + "router_dtype": "fp32", + "first_k_dense_replace": 0, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "head_dim": 64, + "partial_rotary_factor": 0.5, + "use_qk_norm": True, + "use_qkv_bias": False, + "use_bias": False, + "hidden_act": "silu", + "rms_norm_eps": 1e-6, + "max_position_embeddings": 4096, + "rope_parameters": {"rope_theta": 600000, "rope_type": "default"}, + "vocab_size": 1024, + "tie_word_embeddings": False, +} + + +class TestDraftEmbedScan(CustomTestCase): + """PP+spec loads the draft input embedding via a type scan plus a + checkpoint-name table; a draft family whose embedding hangs under a new + attribute name (or grows a second VocabParallelEmbedding) would misload + silently. Pin the scan on the families the runtime GLM/DSv4 tests miss. + """ + + @classmethod + def setUpClass(cls): + from sglang.srt.distributed.parallel_state import ( + init_distributed_environment, + initialize_model_parallel, + ) + from sglang.srt.runtime_context import get_context + from sglang.srt.server_args import ServerArgs + + server_args = ServerArgs(model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + server_args.resolve_once() + get_context().set_server_args(server_args) + + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", "29611") + init_distributed_environment( + world_size=1, rank=0, local_rank=0, distributed_init_method="env://" + ) + initialize_model_parallel(tensor_model_parallel_size=1) + torch.set_default_dtype(torch.bfloat16) + torch.cuda.set_device(0) + + def test_bailing_nextn_embedding_is_found(self): + from sglang.srt.models.bailing_moe_nextn import BailingMoeForCausalLMNextN + from sglang.srt.speculative.eagle_worker_v2 import _find_draft_input_embedding + + config = PretrainedConfig.from_dict(dict(_BAILING_CONFIG)) + with torch.device("cuda"): + model = BailingMoeForCausalLMNextN(config) + self.assertIs(_find_draft_input_embedding(model), model.model.word_embeddings) + + def test_mistral_eagle_embedding_is_found(self): + from sglang.srt.models.mistral_eagle import MistralForCausalLMEagle + from sglang.srt.speculative.eagle_worker_v2 import _find_draft_input_embedding + + config = MistralConfig( + vocab_size=1024, + hidden_size=256, + intermediate_size=512, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=2, + ) + with torch.device("cuda"): + model = MistralForCausalLMEagle(config) + self.assertIs(_find_draft_input_embedding(model), model.model.embed_tokens) + + def test_name_table_matches_published_checkpoints(self): + from sglang.srt.speculative.eagle_worker_v2 import _EMBED_TENSOR_NAMES + + # External-source literals: the embedding tensor's spelling in each + # family's published target checkpoint (inclusionAI/Ling-*-2.0 + # model.safetensors.index.json; Mistral-Large-3 consolidated index). + self.assertIn("model.word_embeddings.weight", _EMBED_TENSOR_NAMES) + self.assertIn("tok_embeddings.weight", _EMBED_TENSOR_NAMES) + + +if __name__ == "__main__": + unittest.main() From 56a9767307039d3ff0666bf3cc2c542c51c22a20 Mon Sep 17 00:00:00 2001 From: YAMY1234 <74099316+YAMY1234@users.noreply.github.com> Date: Tue, 8 Sep 2026 08:57:14 -0700 Subject: [PATCH 23/28] test: move PP speculative tests under e2e taxonomy --- test/registered/{ => e2e}/pp/test_pp_spec.py | 0 test/registered/{ => e2e}/pp/test_pp_spec_embed_scan.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename test/registered/{ => e2e}/pp/test_pp_spec.py (100%) rename test/registered/{ => e2e}/pp/test_pp_spec_embed_scan.py (100%) diff --git a/test/registered/pp/test_pp_spec.py b/test/registered/e2e/pp/test_pp_spec.py similarity index 100% rename from test/registered/pp/test_pp_spec.py rename to test/registered/e2e/pp/test_pp_spec.py diff --git a/test/registered/pp/test_pp_spec_embed_scan.py b/test/registered/e2e/pp/test_pp_spec_embed_scan.py similarity index 100% rename from test/registered/pp/test_pp_spec_embed_scan.py rename to test/registered/e2e/pp/test_pp_spec_embed_scan.py From 7d8cf63c1fa13b586f5c87fe25f589c8ab12c321 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Mon, 14 Sep 2026 16:58:16 -0700 Subject: [PATCH 24/28] test: use Scheduler defaults in sampling-mask PP round trip --- .../unit/managers/test_generation_auxiliary_output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/registered/unit/managers/test_generation_auxiliary_output.py b/test/registered/unit/managers/test_generation_auxiliary_output.py index 21e6d449f25c..ff3fdb0449dc 100644 --- a/test/registered/unit/managers/test_generation_auxiliary_output.py +++ b/test/registered/unit/managers/test_generation_auxiliary_output.py @@ -159,7 +159,7 @@ def test_pipeline_sampling_mask_round_trip_without_logprobs(): next_token_ids=torch.tensor([3]), ) payload = Scheduler._pp_prepare_tensor_dict( - SimpleNamespace(), result, SimpleNamespace(return_logprob=False) + object.__new__(Scheduler), result, SimpleNamespace(return_logprob=False) ) output, _, _ = get_logprob_from_pp_outputs(PPProxyTensors(payload)) for name in ("token_ids", "lengths", "selected_logprobs", "statuses"): From f5c1ad0a5d9bbd799e65f8f81f6ec8910710d815 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Tue, 15 Sep 2026 11:36:54 -0700 Subject: [PATCH 25/28] Fix PP speculative output exchange ordering before tree planning --- python/sglang/srt/managers/scheduler_pp_mixin.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index b9c80c59bfdb..4be4791c77b5 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -126,7 +126,17 @@ def event_loop_pp(self: Scheduler): next_pp_outputs = None next_batch_result = None d2h_event = None - if get_parallel().pp_async_batch_depth > 0: + # With zero async depth, non-last speculative ranks must + # exchange the previous outputs before launching the next batch. + # Tree planning synchronizes CUDA on the host; sending alone + # leaves the peer's return send unmatched and can block that + # synchronization while the peer waits for our next proxy. + # The last rank must launch first to produce its output. + exchange_outputs_before_forward = ( + get_parallel().pp_async_batch_depth > 0 + or (self._pp_spec_relay and not self.pp_group.is_last_rank) + ) + if exchange_outputs_before_forward: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, @@ -142,7 +152,7 @@ def event_loop_pp(self: Scheduler): self.mb_metadata, self.last_rank_comm_queue, ) - if get_parallel().pp_async_batch_depth == 0: + if not exchange_outputs_before_forward: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, From 9141098c94fbf22d6a8749b7a823a37fd00cbcdf Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Tue, 15 Sep 2026 15:43:20 -0700 Subject: [PATCH 26/28] Fix PP speculative fallback topology row strides for trees --- .../sglang/srt/managers/scheduler_pp_mixin.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 20aabc495960..0ca3485c27d4 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -1157,16 +1157,24 @@ def _pp_spec_set_relay(self: Scheduler, batch: ScheduleBatch, relayed) -> None: def _pp_spec_chain_topology( self: Scheduler, bs: int, device: str ) -> Tuple[torch.Tensor, torch.Tensor]: - """topk=1 chain constants, the shape a not-yet-drafted row implies. - Mirrors _rebuild_topk1_chain_buffers: a single-step chain has no parent - entries, and both widths key off num_steps.""" - num_steps = get_spec().speculative_num_steps - parent_width = num_steps if num_steps > 1 else 0 + """Build valid fallback topology using the tree kernel's row strides. + + A not-yet-drafted row still uses the configured topk and token count. + Consecutive selected indices and earlier parents give an acyclic tree; + topk=1 reduces to a chain. Single-step drafts need no parent entries. + """ + spec = get_spec() + num_steps = spec.speculative_num_steps + parent_width = ( + spec.speculative_eagle_topk * (num_steps - 1) + 1 + if num_steps > 1 + else 0 + ) parent_list = torch.arange( -1, parent_width - 1, dtype=torch.long, device=device ).repeat(bs, 1) top_scores_index = torch.arange( - num_steps, dtype=torch.long, device=device + spec.speculative_num_draft_tokens - 1, dtype=torch.long, device=device ).repeat(bs, 1) return parent_list, top_scores_index From 2bd1200da2c787f13f28315a1e6b799616b38307 Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Tue, 15 Sep 2026 15:47:07 -0700 Subject: [PATCH 27/28] Format PP speculative topology dimensions --- python/sglang/srt/managers/scheduler_pp_mixin.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 0ca3485c27d4..f565f2e90236 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -1166,9 +1166,7 @@ def _pp_spec_chain_topology( spec = get_spec() num_steps = spec.speculative_num_steps parent_width = ( - spec.speculative_eagle_topk * (num_steps - 1) + 1 - if num_steps > 1 - else 0 + spec.speculative_eagle_topk * (num_steps - 1) + 1 if num_steps > 1 else 0 ) parent_list = torch.arange( -1, parent_width - 1, dtype=torch.long, device=device From 75c07c828ab1bc94998744381791c988dd256aa4 Mon Sep 17 00:00:00 2001 From: alichen Date: Wed, 16 Sep 2026 03:26:16 -0700 Subject: [PATCH 28/28] Compact the accepted tree path into every PP stage's KV, not just the last A verify forward writes one KV slot per tree node in node order, while every later read assumes the accepted path sits contiguously at the front of each request block. The last stage reconciles the two in _finalize_accept_tree_path; the stages that only ran the target forward advanced seq_lens and skipped it, so their committed prefix kept rejected siblings' KV. A chain is immune because its accepted path is already that prefix and the compaction is an identity, which is why only the tree shape failed. The last stage now puts accept_index on its result and relays it when speculative_eagle_topk > 1; earlier stages apply the same move before seq_lens advances. Two inputs do not survive the round on their own. ScheduleBatch.copy() carries seq_lens_cpu but not seq_lens, so the destination base comes from that snapshot -- which is also already in the forward's row order, where reindexing the live batch would skip precisely the rounds whose composition changed. And eagle_prepare_for_verify writes out_cache_loc inside _forward_isolation, which restores it, so the verify slots travel on the result instead. Measured on 4x H100 with the CI model pair, GSM8K 256, steps 2 / topk 2 / 4 draft tokens: PP2 goes from 0.0195 to 0.2344 against a 0.2305 no-PP baseline, acceptance unchanged at 2.40. The chain shape is unaffected: 0.2266 no-PP vs 0.2344 PP2. Co-Authored-By: Claude Fable 5 --- python/sglang/srt/managers/scheduler.py | 7 ++ .../sglang/srt/managers/scheduler_pp_mixin.py | 68 +++++++++++++++++++ python/sglang/srt/managers/utils.py | 11 +++ .../srt/speculative/eagle_worker_common.py | 1 + 4 files changed, 87 insertions(+) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 2356c6fa0d28..68743f0acbcb 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -4448,6 +4448,13 @@ def run_batch( is_verify=True, ) batch_result.can_run_cuda_graph = can_run_cuda_graph + # The isolation above restores batch.out_cache_loc, but + # this stage still has to compact its own accepted-path + # KV once the last stage relays which nodes it kept, so + # the verify slots have to outlive the forward. + batch_result.spec_verify_out_cache_loc = ( + verify_forward_batch.out_cache_loc + ) else: batch_result = self.model_worker.forward_batch_generation( batch, pp_proxy_tensors=pp_proxy_tensors diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index f565f2e90236..138491e65152 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -61,6 +61,7 @@ class PPBatchMetadata: # result arrives, so relayed tensors must be applied against the # composition that actually ran the forward. fwd_batch: Optional[ScheduleBatch] = None + verify_out_cache_loc: Optional[torch.Tensor] = None class SchedulerPPMixin: @@ -811,6 +812,13 @@ def _pp_prepare_tensor_dict( tensor_dict["spec_accept_lens"] = result.accept_lens tensor_dict["spec_new_seq_lens"] = result.new_seq_lens tensor_dict["spec_bonus_tokens"] = result.next_draft_input.bonus_tokens + if ( + result.accept_index is not None + and get_spec().speculative_eagle_topk > 1 + ): + # Only a tree needs it: a chain's accepted path is already the + # front of each block, so compacting it is an identity. + tensor_dict["spec_accept_index"] = result.accept_index if result.next_verify_chain is not None: # Tail-drafted tree for the next verify round (root = bonus), # with the topology its tokens were arranged by. @@ -989,6 +997,14 @@ def _pp_prep_batch_result( new_seq_lens = pp_outputs["spec_new_seq_lens"] fwd_rids = [req.rid for req in fwd_batch.reqs] live_rids = [req.rid for req in batch.reqs] + self._pp_spec_compact_accept_kv( + batch, + fwd_batch, + fwd_rids, + live_rids, + mb_metadata.verify_out_cache_loc, + pp_outputs, + ) if live_rids == fwd_rids: batch.seq_lens = new_seq_lens if batch.seq_lens_cpu is not None: @@ -1106,6 +1122,57 @@ def _pp_process_batch_result( ): self.process_batch_result(batch, output_result) + def _pp_spec_compact_accept_kv( + self: Scheduler, + batch: ScheduleBatch, + fwd_batch: ScheduleBatch, + fwd_rids: List[str], + live_rids: List[str], + verify_out_cache_loc: Optional[torch.Tensor], + pp_outputs, + ) -> None: + """Move this stage's accepted-path KV to the front of each request block. + + The verify forward writes one KV slot per tree node, in node order. The + committed prefix that every later read assumes is the accepted path laid + out contiguously, so the two have to be reconciled once per round -- and + each stage has to do it for its own layers, since KV is not relayed. + The last stage does it inside verify (_finalize_accept_tree_path); this + is the same step for the stages that only ran the target forward. + + Must run before seq_lens advances: the move writes into the block that + starts at the pre-advance length. + """ + accept_index = pp_outputs.tensors.get("spec_accept_index") + if accept_index is None or fwd_batch.forward_mode.is_idle(): + return + if verify_out_cache_loc is None: + return + from sglang.srt.speculative.spec_utils import ( + move_accept_tokens_to_target_kvcache, + ) + + # The destination base is the length each request had when the forward + # ran. ScheduleBatch.copy() drops seq_lens but keeps seq_lens_cpu, and + # that snapshot is already in the forward's row order -- the live batch + # may have been filtered or merged since, and reindexing it would skip + # exactly the rounds whose composition changed. + device = verify_out_cache_loc.device + if fwd_batch.seq_lens_cpu is not None: + seq_lens = fwd_batch.seq_lens_cpu.to(device=device, dtype=torch.int64) + elif live_rids == fwd_rids: + seq_lens = batch.seq_lens + else: + return + fwd_batch.seq_lens = seq_lens + fwd_batch.out_cache_loc = verify_out_cache_loc + move_accept_tokens_to_target_kvcache( + fwd_batch, + accept_index.to(device), + pp_outputs["spec_accept_lens"].to(device) - 1, + self.token_to_kv_pool_allocator, + ) + def _pp_spec_adopt_relayed_tree( self: Scheduler, batch: ScheduleBatch, @@ -1563,6 +1630,7 @@ def _pp_launch_batch( if not cur_batch.spec_algorithm.is_none() else None ), + verify_out_cache_loc=result.spec_verify_out_cache_loc, ) event = self.device_module.Event() event.record(self.device_module.current_stream()) diff --git a/python/sglang/srt/managers/utils.py b/python/sglang/srt/managers/utils.py index 92dad4210808..f739fb99822c 100644 --- a/python/sglang/srt/managers/utils.py +++ b/python/sglang/srt/managers/utils.py @@ -110,6 +110,17 @@ class GenerationBatchResult: next_verify_parent_list: Optional[torch.Tensor] = None next_verify_top_scores_index: Optional[torch.Tensor] = None + # PP+spec: the verify forward's KV slots on a non-last stage. That stage + # prepares verify inside forward isolation, which restores + # batch.out_cache_loc, so the slots have to travel on the result to survive + # until the accepted path comes back over the relay. + spec_verify_out_cache_loc: Optional[torch.Tensor] = None + + # PP+spec: [bs, spec_steps + 1] global node indices of the accepted path. + # Every stage holds the KV for its own layers, so every stage has to compact + # that path into its committed prefix; only the last stage can compute it. + accept_index: Optional[torch.Tensor] = None + # Refs the worker wants scheduler to keep alive for the same 2-iter window # as batch_record_buf. Used for cross-stream tensor lifetime (e.g. a spec # V2 verify ForwardBatch whose tensors must outlive mid-iter SB rebinds). diff --git a/python/sglang/srt/speculative/eagle_worker_common.py b/python/sglang/srt/speculative/eagle_worker_common.py index c3ce9886f6cf..0c2c53086020 100644 --- a/python/sglang/srt/speculative/eagle_worker_common.py +++ b/python/sglang/srt/speculative/eagle_worker_common.py @@ -667,6 +667,7 @@ def run_eagle_verify( speculative_num_draft_tokens=num_draft_tokens, next_draft_input=next_draft_input, accept_lens=accept_lens, + accept_index=accept_index, new_seq_lens=new_seq_lens, routed_experts_output=forward_batch_output.routed_experts_output, indexer_topk_output=forward_batch_output.indexer_topk_output,