Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ def _static_accept_tail_kernel(
targets = tl.load(target_ptr + row * T + offs, mask=in_draft, other=-2)
eq = (drafts == targets) & in_draft
num_correct = tl.min(tl.where(eq, BLOCK_T, offs), 0)
if HAS_SIM:
correct = tl.load(sim_ptr).to(tl.int32)
else:
correct = num_correct.to(tl.int32)
if HAS_PARTS:
# _simulated_bonus_row_select_kernel's reduction.
s = tl.arange(0, NSPLIT)
Expand All @@ -77,11 +81,7 @@ def _static_accept_tail_kernel(
elif HAS_BONUS_IN:
bonus = tl.load(bonus_in_ptr + row).to(tl.int64)
else:
bonus = tl.load(target_ptr + row * T + num_correct).to(tl.int64)
if HAS_SIM:
correct = tl.load(sim_ptr).to(tl.int32)
else:
correct = num_correct.to(tl.int32)
bonus = tl.load(target_ptr + row * T + correct).to(tl.int64)
commit = correct + 1
new_seq = tl.load(seq_lens_ptr + row) + commit
tl.store(correct_out_ptr + row, correct)
Expand Down Expand Up @@ -218,6 +218,10 @@ def static_accept_tail_torch(
matches = (candidates[:, 1:] == target[:, :gamma]).to(torch.int32)
num_correct = matches.cumprod(dim=1).sum(dim=1).to(torch.int32)
rows = torch.arange(bs, device=candidates.device)
if simulated_correct_len is not None:
correct = simulated_correct_len.repeat(bs).to(torch.int32)
else:
correct = num_correct
if bonus_parts is not None:
part_val, part_idx, vocab = bonus_parts
bonus = simulated_bonus_select_partials(
Expand All @@ -226,11 +230,7 @@ def static_accept_tail_torch(
elif bonus_in is not None:
bonus = bonus_in[:bs].to(torch.int64)
else:
bonus = target[rows, num_correct.long()].to(torch.int64)
if simulated_correct_len is not None:
correct = simulated_correct_len.repeat(bs).to(torch.int32)
else:
correct = num_correct
bonus = target[rows, correct.long()].to(torch.int64)
commit = correct + 1
new_seq = seq_lens[:bs] + commit.to(seq_lens.dtype)
correct_len_out[:bs].copy_(correct)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@
deferred context-KV commit, the verify graph already joined this step's projection, so
the region stages the stores from it.

The caller routes only all-sampled steps without top-k / top-p, logit adjustments,
grammar or a simulated sampled bonus; everything else keeps the eager accept.
The caller routes only all-sampled steps without top-k / top-p, logit adjustments or
grammar; everything else keeps the eager accept. Under SGLANG_SIMULATE_ACC_LEN the region
draws the bonus at the simulated row from the buffers the caller stages with
begin_sim_bonus.
"""

from __future__ import annotations
Expand Down Expand Up @@ -101,6 +103,10 @@ def sampling_accept_commit_lens(
block_verification=block_verification,
rejection_noise=rejection_noise,
)
bonus_logits = io.logits
if epilogue.simulated_correct_len is not None:
bonus = epilogue.sim_bonus(io.logits, bs).to(bonus.dtype).view_as(bonus)
bonus_logits = None
commit_lens = build_accept_outs(
correct_len=correct_len,
bonus=bonus,
Expand All @@ -112,7 +118,7 @@ def sampling_accept_commit_lens(
tp_sync=tp_sync,
sync_site=SpecTpSyncSite.DSPARK_ACCEPT_SAMPLE,
simulated_correct_len=epilogue.simulated_correct_len,
target_logits=io.logits,
target_logits=bonus_logits,
out=epilogue.read_accept(bs),
block_accept_lens_out=epilogue.read_block_accept_lens(bs),
).commit_lens
Expand Down
65 changes: 51 additions & 14 deletions python/sglang/srt/speculative/dspark_components/dspark_verify.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,10 @@
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode, ForwardMode
from sglang.srt.speculative.dflash_info import DFlashVerifyInput
from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2
from sglang.srt.speculative.dflash_utils import apply_dflash_verify_logits_adjustments
from sglang.srt.speculative.dflash_utils import (
apply_dflash_verify_logits_adjustments,
build_speculative_verify_target_probs,
)
from sglang.srt.speculative.dspark_components.dspark_deferred_commit import (
DeferredContextCommit,
)
Expand Down Expand Up @@ -205,9 +208,12 @@ def accept_and_finalize(
block_verification=self._block_verification,
rejection_noise=self._rejection_noise,
)
greedy_bonus = simulate_bonus_sampling_info is None and (
accept_sampling_info is None or accept_sampling_info.is_all_greedy
)
sync_site = (
SpecTpSyncSite.DSPARK_ACCEPT_GREEDY
if accept_sampling_info is None or accept_sampling_info.is_all_greedy
if greedy_bonus
else SpecTpSyncSite.DSPARK_ACCEPT_SAMPLE
)
return build_accept_outs(
Expand Down Expand Up @@ -835,7 +841,7 @@ def _static_accept_tail(
if not self.simulate_fold:
target_predict = _row_argmax(logits, fused=self._fused_argmax).view(-1)
elif self._tail_broadcasts:
bonus_in = self._sim_bonus(logits, bs)
bonus_in = self.sim_bonus(logits, bs)
# Rank 0's length and draw, as build_accept_outs broadcasts them.
self._tp_sync.sync(
SpecTpSyncSite.DSPARK_ACCEPT_GRAPH, self.simulated_correct_len
Expand Down Expand Up @@ -877,7 +883,8 @@ def _sim_bonus_args(self, logits, bs: int) -> dict:
rows_per_request=self.stride,
)

def _sim_bonus(self, logits, bs: int) -> torch.Tensor:
def sim_bonus(self, logits, bs: int) -> torch.Tensor:
"""The simulated row's bonus from the buffers begin_sim_bonus staged; graph-safe."""
return simulated_bonus_sample_graph(**self._sim_bonus_args(logits, bs))

def _log_accept_tail(self, bs: int, *, bonus_in) -> None:
Expand Down Expand Up @@ -1045,7 +1052,7 @@ def _accept(
)
if self.simulate_fold:
# Before the TP broadcast in build_accept_outs: every rank commits rank 0's draw.
bonus = self._sim_bonus(logits, bs).to(bonus.dtype).view_as(bonus)
bonus = self.sim_bonus(logits, bs).to(bonus.dtype).view_as(bonus)
accept = build_accept_outs(
correct_len=correct_len,
bonus=bonus,
Expand All @@ -1057,6 +1064,7 @@ def _accept(
tp_sync=self._tp_sync,
sync_site=SpecTpSyncSite.DSPARK_ACCEPT_GRAPH,
simulated_correct_len=self.simulated_correct_len,
target_logits=None if self.simulate_fold else logits,
out=self.read_accept(bs),
block_accept_lens_out=self.read_block_accept_lens(bs),
)
Expand Down Expand Up @@ -1132,6 +1140,16 @@ def build_accept_outs(
bs=bs,
verify_num_draft_tokens=verify_num_draft_tokens,
)
elif target_logits is not None:
# The accept's bonus is the real-length row's; take the simulated row's argmax.
bonus = (
target_logits.index_select(
0, _simulated_rows(correct_len, verify_num_draft_tokens)
)
.argmax(dim=-1)
.to(bonus.dtype)
.view_as(bonus)
)
tp_sync.sync(sync_site, correct_len)
tp_sync.sync(sync_site, bonus)
tp_sync.sync(sync_site, cap_trim_lens)
Expand Down Expand Up @@ -1180,18 +1198,37 @@ def sample_simulated_bonus(
bs: int,
verify_num_draft_tokens: int,
) -> torch.Tensor:
"""Temperature-sample every verify row and return the row at the (simulated)
correct_len as the bonus token. Temperature only: callers exclude top-p/top-k/min-p."""
bonus = simulated_bonus_sample(
target_logits=target_logits,
correct_len=correct_len,
temperatures=sampling_info.temperatures,
bs=bs,
rows_per_request=verify_num_draft_tokens,
)
"""The bonus token at the (simulated) correct_len row, drawn from the target
distribution the sampling accept uses: temperature, then top-k / top-p."""
if sampling_info.need_top_k_sampling or sampling_info.need_top_p_sampling:
probs = build_speculative_verify_target_probs(
next_token_logits=target_logits.index_select(
0, _simulated_rows(correct_len, verify_num_draft_tokens)
),
sampling_info=sampling_info,
draft_token_num=1,
bs=bs,
).view(bs, -1)
bonus = torch.multinomial(probs, num_samples=1)
else:
bonus = simulated_bonus_sample(
target_logits=target_logits,
correct_len=correct_len,
temperatures=sampling_info.temperatures,
bs=bs,
rows_per_request=verify_num_draft_tokens,
)
return bonus.to(greedy_bonus.dtype).view_as(greedy_bonus)


def _simulated_rows(correct_len: torch.Tensor, rows_per_request: int) -> torch.Tensor:
bs = correct_len.shape[0]
return (
torch.arange(bs, device=correct_len.device) * rows_per_request
+ correct_len.long()
)


_blockwise_rejection_noted = False


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1037,21 +1037,25 @@ def _forward_decode(
self._verify_inputs_ready.record()

sampling_info = batch.sampling_info
# Simulated acceptance overrides correct_len, so only the bonus token needs the
# request's temperature: draft + accept run greedy, the bonus is still sampled.
# Simulated acceptance overrides correct_len; the bonus is drawn at that row
# with the request's sampling params. SGLANG_SIMULATE_ACC_GREEDY also runs the
# draft and accept greedy.
simulate_bonus_sampling_info = None
if (
self._simulate_acc_greedy
self._simulated_correct_len is not None
and sampling_info is not None
and not sampling_info.is_all_greedy
and not batch.has_grammar
and verify_logits_adjustments_are_noop(sampling_info)
and not sampling_info.need_top_p_sampling
and not sampling_info.need_top_k_sampling
and not sampling_info.need_min_p_sampling
):
simulate_bonus_sampling_info = sampling_info
sampling_info = None
if (
self._simulate_acc_greedy
and not batch.has_grammar
and verify_logits_adjustments_are_noop(sampling_info)
and not sampling_info.need_top_p_sampling
and not sampling_info.need_top_k_sampling
and not sampling_info.need_min_p_sampling
):
sampling_info = None
if self._sim_draw_ahead:
self._begin_simulated_step_ahead(
bs=bs, simulate_bonus_sampling_info=simulate_bonus_sampling_info
Expand Down Expand Up @@ -1204,6 +1208,10 @@ def _forward_decode(
simulate_bonus_sampling_info=simulate_bonus_sampling_info,
)
):
if simulate_bonus_sampling_info is not None:
epilogue.begin_sim_bonus(
bs, simulate_bonus_sampling_info.temperatures[:bs]
)
region_accept = self._sampling_accept_graph.run(
bs=bs, target_logits=logits_output.next_token_logits
)
Expand Down Expand Up @@ -1358,7 +1366,11 @@ def _sampling_graph_eligible(
and not sampling_info.need_top_k_sampling
and not sampling_info.need_top_p_sampling
and verify_logits_adjustments_are_noop(sampling_info)
and simulate_bonus_sampling_info is None
# A simulated bonus is drawn in the region from this batch's temperatures.
and (
simulate_bonus_sampling_info is None
or simulate_bonus_sampling_info is sampling_info
)
and not batch.has_grammar
)

Expand Down
6 changes: 5 additions & 1 deletion test/registered/spec/dspark/test_dspark_simulated_bonus.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,11 @@


def _info(temps):
return types.SimpleNamespace(temperatures=torch.tensor(temps, device="cuda"))
return types.SimpleNamespace(
temperatures=torch.tensor(temps, device="cuda"),
need_top_k_sampling=False,
need_top_p_sampling=False,
)


class TestSimulatedBonus(unittest.TestCase):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ def test_routes_only_all_sampled_static_steps():
assert not _eligible(worker, _sampling_info(), folded=False)
assert not _eligible(worker, _sampling_info(), grammar=True)
assert not _eligible(worker, _sampling_info(), sim_bonus=_sampling_info())
info = _sampling_info()
assert _eligible(worker, info, sim_bonus=info)
assert not _eligible(_worker(enabled=False), _sampling_info())
assert not _eligible(_worker(mode="compact"), _sampling_info())

Expand Down
Loading