Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions python/sglang/srt/disaggregation/decode.py
Original file line number Diff line number Diff line change
Expand Up @@ -1583,15 +1583,18 @@ def pop_preallocated(
failed_reqs.append(decode_req)
indices_to_remove.add(i)
continue
if self.scheduler.enable_decode_hicache:
# Prefill is promised only the L3 span the prefetch took.
self._start_hicache_prefetch(decode_req.req, prefix_match)
if prefix_match is not None:
total_prefix_len = prefix_match.decode_prefix_len
dst_kv_indices = self._pre_alloc(
decode_req.req,
prefix_indices,
prefix_len,
total_prefix_len,
)
decode_req.prefix_match = prefix_match
if self.scheduler.enable_decode_hicache:
self._start_hicache_prefetch(decode_req.req, prefix_match)
hisparse_req_budget -= 1
# Recompute from actual pool state for the next queue entry.
# This accounts for page rounding and newly locked evictable cache.
Expand Down
17 changes: 11 additions & 6 deletions python/sglang/srt/disaggregation/decode_hicache_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,8 @@ def _start_hicache_prefetch(
) -> None:
"""Issue L3 storage prefetch after admission succeeds.

On failure, degrades to L2-only restore by clearing l3 fields.
Clears the L3 span unless the prefetch was taken, so prefill is never
promised tokens nothing will load.
"""
if (
prefix_match is None
Expand All @@ -122,27 +123,31 @@ def _start_hicache_prefetch(
return
try:
matched_len = prefix_match.l1_prefix_len + prefix_match.l2_host_hit_length
suffix = req.origin_input_ids[
matched_len : matched_len + prefix_match.l3_storage_hit_length
]
last_hash = self.tree_cache.get_last_hash_value(prefix_match.last_host_node)
prefix_keys = (
self.tree_cache.get_prefix_hash_values(prefix_match.last_host_node)
if self.tree_cache.hicache_storage_pass_prefix_keys
else None
)
self.tree_cache.prefetch_from_storage(
# Same span as the prefill scheduler: the queried hit end, plus the
# bigram boundary token, with no second storage query.
ticketed = self.tree_cache.prefetch_from_storage(
req.cache_request_handle,
prefix_match.last_host_node,
suffix,
req.origin_input_ids[matched_len:],
last_hash,
prefix_keys,
matched_prefix_tokens=req.origin_input_ids[:matched_len],
extra_key=req.extra_key,
cache_salt=req.cache_salt,
storage_hit_end=matched_len + prefix_match.l3_storage_hit_length,
)
prefix_match.prefetch_registered = self.tree_cache.has_ongoing_prefetch(
req.cache_request_handle
)
if not (prefix_match.prefetch_registered or ticketed):
# Declined (below prefetch_threshold or rate limited): nothing loads.
prefix_match.l3_storage_hit_length = 0
except Exception as e:
logger.warning(
"HiCache L3 prefetch failed for rid=%s: %s; falling back to L2-only LoadingBack",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import torch

from sglang.srt.disaggregation.decode_hicache_mixin import DecodeHiCachePreallocMixin
from sglang.srt.disaggregation.kv_events import (
BlockRemoved,
BlockStored,
Expand All @@ -23,6 +24,7 @@
InsertParams,
MatchPrefixParams,
)
from sglang.srt.mem_cache.common import match_kv_cache
from sglang.srt.mem_cache.hicache_storage import (
PoolHitPolicy,
PoolName,
Expand Down Expand Up @@ -1352,6 +1354,59 @@ def _avail():

cons.sanity_check()

@requires_hicache
@requires(
lambda cfg: cfg.components == (ComponentType.FULL,) and cfg.page_size == 4,
"one FULL page_size=4 fixture covers the decode L3 promise",
)
def test_decode_promises_only_the_prefetched_l3_span(self):
threshold = 16
storage_dir = self._make_storage_dir()
short_hit = self._make_seq(1, 2) # below prefetch_threshold
long_hit = self._make_seq(1001, 6)
prod, prod_alloc, prod_rtp = build_fixture(self.cfg)
self._init_hicache(
prod,
storage_backend="file",
storage_dir=storage_dir,
prefetch_threshold=threshold,
)
for seq in (short_hit, long_hit):
self._insert(prod, prod_alloc, prod_rtp, seq)
leaf = prod.match_prefix(
MatchPrefixParams(key=RadixKey(array("q", seq)))
).last_device_node
self._backup_node(prod, leaf)
self._write_path_to_l3(prod, leaf)
self._flush_l3_backups(prod)

cons, _, cons_rtp = build_fixture(self.cfg)
self._init_hicache(
cons,
storage_backend="file",
storage_dir=storage_dir,
prefetch_threshold=threshold,
)
decode = SimpleNamespace(
scheduler=SimpleNamespace(enable_decode_hicache=True), tree_cache=cons
)
for stored, promised in ((short_hit, 0), (long_hit, len(long_hit))):
req = self._make_req(cons_rtp)
req.origin_input_ids = array("q", stored + list(range(5000, 5016)))
result = match_kv_cache(req, cons, req.origin_input_ids)
match = DecodeHiCachePreallocMixin._build_decode_prefix_match(
decode, req, result
)
self.assertEqual(match.l3_storage_hit_length, len(stored))
DecodeHiCachePreallocMixin._start_hicache_prefetch(decode, req, match)
self.assertEqual(match.decode_prefix_len, promised)
if promised:
self._run_prefetch_to_completion(cons, req.cache_request_handle)
hit = cons.match_prefix(
MatchPrefixParams(key=RadixKey(array("q", stored)))
)
self.assertEqual(hit.host_hit_length, promised)

@requires_hicache
@requires(
lambda cfg: cfg.components == (ComponentType.FULL,) and cfg.page_size == 4,
Expand Down
Loading