Skip to content
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions python/sglang/srt/disaggregation/kv_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,26 @@ class StorageMedium(str, enum.Enum):
EXTERNAL = "EXTERNAL" # L4: shared / remote pool (e.g. Mooncake)


class RemovalReason(str, enum.Enum):
"""Why blocks left a storage tier in a ``BlockRemoved`` event.

These are the causes the cache implementations actually distinguish at the
point of removal. The difference that matters to a consumer is whether the
content is gone or merely somewhere else: a miss after ``EVICTED`` means
capacity pressure, a miss after ``DEMOTED`` is still servable from a lower
tier. Request completion and abort are deliberately absent -- finishing a
request only drops a lock reference, and the blocks stay cached until one
of the causes below removes them.
"""

# Capacity pressure. This action kept no copy anywhere else.
EVICTED = "evicted"
# Freed here because a copy exists in a lower tier (e.g. GPU -> host).
DEMOTED = "demoted"
# A redundant copy was reclaimed; the block stays cached in a higher tier.
DUPLICATE_RECLAIMED = "duplicate_reclaimed"


class OffloadedState(msgspec.Struct):
"""Decode-side offload progress for one request, keyed by Req in the manager."""

Expand Down Expand Up @@ -293,6 +313,10 @@ class BlockStored(KVCacheEvent):
class BlockRemoved(KVCacheEvent):
block_hashes: list[int]
medium: Optional[str] = None
# Why the blocks left ``medium``, one of ``RemovalReason``. Optional, so an
# emitter that does not state one produces exactly the map an older
# consumer sees today. "Not stated" is not the same as "evicted".
reason: Optional[str] = None


class AllBlocksCleared(KVCacheEvent):
Expand Down
14 changes: 11 additions & 3 deletions python/sglang/srt/mem_cache/events.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
AllBlocksCleared,
BlockRemoved,
BlockStored,
RemovalReason,
StorageMedium,
)
from sglang.srt.mem_cache.utils import (
Expand Down Expand Up @@ -56,7 +57,7 @@ def enqueue(self, event) -> None:
tail = self._queue[-1]

if isinstance(tail, BlockRemoved) and isinstance(event, BlockRemoved):
if tail.medium == event.medium:
if tail.medium == event.medium and tail.reason == event.reason:
tail.block_hashes.extend(event.block_hashes)
return

Expand Down Expand Up @@ -149,10 +150,15 @@ def record_store(
parent_block_hash = block_hash
page_index += 1

def record_remove(self, node: Any, medium=None) -> None:
def record_remove(
self, node: Any, medium=None, *, reason: str = RemovalReason.EVICTED
) -> None:
# One BlockRemoved per radix node.
# ``medium`` defaults to StorageMedium.GPU but callers may override for
# lower-tier removals (e.g. StorageMedium.CPU when evicting from host).
# ``reason`` defaults to EVICTED because every call site that does not
# pass one is a capacity eviction; the paths that keep a copy elsewhere
# (demotion, duplicate reclaim) name themselves.
if not self.enabled:
return
if medium is None:
Expand All @@ -173,7 +179,9 @@ def record_remove(self, node: Any, medium=None) -> None:
page_index += 1

if block_hashes:
self.enqueue(BlockRemoved(block_hashes=block_hashes, medium=medium))
self.enqueue(
BlockRemoved(block_hashes=block_hashes, medium=medium, reason=reason)
)

def record_all_cleared(self) -> None:
if not self.enabled:
Expand Down
10 changes: 7 additions & 3 deletions python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import msgspec
import torch

from sglang.srt.disaggregation.kv_events import StorageMedium
from sglang.srt.disaggregation.kv_events import RemovalReason, StorageMedium
from sglang.srt.environ import envs
from sglang.srt.mem_cache.base_prefix_cache import (
DecLockRefParams,
Expand Down Expand Up @@ -1946,7 +1946,9 @@ def _release_full_host_duplicate(
"""Free only the Full host layer; aux host slices stay under their own
pools' LRU (a host-only aux slice may be a sole copy)."""
assert self._can_reclaim_full_host_duplicate(node)
self.kv_events.record_remove(node, medium=StorageMedium.CPU)
self.kv_events.record_remove(
node, medium=StorageMedium.CPU, reason=RemovalReason.DUPLICATE_RECLAIMED
)
self._evict_component_and_detach_lru(
node,
self.components_by_type[BASE_COMPONENT_TYPE],
Expand Down Expand Up @@ -2018,7 +2020,9 @@ def _demote(
self._cascade_evict(
node, trigger, tracker, device_frees=device_frees, host_frees=host_frees
)
self.kv_events.record_remove(node, medium=StorageMedium.GPU)
self.kv_events.record_remove(
node, medium=StorageMedium.GPU, reason=RemovalReason.DEMOTED
)

# after device eviction, insert aux components into host LRU.
self._for_each_component_lru(
Expand Down
90 changes: 90 additions & 0 deletions test/registered/unit/disaggregation/test_kv_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
"""

import unittest
from typing import Optional

import msgspec
import zmq
Expand All @@ -17,6 +18,7 @@
BlockRemoved,
BlockStored,
KVEventBatch,
RemovalReason,
StorageMedium,
ZmqEventPublisher,
resolve_load_pub_range,
Expand Down Expand Up @@ -253,6 +255,94 @@ def test_batch_stays_a_positional_array_of_maps(self):
self.assertEqual(len(decoded), 3)


class TestBlockRemovedReason(CustomTestCase):
"""``reason`` says whether the content is gone or merely somewhere else."""

class _LegacyBlockRemoved(
msgspec.Struct,
omit_defaults=True, # type: ignore[call-arg]
gc=False, # type: ignore[call-arg]
tag="BlockRemoved",
):
"""``BlockRemoved`` as it was before ``reason`` existed.

Stands in for a consumer built against the old schema, to check that
the added field does not break it.
"""

block_hashes: list[int]
medium: Optional[str] = None

def test_reason_is_a_named_field(self):
event = BlockRemoved(
block_hashes=[123],
medium=StorageMedium.GPU,
reason=RemovalReason.DEMOTED,
)
decoded = msgspec.msgpack.decode(msgspec.msgpack.encode(event))
self.assertEqual(decoded["type"], "BlockRemoved")
self.assertEqual(decoded["reason"], "demoted")

def test_omitting_the_reason_leaves_todays_wire_shape(self):
# The field is additive: an emitter that states nothing produces the
# exact map an old consumer already reads.
event = BlockRemoved(block_hashes=[123], medium=StorageMedium.GPU)
decoded = msgspec.msgpack.decode(msgspec.msgpack.encode(event))
self.assertEqual(set(decoded), {"type", "block_hashes", "medium"})

def test_an_old_consumer_still_decodes_an_event_carrying_a_reason(self):
# The compatibility direction that matters: new engine, old consumer.
for reason in RemovalReason:
with self.subTest(reason=reason.value):
payload = msgspec.msgpack.encode(
BlockRemoved(
block_hashes=[1, 2],
medium=StorageMedium.CPU,
reason=reason,
)
)
legacy = msgspec.msgpack.decode(payload, type=self._LegacyBlockRemoved)
self.assertEqual(legacy.block_hashes, [1, 2])
self.assertEqual(legacy.medium, "CPU_PINNED")

def test_a_new_consumer_reads_an_event_emitted_without_a_reason(self):
# The other direction: an old recording replayed into a new consumer
# must not be reported as an eviction it never claimed to be.
payload = msgspec.msgpack.encode(
self._LegacyBlockRemoved(block_hashes=[7], medium=StorageMedium.GPU)
)
decoded = msgspec.msgpack.decode(payload, type=BlockRemoved)
self.assertEqual(decoded.block_hashes, [7])
self.assertIsNone(decoded.reason)

def test_reasons_survive_a_mixed_batch(self):
batch = KVEventBatch(
ts=1.0,
events=[
BlockRemoved(block_hashes=[1], medium=StorageMedium.GPU),
BlockRemoved(
block_hashes=[2],
medium=StorageMedium.GPU,
reason=RemovalReason.EVICTED,
),
BlockRemoved(
block_hashes=[3],
medium=StorageMedium.GPU,
reason=RemovalReason.DEMOTED,
),
AllBlocksCleared(),
],
)
round_tripped = msgspec.msgpack.decode(
msgspec.msgpack.encode(batch), type=KVEventBatch
)
self.assertEqual(
[e.reason for e in round_tripped.events[:3]],
[None, "evicted", "demoted"],
)
self.assertIsInstance(round_tripped.events[3], AllBlocksCleared)


class TestReplay(CustomTestCase):
def test_descriptor_advertises_replay_port(self):
def descriptor(**cfg):
Expand Down
49 changes: 49 additions & 0 deletions test/registered/unit/mem_cache/test_radix_cache_unit.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
AllBlocksCleared,
BlockRemoved,
BlockStored,
RemovalReason,
StorageMedium,
)
from sglang.srt.managers.schedule_batch import ReqKvInfo
Expand Down Expand Up @@ -97,6 +98,40 @@ def test_enqueue_coalesces_compatible_removes(self):
self.assertIsInstance(events[0], BlockRemoved)
self.assertEqual(events[0].block_hashes, [1, 2, 3])

def test_enqueue_coalesces_removes_with_the_same_reason(self):
queue = KVCacheEventRecorder(enabled=True, page_size=DEFAULT_PAGE_SIZE)
for hashes in ([1], [2, 3]):
queue.enqueue(
BlockRemoved(
block_hashes=hashes,
medium=StorageMedium.GPU,
reason=RemovalReason.DEMOTED,
)
)

events = queue.take()
self.assertEqual(len(events), 1)
self.assertEqual(events[0].block_hashes, [1, 2, 3])
self.assertEqual(events[0].reason, RemovalReason.DEMOTED)

def test_record_remove_states_a_reason(self):
"""The default is EVICTED because every call site that passes nothing is
a capacity eviction; the paths that keep a copy elsewhere name it."""
queue = KVCacheEventRecorder(enabled=True, page_size=DEFAULT_PAGE_SIZE)
node = TreeNode()
node.key = RadixKey(token_ids=array("i", [1, 2, 3, 4]))
# Hash values are hex digests; the recorder folds them into int64.
node.hash_value = ["a1" * 32]

queue.record_remove(node)
queue.record_remove(node, reason=RemovalReason.DEMOTED)

events = queue.take()
self.assertEqual(
[e.reason for e in events],
[RemovalReason.EVICTED, RemovalReason.DEMOTED],
)

def test_enqueue_preserves_fusion_boundaries(self):
incompatible_stores = [
self._store(2, 1, medium=StorageMedium.CPU),
Expand All @@ -122,6 +157,20 @@ def test_enqueue_preserves_fusion_boundaries(self):
queue.enqueue(BlockRemoved(block_hashes=[2], medium=StorageMedium.CPU))
self.assertEqual(len(queue.take()), 2)

# Same tier, different cause. Merging these would tell a consumer that
# demoted blocks were evicted under capacity pressure.
queue = KVCacheEventRecorder(enabled=True, page_size=DEFAULT_PAGE_SIZE)
for hashes, reason in (
([1], RemovalReason.EVICTED),
([2], RemovalReason.DEMOTED),
):
queue.enqueue(
BlockRemoved(
block_hashes=hashes, medium=StorageMedium.GPU, reason=reason
)
)
self.assertEqual(len(queue.take()), 2)

queue = KVCacheEventRecorder(enabled=True, page_size=DEFAULT_PAGE_SIZE)
queue.enqueue(self._store(1, None, cache_salt="tenant-a"))
queue.enqueue(self._store(2, 1, cache_salt="tenant-b"))
Expand Down
Loading