Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions python/sglang/srt/arg_groups/mamba_hook.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,10 @@ def handle_mamba_backend(server_args: Any):


def handle_int8_mamba_checkpoint(server_args: Any):
# The int8 mamba checkpoint pool is only wired into the built-in
# MambaRadixCache. The host-offload path (enabled by
# --enable-hierarchical-cache) and custom radix-cache backends are NOT
# int8-aware: they would read int8 checkpoint slots as bf16 active slots
# (wrong pool / out-of-range). Reject the combination up front rather than
# silently corrupting state.
# The host-offload path (enabled by --enable-hierarchical-cache) and
# custom radix-cache backends are NOT int8-aware: they would read int8
# checkpoint slots as bf16 active slots (wrong pool / out-of-range).
# Reject the combination up front rather than silently corrupting state.
cfg = resolving_view(server_args)
if not cfg.enable_int8_mamba_checkpoint:
return
Expand Down
2 changes: 1 addition & 1 deletion python/sglang/srt/arg_groups/model_overrides/qwen4_exp.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def _qwen4_exp_overrides(server_args: Any, hf_config: Any) -> dict:
"""Compressed QSA must own ``page_size`` here,
so the qwen3_5 hybrid attention-shape policy is restated rather than shared.
page_size=64 needs page-aligned full-KV allocation (slots are full_slot // ratio),
which MambaRadixCache allows only with mamba extra-buffer or --disable-radix-cache.
which in turn needs the mamba extra-buffer strategy or --disable-radix-cache.
"""
cfg = resolving_view(server_args)
if (
Expand Down
3 changes: 0 additions & 3 deletions python/sglang/srt/environ.py
Original file line number Diff line number Diff line change
Expand Up @@ -678,9 +678,6 @@ class Envs:
SGLANG_ENABLE_UNIFIED_RADIX_TREE = EnvBool(False)
# Registered TreeCore backend serving the unified radix cache.
SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND = EnvStr("python")
# TODO(DSV4): @ispobock this has bug on main branch when retract
SGLANG_OPT_SWA_RADIX_CACHE_COMPACT = EnvBool(False)
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT = EnvBool(False)
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW = EnvBool(False)

# ===================================================================
Expand Down
10 changes: 2 additions & 8 deletions python/sglang/srt/kv_canary/radix_cache_walker.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import torch

from sglang.srt.mem_cache.radix_cache import RadixCache
from sglang.srt.mem_cache.swa_radix_cache import SWARadixCache
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
RadixCacheWalkResult,
)
Expand Down Expand Up @@ -34,7 +33,7 @@ def walk_radix_cache_for_canary(
return radix_cache.tree_core.walk_for_kv_canary(
unlocked_only=unlocked_only, swa_resident_only=swa_resident_only
)
if cache_type is not RadixCache and cache_type is not SWARadixCache:
if cache_type is not RadixCache:
raise NotImplementedError(
f"walk_radix_cache_for_canary does not support {cache_type.__name__}"
)
Expand Down Expand Up @@ -133,9 +132,6 @@ def _node_is_unlocked_for_canary(
if type(radix_cache) is RadixCache:
return node.lock_ref == 0

if type(radix_cache) is SWARadixCache:
return node.full_lock_ref == 0

raise NotImplementedError(
f"walk_radix_cache_for_canary does not support {type(radix_cache).__name__}"
)
Expand All @@ -146,7 +142,5 @@ def _node_is_swa_resident_for_canary(
node: TreeNode,
radix_cache: BasePrefixCache,
) -> bool:
if type(radix_cache) is SWARadixCache:
return not node.swa_tombstone

# RadixCache has no SWA tier, so every node it holds is resident.
return True
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ def _check_full_pool(self, ps: PoolStats, uncached: int = 0) -> Tuple[bool, str]
session_held = self.pool_stats_observer.session_held_full_tokens()
total = ps.full_capacity
elif self.is_hybrid_ssm:
# Branch on cache type for the protected accessor (MambaRadixCache
# splits full/mamba; ChunkCache only has the single protected_size).
# Branch on cache type for the protected accessor (a mamba-capable
# cache splits full/mamba; ChunkCache only has the single protected_size).
# Use the allocator's `.size` for `total`: static max_total_num_tokens for
# non-unified pools, the dynamic byte-coordinated cap (matching
# `available_size`) for the unified pool.
Expand Down
2 changes: 1 addition & 1 deletion python/sglang/srt/mem_cache/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ to keep. The layout is specified in
Two groups sit outside that stack:

- **Radix cache** is its own axis. The per-model variants (`radix_cache.py`,
`swa_radix_cache.py`, `mamba_radix_cache.py`, `hiradix_cache.py`, `chunk_cache.py`)
`hiradix_cache.py`, `chunk_cache.py`)
are converging onto the **Unified Radix Cache** (`unified_cache/`,
[#20415](https://github.com/sgl-project/sglang/issues/20415)), whose Full/SWA/Mamba
component model is documented in
Expand Down
4 changes: 2 additions & 2 deletions python/sglang/srt/mem_cache/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -271,10 +271,10 @@ def retraction_discard(req: Req, tree_cache: BasePrefixCache, backend: str) -> N

def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = True):
assert (not req.kv.holds_kv) == req.kv.is_kv_released
# MambaRadixCache may alloc mamba state before alloc KV cache
# A mamba-capable cache may alloc mamba state before alloc KV cache
if not req.kv.holds_kv:
assert tree_cache.supports_mamba(), (
"Only MambaRadixCache allow freeing before alloc"
"Only a mamba-capable tree cache allows freeing before alloc"
)
# TODO (csy, hanming): clean up this early allocation logic
if req.kv.holds_mamba:
Expand Down
Loading
Loading