Skip to content
Merged
Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
98bb500
perf(hicache): compact DSA indexer producer layers
anish-shanbhag Sep 8, 2026
8b0a4b7
fix(hicache): handle DSA stages without producers
anish-shanbhag Sep 9, 2026
fb98061
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
nvpohanh Sep 10, 2026
825397e
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
YAMY1234 Sep 11, 2026
84411f3
test(hicache): fix indexer fixture and cover compact sidecars
anish-shanbhag Sep 11, 2026
4531aac
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
nvpohanh Sep 14, 2026
6979e27
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
nvpohanh Sep 15, 2026
8c8f40f
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
nvpohanh Sep 15, 2026
b94ff59
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
nvpohanh Sep 17, 2026
27044a1
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
nvpohanh Sep 17, 2026
ae460c7
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
ishandhanani Sep 17, 2026
d35bd62
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
nvpohanh Sep 17, 2026
75897d9
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
nvpohanh Sep 18, 2026
f1640e5
Merge remote-tracking branch 'refs/remotes/sgl-upstream/main' into co…
anish-shanbhag Sep 21, 2026
ca2aa75
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
nvpohanh Sep 22, 2026
4c99598
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
anish-shanbhag Sep 23, 2026
e096fb4
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
nvpohanh Sep 28, 2026
f2e99bf
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
nvpohanh Sep 29, 2026
9d2b08b
Merge branch 'main' into perf/hicache-dsa-producer-sidecar
nvpohanh Sep 29, 2026
b83eec4
[HiCache][PD] Elide DSA index-K for non-producer layers (78 -> 21 on …
Sep 30, 2026
7bd44f2
Merge branch 'main' into pd-dsa-producer-layer-transfer
nvpohanh Oct 1, 2026
fe732db
[PD] Skip zero-length DSA index-K entries in Mooncake registration an…
Oct 1, 2026
032ccf5
Merge upstream/main into pd-dsa-producer-layer-transfer
Oct 2, 2026
ed92a34
Address review: handle DSA elision without changing the PD wire format
wenscarl Oct 5, 2026
b59c106
Upd
wenscarl Oct 8, 2026
c0ae50c
upd
wenscarl Oct 9, 2026
51e4d80
Merge branch 'main' into pd-dsa-producer-layer-transfer
nvpohanh Oct 12, 2026
42444b5
Merge branch 'main' into pd-dsa-producer-layer-transfer
Fridge003 Oct 13, 2026
0a2791c
fix lint
Fridge003 Oct 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion python/sglang/srt/disaggregation/mooncake/conn.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
build_dsa_tail_transfer_blocks,
build_transfer_entry_pairs,
compute_mamba_state_slice_byte_blocks,
is_elided_entry,
resolve_dcp_dst_entry_indices,
should_send_replicated_state,
slice_dsa_tail_dst_ptrs_for_pp,
Expand Down Expand Up @@ -364,6 +365,14 @@ def _registerable_regions(self) -> List[Tuple[int, int]]:

def add(ptrs: List[int], lens: List[int]) -> None:
for ptr, length in zip(ptrs or [], lens or []):
if length == 0:
# DSA index-K elision gives every shared-topk layer a 0-row
# buffer so the per-layer list stays layer-aligned. There is
# nothing to register, and Mooncake rejects it outright:
# transfer_engine_impl.cpp: Transfer Engine does not
# support zero length memory region
# kv_args keeps the entry, so peers still pair positionally.
continue
if (ptr, length) not in seen:
seen.add((ptr, length))
regions.append((ptr, length))
Expand Down Expand Up @@ -832,6 +841,10 @@ def _state_stride_mismatch(
f"{state_type} decode published {len(dst_item_lens)} item "
f"lengths but paired entry dst[{j}] needs one"
)
if is_elided_entry(item_lens[i], dst_item_lens, j):
# One peer elided this shared-topk layer's index-K (e.g. a
# dense HiSparse decode); no bytes move, so no stride to match.
continue
if item_lens[i] != dst_item_lens[j]:
return (
f"{state_type} item length mismatch for paired "
Expand All @@ -853,6 +866,8 @@ def _state_stride_mismatch(
f"lengths for this stage's {layers_current_pp_stage} entries"
)
for layer_id in range(layers_current_pp_stage):
if is_elided_entry(item_lens[layer_id], mapped_dst_lens, layer_id):
continue # Elided by one peer; see the paired loop above.
if item_lens[layer_id] != mapped_dst_lens[layer_id]:
return (
f"{state_type} item length mismatch for positional "
Expand Down Expand Up @@ -1037,22 +1052,34 @@ def _send_kvcache_generic(
len(dst_data_ptrs),
allow_positional_fallback=self.pp_size == 1,
)
# An entry elided on either side moves no bytes and has no
# registered buffer to write into.
layers_params = [
(src_data_ptrs[i], dst_data_ptrs[j], item_lens[i]) for i, j in pairs
(src_data_ptrs[i], dst_data_ptrs[j], item_lens[i])
for i, j in pairs
if not is_elided_entry(item_lens[i], dst_item_lens, j)
]
else:
src_kv_ptrs, dst_kv_ptrs, layers_current_pp_stage = (
self.get_mla_kv_ptrs_with_pp(
src_data_ptrs, dst_data_ptrs, state_type
)
)
mapped_dst_lens = None
if dst_item_lens is not None:
_, mapped_dst_lens, _ = self.get_mla_kv_ptrs_with_pp(
item_lens, dst_item_lens, state_type
)
layers_params = [
(
src_kv_ptrs[layer_id],
dst_kv_ptrs[layer_id],
item_lens[layer_id],
)
for layer_id in range(layers_current_pp_stage)
if not is_elided_entry(
item_lens[layer_id], mapped_dst_lens, layer_id
)
]
else:
src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, layers_current_pp_stage = (
Expand Down
10 changes: 9 additions & 1 deletion python/sglang/srt/disaggregation/mori/conn.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
pack_int_lists,
unpack_int_lists,
)
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.disaggregation.utils import DisaggregationMode, is_elided_entry
from sglang.srt.environ import envs
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils.common import run_with_deadline
Expand Down Expand Up @@ -1493,6 +1493,7 @@ def send_state(
src_descs,
src_lens,
dst_descs,
dst_lens,
st,
)
)
Expand Down Expand Up @@ -1600,6 +1601,7 @@ def _send_swa_dsa_state(
src_state_mem_descs: List[MemoryDesc],
src_state_item_lens: List[int],
dst_state_mem_descs: List[MemoryDesc],
dst_state_item_lens: List[int],
state_type: str,
) -> List[TransferStatus]:
# TP mismatch check for non-MLA SWA
Expand Down Expand Up @@ -1678,6 +1680,12 @@ def _send_swa_dsa_state(
for i, src_desc in enumerate(src_state_mem_descs):
dst_desc = dst_state_mem_descs[i]
state_item_len = src_state_item_lens[i]
if is_elided_entry(state_item_len, dst_state_item_lens, i):
# DSA index-K elision gives every shared-topk layer a 0-row
# buffer; either peer eliding it means nothing moves, and the
# descriptor describes an empty allocation. Skipping keeps the
# index alignment with state_mem_descs that registration relies on.
continue

statuses.extend(
self._submit_batch_transfer_plan(
Expand Down
22 changes: 21 additions & 1 deletion python/sglang/srt/disaggregation/nixl/conn.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
build_dsa_tail_transfer_blocks,
build_transfer_entry_pairs,
compute_mamba_state_slice_byte_blocks,
is_elided_entry,
resolve_dcp_dst_entry_indices,
slice_dsa_tail_dst_ptrs_for_pp,
)
Expand Down Expand Up @@ -1788,28 +1789,47 @@ def _send_kvcache_generic(
# the paired entries must have identical layouts.
if dst_item_lens is not None:
for i, j in pairs:
if is_elided_entry(item_lens[i], dst_item_lens, j):
# A shared-topk layer carries no index-K. One peer
# may elide it while the other does not -- HiSparse
# runs decode-side only -- and neither reads it.
continue
if item_lens[i] != dst_item_lens[j]:
raise RuntimeError(
f"{state_type} item length mismatch for paired "
f"entries src[{i}]={item_lens[i]} "
f"dst[{j}]={dst_item_lens[j]}"
)
# DSA index-K elision leaves a 0-row buffer on every shared-topk
# layer. Neither side registered it, so a descriptor built from
# it would fail in initialize_xfer(); drop it when either peer
# elided the entry.
layers_params = [
(src_data_ptrs[i], dst_data_ptrs[j], item_lens[i]) for i, j in pairs
(src_data_ptrs[i], dst_data_ptrs[j], item_lens[i])
for i, j in pairs
if not is_elided_entry(item_lens[i], dst_item_lens, j)
]
else:
src_kv_ptrs, dst_kv_ptrs, layers_current_pp_stage = (
self.get_mla_kv_ptrs_with_pp(
src_data_ptrs, dst_data_ptrs, state_type
)
)
mapped_dst_lens = None
if dst_item_lens is not None:
_, mapped_dst_lens, _ = self.get_mla_kv_ptrs_with_pp(
item_lens, dst_item_lens, state_type
)
layers_params = [
(
src_kv_ptrs[layer_id],
dst_kv_ptrs[layer_id],
item_lens[layer_id],
)
for layer_id in range(layers_current_pp_stage)
if not is_elided_entry(
item_lens[layer_id], mapped_dst_lens, layer_id
)
]
else:
src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, layers_current_pp_stage = (
Expand Down
15 changes: 15 additions & 0 deletions python/sglang/srt/disaggregation/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
List,
Literal,
Optional,
Sequence,
Tuple,
Type,
overload,
Expand Down Expand Up @@ -912,6 +913,20 @@ def compute_mamba_state_slice_byte_blocks(
return blocks


def is_elided_entry(
src_len: int, dst_lens: Optional[Sequence[int]], dst_index: int
) -> bool:
"""True when either peer elided this entry (a 0-row DSA index-K buffer).

Nothing moves for it, and the elided side never registered the buffer.
"""
if src_len == 0:
return True
return (
dst_lens is not None and dst_index < len(dst_lens) and dst_lens[dst_index] == 0
)


def build_transfer_entry_pairs(
src_layer_ids: List[int],
dst_layer_ids: List[int],
Expand Down
7 changes: 5 additions & 2 deletions python/sglang/srt/mem_cache/kv_cache_configurator.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,13 +112,16 @@


def _should_elide_dsa_index_k(*, is_draft_worker: bool) -> bool:
"""Shared-topk layers get a 0-row index-K buffer; HiCache and PD must skip those entries."""
memory_config = get_memory()
# Ascend PD slices the KV list positionally (index-K rides in it, not in
# StateType.DSA), so a compact layout would mis-slice PP>1 against PP=1.
if _is_npu and get_disagg().disaggregation_mode != "null":
return False
return (
Comment thread
nvpohanh marked this conversation as resolved.
Resolved
not memory_config.enable_hisparse
and not is_draft_worker
and not memory_config.enable_hierarchical_cache
and not memory_config.enable_unified_cache_external_linker
Comment thread
nvpohanh marked this conversation as resolved.
Resolved
and get_disagg().disaggregation_mode == "null"
)


Expand Down
112 changes: 112 additions & 0 deletions test/registered/unit/disaggregation/test_disaggregation_wire.py
Original file line number Diff line number Diff line change
Expand Up @@ -959,6 +959,118 @@ def test_qsa_stage_without_qsa_layers_does_not_register_rope_ring(self):
self.assertEqual(kv_args.state_item_lens[1:], [[], []])
self.assertEqual(kv_args.state_layer_ids[1:], [[], []])

def test_elided_dsa_layers_are_not_registered(self):
"""A 0-row index-K buffer must not reach Mooncake's registration.

The engine rejects it outright ("Transfer Engine does not support zero
length memory region"). Skipping happens in _registerable_regions, not
in kv_args, so the per-layer list keeps its dense shape and peers still
pair positionally.
"""
mgr = MooncakeKVManager.__new__(MooncakeKVManager)
mgr.kv_args = SimpleNamespace(
kv_data_ptrs=[],
kv_data_lens=[],
aux_data_ptrs=[],
aux_data_lens=[],
# 6 layers, 3 producers.
state_data_ptrs=[[100, 101, 102, 103, 104, 105]],
state_data_lens=[[4096, 0, 0, 4096, 0, 4096]],
)
regions = mgr._registerable_regions()
self.assertEqual(regions, [(100, 4096), (103, 4096), (105, 4096)])

def test_stride_check_accepts_one_sided_elided_dsa_entry(self):
"""A prefill that elides a shared-topk layer's index-K must still pair
with a dense decode (and the reverse): the registration-time stride
check used to reject the decode with "prefill=0 decode=64", failing
every request to it. A real stride disagreement is still rejected."""
for layer_ids in ([], [3, 4]):
for src_lens, dst_lens, expect_ok in (
([0, 132], [64, 132], True),
([64, 132], [0, 132], True),
([132, 132], [64, 132], False),
):
with self.subTest(
layer_ids=layer_ids, src_lens=src_lens, dst_lens=dst_lens
):
manager = object.__new__(MooncakeKVManager)
manager.is_mla_backend = True
manager.is_hybrid_mla_backend = False
manager.pp_size = 1
manager.kv_args = SimpleNamespace(
mla_compression_ratios=None, prefill_start_layer=0
)
reason = manager._state_stride_mismatch(
src_data_ptrs=[0x1000, 0x1100],
dst_data_ptrs=[0x2000, 0x2100],
item_lens=src_lens,
dst_item_lens=dst_lens,
state_type=StateType.DSA,
src_layer_ids=layer_ids,
dst_layer_ids=layer_ids,
)
if expect_ok:
self.assertIsNone(reason)
else:
self.assertIn("item length mismatch", reason)

def test_send_drops_entry_elided_by_either_peer(self):
"""An entry elided on either side must not be sent. A dense prefill
against an elided decode used to keep the entry (the filter only looked
at the prefill length), writing index-K into a decode buffer that has no
memory and was never registered."""
for layer_ids in ([], [3, 4, 5]):
for src_lens, dst_lens in (
([132, 132, 132], [0, 132, 132]),
([0, 132, 132], [132, 132, 132]),
):
with self.subTest(
layer_ids=layer_ids, src_lens=src_lens, dst_lens=dst_lens
):
manager = object.__new__(MooncakeKVManager)
manager.is_mla_backend = True
manager.is_hybrid_mla_backend = False
manager.pp_size = 1
manager.kv_args = SimpleNamespace(
mla_compression_ratios=None, prefill_start_layer=0
)
manager.state_strides_validated = set()
manager.enable_custom_mem_pool = False
manager.max_transfer_batch_indices = 0
manager._transfer_data = Mock(return_value=0)

ret = manager._send_kvcache_generic(
"decode",
[0x1000, 0x1100, 0x1200],
[0x2000, 0x2100, 0x2200],
src_lens,
np.array([0], dtype=np.int32),
np.array([0], dtype=np.int32),
executor=None,
state_type=StateType.DSA,
src_layer_ids=layer_ids,
dst_layer_ids=layer_ids,
dst_item_lens=dst_lens,
)

self.assertEqual(ret, 0)
manager._transfer_data.assert_called_once_with(
"decode", [(0x1100, 0x2100, 132), (0x1200, 0x2200, 132)]
)

def test_dense_shape_keeps_positional_pairing_intact(self):
"""Both peers still publish one entry per layer, so pairing is unchanged.

This is what keeps the wire compatible with a peer that elides nothing:
counts match, and build_transfer_entry_pairs stays on its positional
path with no layer ids on either side.
"""
self.assertEqual(
build_transfer_entry_pairs([], [], 6, 6),
[(i, i) for i in range(6)],
)

def test_compact_qsa_entries_map_by_global_layer_id(self):
self.assertEqual(
build_transfer_entry_pairs(
Expand Down
Loading
Loading