Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 52 additions & 2 deletions python/sglang/srt/mem_cache/kv_cache_configurator.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,20 @@
logger = logging.getLogger(__name__)


def _dsa_index_buffer_tokens_for_dcp(
max_total_num_tokens: int, page_size: int, dcp_size: int
) -> int:
# PagedTokenToKVPoolAllocator reserves page 0 and may hand out its final
# page ID. Each DCP logical page spans dcp_size physical index pages.
return max_total_num_tokens * dcp_size + (dcp_size - 1) * page_size


def _dsa_index_pages_required(
max_total_num_tokens: int, page_size: int, dcp_size: int
) -> int:
return (max_total_num_tokens // page_size + 1) * dcp_size


def _should_elide_dsa_index_k(*, is_draft_worker: bool) -> bool:
memory_config = get_memory()
return (
Expand Down Expand Up @@ -1512,6 +1526,26 @@ def _build_dsa_kv_pool(self, *, max_total_num_tokens: int) -> KVCache:
pool_kwargs["layer_shard_size"] = dsa_cp_layer_shard_size
else:
PoolCls = DSATokenToKVPool
dcp_size = get_parallel().attn_dcp_size
pool_size = max_total_num_tokens
pool_page_size = self.pool_page_size
if dcp_size > 1 and not get_memory().enable_hisparse:
# _derive_pool_sizes already widens the draft's token count. Price
# and allocate the logical span once, not DCP squared. DSA kernels
# still use physical 64-token pages, even for a replicated draft.
index_base_tokens = max_total_num_tokens // self.loc_space_scale
pool_page_size = get_schedule().page_size
# The paged allocator addresses logical pages of page_size * DCP,
# while the latent MLA KV remains sharded per rank. The DSA index
# buffer is indexed by those logical locations during PD transfer
# and must cover the allocator's final (nonzero) page ID.
pool_kwargs["index_buf_size"] = _dsa_index_buffer_tokens_for_dcp(
index_base_tokens, pool_page_size, dcp_size
)
if self.is_draft_worker:
# The latent draft pool must also span the whole logical page
# zero and the final allocator page, with physical-page layout.
pool_size = pool_kwargs["index_buf_size"]
if _should_elide_dsa_index_k(is_draft_worker=self.is_draft_worker):
pool_kwargs["skip_topk_layers"] = [
dsa_layer_skips_topk(self.model_config.hf_config, layer_id)
Expand All @@ -1520,8 +1554,8 @@ def _build_dsa_kv_pool(self, *, max_total_num_tokens: int) -> KVCache:
)
]
token_to_kv_pool = PoolCls(
max_total_num_tokens,
page_size=self.pool_page_size,
pool_size,
page_size=pool_page_size,
dtype=self.kv_cache_dtype,
kv_lora_rank=self.model_config.kv_lora_rank,
qk_rope_head_dim=self.model_config.qk_rope_head_dim,
Expand All @@ -1537,6 +1571,22 @@ def _build_dsa_kv_pool(self, *, max_total_num_tokens: int) -> KVCache:
index_head_dim=get_dsa_index_head_dim(self.model_config.hf_config),
**pool_kwargs,
)
if "index_buf_size" in pool_kwargs:
required_pages = _dsa_index_pages_required(
index_base_tokens, pool_page_size, dcp_size
)
# Some DSA layers reuse a preceding top-k and intentionally have
# a zero-row placeholder; validate every materialized index pool.
actual_pages = [
buffer.shape[0]
for buffer in token_to_kv_pool.index_k_with_scale_buffer
if buffer.shape[0] > 0
]
if actual_pages and min(actual_pages) < required_pages:
raise RuntimeError(
f"DSA index pool has {min(actual_pages)} pages, but the "
f"DCP allocator can address {required_pages} pages"
)
return token_to_kv_pool

def _build_hybrid_mla_swa_kv_pool(
Expand Down
22 changes: 21 additions & 1 deletion python/sglang/srt/model_executor/pool_configurator.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
"""Configurator for standard models: MHA, MLA, DSA, FP4.

coeff = cell_size (bytes per token across all layers)
bias = 0
bias = one allocator page for DSA under DCP (including replicated pools)
"""

def __init__(self, kvc: KVCacheConfigurator):
Expand All @@ -168,6 +168,12 @@ def __init__(self, kvc: KVCacheConfigurator):
num_layers = kvc.layer_info.num_effective_layers

self._cell_size = self._compute_cell_size(kvc, num_layers)
self._reserve_dsa_dcp_page = (
kvc.use_mla_backend
and is_deepseek_dsa(kvc.model_config.hf_config)
and get_parallel().attn_dcp_size > 1
and not get_memory().enable_hisparse
)
has_kv_on_another_pp_stage = (
self._cell_size == 0
and mambaish is not None
Expand Down Expand Up @@ -210,6 +216,11 @@ def __init__(self, kvc: KVCacheConfigurator):
draft_kv_size = int(
target_kv_size * draft_num_layers / target_kv_num_layers
)
# The target latent KV is per-rank; the replicated draft
# consumes every virtual allocator location. Its indexer
# replication is already priced below, so scale only KV.
if not get_memory().enable_hisparse:
draft_kv_size *= get_parallel().attn_dcp_size
draft_indexer_size = self._compute_dsa_indexer_cell_size(
kvc=kvc,
num_layers=draft_num_layers,
Expand Down Expand Up @@ -381,6 +392,10 @@ def _compute_dsa_indexer_cell_size(
from sglang.srt.mem_cache.sparsity import parse_hisparse_config

indexer_ratio = parse_hisparse_config().host_to_device_ratio
else:
# Unlike latent MLA KV, DSA index-K is addressed by global logical
# locations. Both target and draft materialize that full span.
indexer_ratio = get_parallel().attn_dcp_size

from sglang.srt.mem_cache.kv_cache_configurator import (
_should_elide_dsa_index_k,
Expand Down Expand Up @@ -432,6 +447,11 @@ def calculate_pool_sizes(
if self._cell_size
else self._zero_kv_max_tokens
)
if self._reserve_dsa_dcp_page:
# Target KV has one physical page zero; replicated index/draft
# pools have DCP physical pages for that same logical page zero.
# Their DCP factors are in _cell_size: reserve it exactly once.
max_total_num_tokens -= page_size
max_total_num_tokens = max_total_num_tokens // page_size * page_size
return MemoryPoolConfig(max_total_num_tokens=max_total_num_tokens)

Expand Down
30 changes: 30 additions & 0 deletions test/registered/unit/mem_cache/test_dsa_dcp_index_capacity.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import pytest

from sglang.srt.mem_cache.kv_cache_configurator import (
_dsa_index_buffer_tokens_for_dcp,
_dsa_index_pages_required,
)


@pytest.mark.parametrize(
("max_total_num_tokens", "page_size", "dcp_size"),
[
(138_048, 64, 2),
(197_568, 64, 4),
(138_049, 64, 2),
],
)
def test_dsa_index_pool_covers_last_dcp_allocator_page(
max_total_num_tokens: int, page_size: int, dcp_size: int
):
index_buffer_tokens = _dsa_index_buffer_tokens_for_dcp(
max_total_num_tokens, page_size, dcp_size
)
# IndexKeyCache uses this page-count formula when allocating index buffers.
allocated_pages = (index_buffer_tokens + page_size + 1) // page_size
required_pages = _dsa_index_pages_required(
max_total_num_tokens, page_size, dcp_size
)
assert allocated_pages >= required_pages
if max_total_num_tokens % page_size == 0:
assert allocated_pages == required_pages
180 changes: 180 additions & 0 deletions test/registered/unit/model_executor/test_dsa_dcp_pool_budget.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
"""Price the real DSA pools, including replicated index/draft and page zero.

CPU tensors exercise the allocation code without loading model weights.
"""

import unittest
from types import SimpleNamespace
from unittest.mock import patch

import pytest
import torch
from test_pool_configurator import (
_configure_dsa_model,
_make_model_runner,
)

from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
from sglang.srt.model_executor.pool_configurator import DefaultPoolConfigurator
from sglang.srt.runtime_context import get_disagg, get_memory, get_parallel
from sglang.test.ci.ci_register import register_cpu_ci

register_cpu_ci(est_time=10, suite="base-a-test-cpu")


@pytest.fixture
def runner():
cleanup = unittest.TestCase()
mr = _make_model_runner(
cleanup,
num_layers=6,
use_mla_backend=True,
page_size=64,
disaggregation_mode="decode",
)
_configure_dsa_model(mr)
mr.kv_cache_dtype = torch.float8_e4m3fn
mr.kv_cache_dtype_str = "fp8_e4m3"
try:
with get_parallel().override(attn_tp_size=1, attn_cp_size=1, attn_dcp_size=1):
yield mr
finally:
cleanup.doCleanups()


def build_pool(runner, tokens, *, draft=False, layers=None, device="cpu"):
"""Use the production builder and its actual draft location/page policy."""
layers = runner.layer_info.num_effective_layers if layers is None else layers
kvc = SimpleNamespace(
model_config=runner.model_config,
is_draft_worker=draft,
use_mla_backend=True,
kv_cache_dtype=runner.kv_cache_dtype,
device=device,
layer_info=SimpleNamespace(
start_layer=0, end_layer=layers, num_effective_layers=layers
),
)
kvc.loc_space_scale = KVCacheConfigurator.loc_space_scale.fget(kvc)
kvc.pool_page_size = KVCacheConfigurator.pool_page_size.fget(kvc)
return KVCacheConfigurator._build_dsa_kv_pool(
kvc, max_total_num_tokens=tokens * kvc.loc_space_scale
)


def pool_bytes(pool):
buffers = pool.kv_buffer + pool.index_k_with_scale_buffer
if pool.layer_shard_enabled:
buffers += [pool.remote_kv_buffer, pool.index_key_cache.remote_buffer]
return sum(b.numel() * b.element_size() for b in buffers)


@pytest.mark.parametrize("dcp", [2, 4])
@pytest.mark.parametrize("draft_layers", [0, 1, 3])
def test_dcp_budget_covers_real_target_and_draft(runner, dcp, draft_layers):
available = 24 * 1024**2
runner.spec_algorithm.is_eagle.return_value = bool(draft_layers)
runner.spec_aux_config.eagle_draft_num_layers = draft_layers
with get_parallel().override(attn_dcp_size=dcp):
cfg = DefaultPoolConfigurator(runner)
resolved = cfg.calculate_pool_sizes(available, 64)
target = build_pool(runner, resolved.max_total_num_tokens)
used = pool_bytes(target)
if draft_layers:
draft = build_pool(
runner, resolved.max_total_num_tokens, draft=True, layers=draft_layers
)
used += pool_bytes(draft)
assert used <= available
# Do not solve under-budgeting with an arbitrary large safety factor.
assert used + cfg._cell_size * 64 > available


@pytest.mark.parametrize("dcp", [2, 4])
def test_replicated_draft_is_not_scaled_twice(runner, dcp):
with get_parallel().override(attn_dcp_size=dcp):
draft = build_pool(runner, 256, draft=True, layers=1)
logical_span = (256 + 64) * dcp
assert draft.page_size == 64
assert draft.kv_buffer[0].shape[0] == logical_span
assert draft.index_k_with_scale_buffer[0].shape[0] * 64 == logical_span
# The final allocator slot must be addressable in both buffers.
draft.kv_buffer[0][logical_span - 1].fill_(7)
draft.index_k_with_scale_buffer[0][logical_span // 64 - 1].fill_(9)


def test_dcp_reserved_page_can_exhaust_budget(runner):
with get_parallel().override(attn_dcp_size=2):
cfg = DefaultPoolConfigurator(runner)
with pytest.raises(RuntimeError, match="Not enough memory"):
cfg.calculate_pool_sizes(cfg._cell_size * 64, 64)


def test_dcp_one_capacity_is_unchanged(runner):
with get_parallel().override(attn_dcp_size=1):
cfg = DefaultPoolConfigurator(runner)
budget = 24 * 1024**2
assert cfg.calculate_pool_sizes(budget, 64).max_total_num_tokens == (
budget // cfg._cell_size // 64 * 64
)


def test_hisparse_ratio_is_not_multiplied_by_dcp(runner):
with (
get_parallel().override(attn_dcp_size=4),
get_memory().override(enable_hisparse=True),
patch(
"sglang.srt.mem_cache.sparsity.parse_hisparse_config",
return_value=SimpleNamespace(host_to_device_ratio=2),
),
):
cfg = DefaultPoolConfigurator(runner)
index_cost = cfg._compute_dsa_indexer_cell_size(kvc=runner, num_layers=6)
assert index_cost == 132 * 6 * 2


@pytest.mark.parametrize("hierarchical_cache", [False, True])
@pytest.mark.parametrize("draft_layers", [0, 1])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
def test_index_layer_elision_matches_allocation(
runner, hierarchical_cache, draft_layers, dtype
):
runner.model_config.hf_config.index_topk_freq = 4
runner.model_config.hf_config.index_skip_topk_offset = 3
runner.kv_cache_dtype = dtype
runner.spec_algorithm.is_eagle.return_value = bool(draft_layers)
runner.spec_aux_config.eagle_draft_num_layers = draft_layers
available = 24 * 1024**2
with (
get_parallel().override(attn_dcp_size=2),
get_disagg().override(disaggregation_mode="null"),
get_memory().override(enable_hierarchical_cache=hierarchical_cache),
):
cfg = DefaultPoolConfigurator(runner)
tokens = cfg.calculate_pool_sizes(available, 64).max_total_num_tokens
target = build_pool(runner, tokens)
assert any(b.numel() == 0 for b in target.index_k_with_scale_buffer) == (
not hierarchical_cache
)
used = pool_bytes(target)
if draft_layers:
draft = build_pool(runner, tokens, draft=True, layers=draft_layers)
assert all(b.numel() for b in draft.index_k_with_scale_buffer)
used += pool_bytes(draft)
assert used <= available
assert used + cfg._cell_size * 64 > available


def test_explicit_token_cap_does_not_reserve_page_twice(runner):
with get_parallel().override(attn_dcp_size=2):
cfg = DefaultPoolConfigurator(runner)
budget = 24 * 1024**2
tokens = cfg.calculate_pool_sizes(budget, 64).max_total_num_tokens
assert (
cfg.calculate_pool_sizes_from_max_tokens(tokens, 64).max_total_num_tokens
== tokens
)
assert (
cfg.calculate_pool_sizes_from_max_tokens(1025, 64).max_total_num_tokens
== 1024
)
Loading