Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions atom/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ class KVCacheTensor:
layer_num: int
k_cache: torch.Tensor = field(default_factory=lambda: torch.tensor([]))
v_cache: torch.Tensor = field(default_factory=lambda: torch.tensor([]))
# Per-block scale planes, block-major like the caches; None when the layer
# has none (a per-tensor scale stays on the layer's own binding).
k_scale: torch.Tensor = None
v_scale: torch.Tensor = None
# DSA sparse layers (GLM-5.2 / DeepSeek-V3.2): indexer key cache, block-major.
Expand Down
12 changes: 9 additions & 3 deletions atom/spec_decode/draft_kv.py
Original file line number Diff line number Diff line change
Expand Up @@ -325,23 +325,29 @@ def build_kv_cache_tensor(self, module):
check_cache = getattr(module, "check_cache", None)
if check_cache is not None:
check_cache(k_cache, v_cache)
# Published to `kv_cache_data`: per-block scale planes only. A declared
# per-tensor scale is one shared scalar the draft's attention reads off
# its own binding; anything walking `kv_cache_data` by block (the
# offload codec) must not see it.
k_scale = v_scale = None
if self._kv_scale is not None:
if self._kv_scale_tensor is None:
self._kv_scale_tensor = torch.full(
(1,), self._kv_scale, dtype=torch.float32, device=k_cache.device
)
module.k_scale = module.v_scale = self._kv_scale_tensor
elif pool.scale_fields: # the layout's, as the target binder asks
module.k_scale, module.v_scale = pool.scale_views(idx)
k_scale, v_scale = pool.scale_views(idx)
module.k_scale, module.v_scale = k_scale, v_scale
module.max_model_len = runner.config.max_model_len
module.k_cache = k_cache
module.v_cache = v_cache
return KVCacheTensor(
layer_num=module.layer_num,
k_cache=k_cache,
v_cache=v_cache,
k_scale=getattr(module, "k_scale", None),
v_scale=getattr(module, "v_scale", None),
k_scale=k_scale,
v_scale=v_scale,
)

def get_kv_transfer_tensors(self):
Expand Down
101 changes: 88 additions & 13 deletions tests/test_draft_kv_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -424,11 +424,9 @@ def model_class(cfg, layer_offset):
assert seen == {"dtype": pool_dtype} and p.config.kv_cache_dtype == target


def test_gqa_draft_layers_share_one_bound_unit_scale_tensor():
"""The DSpark GQA draft's pool binds every layer the same one-element tensor
at the scale its spec declares, at bind time: both its write and its read
take the scale as a device pointer, and nothing may be built for it inside
a captured forward."""
def _gqa_draft(blocks, device="cpu"):
"""A flash pool with the DSpark GQA draft's declared unit scale, and its
builder; returns the builder and the draft's paged layers, numbered from 1."""
pool = MhaKvPool(
layers=2,
block_size=128,
Expand All @@ -437,35 +435,107 @@ def test_gqa_draft_layers_share_one_bound_unit_scale_tensor():
kv_dtype=torch.bfloat16,
flash_layout=True,
)
pool.allocate(2, "cpu")
pool.allocate(blocks, device)
runner = _runner(draft_layers=2)
runner.config = SimpleNamespace(max_model_len=4096, kv_cache_dtype="fp8")
builder = DraftKvBuilder(
runner, lambda **_: pool, kv_scale=DraftKvPoolSpec().kv_scale
)
builder._kv_pool = pool

layers = [m for m in runner.drafter.model if isinstance(m, _PagedAttention)]
for i, m in enumerate(layers, start=1):
m.layer_num = i
return builder, layers


def test_gqa_draft_layers_share_one_bound_unit_scale_tensor():
"""The DSpark GQA draft's pool binds every layer the same one-element tensor
at the scale its spec declares, at bind time: both its write and its read
take the scale as a device pointer, and nothing may be built for it inside
a captured forward."""
builder, layers = _gqa_draft(2)
for m in layers:
m.layer_num = 0
builder.build_kv_cache_tensor(m)

scale = layers[0].k_scale
assert torch.is_tensor(scale) and scale.numel() == 1 and float(scale) == 1.0
assert all(m.k_scale is scale and m.v_scale is scale for m in layers)


def test_a_per_tensor_scale_stays_off_the_blockwise_kv_cache_data():
"""The shared unit scale is the layers' own binding, not per-block data:
the entries published to `kv_cache_data` carry the draft's K/V only, so a
block-wise walker such as the LMCache offload codec accepts them."""
from atom.kv_transfer.offload.dense.kv_byte_codec import DenseKVByteCodec

builder, layers = _gqa_draft(4)
entries = {f"layer_{m.layer_num}": builder.build_kv_cache_tensor(m) for m in layers}

assert all(e.k_scale is None and e.v_scale is None for e in entries.values())
assert all(m.k_scale.numel() == 1 for m in layers) # the binding keeps it
codec = DenseKVByteCodec(entries, num_blocks=4)
assert codec.bytes_per_block > 0


@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused GPU staging")
def test_an_offload_round_trip_restores_the_draft_rows_at_new_block_ids():
"""What an LMCache tier save and a later tier load do to the draft's pool:
the codec gathers some blocks into a staging buffer and scatters it back
into other block ids. The draft's K/V rows come back byte for byte, beside
a target-like block-major cache, and nothing else is written."""
pytest.importorskip("triton")
from atom.config import KVCacheTensor
from atom.kv_transfer.offload.dense.kv_byte_codec import DenseKVByteCodec

blocks = 6
builder, layers = _gqa_draft(blocks, "cuda")
entries = {
# A target layer's block-major cache, as the target builder publishes it.
"layer_0": KVCacheTensor(
layer_num=0, k_cache=torch.empty(blocks, 128, 576, device="cuda")
)
}
for m in layers:
entries[f"layer_{m.layer_num}"] = builder.build_kv_cache_tensor(m)
caches = [
t
for e in entries.values()
for t in (e.k_cache, e.v_cache)
if t is not None and t.numel()
]
for t in caches:
t.copy_(torch.randn(t.shape, device="cuda").to(t.dtype))
before = [t.clone() for t in caches]

codec = DenseKVByteCodec(entries, num_blocks=blocks)
saved, restored = [1, 4], [0, 3]
buf = torch.empty(
len(saved) * codec.bytes_per_block, dtype=torch.uint8, device="cuda"
)
codec.gpu_to_chunk_major_device_buffer(buf, [saved])
sentinel = 7.0 # not zero, so a stray write of zeros shows too
for t in caches:
t.fill_(sentinel)
codec.chunk_major_device_buffer_to_gpu(buf, [restored])
torch.cuda.synchronize()

untouched = [b for b in range(blocks) if b not in restored]
for t, ref in zip(caches, before):
assert torch.equal(t[restored], ref[saved])
assert torch.equal(t[untouched], torch.full_like(t[untouched], sentinel))


def _bind_first_layer(pool, kv_cache_dtype, kv_scale=None):
"""Bind one paged layer through the builder; return the module."""
"""Bind one paged layer through the builder; return the module and the
entry it publishes to `kv_cache_data`."""
pool.allocate(2, "cpu")
runner = _runner(draft_layers=1)
runner.config = SimpleNamespace(max_model_len=4096, kv_cache_dtype=kv_cache_dtype)
builder = DraftKvBuilder(runner, lambda **_: pool, kv_scale=kv_scale)
builder._kv_pool = pool
module = next(m for m in runner.drafter.model if isinstance(m, _PagedAttention))
module.layer_num = 0
builder.build_kv_cache_tensor(module)
return module
return module, builder.build_kv_cache_tensor(module)


def test_a_default_pool_binds_shuffle_views_and_its_own_scale_planes():
Expand All @@ -478,7 +548,7 @@ def test_a_default_pool_binds_shuffle_views_and_its_own_scale_planes():
head_dim=64,
kv_dtype=torch.float8_e4m3fnuz,
)
module = _bind_first_layer(pool, "fp8")
module, entry = _bind_first_layer(pool, "fp8")

k, v = pool.kv_views(0)
k_scale, v_scale = pool.scale_views(0)
Expand All @@ -487,6 +557,9 @@ def test_a_default_pool_binds_shuffle_views_and_its_own_scale_planes():
assert module.k_scale.shape == k_scale.shape
assert module.k_scale.data_ptr() == k_scale.data_ptr()
assert module.v_scale.data_ptr() == v_scale.data_ptr()
# Per-block planes: published to `kv_cache_data` too, for block-wise walkers.
assert entry.k_scale.data_ptr() == k_scale.data_ptr()
assert entry.v_scale.data_ptr() == v_scale.data_ptr()


@pytest.mark.parametrize("blocks_per_entry", [1, 8])
Expand All @@ -505,12 +578,14 @@ def test_a_flash_pool_binds_entry_wide_flash_views_and_the_declared_scale(
blocks_per_entry=blocks_per_entry,
flash_layout=True,
)
module = _bind_first_layer(pool, "fp8", kv_scale=DraftKvPoolSpec().kv_scale)
module, entry = _bind_first_layer(pool, "fp8", kv_scale=DraftKvPoolSpec().kv_scale)

assert module.k_cache.shape == (2, 128 * blocks_per_entry, 2, 64)
assert module.k_cache.shape[1] == pool.entry_tokens
assert module.v_cache.shape == pool.fa_views(0)[1].shape
assert module.k_scale.numel() == 1 and float(module.k_scale) == 1.0
# The declared per-tensor scale stays on the binding, off `kv_cache_data`.
assert entry.k_scale is None and entry.v_scale is None


@pytest.mark.parametrize("dcp", [1, 8])
Expand Down
Loading