Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions atom/spec_decode/draft_kv.py
Original file line number Diff line number Diff line change
Expand Up @@ -336,12 +336,16 @@ def build_kv_cache_tensor(self, module):
module.max_model_len = runner.config.max_model_len
module.k_cache = k_cache
module.v_cache = v_cache
# A declared per-tensor scale is one shared scalar, not per-block data:
# the draft's attention reads it off its own binding, and anything that
# walks `kv_cache_data` by block (the offload codec) must not see it.
per_tensor = self._kv_scale is not None
return KVCacheTensor(
layer_num=module.layer_num,
k_cache=k_cache,
v_cache=v_cache,
k_scale=getattr(module, "k_scale", None),
v_scale=getattr(module, "v_scale", None),
k_scale=None if per_tensor else getattr(module, "k_scale", None),
v_scale=None if per_tensor else getattr(module, "v_scale", None),
)

def get_kv_transfer_tensors(self):
Expand Down
97 changes: 97 additions & 0 deletions tests/test_draft_kv_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,103 @@ def test_gqa_draft_layers_share_one_bound_unit_scale_tensor():
assert all(m.k_scale is scale and m.v_scale is scale for m in layers)


def test_a_per_tensor_scale_stays_off_the_blockwise_kv_cache_data():
"""The shared unit scale is the layers' own binding, not per-block data:
the entries published to `kv_cache_data` carry the draft's K/V only, so a
block-wise walker such as the LMCache offload codec accepts them."""
from atom.kv_transfer.offload.dense.kv_byte_codec import DenseKVByteCodec

pool = MhaKvPool(
layers=2,
block_size=128,
num_kv_heads=2,
head_dim=64,
kv_dtype=torch.bfloat16,
flash_layout=True,
)
pool.allocate(4, "cpu")
runner = _runner(draft_layers=2)
runner.config = SimpleNamespace(max_model_len=4096, kv_cache_dtype="fp8")
builder = DraftKvBuilder(
runner, lambda **_: pool, kv_scale=DraftKvPoolSpec().kv_scale
)
builder._kv_pool = pool

layers = [m for m in runner.drafter.model if isinstance(m, _PagedAttention)]
entries = {}
for i, m in enumerate(layers):
m.layer_num = i
entries[f"layer_{i}"] = builder.build_kv_cache_tensor(m)

assert all(e.k_scale is None and e.v_scale is None for e in entries.values())
assert all(m.k_scale.numel() == 1 for m in layers) # the binding keeps it
codec = DenseKVByteCodec(entries, num_blocks=4)
assert codec.bytes_per_block > 0


@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused GPU staging")
def test_an_offload_round_trip_restores_the_draft_rows_at_new_block_ids():
"""What an LMCache tier save and a later tier load do to the draft's pool:
the codec gathers some blocks into a staging buffer and scatters it back
into other block ids. The draft's K/V rows come back byte for byte, beside
a target-like block-major cache, and nothing else is written."""
pytest.importorskip("triton")
from atom.config import KVCacheTensor
from atom.kv_transfer.offload.dense.kv_byte_codec import DenseKVByteCodec

blocks = 6
pool = MhaKvPool(
layers=2,
block_size=128,
num_kv_heads=2,
head_dim=64,
kv_dtype=torch.bfloat16,
flash_layout=True,
)
pool.allocate(blocks, "cuda")
runner = _runner(draft_layers=2)
runner.config = SimpleNamespace(max_model_len=4096, kv_cache_dtype="fp8")
builder = DraftKvBuilder(
runner, lambda **_: pool, kv_scale=DraftKvPoolSpec().kv_scale
)
builder._kv_pool = pool
layers = [m for m in runner.drafter.model if isinstance(m, _PagedAttention)]
entries = {
# A target layer's block-major cache, as the target builder publishes it.
"layer_0": KVCacheTensor(
layer_num=0, k_cache=torch.empty(blocks, 128, 576, device="cuda")
)
}
for i, m in enumerate(layers, start=1):
m.layer_num = i
entries[f"layer_{i}"] = builder.build_kv_cache_tensor(m)
caches = [
t
for e in entries.values()
for t in (e.k_cache, e.v_cache)
if t is not None and t.numel()
]
for t in caches:
t.copy_(torch.randn(t.shape, device="cuda").to(t.dtype))
before = [t.clone() for t in caches]

codec = DenseKVByteCodec(entries, num_blocks=blocks)
saved, restored = [1, 4], [0, 3]
buf = torch.empty(
len(saved) * codec.bytes_per_block, dtype=torch.uint8, device="cuda"
)
codec.gpu_to_chunk_major_device_buffer(buf, [saved])
for t in caches:
t.zero_()
codec.chunk_major_device_buffer_to_gpu(buf, [restored])
torch.cuda.synchronize()

for t, ref in zip(caches, before):
assert torch.equal(t[restored], ref[saved])
untouched = [b for b in range(blocks) if b not in restored]
assert not t[untouched].any()


def _bind_first_layer(pool, kv_cache_dtype, kv_scale=None):
"""Bind one paged layer through the builder; return the module."""
pool.allocate(2, "cpu")
Expand Down
Loading