Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions atom/model_engine/model_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -3855,6 +3855,8 @@ def pause_gc():
full_q_len,
)

self.attn_metadata_builder.blank_cache_write_targets()

# Whether this backend's capture builder supports a dynamic (per-bucket)
build_capture = self.attn_metadata_builder.build_for_cudagraph_capture
_build_params = inspect.signature(build_capture).parameters
Expand Down
27 changes: 27 additions & 0 deletions atom/model_ops/attentions/backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@
logger = logging.getLogger("atom")
T = TypeVar("T", bound="BroadcastableModelInput")

# "Written nowhere" -- see `_write_prefill_slots`. Same value the cache kernels
# and `fla_ops.replayssm` skip on.
PAD_SLOT_ID = -1


class BroadcastableModelInput(ABC):

Expand Down Expand Up @@ -183,6 +187,29 @@ def prepare_mtp_decode(
def build_for_cudagraph_capture(self, bs: int) -> AttentionMetaData:
raise NotImplementedError

def cache_write_targets(self) -> list[CpuGpuBuffer]:
"""Index buffers a CUDA graph capture would write cache through.

Declared here rather than matched on `forward_vars` key names, because a
backend can aim cache writes with a buffer that is named differently or
not registered there at all. The default is the paged slot mapping, by
suffix so TBO's per-ubatch mirrors come along; a backend with other write
targets overrides this, and must accept `PAD_SLOT_ID` in each of them as
"skip this row".
"""
var = self.model_runner.forward_vars
return [buf for name, buf in var.items() if name.endswith("slot_mapping")]

def blank_cache_write_targets(self) -> None:
"""Point the capture's cache writes at nothing.

A capture runs the model for real, so it writes through these; replay
overwrites them with real rows.
"""
for buf in self.cache_write_targets():
buf.np[:] = PAD_SLOT_ID
buf.copy_to_gpu()

# ------------------------------------------------------------------ #
# Cache sizing — one byte currency for every cache class. #
# ------------------------------------------------------------------ #
Expand Down
26 changes: 15 additions & 11 deletions atom/rollout/memory_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -402,19 +402,23 @@ def _resume_kv_cache(self) -> None:
logger.warning(f"{self.label}: No KV cache num_blocks to resume from")
return
saved_blocks = self._kv_cache_num_blocks
self._kv_cache_num_blocks = None
torch.cuda.empty_cache()
torch.cuda.reset_peak_memory_stats()
available_blocks = self.get_num_blocks()["num_kvcache_blocks"]
num_blocks = min(saved_blocks, available_blocks)
if num_blocks < saved_blocks:
logger.warning(
f"{self.label}: KV cache blocks reduced from {saved_blocks} to "
f"{num_blocks} due to changed GPU memory availability"
)
self.allocate_kv_cache(num_blocks)
# The size the pool slept at, not a fresh reading: `BlockManager`'s
# `BlockPool` is sized in the engine process from the startup count and
# nothing carries a wake-time one back, and the decode graphs were
# captured against the original pool.
free, total = torch.cuda.mem_get_info()
logger.info(
f"{self.label}: re-allocating {saved_blocks} KV blocks "
f"({free / (1 << 30):.2f}GB free of {total / (1 << 30):.2f}GB)"
)
# After the allocation, which can now OOM: clearing it first would lose
# the only record of the size, and the next wake would then take the
# guard above and report success with no pool.
self.allocate_kv_cache(saved_blocks)
self._kv_cache_num_blocks = None
logger.info(
f"{self.label}: KV cache re-allocated and bound ({num_blocks} blocks)"
f"{self.label}: KV cache re-allocated and bound ({saved_blocks} blocks)"
)

def _recapture_cudagraphs_if_needed(self) -> None:
Expand Down
140 changes: 140 additions & 0 deletions tests/test_cache_write_targets.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
# SPDX-License-Identifier: MIT
# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.

"""A capture must not write cache through a live batch's index buffers.

`capture_cudagraph` runs the model for real, so it writes cache wherever these
point, and `forward_vars` is not released on sleep. Where that landed decided the
two symptoms: outside a pool that came back smaller it was `Memory access fault`
during capture on 30B, and on live slots a decode row of all-NaN logprobs on 8B.

Which buffers a backend writes through is its own knowledge, so these pin the
`cache_write_targets` contract -- a builder's targets get blanked whether or not
they are in `forward_vars` -- rather than only the key-name rule behind the
default.
"""

from __future__ import annotations

import numpy as np
import pytest

try:
from atom.model_ops.attentions.backends import (
PAD_SLOT_ID,
AttentionMetadataBuilder,
)
except ImportError as _e: # pragma: no cover - environment, not the code
from import_guard import skip_if_dependency_missing

skip_if_dependency_missing(_e, "the attention builders import AITER")


class _FakeBuffer:
"""A `CpuGpuBuffer` down to the two halves and the upload between them."""

def __init__(self, size: int, fill: int = 0):
self.np = np.full(size, fill, dtype=np.int64)
self.gpu = np.full(size, fill, dtype=np.int64)
self.uploads = 0

def copy_to_gpu(self, n: int | None = None):
self.uploads += 1
self.gpu[:] = self.np
return self.gpu


class _Builder:
"""The two methods under test, over a `forward_vars` a served step left.

Not a subclass: the real methods are called unbound, so the test does not
have to satisfy the rest of the abstract surface to reach them.
"""

def __init__(self, extra_targets=()):
self.model_runner = type("R", (), {})()
self.model_runner.forward_vars = {
"slot_mapping": _FakeBuffer(4, fill=30017),
"ub0_slot_mapping": _FakeBuffer(2, fill=30021),
"ub1_slot_mapping": _FakeBuffer(2, fill=30022),
"context_lens": _FakeBuffer(4, fill=99),
"block_tables": _FakeBuffer(4, fill=1873),
# GDN registers bare tensors here too, so anything that scanned the
# dict and duck-typed on `.np` would trip over them.
"spec_sequence_masks": object(),
}
self._extra = list(extra_targets)

def cache_write_targets(self):
return AttentionMetadataBuilder.cache_write_targets(self) + self._extra

def blank_cache_write_targets(self):
return AttentionMetadataBuilder.blank_cache_write_targets(self)


def test_the_default_covers_the_base_buffer_and_every_ubatch_mirror():
"""TBO's capture reads per-ubatch copies rather than the base buffer."""
b = _Builder()
var = b.model_runner.forward_vars

found = AttentionMetadataBuilder.cache_write_targets(b)

assert {id(x) for x in found} == {
id(var["slot_mapping"]),
id(var["ub0_slot_mapping"]),
id(var["ub1_slot_mapping"]),
}


def test_the_default_leaves_the_read_side_alone():
"""The capture builders synthesize those; blanking them would cut across it."""
b = _Builder()
var = b.model_runner.forward_vars

found = AttentionMetadataBuilder.cache_write_targets(b)

assert var["context_lens"] not in found
assert var["block_tables"] not in found


def test_a_capture_writes_nowhere():
b = _Builder()

b.blank_cache_write_targets()

for name, buf in b.model_runner.forward_vars.items():
if name.endswith("slot_mapping"):
assert list(buf.np) == [PAD_SLOT_ID] * len(buf.np)


def test_the_device_half_is_what_a_capture_reads():
"""The builders hand the graph `.gpu`, so a host-only fill writes nothing."""
b = _Builder()
base = b.model_runner.forward_vars["slot_mapping"]

b.blank_cache_write_targets()

assert list(base.gpu) == [PAD_SLOT_ID] * 4
assert base.uploads == 1


def test_a_target_outside_forward_vars_is_covered():
"""GDN's read fork is allocated on the builder and never registered there,
yet its capture metadata hands the graph a live view of it."""
unregistered = _FakeBuffer(4, fill=7)

_Builder(extra_targets=[unregistered]).blank_cache_write_targets()

assert list(unregistered.gpu) == [PAD_SLOT_ID] * 4


@pytest.mark.parametrize("stale", [0, 1, 30017, 2**31])
def test_no_row_survives_whatever_the_last_batch_held(stale):
# 0 included on purpose: it is what a fresh `CpuGpuBuffer` holds, which is
# why a startup capture writes row 0 rather than nothing.
b = _Builder()
b.model_runner.forward_vars["slot_mapping"].np[:] = stale

b.blank_cache_write_targets()

assert (b.model_runner.forward_vars["slot_mapping"].gpu < 0).all()
78 changes: 76 additions & 2 deletions tests/test_rollout_memory_manager_sleep.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@
class _Runner(MemoryManagerMixin):
"""The surface `MemoryManagerMixin` documents, and nothing else."""

def __init__(self, *, enforce_eager, keep_resident, with_graphs=True):
def __init__(
self, *, enforce_eager, keep_resident, with_graphs=True, available_blocks=7
):
self.device = torch.device("cpu")
self.label = "test"
self.enforce_eager = enforce_eager
Expand All @@ -56,12 +58,18 @@ def __init__(self, *, enforce_eager, keep_resident, with_graphs=True):
self.tokenID_processor = SimpleNamespace(clean=lambda: None)
self.allocated_blocks = []
self.captures = 0
# What a fresh sizing would come back with, and how often one is asked
# for. Default equals the startup count so a re-derivation is invisible;
# pass less to stand in for a peer holding memory at wake time.
self._available_blocks = available_blocks
self.sizings = 0

def _get_models_with_kv(self):
return [self.model]

def get_num_blocks(self):
return {"num_kvcache_blocks": 7}
self.sizings += 1
return {"num_kvcache_blocks": self._available_blocks}

def allocate_kv_cache(self, num_blocks):
self.allocated_blocks.append(num_blocks)
Expand All @@ -77,6 +85,8 @@ def _no_gpu_calls(monkeypatch):
monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)
monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None)
monkeypatch.setattr(torch.cuda, "reset_peak_memory_stats", lambda *a, **k: None)
# `_resume_kv_cache` logs the headroom it is about to allocate into.
monkeypatch.setattr(torch.cuda, "mem_get_info", lambda *a, **k: (1 << 30, 2 << 30))
monkeypatch.setattr(memory_manager, "set_kv_cache_data", lambda _value: None)


Expand Down Expand Up @@ -182,6 +192,70 @@ def test_default_wake_recaptures_the_graphs_it_released():
assert not hasattr(runner, "_graphs_backup_keys")


def test_a_wake_allocates_the_size_it_slept_at_however_little_is_free():
"""The regression: a peer holding memory at wake time must not resize us.

A wake used to re-derive the count and `min()` it with the saved one, which
silently handed back a smaller pool. Two things outside this process are
built against the startup count and neither hears about the new one: the
decode graphs captured against the original pool, and -- the one that
corrupts -- `BlockManager`'s `BlockPool`, sized in the *engine* process from
the count `EngineCore.__init__` copied out of the startup reply. The
scheduler went on issuing block ids past the end of the reallocated pool.

Measured on Qwen3-30B-A3B: 33562 blocks at startup, 8316 on the first wake
with 176GB still free, then `Memory access fault by GPU node-N` on all eight
replicas.
"""
runner = _Runner(enforce_eager=False, keep_resident=False, available_blocks=3)

runner.release_memory(tags=["kv_cache"])
_report_weights_on_device(runner)
runner.resume_memory(tags=["kv_cache"])

assert runner.allocated_blocks == [7]


def test_a_wake_that_cannot_allocate_keeps_the_size_for_the_next_one():
"""Dropping the clamp made `allocate_kv_cache` the thing that raises.

`_kv_cache_num_blocks` is the only record of how big the pool was, so
clearing it before the allocation loses that record on the way out. The next
wake then takes the "No KV cache num_blocks to resume from" guard, returns,
and `resume_memory` reports success for an engine that has no pool at all --
which faults on the first forward instead of at the point of failure.
"""
runner = _Runner(enforce_eager=False, keep_resident=False)
runner.release_memory(tags=["kv_cache"])

def _oom(num_blocks):
raise torch.cuda.OutOfMemoryError("out of memory")

runner.allocate_kv_cache = _oom
with pytest.raises(torch.cuda.OutOfMemoryError):
runner._resume_kv_cache()

assert runner._kv_cache_num_blocks == 7
assert runner.kv_cache is None


def test_a_wake_does_not_size_the_pool_at_all():
"""Not just the same answer -- the question is not asked.

`resume_memory` broadcasts the command rather than a count, so every TP rank
ran its own sizing against its own `mem_get_info()`. Ranks could therefore
disagree with each other, and a block id valid on rank 0 be out of bounds on
rank 3. One `get_num_blocks` at startup is what makes them agree.
"""
runner = _Runner(enforce_eager=False, keep_resident=False)

runner.release_memory(tags=["kv_cache"])
_report_weights_on_device(runner)
runner.resume_memory(tags=["kv_cache"])

assert runner.sizings == 0


def test_releasing_only_the_kv_pool_still_invalidates_the_graphs():
"""`AsyncLLMEngine.sleep(level=1)`, the default, frees the pool and nothing else.

Expand Down
Loading