From 5177a5c8f05648009d13aa0aec81028bf20bd6f0 Mon Sep 17 00:00:00 2001 From: Lingpeng Jin <103567126+valarLip@users.noreply.github.com> Date: Sun, 23 Aug 2026 09:37:15 +0000 Subject: [PATCH 1/5] fix(engine): a fourth consumer of token ids, and a guard for the next one `Sequence.token_ids` became an `array("i")` in #1989. That commit named three places that compare token ids and fixed each. Sweeping the rest of the chain turned up a fourth consumer of a different kind -- one that serializes rather than compares -- and it fails silently too. `publish_loaded_prefix` hands `_hash_block_tokens`' output straight into a `BlockStored` event, which is msgpack-encoded. msgspec has no encoding for `array.array`, and `KVEventPublisher.publish` counts encode failures rather than raising, so with KV events enabled the whole event stream goes dark after one log line. The other two BlockStored sites accumulate into a list via `extend` and were never affected. Rather than convert at each site, the two boundaries where the type is load-bearing now assert it: - `Block.update` takes an array and refuses a list. A list there costs twice and neither cost announces itself: it never compares equal to the arrays the other publish paths store, so every hit on that block reads as a collision; and it hands the collector one traversal slot per token. - `_make_block_stored` takes a list and refuses an array, for the encoder. The rest of the chain follows the same axis: everything that grows once per token is an array now. `Sequence.output_tokens` (the completion half of `token_ids`, kept in step with it by every writer), `Sequence.logprobs`, the API server's three per-request accumulators, the streaming detokenizer's buffer, and atomesh's two. `tokenizer.decode` takes an array as is, so only the JSON edges convert back -- and only `LLMEngine`'s, because the HTTP response builders read `text` and the counters and never `token_ids`. A test pins that, so a builder that starts forwarding the key fails there rather than in production. Four annotations said `list[int]` where an array now flows. Measured on a live V4-Flash-DSpark tp1 (94,681 blocks) with a `gc.callbacks` probe: the EngineCore's stop-the-world gen-2 pause is 288.6ms with list storage and 242.8ms with arrays, and with arrays it stops growing as the prefix cache fills. Throughput is unchanged at this scale -- gen-2 fires about once per seven minutes in steady state here, so the pause matters at the concurrency #1980 measured at, not at this one. --- .../atomesh/atom_standalone_service.py | 8 +- atom/entrypoints/openai/api_server.py | 13 +- atom/entrypoints/openai/streaming_dispatch.py | 8 +- atom/model_engine/block_manager.py | 17 +- atom/model_engine/block_pool.py | 9 +- atom/model_engine/kv_block.py | 22 ++- atom/model_engine/llm_engine.py | 3 +- atom/model_engine/sequence.py | 10 +- tests/entrypoints/test_streaming_dispatch.py | 7 +- tests/test_block_pool.py | 19 +- tests/test_scheduler.py | 2 +- tests/test_sequence.py | 2 +- tests/test_token_ids_storage.py | 165 +++++++++++++++++- 13 files changed, 250 insertions(+), 35 deletions(-) diff --git a/atom/entrypoints/atomesh/atom_standalone_service.py b/atom/entrypoints/atomesh/atom_standalone_service.py index 2cb84aea29..a38e48b1ec 100644 --- a/atom/entrypoints/atomesh/atom_standalone_service.py +++ b/atom/entrypoints/atomesh/atom_standalone_service.py @@ -46,6 +46,7 @@ ) from atom.entrypoints.openai.sse import data_frame from atom.entrypoints.openai.tool_parser import ToolCallStreamParser +from atom.model_engine.sequence import new_token_ids logger = logging.getLogger("atom") @@ -331,7 +332,10 @@ def __init__( self.started_at = time.time() self.first_token_at: float | None = None self.last_token_at: float | None = None - self.token_ids: list[int] = [] + # An array for the same reason the engine's `token_ids` is one. It + # stays one: this dict goes to `build_*_response`, which reads + # `text` and the counters and never `token_ids`. + self.token_ids = new_token_ids() self.finish_reason: str | None = None self.num_tokens_input = 0 self.kv_transfer_output_meta_info: Any = None @@ -400,7 +404,7 @@ def __init__( self.future = future self.n = n self.started_at = time.time() - self.per_tokens: list[list[int]] = [[] for _ in range(n)] + self.per_tokens = [new_token_ids() for _ in range(n)] self.per_first_token_at: list[float | None] = [None] * n self.per_last_token_at: list[float | None] = [None] * n self.per_finish_reason: list[str | None] = [None] * n diff --git a/atom/entrypoints/openai/api_server.py b/atom/entrypoints/openai/api_server.py index 0026720148..239d2158be 100644 --- a/atom/entrypoints/openai/api_server.py +++ b/atom/entrypoints/openai/api_server.py @@ -38,6 +38,7 @@ from atom.model_engine.llm_engine import _load_tokenizer from atom.model_engine.multimodal import build_multimodal_inputs from atom.model_engine.request import RequestOutput +from atom.model_engine.sequence import new_token_ids from atom.utils import tune_gc from atom.utils.arg_parser import FlexibleArgumentParser @@ -475,7 +476,13 @@ async def generate_async( started_at = time.time() first_token_at: float | None = None last_token_at: float | None = None - all_token_ids: list[int] = [] + # An array, not a list: this grows for the whole life of the request, + # and one boxed PyInt per token is what the collector then walks on + # every pass. It stays an array all the way out -- the dict below is + # an internal hand-off to `build_*_response`, which reads `text` and + # the counters and never `token_ids`, so nothing serializes it. A + # consumer that starts reading that key has to convert. + all_token_ids = new_token_ids() finish_reason: str | None = None seq = None kv_transfer_output_meta_info = None @@ -594,7 +601,7 @@ async def generate_async_multimodal( started_at = time.time() first_token_at: float | None = None last_token_at: float | None = None - all_token_ids: list[int] = [] + all_token_ids = new_token_ids() finish_reason: str | None = None seq = None @@ -700,7 +707,7 @@ async def generate_async_fanout( loop = asyncio.get_running_loop() started_at = time.time() - per_tokens: list[list[int]] = [[] for _ in range(n)] + per_tokens = [new_token_ids() for _ in range(n)] per_first_token_at: list[float | None] = [None] * n per_last_token_at: list[float | None] = [None] * n per_finish_reason: list[str | None] = [None] * n diff --git a/atom/entrypoints/openai/streaming_dispatch.py b/atom/entrypoints/openai/streaming_dispatch.py index 900efd5b82..43262ee908 100644 --- a/atom/entrypoints/openai/streaming_dispatch.py +++ b/atom/entrypoints/openai/streaming_dispatch.py @@ -9,11 +9,14 @@ landing point each stream's SSE generator reads from. """ +import array import threading from asyncio import AbstractEventLoop, Event from dataclasses import dataclass, field from typing import Any, NamedTuple +from atom.model_engine.sequence import new_token_ids + # Fields a later chunk overrides on the one it merges into, when it has a value # of its own. The SSE consumers keep the newest non-empty value they see, so # merging this way hands them what reading each chunk separately would have. @@ -25,7 +28,10 @@ class IncrementalStreamDetokenizer: """Decode token deltas without emitting incomplete UTF-8 characters.""" tokenizer: Any - tokens: list[int] = field(default_factory=list) + # Grows for the whole life of a stream, one entry per token, and only ever + # sliced into `tokenizer.decode` -- which takes an array. Nothing here is + # serialized, so this one has no boundary to convert back at. + tokens: array.array = field(default_factory=new_token_ids) prefix_offset: int = 0 read_offset: int = 0 diff --git a/atom/model_engine/block_manager.py b/atom/model_engine/block_manager.py index 07a89249e9..26966bfaee 100644 --- a/atom/model_engine/block_manager.py +++ b/atom/model_engine/block_manager.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: MIT # Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +import array import logging from math import inf, isinf @@ -40,6 +41,13 @@ def _make_block_stored( medium: str = MEDIUM_GPU, ) -> BlockStored: """Construct a BlockStored event from a coalesced run of new blocks.""" + # A list, not the `array("i")` the publish paths carry: the event is + # msgpack-encoded and msgspec has no encoding for an array. The publisher + # counts encode failures rather than raising, so an array here takes the + # event stream down without stopping anything. + assert isinstance( + tokens, list + ), f"BlockStored.token_ids must be a list, got {type(tokens).__name__}" return BlockStored( block_hashes=hashes, parent_block_hash=parent, @@ -179,7 +187,7 @@ def __init__( self.demands_declined_no_room: int = 0 @classmethod - def compute_hash(cls, token_ids: list[int], prefix: int = -1): + def compute_hash(cls, token_ids: list[int] | array.array, prefix: int = -1): h = xxhash.xxh64() if prefix != -1: h.update(prefix.to_bytes(8, "little")) @@ -308,7 +316,7 @@ def _n_hash_blocks(self, seq: Sequence) -> int: hbs = self.hash_block_size return (len(seq) + hbs - 1) // hbs - def _hash_block_tokens(self, seq: Sequence, i: int) -> list[int]: + def _hash_block_tokens(self, seq: Sequence, i: int) -> array.array: hbs = self.hash_block_size return seq.token_ids[i * hbs : (i + 1) * hbs] @@ -1043,7 +1051,10 @@ def publish_loaded_prefix( self._event_log.append( _make_block_stored( [block_hash], - token_ids, + # The only BlockStored site fed straight from + # `_hash_block_tokens`; the other two accumulate + # into a list already. See `_make_block_stored`. + list(token_ids), parent_hash if parent_hash != -1 else None, self.block_size, ) diff --git a/atom/model_engine/block_pool.py b/atom/model_engine/block_pool.py index 326b68ed1b..8afbb8bfbc 100644 --- a/atom/model_engine/block_pool.py +++ b/atom/model_engine/block_pool.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: MIT # Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +import array from collections import OrderedDict from collections.abc import Callable, Hashable, Iterable from dataclasses import dataclass @@ -115,7 +116,7 @@ def lookup(self, h: int) -> int: """Block id indexed under content hash `h`, or -1.""" return self._hash_to_block_id.get(h, -1) - def publish(self, block_id: int, h: int, token_ids: list[int]) -> None: + def publish(self, block_id: int, h: int, token_ids: array.array) -> None: """Index `block_id` under the content hash of the tokens it now holds.""" block = self.blocks[block_id] block.update(h, token_ids) @@ -131,7 +132,7 @@ def clear_index(self) -> None: for block in self.blocks: if block.ref_count == 0: block.hash = -1 - block.token_ids = [] + block.token_ids = array.array("i") # Every free block is vacant now, and which container an id is in is # only ever decided from its hash — so the split has to be redrawn # here rather than left to drift. @@ -147,7 +148,7 @@ def _unindex(self, block_id: int) -> None: if self._on_evict is not None: self._on_evict(block.hash) block.hash = -1 - block.token_ids = [] + block.token_ids = array.array("i") # ---------------------------- allocation ------------------------------- # def _take_free(self) -> int: @@ -331,4 +332,4 @@ def _adopt(self, destination: int, source: int) -> None: self._hash_to_block_id[src.hash] = destination self._used.discard(source) self._used.add(destination) - src.ref_count, src.hash, src.token_ids = 0, -1, [] + src.ref_count, src.hash, src.token_ids = 0, -1, array.array("i") diff --git a/atom/model_engine/kv_block.py b/atom/model_engine/kv_block.py index 7c64d0a271..e8af6cf899 100644 --- a/atom/model_engine/kv_block.py +++ b/atom/model_engine/kv_block.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: MIT # Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +import array + class Block: """One physical KV cache block of the compressed pool (BlockManager). @@ -12,16 +14,28 @@ def __init__(self, block_id): self.block_id = block_id self.ref_count = 0 self.hash = -1 - self.token_ids = [] - - def update(self, hash: int, token_ids: list[int]): + # A slice of `Sequence.token_ids`, kept only so a hash hit can be + # checked against the tokens it claims to stand for. See `update` for + # why the type is pinned. + self.token_ids: array.array = array.array("i") + + def update(self, hash: int, token_ids: array.array): + # Pinned rather than accepted as either: a list here fails twice and + # neither failure says so. It never compares equal to the `array("i")` + # the other publish paths store, so every hit on this block reads as a + # collision; and a list of ids is one traversal slot per token for the + # collector, which across a full pool is a stop-the-world pause of + # ~200ms per gen-2 pass against ~5ms for arrays. + assert isinstance( + token_ids, array.array + ), f"Block.token_ids must be an array('i'), got {type(token_ids).__name__}" self.hash = hash self.token_ids = token_ids def reset(self): self.ref_count = 1 self.hash = -1 - self.token_ids = [] + self.token_ids = array.array("i") # Name of the sub-pool sizing class backing the per-request state slots diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index de59199a17..0222bb6eba 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -754,7 +754,8 @@ def postprocess(self, reqs: list[Sequence]): # `list`, not the `array("i")` slice: this is what `generate()` # hands a caller, and the storage type is ours to change. "token_ids": list(req.completion_token_ids), - "logprobs": req.logprobs if req.return_logprobs else None, + # `list` for the same reason as `token_ids` above. + "logprobs": list(req.logprobs) if req.return_logprobs else None, "latency": req.leave_time - req.arrive_time, "finish_reason": req.leave_reason, "num_tokens_input": req.num_prompt_tokens, diff --git a/atom/model_engine/sequence.py b/atom/model_engine/sequence.py index 65d4793fab..438a36626b 100644 --- a/atom/model_engine/sequence.py +++ b/atom/model_engine/sequence.py @@ -190,10 +190,16 @@ def __init__( # would never fire for the prefill blocks; this flag does. self.prefix_hashes_published = False self.return_logprobs = bool(getattr(sampling_params, "logprobs", False)) - self.logprobs: list[float] = [] + # One entry per completion token, so the same reason `token_ids` is an + # array applies: a list would box a PyFloat per token and hand the + # collector a slot to walk for each one. `json.dumps` is the only + # consumer that needs a list, and it converts at its own boundary. + self.logprobs: array.array = array.array("d") # stream callback self.stream_callback = stream_callback - self.output_tokens = [] # cache for newly generate tokens + # The completion half of `token_ids`, kept in step with it by every + # writer, so it is the same array type for the same reasons. + self.output_tokens = new_token_ids() # Placeholders from previous postprocess; overwritten in place. self.num_placeholder_tokens: int = 0 diff --git a/tests/entrypoints/test_streaming_dispatch.py b/tests/entrypoints/test_streaming_dispatch.py index 8d6e5abf82..3f5cbf1017 100644 --- a/tests/entrypoints/test_streaming_dispatch.py +++ b/tests/entrypoints/test_streaming_dispatch.py @@ -10,7 +10,10 @@ class _Utf8ByteTokenizer: def decode(self, token_ids, skip_special_tokens=True): - return bytes(token_ids).decode("utf-8", errors="replace") + # `bytes(x)` of an `array("i")` copies its buffer -- four bytes per id + # -- where from a list it takes the values. A real tokenizer reads ids, + # so this double has to as well; keep the `list`. + return bytes(list(token_ids)).decode("utf-8", errors="replace") class _ImmediateLoop: @@ -378,4 +381,4 @@ def test_each_stream_gets_its_own_detokenizer(): first, second = dispatcher.new_state(), dispatcher.new_state() assert first is not second - assert first.tokens == [] and second.tokens == [] + assert not first.tokens and not second.tokens diff --git a/tests/test_block_pool.py b/tests/test_block_pool.py index 8330d63710..1e07c7eb58 100644 --- a/tests/test_block_pool.py +++ b/tests/test_block_pool.py @@ -10,15 +10,26 @@ compress/state boundary does. """ +import array + import pytest from atom.model_engine.block_pool import BlockPool +def toks(*ids: int) -> array.array: + """What the block manager publishes: a slice of `Sequence.token_ids`. + + A list here is what `Block.update` refuses, so the tests have to hold the + production type or they stop covering the comparison the pool relies on. + """ + return array.array("i", ids) + + def published(pool: BlockPool, block_id: int, h: int) -> int: """Allocate, publish under `h`, and release — a cached, free block.""" pool.allocate(block_id) - pool.publish(block_id, h, [h]) + pool.publish(block_id, h, toks(h)) pool.free(block_id) return block_id @@ -81,7 +92,7 @@ def test_a_cached_top_loses_its_content_and_says_so(self): def test_a_held_top_moves_and_keeps_its_identity(self): pool = BlockPool(num_blocks=3) pool.allocate(2) - pool.publish(2, 100, [7, 8]) + pool.publish(2, 100, toks(7, 8)) pool.claim(2) # a second holder, so the ref count has to travel too retirement = pool.retire_top() @@ -89,7 +100,7 @@ def test_a_held_top_moves_and_keeps_its_identity(self): assert 0 <= retirement.moved_to < 2 moved = pool.block(retirement.moved_to) - assert (moved.hash, moved.token_ids, moved.ref_count) == (100, [7, 8], 2) + assert (moved.hash, moved.token_ids, moved.ref_count) == (100, toks(7, 8), 2) assert pool.lookup(100) == retirement.moved_to assert pool.is_used(retirement.moved_to) assert not pool.is_used(2) @@ -101,7 +112,7 @@ def test_moving_evicts_whatever_the_destination_held(self): published(pool, 0, h=100) published(pool, 1, h=200) pool.allocate(2) - pool.publish(2, 300, [3]) + pool.publish(2, 300, toks(3)) retirement = pool.retire_top() # The destination is a cached block, so its content is the price. diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index b218e13856..4b8c1837ff 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -562,7 +562,7 @@ def test_offload_partial_handoff_keeps_resumed_deferred_output(self, seq_factory ) assert seq.num_cached_tokens == 10 - assert seq.output_tokens == [999, sched.eos_token_id] + assert list(seq.output_tokens) == [999, sched.eos_token_id] # ── _waiting_new_token_count (PrefillDelayer queue signal) ───────────────── diff --git a/tests/test_sequence.py b/tests/test_sequence.py index fb4427fbb3..676b0069b1 100644 --- a/tests/test_sequence.py +++ b/tests/test_sequence.py @@ -79,7 +79,7 @@ def test_append_tracks_output_tokens(self, seq_factory): seq = seq_factory([1, 2]) seq.append_token(3) seq.append_token(4) - assert seq.output_tokens == [3, 4] + assert list(seq.output_tokens) == [3, 4] def test_append_crosses_block_boundary(self, seq_factory): seq = seq_factory([1, 2, 3, 4], block_size=4) diff --git a/tests/test_token_ids_storage.py b/tests/test_token_ids_storage.py index 8767453621..6c1c71d8e6 100644 --- a/tests/test_token_ids_storage.py +++ b/tests/test_token_ids_storage.py @@ -6,22 +6,42 @@ `scheduled_tokens` becomes a memcpy instead of one CPython int unboxing per token, and a 100k-token prompt costs 0.38 MiB instead of 3.4. -What it risks is narrower and entirely one shape: an `array("i")` never -compares equal to a list. Three places compare token ids, and in every one a -mismatch is silent -- a stop sequence that stops nothing, a prefix cache that -never hits, a hash that changes with its argument's Python type. Each is -pinned here with a positive control, because a test that only checked the -happy path would pass just as well against the versions that were broken. +What it risks is one axis, not one bug: every consumer that was written +against a list. Two kinds, and a silent failure in both -- + + compared against a list an `array("i")` never equals one, so the answer is + always "different": a stop sequence that stops + nothing, a prefix cache that never hits, a hash + that changes with its argument's Python type. + + serialized as a list msgspec has no encoding for an array, and the KV + event publisher counts encode failures instead of + raising -- the whole event stream goes dark. + +Each is pinned here with a positive control, because a test that only checked +the happy path would pass just as well against the versions that were broken. +The two boundaries where the type is load-bearing assert it rather than accept +either, so a consumer drifting back to a list fails loudly at the boundary +instead of quietly downstream. """ from __future__ import annotations import array +import gc +import json +from types import SimpleNamespace +import msgspec import numpy as np import pytest +from conftest import MockConfig -from atom.model_engine.block_manager import BlockManager +from atom.distributed.kv_events import BlockStored +from atom.entrypoints.openai.serving_chat import build_chat_response +from atom.entrypoints.openai.serving_completion import build_completion_response +from atom.model_engine.block_manager import BlockManager, _make_block_stored +from atom.model_engine.kv_block import Block from atom.model_engine.sequence import Sequence, new_token_ids from atom.sampling_params import SamplingParams @@ -32,6 +52,26 @@ def _seq(token_ids, **kw): ) +def test_every_per_token_field_on_a_sequence_is_an_array(): + """Named one by one rather than checked as a class, because the point is + the list of fields: anything growing once per token belongs here, and a + new one added as a list is the regression this catches.""" + seq = _seq([1, 2, 3]) + seq.append_token(4) + fields = { + "token_ids": "i", + "output_tokens": "i", # the completion half of token_ids + "block_table": "i", + "logprobs": "d", # floats, same boxing cost per token + } + for name, typecode in fields.items(): + held = getattr(seq, name) + assert isinstance(held, array.array), f"{name} is a {type(held).__name__}" + assert held.typecode == typecode, name + for stop in _seq([1], stop_token_sequences=[[1]]).stop_token_sequences: + assert isinstance(stop, array.array) + + def test_storage_is_int32_and_holds_what_a_list_held(): seq = _seq([0, 1, 129279, -1]) assert isinstance(seq.token_ids, array.array) @@ -101,6 +141,117 @@ def test_a_block_and_the_slice_it_is_compared_against_share_a_type(): assert published != list(fresh) # control: the mismatch is real +# --- the two boundaries that refuse the wrong type ----------------------- + + +def test_a_block_refuses_to_store_a_list(): + """`Block.token_ids` is what a hash hit is verified against, and the pool + holds one per block -- 94602 of them on a V4-Flash-DSpark tp1. A list costs + twice: the verify always reads as a collision, and the collector walks one + slot per token on every gen-2 pass, 207.7ms of stop-the-world against 5.6 + at that pool size (`/app/logs_claude/o27_branch_gc_heap.py`).""" + block = Block(0) + block.update(7, new_token_ids([1, 2, 3])) + assert block.token_ids == new_token_ids([1, 2, 3]) + + with pytest.raises(AssertionError, match="must be an array"): + block.update(7, [1, 2, 3]) + + # Control: the two costs, neither of which announces itself. The second is + # `gc.get_referents`, the traversal a collection performs -- a list is one + # visit per token, an array is a fixed visit however many it holds. + assert block.token_ids != [1, 2, 3] + assert [len(gc.get_referents(list(range(n)))) for n in (3, 300)] == [3, 300] + assert len({len(gc.get_referents(new_token_ids(range(n)))) for n in (3, 300)}) == 1 + + +def test_a_kv_event_refuses_to_carry_an_array(): + """`BlockStored` is msgpack-encoded, and `KVEventPublisher.publish` counts + encode errors rather than raising -- so an array reaches the wire as + nothing at all.""" + tokens = [1, 2, 3] + event = _make_block_stored([100], tokens, None, block_size=4) + assert msgspec.msgpack.Encoder().encode(event) + + with pytest.raises(AssertionError, match="must be a list"): + _make_block_stored([100], new_token_ids(tokens), None, block_size=4) + + # Control: the encoder is what the assert is standing in front of. + with pytest.raises(TypeError, match="array"): + msgspec.msgpack.Encoder().encode( + BlockStored( + block_hashes=[100], + parent_block_hash=None, + token_ids=new_token_ids(tokens), + block_size=4, + ) + ) + + +def test_every_event_a_publish_path_emits_reaches_the_wire(seq_factory): + """The boundary assert only fires on a path a test actually walks. + + `publish_loaded_prefix` is the one BlockStored site fed straight from + `_hash_block_tokens`, and it emits nothing unless KV events are on -- so + the suite drove it for its return value and never looked at what it put in + the log. Encoding the whole batch is what makes the type load-bearing. + """ + cfg = MockConfig( + num_kvcache_blocks=16, + kv_cache_block_size=4, + enable_prefix_caching=True, + kv_events_config=SimpleNamespace(enable=True), + ) + bm = BlockManager(cfg) + assert bm._event_log is not None, "events did not turn on; the rest is vacuous" + + # Loaded first and on tokens nothing has indexed yet: with a canonical + # mapping already in place this takes the branch that reuses it and emits + # nothing at all, which is how the path stayed uncovered. + loaded = seq_factory(list(range(16))) + bm.allocate(loaded) + assert bm.publish_loaded_prefix(loaded, start_token=0, end_token=16) == 16 + + hashed = seq_factory(list(range(100, 116))) + bm.allocate(hashed) + bm.hash_blocks(hashed, 16, start_tokens=0) + + stored = [e for e in bm._event_log if isinstance(e, BlockStored)] + assert len(stored) >= 2, "a publish path emitted nothing; nothing was checked" + encoder = msgspec.msgpack.Encoder() + for event in bm._event_log: + assert encoder.encode(event) + + +def test_a_response_builder_never_puts_token_ids_on_the_wire(): + """Why the API server's accumulators need no conversion back. + + `generate_async` hands its dict to `build_*_response`, which reads `text`, + the finish reason and the counters. `token_ids` is not among them, so the + array never meets a serializer -- and the accumulator can stay an array + for the whole request instead of holding one boxed PyInt per token. + + Passing an array in and serializing the result is what pins that: a + builder that starts forwarding the key fails here rather than in + production, where it would be a 500 on a response that used to work. + """ + final_output = { + "text": "hi", + "token_ids": new_token_ids([1, 2, 3]), + "finish_reason": "stop", + "num_tokens_input": 4, + "num_tokens_output": 3, + } + chat = build_chat_response("id", "m", final_output["text"], dict(final_output)) + completion = build_completion_response("id", "m", dict(final_output)) + for response in (chat, completion): + assert json.loads(response.model_dump_json())["choices"][0] + + # Control: the failure the assertion above stands in for. + with pytest.raises(TypeError, match="array"): + json.dumps(final_output) + + # --- what the change is for ---------------------------------------------- From 6e8025459b0a4e5e9402b7125f229df289f3f659 Mon Sep 17 00:00:00 2001 From: Lingpeng Jin <103567126+valarLip@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:45:49 +0000 Subject: [PATCH 2/5] fix(engine): two more token-id annotations that said list `compute_hash` was annotated `list[int] | array.array` on the reasoning that one caller still passed a list. It does not. All six production call sites pass an `array("i")`: - `block_manager.py` x5, from `_hash_block_tokens` / `seq.block(i)`, both slices of `Sequence.token_ids` - `policy.py`, from `_chained_prefix_hashes(seq.token_ids, ...)` The union came from reading `_chained_prefix_hashes(token_ids: list[int], ...)` and taking the signature for the fact -- but that annotation is stale in the same way, since `connector.py` hands it `seq.token_ids`. Both are narrowed to what actually arrives. The int64 pin stays. Its reason changes rather than disappearing: nothing passes a list today, so it is no longer reconciling two live callers, but it is what stops a caller who does pass one from silently computing a different digest for the same tokens. Only tests pass lists now, and `test_compute_hash_does_not_depend_on_its_ argument_type` keeps asserting the two agree -- the annotation says what callers should pass, that test says the function is robust if they do not. --- atom/kv_transfer/offload/hybrid/dsv4/policy.py | 3 ++- atom/model_engine/block_manager.py | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/atom/kv_transfer/offload/hybrid/dsv4/policy.py b/atom/kv_transfer/offload/hybrid/dsv4/policy.py index f6c5e4c2d2..f3e019757c 100644 --- a/atom/kv_transfer/offload/hybrid/dsv4/policy.py +++ b/atom/kv_transfer/offload/hybrid/dsv4/policy.py @@ -10,6 +10,7 @@ from __future__ import annotations +import array import hashlib import heapq import json @@ -290,7 +291,7 @@ def _committed_sidecar_capacity(kvc) -> int: def _chained_prefix_hashes( - token_ids: list[int], + token_ids: array.array, hash_block_size: int, ) -> dict[int, int]: """Return each full-block prefix hash using BlockManager's exact chain.""" diff --git a/atom/model_engine/block_manager.py b/atom/model_engine/block_manager.py index 26966bfaee..0479034c41 100644 --- a/atom/model_engine/block_manager.py +++ b/atom/model_engine/block_manager.py @@ -187,14 +187,16 @@ def __init__( self.demands_declined_no_room: int = 0 @classmethod - def compute_hash(cls, token_ids: list[int] | array.array, prefix: int = -1): + def compute_hash(cls, token_ids: array.array, prefix: int = -1): h = xxhash.xxh64() if prefix != -1: h.update(prefix.to_bytes(8, "little")) - # dtype pinned: `np.array` infers int64 from a list and int32 from an - # `array("i")`, so the digest used to depend on the caller's Python - # type. int64 is what lists gave, so pinning it leaves every existing - # hash where it was. + # dtype pinned even though every caller now passes an `array("i")`: + # `np.array` infers int64 from a list and int32 from an array, so the + # digest used to depend on the caller's Python type. int64 is what + # lists gave, which leaves every hash recorded before that where it + # was -- and keeps a caller who does pass a list from silently + # computing a different one. h.update(np.asarray(token_ids, dtype=np.int64).tobytes()) return h.intdigest() From 49c86ece043755f20741cb0b1d0f60a625177c36 Mon Sep 17 00:00:00 2001 From: Lingpeng Jin <103567126+valarLip@users.noreply.github.com> Date: Sun, 23 Aug 2026 19:24:28 +0800 Subject: [PATCH 3/5] perf(engine): stop the collector walking a heap it never reclaims (#1991) * perf(engine): stop the collector walking a heap it never reclaims At steady state the garbage collector in these processes finds nothing. Measured on DeepSeek-V4-Flash-DSpark tp1 with a `gc.callbacks` probe: over 1754 gen-0, 158 gen-1 and 50 gen-2 passes after startup, it reclaimed **zero** objects. Everything the hot loop allocates is acyclic and reference counting takes it. What each pass does cost, being stop-the-world and proportional to the live heap: EngineCore 268 ms of 819,923 tracked TP0 worker 979 ms of 959,694 api_server 265 ms of 632,424 Almost all of that heap is startup state -- model, compiled graph, tokenizer, KV block pool -- so `gc.freeze()` after warmup removes the work rather than merely spacing it out, which is all raising thresholds (#1980) can do. With it on, gen-2 stops firing entirely in all three processes for the rest of the run, while gen-0/1 keep going at under 2 ms: this narrows the collector, it does not disable it, so a cycle written by later code is still caught. `unfreeze_gc_heap` on shutdown is not optional. A frozen object is invisible to the collector, so an engine destroyed inside a live interpreter -- a test, an RL loop that rebuilds it -- would leave its weights and KV cache unreachable *and* uncollectable, which presents as a GPU memory leak with nothing about GC in the symptom. vLLM hit this and handles it the same way. Placement is one call per process at the boundary where startup ends: - api_server, in the lifespan just before it yields - EngineCore, at the end of `__init__` -- KV cache allocated, graphs captured, BlockPool built (94,763 `Block` objects on a V4-Flash tp1) - each ModelRunner worker, over an RPC the EngineCore sends, because only the caller knows warmup is over: weights, compile and capture all arrive as RPCs it issues, and `--enforce-eager` has no capture step to hook Disaggregated decode is the exception and is left partly covered. Its block pool is built in `_init_disagg`, after `super().__init__()` has already sent READY, so a freeze there could make queued requests permanent. It keeps the base freeze and forgoes the pool. `tune_gc` moves into the new module and stays as the fallback for `ATOM_GC_FREEZE=0`. `ATOM_GC_DEBUG` is added alongside: it logs every collection's generation, duration and reclaimed count, which is the only way to see these pauses -- a stall in the EngineCore idles the workers, and an idle worker emits no trace event at all. It is expensive (~90s of extra startup) and off by default. Two gaps found next door and closed, since the fix needed a shared per-process setup anyway: `PrefillEngineCore.run_engine` and `DecodeEngineCore.run_engine` override the base without calling it, so a disaggregated deployment got neither `enable_orphan_reaping` nor `tune_gc`, and neither ever set a process title -- both showed as bare `python` in `ps`. `_setup_engine_process` now owns identity, reaping and GC policy for all three, which also collapses the base's three `set_process_title` branches into one and makes the process title and the GC log lines agree by construction. Throughput is unchanged at this scale (32098 vs 32122 tok/s, and 32098 with freezing off), because gen-2 fires roughly once per seven minutes in steady state here. This is a tail-latency change; it pays at the concurrency #1980 measured at, which is not reproduced here. * fix(engine): one name per worker process, not three The worker side had the drift the EngineCore side just lost. Three places produced its name and only one knew about data parallelism: - `set_process_title` in `AsyncIOProc.__init__`, dp-aware (`DP0TP0`) - the GC debug callback, `f"TP{rank}"` - `ModelRunner.freeze_gc_heap`, `f"TP{rank}"` So under dp>1 every rank's worker logged as `TP0`, which is exactly the case worth telling apart -- and `ps` disagreed with both log lines. `engine_process_name` and `worker_process_name` now live next to `set_process_title`, since naming a process is what that function is for, and all five call sites take their string from them. `AsyncIOProc` names itself once, before anything logs, which also retires a `try/except Exception` that existed only to repeat the fallback the helper now expresses directly. `EngineCore._process_name_for` moves out to join them rather than staying a method, so a reader asking "what are these processes called" finds one answer. * fix(entrypoints): the atomesh frontend was the third process to miss this Enumerating which processes get the GC policy has now gone stale twice. #1980 reached the API server but not the disaggregated EngineCores, whose `run_engine` does not call the base one. The first version of this branch reached those but not atomesh, which builds its own engine in `launch_atom_standalone` and never runs the FastAPI lifespan the API server applies the policy from -- so that frontend had neither `tune_gc` (since #1980) nor the freeze, while the EngineCore and workers it spawns had both. It is the API server's counterpart, with the same profile: the tokenizer, the per-request accumulators, ~632k tracked objects and a 265 ms gen-2 pause. Rather than fix the third instance and wait for a fourth, the rule is now tested: `test_every_serving_frontend_applies_the_gc_policy` walks every module that builds an engine and serves, and fails naming the ones that do not apply the policy. Written the obvious way it was vacuous -- an unused import satisfies a search for the name -- so it requires the call. The prose enumerations in `tune_gc` and the env-var docs are replaced by the rule they kept failing to track. * fix(engine): restore the decode freeze, dropped on a misreading An earlier commit on this branch removed `_freeze_after_startup` from `DecodeEngineCore` on the grounds that READY had already gone out, so freezing could catch requests the input thread had queued and make them permanent. That is not what the code does. `DecodeEngineCore.__init__` sets `_ready_deferred = True` before calling `super().__init__()`, precisely so the base's ready signal is suppressed; the real one is sent nine lines *after* the block pool is built, once the kvcache IPC import and graph capture are done. Nothing can have been admitted at that point, so there is no window and the freeze is safe -- and it is what covers decode's 94,763 `Block` objects. The comment left in its place asserted the false premise in so many words, which would have taught the next reader the same wrong thing. --- atom/entrypoints/atomesh/server.py | 13 ++ atom/entrypoints/openai/api_server.py | 10 +- atom/model_engine/async_proc.py | 23 ++- atom/model_engine/engine_core.py | 79 ++++++--- atom/model_engine/model_runner.py | 11 ++ atom/utils/__init__.py | 56 +++---- atom/utils/envs.py | 12 +- atom/utils/gc_utils.py | 121 ++++++++++++++ docs/environment_variables.md | 15 ++ tests/test_gc_utils.py | 226 ++++++++++++++++++++++++++ 10 files changed, 492 insertions(+), 74 deletions(-) create mode 100644 atom/utils/gc_utils.py create mode 100644 tests/test_gc_utils.py diff --git a/atom/entrypoints/atomesh/server.py b/atom/entrypoints/atomesh/server.py index 3c3ce3630b..7ae2216249 100644 --- a/atom/entrypoints/atomesh/server.py +++ b/atom/entrypoints/atomesh/server.py @@ -17,6 +17,12 @@ import sys from typing import Any +from atom.utils.gc_utils import ( + freeze_gc_heap, + maybe_attach_gc_debug_callback, + tune_gc, +) + logger = logging.getLogger("atom") engine: Any | None = None @@ -186,6 +192,13 @@ def launch_atom_standalone(atomesh_runner: Any, raw_args: list[str]) -> None: standalone_args.default_chat_template_kwargs, ) + # This frontend is the api_server's counterpart -- same tokenizer, same + # per-request accumulators -- but it builds its engine itself and never + # runs that FastAPI lifespan, so it has to apply the GC policy here. + tune_gc() + maybe_attach_gc_debug_callback("atomesh") + freeze_gc_heap("atomesh") + print("\033[32mATOM starting...\033[0m") print(f"\033[32mHost: {cli_args['host']}:{cli_args['port']}\033[0m") atomesh_runner.launch_mesh( diff --git a/atom/entrypoints/openai/api_server.py b/atom/entrypoints/openai/api_server.py index 239d2158be..7ac5d825ad 100644 --- a/atom/entrypoints/openai/api_server.py +++ b/atom/entrypoints/openai/api_server.py @@ -39,8 +39,12 @@ from atom.model_engine.multimodal import build_multimodal_inputs from atom.model_engine.request import RequestOutput from atom.model_engine.sequence import new_token_ids -from atom.utils import tune_gc from atom.utils.arg_parser import FlexibleArgumentParser +from atom.utils.gc_utils import ( + freeze_gc_heap, + maybe_attach_gc_debug_callback, + tune_gc, +) from .chat_encoders import apply_chat_template, load_custom_message_encoder from .metrics import AtomMetricsExporter @@ -1140,8 +1144,12 @@ async def lifespan(app: FastAPI): global _metrics_refresh_task logger.info("Server started successfully and ready to accept requests") tune_gc() + maybe_attach_gc_debug_callback("api_server") await _refresh_metrics_once() _metrics_refresh_task = asyncio.create_task(_metrics_refresh_loop()) + # The engine was built in `main()`, so this is the last point before the + # first request at which everything reachable is still startup state. + freeze_gc_heap("api_server") try: yield finally: diff --git a/atom/model_engine/async_proc.py b/atom/model_engine/async_proc.py index 4adf40ddc5..e13c80c544 100644 --- a/atom/model_engine/async_proc.py +++ b/atom/model_engine/async_proc.py @@ -35,8 +35,9 @@ resolve_obj_by_qualname, set_process_title, shutdown_all_processes, - tune_gc, + worker_process_name, ) +from atom.utils.gc_utils import maybe_attach_gc_debug_callback, tune_gc from atom.utils.numa_utils import numa_bind_to_node logger = logging.getLogger("atom") @@ -85,11 +86,19 @@ def __init__( enable_orphan_reaping() + # Named once, before anything logs: `ps`, the GC lines here and the + # freeze line the runner emits over RPC all have to be the same string, + # or a dp deployment's workers are indistinguishable from each other. + name = worker_process_name(args[0] if args else None, rank) + set_process_title(name) + # Second-order next to the EngineCore's call but not zero: without it # a freeze still lands at the wave boundary, where the prefill burst # churns enough objects to trigger gen-2 here. Runs before the model - # is built so the thresholds cover the startup heap. + # is built so the thresholds cover the startup heap. Freezing that heap + # happens later, via the EngineCore's `freeze_gc_heap` RPC. tune_gc() + maybe_attach_gc_debug_callback(name) # NUMA-local CPU/memory pinning (see atom.utils.numa_utils). # Auto-detects the GPU's local node by default; gated by @@ -106,16 +115,6 @@ def __init__( except Exception as e: # noqa: BLE001 logger.warning(f"AsyncIOProc({label}): NUMA bind skipped: {e}") self.label = f"AsyncIOProc({label})" - # Set process title so this GPU worker is distinguishable by rank in - # ps/top/rocm-smi (otherwise all workers show as "python"). - try: - cfg = args[0] - if cfg.parallel_config.data_parallel_size > 1: - set_process_title(f"DP{cfg.parallel_config.data_parallel_rank}TP{rank}") - else: - set_process_title(f"TP{rank}") - except Exception: - set_process_title(f"TP{rank}") self.io_addrs = io_addrs self.io_queues = queue.Queue(), queue.Queue() self.io_threads: list[threading.Thread] = [] diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 3f9930be65..0fb1ad59e2 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -26,15 +26,21 @@ ) from atom.model_engine.state_runtime import StateRuntime from atom.utils import ( + engine_process_name, envs, init_exit_handler, make_zmq_socket, set_process_title, - tune_gc, ) from atom.utils.distributed.utils import ( stateless_destroy_torch_distributed_process_group, ) +from atom.utils.gc_utils import ( + freeze_gc_heap, + maybe_attach_gc_debug_callback, + tune_gc, + unfreeze_gc_heap, +) logger = logging.getLogger("atom") @@ -55,6 +61,11 @@ class EngineCore: + # This process's name, for the title and every GC log line. A class + # attribute because it is per-process state and each engine is spawned into + # its own interpreter; `_setup_engine_process` is the only writer. + _process_name = "EngineCore" + def __init__(self, config: Config, input_address: str, output_address: str): self.label = "Engine Core" self.input_queue = queue.Queue[Sequence]() @@ -174,9 +185,42 @@ def __init__(self, config: Config, input_address: str, output_address: str): scheduler=self.scheduler, ) + # KV cache allocated, graphs captured, BlockPool built: everything this + # process holds for its lifetime exists, and the next thing is traffic. + self._freeze_after_startup() + self._send_ready_signal() logger.info(f"{self.label}: EngineCore fully initialized and ready") + def _freeze_after_startup(self): + """Freeze this process and its ModelRunner workers. + + The workers are driven from here because only the caller knows warmup + is over -- weights, compile and capture all land via RPCs it sends, and + `--enforce-eager` skips the capture step entirely. + """ + freeze_gc_heap(self._process_name) + try: + self.runner_mgr.call_func("freeze_gc_heap", wait_out=True) + except Exception as e: # noqa: BLE001 - never fail startup over this + logger.warning(f"{self._process_name}: worker heap freeze skipped: {e}") + + @staticmethod + def _setup_engine_process(name: str): + """Identity, orphan reaping and GC policy, for every `run_engine`. + + The disaggregated overrides do not call the base one, and each omission + is silent: an unreaped orphan pins VRAM, an untitled process is + `python` in `ps`, an unattached callback leaves ATOM_GC_DEBUG inert. + """ + from atom.utils import enable_orphan_reaping + + EngineCore._process_name = name + set_process_title(name) + enable_orphan_reaping() # orphans pin VRAM + IPC handles; see its docs + tune_gc() + maybe_attach_gc_debug_callback(name) + def _send_ready_signal(self): self.output_queue.put_nowait(("READY", None)) @@ -192,6 +236,9 @@ def exit(self): if not self.still_running: return self.still_running = False + # Frozen weights and KV cache are unreachable *and* uncollectable, so + # an engine destroyed in-process would read as a GPU memory leak. + unfreeze_gc_heap() if not hasattr(self, "runner_mgr"): self._send_engine_dead() return @@ -217,37 +264,17 @@ def _send_engine_dead(self): @staticmethod def run_engine(config: Config, input_address: str, output_address: str): - # Bind this EngineCore's lifetime to its parent (the server / - # CoreManager): if the parent exits, have the kernel reap this process — - # and, transitively, the ModelRunner workers it spawns — instead of - # leaving them orphaned. Orphans keep pinning GPU VRAM + the custom - # all-reduce IPC handles / rendezvous TCPStore, which makes the next - # restart reuse a stale hipIpc handle and crash. See - # atom.utils.enable_orphan_reaping for the full rationale. - from atom.utils import enable_orphan_reaping - - enable_orphan_reaping() - - # The process whose GC pauses idle the GPUs: a gen-2 pass here stops - # the scheduler, and every ModelRunner worker then has nothing to run. - tune_gc() + EngineCore._setup_engine_process(engine_process_name(config)) engine: EngineCore = None try: if config.pipeline_parallel_size > 1: from atom.model_engine.pp_engine_core import PPEngineCoreProc - set_process_title( - f"EngineCore_PP{config.parallel_config.pipeline_parallel_rank}" - ) engine = PPEngineCoreProc(config, input_address, output_address) elif config.parallel_config.data_parallel_size > 1: - set_process_title( - f"EngineCore_DP{config.parallel_config.data_parallel_rank}" - ) engine = DPEngineCoreProc(config, input_address, output_address) else: - set_process_title("EngineCore") engine = EngineCore(config, input_address, output_address) engine.busy_loop() except Exception as e: @@ -968,6 +995,7 @@ def _process_engine_step(self): @staticmethod def run_engine(config: Config, input_address: str, output_address: str): + EngineCore._setup_engine_process("PrefillEngineCore") engine = None try: engine = PrefillEngineCore(config, input_address, output_address) @@ -1062,6 +1090,12 @@ def __init__(self, config: Config, input_address: str, output_address: str): disagg_cu_shm_name=config.disagg_cu_shm_name, state_runtime=self.state_runtime, ) + # The block pool exists only now, so the freeze inside + # `super().__init__()` ran before there was one to take. Safe to repeat: + # `_ready_deferred` held READY back, so nothing has been admitted yet + # and `gc.freeze()` is additive. + self._freeze_after_startup() + # EngineUtilityHandler was built in super().__init__() with scheduler=None # (decode defers scheduler creation); wire the real one in for MTP stats. self.utility_handler.scheduler = self.scheduler @@ -1224,6 +1258,7 @@ def _process_engine_step(self): @staticmethod def run_engine(config: Config, input_address: str, output_address: str): + EngineCore._setup_engine_process("DecodeEngineCore") engine = None try: engine = DecodeEngineCore(config, input_address, output_address) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 4f3183a25f..bd4096fb2a 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -77,6 +77,7 @@ get_hf_text_config, init_exit_handler, resolve_obj_by_qualname, + worker_process_name, ) from atom.utils.cuda_graph import BatchDescriptor from atom.utils.forward_context import ( @@ -89,6 +90,7 @@ set_forward_context, set_kv_cache_data, ) +from atom.utils.gc_utils import freeze_gc_heap from atom.utils.selector import get_attn_backend from atom.utils.tbo import ( UBatchSlice, @@ -1587,6 +1589,15 @@ def _estimate_cudagraph_overhead(self): ) return int(overhead) + def freeze_gc_heap(self) -> int: + """RPC target: freeze this worker's startup heap. Pauses here reached + 979 ms, the largest of any process. + + The count is returned because `busy_loop` replies only `if out is not + None` -- an RPC target returning None hangs its `wait_out=True` caller. + """ + return freeze_gc_heap(worker_process_name(self.config, self.rank)) + def get_num_blocks(self) -> dict[str, object]: torch.set_default_device(self.device) config = self.config diff --git a/atom/utils/__init__.py b/atom/utils/__init__.py index ba4426e45b..0b06bcc753 100644 --- a/atom/utils/__init__.py +++ b/atom/utils/__init__.py @@ -4,7 +4,6 @@ import contextlib import copy import dataclasses -import gc import importlib import ipaddress import json @@ -195,6 +194,25 @@ def set_process_title(name: str, suffix: str = "", prefix: str | None = None) -> setproctitle.setproctitle(f"{prefix}::{name}") +def engine_process_name(config: "Config") -> str: + """What an EngineCore process is called, in `ps` and in its own logs.""" + pc = config.parallel_config + if config.pipeline_parallel_size > 1: + return f"EngineCore_PP{pc.pipeline_parallel_rank}" + if pc.data_parallel_size > 1: + return f"EngineCore_DP{pc.data_parallel_rank}" + return "EngineCore" + + +def worker_process_name(config: "Config | None", rank: int) -> str: + """What a ModelRunner worker is called. `config` may be absent on paths + that build a worker without one.""" + pc = getattr(config, "parallel_config", None) + if pc is not None and pc.data_parallel_size > 1: + return f"DP{pc.data_parallel_rank}TP{rank}" + return f"TP{rank}" + + def shutdown_all_processes(procs: list[BaseProcess], allowed_seconds: int = 2): # First join any already-exited processes (instant, no wait). for proc in procs: @@ -293,42 +311,6 @@ def enable_orphan_reaping(sig: int = signal.SIGKILL) -> bool: return True -def tune_gc() -> None: - """Stretch the interval between full (generation-2) collections. - - Only gen-2 matters: it is stop-the-world and rescans every tracked - container, so its cost tracks live objects rather than recent garbage, - and at high concurrency it dominates while the gen-0/1 passes are noise. - Raising the thresholds is close to free because reference counting, not - the collector, reclaims everything acyclic -- peak RSS falls too, since a - larger gen-0 threshold lets more objects die before a collection can - promote them. - - gc.freeze() was tried here and removed: once gen-2 stops running there is - nothing left for it to make cheaper, and alone it made collections more - frequent by emptying gen-2, which is the denominator CPython gates full - collections on (long_lived_pending > long_lived_total / 4). - - Thresholds are per-interpreter, so every process has to call this itself: - subprocesses are spawned, and the API server's lifespan runs long after - they exist. Called from the API server, each EngineCore and each - ModelRunner worker; the EngineCore is the one that matters, since its - pauses stall the scheduler and the workers then have nothing to run. - """ - from atom.utils import envs - - thresholds = envs.ATOM_GC_THRESHOLD - if not thresholds: - return - try: - t = tuple(int(x) for x in thresholds.split(",")) - old = gc.get_threshold() - gc.set_threshold(*t) - logger.info("[gc] thresholds %s -> %s", old, t) - except (ValueError, TypeError): - logger.warning("[gc] bad ATOM_GC_THRESHOLD=%r, ignored", thresholds) - - def kill_process_tree(pid: int): """ Kills all descendant processes of the given pid by sending SIGKILL. diff --git a/atom/utils/envs.py b/atom/utils/envs.py index bc39d51433..04030918ef 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -208,10 +208,18 @@ ), # --- Profiling & Logging --- "ATOM_TORCH_PROFILER_DIR": lambda: os.getenv("ATOM_TORCH_PROFILER_DIR", None), + # Move the startup heap (model, compiled graph, tokenizer, KV block pool) + # into CPython's permanent generation once warmup is done, so collections + # stop scanning it. On by default; set 0 to keep the old behaviour. + # See freeze_gc_heap in atom/utils/gc_utils.py. + "ATOM_GC_FREEZE": lambda: os.getenv("ATOM_GC_FREEZE", "1") == "1", + # Log every garbage collection: generation, duration, objects reclaimed. + "ATOM_GC_DEBUG": lambda: os.getenv("ATOM_GC_DEBUG", "0") == "1", # "t0,t1,t2" for gc.set_threshold(); empty keeps CPython's default. # Read independently by the API server, each EngineCore and each - # ModelRunner worker -- thresholds are per-interpreter. The EngineCore is - # the one that matters. See tune_gc in atom/utils/__init__.py. + # ModelRunner worker -- thresholds are per-interpreter. A fallback for + # ATOM_GC_FREEZE=0: freezing removes the cost of a pass, this only spaces + # the passes out. See tune_gc in atom/utils/gc_utils.py. "ATOM_GC_THRESHOLD": lambda: os.getenv("ATOM_GC_THRESHOLD", "").strip(), "ATOM_PROFILER_MORE": lambda: os.getenv("ATOM_PROFILER_MORE", "0") == "1", # When profiling is active, append detailed attention aggregates (sqsq, sqsk, sk) diff --git a/atom/utils/gc_utils.py b/atom/utils/gc_utils.py new file mode 100644 index 0000000000..eea37e803f --- /dev/null +++ b/atom/utils/gc_utils.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + +"""Garbage-collector policy for the serving processes. + +CPython's gen-2 pass is stop-the-world and traverses every tracked container, +so its cost tracks the live heap -- which here is almost all startup state +(model, compiled graph, tokenizer, KV block pool) that is never garbage. +Measured on DeepSeek-V4-Flash-DSpark tp1: 268 ms in the EngineCore, 979 ms in a +ModelRunner worker, 265 ms in the API server, each reclaiming 0 objects. + +That is the argument for freezing rather than tuning thresholds: the collector +is not merely slow here, it finds nothing. Everything the hot loop allocates is +acyclic, so reference counting takes it. +""" + +import gc +import logging +import time + +logger = logging.getLogger("atom") + + +def tune_gc() -> None: + """Raise the collection thresholds from ``ATOM_GC_THRESHOLD``. + + Per-interpreter, so every process that serves calls it for itself -- an + enumeration here has gone stale twice, so the rule is the documentation and + `tests/test_gc_utils.py` is what checks it. This spaces passes out; + `freeze_gc_heap` removes what one costs and is on by default, so with + `ATOM_GC_FREEZE=1` there is little left for this to do. + """ + from atom.utils import envs + + thresholds = envs.ATOM_GC_THRESHOLD + if not thresholds: + return + try: + t = tuple(int(x) for x in thresholds.split(",")) + old = gc.get_threshold() + gc.set_threshold(*t) + logger.info("[gc] thresholds %s -> %s", old, t) + except (ValueError, TypeError): + logger.warning("[gc] bad ATOM_GC_THRESHOLD=%r, ignored", thresholds) + + +def freeze_gc_heap(context: str) -> int: + """Move everything alive now into the permanent generation, which + collections skip. Call once, between "startup done" and "traffic starts". + Returns the total frozen count. + + Collecting first is not optional: `gc.freeze()` takes every generation as + it finds it, so current garbage would be made permanently unreclaimable. + One full pass suffices -- `gc.collect()` covers all three generations, and + freezing does not care which one an object ended up in. + + Objects created afterwards are still tracked and collected, so this is not + `gc.disable()` -- a cycle written by later code is still caught. What it + forfeits is anything alive *now* that later becomes garbage, which in these + processes outlives the process anyway. `unfreeze_gc_heap` covers the one + case where that is false: tearing an engine down in-process. + """ + from atom.utils import envs + + if not envs.ATOM_GC_FREEZE: + return 0 + gc.collect() + before = gc.get_freeze_count() + gc.freeze() + total = gc.get_freeze_count() + logger.info( + "[gc] %s: froze %d objects (%d already frozen)", context, total - before, before + ) + return total + + +def unfreeze_gc_heap() -> None: + """Hand the permanent generation back. Required on engine shutdown: a + frozen object is invisible to the collector, so an engine destroyed inside + a live interpreter would leave its weights and KV cache unreachable *and* + uncollectable, which presents as a GPU memory leak.""" + frozen = gc.get_freeze_count() + if frozen: + logger.info("[gc] unfroze %d objects", frozen) + gc.unfreeze() + + +def maybe_attach_gc_debug_callback(context: str) -> None: + """Under ``ATOM_GC_DEBUG``, log every collection. + + Deliberately expensive: it counts the tracked set on each pass, which cost + ~90s of extra startup on a V4-Flash tp1. It is also the only way to see + these pauses -- a stall in the EngineCore idles the workers, and an idle + worker emits no trace event at all. + """ + from atom.utils import envs + + if not envs.ATOM_GC_DEBUG: + return + + started: dict[str, float | int] = {"at": 0.0, "tracked": 0} + + def _log(phase: str, info: dict) -> None: + gen = info.get("generation") + if gen is None: + return + if phase == "start": + started["at"] = time.perf_counter() + started["tracked"] = len(gc.get_objects(gen)) + return + logger.info( + "[gc] %s: gen-%d took %.2f ms, reclaimed %s of %d tracked", + context, + gen, + (time.perf_counter() - started["at"]) * 1e3, + info.get("collected", "?"), + started["tracked"], + ) + + gc.callbacks.append(_log) + logger.info("[gc] %s: debug callback attached", context) diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 3e42418990..9bc8f63407 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -158,6 +158,21 @@ land. See `atom/model_ops/v4_backend_gate.py` for the selector. | **ATOM_ENABLE_DETAILED_ANNOTATION** | bool | 0 (false) | When profiling is active, appends detailed attention aggregates to the `prefill[]`/`decode[]` trace labels: `sqsq` (Σ N_Q²), `sqsk` (Σ N_Q·N_KV), and `sk` (Σ N_KV), where N_Q is the scheduled query tokens and N_KV the KV length per request. Used to estimate attention FLOPs for downstream roofline analysis. | | **ATOM_LOG_MORE** | bool | 0 (false) | If set to `1`, use verbose logging format (includes process name, PID, path, line number, function name). | +## Garbage collection + +CPython's generation-2 pass is stop-the-world and walks every tracked +container, so its cost tracks the live heap — which in a serving process is +almost entirely startup state (model, compiled graph, tokenizer, KV block +pool) that is never garbage. Measured on DeepSeek-V4-Flash-DSpark tp1: 242.8 ms +in the EngineCore, up to 596 ms in a ModelRunner worker, while reclaiming zero +objects once startup was done. See `atom/utils/gc_utils.py`. + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| **ATOM_GC_FREEZE** | bool | 1 (true) | Move the startup heap into CPython's permanent generation once warmup is done, so collections stop scanning it. Applied in every process that outlives startup — the API server, the atomesh frontend, every EngineCore and every ModelRunner worker; undone on engine shutdown so an in-process teardown does not leak. Set `0` to keep the pre-freeze behaviour. | +| **ATOM_GC_DEBUG** | bool | 0 (false) | Log every collection: generation, duration, objects reclaimed, objects tracked. Costly — counting the tracked set on every pass added ~90s of startup on a V4-Flash tp1 — but the only way to see these pauses, since a stall in the EngineCore idles the workers with no event in their torch trace. | +| **ATOM_GC_THRESHOLD** | csv int | "" (= CPython default 700,10,10) | `t0,t1,t2` for `gc.set_threshold()`. Thresholds are per-interpreter, so each process reads it independently. A fallback for `ATOM_GC_FREEZE=0`: this spaces collections out, freezing removes what one costs. | + ### Debug dump (`atom.utils.debug_helper`) Env-gated dump / compare / monkey-patch primitives for forward bisect & diff --git a/tests/test_gc_utils.py b/tests/test_gc_utils.py new file mode 100644 index 0000000000..3a20a1d386 --- /dev/null +++ b/tests/test_gc_utils.py @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: MIT + +"""`gc.freeze()` is what removes the collector's cost; these pin what it costs. + +Freezing is the only knob here that changes what the collector *may not touch*, +so the two things worth a test are the two ways it can be wrong: freezing too +much (garbage made permanent, or never handed back) and freezing too little +(the safety net gone, which would make it `gc.disable()` in disguise). + +Each case carries the failure shape it guards, because a happy-path assertion +would pass against a `freeze_gc_heap` that did nothing at all. +""" + +from __future__ import annotations + +import gc + +import pytest + +from atom.utils.gc_utils import freeze_gc_heap, unfreeze_gc_heap + + +class _Cycle: + """A reference cycle: unreachable by refcount, only the collector frees it.""" + + def __init__(self): + self.self_ref = self + + +@pytest.fixture(autouse=True) +def _leave_gc_as_found(): + """Every test here mutates interpreter-global state.""" + was_enabled = gc.isenabled() + yield + gc.unfreeze() + gc.collect() + if was_enabled: + gc.enable() + + +def test_freezing_takes_the_live_heap_out_of_the_collectors_reach(): + live = [object() for _ in range(64)] + gc.collect() + before = gc.get_freeze_count() + + freeze_gc_heap("test") + + assert gc.get_freeze_count() > before, "nothing was frozen" + # Control: what the frozen set is for. `gc.get_objects()` reports only what + # a collection would still walk, so the drop is the cost that goes away. + assert len(gc.get_objects()) < len(live), ( + "the live heap is still visible to the collector, so freezing bought " "nothing" + ) + + +def test_new_objects_are_still_collected_after_a_freeze(): + """This is what separates freezing from `gc.disable()`. + + Freezing forfeits only what was alive at that instant. A cycle created by + later code -- a code path added next year -- must still be reclaimed, or + this becomes an unbounded leak instead of a bounded one. + """ + freeze_gc_heap("test") + + # `gc.collect()` runs even when the collector is disabled, so calling it + # proves only that the object is not frozen -- it would pass just as well + # against a `freeze_gc_heap` that also called `gc.disable()`. What has to + # be shown is that a collection still fires *on its own*. + assert gc.isenabled(), "freezing disabled the collector" + + fired: list[int] = [] + gc.callbacks.append( + lambda phase, info: fired.append(1) if phase == "stop" else None + ) + try: + threshold = gc.get_threshold()[0] + for _ in range(threshold * 4): + _Cycle() # dropped immediately; only the collector can free it + if fired: + break + finally: + gc.callbacks.pop() + + assert fired, ( + "no automatic collection ran after the freeze -- the safety net for " + "cycles written by later code is gone, which makes this gc.disable()" + ) + + +def test_garbage_alive_at_freeze_time_is_not_made_permanent(): + """`gc.freeze()` moves every generation across exactly as it finds it, so + freezing without collecting first would make current garbage permanently + unreclaimable. `freeze_gc_heap` collects all three generations first.""" + _Cycle() # garbage right now, but no collection has run to notice + freeze_gc_heap("test") + + unfreeze_gc_heap() + # Nothing left for a collection to find: the freeze helper already took it. + assert gc.collect() == 0, "a cycle was carried into the permanent generation" + + +def test_unfreezing_makes_the_startup_heap_reclaimable_again(): + """Required on engine shutdown. Without it an engine torn down inside a + live interpreter leaves its weights unreachable *and* uncollectable, which + presents as a GPU memory leak rather than as anything about GC.""" + doomed = _Cycle() + gc.collect() # promote it, so it is part of the heap being frozen + freeze_gc_heap("test") + del doomed + + # Control: frozen, it is beyond the collector's reach. + assert gc.collect() == 0, "the frozen object was collected; nothing to prove" + + unfreeze_gc_heap() + assert gc.get_freeze_count() == 0 + assert gc.collect() > 0, "unfreezing did not hand the object back" + + +def test_freezing_twice_is_additive_and_harmless(): + """The disaggregated decode path freezes a second time, once its block pool + exists -- it is built later in `DecodeEngineCore.__init__`, after the base + freeze has already run.""" + freeze_gc_heap("first") + first = gc.get_freeze_count() + later = [object() for _ in range(64)] + freeze_gc_heap("second") + + assert gc.get_freeze_count() > first, "the second freeze caught nothing" + assert len(later) == 64 # and did not disturb what it froze + + +def test_every_serving_frontend_applies_the_gc_policy(): + """The axis, not one instance of it. + + This coverage has gone stale twice already: #1980 reached the API server + but not the disaggregated EngineCores, whose `run_engine` does not call the + base one; and the first version of this change reached those but not + atomesh, which builds its own engine and never runs the FastAPI lifespan. + Both misses are silent -- the process simply keeps its 200ms pauses. + + So the rule is checked rather than the instances: a process that builds an + engine and then serves has to apply the policy. + """ + import pathlib + + root = pathlib.Path(__file__).resolve().parent.parent / "atom" + frontends = sorted( + p + for p in root.rglob("*.py") + # `examples/` are batch scripts: they exit, they do not serve. + if ".create_engine(" in p.read_text() and "examples/" not in p.as_posix() + ) + assert frontends, "no engine frontend found; this test has stopped checking" + missing = [ + p.relative_to(root).as_posix() + for p in frontends + # The call, not the import: an unused import passes for the name. + if "freeze_gc_heap(" not in p.read_text() + ] + assert not missing, ( + f"these build an engine and serve, but never apply the GC policy: " + f"{missing}. Add tune_gc/maybe_attach_gc_debug_callback/freeze_gc_heap " + f"once startup is done, as atom/entrypoints/openai/api_server.py does." + ) + + +def test_a_process_has_exactly_one_name(): + """`ps`, the freeze line and the debug callback all take a name, and they + have to be the same string: under dp>1 a name that omits the dp rank makes + every rank's logs identical, which is the case worth telling apart.""" + from types import SimpleNamespace as NS + + from atom.utils import engine_process_name, worker_process_name + + def cfg(pp=1, dp=1, pp_rank=0, dp_rank=0): + return NS( + pipeline_parallel_size=pp, + parallel_config=NS( + pipeline_parallel_rank=pp_rank, + data_parallel_size=dp, + data_parallel_rank=dp_rank, + ), + ) + + assert engine_process_name(cfg()) == "EngineCore" + assert engine_process_name(cfg(dp=4, dp_rank=2)) == "EngineCore_DP2" + assert engine_process_name(cfg(pp=2, pp_rank=1)) == "EngineCore_PP1" + + assert worker_process_name(cfg(), 3) == "TP3" + assert worker_process_name(cfg(dp=4, dp_rank=2), 3) == "DP2TP3" + # A worker can be built without a config; naming must not be what fails. + assert worker_process_name(None, 3) == "TP3" + + # Control: dp ranks must not collide, which is what a rank-blind name does. + names = {worker_process_name(cfg(dp=4, dp_rank=r), 0) for r in range(4)} + assert len(names) == 4, f"dp ranks share a name: {names}" + + +def test_the_worker_rpc_returns_something(): + """`AsyncIOProc.busy_loop` replies only `if out is not None`, so an RPC + target that returns None hangs its `wait_out=True` caller forever. The + EngineCore freezes its workers through exactly such a call, and a server + started with the first version of it never reached "ready". + + Checked on the unbound function so this needs no GPU and no runner. + """ + import inspect + + from atom.model_engine.model_runner import ModelRunner + + src = inspect.getsource(ModelRunner.freeze_gc_heap) + assert "return " in src, ( + "ModelRunner.freeze_gc_heap returns None, which deadlocks the " + "EngineCore's call_func(..., wait_out=True)" + ) + assert inspect.signature(ModelRunner.freeze_gc_heap).return_annotation is not None + + +def test_the_env_gate_turns_it_off(monkeypatch): + from atom.utils import envs + + monkeypatch.setattr(envs, "ATOM_GC_FREEZE", False) + gc.collect() + before = gc.get_freeze_count() + freeze_gc_heap("test") + assert gc.get_freeze_count() == before From 7719cfdf7805cf50f04860604fb0dae141be4fff Mon Sep 17 00:00:00 2001 From: Lingpeng Jin <103567126+valarLip@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:31:52 +0000 Subject: [PATCH 4/5] style(entrypoints): make atomesh/server.py clean, not just no-worse CI runs ruff through reviewdog with `-filter-mode=diff_context`, so it reports anything landing on a changed line or in the context around it -- pre-existing or not. Adding an import inside this file's already-unsorted block pulled a long-standing `I001` into review and failed the check. Comparing whole-file findings against `main` is the wrong local check for that: identical counts still fail if one of them is near your edit. Cheaper to leave the file with none. Sorting the imports is the fix CI asked for. The other two go with it: `initialize_standalone_service` declared `global engine, tokenizer` but only reads them, which `global` is not for; and the blind except around the version banner gets the reason it always had. --- atom/entrypoints/atomesh/server.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/atom/entrypoints/atomesh/server.py b/atom/entrypoints/atomesh/server.py index 7ae2216249..215028c8af 100644 --- a/atom/entrypoints/atomesh/server.py +++ b/atom/entrypoints/atomesh/server.py @@ -8,13 +8,13 @@ from __future__ import annotations import argparse -from dataclasses import dataclass import importlib import importlib.util import json import logging -from pathlib import Path import sys +from dataclasses import dataclass +from pathlib import Path from typing import Any from atom.utils.gc_utils import ( @@ -82,7 +82,7 @@ def print_version(verbose: bool = False) -> None: else atomesh_runner.version_string ) print(version_fn()) - except Exception: + except Exception: # noqa: BLE001 - a version banner must not fail the CLI print("Atomesh Python interface") @@ -107,7 +107,6 @@ def initialize_standalone_service( ) -> Any: from atom.entrypoints.atomesh.atom_standalone_service import AtomStandaloneService - global engine, tokenizer return AtomStandaloneService( engine=engine, tokenizer=tokenizer, From 7e9171be08ec44ebd6aadec6883d26ff3a147e36 Mon Sep 17 00:00:00 2001 From: Lingpeng Jin <103567126+valarLip@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:38:21 +0000 Subject: [PATCH 5/5] fix(tests): the worker-RPC contract test needed no aiter, and no import CI's non-GPU runner has no aiter, and `atom.model_engine.model_runner` imports it. Importing `ModelRunner` inside the test to read `freeze_gc_heap` off it therefore failed there with `ImportError: cannot import name destroy_dist_env`. The import-without-aiter probe missed it because that probe models collection: it imports each test module and stops. An import inside a test body only runs when the test does. The contract is about what the source says -- an RPC target must return something or `call_func(..., wait_out=True)` hangs -- so it is now read with `ast` straight from the file. No import, no aiter, no GPU, and the whole module drops from 5.7s to 1.6s. Re-verified by running every test file this branch touches with `aiter` and `triton` blocked at the import hook: 142 passed. --- tests/test_gc_utils.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/tests/test_gc_utils.py b/tests/test_gc_utils.py index 3a20a1d386..678ee9ca83 100644 --- a/tests/test_gc_utils.py +++ b/tests/test_gc_utils.py @@ -202,18 +202,36 @@ def test_the_worker_rpc_returns_something(): EngineCore freezes its workers through exactly such a call, and a server started with the first version of it never reached "ready". - Checked on the unbound function so this needs no GPU and no runner. + Read from source rather than imported: `model_runner` pulls in aiter, which + the non-GPU CI runner does not have, and a contract about what the code + says needs no runtime anyway. """ - import inspect - - from atom.model_engine.model_runner import ModelRunner + import ast + import pathlib - src = inspect.getsource(ModelRunner.freeze_gc_heap) - assert "return " in src, ( + src = ( + pathlib.Path(__file__).resolve().parent.parent + / "atom" + / "model_engine" + / "model_runner.py" + ).read_text() + fn = next( + ( + node + for cls in ast.parse(src).body + if isinstance(cls, ast.ClassDef) and cls.name == "ModelRunner" + for node in cls.body + if isinstance(node, ast.FunctionDef) and node.name == "freeze_gc_heap" + ), + None, + ) + assert fn is not None, "ModelRunner.freeze_gc_heap is gone; who freezes workers?" + assert any( + isinstance(n, ast.Return) and n.value is not None for n in ast.walk(fn) + ), ( "ModelRunner.freeze_gc_heap returns None, which deadlocks the " "EngineCore's call_func(..., wait_out=True)" ) - assert inspect.signature(ModelRunner.freeze_gc_heap).return_annotation is not None def test_the_env_gate_turns_it_off(monkeypatch):