diff --git a/atom/entrypoints/atomesh/atom_standalone_service.py b/atom/entrypoints/atomesh/atom_standalone_service.py index 2cb84aea29..a38e48b1ec 100644 --- a/atom/entrypoints/atomesh/atom_standalone_service.py +++ b/atom/entrypoints/atomesh/atom_standalone_service.py @@ -46,6 +46,7 @@ ) from atom.entrypoints.openai.sse import data_frame from atom.entrypoints.openai.tool_parser import ToolCallStreamParser +from atom.model_engine.sequence import new_token_ids logger = logging.getLogger("atom") @@ -331,7 +332,10 @@ def __init__( self.started_at = time.time() self.first_token_at: float | None = None self.last_token_at: float | None = None - self.token_ids: list[int] = [] + # An array for the same reason the engine's `token_ids` is one. It + # stays one: this dict goes to `build_*_response`, which reads + # `text` and the counters and never `token_ids`. + self.token_ids = new_token_ids() self.finish_reason: str | None = None self.num_tokens_input = 0 self.kv_transfer_output_meta_info: Any = None @@ -400,7 +404,7 @@ def __init__( self.future = future self.n = n self.started_at = time.time() - self.per_tokens: list[list[int]] = [[] for _ in range(n)] + self.per_tokens = [new_token_ids() for _ in range(n)] self.per_first_token_at: list[float | None] = [None] * n self.per_last_token_at: list[float | None] = [None] * n self.per_finish_reason: list[str | None] = [None] * n diff --git a/atom/entrypoints/atomesh/server.py b/atom/entrypoints/atomesh/server.py index 3c3ce3630b..215028c8af 100644 --- a/atom/entrypoints/atomesh/server.py +++ b/atom/entrypoints/atomesh/server.py @@ -8,15 +8,21 @@ from __future__ import annotations import argparse -from dataclasses import dataclass import importlib import importlib.util import json import logging -from pathlib import Path import sys +from dataclasses import dataclass +from pathlib import Path from typing import Any +from atom.utils.gc_utils import ( + freeze_gc_heap, + maybe_attach_gc_debug_callback, + tune_gc, +) + logger = logging.getLogger("atom") engine: Any | None = None @@ -76,7 +82,7 @@ def print_version(verbose: bool = False) -> None: else atomesh_runner.version_string ) print(version_fn()) - except Exception: + except Exception: # noqa: BLE001 - a version banner must not fail the CLI print("Atomesh Python interface") @@ -101,7 +107,6 @@ def initialize_standalone_service( ) -> Any: from atom.entrypoints.atomesh.atom_standalone_service import AtomStandaloneService - global engine, tokenizer return AtomStandaloneService( engine=engine, tokenizer=tokenizer, @@ -186,6 +191,13 @@ def launch_atom_standalone(atomesh_runner: Any, raw_args: list[str]) -> None: standalone_args.default_chat_template_kwargs, ) + # This frontend is the api_server's counterpart -- same tokenizer, same + # per-request accumulators -- but it builds its engine itself and never + # runs that FastAPI lifespan, so it has to apply the GC policy here. + tune_gc() + maybe_attach_gc_debug_callback("atomesh") + freeze_gc_heap("atomesh") + print("\033[32mATOM starting...\033[0m") print(f"\033[32mHost: {cli_args['host']}:{cli_args['port']}\033[0m") atomesh_runner.launch_mesh( diff --git a/atom/entrypoints/openai/api_server.py b/atom/entrypoints/openai/api_server.py index 0026720148..7ac5d825ad 100644 --- a/atom/entrypoints/openai/api_server.py +++ b/atom/entrypoints/openai/api_server.py @@ -38,8 +38,13 @@ from atom.model_engine.llm_engine import _load_tokenizer from atom.model_engine.multimodal import build_multimodal_inputs from atom.model_engine.request import RequestOutput -from atom.utils import tune_gc +from atom.model_engine.sequence import new_token_ids from atom.utils.arg_parser import FlexibleArgumentParser +from atom.utils.gc_utils import ( + freeze_gc_heap, + maybe_attach_gc_debug_callback, + tune_gc, +) from .chat_encoders import apply_chat_template, load_custom_message_encoder from .metrics import AtomMetricsExporter @@ -475,7 +480,13 @@ async def generate_async( started_at = time.time() first_token_at: float | None = None last_token_at: float | None = None - all_token_ids: list[int] = [] + # An array, not a list: this grows for the whole life of the request, + # and one boxed PyInt per token is what the collector then walks on + # every pass. It stays an array all the way out -- the dict below is + # an internal hand-off to `build_*_response`, which reads `text` and + # the counters and never `token_ids`, so nothing serializes it. A + # consumer that starts reading that key has to convert. + all_token_ids = new_token_ids() finish_reason: str | None = None seq = None kv_transfer_output_meta_info = None @@ -594,7 +605,7 @@ async def generate_async_multimodal( started_at = time.time() first_token_at: float | None = None last_token_at: float | None = None - all_token_ids: list[int] = [] + all_token_ids = new_token_ids() finish_reason: str | None = None seq = None @@ -700,7 +711,7 @@ async def generate_async_fanout( loop = asyncio.get_running_loop() started_at = time.time() - per_tokens: list[list[int]] = [[] for _ in range(n)] + per_tokens = [new_token_ids() for _ in range(n)] per_first_token_at: list[float | None] = [None] * n per_last_token_at: list[float | None] = [None] * n per_finish_reason: list[str | None] = [None] * n @@ -1133,8 +1144,12 @@ async def lifespan(app: FastAPI): global _metrics_refresh_task logger.info("Server started successfully and ready to accept requests") tune_gc() + maybe_attach_gc_debug_callback("api_server") await _refresh_metrics_once() _metrics_refresh_task = asyncio.create_task(_metrics_refresh_loop()) + # The engine was built in `main()`, so this is the last point before the + # first request at which everything reachable is still startup state. + freeze_gc_heap("api_server") try: yield finally: diff --git a/atom/entrypoints/openai/streaming_dispatch.py b/atom/entrypoints/openai/streaming_dispatch.py index 900efd5b82..43262ee908 100644 --- a/atom/entrypoints/openai/streaming_dispatch.py +++ b/atom/entrypoints/openai/streaming_dispatch.py @@ -9,11 +9,14 @@ landing point each stream's SSE generator reads from. """ +import array import threading from asyncio import AbstractEventLoop, Event from dataclasses import dataclass, field from typing import Any, NamedTuple +from atom.model_engine.sequence import new_token_ids + # Fields a later chunk overrides on the one it merges into, when it has a value # of its own. The SSE consumers keep the newest non-empty value they see, so # merging this way hands them what reading each chunk separately would have. @@ -25,7 +28,10 @@ class IncrementalStreamDetokenizer: """Decode token deltas without emitting incomplete UTF-8 characters.""" tokenizer: Any - tokens: list[int] = field(default_factory=list) + # Grows for the whole life of a stream, one entry per token, and only ever + # sliced into `tokenizer.decode` -- which takes an array. Nothing here is + # serialized, so this one has no boundary to convert back at. + tokens: array.array = field(default_factory=new_token_ids) prefix_offset: int = 0 read_offset: int = 0 diff --git a/atom/kv_transfer/offload/hybrid/dsv4/policy.py b/atom/kv_transfer/offload/hybrid/dsv4/policy.py index f6c5e4c2d2..f3e019757c 100644 --- a/atom/kv_transfer/offload/hybrid/dsv4/policy.py +++ b/atom/kv_transfer/offload/hybrid/dsv4/policy.py @@ -10,6 +10,7 @@ from __future__ import annotations +import array import hashlib import heapq import json @@ -290,7 +291,7 @@ def _committed_sidecar_capacity(kvc) -> int: def _chained_prefix_hashes( - token_ids: list[int], + token_ids: array.array, hash_block_size: int, ) -> dict[int, int]: """Return each full-block prefix hash using BlockManager's exact chain.""" diff --git a/atom/model_engine/async_proc.py b/atom/model_engine/async_proc.py index 4adf40ddc5..e13c80c544 100644 --- a/atom/model_engine/async_proc.py +++ b/atom/model_engine/async_proc.py @@ -35,8 +35,9 @@ resolve_obj_by_qualname, set_process_title, shutdown_all_processes, - tune_gc, + worker_process_name, ) +from atom.utils.gc_utils import maybe_attach_gc_debug_callback, tune_gc from atom.utils.numa_utils import numa_bind_to_node logger = logging.getLogger("atom") @@ -85,11 +86,19 @@ def __init__( enable_orphan_reaping() + # Named once, before anything logs: `ps`, the GC lines here and the + # freeze line the runner emits over RPC all have to be the same string, + # or a dp deployment's workers are indistinguishable from each other. + name = worker_process_name(args[0] if args else None, rank) + set_process_title(name) + # Second-order next to the EngineCore's call but not zero: without it # a freeze still lands at the wave boundary, where the prefill burst # churns enough objects to trigger gen-2 here. Runs before the model - # is built so the thresholds cover the startup heap. + # is built so the thresholds cover the startup heap. Freezing that heap + # happens later, via the EngineCore's `freeze_gc_heap` RPC. tune_gc() + maybe_attach_gc_debug_callback(name) # NUMA-local CPU/memory pinning (see atom.utils.numa_utils). # Auto-detects the GPU's local node by default; gated by @@ -106,16 +115,6 @@ def __init__( except Exception as e: # noqa: BLE001 logger.warning(f"AsyncIOProc({label}): NUMA bind skipped: {e}") self.label = f"AsyncIOProc({label})" - # Set process title so this GPU worker is distinguishable by rank in - # ps/top/rocm-smi (otherwise all workers show as "python"). - try: - cfg = args[0] - if cfg.parallel_config.data_parallel_size > 1: - set_process_title(f"DP{cfg.parallel_config.data_parallel_rank}TP{rank}") - else: - set_process_title(f"TP{rank}") - except Exception: - set_process_title(f"TP{rank}") self.io_addrs = io_addrs self.io_queues = queue.Queue(), queue.Queue() self.io_threads: list[threading.Thread] = [] diff --git a/atom/model_engine/block_manager.py b/atom/model_engine/block_manager.py index 07a89249e9..0479034c41 100644 --- a/atom/model_engine/block_manager.py +++ b/atom/model_engine/block_manager.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: MIT # Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +import array import logging from math import inf, isinf @@ -40,6 +41,13 @@ def _make_block_stored( medium: str = MEDIUM_GPU, ) -> BlockStored: """Construct a BlockStored event from a coalesced run of new blocks.""" + # A list, not the `array("i")` the publish paths carry: the event is + # msgpack-encoded and msgspec has no encoding for an array. The publisher + # counts encode failures rather than raising, so an array here takes the + # event stream down without stopping anything. + assert isinstance( + tokens, list + ), f"BlockStored.token_ids must be a list, got {type(tokens).__name__}" return BlockStored( block_hashes=hashes, parent_block_hash=parent, @@ -179,14 +187,16 @@ def __init__( self.demands_declined_no_room: int = 0 @classmethod - def compute_hash(cls, token_ids: list[int], prefix: int = -1): + def compute_hash(cls, token_ids: array.array, prefix: int = -1): h = xxhash.xxh64() if prefix != -1: h.update(prefix.to_bytes(8, "little")) - # dtype pinned: `np.array` infers int64 from a list and int32 from an - # `array("i")`, so the digest used to depend on the caller's Python - # type. int64 is what lists gave, so pinning it leaves every existing - # hash where it was. + # dtype pinned even though every caller now passes an `array("i")`: + # `np.array` infers int64 from a list and int32 from an array, so the + # digest used to depend on the caller's Python type. int64 is what + # lists gave, which leaves every hash recorded before that where it + # was -- and keeps a caller who does pass a list from silently + # computing a different one. h.update(np.asarray(token_ids, dtype=np.int64).tobytes()) return h.intdigest() @@ -308,7 +318,7 @@ def _n_hash_blocks(self, seq: Sequence) -> int: hbs = self.hash_block_size return (len(seq) + hbs - 1) // hbs - def _hash_block_tokens(self, seq: Sequence, i: int) -> list[int]: + def _hash_block_tokens(self, seq: Sequence, i: int) -> array.array: hbs = self.hash_block_size return seq.token_ids[i * hbs : (i + 1) * hbs] @@ -1043,7 +1053,10 @@ def publish_loaded_prefix( self._event_log.append( _make_block_stored( [block_hash], - token_ids, + # The only BlockStored site fed straight from + # `_hash_block_tokens`; the other two accumulate + # into a list already. See `_make_block_stored`. + list(token_ids), parent_hash if parent_hash != -1 else None, self.block_size, ) diff --git a/atom/model_engine/block_pool.py b/atom/model_engine/block_pool.py index 326b68ed1b..8afbb8bfbc 100644 --- a/atom/model_engine/block_pool.py +++ b/atom/model_engine/block_pool.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: MIT # Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +import array from collections import OrderedDict from collections.abc import Callable, Hashable, Iterable from dataclasses import dataclass @@ -115,7 +116,7 @@ def lookup(self, h: int) -> int: """Block id indexed under content hash `h`, or -1.""" return self._hash_to_block_id.get(h, -1) - def publish(self, block_id: int, h: int, token_ids: list[int]) -> None: + def publish(self, block_id: int, h: int, token_ids: array.array) -> None: """Index `block_id` under the content hash of the tokens it now holds.""" block = self.blocks[block_id] block.update(h, token_ids) @@ -131,7 +132,7 @@ def clear_index(self) -> None: for block in self.blocks: if block.ref_count == 0: block.hash = -1 - block.token_ids = [] + block.token_ids = array.array("i") # Every free block is vacant now, and which container an id is in is # only ever decided from its hash — so the split has to be redrawn # here rather than left to drift. @@ -147,7 +148,7 @@ def _unindex(self, block_id: int) -> None: if self._on_evict is not None: self._on_evict(block.hash) block.hash = -1 - block.token_ids = [] + block.token_ids = array.array("i") # ---------------------------- allocation ------------------------------- # def _take_free(self) -> int: @@ -331,4 +332,4 @@ def _adopt(self, destination: int, source: int) -> None: self._hash_to_block_id[src.hash] = destination self._used.discard(source) self._used.add(destination) - src.ref_count, src.hash, src.token_ids = 0, -1, [] + src.ref_count, src.hash, src.token_ids = 0, -1, array.array("i") diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 3f9930be65..0fb1ad59e2 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -26,15 +26,21 @@ ) from atom.model_engine.state_runtime import StateRuntime from atom.utils import ( + engine_process_name, envs, init_exit_handler, make_zmq_socket, set_process_title, - tune_gc, ) from atom.utils.distributed.utils import ( stateless_destroy_torch_distributed_process_group, ) +from atom.utils.gc_utils import ( + freeze_gc_heap, + maybe_attach_gc_debug_callback, + tune_gc, + unfreeze_gc_heap, +) logger = logging.getLogger("atom") @@ -55,6 +61,11 @@ class EngineCore: + # This process's name, for the title and every GC log line. A class + # attribute because it is per-process state and each engine is spawned into + # its own interpreter; `_setup_engine_process` is the only writer. + _process_name = "EngineCore" + def __init__(self, config: Config, input_address: str, output_address: str): self.label = "Engine Core" self.input_queue = queue.Queue[Sequence]() @@ -174,9 +185,42 @@ def __init__(self, config: Config, input_address: str, output_address: str): scheduler=self.scheduler, ) + # KV cache allocated, graphs captured, BlockPool built: everything this + # process holds for its lifetime exists, and the next thing is traffic. + self._freeze_after_startup() + self._send_ready_signal() logger.info(f"{self.label}: EngineCore fully initialized and ready") + def _freeze_after_startup(self): + """Freeze this process and its ModelRunner workers. + + The workers are driven from here because only the caller knows warmup + is over -- weights, compile and capture all land via RPCs it sends, and + `--enforce-eager` skips the capture step entirely. + """ + freeze_gc_heap(self._process_name) + try: + self.runner_mgr.call_func("freeze_gc_heap", wait_out=True) + except Exception as e: # noqa: BLE001 - never fail startup over this + logger.warning(f"{self._process_name}: worker heap freeze skipped: {e}") + + @staticmethod + def _setup_engine_process(name: str): + """Identity, orphan reaping and GC policy, for every `run_engine`. + + The disaggregated overrides do not call the base one, and each omission + is silent: an unreaped orphan pins VRAM, an untitled process is + `python` in `ps`, an unattached callback leaves ATOM_GC_DEBUG inert. + """ + from atom.utils import enable_orphan_reaping + + EngineCore._process_name = name + set_process_title(name) + enable_orphan_reaping() # orphans pin VRAM + IPC handles; see its docs + tune_gc() + maybe_attach_gc_debug_callback(name) + def _send_ready_signal(self): self.output_queue.put_nowait(("READY", None)) @@ -192,6 +236,9 @@ def exit(self): if not self.still_running: return self.still_running = False + # Frozen weights and KV cache are unreachable *and* uncollectable, so + # an engine destroyed in-process would read as a GPU memory leak. + unfreeze_gc_heap() if not hasattr(self, "runner_mgr"): self._send_engine_dead() return @@ -217,37 +264,17 @@ def _send_engine_dead(self): @staticmethod def run_engine(config: Config, input_address: str, output_address: str): - # Bind this EngineCore's lifetime to its parent (the server / - # CoreManager): if the parent exits, have the kernel reap this process — - # and, transitively, the ModelRunner workers it spawns — instead of - # leaving them orphaned. Orphans keep pinning GPU VRAM + the custom - # all-reduce IPC handles / rendezvous TCPStore, which makes the next - # restart reuse a stale hipIpc handle and crash. See - # atom.utils.enable_orphan_reaping for the full rationale. - from atom.utils import enable_orphan_reaping - - enable_orphan_reaping() - - # The process whose GC pauses idle the GPUs: a gen-2 pass here stops - # the scheduler, and every ModelRunner worker then has nothing to run. - tune_gc() + EngineCore._setup_engine_process(engine_process_name(config)) engine: EngineCore = None try: if config.pipeline_parallel_size > 1: from atom.model_engine.pp_engine_core import PPEngineCoreProc - set_process_title( - f"EngineCore_PP{config.parallel_config.pipeline_parallel_rank}" - ) engine = PPEngineCoreProc(config, input_address, output_address) elif config.parallel_config.data_parallel_size > 1: - set_process_title( - f"EngineCore_DP{config.parallel_config.data_parallel_rank}" - ) engine = DPEngineCoreProc(config, input_address, output_address) else: - set_process_title("EngineCore") engine = EngineCore(config, input_address, output_address) engine.busy_loop() except Exception as e: @@ -968,6 +995,7 @@ def _process_engine_step(self): @staticmethod def run_engine(config: Config, input_address: str, output_address: str): + EngineCore._setup_engine_process("PrefillEngineCore") engine = None try: engine = PrefillEngineCore(config, input_address, output_address) @@ -1062,6 +1090,12 @@ def __init__(self, config: Config, input_address: str, output_address: str): disagg_cu_shm_name=config.disagg_cu_shm_name, state_runtime=self.state_runtime, ) + # The block pool exists only now, so the freeze inside + # `super().__init__()` ran before there was one to take. Safe to repeat: + # `_ready_deferred` held READY back, so nothing has been admitted yet + # and `gc.freeze()` is additive. + self._freeze_after_startup() + # EngineUtilityHandler was built in super().__init__() with scheduler=None # (decode defers scheduler creation); wire the real one in for MTP stats. self.utility_handler.scheduler = self.scheduler @@ -1224,6 +1258,7 @@ def _process_engine_step(self): @staticmethod def run_engine(config: Config, input_address: str, output_address: str): + EngineCore._setup_engine_process("DecodeEngineCore") engine = None try: engine = DecodeEngineCore(config, input_address, output_address) diff --git a/atom/model_engine/kv_block.py b/atom/model_engine/kv_block.py index 7c64d0a271..e8af6cf899 100644 --- a/atom/model_engine/kv_block.py +++ b/atom/model_engine/kv_block.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: MIT # Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +import array + class Block: """One physical KV cache block of the compressed pool (BlockManager). @@ -12,16 +14,28 @@ def __init__(self, block_id): self.block_id = block_id self.ref_count = 0 self.hash = -1 - self.token_ids = [] - - def update(self, hash: int, token_ids: list[int]): + # A slice of `Sequence.token_ids`, kept only so a hash hit can be + # checked against the tokens it claims to stand for. See `update` for + # why the type is pinned. + self.token_ids: array.array = array.array("i") + + def update(self, hash: int, token_ids: array.array): + # Pinned rather than accepted as either: a list here fails twice and + # neither failure says so. It never compares equal to the `array("i")` + # the other publish paths store, so every hit on this block reads as a + # collision; and a list of ids is one traversal slot per token for the + # collector, which across a full pool is a stop-the-world pause of + # ~200ms per gen-2 pass against ~5ms for arrays. + assert isinstance( + token_ids, array.array + ), f"Block.token_ids must be an array('i'), got {type(token_ids).__name__}" self.hash = hash self.token_ids = token_ids def reset(self): self.ref_count = 1 self.hash = -1 - self.token_ids = [] + self.token_ids = array.array("i") # Name of the sub-pool sizing class backing the per-request state slots diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index de59199a17..0222bb6eba 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -754,7 +754,8 @@ def postprocess(self, reqs: list[Sequence]): # `list`, not the `array("i")` slice: this is what `generate()` # hands a caller, and the storage type is ours to change. "token_ids": list(req.completion_token_ids), - "logprobs": req.logprobs if req.return_logprobs else None, + # `list` for the same reason as `token_ids` above. + "logprobs": list(req.logprobs) if req.return_logprobs else None, "latency": req.leave_time - req.arrive_time, "finish_reason": req.leave_reason, "num_tokens_input": req.num_prompt_tokens, diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 4f3183a25f..bd4096fb2a 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -77,6 +77,7 @@ get_hf_text_config, init_exit_handler, resolve_obj_by_qualname, + worker_process_name, ) from atom.utils.cuda_graph import BatchDescriptor from atom.utils.forward_context import ( @@ -89,6 +90,7 @@ set_forward_context, set_kv_cache_data, ) +from atom.utils.gc_utils import freeze_gc_heap from atom.utils.selector import get_attn_backend from atom.utils.tbo import ( UBatchSlice, @@ -1587,6 +1589,15 @@ def _estimate_cudagraph_overhead(self): ) return int(overhead) + def freeze_gc_heap(self) -> int: + """RPC target: freeze this worker's startup heap. Pauses here reached + 979 ms, the largest of any process. + + The count is returned because `busy_loop` replies only `if out is not + None` -- an RPC target returning None hangs its `wait_out=True` caller. + """ + return freeze_gc_heap(worker_process_name(self.config, self.rank)) + def get_num_blocks(self) -> dict[str, object]: torch.set_default_device(self.device) config = self.config diff --git a/atom/model_engine/sequence.py b/atom/model_engine/sequence.py index 65d4793fab..438a36626b 100644 --- a/atom/model_engine/sequence.py +++ b/atom/model_engine/sequence.py @@ -190,10 +190,16 @@ def __init__( # would never fire for the prefill blocks; this flag does. self.prefix_hashes_published = False self.return_logprobs = bool(getattr(sampling_params, "logprobs", False)) - self.logprobs: list[float] = [] + # One entry per completion token, so the same reason `token_ids` is an + # array applies: a list would box a PyFloat per token and hand the + # collector a slot to walk for each one. `json.dumps` is the only + # consumer that needs a list, and it converts at its own boundary. + self.logprobs: array.array = array.array("d") # stream callback self.stream_callback = stream_callback - self.output_tokens = [] # cache for newly generate tokens + # The completion half of `token_ids`, kept in step with it by every + # writer, so it is the same array type for the same reasons. + self.output_tokens = new_token_ids() # Placeholders from previous postprocess; overwritten in place. self.num_placeholder_tokens: int = 0 diff --git a/atom/utils/__init__.py b/atom/utils/__init__.py index ba4426e45b..0b06bcc753 100644 --- a/atom/utils/__init__.py +++ b/atom/utils/__init__.py @@ -4,7 +4,6 @@ import contextlib import copy import dataclasses -import gc import importlib import ipaddress import json @@ -195,6 +194,25 @@ def set_process_title(name: str, suffix: str = "", prefix: str | None = None) -> setproctitle.setproctitle(f"{prefix}::{name}") +def engine_process_name(config: "Config") -> str: + """What an EngineCore process is called, in `ps` and in its own logs.""" + pc = config.parallel_config + if config.pipeline_parallel_size > 1: + return f"EngineCore_PP{pc.pipeline_parallel_rank}" + if pc.data_parallel_size > 1: + return f"EngineCore_DP{pc.data_parallel_rank}" + return "EngineCore" + + +def worker_process_name(config: "Config | None", rank: int) -> str: + """What a ModelRunner worker is called. `config` may be absent on paths + that build a worker without one.""" + pc = getattr(config, "parallel_config", None) + if pc is not None and pc.data_parallel_size > 1: + return f"DP{pc.data_parallel_rank}TP{rank}" + return f"TP{rank}" + + def shutdown_all_processes(procs: list[BaseProcess], allowed_seconds: int = 2): # First join any already-exited processes (instant, no wait). for proc in procs: @@ -293,42 +311,6 @@ def enable_orphan_reaping(sig: int = signal.SIGKILL) -> bool: return True -def tune_gc() -> None: - """Stretch the interval between full (generation-2) collections. - - Only gen-2 matters: it is stop-the-world and rescans every tracked - container, so its cost tracks live objects rather than recent garbage, - and at high concurrency it dominates while the gen-0/1 passes are noise. - Raising the thresholds is close to free because reference counting, not - the collector, reclaims everything acyclic -- peak RSS falls too, since a - larger gen-0 threshold lets more objects die before a collection can - promote them. - - gc.freeze() was tried here and removed: once gen-2 stops running there is - nothing left for it to make cheaper, and alone it made collections more - frequent by emptying gen-2, which is the denominator CPython gates full - collections on (long_lived_pending > long_lived_total / 4). - - Thresholds are per-interpreter, so every process has to call this itself: - subprocesses are spawned, and the API server's lifespan runs long after - they exist. Called from the API server, each EngineCore and each - ModelRunner worker; the EngineCore is the one that matters, since its - pauses stall the scheduler and the workers then have nothing to run. - """ - from atom.utils import envs - - thresholds = envs.ATOM_GC_THRESHOLD - if not thresholds: - return - try: - t = tuple(int(x) for x in thresholds.split(",")) - old = gc.get_threshold() - gc.set_threshold(*t) - logger.info("[gc] thresholds %s -> %s", old, t) - except (ValueError, TypeError): - logger.warning("[gc] bad ATOM_GC_THRESHOLD=%r, ignored", thresholds) - - def kill_process_tree(pid: int): """ Kills all descendant processes of the given pid by sending SIGKILL. diff --git a/atom/utils/envs.py b/atom/utils/envs.py index bc39d51433..04030918ef 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -208,10 +208,18 @@ ), # --- Profiling & Logging --- "ATOM_TORCH_PROFILER_DIR": lambda: os.getenv("ATOM_TORCH_PROFILER_DIR", None), + # Move the startup heap (model, compiled graph, tokenizer, KV block pool) + # into CPython's permanent generation once warmup is done, so collections + # stop scanning it. On by default; set 0 to keep the old behaviour. + # See freeze_gc_heap in atom/utils/gc_utils.py. + "ATOM_GC_FREEZE": lambda: os.getenv("ATOM_GC_FREEZE", "1") == "1", + # Log every garbage collection: generation, duration, objects reclaimed. + "ATOM_GC_DEBUG": lambda: os.getenv("ATOM_GC_DEBUG", "0") == "1", # "t0,t1,t2" for gc.set_threshold(); empty keeps CPython's default. # Read independently by the API server, each EngineCore and each - # ModelRunner worker -- thresholds are per-interpreter. The EngineCore is - # the one that matters. See tune_gc in atom/utils/__init__.py. + # ModelRunner worker -- thresholds are per-interpreter. A fallback for + # ATOM_GC_FREEZE=0: freezing removes the cost of a pass, this only spaces + # the passes out. See tune_gc in atom/utils/gc_utils.py. "ATOM_GC_THRESHOLD": lambda: os.getenv("ATOM_GC_THRESHOLD", "").strip(), "ATOM_PROFILER_MORE": lambda: os.getenv("ATOM_PROFILER_MORE", "0") == "1", # When profiling is active, append detailed attention aggregates (sqsq, sqsk, sk) diff --git a/atom/utils/gc_utils.py b/atom/utils/gc_utils.py new file mode 100644 index 0000000000..eea37e803f --- /dev/null +++ b/atom/utils/gc_utils.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + +"""Garbage-collector policy for the serving processes. + +CPython's gen-2 pass is stop-the-world and traverses every tracked container, +so its cost tracks the live heap -- which here is almost all startup state +(model, compiled graph, tokenizer, KV block pool) that is never garbage. +Measured on DeepSeek-V4-Flash-DSpark tp1: 268 ms in the EngineCore, 979 ms in a +ModelRunner worker, 265 ms in the API server, each reclaiming 0 objects. + +That is the argument for freezing rather than tuning thresholds: the collector +is not merely slow here, it finds nothing. Everything the hot loop allocates is +acyclic, so reference counting takes it. +""" + +import gc +import logging +import time + +logger = logging.getLogger("atom") + + +def tune_gc() -> None: + """Raise the collection thresholds from ``ATOM_GC_THRESHOLD``. + + Per-interpreter, so every process that serves calls it for itself -- an + enumeration here has gone stale twice, so the rule is the documentation and + `tests/test_gc_utils.py` is what checks it. This spaces passes out; + `freeze_gc_heap` removes what one costs and is on by default, so with + `ATOM_GC_FREEZE=1` there is little left for this to do. + """ + from atom.utils import envs + + thresholds = envs.ATOM_GC_THRESHOLD + if not thresholds: + return + try: + t = tuple(int(x) for x in thresholds.split(",")) + old = gc.get_threshold() + gc.set_threshold(*t) + logger.info("[gc] thresholds %s -> %s", old, t) + except (ValueError, TypeError): + logger.warning("[gc] bad ATOM_GC_THRESHOLD=%r, ignored", thresholds) + + +def freeze_gc_heap(context: str) -> int: + """Move everything alive now into the permanent generation, which + collections skip. Call once, between "startup done" and "traffic starts". + Returns the total frozen count. + + Collecting first is not optional: `gc.freeze()` takes every generation as + it finds it, so current garbage would be made permanently unreclaimable. + One full pass suffices -- `gc.collect()` covers all three generations, and + freezing does not care which one an object ended up in. + + Objects created afterwards are still tracked and collected, so this is not + `gc.disable()` -- a cycle written by later code is still caught. What it + forfeits is anything alive *now* that later becomes garbage, which in these + processes outlives the process anyway. `unfreeze_gc_heap` covers the one + case where that is false: tearing an engine down in-process. + """ + from atom.utils import envs + + if not envs.ATOM_GC_FREEZE: + return 0 + gc.collect() + before = gc.get_freeze_count() + gc.freeze() + total = gc.get_freeze_count() + logger.info( + "[gc] %s: froze %d objects (%d already frozen)", context, total - before, before + ) + return total + + +def unfreeze_gc_heap() -> None: + """Hand the permanent generation back. Required on engine shutdown: a + frozen object is invisible to the collector, so an engine destroyed inside + a live interpreter would leave its weights and KV cache unreachable *and* + uncollectable, which presents as a GPU memory leak.""" + frozen = gc.get_freeze_count() + if frozen: + logger.info("[gc] unfroze %d objects", frozen) + gc.unfreeze() + + +def maybe_attach_gc_debug_callback(context: str) -> None: + """Under ``ATOM_GC_DEBUG``, log every collection. + + Deliberately expensive: it counts the tracked set on each pass, which cost + ~90s of extra startup on a V4-Flash tp1. It is also the only way to see + these pauses -- a stall in the EngineCore idles the workers, and an idle + worker emits no trace event at all. + """ + from atom.utils import envs + + if not envs.ATOM_GC_DEBUG: + return + + started: dict[str, float | int] = {"at": 0.0, "tracked": 0} + + def _log(phase: str, info: dict) -> None: + gen = info.get("generation") + if gen is None: + return + if phase == "start": + started["at"] = time.perf_counter() + started["tracked"] = len(gc.get_objects(gen)) + return + logger.info( + "[gc] %s: gen-%d took %.2f ms, reclaimed %s of %d tracked", + context, + gen, + (time.perf_counter() - started["at"]) * 1e3, + info.get("collected", "?"), + started["tracked"], + ) + + gc.callbacks.append(_log) + logger.info("[gc] %s: debug callback attached", context) diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 3e42418990..9bc8f63407 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -158,6 +158,21 @@ land. See `atom/model_ops/v4_backend_gate.py` for the selector. | **ATOM_ENABLE_DETAILED_ANNOTATION** | bool | 0 (false) | When profiling is active, appends detailed attention aggregates to the `prefill[]`/`decode[]` trace labels: `sqsq` (Σ N_Q²), `sqsk` (Σ N_Q·N_KV), and `sk` (Σ N_KV), where N_Q is the scheduled query tokens and N_KV the KV length per request. Used to estimate attention FLOPs for downstream roofline analysis. | | **ATOM_LOG_MORE** | bool | 0 (false) | If set to `1`, use verbose logging format (includes process name, PID, path, line number, function name). | +## Garbage collection + +CPython's generation-2 pass is stop-the-world and walks every tracked +container, so its cost tracks the live heap — which in a serving process is +almost entirely startup state (model, compiled graph, tokenizer, KV block +pool) that is never garbage. Measured on DeepSeek-V4-Flash-DSpark tp1: 242.8 ms +in the EngineCore, up to 596 ms in a ModelRunner worker, while reclaiming zero +objects once startup was done. See `atom/utils/gc_utils.py`. + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| **ATOM_GC_FREEZE** | bool | 1 (true) | Move the startup heap into CPython's permanent generation once warmup is done, so collections stop scanning it. Applied in every process that outlives startup — the API server, the atomesh frontend, every EngineCore and every ModelRunner worker; undone on engine shutdown so an in-process teardown does not leak. Set `0` to keep the pre-freeze behaviour. | +| **ATOM_GC_DEBUG** | bool | 0 (false) | Log every collection: generation, duration, objects reclaimed, objects tracked. Costly — counting the tracked set on every pass added ~90s of startup on a V4-Flash tp1 — but the only way to see these pauses, since a stall in the EngineCore idles the workers with no event in their torch trace. | +| **ATOM_GC_THRESHOLD** | csv int | "" (= CPython default 700,10,10) | `t0,t1,t2` for `gc.set_threshold()`. Thresholds are per-interpreter, so each process reads it independently. A fallback for `ATOM_GC_FREEZE=0`: this spaces collections out, freezing removes what one costs. | + ### Debug dump (`atom.utils.debug_helper`) Env-gated dump / compare / monkey-patch primitives for forward bisect & diff --git a/tests/entrypoints/test_streaming_dispatch.py b/tests/entrypoints/test_streaming_dispatch.py index 8d6e5abf82..3f5cbf1017 100644 --- a/tests/entrypoints/test_streaming_dispatch.py +++ b/tests/entrypoints/test_streaming_dispatch.py @@ -10,7 +10,10 @@ class _Utf8ByteTokenizer: def decode(self, token_ids, skip_special_tokens=True): - return bytes(token_ids).decode("utf-8", errors="replace") + # `bytes(x)` of an `array("i")` copies its buffer -- four bytes per id + # -- where from a list it takes the values. A real tokenizer reads ids, + # so this double has to as well; keep the `list`. + return bytes(list(token_ids)).decode("utf-8", errors="replace") class _ImmediateLoop: @@ -378,4 +381,4 @@ def test_each_stream_gets_its_own_detokenizer(): first, second = dispatcher.new_state(), dispatcher.new_state() assert first is not second - assert first.tokens == [] and second.tokens == [] + assert not first.tokens and not second.tokens diff --git a/tests/test_block_pool.py b/tests/test_block_pool.py index 8330d63710..1e07c7eb58 100644 --- a/tests/test_block_pool.py +++ b/tests/test_block_pool.py @@ -10,15 +10,26 @@ compress/state boundary does. """ +import array + import pytest from atom.model_engine.block_pool import BlockPool +def toks(*ids: int) -> array.array: + """What the block manager publishes: a slice of `Sequence.token_ids`. + + A list here is what `Block.update` refuses, so the tests have to hold the + production type or they stop covering the comparison the pool relies on. + """ + return array.array("i", ids) + + def published(pool: BlockPool, block_id: int, h: int) -> int: """Allocate, publish under `h`, and release — a cached, free block.""" pool.allocate(block_id) - pool.publish(block_id, h, [h]) + pool.publish(block_id, h, toks(h)) pool.free(block_id) return block_id @@ -81,7 +92,7 @@ def test_a_cached_top_loses_its_content_and_says_so(self): def test_a_held_top_moves_and_keeps_its_identity(self): pool = BlockPool(num_blocks=3) pool.allocate(2) - pool.publish(2, 100, [7, 8]) + pool.publish(2, 100, toks(7, 8)) pool.claim(2) # a second holder, so the ref count has to travel too retirement = pool.retire_top() @@ -89,7 +100,7 @@ def test_a_held_top_moves_and_keeps_its_identity(self): assert 0 <= retirement.moved_to < 2 moved = pool.block(retirement.moved_to) - assert (moved.hash, moved.token_ids, moved.ref_count) == (100, [7, 8], 2) + assert (moved.hash, moved.token_ids, moved.ref_count) == (100, toks(7, 8), 2) assert pool.lookup(100) == retirement.moved_to assert pool.is_used(retirement.moved_to) assert not pool.is_used(2) @@ -101,7 +112,7 @@ def test_moving_evicts_whatever_the_destination_held(self): published(pool, 0, h=100) published(pool, 1, h=200) pool.allocate(2) - pool.publish(2, 300, [3]) + pool.publish(2, 300, toks(3)) retirement = pool.retire_top() # The destination is a cached block, so its content is the price. diff --git a/tests/test_gc_utils.py b/tests/test_gc_utils.py new file mode 100644 index 0000000000..678ee9ca83 --- /dev/null +++ b/tests/test_gc_utils.py @@ -0,0 +1,244 @@ +# SPDX-License-Identifier: MIT + +"""`gc.freeze()` is what removes the collector's cost; these pin what it costs. + +Freezing is the only knob here that changes what the collector *may not touch*, +so the two things worth a test are the two ways it can be wrong: freezing too +much (garbage made permanent, or never handed back) and freezing too little +(the safety net gone, which would make it `gc.disable()` in disguise). + +Each case carries the failure shape it guards, because a happy-path assertion +would pass against a `freeze_gc_heap` that did nothing at all. +""" + +from __future__ import annotations + +import gc + +import pytest + +from atom.utils.gc_utils import freeze_gc_heap, unfreeze_gc_heap + + +class _Cycle: + """A reference cycle: unreachable by refcount, only the collector frees it.""" + + def __init__(self): + self.self_ref = self + + +@pytest.fixture(autouse=True) +def _leave_gc_as_found(): + """Every test here mutates interpreter-global state.""" + was_enabled = gc.isenabled() + yield + gc.unfreeze() + gc.collect() + if was_enabled: + gc.enable() + + +def test_freezing_takes_the_live_heap_out_of_the_collectors_reach(): + live = [object() for _ in range(64)] + gc.collect() + before = gc.get_freeze_count() + + freeze_gc_heap("test") + + assert gc.get_freeze_count() > before, "nothing was frozen" + # Control: what the frozen set is for. `gc.get_objects()` reports only what + # a collection would still walk, so the drop is the cost that goes away. + assert len(gc.get_objects()) < len(live), ( + "the live heap is still visible to the collector, so freezing bought " "nothing" + ) + + +def test_new_objects_are_still_collected_after_a_freeze(): + """This is what separates freezing from `gc.disable()`. + + Freezing forfeits only what was alive at that instant. A cycle created by + later code -- a code path added next year -- must still be reclaimed, or + this becomes an unbounded leak instead of a bounded one. + """ + freeze_gc_heap("test") + + # `gc.collect()` runs even when the collector is disabled, so calling it + # proves only that the object is not frozen -- it would pass just as well + # against a `freeze_gc_heap` that also called `gc.disable()`. What has to + # be shown is that a collection still fires *on its own*. + assert gc.isenabled(), "freezing disabled the collector" + + fired: list[int] = [] + gc.callbacks.append( + lambda phase, info: fired.append(1) if phase == "stop" else None + ) + try: + threshold = gc.get_threshold()[0] + for _ in range(threshold * 4): + _Cycle() # dropped immediately; only the collector can free it + if fired: + break + finally: + gc.callbacks.pop() + + assert fired, ( + "no automatic collection ran after the freeze -- the safety net for " + "cycles written by later code is gone, which makes this gc.disable()" + ) + + +def test_garbage_alive_at_freeze_time_is_not_made_permanent(): + """`gc.freeze()` moves every generation across exactly as it finds it, so + freezing without collecting first would make current garbage permanently + unreclaimable. `freeze_gc_heap` collects all three generations first.""" + _Cycle() # garbage right now, but no collection has run to notice + freeze_gc_heap("test") + + unfreeze_gc_heap() + # Nothing left for a collection to find: the freeze helper already took it. + assert gc.collect() == 0, "a cycle was carried into the permanent generation" + + +def test_unfreezing_makes_the_startup_heap_reclaimable_again(): + """Required on engine shutdown. Without it an engine torn down inside a + live interpreter leaves its weights unreachable *and* uncollectable, which + presents as a GPU memory leak rather than as anything about GC.""" + doomed = _Cycle() + gc.collect() # promote it, so it is part of the heap being frozen + freeze_gc_heap("test") + del doomed + + # Control: frozen, it is beyond the collector's reach. + assert gc.collect() == 0, "the frozen object was collected; nothing to prove" + + unfreeze_gc_heap() + assert gc.get_freeze_count() == 0 + assert gc.collect() > 0, "unfreezing did not hand the object back" + + +def test_freezing_twice_is_additive_and_harmless(): + """The disaggregated decode path freezes a second time, once its block pool + exists -- it is built later in `DecodeEngineCore.__init__`, after the base + freeze has already run.""" + freeze_gc_heap("first") + first = gc.get_freeze_count() + later = [object() for _ in range(64)] + freeze_gc_heap("second") + + assert gc.get_freeze_count() > first, "the second freeze caught nothing" + assert len(later) == 64 # and did not disturb what it froze + + +def test_every_serving_frontend_applies_the_gc_policy(): + """The axis, not one instance of it. + + This coverage has gone stale twice already: #1980 reached the API server + but not the disaggregated EngineCores, whose `run_engine` does not call the + base one; and the first version of this change reached those but not + atomesh, which builds its own engine and never runs the FastAPI lifespan. + Both misses are silent -- the process simply keeps its 200ms pauses. + + So the rule is checked rather than the instances: a process that builds an + engine and then serves has to apply the policy. + """ + import pathlib + + root = pathlib.Path(__file__).resolve().parent.parent / "atom" + frontends = sorted( + p + for p in root.rglob("*.py") + # `examples/` are batch scripts: they exit, they do not serve. + if ".create_engine(" in p.read_text() and "examples/" not in p.as_posix() + ) + assert frontends, "no engine frontend found; this test has stopped checking" + missing = [ + p.relative_to(root).as_posix() + for p in frontends + # The call, not the import: an unused import passes for the name. + if "freeze_gc_heap(" not in p.read_text() + ] + assert not missing, ( + f"these build an engine and serve, but never apply the GC policy: " + f"{missing}. Add tune_gc/maybe_attach_gc_debug_callback/freeze_gc_heap " + f"once startup is done, as atom/entrypoints/openai/api_server.py does." + ) + + +def test_a_process_has_exactly_one_name(): + """`ps`, the freeze line and the debug callback all take a name, and they + have to be the same string: under dp>1 a name that omits the dp rank makes + every rank's logs identical, which is the case worth telling apart.""" + from types import SimpleNamespace as NS + + from atom.utils import engine_process_name, worker_process_name + + def cfg(pp=1, dp=1, pp_rank=0, dp_rank=0): + return NS( + pipeline_parallel_size=pp, + parallel_config=NS( + pipeline_parallel_rank=pp_rank, + data_parallel_size=dp, + data_parallel_rank=dp_rank, + ), + ) + + assert engine_process_name(cfg()) == "EngineCore" + assert engine_process_name(cfg(dp=4, dp_rank=2)) == "EngineCore_DP2" + assert engine_process_name(cfg(pp=2, pp_rank=1)) == "EngineCore_PP1" + + assert worker_process_name(cfg(), 3) == "TP3" + assert worker_process_name(cfg(dp=4, dp_rank=2), 3) == "DP2TP3" + # A worker can be built without a config; naming must not be what fails. + assert worker_process_name(None, 3) == "TP3" + + # Control: dp ranks must not collide, which is what a rank-blind name does. + names = {worker_process_name(cfg(dp=4, dp_rank=r), 0) for r in range(4)} + assert len(names) == 4, f"dp ranks share a name: {names}" + + +def test_the_worker_rpc_returns_something(): + """`AsyncIOProc.busy_loop` replies only `if out is not None`, so an RPC + target that returns None hangs its `wait_out=True` caller forever. The + EngineCore freezes its workers through exactly such a call, and a server + started with the first version of it never reached "ready". + + Read from source rather than imported: `model_runner` pulls in aiter, which + the non-GPU CI runner does not have, and a contract about what the code + says needs no runtime anyway. + """ + import ast + import pathlib + + src = ( + pathlib.Path(__file__).resolve().parent.parent + / "atom" + / "model_engine" + / "model_runner.py" + ).read_text() + fn = next( + ( + node + for cls in ast.parse(src).body + if isinstance(cls, ast.ClassDef) and cls.name == "ModelRunner" + for node in cls.body + if isinstance(node, ast.FunctionDef) and node.name == "freeze_gc_heap" + ), + None, + ) + assert fn is not None, "ModelRunner.freeze_gc_heap is gone; who freezes workers?" + assert any( + isinstance(n, ast.Return) and n.value is not None for n in ast.walk(fn) + ), ( + "ModelRunner.freeze_gc_heap returns None, which deadlocks the " + "EngineCore's call_func(..., wait_out=True)" + ) + + +def test_the_env_gate_turns_it_off(monkeypatch): + from atom.utils import envs + + monkeypatch.setattr(envs, "ATOM_GC_FREEZE", False) + gc.collect() + before = gc.get_freeze_count() + freeze_gc_heap("test") + assert gc.get_freeze_count() == before diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index b218e13856..4b8c1837ff 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -562,7 +562,7 @@ def test_offload_partial_handoff_keeps_resumed_deferred_output(self, seq_factory ) assert seq.num_cached_tokens == 10 - assert seq.output_tokens == [999, sched.eos_token_id] + assert list(seq.output_tokens) == [999, sched.eos_token_id] # ── _waiting_new_token_count (PrefillDelayer queue signal) ───────────────── diff --git a/tests/test_sequence.py b/tests/test_sequence.py index fb4427fbb3..676b0069b1 100644 --- a/tests/test_sequence.py +++ b/tests/test_sequence.py @@ -79,7 +79,7 @@ def test_append_tracks_output_tokens(self, seq_factory): seq = seq_factory([1, 2]) seq.append_token(3) seq.append_token(4) - assert seq.output_tokens == [3, 4] + assert list(seq.output_tokens) == [3, 4] def test_append_crosses_block_boundary(self, seq_factory): seq = seq_factory([1, 2, 3, 4], block_size=4) diff --git a/tests/test_token_ids_storage.py b/tests/test_token_ids_storage.py index 8767453621..6c1c71d8e6 100644 --- a/tests/test_token_ids_storage.py +++ b/tests/test_token_ids_storage.py @@ -6,22 +6,42 @@ `scheduled_tokens` becomes a memcpy instead of one CPython int unboxing per token, and a 100k-token prompt costs 0.38 MiB instead of 3.4. -What it risks is narrower and entirely one shape: an `array("i")` never -compares equal to a list. Three places compare token ids, and in every one a -mismatch is silent -- a stop sequence that stops nothing, a prefix cache that -never hits, a hash that changes with its argument's Python type. Each is -pinned here with a positive control, because a test that only checked the -happy path would pass just as well against the versions that were broken. +What it risks is one axis, not one bug: every consumer that was written +against a list. Two kinds, and a silent failure in both -- + + compared against a list an `array("i")` never equals one, so the answer is + always "different": a stop sequence that stops + nothing, a prefix cache that never hits, a hash + that changes with its argument's Python type. + + serialized as a list msgspec has no encoding for an array, and the KV + event publisher counts encode failures instead of + raising -- the whole event stream goes dark. + +Each is pinned here with a positive control, because a test that only checked +the happy path would pass just as well against the versions that were broken. +The two boundaries where the type is load-bearing assert it rather than accept +either, so a consumer drifting back to a list fails loudly at the boundary +instead of quietly downstream. """ from __future__ import annotations import array +import gc +import json +from types import SimpleNamespace +import msgspec import numpy as np import pytest +from conftest import MockConfig -from atom.model_engine.block_manager import BlockManager +from atom.distributed.kv_events import BlockStored +from atom.entrypoints.openai.serving_chat import build_chat_response +from atom.entrypoints.openai.serving_completion import build_completion_response +from atom.model_engine.block_manager import BlockManager, _make_block_stored +from atom.model_engine.kv_block import Block from atom.model_engine.sequence import Sequence, new_token_ids from atom.sampling_params import SamplingParams @@ -32,6 +52,26 @@ def _seq(token_ids, **kw): ) +def test_every_per_token_field_on_a_sequence_is_an_array(): + """Named one by one rather than checked as a class, because the point is + the list of fields: anything growing once per token belongs here, and a + new one added as a list is the regression this catches.""" + seq = _seq([1, 2, 3]) + seq.append_token(4) + fields = { + "token_ids": "i", + "output_tokens": "i", # the completion half of token_ids + "block_table": "i", + "logprobs": "d", # floats, same boxing cost per token + } + for name, typecode in fields.items(): + held = getattr(seq, name) + assert isinstance(held, array.array), f"{name} is a {type(held).__name__}" + assert held.typecode == typecode, name + for stop in _seq([1], stop_token_sequences=[[1]]).stop_token_sequences: + assert isinstance(stop, array.array) + + def test_storage_is_int32_and_holds_what_a_list_held(): seq = _seq([0, 1, 129279, -1]) assert isinstance(seq.token_ids, array.array) @@ -101,6 +141,117 @@ def test_a_block_and_the_slice_it_is_compared_against_share_a_type(): assert published != list(fresh) # control: the mismatch is real +# --- the two boundaries that refuse the wrong type ----------------------- + + +def test_a_block_refuses_to_store_a_list(): + """`Block.token_ids` is what a hash hit is verified against, and the pool + holds one per block -- 94602 of them on a V4-Flash-DSpark tp1. A list costs + twice: the verify always reads as a collision, and the collector walks one + slot per token on every gen-2 pass, 207.7ms of stop-the-world against 5.6 + at that pool size (`/app/logs_claude/o27_branch_gc_heap.py`).""" + block = Block(0) + block.update(7, new_token_ids([1, 2, 3])) + assert block.token_ids == new_token_ids([1, 2, 3]) + + with pytest.raises(AssertionError, match="must be an array"): + block.update(7, [1, 2, 3]) + + # Control: the two costs, neither of which announces itself. The second is + # `gc.get_referents`, the traversal a collection performs -- a list is one + # visit per token, an array is a fixed visit however many it holds. + assert block.token_ids != [1, 2, 3] + assert [len(gc.get_referents(list(range(n)))) for n in (3, 300)] == [3, 300] + assert len({len(gc.get_referents(new_token_ids(range(n)))) for n in (3, 300)}) == 1 + + +def test_a_kv_event_refuses_to_carry_an_array(): + """`BlockStored` is msgpack-encoded, and `KVEventPublisher.publish` counts + encode errors rather than raising -- so an array reaches the wire as + nothing at all.""" + tokens = [1, 2, 3] + event = _make_block_stored([100], tokens, None, block_size=4) + assert msgspec.msgpack.Encoder().encode(event) + + with pytest.raises(AssertionError, match="must be a list"): + _make_block_stored([100], new_token_ids(tokens), None, block_size=4) + + # Control: the encoder is what the assert is standing in front of. + with pytest.raises(TypeError, match="array"): + msgspec.msgpack.Encoder().encode( + BlockStored( + block_hashes=[100], + parent_block_hash=None, + token_ids=new_token_ids(tokens), + block_size=4, + ) + ) + + +def test_every_event_a_publish_path_emits_reaches_the_wire(seq_factory): + """The boundary assert only fires on a path a test actually walks. + + `publish_loaded_prefix` is the one BlockStored site fed straight from + `_hash_block_tokens`, and it emits nothing unless KV events are on -- so + the suite drove it for its return value and never looked at what it put in + the log. Encoding the whole batch is what makes the type load-bearing. + """ + cfg = MockConfig( + num_kvcache_blocks=16, + kv_cache_block_size=4, + enable_prefix_caching=True, + kv_events_config=SimpleNamespace(enable=True), + ) + bm = BlockManager(cfg) + assert bm._event_log is not None, "events did not turn on; the rest is vacuous" + + # Loaded first and on tokens nothing has indexed yet: with a canonical + # mapping already in place this takes the branch that reuses it and emits + # nothing at all, which is how the path stayed uncovered. + loaded = seq_factory(list(range(16))) + bm.allocate(loaded) + assert bm.publish_loaded_prefix(loaded, start_token=0, end_token=16) == 16 + + hashed = seq_factory(list(range(100, 116))) + bm.allocate(hashed) + bm.hash_blocks(hashed, 16, start_tokens=0) + + stored = [e for e in bm._event_log if isinstance(e, BlockStored)] + assert len(stored) >= 2, "a publish path emitted nothing; nothing was checked" + encoder = msgspec.msgpack.Encoder() + for event in bm._event_log: + assert encoder.encode(event) + + +def test_a_response_builder_never_puts_token_ids_on_the_wire(): + """Why the API server's accumulators need no conversion back. + + `generate_async` hands its dict to `build_*_response`, which reads `text`, + the finish reason and the counters. `token_ids` is not among them, so the + array never meets a serializer -- and the accumulator can stay an array + for the whole request instead of holding one boxed PyInt per token. + + Passing an array in and serializing the result is what pins that: a + builder that starts forwarding the key fails here rather than in + production, where it would be a 500 on a response that used to work. + """ + final_output = { + "text": "hi", + "token_ids": new_token_ids([1, 2, 3]), + "finish_reason": "stop", + "num_tokens_input": 4, + "num_tokens_output": 3, + } + chat = build_chat_response("id", "m", final_output["text"], dict(final_output)) + completion = build_completion_response("id", "m", dict(final_output)) + for response in (chat, completion): + assert json.loads(response.model_dump_json())["choices"][0] + + # Control: the failure the assertion above stands in for. + with pytest.raises(TypeError, match="array"): + json.dumps(final_output) + + # --- what the change is for ----------------------------------------------