diff --git a/atom/entrypoints/atomesh/server.py b/atom/entrypoints/atomesh/server.py index 3c3ce3630b..7ae2216249 100644 --- a/atom/entrypoints/atomesh/server.py +++ b/atom/entrypoints/atomesh/server.py @@ -17,6 +17,12 @@ import sys from typing import Any +from atom.utils.gc_utils import ( + freeze_gc_heap, + maybe_attach_gc_debug_callback, + tune_gc, +) + logger = logging.getLogger("atom") engine: Any | None = None @@ -186,6 +192,13 @@ def launch_atom_standalone(atomesh_runner: Any, raw_args: list[str]) -> None: standalone_args.default_chat_template_kwargs, ) + # This frontend is the api_server's counterpart -- same tokenizer, same + # per-request accumulators -- but it builds its engine itself and never + # runs that FastAPI lifespan, so it has to apply the GC policy here. + tune_gc() + maybe_attach_gc_debug_callback("atomesh") + freeze_gc_heap("atomesh") + print("\033[32mATOM starting...\033[0m") print(f"\033[32mHost: {cli_args['host']}:{cli_args['port']}\033[0m") atomesh_runner.launch_mesh( diff --git a/atom/entrypoints/openai/api_server.py b/atom/entrypoints/openai/api_server.py index 239d2158be..7ac5d825ad 100644 --- a/atom/entrypoints/openai/api_server.py +++ b/atom/entrypoints/openai/api_server.py @@ -39,8 +39,12 @@ from atom.model_engine.multimodal import build_multimodal_inputs from atom.model_engine.request import RequestOutput from atom.model_engine.sequence import new_token_ids -from atom.utils import tune_gc from atom.utils.arg_parser import FlexibleArgumentParser +from atom.utils.gc_utils import ( + freeze_gc_heap, + maybe_attach_gc_debug_callback, + tune_gc, +) from .chat_encoders import apply_chat_template, load_custom_message_encoder from .metrics import AtomMetricsExporter @@ -1140,8 +1144,12 @@ async def lifespan(app: FastAPI): global _metrics_refresh_task logger.info("Server started successfully and ready to accept requests") tune_gc() + maybe_attach_gc_debug_callback("api_server") await _refresh_metrics_once() _metrics_refresh_task = asyncio.create_task(_metrics_refresh_loop()) + # The engine was built in `main()`, so this is the last point before the + # first request at which everything reachable is still startup state. + freeze_gc_heap("api_server") try: yield finally: diff --git a/atom/model_engine/async_proc.py b/atom/model_engine/async_proc.py index 4adf40ddc5..e13c80c544 100644 --- a/atom/model_engine/async_proc.py +++ b/atom/model_engine/async_proc.py @@ -35,8 +35,9 @@ resolve_obj_by_qualname, set_process_title, shutdown_all_processes, - tune_gc, + worker_process_name, ) +from atom.utils.gc_utils import maybe_attach_gc_debug_callback, tune_gc from atom.utils.numa_utils import numa_bind_to_node logger = logging.getLogger("atom") @@ -85,11 +86,19 @@ def __init__( enable_orphan_reaping() + # Named once, before anything logs: `ps`, the GC lines here and the + # freeze line the runner emits over RPC all have to be the same string, + # or a dp deployment's workers are indistinguishable from each other. + name = worker_process_name(args[0] if args else None, rank) + set_process_title(name) + # Second-order next to the EngineCore's call but not zero: without it # a freeze still lands at the wave boundary, where the prefill burst # churns enough objects to trigger gen-2 here. Runs before the model - # is built so the thresholds cover the startup heap. + # is built so the thresholds cover the startup heap. Freezing that heap + # happens later, via the EngineCore's `freeze_gc_heap` RPC. tune_gc() + maybe_attach_gc_debug_callback(name) # NUMA-local CPU/memory pinning (see atom.utils.numa_utils). # Auto-detects the GPU's local node by default; gated by @@ -106,16 +115,6 @@ def __init__( except Exception as e: # noqa: BLE001 logger.warning(f"AsyncIOProc({label}): NUMA bind skipped: {e}") self.label = f"AsyncIOProc({label})" - # Set process title so this GPU worker is distinguishable by rank in - # ps/top/rocm-smi (otherwise all workers show as "python"). - try: - cfg = args[0] - if cfg.parallel_config.data_parallel_size > 1: - set_process_title(f"DP{cfg.parallel_config.data_parallel_rank}TP{rank}") - else: - set_process_title(f"TP{rank}") - except Exception: - set_process_title(f"TP{rank}") self.io_addrs = io_addrs self.io_queues = queue.Queue(), queue.Queue() self.io_threads: list[threading.Thread] = [] diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 3f9930be65..0fb1ad59e2 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -26,15 +26,21 @@ ) from atom.model_engine.state_runtime import StateRuntime from atom.utils import ( + engine_process_name, envs, init_exit_handler, make_zmq_socket, set_process_title, - tune_gc, ) from atom.utils.distributed.utils import ( stateless_destroy_torch_distributed_process_group, ) +from atom.utils.gc_utils import ( + freeze_gc_heap, + maybe_attach_gc_debug_callback, + tune_gc, + unfreeze_gc_heap, +) logger = logging.getLogger("atom") @@ -55,6 +61,11 @@ class EngineCore: + # This process's name, for the title and every GC log line. A class + # attribute because it is per-process state and each engine is spawned into + # its own interpreter; `_setup_engine_process` is the only writer. + _process_name = "EngineCore" + def __init__(self, config: Config, input_address: str, output_address: str): self.label = "Engine Core" self.input_queue = queue.Queue[Sequence]() @@ -174,9 +185,42 @@ def __init__(self, config: Config, input_address: str, output_address: str): scheduler=self.scheduler, ) + # KV cache allocated, graphs captured, BlockPool built: everything this + # process holds for its lifetime exists, and the next thing is traffic. + self._freeze_after_startup() + self._send_ready_signal() logger.info(f"{self.label}: EngineCore fully initialized and ready") + def _freeze_after_startup(self): + """Freeze this process and its ModelRunner workers. + + The workers are driven from here because only the caller knows warmup + is over -- weights, compile and capture all land via RPCs it sends, and + `--enforce-eager` skips the capture step entirely. + """ + freeze_gc_heap(self._process_name) + try: + self.runner_mgr.call_func("freeze_gc_heap", wait_out=True) + except Exception as e: # noqa: BLE001 - never fail startup over this + logger.warning(f"{self._process_name}: worker heap freeze skipped: {e}") + + @staticmethod + def _setup_engine_process(name: str): + """Identity, orphan reaping and GC policy, for every `run_engine`. + + The disaggregated overrides do not call the base one, and each omission + is silent: an unreaped orphan pins VRAM, an untitled process is + `python` in `ps`, an unattached callback leaves ATOM_GC_DEBUG inert. + """ + from atom.utils import enable_orphan_reaping + + EngineCore._process_name = name + set_process_title(name) + enable_orphan_reaping() # orphans pin VRAM + IPC handles; see its docs + tune_gc() + maybe_attach_gc_debug_callback(name) + def _send_ready_signal(self): self.output_queue.put_nowait(("READY", None)) @@ -192,6 +236,9 @@ def exit(self): if not self.still_running: return self.still_running = False + # Frozen weights and KV cache are unreachable *and* uncollectable, so + # an engine destroyed in-process would read as a GPU memory leak. + unfreeze_gc_heap() if not hasattr(self, "runner_mgr"): self._send_engine_dead() return @@ -217,37 +264,17 @@ def _send_engine_dead(self): @staticmethod def run_engine(config: Config, input_address: str, output_address: str): - # Bind this EngineCore's lifetime to its parent (the server / - # CoreManager): if the parent exits, have the kernel reap this process — - # and, transitively, the ModelRunner workers it spawns — instead of - # leaving them orphaned. Orphans keep pinning GPU VRAM + the custom - # all-reduce IPC handles / rendezvous TCPStore, which makes the next - # restart reuse a stale hipIpc handle and crash. See - # atom.utils.enable_orphan_reaping for the full rationale. - from atom.utils import enable_orphan_reaping - - enable_orphan_reaping() - - # The process whose GC pauses idle the GPUs: a gen-2 pass here stops - # the scheduler, and every ModelRunner worker then has nothing to run. - tune_gc() + EngineCore._setup_engine_process(engine_process_name(config)) engine: EngineCore = None try: if config.pipeline_parallel_size > 1: from atom.model_engine.pp_engine_core import PPEngineCoreProc - set_process_title( - f"EngineCore_PP{config.parallel_config.pipeline_parallel_rank}" - ) engine = PPEngineCoreProc(config, input_address, output_address) elif config.parallel_config.data_parallel_size > 1: - set_process_title( - f"EngineCore_DP{config.parallel_config.data_parallel_rank}" - ) engine = DPEngineCoreProc(config, input_address, output_address) else: - set_process_title("EngineCore") engine = EngineCore(config, input_address, output_address) engine.busy_loop() except Exception as e: @@ -968,6 +995,7 @@ def _process_engine_step(self): @staticmethod def run_engine(config: Config, input_address: str, output_address: str): + EngineCore._setup_engine_process("PrefillEngineCore") engine = None try: engine = PrefillEngineCore(config, input_address, output_address) @@ -1062,6 +1090,12 @@ def __init__(self, config: Config, input_address: str, output_address: str): disagg_cu_shm_name=config.disagg_cu_shm_name, state_runtime=self.state_runtime, ) + # The block pool exists only now, so the freeze inside + # `super().__init__()` ran before there was one to take. Safe to repeat: + # `_ready_deferred` held READY back, so nothing has been admitted yet + # and `gc.freeze()` is additive. + self._freeze_after_startup() + # EngineUtilityHandler was built in super().__init__() with scheduler=None # (decode defers scheduler creation); wire the real one in for MTP stats. self.utility_handler.scheduler = self.scheduler @@ -1224,6 +1258,7 @@ def _process_engine_step(self): @staticmethod def run_engine(config: Config, input_address: str, output_address: str): + EngineCore._setup_engine_process("DecodeEngineCore") engine = None try: engine = DecodeEngineCore(config, input_address, output_address) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 4f3183a25f..bd4096fb2a 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -77,6 +77,7 @@ get_hf_text_config, init_exit_handler, resolve_obj_by_qualname, + worker_process_name, ) from atom.utils.cuda_graph import BatchDescriptor from atom.utils.forward_context import ( @@ -89,6 +90,7 @@ set_forward_context, set_kv_cache_data, ) +from atom.utils.gc_utils import freeze_gc_heap from atom.utils.selector import get_attn_backend from atom.utils.tbo import ( UBatchSlice, @@ -1587,6 +1589,15 @@ def _estimate_cudagraph_overhead(self): ) return int(overhead) + def freeze_gc_heap(self) -> int: + """RPC target: freeze this worker's startup heap. Pauses here reached + 979 ms, the largest of any process. + + The count is returned because `busy_loop` replies only `if out is not + None` -- an RPC target returning None hangs its `wait_out=True` caller. + """ + return freeze_gc_heap(worker_process_name(self.config, self.rank)) + def get_num_blocks(self) -> dict[str, object]: torch.set_default_device(self.device) config = self.config diff --git a/atom/utils/__init__.py b/atom/utils/__init__.py index ba4426e45b..0b06bcc753 100644 --- a/atom/utils/__init__.py +++ b/atom/utils/__init__.py @@ -4,7 +4,6 @@ import contextlib import copy import dataclasses -import gc import importlib import ipaddress import json @@ -195,6 +194,25 @@ def set_process_title(name: str, suffix: str = "", prefix: str | None = None) -> setproctitle.setproctitle(f"{prefix}::{name}") +def engine_process_name(config: "Config") -> str: + """What an EngineCore process is called, in `ps` and in its own logs.""" + pc = config.parallel_config + if config.pipeline_parallel_size > 1: + return f"EngineCore_PP{pc.pipeline_parallel_rank}" + if pc.data_parallel_size > 1: + return f"EngineCore_DP{pc.data_parallel_rank}" + return "EngineCore" + + +def worker_process_name(config: "Config | None", rank: int) -> str: + """What a ModelRunner worker is called. `config` may be absent on paths + that build a worker without one.""" + pc = getattr(config, "parallel_config", None) + if pc is not None and pc.data_parallel_size > 1: + return f"DP{pc.data_parallel_rank}TP{rank}" + return f"TP{rank}" + + def shutdown_all_processes(procs: list[BaseProcess], allowed_seconds: int = 2): # First join any already-exited processes (instant, no wait). for proc in procs: @@ -293,42 +311,6 @@ def enable_orphan_reaping(sig: int = signal.SIGKILL) -> bool: return True -def tune_gc() -> None: - """Stretch the interval between full (generation-2) collections. - - Only gen-2 matters: it is stop-the-world and rescans every tracked - container, so its cost tracks live objects rather than recent garbage, - and at high concurrency it dominates while the gen-0/1 passes are noise. - Raising the thresholds is close to free because reference counting, not - the collector, reclaims everything acyclic -- peak RSS falls too, since a - larger gen-0 threshold lets more objects die before a collection can - promote them. - - gc.freeze() was tried here and removed: once gen-2 stops running there is - nothing left for it to make cheaper, and alone it made collections more - frequent by emptying gen-2, which is the denominator CPython gates full - collections on (long_lived_pending > long_lived_total / 4). - - Thresholds are per-interpreter, so every process has to call this itself: - subprocesses are spawned, and the API server's lifespan runs long after - they exist. Called from the API server, each EngineCore and each - ModelRunner worker; the EngineCore is the one that matters, since its - pauses stall the scheduler and the workers then have nothing to run. - """ - from atom.utils import envs - - thresholds = envs.ATOM_GC_THRESHOLD - if not thresholds: - return - try: - t = tuple(int(x) for x in thresholds.split(",")) - old = gc.get_threshold() - gc.set_threshold(*t) - logger.info("[gc] thresholds %s -> %s", old, t) - except (ValueError, TypeError): - logger.warning("[gc] bad ATOM_GC_THRESHOLD=%r, ignored", thresholds) - - def kill_process_tree(pid: int): """ Kills all descendant processes of the given pid by sending SIGKILL. diff --git a/atom/utils/envs.py b/atom/utils/envs.py index bc39d51433..04030918ef 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -208,10 +208,18 @@ ), # --- Profiling & Logging --- "ATOM_TORCH_PROFILER_DIR": lambda: os.getenv("ATOM_TORCH_PROFILER_DIR", None), + # Move the startup heap (model, compiled graph, tokenizer, KV block pool) + # into CPython's permanent generation once warmup is done, so collections + # stop scanning it. On by default; set 0 to keep the old behaviour. + # See freeze_gc_heap in atom/utils/gc_utils.py. + "ATOM_GC_FREEZE": lambda: os.getenv("ATOM_GC_FREEZE", "1") == "1", + # Log every garbage collection: generation, duration, objects reclaimed. + "ATOM_GC_DEBUG": lambda: os.getenv("ATOM_GC_DEBUG", "0") == "1", # "t0,t1,t2" for gc.set_threshold(); empty keeps CPython's default. # Read independently by the API server, each EngineCore and each - # ModelRunner worker -- thresholds are per-interpreter. The EngineCore is - # the one that matters. See tune_gc in atom/utils/__init__.py. + # ModelRunner worker -- thresholds are per-interpreter. A fallback for + # ATOM_GC_FREEZE=0: freezing removes the cost of a pass, this only spaces + # the passes out. See tune_gc in atom/utils/gc_utils.py. "ATOM_GC_THRESHOLD": lambda: os.getenv("ATOM_GC_THRESHOLD", "").strip(), "ATOM_PROFILER_MORE": lambda: os.getenv("ATOM_PROFILER_MORE", "0") == "1", # When profiling is active, append detailed attention aggregates (sqsq, sqsk, sk) diff --git a/atom/utils/gc_utils.py b/atom/utils/gc_utils.py new file mode 100644 index 0000000000..eea37e803f --- /dev/null +++ b/atom/utils/gc_utils.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + +"""Garbage-collector policy for the serving processes. + +CPython's gen-2 pass is stop-the-world and traverses every tracked container, +so its cost tracks the live heap -- which here is almost all startup state +(model, compiled graph, tokenizer, KV block pool) that is never garbage. +Measured on DeepSeek-V4-Flash-DSpark tp1: 268 ms in the EngineCore, 979 ms in a +ModelRunner worker, 265 ms in the API server, each reclaiming 0 objects. + +That is the argument for freezing rather than tuning thresholds: the collector +is not merely slow here, it finds nothing. Everything the hot loop allocates is +acyclic, so reference counting takes it. +""" + +import gc +import logging +import time + +logger = logging.getLogger("atom") + + +def tune_gc() -> None: + """Raise the collection thresholds from ``ATOM_GC_THRESHOLD``. + + Per-interpreter, so every process that serves calls it for itself -- an + enumeration here has gone stale twice, so the rule is the documentation and + `tests/test_gc_utils.py` is what checks it. This spaces passes out; + `freeze_gc_heap` removes what one costs and is on by default, so with + `ATOM_GC_FREEZE=1` there is little left for this to do. + """ + from atom.utils import envs + + thresholds = envs.ATOM_GC_THRESHOLD + if not thresholds: + return + try: + t = tuple(int(x) for x in thresholds.split(",")) + old = gc.get_threshold() + gc.set_threshold(*t) + logger.info("[gc] thresholds %s -> %s", old, t) + except (ValueError, TypeError): + logger.warning("[gc] bad ATOM_GC_THRESHOLD=%r, ignored", thresholds) + + +def freeze_gc_heap(context: str) -> int: + """Move everything alive now into the permanent generation, which + collections skip. Call once, between "startup done" and "traffic starts". + Returns the total frozen count. + + Collecting first is not optional: `gc.freeze()` takes every generation as + it finds it, so current garbage would be made permanently unreclaimable. + One full pass suffices -- `gc.collect()` covers all three generations, and + freezing does not care which one an object ended up in. + + Objects created afterwards are still tracked and collected, so this is not + `gc.disable()` -- a cycle written by later code is still caught. What it + forfeits is anything alive *now* that later becomes garbage, which in these + processes outlives the process anyway. `unfreeze_gc_heap` covers the one + case where that is false: tearing an engine down in-process. + """ + from atom.utils import envs + + if not envs.ATOM_GC_FREEZE: + return 0 + gc.collect() + before = gc.get_freeze_count() + gc.freeze() + total = gc.get_freeze_count() + logger.info( + "[gc] %s: froze %d objects (%d already frozen)", context, total - before, before + ) + return total + + +def unfreeze_gc_heap() -> None: + """Hand the permanent generation back. Required on engine shutdown: a + frozen object is invisible to the collector, so an engine destroyed inside + a live interpreter would leave its weights and KV cache unreachable *and* + uncollectable, which presents as a GPU memory leak.""" + frozen = gc.get_freeze_count() + if frozen: + logger.info("[gc] unfroze %d objects", frozen) + gc.unfreeze() + + +def maybe_attach_gc_debug_callback(context: str) -> None: + """Under ``ATOM_GC_DEBUG``, log every collection. + + Deliberately expensive: it counts the tracked set on each pass, which cost + ~90s of extra startup on a V4-Flash tp1. It is also the only way to see + these pauses -- a stall in the EngineCore idles the workers, and an idle + worker emits no trace event at all. + """ + from atom.utils import envs + + if not envs.ATOM_GC_DEBUG: + return + + started: dict[str, float | int] = {"at": 0.0, "tracked": 0} + + def _log(phase: str, info: dict) -> None: + gen = info.get("generation") + if gen is None: + return + if phase == "start": + started["at"] = time.perf_counter() + started["tracked"] = len(gc.get_objects(gen)) + return + logger.info( + "[gc] %s: gen-%d took %.2f ms, reclaimed %s of %d tracked", + context, + gen, + (time.perf_counter() - started["at"]) * 1e3, + info.get("collected", "?"), + started["tracked"], + ) + + gc.callbacks.append(_log) + logger.info("[gc] %s: debug callback attached", context) diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 3e42418990..9bc8f63407 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -158,6 +158,21 @@ land. See `atom/model_ops/v4_backend_gate.py` for the selector. | **ATOM_ENABLE_DETAILED_ANNOTATION** | bool | 0 (false) | When profiling is active, appends detailed attention aggregates to the `prefill[]`/`decode[]` trace labels: `sqsq` (Σ N_Q²), `sqsk` (Σ N_Q·N_KV), and `sk` (Σ N_KV), where N_Q is the scheduled query tokens and N_KV the KV length per request. Used to estimate attention FLOPs for downstream roofline analysis. | | **ATOM_LOG_MORE** | bool | 0 (false) | If set to `1`, use verbose logging format (includes process name, PID, path, line number, function name). | +## Garbage collection + +CPython's generation-2 pass is stop-the-world and walks every tracked +container, so its cost tracks the live heap — which in a serving process is +almost entirely startup state (model, compiled graph, tokenizer, KV block +pool) that is never garbage. Measured on DeepSeek-V4-Flash-DSpark tp1: 242.8 ms +in the EngineCore, up to 596 ms in a ModelRunner worker, while reclaiming zero +objects once startup was done. See `atom/utils/gc_utils.py`. + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| **ATOM_GC_FREEZE** | bool | 1 (true) | Move the startup heap into CPython's permanent generation once warmup is done, so collections stop scanning it. Applied in every process that outlives startup — the API server, the atomesh frontend, every EngineCore and every ModelRunner worker; undone on engine shutdown so an in-process teardown does not leak. Set `0` to keep the pre-freeze behaviour. | +| **ATOM_GC_DEBUG** | bool | 0 (false) | Log every collection: generation, duration, objects reclaimed, objects tracked. Costly — counting the tracked set on every pass added ~90s of startup on a V4-Flash tp1 — but the only way to see these pauses, since a stall in the EngineCore idles the workers with no event in their torch trace. | +| **ATOM_GC_THRESHOLD** | csv int | "" (= CPython default 700,10,10) | `t0,t1,t2` for `gc.set_threshold()`. Thresholds are per-interpreter, so each process reads it independently. A fallback for `ATOM_GC_FREEZE=0`: this spaces collections out, freezing removes what one costs. | + ### Debug dump (`atom.utils.debug_helper`) Env-gated dump / compare / monkey-patch primitives for forward bisect & diff --git a/tests/test_gc_utils.py b/tests/test_gc_utils.py new file mode 100644 index 0000000000..3a20a1d386 --- /dev/null +++ b/tests/test_gc_utils.py @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: MIT + +"""`gc.freeze()` is what removes the collector's cost; these pin what it costs. + +Freezing is the only knob here that changes what the collector *may not touch*, +so the two things worth a test are the two ways it can be wrong: freezing too +much (garbage made permanent, or never handed back) and freezing too little +(the safety net gone, which would make it `gc.disable()` in disguise). + +Each case carries the failure shape it guards, because a happy-path assertion +would pass against a `freeze_gc_heap` that did nothing at all. +""" + +from __future__ import annotations + +import gc + +import pytest + +from atom.utils.gc_utils import freeze_gc_heap, unfreeze_gc_heap + + +class _Cycle: + """A reference cycle: unreachable by refcount, only the collector frees it.""" + + def __init__(self): + self.self_ref = self + + +@pytest.fixture(autouse=True) +def _leave_gc_as_found(): + """Every test here mutates interpreter-global state.""" + was_enabled = gc.isenabled() + yield + gc.unfreeze() + gc.collect() + if was_enabled: + gc.enable() + + +def test_freezing_takes_the_live_heap_out_of_the_collectors_reach(): + live = [object() for _ in range(64)] + gc.collect() + before = gc.get_freeze_count() + + freeze_gc_heap("test") + + assert gc.get_freeze_count() > before, "nothing was frozen" + # Control: what the frozen set is for. `gc.get_objects()` reports only what + # a collection would still walk, so the drop is the cost that goes away. + assert len(gc.get_objects()) < len(live), ( + "the live heap is still visible to the collector, so freezing bought " "nothing" + ) + + +def test_new_objects_are_still_collected_after_a_freeze(): + """This is what separates freezing from `gc.disable()`. + + Freezing forfeits only what was alive at that instant. A cycle created by + later code -- a code path added next year -- must still be reclaimed, or + this becomes an unbounded leak instead of a bounded one. + """ + freeze_gc_heap("test") + + # `gc.collect()` runs even when the collector is disabled, so calling it + # proves only that the object is not frozen -- it would pass just as well + # against a `freeze_gc_heap` that also called `gc.disable()`. What has to + # be shown is that a collection still fires *on its own*. + assert gc.isenabled(), "freezing disabled the collector" + + fired: list[int] = [] + gc.callbacks.append( + lambda phase, info: fired.append(1) if phase == "stop" else None + ) + try: + threshold = gc.get_threshold()[0] + for _ in range(threshold * 4): + _Cycle() # dropped immediately; only the collector can free it + if fired: + break + finally: + gc.callbacks.pop() + + assert fired, ( + "no automatic collection ran after the freeze -- the safety net for " + "cycles written by later code is gone, which makes this gc.disable()" + ) + + +def test_garbage_alive_at_freeze_time_is_not_made_permanent(): + """`gc.freeze()` moves every generation across exactly as it finds it, so + freezing without collecting first would make current garbage permanently + unreclaimable. `freeze_gc_heap` collects all three generations first.""" + _Cycle() # garbage right now, but no collection has run to notice + freeze_gc_heap("test") + + unfreeze_gc_heap() + # Nothing left for a collection to find: the freeze helper already took it. + assert gc.collect() == 0, "a cycle was carried into the permanent generation" + + +def test_unfreezing_makes_the_startup_heap_reclaimable_again(): + """Required on engine shutdown. Without it an engine torn down inside a + live interpreter leaves its weights unreachable *and* uncollectable, which + presents as a GPU memory leak rather than as anything about GC.""" + doomed = _Cycle() + gc.collect() # promote it, so it is part of the heap being frozen + freeze_gc_heap("test") + del doomed + + # Control: frozen, it is beyond the collector's reach. + assert gc.collect() == 0, "the frozen object was collected; nothing to prove" + + unfreeze_gc_heap() + assert gc.get_freeze_count() == 0 + assert gc.collect() > 0, "unfreezing did not hand the object back" + + +def test_freezing_twice_is_additive_and_harmless(): + """The disaggregated decode path freezes a second time, once its block pool + exists -- it is built later in `DecodeEngineCore.__init__`, after the base + freeze has already run.""" + freeze_gc_heap("first") + first = gc.get_freeze_count() + later = [object() for _ in range(64)] + freeze_gc_heap("second") + + assert gc.get_freeze_count() > first, "the second freeze caught nothing" + assert len(later) == 64 # and did not disturb what it froze + + +def test_every_serving_frontend_applies_the_gc_policy(): + """The axis, not one instance of it. + + This coverage has gone stale twice already: #1980 reached the API server + but not the disaggregated EngineCores, whose `run_engine` does not call the + base one; and the first version of this change reached those but not + atomesh, which builds its own engine and never runs the FastAPI lifespan. + Both misses are silent -- the process simply keeps its 200ms pauses. + + So the rule is checked rather than the instances: a process that builds an + engine and then serves has to apply the policy. + """ + import pathlib + + root = pathlib.Path(__file__).resolve().parent.parent / "atom" + frontends = sorted( + p + for p in root.rglob("*.py") + # `examples/` are batch scripts: they exit, they do not serve. + if ".create_engine(" in p.read_text() and "examples/" not in p.as_posix() + ) + assert frontends, "no engine frontend found; this test has stopped checking" + missing = [ + p.relative_to(root).as_posix() + for p in frontends + # The call, not the import: an unused import passes for the name. + if "freeze_gc_heap(" not in p.read_text() + ] + assert not missing, ( + f"these build an engine and serve, but never apply the GC policy: " + f"{missing}. Add tune_gc/maybe_attach_gc_debug_callback/freeze_gc_heap " + f"once startup is done, as atom/entrypoints/openai/api_server.py does." + ) + + +def test_a_process_has_exactly_one_name(): + """`ps`, the freeze line and the debug callback all take a name, and they + have to be the same string: under dp>1 a name that omits the dp rank makes + every rank's logs identical, which is the case worth telling apart.""" + from types import SimpleNamespace as NS + + from atom.utils import engine_process_name, worker_process_name + + def cfg(pp=1, dp=1, pp_rank=0, dp_rank=0): + return NS( + pipeline_parallel_size=pp, + parallel_config=NS( + pipeline_parallel_rank=pp_rank, + data_parallel_size=dp, + data_parallel_rank=dp_rank, + ), + ) + + assert engine_process_name(cfg()) == "EngineCore" + assert engine_process_name(cfg(dp=4, dp_rank=2)) == "EngineCore_DP2" + assert engine_process_name(cfg(pp=2, pp_rank=1)) == "EngineCore_PP1" + + assert worker_process_name(cfg(), 3) == "TP3" + assert worker_process_name(cfg(dp=4, dp_rank=2), 3) == "DP2TP3" + # A worker can be built without a config; naming must not be what fails. + assert worker_process_name(None, 3) == "TP3" + + # Control: dp ranks must not collide, which is what a rank-blind name does. + names = {worker_process_name(cfg(dp=4, dp_rank=r), 0) for r in range(4)} + assert len(names) == 4, f"dp ranks share a name: {names}" + + +def test_the_worker_rpc_returns_something(): + """`AsyncIOProc.busy_loop` replies only `if out is not None`, so an RPC + target that returns None hangs its `wait_out=True` caller forever. The + EngineCore freezes its workers through exactly such a call, and a server + started with the first version of it never reached "ready". + + Checked on the unbound function so this needs no GPU and no runner. + """ + import inspect + + from atom.model_engine.model_runner import ModelRunner + + src = inspect.getsource(ModelRunner.freeze_gc_heap) + assert "return " in src, ( + "ModelRunner.freeze_gc_heap returns None, which deadlocks the " + "EngineCore's call_func(..., wait_out=True)" + ) + assert inspect.signature(ModelRunner.freeze_gc_heap).return_annotation is not None + + +def test_the_env_gate_turns_it_off(monkeypatch): + from atom.utils import envs + + monkeypatch.setattr(envs, "ATOM_GC_FREEZE", False) + gc.collect() + before = gc.get_freeze_count() + freeze_gc_heap("test") + assert gc.get_freeze_count() == before