From bf8fcc9d0d8998376b3394f042da051607247e2e Mon Sep 17 00:00:00 2001 From: Lingpeng Jin <103567126+valarLip@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:33:15 +0000 Subject: [PATCH 1/4] perf(engine): stop the collector walking a heap it never reclaims At steady state the garbage collector in these processes finds nothing. Measured on DeepSeek-V4-Flash-DSpark tp1 with a `gc.callbacks` probe: over 1754 gen-0, 158 gen-1 and 50 gen-2 passes after startup, it reclaimed **zero** objects. Everything the hot loop allocates is acyclic and reference counting takes it. What each pass does cost, being stop-the-world and proportional to the live heap: EngineCore 268 ms of 819,923 tracked TP0 worker 979 ms of 959,694 api_server 265 ms of 632,424 Almost all of that heap is startup state -- model, compiled graph, tokenizer, KV block pool -- so `gc.freeze()` after warmup removes the work rather than merely spacing it out, which is all raising thresholds (#1980) can do. With it on, gen-2 stops firing entirely in all three processes for the rest of the run, while gen-0/1 keep going at under 2 ms: this narrows the collector, it does not disable it, so a cycle written by later code is still caught. `unfreeze_gc_heap` on shutdown is not optional. A frozen object is invisible to the collector, so an engine destroyed inside a live interpreter -- a test, an RL loop that rebuilds it -- would leave its weights and KV cache unreachable *and* uncollectable, which presents as a GPU memory leak with nothing about GC in the symptom. vLLM hit this and handles it the same way. Placement is one call per process at the boundary where startup ends: - api_server, in the lifespan just before it yields - EngineCore, at the end of `__init__` -- KV cache allocated, graphs captured, BlockPool built (94,763 `Block` objects on a V4-Flash tp1) - each ModelRunner worker, over an RPC the EngineCore sends, because only the caller knows warmup is over: weights, compile and capture all arrive as RPCs it issues, and `--enforce-eager` has no capture step to hook Disaggregated decode is the exception and is left partly covered. Its block pool is built in `_init_disagg`, after `super().__init__()` has already sent READY, so a freeze there could make queued requests permanent. It keeps the base freeze and forgoes the pool. `tune_gc` moves into the new module and stays as the fallback for `ATOM_GC_FREEZE=0`. `ATOM_GC_DEBUG` is added alongside: it logs every collection's generation, duration and reclaimed count, which is the only way to see these pauses -- a stall in the EngineCore idles the workers, and an idle worker emits no trace event at all. It is expensive (~90s of extra startup) and off by default. Two gaps found next door and closed, since the fix needed a shared per-process setup anyway: `PrefillEngineCore.run_engine` and `DecodeEngineCore.run_engine` override the base without calling it, so a disaggregated deployment got neither `enable_orphan_reaping` nor `tune_gc`, and neither ever set a process title -- both showed as bare `python` in `ps`. `_setup_engine_process` now owns identity, reaping and GC policy for all three, which also collapses the base's three `set_process_title` branches into one and makes the process title and the GC log lines agree by construction. Throughput is unchanged at this scale (32098 vs 32122 tok/s, and 32098 with freezing off), because gen-2 fires roughly once per seven minutes in steady state here. This is a tail-latency change; it pays at the concurrency #1980 measured at, which is not reproduced here. --- atom/entrypoints/openai/api_server.py | 10 +- atom/model_engine/async_proc.py | 6 +- atom/model_engine/engine_core.py | 86 ++++++++++---- atom/model_engine/model_runner.py | 10 ++ atom/utils/__init__.py | 37 ------ atom/utils/envs.py | 12 +- atom/utils/gc_utils.py | 119 +++++++++++++++++++ docs/environment_variables.md | 15 +++ tests/test_gc_utils.py | 158 ++++++++++++++++++++++++++ 9 files changed, 389 insertions(+), 64 deletions(-) create mode 100644 atom/utils/gc_utils.py create mode 100644 tests/test_gc_utils.py diff --git a/atom/entrypoints/openai/api_server.py b/atom/entrypoints/openai/api_server.py index 239d2158be..7ac5d825ad 100644 --- a/atom/entrypoints/openai/api_server.py +++ b/atom/entrypoints/openai/api_server.py @@ -39,8 +39,12 @@ from atom.model_engine.multimodal import build_multimodal_inputs from atom.model_engine.request import RequestOutput from atom.model_engine.sequence import new_token_ids -from atom.utils import tune_gc from atom.utils.arg_parser import FlexibleArgumentParser +from atom.utils.gc_utils import ( + freeze_gc_heap, + maybe_attach_gc_debug_callback, + tune_gc, +) from .chat_encoders import apply_chat_template, load_custom_message_encoder from .metrics import AtomMetricsExporter @@ -1140,8 +1144,12 @@ async def lifespan(app: FastAPI): global _metrics_refresh_task logger.info("Server started successfully and ready to accept requests") tune_gc() + maybe_attach_gc_debug_callback("api_server") await _refresh_metrics_once() _metrics_refresh_task = asyncio.create_task(_metrics_refresh_loop()) + # The engine was built in `main()`, so this is the last point before the + # first request at which everything reachable is still startup state. + freeze_gc_heap("api_server") try: yield finally: diff --git a/atom/model_engine/async_proc.py b/atom/model_engine/async_proc.py index 4adf40ddc5..e1d49a172c 100644 --- a/atom/model_engine/async_proc.py +++ b/atom/model_engine/async_proc.py @@ -35,8 +35,8 @@ resolve_obj_by_qualname, set_process_title, shutdown_all_processes, - tune_gc, ) +from atom.utils.gc_utils import maybe_attach_gc_debug_callback, tune_gc from atom.utils.numa_utils import numa_bind_to_node logger = logging.getLogger("atom") @@ -88,8 +88,10 @@ def __init__( # Second-order next to the EngineCore's call but not zero: without it # a freeze still lands at the wave boundary, where the prefill burst # churns enough objects to trigger gen-2 here. Runs before the model - # is built so the thresholds cover the startup heap. + # is built so the thresholds cover the startup heap. Freezing that heap + # happens later, via the EngineCore's `freeze_gc_heap` RPC. tune_gc() + maybe_attach_gc_debug_callback(f"TP{rank}") # NUMA-local CPU/memory pinning (see atom.utils.numa_utils). # Auto-detects the GPU's local node by default; gated by diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 3f9930be65..3de1014b2c 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -30,11 +30,16 @@ init_exit_handler, make_zmq_socket, set_process_title, - tune_gc, ) from atom.utils.distributed.utils import ( stateless_destroy_torch_distributed_process_group, ) +from atom.utils.gc_utils import ( + freeze_gc_heap, + maybe_attach_gc_debug_callback, + tune_gc, + unfreeze_gc_heap, +) logger = logging.getLogger("atom") @@ -55,6 +60,11 @@ class EngineCore: + # This process's name, for the title and every GC log line. A class + # attribute because it is per-process state and each engine is spawned into + # its own interpreter; `_setup_engine_process` is the only writer. + _process_name = "EngineCore" + def __init__(self, config: Config, input_address: str, output_address: str): self.label = "Engine Core" self.input_queue = queue.Queue[Sequence]() @@ -174,9 +184,52 @@ def __init__(self, config: Config, input_address: str, output_address: str): scheduler=self.scheduler, ) + # KV cache allocated, graphs captured, BlockPool built: everything this + # process holds for its lifetime exists, and the next thing is traffic. + self._freeze_after_startup() + self._send_ready_signal() logger.info(f"{self.label}: EngineCore fully initialized and ready") + def _freeze_after_startup(self): + """Freeze this process and its ModelRunner workers. + + The workers are driven from here because only the caller knows warmup + is over -- weights, compile and capture all land via RPCs it sends, and + `--enforce-eager` skips the capture step entirely. + """ + freeze_gc_heap(self._process_name) + try: + self.runner_mgr.call_func("freeze_gc_heap", wait_out=True) + except Exception as e: # noqa: BLE001 - never fail startup over this + logger.warning(f"{self._process_name}: worker heap freeze skipped: {e}") + + @staticmethod + def _process_name_for(config: Config) -> str: + """Title of the EngineCore process a `config` will run.""" + pc = config.parallel_config + if config.pipeline_parallel_size > 1: + return f"EngineCore_PP{pc.pipeline_parallel_rank}" + if pc.data_parallel_size > 1: + return f"EngineCore_DP{pc.data_parallel_rank}" + return "EngineCore" + + @staticmethod + def _setup_engine_process(name: str): + """Identity, orphan reaping and GC policy, for every `run_engine`. + + The disaggregated overrides do not call the base one, and each omission + is silent: an unreaped orphan pins VRAM, an untitled process is + `python` in `ps`, an unattached callback leaves ATOM_GC_DEBUG inert. + """ + from atom.utils import enable_orphan_reaping + + EngineCore._process_name = name + set_process_title(name) + enable_orphan_reaping() # orphans pin VRAM + IPC handles; see its docs + tune_gc() + maybe_attach_gc_debug_callback(name) + def _send_ready_signal(self): self.output_queue.put_nowait(("READY", None)) @@ -192,6 +245,9 @@ def exit(self): if not self.still_running: return self.still_running = False + # Frozen weights and KV cache are unreachable *and* uncollectable, so + # an engine destroyed in-process would read as a GPU memory leak. + unfreeze_gc_heap() if not hasattr(self, "runner_mgr"): self._send_engine_dead() return @@ -217,37 +273,17 @@ def _send_engine_dead(self): @staticmethod def run_engine(config: Config, input_address: str, output_address: str): - # Bind this EngineCore's lifetime to its parent (the server / - # CoreManager): if the parent exits, have the kernel reap this process — - # and, transitively, the ModelRunner workers it spawns — instead of - # leaving them orphaned. Orphans keep pinning GPU VRAM + the custom - # all-reduce IPC handles / rendezvous TCPStore, which makes the next - # restart reuse a stale hipIpc handle and crash. See - # atom.utils.enable_orphan_reaping for the full rationale. - from atom.utils import enable_orphan_reaping - - enable_orphan_reaping() - - # The process whose GC pauses idle the GPUs: a gen-2 pass here stops - # the scheduler, and every ModelRunner worker then has nothing to run. - tune_gc() + EngineCore._setup_engine_process(EngineCore._process_name_for(config)) engine: EngineCore = None try: if config.pipeline_parallel_size > 1: from atom.model_engine.pp_engine_core import PPEngineCoreProc - set_process_title( - f"EngineCore_PP{config.parallel_config.pipeline_parallel_rank}" - ) engine = PPEngineCoreProc(config, input_address, output_address) elif config.parallel_config.data_parallel_size > 1: - set_process_title( - f"EngineCore_DP{config.parallel_config.data_parallel_rank}" - ) engine = DPEngineCoreProc(config, input_address, output_address) else: - set_process_title("EngineCore") engine = EngineCore(config, input_address, output_address) engine.busy_loop() except Exception as e: @@ -968,6 +1004,7 @@ def _process_engine_step(self): @staticmethod def run_engine(config: Config, input_address: str, output_address: str): + EngineCore._setup_engine_process("PrefillEngineCore") engine = None try: engine = PrefillEngineCore(config, input_address, output_address) @@ -1062,6 +1099,10 @@ def __init__(self, config: Config, input_address: str, output_address: str): disagg_cu_shm_name=config.disagg_cu_shm_name, state_runtime=self.state_runtime, ) + # Deliberately not frozen here, though the block pool only exists now: + # READY went out in `super().__init__()`, so the input thread may + # already hold requests that freezing would make permanent. + # EngineUtilityHandler was built in super().__init__() with scheduler=None # (decode defers scheduler creation); wire the real one in for MTP stats. self.utility_handler.scheduler = self.scheduler @@ -1224,6 +1265,7 @@ def _process_engine_step(self): @staticmethod def run_engine(config: Config, input_address: str, output_address: str): + EngineCore._setup_engine_process("DecodeEngineCore") engine = None try: engine = DecodeEngineCore(config, input_address, output_address) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 4f3183a25f..7e904676d6 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -89,6 +89,7 @@ set_forward_context, set_kv_cache_data, ) +from atom.utils.gc_utils import freeze_gc_heap from atom.utils.selector import get_attn_backend from atom.utils.tbo import ( UBatchSlice, @@ -1587,6 +1588,15 @@ def _estimate_cudagraph_overhead(self): ) return int(overhead) + def freeze_gc_heap(self) -> int: + """RPC target: freeze this worker's startup heap. Pauses here reached + 979 ms, the largest of any process. + + The count is returned because `busy_loop` replies only `if out is not + None` -- an RPC target returning None hangs its `wait_out=True` caller. + """ + return freeze_gc_heap(f"TP{self.rank}") + def get_num_blocks(self) -> dict[str, object]: torch.set_default_device(self.device) config = self.config diff --git a/atom/utils/__init__.py b/atom/utils/__init__.py index ba4426e45b..488f5602de 100644 --- a/atom/utils/__init__.py +++ b/atom/utils/__init__.py @@ -4,7 +4,6 @@ import contextlib import copy import dataclasses -import gc import importlib import ipaddress import json @@ -293,42 +292,6 @@ def enable_orphan_reaping(sig: int = signal.SIGKILL) -> bool: return True -def tune_gc() -> None: - """Stretch the interval between full (generation-2) collections. - - Only gen-2 matters: it is stop-the-world and rescans every tracked - container, so its cost tracks live objects rather than recent garbage, - and at high concurrency it dominates while the gen-0/1 passes are noise. - Raising the thresholds is close to free because reference counting, not - the collector, reclaims everything acyclic -- peak RSS falls too, since a - larger gen-0 threshold lets more objects die before a collection can - promote them. - - gc.freeze() was tried here and removed: once gen-2 stops running there is - nothing left for it to make cheaper, and alone it made collections more - frequent by emptying gen-2, which is the denominator CPython gates full - collections on (long_lived_pending > long_lived_total / 4). - - Thresholds are per-interpreter, so every process has to call this itself: - subprocesses are spawned, and the API server's lifespan runs long after - they exist. Called from the API server, each EngineCore and each - ModelRunner worker; the EngineCore is the one that matters, since its - pauses stall the scheduler and the workers then have nothing to run. - """ - from atom.utils import envs - - thresholds = envs.ATOM_GC_THRESHOLD - if not thresholds: - return - try: - t = tuple(int(x) for x in thresholds.split(",")) - old = gc.get_threshold() - gc.set_threshold(*t) - logger.info("[gc] thresholds %s -> %s", old, t) - except (ValueError, TypeError): - logger.warning("[gc] bad ATOM_GC_THRESHOLD=%r, ignored", thresholds) - - def kill_process_tree(pid: int): """ Kills all descendant processes of the given pid by sending SIGKILL. diff --git a/atom/utils/envs.py b/atom/utils/envs.py index bc39d51433..04030918ef 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -208,10 +208,18 @@ ), # --- Profiling & Logging --- "ATOM_TORCH_PROFILER_DIR": lambda: os.getenv("ATOM_TORCH_PROFILER_DIR", None), + # Move the startup heap (model, compiled graph, tokenizer, KV block pool) + # into CPython's permanent generation once warmup is done, so collections + # stop scanning it. On by default; set 0 to keep the old behaviour. + # See freeze_gc_heap in atom/utils/gc_utils.py. + "ATOM_GC_FREEZE": lambda: os.getenv("ATOM_GC_FREEZE", "1") == "1", + # Log every garbage collection: generation, duration, objects reclaimed. + "ATOM_GC_DEBUG": lambda: os.getenv("ATOM_GC_DEBUG", "0") == "1", # "t0,t1,t2" for gc.set_threshold(); empty keeps CPython's default. # Read independently by the API server, each EngineCore and each - # ModelRunner worker -- thresholds are per-interpreter. The EngineCore is - # the one that matters. See tune_gc in atom/utils/__init__.py. + # ModelRunner worker -- thresholds are per-interpreter. A fallback for + # ATOM_GC_FREEZE=0: freezing removes the cost of a pass, this only spaces + # the passes out. See tune_gc in atom/utils/gc_utils.py. "ATOM_GC_THRESHOLD": lambda: os.getenv("ATOM_GC_THRESHOLD", "").strip(), "ATOM_PROFILER_MORE": lambda: os.getenv("ATOM_PROFILER_MORE", "0") == "1", # When profiling is active, append detailed attention aggregates (sqsq, sqsk, sk) diff --git a/atom/utils/gc_utils.py b/atom/utils/gc_utils.py new file mode 100644 index 0000000000..d20ea6f3c7 --- /dev/null +++ b/atom/utils/gc_utils.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + +"""Garbage-collector policy for the serving processes. + +CPython's gen-2 pass is stop-the-world and traverses every tracked container, +so its cost tracks the live heap -- which here is almost all startup state +(model, compiled graph, tokenizer, KV block pool) that is never garbage. +Measured on DeepSeek-V4-Flash-DSpark tp1: 268 ms in the EngineCore, 979 ms in a +ModelRunner worker, 265 ms in the API server, each reclaiming 0 objects. + +That is the argument for freezing rather than tuning thresholds: the collector +is not merely slow here, it finds nothing. Everything the hot loop allocates is +acyclic, so reference counting takes it. +""" + +import gc +import logging +import time + +logger = logging.getLogger("atom") + + +def tune_gc() -> None: + """Raise the collection thresholds from ``ATOM_GC_THRESHOLD``. + + Per-interpreter, so every process calls it for itself. This spaces passes + out; `freeze_gc_heap` removes what one costs and is on by default, so with + `ATOM_GC_FREEZE=1` there is little left for this to do. + """ + from atom.utils import envs + + thresholds = envs.ATOM_GC_THRESHOLD + if not thresholds: + return + try: + t = tuple(int(x) for x in thresholds.split(",")) + old = gc.get_threshold() + gc.set_threshold(*t) + logger.info("[gc] thresholds %s -> %s", old, t) + except (ValueError, TypeError): + logger.warning("[gc] bad ATOM_GC_THRESHOLD=%r, ignored", thresholds) + + +def freeze_gc_heap(context: str) -> int: + """Move everything alive now into the permanent generation, which + collections skip. Call once, between "startup done" and "traffic starts". + Returns the total frozen count. + + Collecting first is not optional: `gc.freeze()` takes every generation as + it finds it, so current garbage would be made permanently unreclaimable. + One full pass suffices -- `gc.collect()` covers all three generations, and + freezing does not care which one an object ended up in. + + Objects created afterwards are still tracked and collected, so this is not + `gc.disable()` -- a cycle written by later code is still caught. What it + forfeits is anything alive *now* that later becomes garbage, which in these + processes outlives the process anyway. `unfreeze_gc_heap` covers the one + case where that is false: tearing an engine down in-process. + """ + from atom.utils import envs + + if not envs.ATOM_GC_FREEZE: + return 0 + gc.collect() + before = gc.get_freeze_count() + gc.freeze() + total = gc.get_freeze_count() + logger.info( + "[gc] %s: froze %d objects (%d already frozen)", context, total - before, before + ) + return total + + +def unfreeze_gc_heap() -> None: + """Hand the permanent generation back. Required on engine shutdown: a + frozen object is invisible to the collector, so an engine destroyed inside + a live interpreter would leave its weights and KV cache unreachable *and* + uncollectable, which presents as a GPU memory leak.""" + frozen = gc.get_freeze_count() + if frozen: + logger.info("[gc] unfroze %d objects", frozen) + gc.unfreeze() + + +def maybe_attach_gc_debug_callback(context: str) -> None: + """Under ``ATOM_GC_DEBUG``, log every collection. + + Deliberately expensive: it counts the tracked set on each pass, which cost + ~90s of extra startup on a V4-Flash tp1. It is also the only way to see + these pauses -- a stall in the EngineCore idles the workers, and an idle + worker emits no trace event at all. + """ + from atom.utils import envs + + if not envs.ATOM_GC_DEBUG: + return + + started: dict[str, float | int] = {"at": 0.0, "tracked": 0} + + def _log(phase: str, info: dict) -> None: + gen = info.get("generation") + if gen is None: + return + if phase == "start": + started["at"] = time.perf_counter() + started["tracked"] = len(gc.get_objects(gen)) + return + logger.info( + "[gc] %s: gen-%d took %.2f ms, reclaimed %s of %d tracked", + context, + gen, + (time.perf_counter() - started["at"]) * 1e3, + info.get("collected", "?"), + started["tracked"], + ) + + gc.callbacks.append(_log) + logger.info("[gc] %s: debug callback attached", context) diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 3e42418990..bd280d63ee 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -158,6 +158,21 @@ land. See `atom/model_ops/v4_backend_gate.py` for the selector. | **ATOM_ENABLE_DETAILED_ANNOTATION** | bool | 0 (false) | When profiling is active, appends detailed attention aggregates to the `prefill[]`/`decode[]` trace labels: `sqsq` (Σ N_Q²), `sqsk` (Σ N_Q·N_KV), and `sk` (Σ N_KV), where N_Q is the scheduled query tokens and N_KV the KV length per request. Used to estimate attention FLOPs for downstream roofline analysis. | | **ATOM_LOG_MORE** | bool | 0 (false) | If set to `1`, use verbose logging format (includes process name, PID, path, line number, function name). | +## Garbage collection + +CPython's generation-2 pass is stop-the-world and walks every tracked +container, so its cost tracks the live heap — which in a serving process is +almost entirely startup state (model, compiled graph, tokenizer, KV block +pool) that is never garbage. Measured on DeepSeek-V4-Flash-DSpark tp1: 242.8 ms +in the EngineCore, up to 596 ms in a ModelRunner worker, while reclaiming zero +objects once startup was done. See `atom/utils/gc_utils.py`. + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| **ATOM_GC_FREEZE** | bool | 1 (true) | Move the startup heap into CPython's permanent generation once warmup is done, so collections stop scanning it. Applied in the API server, every EngineCore and every ModelRunner worker; undone on engine shutdown so an in-process teardown does not leak. Disaggregated decode is frozen before its block pool exists (it signals ready first), so it covers the model, graph and tokenizer but not the pool. Set `0` to keep the pre-freeze behaviour. | +| **ATOM_GC_DEBUG** | bool | 0 (false) | Log every collection: generation, duration, objects reclaimed, objects tracked. Costly — counting the tracked set on every pass added ~90s of startup on a V4-Flash tp1 — but the only way to see these pauses, since a stall in the EngineCore idles the workers with no event in their torch trace. | +| **ATOM_GC_THRESHOLD** | csv int | "" (= CPython default 700,10,10) | `t0,t1,t2` for `gc.set_threshold()`. Thresholds are per-interpreter, so each process reads it independently. A fallback for `ATOM_GC_FREEZE=0`: this spaces collections out, freezing removes what one costs. | + ### Debug dump (`atom.utils.debug_helper`) Env-gated dump / compare / monkey-patch primitives for forward bisect & diff --git a/tests/test_gc_utils.py b/tests/test_gc_utils.py new file mode 100644 index 0000000000..03156c499e --- /dev/null +++ b/tests/test_gc_utils.py @@ -0,0 +1,158 @@ +# SPDX-License-Identifier: MIT + +"""`gc.freeze()` is what removes the collector's cost; these pin what it costs. + +Freezing is the only knob here that changes what the collector *may not touch*, +so the two things worth a test are the two ways it can be wrong: freezing too +much (garbage made permanent, or never handed back) and freezing too little +(the safety net gone, which would make it `gc.disable()` in disguise). + +Each case carries the failure shape it guards, because a happy-path assertion +would pass against a `freeze_gc_heap` that did nothing at all. +""" + +from __future__ import annotations + +import gc + +import pytest + +from atom.utils.gc_utils import freeze_gc_heap, unfreeze_gc_heap + + +class _Cycle: + """A reference cycle: unreachable by refcount, only the collector frees it.""" + + def __init__(self): + self.self_ref = self + + +@pytest.fixture(autouse=True) +def _leave_gc_as_found(): + """Every test here mutates interpreter-global state.""" + was_enabled = gc.isenabled() + yield + gc.unfreeze() + gc.collect() + if was_enabled: + gc.enable() + + +def test_freezing_takes_the_live_heap_out_of_the_collectors_reach(): + live = [object() for _ in range(64)] + gc.collect() + before = gc.get_freeze_count() + + freeze_gc_heap("test") + + assert gc.get_freeze_count() > before, "nothing was frozen" + # Control: what the frozen set is for. `gc.get_objects()` reports only what + # a collection would still walk, so the drop is the cost that goes away. + assert len(gc.get_objects()) < len(live), ( + "the live heap is still visible to the collector, so freezing bought " "nothing" + ) + + +def test_new_objects_are_still_collected_after_a_freeze(): + """This is what separates freezing from `gc.disable()`. + + Freezing forfeits only what was alive at that instant. A cycle created by + later code -- a code path added next year -- must still be reclaimed, or + this becomes an unbounded leak instead of a bounded one. + """ + freeze_gc_heap("test") + + # `gc.collect()` runs even when the collector is disabled, so calling it + # proves only that the object is not frozen -- it would pass just as well + # against a `freeze_gc_heap` that also called `gc.disable()`. What has to + # be shown is that a collection still fires *on its own*. + assert gc.isenabled(), "freezing disabled the collector" + + fired: list[int] = [] + gc.callbacks.append( + lambda phase, info: fired.append(1) if phase == "stop" else None + ) + try: + threshold = gc.get_threshold()[0] + for _ in range(threshold * 4): + _Cycle() # dropped immediately; only the collector can free it + if fired: + break + finally: + gc.callbacks.pop() + + assert fired, ( + "no automatic collection ran after the freeze -- the safety net for " + "cycles written by later code is gone, which makes this gc.disable()" + ) + + +def test_garbage_alive_at_freeze_time_is_not_made_permanent(): + """`gc.freeze()` moves every generation across exactly as it finds it, so + freezing without collecting first would make current garbage permanently + unreclaimable. `freeze_gc_heap` collects all three generations first.""" + _Cycle() # garbage right now, but no collection has run to notice + freeze_gc_heap("test") + + unfreeze_gc_heap() + # Nothing left for a collection to find: the freeze helper already took it. + assert gc.collect() == 0, "a cycle was carried into the permanent generation" + + +def test_unfreezing_makes_the_startup_heap_reclaimable_again(): + """Required on engine shutdown. Without it an engine torn down inside a + live interpreter leaves its weights unreachable *and* uncollectable, which + presents as a GPU memory leak rather than as anything about GC.""" + doomed = _Cycle() + gc.collect() # promote it, so it is part of the heap being frozen + freeze_gc_heap("test") + del doomed + + # Control: frozen, it is beyond the collector's reach. + assert gc.collect() == 0, "the frozen object was collected; nothing to prove" + + unfreeze_gc_heap() + assert gc.get_freeze_count() == 0 + assert gc.collect() > 0, "unfreezing did not hand the object back" + + +def test_freezing_twice_is_additive_and_harmless(): + """The disaggregated decode path freezes a second time, once its scheduler + exists -- its block pool is built after `EngineCore.__init__` returns.""" + freeze_gc_heap("first") + first = gc.get_freeze_count() + later = [object() for _ in range(64)] + freeze_gc_heap("second") + + assert gc.get_freeze_count() > first, "the second freeze caught nothing" + assert len(later) == 64 # and did not disturb what it froze + + +def test_the_worker_rpc_returns_something(): + """`AsyncIOProc.busy_loop` replies only `if out is not None`, so an RPC + target that returns None hangs its `wait_out=True` caller forever. The + EngineCore freezes its workers through exactly such a call, and a server + started with the first version of it never reached "ready". + + Checked on the unbound function so this needs no GPU and no runner. + """ + import inspect + + from atom.model_engine.model_runner import ModelRunner + + src = inspect.getsource(ModelRunner.freeze_gc_heap) + assert "return " in src, ( + "ModelRunner.freeze_gc_heap returns None, which deadlocks the " + "EngineCore's call_func(..., wait_out=True)" + ) + assert inspect.signature(ModelRunner.freeze_gc_heap).return_annotation is not None + + +def test_the_env_gate_turns_it_off(monkeypatch): + from atom.utils import envs + + monkeypatch.setattr(envs, "ATOM_GC_FREEZE", False) + gc.collect() + before = gc.get_freeze_count() + freeze_gc_heap("test") + assert gc.get_freeze_count() == before From 77c870240bb14340e8cb0b0ebd5afaceca062490 Mon Sep 17 00:00:00 2001 From: Lingpeng Jin <103567126+valarLip@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:56:39 +0000 Subject: [PATCH 2/4] fix(engine): one name per worker process, not three The worker side had the drift the EngineCore side just lost. Three places produced its name and only one knew about data parallelism: - `set_process_title` in `AsyncIOProc.__init__`, dp-aware (`DP0TP0`) - the GC debug callback, `f"TP{rank}"` - `ModelRunner.freeze_gc_heap`, `f"TP{rank}"` So under dp>1 every rank's worker logged as `TP0`, which is exactly the case worth telling apart -- and `ps` disagreed with both log lines. `engine_process_name` and `worker_process_name` now live next to `set_process_title`, since naming a process is what that function is for, and all five call sites take their string from them. `AsyncIOProc` names itself once, before anything logs, which also retires a `try/except Exception` that existed only to repeat the fallback the helper now expresses directly. `EngineCore._process_name_for` moves out to join them rather than staying a method, so a reader asking "what are these processes called" finds one answer. --- atom/model_engine/async_proc.py | 19 ++++++++---------- atom/model_engine/engine_core.py | 13 ++----------- atom/model_engine/model_runner.py | 3 ++- atom/utils/__init__.py | 19 ++++++++++++++++++ tests/test_gc_utils.py | 32 +++++++++++++++++++++++++++++++ 5 files changed, 63 insertions(+), 23 deletions(-) diff --git a/atom/model_engine/async_proc.py b/atom/model_engine/async_proc.py index e1d49a172c..e13c80c544 100644 --- a/atom/model_engine/async_proc.py +++ b/atom/model_engine/async_proc.py @@ -35,6 +35,7 @@ resolve_obj_by_qualname, set_process_title, shutdown_all_processes, + worker_process_name, ) from atom.utils.gc_utils import maybe_attach_gc_debug_callback, tune_gc from atom.utils.numa_utils import numa_bind_to_node @@ -85,13 +86,19 @@ def __init__( enable_orphan_reaping() + # Named once, before anything logs: `ps`, the GC lines here and the + # freeze line the runner emits over RPC all have to be the same string, + # or a dp deployment's workers are indistinguishable from each other. + name = worker_process_name(args[0] if args else None, rank) + set_process_title(name) + # Second-order next to the EngineCore's call but not zero: without it # a freeze still lands at the wave boundary, where the prefill burst # churns enough objects to trigger gen-2 here. Runs before the model # is built so the thresholds cover the startup heap. Freezing that heap # happens later, via the EngineCore's `freeze_gc_heap` RPC. tune_gc() - maybe_attach_gc_debug_callback(f"TP{rank}") + maybe_attach_gc_debug_callback(name) # NUMA-local CPU/memory pinning (see atom.utils.numa_utils). # Auto-detects the GPU's local node by default; gated by @@ -108,16 +115,6 @@ def __init__( except Exception as e: # noqa: BLE001 logger.warning(f"AsyncIOProc({label}): NUMA bind skipped: {e}") self.label = f"AsyncIOProc({label})" - # Set process title so this GPU worker is distinguishable by rank in - # ps/top/rocm-smi (otherwise all workers show as "python"). - try: - cfg = args[0] - if cfg.parallel_config.data_parallel_size > 1: - set_process_title(f"DP{cfg.parallel_config.data_parallel_rank}TP{rank}") - else: - set_process_title(f"TP{rank}") - except Exception: - set_process_title(f"TP{rank}") self.io_addrs = io_addrs self.io_queues = queue.Queue(), queue.Queue() self.io_threads: list[threading.Thread] = [] diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 3de1014b2c..63a3a30da1 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -26,6 +26,7 @@ ) from atom.model_engine.state_runtime import StateRuntime from atom.utils import ( + engine_process_name, envs, init_exit_handler, make_zmq_socket, @@ -204,16 +205,6 @@ def _freeze_after_startup(self): except Exception as e: # noqa: BLE001 - never fail startup over this logger.warning(f"{self._process_name}: worker heap freeze skipped: {e}") - @staticmethod - def _process_name_for(config: Config) -> str: - """Title of the EngineCore process a `config` will run.""" - pc = config.parallel_config - if config.pipeline_parallel_size > 1: - return f"EngineCore_PP{pc.pipeline_parallel_rank}" - if pc.data_parallel_size > 1: - return f"EngineCore_DP{pc.data_parallel_rank}" - return "EngineCore" - @staticmethod def _setup_engine_process(name: str): """Identity, orphan reaping and GC policy, for every `run_engine`. @@ -273,7 +264,7 @@ def _send_engine_dead(self): @staticmethod def run_engine(config: Config, input_address: str, output_address: str): - EngineCore._setup_engine_process(EngineCore._process_name_for(config)) + EngineCore._setup_engine_process(engine_process_name(config)) engine: EngineCore = None try: diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 7e904676d6..bd4096fb2a 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -77,6 +77,7 @@ get_hf_text_config, init_exit_handler, resolve_obj_by_qualname, + worker_process_name, ) from atom.utils.cuda_graph import BatchDescriptor from atom.utils.forward_context import ( @@ -1595,7 +1596,7 @@ def freeze_gc_heap(self) -> int: The count is returned because `busy_loop` replies only `if out is not None` -- an RPC target returning None hangs its `wait_out=True` caller. """ - return freeze_gc_heap(f"TP{self.rank}") + return freeze_gc_heap(worker_process_name(self.config, self.rank)) def get_num_blocks(self) -> dict[str, object]: torch.set_default_device(self.device) diff --git a/atom/utils/__init__.py b/atom/utils/__init__.py index 488f5602de..0b06bcc753 100644 --- a/atom/utils/__init__.py +++ b/atom/utils/__init__.py @@ -194,6 +194,25 @@ def set_process_title(name: str, suffix: str = "", prefix: str | None = None) -> setproctitle.setproctitle(f"{prefix}::{name}") +def engine_process_name(config: "Config") -> str: + """What an EngineCore process is called, in `ps` and in its own logs.""" + pc = config.parallel_config + if config.pipeline_parallel_size > 1: + return f"EngineCore_PP{pc.pipeline_parallel_rank}" + if pc.data_parallel_size > 1: + return f"EngineCore_DP{pc.data_parallel_rank}" + return "EngineCore" + + +def worker_process_name(config: "Config | None", rank: int) -> str: + """What a ModelRunner worker is called. `config` may be absent on paths + that build a worker without one.""" + pc = getattr(config, "parallel_config", None) + if pc is not None and pc.data_parallel_size > 1: + return f"DP{pc.data_parallel_rank}TP{rank}" + return f"TP{rank}" + + def shutdown_all_processes(procs: list[BaseProcess], allowed_seconds: int = 2): # First join any already-exited processes (instant, no wait). for proc in procs: diff --git a/tests/test_gc_utils.py b/tests/test_gc_utils.py index 03156c499e..873925f5ea 100644 --- a/tests/test_gc_utils.py +++ b/tests/test_gc_utils.py @@ -128,6 +128,38 @@ def test_freezing_twice_is_additive_and_harmless(): assert len(later) == 64 # and did not disturb what it froze +def test_a_process_has_exactly_one_name(): + """`ps`, the freeze line and the debug callback all take a name, and they + have to be the same string: under dp>1 a name that omits the dp rank makes + every rank's logs identical, which is the case worth telling apart.""" + from types import SimpleNamespace as NS + + from atom.utils import engine_process_name, worker_process_name + + def cfg(pp=1, dp=1, pp_rank=0, dp_rank=0): + return NS( + pipeline_parallel_size=pp, + parallel_config=NS( + pipeline_parallel_rank=pp_rank, + data_parallel_size=dp, + data_parallel_rank=dp_rank, + ), + ) + + assert engine_process_name(cfg()) == "EngineCore" + assert engine_process_name(cfg(dp=4, dp_rank=2)) == "EngineCore_DP2" + assert engine_process_name(cfg(pp=2, pp_rank=1)) == "EngineCore_PP1" + + assert worker_process_name(cfg(), 3) == "TP3" + assert worker_process_name(cfg(dp=4, dp_rank=2), 3) == "DP2TP3" + # A worker can be built without a config; naming must not be what fails. + assert worker_process_name(None, 3) == "TP3" + + # Control: dp ranks must not collide, which is what a rank-blind name does. + names = {worker_process_name(cfg(dp=4, dp_rank=r), 0) for r in range(4)} + assert len(names) == 4, f"dp ranks share a name: {names}" + + def test_the_worker_rpc_returns_something(): """`AsyncIOProc.busy_loop` replies only `if out is not None`, so an RPC target that returns None hangs its `wait_out=True` caller forever. The From 52017844d927f9e2e23e580cefef15edf8d9f563 Mon Sep 17 00:00:00 2001 From: Lingpeng Jin <103567126+valarLip@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:05:49 +0000 Subject: [PATCH 3/4] fix(entrypoints): the atomesh frontend was the third process to miss this Enumerating which processes get the GC policy has now gone stale twice. #1980 reached the API server but not the disaggregated EngineCores, whose `run_engine` does not call the base one. The first version of this branch reached those but not atomesh, which builds its own engine in `launch_atom_standalone` and never runs the FastAPI lifespan the API server applies the policy from -- so that frontend had neither `tune_gc` (since #1980) nor the freeze, while the EngineCore and workers it spawns had both. It is the API server's counterpart, with the same profile: the tokenizer, the per-request accumulators, ~632k tracked objects and a 265 ms gen-2 pause. Rather than fix the third instance and wait for a fourth, the rule is now tested: `test_every_serving_frontend_applies_the_gc_policy` walks every module that builds an engine and serves, and fails naming the ones that do not apply the policy. Written the obvious way it was vacuous -- an unused import satisfies a search for the name -- so it requires the call. The prose enumerations in `tune_gc` and the env-var docs are replaced by the rule they kept failing to track. --- atom/entrypoints/atomesh/server.py | 13 +++++++++++ atom/utils/gc_utils.py | 6 +++-- docs/environment_variables.md | 2 +- tests/test_gc_utils.py | 35 ++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/atom/entrypoints/atomesh/server.py b/atom/entrypoints/atomesh/server.py index 3c3ce3630b..7ae2216249 100644 --- a/atom/entrypoints/atomesh/server.py +++ b/atom/entrypoints/atomesh/server.py @@ -17,6 +17,12 @@ import sys from typing import Any +from atom.utils.gc_utils import ( + freeze_gc_heap, + maybe_attach_gc_debug_callback, + tune_gc, +) + logger = logging.getLogger("atom") engine: Any | None = None @@ -186,6 +192,13 @@ def launch_atom_standalone(atomesh_runner: Any, raw_args: list[str]) -> None: standalone_args.default_chat_template_kwargs, ) + # This frontend is the api_server's counterpart -- same tokenizer, same + # per-request accumulators -- but it builds its engine itself and never + # runs that FastAPI lifespan, so it has to apply the GC policy here. + tune_gc() + maybe_attach_gc_debug_callback("atomesh") + freeze_gc_heap("atomesh") + print("\033[32mATOM starting...\033[0m") print(f"\033[32mHost: {cli_args['host']}:{cli_args['port']}\033[0m") atomesh_runner.launch_mesh( diff --git a/atom/utils/gc_utils.py b/atom/utils/gc_utils.py index d20ea6f3c7..eea37e803f 100644 --- a/atom/utils/gc_utils.py +++ b/atom/utils/gc_utils.py @@ -24,8 +24,10 @@ def tune_gc() -> None: """Raise the collection thresholds from ``ATOM_GC_THRESHOLD``. - Per-interpreter, so every process calls it for itself. This spaces passes - out; `freeze_gc_heap` removes what one costs and is on by default, so with + Per-interpreter, so every process that serves calls it for itself -- an + enumeration here has gone stale twice, so the rule is the documentation and + `tests/test_gc_utils.py` is what checks it. This spaces passes out; + `freeze_gc_heap` removes what one costs and is on by default, so with `ATOM_GC_FREEZE=1` there is little left for this to do. """ from atom.utils import envs diff --git a/docs/environment_variables.md b/docs/environment_variables.md index bd280d63ee..18f1be9b65 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -169,7 +169,7 @@ objects once startup was done. See `atom/utils/gc_utils.py`. | Variable | Type | Default | Description | |----------|------|---------|-------------| -| **ATOM_GC_FREEZE** | bool | 1 (true) | Move the startup heap into CPython's permanent generation once warmup is done, so collections stop scanning it. Applied in the API server, every EngineCore and every ModelRunner worker; undone on engine shutdown so an in-process teardown does not leak. Disaggregated decode is frozen before its block pool exists (it signals ready first), so it covers the model, graph and tokenizer but not the pool. Set `0` to keep the pre-freeze behaviour. | +| **ATOM_GC_FREEZE** | bool | 1 (true) | Move the startup heap into CPython's permanent generation once warmup is done, so collections stop scanning it. Applied in every process that outlives startup — the API server, the atomesh frontend, every EngineCore and every ModelRunner worker; undone on engine shutdown so an in-process teardown does not leak. Disaggregated decode is frozen before its block pool exists (it signals ready first), so it covers the model, graph and tokenizer but not the pool. Set `0` to keep the pre-freeze behaviour. | | **ATOM_GC_DEBUG** | bool | 0 (false) | Log every collection: generation, duration, objects reclaimed, objects tracked. Costly — counting the tracked set on every pass added ~90s of startup on a V4-Flash tp1 — but the only way to see these pauses, since a stall in the EngineCore idles the workers with no event in their torch trace. | | **ATOM_GC_THRESHOLD** | csv int | "" (= CPython default 700,10,10) | `t0,t1,t2` for `gc.set_threshold()`. Thresholds are per-interpreter, so each process reads it independently. A fallback for `ATOM_GC_FREEZE=0`: this spaces collections out, freezing removes what one costs. | diff --git a/tests/test_gc_utils.py b/tests/test_gc_utils.py index 873925f5ea..ec6e81f1d7 100644 --- a/tests/test_gc_utils.py +++ b/tests/test_gc_utils.py @@ -128,6 +128,41 @@ def test_freezing_twice_is_additive_and_harmless(): assert len(later) == 64 # and did not disturb what it froze +def test_every_serving_frontend_applies_the_gc_policy(): + """The axis, not one instance of it. + + This coverage has gone stale twice already: #1980 reached the API server + but not the disaggregated EngineCores, whose `run_engine` does not call the + base one; and the first version of this change reached those but not + atomesh, which builds its own engine and never runs the FastAPI lifespan. + Both misses are silent -- the process simply keeps its 200ms pauses. + + So the rule is checked rather than the instances: a process that builds an + engine and then serves has to apply the policy. + """ + import pathlib + + root = pathlib.Path(__file__).resolve().parent.parent / "atom" + frontends = sorted( + p + for p in root.rglob("*.py") + # `examples/` are batch scripts: they exit, they do not serve. + if ".create_engine(" in p.read_text() and "examples/" not in p.as_posix() + ) + assert frontends, "no engine frontend found; this test has stopped checking" + missing = [ + p.relative_to(root).as_posix() + for p in frontends + # The call, not the import: an unused import passes for the name. + if "freeze_gc_heap(" not in p.read_text() + ] + assert not missing, ( + f"these build an engine and serve, but never apply the GC policy: " + f"{missing}. Add tune_gc/maybe_attach_gc_debug_callback/freeze_gc_heap " + f"once startup is done, as atom/entrypoints/openai/api_server.py does." + ) + + def test_a_process_has_exactly_one_name(): """`ps`, the freeze line and the debug callback all take a name, and they have to be the same string: under dp>1 a name that omits the dp rank makes From b4d5b097ffa38a7294defaf8a4f88224d4092572 Mon Sep 17 00:00:00 2001 From: Lingpeng Jin <103567126+valarLip@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:22:36 +0000 Subject: [PATCH 4/4] fix(engine): restore the decode freeze, dropped on a misreading An earlier commit on this branch removed `_freeze_after_startup` from `DecodeEngineCore` on the grounds that READY had already gone out, so freezing could catch requests the input thread had queued and make them permanent. That is not what the code does. `DecodeEngineCore.__init__` sets `_ready_deferred = True` before calling `super().__init__()`, precisely so the base's ready signal is suppressed; the real one is sent nine lines *after* the block pool is built, once the kvcache IPC import and graph capture are done. Nothing can have been admitted at that point, so there is no window and the freeze is safe -- and it is what covers decode's 94,763 `Block` objects. The comment left in its place asserted the false premise in so many words, which would have taught the next reader the same wrong thing. --- atom/model_engine/engine_core.py | 8 +++++--- docs/environment_variables.md | 2 +- tests/test_gc_utils.py | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 63a3a30da1..0fb1ad59e2 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -1090,9 +1090,11 @@ def __init__(self, config: Config, input_address: str, output_address: str): disagg_cu_shm_name=config.disagg_cu_shm_name, state_runtime=self.state_runtime, ) - # Deliberately not frozen here, though the block pool only exists now: - # READY went out in `super().__init__()`, so the input thread may - # already hold requests that freezing would make permanent. + # The block pool exists only now, so the freeze inside + # `super().__init__()` ran before there was one to take. Safe to repeat: + # `_ready_deferred` held READY back, so nothing has been admitted yet + # and `gc.freeze()` is additive. + self._freeze_after_startup() # EngineUtilityHandler was built in super().__init__() with scheduler=None # (decode defers scheduler creation); wire the real one in for MTP stats. diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 18f1be9b65..9bc8f63407 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -169,7 +169,7 @@ objects once startup was done. See `atom/utils/gc_utils.py`. | Variable | Type | Default | Description | |----------|------|---------|-------------| -| **ATOM_GC_FREEZE** | bool | 1 (true) | Move the startup heap into CPython's permanent generation once warmup is done, so collections stop scanning it. Applied in every process that outlives startup — the API server, the atomesh frontend, every EngineCore and every ModelRunner worker; undone on engine shutdown so an in-process teardown does not leak. Disaggregated decode is frozen before its block pool exists (it signals ready first), so it covers the model, graph and tokenizer but not the pool. Set `0` to keep the pre-freeze behaviour. | +| **ATOM_GC_FREEZE** | bool | 1 (true) | Move the startup heap into CPython's permanent generation once warmup is done, so collections stop scanning it. Applied in every process that outlives startup — the API server, the atomesh frontend, every EngineCore and every ModelRunner worker; undone on engine shutdown so an in-process teardown does not leak. Set `0` to keep the pre-freeze behaviour. | | **ATOM_GC_DEBUG** | bool | 0 (false) | Log every collection: generation, duration, objects reclaimed, objects tracked. Costly — counting the tracked set on every pass added ~90s of startup on a V4-Flash tp1 — but the only way to see these pauses, since a stall in the EngineCore idles the workers with no event in their torch trace. | | **ATOM_GC_THRESHOLD** | csv int | "" (= CPython default 700,10,10) | `t0,t1,t2` for `gc.set_threshold()`. Thresholds are per-interpreter, so each process reads it independently. A fallback for `ATOM_GC_FREEZE=0`: this spaces collections out, freezing removes what one costs. | diff --git a/tests/test_gc_utils.py b/tests/test_gc_utils.py index ec6e81f1d7..3a20a1d386 100644 --- a/tests/test_gc_utils.py +++ b/tests/test_gc_utils.py @@ -117,8 +117,9 @@ def test_unfreezing_makes_the_startup_heap_reclaimable_again(): def test_freezing_twice_is_additive_and_harmless(): - """The disaggregated decode path freezes a second time, once its scheduler - exists -- its block pool is built after `EngineCore.__init__` returns.""" + """The disaggregated decode path freezes a second time, once its block pool + exists -- it is built later in `DecodeEngineCore.__init__`, after the base + freeze has already run.""" freeze_gc_heap("first") first = gc.get_freeze_count() later = [object() for _ in range(64)]