Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions atom/entrypoints/atomesh/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@
import sys
from typing import Any

from atom.utils.gc_utils import (
freeze_gc_heap,
maybe_attach_gc_debug_callback,
tune_gc,
)

logger = logging.getLogger("atom")

engine: Any | None = None
Expand Down Expand Up @@ -186,6 +192,13 @@ def launch_atom_standalone(atomesh_runner: Any, raw_args: list[str]) -> None:
standalone_args.default_chat_template_kwargs,
)

# This frontend is the api_server's counterpart -- same tokenizer, same
# per-request accumulators -- but it builds its engine itself and never
# runs that FastAPI lifespan, so it has to apply the GC policy here.
tune_gc()
maybe_attach_gc_debug_callback("atomesh")
freeze_gc_heap("atomesh")

print("\033[32mATOM starting...\033[0m")
print(f"\033[32mHost: {cli_args['host']}:{cli_args['port']}\033[0m")
atomesh_runner.launch_mesh(
Expand Down
10 changes: 9 additions & 1 deletion atom/entrypoints/openai/api_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,12 @@
from atom.model_engine.multimodal import build_multimodal_inputs
from atom.model_engine.request import RequestOutput
from atom.model_engine.sequence import new_token_ids
from atom.utils import tune_gc
from atom.utils.arg_parser import FlexibleArgumentParser
from atom.utils.gc_utils import (
freeze_gc_heap,
maybe_attach_gc_debug_callback,
tune_gc,
)

from .chat_encoders import apply_chat_template, load_custom_message_encoder
from .metrics import AtomMetricsExporter
Expand Down Expand Up @@ -1140,8 +1144,12 @@ async def lifespan(app: FastAPI):
global _metrics_refresh_task
logger.info("Server started successfully and ready to accept requests")
tune_gc()
maybe_attach_gc_debug_callback("api_server")
await _refresh_metrics_once()
_metrics_refresh_task = asyncio.create_task(_metrics_refresh_loop())
# The engine was built in `main()`, so this is the last point before the
# first request at which everything reachable is still startup state.
freeze_gc_heap("api_server")
try:
yield
finally:
Expand Down
23 changes: 11 additions & 12 deletions atom/model_engine/async_proc.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,9 @@
resolve_obj_by_qualname,
set_process_title,
shutdown_all_processes,
tune_gc,
worker_process_name,
)
from atom.utils.gc_utils import maybe_attach_gc_debug_callback, tune_gc
from atom.utils.numa_utils import numa_bind_to_node

logger = logging.getLogger("atom")
Expand Down Expand Up @@ -85,11 +86,19 @@ def __init__(

enable_orphan_reaping()

# Named once, before anything logs: `ps`, the GC lines here and the
# freeze line the runner emits over RPC all have to be the same string,
# or a dp deployment's workers are indistinguishable from each other.
name = worker_process_name(args[0] if args else None, rank)
set_process_title(name)

# Second-order next to the EngineCore's call but not zero: without it
# a freeze still lands at the wave boundary, where the prefill burst
# churns enough objects to trigger gen-2 here. Runs before the model
# is built so the thresholds cover the startup heap.
# is built so the thresholds cover the startup heap. Freezing that heap
# happens later, via the EngineCore's `freeze_gc_heap` RPC.
tune_gc()
maybe_attach_gc_debug_callback(name)

# NUMA-local CPU/memory pinning (see atom.utils.numa_utils).
# Auto-detects the GPU's local node by default; gated by
Expand All @@ -106,16 +115,6 @@ def __init__(
except Exception as e: # noqa: BLE001
logger.warning(f"AsyncIOProc({label}): NUMA bind skipped: {e}")
self.label = f"AsyncIOProc({label})"
# Set process title so this GPU worker is distinguishable by rank in
# ps/top/rocm-smi (otherwise all workers show as "python").
try:
cfg = args[0]
if cfg.parallel_config.data_parallel_size > 1:
set_process_title(f"DP{cfg.parallel_config.data_parallel_rank}TP{rank}")
else:
set_process_title(f"TP{rank}")
except Exception:
set_process_title(f"TP{rank}")
self.io_addrs = io_addrs
self.io_queues = queue.Queue(), queue.Queue()
self.io_threads: list[threading.Thread] = []
Expand Down
79 changes: 57 additions & 22 deletions atom/model_engine/engine_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,21 @@
)
from atom.model_engine.state_runtime import StateRuntime
from atom.utils import (
engine_process_name,
envs,
init_exit_handler,
make_zmq_socket,
set_process_title,
tune_gc,
)
from atom.utils.distributed.utils import (
stateless_destroy_torch_distributed_process_group,
)
from atom.utils.gc_utils import (
freeze_gc_heap,
maybe_attach_gc_debug_callback,
tune_gc,
unfreeze_gc_heap,
)

logger = logging.getLogger("atom")

Expand All @@ -55,6 +61,11 @@


class EngineCore:
# This process's name, for the title and every GC log line. A class
# attribute because it is per-process state and each engine is spawned into
# its own interpreter; `_setup_engine_process` is the only writer.
_process_name = "EngineCore"

def __init__(self, config: Config, input_address: str, output_address: str):
self.label = "Engine Core"
self.input_queue = queue.Queue[Sequence]()
Expand Down Expand Up @@ -174,9 +185,42 @@ def __init__(self, config: Config, input_address: str, output_address: str):
scheduler=self.scheduler,
)

# KV cache allocated, graphs captured, BlockPool built: everything this
# process holds for its lifetime exists, and the next thing is traffic.
self._freeze_after_startup()

self._send_ready_signal()
logger.info(f"{self.label}: EngineCore fully initialized and ready")

def _freeze_after_startup(self):
"""Freeze this process and its ModelRunner workers.

The workers are driven from here because only the caller knows warmup
is over -- weights, compile and capture all land via RPCs it sends, and
`--enforce-eager` skips the capture step entirely.
"""
freeze_gc_heap(self._process_name)
try:
self.runner_mgr.call_func("freeze_gc_heap", wait_out=True)
except Exception as e: # noqa: BLE001 - never fail startup over this
logger.warning(f"{self._process_name}: worker heap freeze skipped: {e}")

@staticmethod
def _setup_engine_process(name: str):
"""Identity, orphan reaping and GC policy, for every `run_engine`.

The disaggregated overrides do not call the base one, and each omission
is silent: an unreaped orphan pins VRAM, an untitled process is
`python` in `ps`, an unattached callback leaves ATOM_GC_DEBUG inert.
"""
from atom.utils import enable_orphan_reaping

EngineCore._process_name = name
set_process_title(name)
enable_orphan_reaping() # orphans pin VRAM + IPC handles; see its docs
tune_gc()
maybe_attach_gc_debug_callback(name)

def _send_ready_signal(self):
self.output_queue.put_nowait(("READY", None))

Expand All @@ -192,6 +236,9 @@ def exit(self):
if not self.still_running:
return
self.still_running = False
# Frozen weights and KV cache are unreachable *and* uncollectable, so
# an engine destroyed in-process would read as a GPU memory leak.
unfreeze_gc_heap()
if not hasattr(self, "runner_mgr"):
self._send_engine_dead()
return
Expand All @@ -217,37 +264,17 @@ def _send_engine_dead(self):

@staticmethod
def run_engine(config: Config, input_address: str, output_address: str):
# Bind this EngineCore's lifetime to its parent (the server /
# CoreManager): if the parent exits, have the kernel reap this process —
# and, transitively, the ModelRunner workers it spawns — instead of
# leaving them orphaned. Orphans keep pinning GPU VRAM + the custom
# all-reduce IPC handles / rendezvous TCPStore, which makes the next
# restart reuse a stale hipIpc handle and crash. See
# atom.utils.enable_orphan_reaping for the full rationale.
from atom.utils import enable_orphan_reaping

enable_orphan_reaping()

# The process whose GC pauses idle the GPUs: a gen-2 pass here stops
# the scheduler, and every ModelRunner worker then has nothing to run.
tune_gc()
EngineCore._setup_engine_process(engine_process_name(config))

engine: EngineCore = None
try:
if config.pipeline_parallel_size > 1:
from atom.model_engine.pp_engine_core import PPEngineCoreProc

set_process_title(
f"EngineCore_PP{config.parallel_config.pipeline_parallel_rank}"
)
engine = PPEngineCoreProc(config, input_address, output_address)
elif config.parallel_config.data_parallel_size > 1:
set_process_title(
f"EngineCore_DP{config.parallel_config.data_parallel_rank}"
)
engine = DPEngineCoreProc(config, input_address, output_address)
else:
set_process_title("EngineCore")
engine = EngineCore(config, input_address, output_address)
engine.busy_loop()
except Exception as e:
Expand Down Expand Up @@ -968,6 +995,7 @@ def _process_engine_step(self):

@staticmethod
def run_engine(config: Config, input_address: str, output_address: str):
EngineCore._setup_engine_process("PrefillEngineCore")
engine = None
try:
engine = PrefillEngineCore(config, input_address, output_address)
Expand Down Expand Up @@ -1062,6 +1090,12 @@ def __init__(self, config: Config, input_address: str, output_address: str):
disagg_cu_shm_name=config.disagg_cu_shm_name,
state_runtime=self.state_runtime,
)
# The block pool exists only now, so the freeze inside
# `super().__init__()` ran before there was one to take. Safe to repeat:
# `_ready_deferred` held READY back, so nothing has been admitted yet
# and `gc.freeze()` is additive.
self._freeze_after_startup()

# EngineUtilityHandler was built in super().__init__() with scheduler=None
# (decode defers scheduler creation); wire the real one in for MTP stats.
self.utility_handler.scheduler = self.scheduler
Expand Down Expand Up @@ -1224,6 +1258,7 @@ def _process_engine_step(self):

@staticmethod
def run_engine(config: Config, input_address: str, output_address: str):
EngineCore._setup_engine_process("DecodeEngineCore")
engine = None
try:
engine = DecodeEngineCore(config, input_address, output_address)
Expand Down
11 changes: 11 additions & 0 deletions atom/model_engine/model_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
get_hf_text_config,
init_exit_handler,
resolve_obj_by_qualname,
worker_process_name,
)
from atom.utils.cuda_graph import BatchDescriptor
from atom.utils.forward_context import (
Expand All @@ -89,6 +90,7 @@
set_forward_context,
set_kv_cache_data,
)
from atom.utils.gc_utils import freeze_gc_heap
from atom.utils.selector import get_attn_backend
from atom.utils.tbo import (
UBatchSlice,
Expand Down Expand Up @@ -1587,6 +1589,15 @@ def _estimate_cudagraph_overhead(self):
)
return int(overhead)

def freeze_gc_heap(self) -> int:
"""RPC target: freeze this worker's startup heap. Pauses here reached
979 ms, the largest of any process.

The count is returned because `busy_loop` replies only `if out is not
None` -- an RPC target returning None hangs its `wait_out=True` caller.
"""
return freeze_gc_heap(worker_process_name(self.config, self.rank))

def get_num_blocks(self) -> dict[str, object]:
torch.set_default_device(self.device)
config = self.config
Expand Down
56 changes: 19 additions & 37 deletions atom/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import contextlib
import copy
import dataclasses
import gc
import importlib
import ipaddress
import json
Expand Down Expand Up @@ -195,6 +194,25 @@ def set_process_title(name: str, suffix: str = "", prefix: str | None = None) ->
setproctitle.setproctitle(f"{prefix}::{name}")


def engine_process_name(config: "Config") -> str:
"""What an EngineCore process is called, in `ps` and in its own logs."""
pc = config.parallel_config
if config.pipeline_parallel_size > 1:
return f"EngineCore_PP{pc.pipeline_parallel_rank}"
if pc.data_parallel_size > 1:
return f"EngineCore_DP{pc.data_parallel_rank}"
return "EngineCore"


def worker_process_name(config: "Config | None", rank: int) -> str:
"""What a ModelRunner worker is called. `config` may be absent on paths
that build a worker without one."""
pc = getattr(config, "parallel_config", None)
if pc is not None and pc.data_parallel_size > 1:
return f"DP{pc.data_parallel_rank}TP{rank}"
return f"TP{rank}"


def shutdown_all_processes(procs: list[BaseProcess], allowed_seconds: int = 2):
# First join any already-exited processes (instant, no wait).
for proc in procs:
Expand Down Expand Up @@ -293,42 +311,6 @@ def enable_orphan_reaping(sig: int = signal.SIGKILL) -> bool:
return True


def tune_gc() -> None:
"""Stretch the interval between full (generation-2) collections.

Only gen-2 matters: it is stop-the-world and rescans every tracked
container, so its cost tracks live objects rather than recent garbage,
and at high concurrency it dominates while the gen-0/1 passes are noise.
Raising the thresholds is close to free because reference counting, not
the collector, reclaims everything acyclic -- peak RSS falls too, since a
larger gen-0 threshold lets more objects die before a collection can
promote them.

gc.freeze() was tried here and removed: once gen-2 stops running there is
nothing left for it to make cheaper, and alone it made collections more
frequent by emptying gen-2, which is the denominator CPython gates full
collections on (long_lived_pending > long_lived_total / 4).

Thresholds are per-interpreter, so every process has to call this itself:
subprocesses are spawned, and the API server's lifespan runs long after
they exist. Called from the API server, each EngineCore and each
ModelRunner worker; the EngineCore is the one that matters, since its
pauses stall the scheduler and the workers then have nothing to run.
"""
from atom.utils import envs

thresholds = envs.ATOM_GC_THRESHOLD
if not thresholds:
return
try:
t = tuple(int(x) for x in thresholds.split(","))
old = gc.get_threshold()
gc.set_threshold(*t)
logger.info("[gc] thresholds %s -> %s", old, t)
except (ValueError, TypeError):
logger.warning("[gc] bad ATOM_GC_THRESHOLD=%r, ignored", thresholds)


def kill_process_tree(pid: int):
"""
Kills all descendant processes of the given pid by sending SIGKILL.
Expand Down
12 changes: 10 additions & 2 deletions atom/utils/envs.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,10 +208,18 @@
),
# --- Profiling & Logging ---
"ATOM_TORCH_PROFILER_DIR": lambda: os.getenv("ATOM_TORCH_PROFILER_DIR", None),
# Move the startup heap (model, compiled graph, tokenizer, KV block pool)
# into CPython's permanent generation once warmup is done, so collections
# stop scanning it. On by default; set 0 to keep the old behaviour.
# See freeze_gc_heap in atom/utils/gc_utils.py.
"ATOM_GC_FREEZE": lambda: os.getenv("ATOM_GC_FREEZE", "1") == "1",
# Log every garbage collection: generation, duration, objects reclaimed.
"ATOM_GC_DEBUG": lambda: os.getenv("ATOM_GC_DEBUG", "0") == "1",
# "t0,t1,t2" for gc.set_threshold(); empty keeps CPython's default.
# Read independently by the API server, each EngineCore and each
# ModelRunner worker -- thresholds are per-interpreter. The EngineCore is
# the one that matters. See tune_gc in atom/utils/__init__.py.
# ModelRunner worker -- thresholds are per-interpreter. A fallback for
# ATOM_GC_FREEZE=0: freezing removes the cost of a pass, this only spaces
# the passes out. See tune_gc in atom/utils/gc_utils.py.
"ATOM_GC_THRESHOLD": lambda: os.getenv("ATOM_GC_THRESHOLD", "").strip(),
"ATOM_PROFILER_MORE": lambda: os.getenv("ATOM_PROFILER_MORE", "0") == "1",
# When profiling is active, append detailed attention aggregates (sqsq, sqsk, sk)
Expand Down
Loading