Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions atom/entrypoints/atomesh/atom_standalone_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
)
from atom.entrypoints.openai.sse import data_frame
from atom.entrypoints.openai.tool_parser import ToolCallStreamParser
from atom.model_engine.sequence import new_token_ids

logger = logging.getLogger("atom")

Expand Down Expand Up @@ -331,7 +332,10 @@ def __init__(
self.started_at = time.time()
self.first_token_at: float | None = None
self.last_token_at: float | None = None
self.token_ids: list[int] = []
# An array for the same reason the engine's `token_ids` is one. It
# stays one: this dict goes to `build_*_response`, which reads
# `text` and the counters and never `token_ids`.
self.token_ids = new_token_ids()
self.finish_reason: str | None = None
self.num_tokens_input = 0
self.kv_transfer_output_meta_info: Any = None
Expand Down Expand Up @@ -400,7 +404,7 @@ def __init__(
self.future = future
self.n = n
self.started_at = time.time()
self.per_tokens: list[list[int]] = [[] for _ in range(n)]
self.per_tokens = [new_token_ids() for _ in range(n)]
self.per_first_token_at: list[float | None] = [None] * n
self.per_last_token_at: list[float | None] = [None] * n
self.per_finish_reason: list[str | None] = [None] * n
Expand Down
20 changes: 16 additions & 4 deletions atom/entrypoints/atomesh/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,21 @@
from __future__ import annotations

import argparse
from dataclasses import dataclass
import importlib
import importlib.util
import json
import logging
from pathlib import Path
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from atom.utils.gc_utils import (
freeze_gc_heap,
maybe_attach_gc_debug_callback,
tune_gc,
)

logger = logging.getLogger("atom")

engine: Any | None = None
Expand Down Expand Up @@ -76,7 +82,7 @@ def print_version(verbose: bool = False) -> None:
else atomesh_runner.version_string
)
print(version_fn())
except Exception:
except Exception: # noqa: BLE001 - a version banner must not fail the CLI
print("Atomesh Python interface")


Expand All @@ -101,7 +107,6 @@ def initialize_standalone_service(
) -> Any:
from atom.entrypoints.atomesh.atom_standalone_service import AtomStandaloneService

global engine, tokenizer
return AtomStandaloneService(
engine=engine,
tokenizer=tokenizer,
Expand Down Expand Up @@ -186,6 +191,13 @@ def launch_atom_standalone(atomesh_runner: Any, raw_args: list[str]) -> None:
standalone_args.default_chat_template_kwargs,
)

# This frontend is the api_server's counterpart -- same tokenizer, same
# per-request accumulators -- but it builds its engine itself and never
# runs that FastAPI lifespan, so it has to apply the GC policy here.
tune_gc()
maybe_attach_gc_debug_callback("atomesh")
freeze_gc_heap("atomesh")

print("\033[32mATOM starting...\033[0m")
print(f"\033[32mHost: {cli_args['host']}:{cli_args['port']}\033[0m")
atomesh_runner.launch_mesh(
Expand Down
23 changes: 19 additions & 4 deletions atom/entrypoints/openai/api_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,13 @@
from atom.model_engine.llm_engine import _load_tokenizer
from atom.model_engine.multimodal import build_multimodal_inputs
from atom.model_engine.request import RequestOutput
from atom.utils import tune_gc
from atom.model_engine.sequence import new_token_ids
from atom.utils.arg_parser import FlexibleArgumentParser
from atom.utils.gc_utils import (
freeze_gc_heap,
maybe_attach_gc_debug_callback,
tune_gc,
)

from .chat_encoders import apply_chat_template, load_custom_message_encoder
from .metrics import AtomMetricsExporter
Expand Down Expand Up @@ -475,7 +480,13 @@ async def generate_async(
started_at = time.time()
first_token_at: float | None = None
last_token_at: float | None = None
all_token_ids: list[int] = []
# An array, not a list: this grows for the whole life of the request,
# and one boxed PyInt per token is what the collector then walks on
# every pass. It stays an array all the way out -- the dict below is
# an internal hand-off to `build_*_response`, which reads `text` and
# the counters and never `token_ids`, so nothing serializes it. A
# consumer that starts reading that key has to convert.
all_token_ids = new_token_ids()
finish_reason: str | None = None
seq = None
kv_transfer_output_meta_info = None
Expand Down Expand Up @@ -594,7 +605,7 @@ async def generate_async_multimodal(
started_at = time.time()
first_token_at: float | None = None
last_token_at: float | None = None
all_token_ids: list[int] = []
all_token_ids = new_token_ids()
finish_reason: str | None = None
seq = None

Expand Down Expand Up @@ -700,7 +711,7 @@ async def generate_async_fanout(
loop = asyncio.get_running_loop()

started_at = time.time()
per_tokens: list[list[int]] = [[] for _ in range(n)]
per_tokens = [new_token_ids() for _ in range(n)]
per_first_token_at: list[float | None] = [None] * n
per_last_token_at: list[float | None] = [None] * n
per_finish_reason: list[str | None] = [None] * n
Expand Down Expand Up @@ -1133,8 +1144,12 @@ async def lifespan(app: FastAPI):
global _metrics_refresh_task
logger.info("Server started successfully and ready to accept requests")
tune_gc()
maybe_attach_gc_debug_callback("api_server")
await _refresh_metrics_once()
_metrics_refresh_task = asyncio.create_task(_metrics_refresh_loop())
# The engine was built in `main()`, so this is the last point before the
# first request at which everything reachable is still startup state.
freeze_gc_heap("api_server")
try:
yield
finally:
Expand Down
8 changes: 7 additions & 1 deletion atom/entrypoints/openai/streaming_dispatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,14 @@
landing point each stream's SSE generator reads from.
"""

import array
import threading
from asyncio import AbstractEventLoop, Event
from dataclasses import dataclass, field
from typing import Any, NamedTuple

from atom.model_engine.sequence import new_token_ids

# Fields a later chunk overrides on the one it merges into, when it has a value
# of its own. The SSE consumers keep the newest non-empty value they see, so
# merging this way hands them what reading each chunk separately would have.
Expand All @@ -25,7 +28,10 @@ class IncrementalStreamDetokenizer:
"""Decode token deltas without emitting incomplete UTF-8 characters."""

tokenizer: Any
tokens: list[int] = field(default_factory=list)
# Grows for the whole life of a stream, one entry per token, and only ever
# sliced into `tokenizer.decode` -- which takes an array. Nothing here is
# serialized, so this one has no boundary to convert back at.
tokens: array.array = field(default_factory=new_token_ids)
prefix_offset: int = 0
read_offset: int = 0

Expand Down
3 changes: 2 additions & 1 deletion atom/kv_transfer/offload/hybrid/dsv4/policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from __future__ import annotations

import array
import hashlib
import heapq
import json
Expand Down Expand Up @@ -290,7 +291,7 @@ def _committed_sidecar_capacity(kvc) -> int:


def _chained_prefix_hashes(
token_ids: list[int],
token_ids: array.array,
hash_block_size: int,
) -> dict[int, int]:
"""Return each full-block prefix hash using BlockManager's exact chain."""
Expand Down
23 changes: 11 additions & 12 deletions atom/model_engine/async_proc.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,9 @@
resolve_obj_by_qualname,
set_process_title,
shutdown_all_processes,
tune_gc,
worker_process_name,
)
from atom.utils.gc_utils import maybe_attach_gc_debug_callback, tune_gc
from atom.utils.numa_utils import numa_bind_to_node

logger = logging.getLogger("atom")
Expand Down Expand Up @@ -85,11 +86,19 @@ def __init__(

enable_orphan_reaping()

# Named once, before anything logs: `ps`, the GC lines here and the
# freeze line the runner emits over RPC all have to be the same string,
# or a dp deployment's workers are indistinguishable from each other.
name = worker_process_name(args[0] if args else None, rank)
set_process_title(name)

# Second-order next to the EngineCore's call but not zero: without it
# a freeze still lands at the wave boundary, where the prefill burst
# churns enough objects to trigger gen-2 here. Runs before the model
# is built so the thresholds cover the startup heap.
# is built so the thresholds cover the startup heap. Freezing that heap
# happens later, via the EngineCore's `freeze_gc_heap` RPC.
tune_gc()
maybe_attach_gc_debug_callback(name)

# NUMA-local CPU/memory pinning (see atom.utils.numa_utils).
# Auto-detects the GPU's local node by default; gated by
Expand All @@ -106,16 +115,6 @@ def __init__(
except Exception as e: # noqa: BLE001
logger.warning(f"AsyncIOProc({label}): NUMA bind skipped: {e}")
self.label = f"AsyncIOProc({label})"
# Set process title so this GPU worker is distinguishable by rank in
# ps/top/rocm-smi (otherwise all workers show as "python").
try:
cfg = args[0]
if cfg.parallel_config.data_parallel_size > 1:
set_process_title(f"DP{cfg.parallel_config.data_parallel_rank}TP{rank}")
else:
set_process_title(f"TP{rank}")
except Exception:
set_process_title(f"TP{rank}")
self.io_addrs = io_addrs
self.io_queues = queue.Queue(), queue.Queue()
self.io_threads: list[threading.Thread] = []
Expand Down
27 changes: 20 additions & 7 deletions atom/model_engine/block_manager.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: MIT
# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.

import array
import logging
from math import inf, isinf

Expand Down Expand Up @@ -40,6 +41,13 @@ def _make_block_stored(
medium: str = MEDIUM_GPU,
) -> BlockStored:
"""Construct a BlockStored event from a coalesced run of new blocks."""
# A list, not the `array("i")` the publish paths carry: the event is
# msgpack-encoded and msgspec has no encoding for an array. The publisher
# counts encode failures rather than raising, so an array here takes the
# event stream down without stopping anything.
assert isinstance(
tokens, list
), f"BlockStored.token_ids must be a list, got {type(tokens).__name__}"
return BlockStored(
block_hashes=hashes,
parent_block_hash=parent,
Expand Down Expand Up @@ -179,14 +187,16 @@ def __init__(
self.demands_declined_no_room: int = 0

@classmethod
def compute_hash(cls, token_ids: list[int], prefix: int = -1):
def compute_hash(cls, token_ids: array.array, prefix: int = -1):
h = xxhash.xxh64()
if prefix != -1:
h.update(prefix.to_bytes(8, "little"))
# dtype pinned: `np.array` infers int64 from a list and int32 from an
# `array("i")`, so the digest used to depend on the caller's Python
# type. int64 is what lists gave, so pinning it leaves every existing
# hash where it was.
# dtype pinned even though every caller now passes an `array("i")`:
# `np.array` infers int64 from a list and int32 from an array, so the
# digest used to depend on the caller's Python type. int64 is what
# lists gave, which leaves every hash recorded before that where it
# was -- and keeps a caller who does pass a list from silently
# computing a different one.
h.update(np.asarray(token_ids, dtype=np.int64).tobytes())
return h.intdigest()

Expand Down Expand Up @@ -308,7 +318,7 @@ def _n_hash_blocks(self, seq: Sequence) -> int:
hbs = self.hash_block_size
return (len(seq) + hbs - 1) // hbs

def _hash_block_tokens(self, seq: Sequence, i: int) -> list[int]:
def _hash_block_tokens(self, seq: Sequence, i: int) -> array.array:
hbs = self.hash_block_size
return seq.token_ids[i * hbs : (i + 1) * hbs]

Expand Down Expand Up @@ -1043,7 +1053,10 @@ def publish_loaded_prefix(
self._event_log.append(
_make_block_stored(
[block_hash],
token_ids,
# The only BlockStored site fed straight from
# `_hash_block_tokens`; the other two accumulate
# into a list already. See `_make_block_stored`.
list(token_ids),
parent_hash if parent_hash != -1 else None,
self.block_size,
)
Expand Down
9 changes: 5 additions & 4 deletions atom/model_engine/block_pool.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: MIT
# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.

import array
from collections import OrderedDict
from collections.abc import Callable, Hashable, Iterable
from dataclasses import dataclass
Expand Down Expand Up @@ -115,7 +116,7 @@ def lookup(self, h: int) -> int:
"""Block id indexed under content hash `h`, or -1."""
return self._hash_to_block_id.get(h, -1)

def publish(self, block_id: int, h: int, token_ids: list[int]) -> None:
def publish(self, block_id: int, h: int, token_ids: array.array) -> None:
"""Index `block_id` under the content hash of the tokens it now holds."""
block = self.blocks[block_id]
block.update(h, token_ids)
Expand All @@ -131,7 +132,7 @@ def clear_index(self) -> None:
for block in self.blocks:
if block.ref_count == 0:
block.hash = -1
block.token_ids = []
block.token_ids = array.array("i")
# Every free block is vacant now, and which container an id is in is
# only ever decided from its hash — so the split has to be redrawn
# here rather than left to drift.
Expand All @@ -147,7 +148,7 @@ def _unindex(self, block_id: int) -> None:
if self._on_evict is not None:
self._on_evict(block.hash)
block.hash = -1
block.token_ids = []
block.token_ids = array.array("i")

# ---------------------------- allocation ------------------------------- #
def _take_free(self) -> int:
Expand Down Expand Up @@ -331,4 +332,4 @@ def _adopt(self, destination: int, source: int) -> None:
self._hash_to_block_id[src.hash] = destination
self._used.discard(source)
self._used.add(destination)
src.ref_count, src.hash, src.token_ids = 0, -1, []
src.ref_count, src.hash, src.token_ids = 0, -1, array.array("i")
Loading
Loading