Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,7 @@ def update_state_after_alloc(self, seq: Sequence) -> None:
assert (
not self.is_producer
), "Only the decode (consumer) side handles do_remote_prefill"
self._reqs_need_recv[seq.id] = (seq, seq.block_table, slot_index)
self._reqs_need_recv[seq.id] = (seq, list(seq.block_table), slot_index)
params["do_remote_prefill"] = False
params["local_slot_index"] = slot_index
# PD incremental: skip leading blocks already in the decode node's
Expand All @@ -404,7 +404,7 @@ def update_state_after_alloc(self, seq: Sequence) -> None:
# Producer side: queue block_ids for the write listener to look up
if params.get("do_remote_decode"):
assert self.is_producer, "Only the producer side handles do_remote_decode"
self._reqs_need_save[seq.id] = (seq, seq.block_table, slot_index)
self._reqs_need_save[seq.id] = (seq, list(seq.block_table), slot_index)
logger.debug(
"Queued req %s for KV save (%d blocks, slot=%d)",
seq.id,
Expand All @@ -421,7 +421,7 @@ def request_finished(self, seq: Sequence) -> None:
seq.kv_transfer_params_output = {
"do_remote_prefill": True,
"do_remote_decode": False,
"remote_block_ids": seq.block_table.copy(),
"remote_block_ids": list(seq.block_table),
# The consumer's SWA ring slot; the producer keys the SWA region
# transfer by it. Empty for backends with no SWA state.
"remote_swa_block_ids": _swa_ring_ids(seq),
Expand Down
10 changes: 5 additions & 5 deletions atom/kv_transfer/disaggregation/moriio/moriio_connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,19 @@
import msgspec
import numpy as np
import zmq
from aiter.dist.parallel_state import get_dp_group, get_tp_group

from atom.config import Config
from atom.kv_transfer.disaggregation.base import (
KVConnectorBase,
KVConnectorSchedulerBase,
)
from atom.kv_transfer.disaggregation.moriio.moriio_common import (
_MORIIO_AVAILABLE,
MoRIIOAgentMetadata,
MoRIIOConstants,
_MORIIO_AVAILABLE,
get_port_offset,
)
from atom.kv_transfer.disaggregation.utils import chunk_tensor_for_rdma
from atom.kv_transfer.disaggregation.moriio.moriio_engine import MoRIIOWrapper
from atom.kv_transfer.disaggregation.types import (
ConnectorMetadata,
Expand All @@ -43,14 +43,14 @@
ReqMeta,
TransferId,
)
from atom.kv_transfer.disaggregation.utils import chunk_tensor_for_rdma
from atom.model_engine.sequence import Sequence
from atom.utils import (
get_open_port,
make_zmq_path,
zmq_socket_ctx,
)
from atom.utils.network import get_ip
from aiter.dist.parallel_state import get_dp_group, get_tp_group

if _MORIIO_AVAILABLE:
from mori.io import (
Expand Down Expand Up @@ -959,7 +959,7 @@ def update_state_after_alloc(self, seq: Sequence) -> None:
assert (
not self.is_producer
), "Only the decode (consumer) side handles do_remote_prefill"
self._reqs_need_recv[seq.id] = (seq, seq.block_table)
self._reqs_need_recv[seq.id] = (seq, list(seq.block_table))
params["do_remote_prefill"] = False
logger.debug(
"Queued req %s for remote KV loading (%d blocks)",
Expand All @@ -983,7 +983,7 @@ def request_finished(self, seq: Sequence) -> None:
seq.kv_transfer_params_output = {
"do_remote_prefill": True,
"do_remote_decode": False,
"remote_block_ids": seq.block_table.copy(),
"remote_block_ids": list(seq.block_table),
"remote_engine_id": self.engine_id,
"remote_host": self.host_ip,
"remote_port": self.handshake_port,
Expand Down
8 changes: 6 additions & 2 deletions atom/model_engine/block_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,11 @@ def compute_hash(cls, token_ids: list[int], prefix: int = -1):
h = xxhash.xxh64()
if prefix != -1:
h.update(prefix.to_bytes(8, "little"))
h.update(np.array(token_ids).tobytes())
# dtype pinned: `np.array` infers int64 from a list and int32 from an
# `array("i")`, so the digest used to depend on the caller's Python
# type. int64 is what lists gave, so pinning it leaves every existing
# hash where it was.
h.update(np.asarray(token_ids, dtype=np.int64).tobytes())
return h.intdigest()

def complete_previous_state_batch(self) -> None:
Expand Down Expand Up @@ -1098,7 +1102,7 @@ def deallocate(self, seq: Sequence):
# go back on the free list, so the intent dies with it.
if self.paged_state_checkpoints is not None:
self.paged_state_checkpoints.forget_pending(seq)
seq.block_table.clear()
del seq.block_table[:] # `array("i")` has no `.clear()`
if seq.has_per_req_cache and seq.per_req_cache_group >= 0:
# No next forward will read a pending fork source after deallocation.
self.state.release(seq.per_req_cache_group)
Expand Down
9 changes: 7 additions & 2 deletions atom/model_engine/engine_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,12 @@
from atom.model_engine.engine_core_protocol import EngineCoreRequestType
from atom.model_engine.engine_utility import EngineUtilityHandler
from atom.model_engine.scheduler import DecodeScheduler, PrefillScheduler, Scheduler
from atom.model_engine.sequence import Sequence, SequenceStatus, get_exit_sequence
from atom.model_engine.sequence import (
Sequence,
SequenceStatus,
get_exit_sequence,
new_block_table,
)
from atom.model_engine.state_runtime import StateRuntime
from atom.utils import (
envs,
Expand Down Expand Up @@ -904,7 +909,7 @@ def _apply_pending_assignments(self):
for seq in self.scheduler.waiting:
if seq.id in self._pending_assignments:
assignment = self._pending_assignments.pop(seq.id)
seq.block_table = list(assignment.block_table)
seq.block_table = new_block_table(assignment.block_table)
seq.num_cached_tokens = assignment.num_cached_tokens

def _process_engine_step(self):
Expand Down
4 changes: 3 additions & 1 deletion atom/model_engine/llm_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -751,7 +751,9 @@ def postprocess(self, reqs: list[Sequence]):
)
outputs[req.id] = {
"text": output_str,
"token_ids": req.completion_token_ids,
# `list`, not the `array("i")` slice: this is what `generate()`
# hands a caller, and the storage type is ours to change.
"token_ids": list(req.completion_token_ids),
"logprobs": req.logprobs if req.return_logprobs else None,
"latency": req.leave_time - req.arrive_time,
"finish_reason": req.leave_reason,
Expand Down
18 changes: 12 additions & 6 deletions atom/model_engine/model_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,12 @@
from atom.model_engine.page_unit_checkpoint import PagedStateCheckpointSpec
from atom.model_engine.run_labels import build_run_label
from atom.model_engine.scheduler import ScheduledBatch, ScheduledBatchOutput
from atom.model_engine.sequence import Sequence, SequenceStatus, SequenceType
from atom.model_engine.sequence import (
Sequence,
SequenceStatus,
SequenceType,
new_block_table,
)
from atom.model_engine.state_runtime import StateRuntime
from atom.model_loader.loader import load_model
from atom.model_ops.attentions.sub_pool_spec import (
Expand Down Expand Up @@ -1146,7 +1151,7 @@ def dummy_execution(self):
)
seq.status = SequenceStatus.RUNNING
seq.type = SequenceType.DECODE
seq.block_table = [0]
seq.block_table = new_block_table([0])

spec_tokens = {seq.id: np.zeros(mtp_k, dtype=np.int32)} if mtp_k > 0 else None
dummy_batch = ScheduledBatch(
Expand Down Expand Up @@ -1320,10 +1325,11 @@ def _init_forward_vars_ring(self):
)

def _clone_slot(src: dict) -> dict:
# Only CpuGpuBuffers are per-forward host-pinned staging buffers that
# get overwritten each forward. Everything else (the eager `outputs`
# tensor, scalar `mtp_k`, ...) is either unused on the eager PP path
# or immutable, so share it by reference.
# CpuGpuBuffers are the per-forward staging buffers, and only their
# host half can be rewritten while an earlier microbatch's kernels
# are still reading. Everything else is immutable, unused on the
# eager PP path, or device-only, where the stream orders the writing
# kernel after those readers.
return {
k: (v.clone() if isinstance(v, CpuGpuBuffer) else v)
for k, v in src.items()
Expand Down
89 changes: 60 additions & 29 deletions atom/model_engine/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,12 @@
from atom.kv_transfer.disaggregation import KVConnectorOutput
from atom.model_engine.block_manager import BlockManager
from atom.model_engine.request import RequestOutput
from atom.model_engine.sequence import Sequence, SequenceStatus, SequenceType
from atom.model_engine.sequence import (
Sequence,
SequenceStatus,
SequenceType,
new_token_ids,
)
from atom.model_engine.state_runtime import (
DEFAULT_STATE_RUNTIME,
StateMaintenanceOps,
Expand Down Expand Up @@ -381,9 +386,11 @@ def __init__(
# `context_lens` is set further down, once `num_cached_tokens` is known:
# a chunked prefill's context ends at this chunk, not at the whole
# prompt, so `seq.num_tokens` is only right for decode.
self.num_rejected = np.asarray(
[seq.num_rejected for seq in seqs.values()], dtype=np.int32
)
# Kept as a list too: the offset loop below reads it per sequence, and
# `arr[i]` builds a numpy scalar where a list index does not. The list
# is what `np.asarray` consumes anyway, so holding onto it is free.
num_rejected = [seq.num_rejected for seq in seqs.values()]
self.num_rejected = np.asarray(num_rejected, dtype=np.int32)
self.num_bonus = np.asarray(
[seq.num_bonus_tokens for seq in seqs.values()], dtype=np.int32
)
Expand Down Expand Up @@ -458,38 +465,62 @@ def __init__(
dtype=np.int32,
)

# Compute token offsets: prefill uses num_cached_tokens, decode uses existing formula
self.scheduled_tokens = np.empty(total_tokens_num, dtype=np.int32)
pos = 0
# Each sequence's window, staged into one array rather than assigned
# per sequence: a numpy slice-assign costs ~245ns of dispatch whatever
# its length, and at decode a window is a single token. `extend`
# between two `array("i")` is a memcpy. 4x at bs=256.
staged = new_token_ids()
for i, (seq, num) in enumerate(zip(seqs.values(), num_scheduled_tokens)):
if seq.type == SequenceType.PREFILL:
offset = self.num_cached_tokens[i]
else:
offset = seq.num_tokens - self.num_rejected[i] - num
self.scheduled_tokens[pos : pos + num] = seq.token_ids[
offset : offset + num
]
pos += num
offset = seq.num_tokens - num_rejected[i] - num
staged.extend(seq.token_ids[offset : offset + num])
# Checked here because nothing downstream will: the array below wraps
# whatever length was staged, and a sequence too short to fill its
# window would otherwise leave the batch quietly short.
if len(staged) != total_tokens_num:
raise ValueError(
f"staged {len(staged)} tokens for a batch of {total_tokens_num}: "
"a sequence is shorter than the window scheduled for it"
)
# Wrapped, not copied into a fresh array: consumers only ever read this
# or rebind it, so the staging buffer can be the buffer.
self.scheduled_tokens = np.frombuffer(staged, dtype=np.int32)

if num_spec_step > 0 and scheduled_spec_decode_tokens is not None:
# One row per sequence, in batch order, zero-filled where a sequence
# has no drafts yet — which is every sequence entering decode
# straight off its own prefill. The caller's dict is keyed by
# request id and only holds sequences that DO have drafts, so
# densifying it with `list(...values())` drops those rows and shifts
# every later one onto the wrong sequence. Consumers index this by
# batch position (`prepare_input_ids`), not by request id, so that
# shift silently feeds one sequence another's draft tokens, and
# raises IndexError once the array is short enough.
self.scheduled_spec_decode_tokens = np.zeros(
(len(seqs), num_spec_step), dtype=np.int32
)
for i, req_id in enumerate(self.req_ids):
# One row per sequence, in batch order. The caller's dict is keyed
# by request id and holds only sequences that DO have drafts, so
# densifying it with `list(...values())` drops the others and
# shifts every later row onto the wrong sequence — consumers index
# this by batch position (`prepare_input_ids`), which turns that
# shift into one sequence silently reading another's drafts.
#
# Rows are collected and written in one assign: a per-row write
# costs ~245ns of dispatch against a row this short, so the marshal
# was mostly dispatch (3.7x at bs=256). Rows that are not
# full width are padded rather than dropping the batch back to the
# per-row form, because at a large batch a sequence joins decode
# nearly every step and would take every other sequence with it.
no_drafts = np.zeros(num_spec_step, dtype=np.int32)
rows = []
for req_id in self.req_ids:
drafts = scheduled_spec_decode_tokens.get(req_id)
if drafts is None or drafts.size == 0:
continue
width = min(drafts.size, num_spec_step)
self.scheduled_spec_decode_tokens[i, :width] = drafts[:width]
if drafts is not None and drafts.size == num_spec_step:
rows.append(drafts)
elif drafts is None or drafts.size == 0:
rows.append(no_drafts)
else: # short or long: onto a row of its own, never the shared one
padded = no_drafts.copy()
padded[: min(drafts.size, num_spec_step)] = drafts[:num_spec_step]
rows.append(padded)
# `concatenate` rejects an empty list, which a warmup batch is.
flat = (
np.concatenate(rows, dtype=np.int32)
if rows
else np.empty(0, dtype=np.int32)
)
self.scheduled_spec_decode_tokens = flat.reshape(len(seqs), num_spec_step)
self.block_tables = [
seq.block_table for seq in seqs.values() if seq.block_table
]
Expand Down
42 changes: 38 additions & 4 deletions atom/model_engine/sequence.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# SPDX-License-Identifier: MIT
# Copyright (C) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.

import array
from collections.abc import Callable
from copy import copy
from enum import Enum, auto
from itertools import count
from typing import Any
Expand All @@ -12,6 +12,36 @@
from atom.sampling_params import SamplingParams


def new_token_ids(token_ids=()) -> array.array:
"""A sequence's token ids.

An `array("i")` rather than a list for two reasons that both scale with
context. The scheduler copies a slice of this into the flat
`scheduled_tokens` buffer every step -- a whole chunk of it on a prefill --
and from a list that is one CPython int unboxing per token, 0.28 ms for a
16k chunk against 0.001 ms from an array. And a list of 100k ids costs
3.4 MiB of boxed ints where the array costs 0.38.

Behaves as a list for append/pop/index/len/iterate/slice-delete, and holds
negative ids (the exit sentinel is -1). It does NOT compare equal to a
list, which is why `stop_token_sequences` below is converted too, and why
`BlockManager.compute_hash` pins its dtype.
"""
return array.array("i", token_ids)


def new_block_table(block_ids=()) -> array.array:
"""A sequence's physical block ids.

An `array("i")` rather than a list because every forward marshals these
into the int32 `block_tables` buffer, where a list costs one CPython int
unboxing per block (~17k per step at 50 seqs x 100k ctx) and an array is a
memcpy. It behaves as a list for append/pop/index/len/iterate; it has no
`.clear()` (use `del bt[:]`) and no `.copy()` (use `list(bt)`).
"""
return array.array("i", block_ids)


class SequenceStatus(Enum):
WAITING_FOR_REMOTE_KVS = auto()
WAITING = auto()
Expand Down Expand Up @@ -69,7 +99,7 @@ def __init__(
self.external_request_id = request_id
self.status = SequenceStatus.WAITING
self.type = SequenceType.DUMMY
self.token_ids = copy(token_ids)
self.token_ids = new_token_ids(token_ids)
self.last_token = token_ids[-1]
self.num_draft_tokens = num_draft_tokens
# `has_per_req_cache=True` means this seq's attention type maintains
Expand Down Expand Up @@ -127,7 +157,7 @@ def __init__(
# garbage sampled tokens from intermediate chunks and to skip the
# scheduler's Phase 1 scan when no partials exist.
self.is_partial_prefill = False
self.block_table = []
self.block_table = new_block_table()
# Per-request cache slot index (filled by BlockManager.allocate()).
# -1 = unallocated. The slot indexes into the per-req cache tensors
# owned by ModelRunner (e.g. mamba_k_cache for GDN).
Expand All @@ -145,7 +175,11 @@ def __init__(
self.max_tokens = sampling_params.max_tokens
self.ignore_eos = sampling_params.ignore_eos
self.stop_strings = sampling_params.stop_strings
self.stop_token_sequences = stop_token_sequences or []
# Same type as `token_ids`, because the stop check compares a slice of
# that against these and an `array("i")` never equals a list.
self.stop_token_sequences = [
new_token_ids(s) for s in (stop_token_sequences or [])
]
self.is_first_decode = False
# Set to True by Scheduler.postprocess after BlockManager.hash_blocks
# has registered the prompt blocks for prefix caching. The trigger has
Expand Down
Loading
Loading