Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,9 @@ Key entry points:
- Env vars: `atom/utils/envs.py` (all `ATOM_*` variable definitions)
- Mono decode (fused per-layer decode kernels): the shared mechanisms in `atom/mono/`
(`runtime/`: TP consensus and a runner's bind-then-agree construction
(`consensus.bind_agreed`), peer memory with the step fence and its owned
buffer (`lifecycle`), the step's mailbox clear
(`mailboxes.StepMailboxes.begin_step`), per-width builds (`widths`), the routing wrapper's
(`consensus.bind_agreed`), peer memory and its owned buffer (`lifecycle`),
the step's mailbox zeroing and peer fence, one FlyDSL launch
(`mailboxes.StepMailboxes.begin_step` -> `step_begin`), per-width builds (`widths`), the routing wrapper's
lazy runner (`route`), debug wait reports, kernel argument tables, compile-only
builds and their traced hand-offs (`compile.trace` / `compile.check_traced`),
the refusals every model shares (`deployment`), check-mode
Expand Down
5 changes: 2 additions & 3 deletions atom/models/deepseek_v41/mono/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,9 +145,8 @@ def _bind_agreed(self, bind) -> None:
self.rank,
self.tp,
self.scratch.device,
debug=self.debug,
)
self.mailboxes = StepMailboxes(self.peers, self.scratch)
self.mailboxes = StepMailboxes(self.peers, self.scratch, debug=self.debug)
self.widths = WidthBuilds(
self._rows_build, self._row_kernels, group, "V4.1 mono"
)
Expand Down Expand Up @@ -617,7 +616,7 @@ def _raise_on_given_up_waits(self, rows: int) -> None:
}
diag = self.scratch[self.diag_off : self.diag_off + DIAG_BYTES]
waits = given_up_waits(diag, kernel_debug.region_names(regions))
raise_if_given_up("V4.1 mono", self.rank, rows, waits, self.peers)
raise_if_given_up("V4.1 mono", self.rank, rows, waits, self.mailboxes)

def _rope_positions(self, cache, step) -> torch.Tensor:
"""Each row's position for RoPE (int64 [S])."""
Expand Down
9 changes: 5 additions & 4 deletions atom/models/minimax_m3/mono/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,10 +274,11 @@ def _allocate(self, group) -> None:
self.rank,
self.npes,
dev,
debug=self.debug,
)
# every mailbox pair, cleared at each step's start (forward)
self.mailboxes = StepMailboxes(self.peers, self.scratch1, self.scratch4)
# every mailbox pair, zeroed at each step's start (forward)
self.mailboxes = StepMailboxes(
self.peers, self.scratch1, self.scratch4, debug=self.debug
)
bf16 = torch.bfloat16
# row k = token k of the step; sparse layer i reads ars[i % 2] (the
# previous layer's output) and writes ars[(i + 1) % 2], likewise h_mids
Expand Down Expand Up @@ -489,4 +490,4 @@ def raise_on_given_up_waits(self, n: int) -> None:
self.scratch4[off : off + DIAG_BYTES],
region_namer(diag_region_names(n, self.index_heads)),
)
raise_if_given_up("MiniMax-M3 mono", self.rank, n, waits, self.peers)
raise_if_given_up("MiniMax-M3 mono", self.rank, n, waits, self.mailboxes)
4 changes: 2 additions & 2 deletions atom/mono/device/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,8 @@ class Mailbox:
Every 32-bit value is stored next to this launch's tag; a consumer
polls the payload until all tags match, so a hand-off costs one round trip
(no store drain, no separate flag). The tag is ``layer + 1``: it tells the
layers of one step apart, and the runner zeroes every mailbox between steps
(``MonoDecodeRunner.finish_step``), so a pair from an earlier step never
layers of one step apart, and the runner zeroes every mailbox at a step's
start (``StepMailboxes.begin_step``), so a pair from an earlier step never
carries a live tag. Every address is an ``Addr``: its space picks the scope.
"""

Expand Down
6 changes: 3 additions & 3 deletions atom/mono/runtime/debug.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved.
"""An ``ATOM_MONO_DEBUG`` step's report: the mailbox waits that gave up
(``atom.mono.device.sync._spin_bounded`` records, one a CTA) and the ranks the
step fence gave up on (``PeerBuffer.missing_ranks``). Synchronizes: eager only."""
step fence gave up on (``StepMailboxes.missing_ranks``). Synchronizes: eager only."""

import zlib

Expand Down Expand Up @@ -46,10 +46,10 @@ def given_up_waits(diag: torch.Tensor, name_of) -> list[str]:
]


def raise_if_given_up(what: str, rank: int, rows: int, waits, peers) -> None:
def raise_if_given_up(what: str, rank: int, rows: int, waits, mailboxes) -> None:
"""Raise with ``waits`` and the fence's missing ranks, if any."""
lines = list(waits) + [
f"step fence: rank {r} never arrived" for r in peers.missing_ranks()
f"step fence: rank {r} never arrived" for r in mailboxes.missing_ranks()
]
if lines:
raise RuntimeError(
Expand Down
4 changes: 2 additions & 2 deletions atom/mono/runtime/lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@
from atom.mono.runtime.peer_memory import PeerBuffer


def owned_peer_buffer(owner, nbytes, group, rank, npes, device, *, debug=False):
def owned_peer_buffer(owner, nbytes, group, rank, npes, device):
"""``(PeerBuffer, finalizer)``: the buffer is freed when ``owner`` is dropped
(a model reload), not at interpreter exit, where the HIP runtime may already
be gone; calling the finalizer frees it now. Collective (``PeerBuffer``)."""
peers = PeerBuffer(nbytes, group, rank, npes, device, debug=debug)
peers = PeerBuffer(nbytes, group, rank, npes, device)
finalizer = weakref.finalize(owner, peers.close)
finalizer.atexit = False
return peers, finalizer
37 changes: 26 additions & 11 deletions atom/mono/runtime/mailboxes.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,42 @@
"""A runner's mailboxes and where a step begins with them.

A mailbox tag is a layer's, so a pair left from one step would pass for the
next step's: every buffer holding pairs is cleared before a step's first
kernel, then the peer fence (``PeerBuffer.fence``) holds every rank until all
of them cleared, so no peer's first push of the step lands on an uncleared
buffer or is wiped by a late clear. At the step's start, not its end: a step
that stopped half-way leaves nothing for the next one to trip on.
next step's: every buffer holding pairs is zeroed before a step's first
kernel, then the peer fence holds every rank until all of them zeroed, so no
peer's first push of the step lands on an unzeroed buffer or is wiped by a
late zeroing (``step_begin``, one launch). At the step's start, not its end:
a step that stopped half-way leaves nothing for the next one to trip on.
"""

import torch

from atom.mono.runtime.peer_memory import PeerBuffer
from atom.mono.runtime.step_begin import MISSING, STATE_WORDS, step_begin


class StepMailboxes:
"""``peers``' data region and this rank's ``scratch`` buffers, cleared
together by ``begin_step``."""
"""``peers``' data region and this rank's ``scratch`` buffers, zeroed
together by ``begin_step``. ``debug``: a bounded fence wait, a rank that
never arrived reported by ``missing_ranks``."""

def __init__(self, peers: PeerBuffer, *scratch: torch.Tensor) -> None:
def __init__(
self, peers: PeerBuffer, *scratch: torch.Tensor, debug: bool = False
) -> None:
self.peers = peers
self._buffers = [*scratch, peers.bytes]
self._scratch = scratch
self._debug = debug
self._state = torch.zeros(
STATE_WORDS, dtype=torch.int64, device=peers.bytes.device
)

def begin_step(self) -> None:
"""Before the step's first kernel, on its stream (a graph records it)."""
torch._foreach_zero_(self._buffers)
self.peers.fence()
step_begin(self._scratch, self.peers, self._state, self._debug)

def missing_ranks(self) -> list[int]:
"""The ranks the fences since the last call gave up on (a debug build;
synchronizes: eager only)."""
missing = self._state[MISSING : MISSING + self.peers.addresses.numel()]
ranks = [r for r, m in enumerate(missing.tolist()) if m]
missing.zero_()
return ranks
30 changes: 3 additions & 27 deletions atom/mono/runtime/peer_memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from aiter.ops.flydsl.quick_allreduce_int4_ipc import UncachedIpcHeap

from atom.mono.runtime.consensus import MonoUnsupported, tp_agree
from atom.mono.runtime.step_fence import FENCE_BYTES, step_fence
from atom.mono.runtime.step_begin import FENCE_BYTES


class _DeviceBytes:
Expand All @@ -27,10 +27,8 @@ class PeerBuffer:
(``addresses``, int64, indexed by TP rank). What lives where, and why no rank
overwrites a peer's unread data, is the model's layout's to say.

``bytes`` is the data region, the part cleared between steps; the step
fence's slots sit past it (``fence``, called after every clear on the same
stream: ``mailboxes.StepMailboxes.begin_step``). ``debug``: bounded fence
waits, a rank that never arrived reported by ``missing_ranks``.
``bytes`` is the data region, the part zeroed between steps; the step
fence's slots sit past it (``step_begin``).

The constructor is collective over ``group`` (the handle exchange, then an
agreement that every rank mapped every peer): every rank must reach it, so a
Expand All @@ -45,18 +43,11 @@ def __init__(
rank: int,
npes: int,
device: torch.device,
*,
debug: bool = False,
):
self.local = UncachedIpcHeap.alloc_uncached(nbytes + FENCE_BYTES)
# a view, not an owner: close() frees the memory under it
self.bytes = torch.as_tensor(_DeviceBytes(self.local, nbytes), device=device)
self.rank = rank
self._fence_off = nbytes
self._epoch = torch.zeros(1, dtype=torch.int64, device=device)
self._missing = (
torch.zeros(npes, dtype=torch.int64, device=device) if debug else None
)
self._opened: list[int] = []
addresses = [self.local]
if npes > 1:
Expand Down Expand Up @@ -91,21 +82,6 @@ def kernel_args(self) -> dict:
"rank": self.rank,
}

def fence(self) -> None:
"""The step fence, after this rank's clear of ``bytes`` (same stream)."""
step_fence(
self._epoch, self.addresses, self._fence_off, self.rank, self._missing
)

def missing_ranks(self) -> list[int]:
"""A debug buffer's ranks the fences since the last call gave up on
(synchronizes: eager only)."""
if self._missing is None:
return []
missing = [r for r, m in enumerate(self._missing.tolist()) if m]
self._missing.zero_()
return missing

def close(self) -> None:
"""Close the peer mappings and free the local buffer; idempotent. Only once no
kernel of any rank can still touch it (the runner is being dropped)."""
Expand Down
Loading