From 34f0703285048f74f25ea41548374d875909e0bc Mon Sep 17 00:00:00 2001 From: karverma-amd Date: Fri, 2 Oct 2026 22:55:24 +0000 Subject: [PATCH 1/6] feat(dp): backpressure (lazy) DP dispatch to avoid forced-eager tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Offline batch inference hands the engine all requests at once; CoreManager sprayed them across DP ranks up front, so a rank that drew shorter sequences drained early, went idle, and — because DP runs in lockstep — forced the whole group onto the slow eager path (observed as a large end-of-run throughput collapse). Add an optional backpressure dispatch mode gated by ATOM_DP_DISPATCH_DEPTH (default 0 = unchanged legacy spray). When >0 it is the per-rank in-flight cap: load-balanced seqs wait in a manager-side pending pool and are released to the lightest rank only up to the cap, then re-fed on each completion. A fast rank keeps getting work instead of running dry, so all ranks drain together and the eager tail is eliminated. - _bp_pending deque + _drain_pending_locked + _bp_dispatch_or_refill - backpressure branch in _dispatch_to_dp_ranks (plain load-balanced batches only; explicit hints / sticky sessions unchanged) - refill hook on offline completion path; pending cleared in reset_dp_router - _input_send_lock serializes input_sockets sends (refill now runs on the per-rank output threads too; a ZMQ socket is not thread-safe) - class-level _bp_depth/_input_send_lock defaults so bare __new__ test objects and _init_shared_state-skipping subclasses keep working Measured on DeepSeek-R1 MXFP4 Offline, 4x MI455X (gfx1250), depth 1860: +39% tokens/sec (38.4k -> 53.5k), ~34 min faster, VALID; accuracy unchanged. tests: add tests/test_dp_backpressure.py (cap / refill / full-drain); existing tests/test_dp_load_balance.py unaffected (38 passed). Co-authored-by: Cursor --- atom/model_engine/engine_core_mgr.py | 132 ++++++++++++++++++++++++++- tests/test_dp_backpressure.py | 125 +++++++++++++++++++++++++ 2 files changed, 254 insertions(+), 3 deletions(-) create mode 100644 tests/test_dp_backpressure.py diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 68f07ecda3..704d02fdca 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -10,6 +10,7 @@ import pickle import queue import weakref +from collections import deque from dataclasses import dataclass from threading import Lock, Thread @@ -148,6 +149,14 @@ def _resolve_dp_engine_count(config: Config, logical: int) -> int: class CoreManager: + # Class-level defaults so bare instances built via ``__new__`` (the DP + # load-balance unit tests) and any subclass that skips ``_init_shared_state`` + # still have working backpressure state. Real instances override + # _input_send_lock with their own in _init_shared_state; _bp_depth defaults + # to 0 (backpressure disabled). + _bp_depth = 0 + _input_send_lock = Lock() + def _init_shared_state( self, config: Config, @@ -253,6 +262,25 @@ def _init_shared_state( # what dispatch added, and only for ranks that were actually charged. self._seq_load = {} self._lb_lock = Lock() + # --- Backpressure (lazy) DP dispatch -------------------------------- + # ATOM_DP_DISPATCH_DEPTH > 0 enables it; 0 keeps the legacy behaviour of + # spraying every seq across ranks up front. The value is the per-rank cap + # on in-flight (charged-but-unfinished) requests. Un-dispatched + # load-balanced seqs wait in _bp_pending and are released to the lightest + # rank only as that rank drains, so a rank that finishes early keeps + # getting re-fed instead of running dry and dragging the lockstep DP + # group onto the forced-eager path. Must stay >= max_num_seqs so each + # rank can still fill a full running batch; ~1.2-1.5x is a good start. + try: + self._bp_depth = int(os.getenv("ATOM_DP_DISPATCH_DEPTH", "0")) + except ValueError: + self._bp_depth = 0 + self._bp_pending: deque = deque() + # Backpressure refill runs on the per-rank output threads, which then + # also write input_sockets. A ZMQ socket is not thread-safe, so every + # input_sockets send is serialized through this lock (see _send_request). + # Uncontended in the legacy path, so it costs nothing when disabled. + self._input_send_lock = Lock() # Control traffic (utility commands, abort, shutdown) travels on its own # sockets so that input_sockets keeps a single writer -- see # _send_request. These have several writer threads and so do need @@ -717,6 +745,12 @@ def process_outputs_socket(): exc_info=True, ) self.outputs_queue.put_nowait(seqs) + # These completions freed slots on their ranks — re-feed + # from the pending pool so a fast rank never drains to + # empty while peers still have work. No-op when + # ATOM_DP_DISPATCH_DEPTH is unset (legacy spray path). + if getattr(self, "_bp_depth", 0) > 0 and self._bp_pending: + self._bp_dispatch_or_refill(raise_on_error=False) finally: # Close sockets. shutdown_socket.close(linger=0) @@ -856,9 +890,13 @@ def _send_request(self, dp_rank: int, payload: bytes) -> None: anywhere but that thread's own traceback. So: never send to ``input_sockets`` from anywhere but here. """ - self.input_sockets[dp_rank].send_multipart( - [self.engine_core_identities[dp_rank], payload], copy=False - ) + # Serialized because backpressure refill (ATOM_DP_DISPATCH_DEPTH) sends + # from the per-rank output threads too, not just the single request + # thread. The lock is uncontended when backpressure is off. + with self._input_send_lock: + self.input_sockets[dp_rank].send_multipart( + [self.engine_core_identities[dp_rank], payload], copy=False + ) def _send_control(self, dp_rank: int, payload: bytes, copy: bool = False) -> None: """Send one already-pickled control message. Serialized, never hot. @@ -945,6 +983,19 @@ def _dispatch_to_dp_ranks(self, seqs: list[Sequence]) -> None: # getattr/int pass per seq. hints = self._resolve_and_validate_hints(seqs) + # Backpressure (lazy) dispatch. Only engages for plain load-balanced + # batches (no explicit rank hints, no sticky sessions) so the + # hinted/affinity semantics below are never altered. Offline batch + # inference takes exactly this path, which is where a rank that drains + # early otherwise forces the lockstep DP group onto the eager path. + if ( + getattr(self, "_bp_depth", 0) > 0 + and not self._dp_session_affinity_enabled + and all(h is None for h in hints) + ): + self._bp_dispatch_or_refill(new_seqs=seqs, raise_on_error=True) + return + # round_robin normally skips load bookkeeping. Session affinity still # needs queued-prefill counters even if the fallback strategy is RR. track_load = ( @@ -1014,6 +1065,79 @@ def _dispatch_to_dp_ranks(self, seqs: list[Sequence]) -> None: else: logger.info("%s: add %s", self.label, ", ".join(added)) + def _drain_pending_locked(self) -> list[tuple[int, list]]: + """Release pending seqs up to the per-rank depth cap. Hold _lb_lock. + + Fills the lightest rank first (fewest in-flight requests), charging each + seq as it goes, until every rank is at the cap or the pending pool is + empty. Balancing on request count keeps the lockstep DP ranks in phase; + the cap is what leaves work queued so a fast rank can be re-fed on its + next completion instead of draining to empty. Returns per-rank + sub-batches to send after the lock is dropped (ZMQ sends must not hold + _lb_lock). + """ + n = self._routable_engine_count + batches: list[list] = [[] for _ in range(n)] + while self._bp_pending: + # Lightest rank by (in-flight reqs, rank) for a stable tie-break. + best = min(range(n), key=lambda r: (self._rank_reqs[r], r)) + if self._rank_reqs[best] >= self._bp_depth: + break # every rank is at/above the cap; hold the rest. + seq = self._bp_pending.popleft() + self._charge_seq_load_locked(seq, best) + batches[best].append(seq) + return [(r, s) for r, s in enumerate(batches) if s] + + def _bp_dispatch_or_refill(self, new_seqs=None, raise_on_error: bool = False): + """Backpressure dispatch: enqueue, then top ranks up to the cap. + + Called both on add (``new_seqs`` set, request thread) and after each + completion (``new_seqs`` None, per-rank output thread) to re-feed the + rank that just freed a slot. Charging happens under ``_lb_lock``; the + actual ZMQ sends happen after the lock is dropped. On a send failure the + just-charged seqs are released so the per-rank counters never leak. + + ``raise_on_error`` propagates on the add path (the offline ``generate`` + caller fails the run cleanly). The completion path swallows + logs so a + broken socket cannot silently kill an output thread. + """ + with self._lb_lock: + if new_seqs: + self._bp_pending.extend(new_seqs) + sends = self._drain_pending_locked() + if not sends: + return + added = [] + for dp_rank, rank_seqs in sends: + try: + self._send_request( + dp_rank, pickle.dumps((EngineCoreRequestType.ADD, rank_seqs)) + ) + added.append(f"rank{dp_rank}: {len(rank_seqs)} req") + except Exception: + # Roll back this rank's charge; it will never finish to release. + for seq in rank_seqs: + self._release_seq_load(seq.id) + if raise_on_error: + raise + logger.exception( + "%s: backpressure send to rank%d failed; dropped %d seq(s)", + self.label, + dp_rank, + len(rank_seqs), + ) + if added: + with self._lb_lock: + reqs_snapshot = list(self._rank_reqs) + pending = len(self._bp_pending) + logger.info( + "%s: bp-dispatch %s | in-flight reqs=%s pending=%d", + self.label, + ", ".join(added), + reqs_snapshot, + pending, + ) + def _select_dp_rank_locked(self) -> int: """Pick a DP engine rank for a new request. Caller must hold _lb_lock. @@ -1309,6 +1433,8 @@ def reset_dp_router(self) -> None: self._rank_reqs = [0] * self._routable_engine_count self._rank_tokens = [0] * self._routable_engine_count self._seq_load.clear() + if getattr(self, "_bp_pending", None) is not None: + self._bp_pending.clear() self._dp_session_owners.clear() self._dp_session_prompt_tokens.clear() diff --git a/tests/test_dp_backpressure.py b/tests/test_dp_backpressure.py new file mode 100644 index 0000000000..e6d9a1bf4b --- /dev/null +++ b/tests/test_dp_backpressure.py @@ -0,0 +1,125 @@ +"""Fast CPU-only test of Fix A backpressure dispatch (ATOM_DP_DISPATCH_DEPTH). + +Builds a bare CoreManager via __new__ (same pattern as tests/test_dp_load_balance.py), +stubs _send_request, and checks: (1) per-rank cap is honored, (2) the rest waits in +the pending pool, (3) completing a seq re-feeds the drained rank, (4) a fast rank +ends up processing MORE total requests than a slow one (the whole point of Fix A). +""" +import pickle +from collections import deque +from threading import Lock + +from atom.model_engine.engine_core_mgr import CoreManager +from atom.model_engine.engine_core_protocol import EngineCoreRequestType + + +class Seq: + _n = 0 + + def __init__(self, prompt=10): + Seq._n += 1 + self.id = f"s{Seq._n}" + self.num_prompt_tokens = prompt + self.data_parallel_rank = None + self.dp_session_id = None + self.dp_parent_session_id = None + self.stream_callback = None + + +def make_mgr(n_ranks=4, depth=3): + mgr = CoreManager.__new__(CoreManager) + mgr.label = "test" + mgr._dp_lb_strategy = "least_requests" + mgr._dp_lb_req_equiv = 512 + mgr._dp_session_affinity_enabled = False + mgr.global_engine_count = n_ranks + mgr.local_engine_count = n_ranks + mgr.pp_size = 1 + mgr._rank_rotation_cursor = 0 + mgr._rank_reqs = [0] * n_ranks + mgr._rank_tokens = [0] * n_ranks + mgr._rank_routed_total = [0] * n_ranks + mgr._dp_route_counters = {"explicit_total": 0, "load_balanced_total": 0} + mgr._seq_load = {} + mgr._lb_lock = Lock() + mgr._input_send_lock = Lock() + mgr._bp_depth = depth + mgr._bp_pending = deque() + mgr._seq_id_to_callback = {} + # capture what got sent to each rank + mgr.sent = [[] for _ in range(n_ranks)] + + def fake_send(dp_rank, payload): + _type, seqs = pickle.loads(payload) + assert _type == EngineCoreRequestType.ADD + mgr.sent[dp_rank].extend(s.id for s in seqs) + + mgr._send_request = fake_send + return mgr + + +def test_cap_and_pending(): + mgr = make_mgr(n_ranks=4, depth=3) + seqs = [Seq() for _ in range(100)] + mgr.add_request(seqs) + # 4 ranks * depth 3 = 12 dispatched, rest pending. + assert mgr._rank_reqs == [3, 3, 3, 3], mgr._rank_reqs + assert sum(len(s) for s in mgr.sent) == 12 + assert len(mgr._bp_pending) == 88 + print("OK cap+pending: dispatched=12 pending=88 reqs=", mgr._rank_reqs) + + +def test_refill_feeds_fast_rank_more(): + mgr = make_mgr(n_ranks=4, depth=3) + seqs = [Seq() for _ in range(40)] + mgr.add_request(seqs) + assert mgr._rank_reqs == [3, 3, 3, 3] + + # Simulate rank 0 completing fast: finish 10 of its seqs one at a time, + # each completion triggering a refill (as the output thread does). + def finish_one_on_rank(rank): + # pick a charged seq currently on `rank` + sid = next(s for s, (r, _, _) in mgr._seq_load.items() if r == rank) + mgr._release_seq_load(sid) + if mgr._bp_depth > 0 and mgr._bp_pending: + mgr._bp_dispatch_or_refill(raise_on_error=False) + return sid + + for _ in range(10): + finish_one_on_rank(0) + + # Rank 0 should have been re-fed each time -> processed more than peers. + assert len(mgr.sent[0]) > len(mgr.sent[1]), (len(mgr.sent[0]), mgr.sent[1:]) + # Every rank still capped at depth in-flight (never exceeded). + assert all(r <= 3 for r in mgr._rank_reqs), mgr._rank_reqs + # Pending shrank by the 10 refills. + assert len(mgr._bp_pending) == 40 - 12 - 10, len(mgr._bp_pending) + print("OK refill: rank0 processed", len(mgr.sent[0]), + "vs peers", [len(mgr.sent[i]) for i in (1, 2, 3)], + "| in-flight", mgr._rank_reqs, "| pending", len(mgr._bp_pending)) + + +def test_drains_everything(): + mgr = make_mgr(n_ranks=2, depth=2) + seqs = [Seq() for _ in range(20)] + mgr.add_request(seqs) + # finish all in-flight repeatedly until pending + in-flight empty + guard = 0 + while mgr._seq_load: + sid = next(iter(mgr._seq_load)) + mgr._release_seq_load(sid) + if mgr._bp_pending: + mgr._bp_dispatch_or_refill(raise_on_error=False) + guard += 1 + assert guard < 1000 + assert len(mgr._bp_pending) == 0 + assert sum(len(s) for s in mgr.sent) == 20 # every seq dispatched exactly once + assert len(set(sum(mgr.sent, []))) == 20 # no duplicates + print("OK drain: all 20 dispatched exactly once, nothing stuck") + + +if __name__ == "__main__": + test_cap_and_pending() + test_refill_feeds_fast_rank_more() + test_drains_everything() + print("\nALL BACKPRESSURE TESTS PASSED") From 32d6beafaf92a443b9d508317ca17b1d3531ea5c Mon Sep 17 00:00:00 2001 From: karverma-amd Date: Sat, 10 Oct 2026 23:05:12 +0000 Subject: [PATCH 2/6] style: satisfy black and ruff on the DP backpressure test Black wraps the refill print and wants a blank line after the module docstring. Ruff RUF017 rejects sum(mgr.sent, []) as a quadratic flatten. Co-authored-by: Cursor --- tests/test_dp_backpressure.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/test_dp_backpressure.py b/tests/test_dp_backpressure.py index e6d9a1bf4b..c163a53d40 100644 --- a/tests/test_dp_backpressure.py +++ b/tests/test_dp_backpressure.py @@ -5,6 +5,7 @@ the pending pool, (3) completing a seq re-feeds the drained rank, (4) a fast rank ends up processing MORE total requests than a slow one (the whole point of Fix A). """ + import pickle from collections import deque from threading import Lock @@ -94,9 +95,16 @@ def finish_one_on_rank(rank): assert all(r <= 3 for r in mgr._rank_reqs), mgr._rank_reqs # Pending shrank by the 10 refills. assert len(mgr._bp_pending) == 40 - 12 - 10, len(mgr._bp_pending) - print("OK refill: rank0 processed", len(mgr.sent[0]), - "vs peers", [len(mgr.sent[i]) for i in (1, 2, 3)], - "| in-flight", mgr._rank_reqs, "| pending", len(mgr._bp_pending)) + print( + "OK refill: rank0 processed", + len(mgr.sent[0]), + "vs peers", + [len(mgr.sent[i]) for i in (1, 2, 3)], + "| in-flight", + mgr._rank_reqs, + "| pending", + len(mgr._bp_pending), + ) def test_drains_everything(): @@ -113,8 +121,9 @@ def test_drains_everything(): guard += 1 assert guard < 1000 assert len(mgr._bp_pending) == 0 - assert sum(len(s) for s in mgr.sent) == 20 # every seq dispatched exactly once - assert len(set(sum(mgr.sent, []))) == 20 # no duplicates + dispatched = [sid for rank_ids in mgr.sent for sid in rank_ids] + assert len(dispatched) == 20 # every seq dispatched exactly once + assert len(set(dispatched)) == 20 # no duplicates print("OK drain: all 20 dispatched exactly once, nothing stuck") From 6b55087489b86fe22061cb11573ae607957361b0 Mon Sep 17 00:00:00 2001 From: karverma-amd Date: Mon, 12 Oct 2026 15:10:34 +0000 Subject: [PATCH 3/6] [AMD][DSR1] address DP backpressure review notes on wording and env The tail leaves CUDA-graph replay because a prefill on any rank makes the lockstep group eager, not because an idle rank's dummy batch does. Say that in the comments, register ATOM_DP_DISPATCH_DEPTH, and warn on a non-integer, negative, or below-max_num_seqs value. Rewrite _send_request for the output thread refill under _input_send_lock, warn when reset drops a non-empty pending pool, and drop the test prints and __main__ runner. Co-authored-by: Cursor --- atom/model_engine/engine_core_mgr.py | 85 ++++++++++++++++------------ atom/utils/envs.py | 7 +++ docs/environment_variables.md | 1 + tests/test_dp_backpressure.py | 19 ------- 4 files changed, 56 insertions(+), 56 deletions(-) diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 704d02fdca..850008e877 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -67,7 +67,8 @@ def build_internode_dp_socket_plan( selects the IPC path (and, with it, the connect/bind polarity) further down. Three ports per engine: requests, outputs, and control. Control is separate - so the request socket keeps a single writer thread (see _send_request). + so utility, abort, and shutdown never share ``input_sockets`` with add and + backpressure refill (see _send_request). """ base = master_port + INTERNODE_DP_SOCKET_PORT_OFFSET return [ @@ -263,18 +264,23 @@ def _init_shared_state( self._seq_load = {} self._lb_lock = Lock() # --- Backpressure (lazy) DP dispatch -------------------------------- - # ATOM_DP_DISPATCH_DEPTH > 0 enables it; 0 keeps the legacy behaviour of - # spraying every seq across ranks up front. The value is the per-rank cap - # on in-flight (charged-but-unfinished) requests. Un-dispatched - # load-balanced seqs wait in _bp_pending and are released to the lightest - # rank only as that rank drains, so a rank that finishes early keeps - # getting re-fed instead of running dry and dragging the lockstep DP - # group onto the forced-eager path. Must stay >= max_num_seqs so each - # rank can still fill a full running batch; ~1.2-1.5x is a good start. - try: - self._bp_depth = int(os.getenv("ATOM_DP_DISPATCH_DEPTH", "0")) - except ValueError: - self._bp_depth = 0 + # ATOM_DP_DISPATCH_DEPTH > 0 holds load-balanced seqs in _bp_pending and + # releases them to the lightest rank only up to that in-flight cap. A + # rank that finishes its share early is re-fed instead of sitting on a + # dummy decode while peers still open prefills. A prefill on any rank + # takes the whole lockstep group off CUDA-graph replay; an idle rank's + # dummy batch does not. Finishing prefills together lets the tail stay + # all-decode and on the graph. 0 keeps the legacy up-front spray. Depth + # should stay >= max_num_seqs so each rank can fill a full running batch. + self._bp_depth = envs.ATOM_DP_DISPATCH_DEPTH + if 0 < self._bp_depth < config.max_num_seqs: + logger.warning( + "%s: ATOM_DP_DISPATCH_DEPTH=%d is below max_num_seqs=%d; " + "each rank's running batch is capped below a full step", + label, + self._bp_depth, + config.max_num_seqs, + ) self._bp_pending: deque = deque() # Backpressure refill runs on the per-rank output threads, which then # also write input_sockets. A ZMQ socket is not thread-safe, so every @@ -875,24 +881,22 @@ def close(self): def _send_request(self, dp_rank: int, payload: bytes) -> None: """Send one already-pickled request to an engine core. Hot path. - Deliberately unsynchronized: ``input_sockets`` carries nothing but - ``add_request``, which runs on a single thread (the API server's event - loop online, the caller's thread offline). Everything that can be sent - from another thread -- utility commands, abort, shutdown -- goes to - :meth:`_send_control` on a separate socket instead. - - That separation is load-bearing, not stylistic. A ZMQ socket is not - thread-safe: two unserialized ``send_multipart`` calls interleave their - frames, the DEALER on the other end then reads a routing identity where - a payload should be, and its input thread dies on ``UnpicklingError``. - Nothing recovers from that -- the engine spins on a forever-empty input - queue, the workers idle, and every client hangs with no error logged - anywhere but that thread's own traceback. So: never send to - ``input_sockets`` from anywhere but here. + Every ``input_sockets`` write goes through here and holds + ``_input_send_lock``. Two caller classes share the socket: the request + thread (``add_request``: the API server's event loop online, the + caller's thread offline) and the per-rank output threads (backpressure + refill when ``ATOM_DP_DISPATCH_DEPTH`` > 0). A ZMQ socket is not + thread-safe: unserialized ``send_multipart`` calls interleave frames, + the DEALER reads a routing identity where a payload should be, and its + input thread dies on ``UnpicklingError``. Nothing recovers from that. + + Refill sends the same ADD payload as the request thread, so it reuses + this socket under the lock. A second ROUTER would need another + engine-side reader for the same queue. Utility commands, abort, and + shutdown stay on :meth:`_send_control`, which is why control has its + own socket. Never send to ``input_sockets`` from anywhere but here. + The lock is uncontended when backpressure is off. """ - # Serialized because backpressure refill (ATOM_DP_DISPATCH_DEPTH) sends - # from the per-rank output threads too, not just the single request - # thread. The lock is uncontended when backpressure is off. with self._input_send_lock: self.input_sockets[dp_rank].send_multipart( [self.engine_core_identities[dp_rank], payload], copy=False @@ -983,11 +987,11 @@ def _dispatch_to_dp_ranks(self, seqs: list[Sequence]) -> None: # getattr/int pass per seq. hints = self._resolve_and_validate_hints(seqs) - # Backpressure (lazy) dispatch. Only engages for plain load-balanced - # batches (no explicit rank hints, no sticky sessions) so the - # hinted/affinity semantics below are never altered. Offline batch - # inference takes exactly this path, which is where a rank that drains - # early otherwise forces the lockstep DP group onto the eager path. + # Backpressure (lazy) dispatch. Only plain load-balanced batches (no + # explicit rank hints, no sticky sessions), which is the offline path. + # Holding work back makes every rank finish prefilling together, so the + # tail can stay all-decode and on CUDA-graph replay. A prefill on any + # rank is what takes the group off the graph; an idle dummy batch is not. if ( getattr(self, "_bp_depth", 0) > 0 and not self._dp_session_affinity_enabled @@ -1433,8 +1437,15 @@ def reset_dp_router(self) -> None: self._rank_reqs = [0] * self._routable_engine_count self._rank_tokens = [0] * self._routable_engine_count self._seq_load.clear() - if getattr(self, "_bp_pending", None) is not None: - self._bp_pending.clear() + pending = getattr(self, "_bp_pending", None) + if pending: + logger.warning( + "%s: reset_dp_router() dropping %d undispatched " + "request(s) still in the backpressure pool", + self.label, + len(pending), + ) + pending.clear() self._dp_session_owners.clear() self._dp_session_prompt_tokens.clear() diff --git a/atom/utils/envs.py b/atom/utils/envs.py index 747814494e..2f029911cb 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -155,6 +155,13 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: # pressure) and a smaller value toward prompt-token balance (prefill # pressure). See engine_core_mgr.CoreManager._select_dp_rank_locked. "ATOM_DP_LB_REQ_EQUIV": lambda: int(os.getenv("ATOM_DP_LB_REQ_EQUIV", "512")), + # Per-rank cap on in-flight load-balanced requests. 0 sprays the whole + # offline batch up front. >0 holds the rest and refills a rank only as it + # finishes, so ranks finish prefilling together and the tail stays + # all-decode (CUDA-graph replay). A non-integer or negative value warns + # and reads as 0. Keep it >= max_num_seqs; a smaller cap truncates every + # rank's running batch. CoreManager warns when that happens. + "ATOM_DP_DISPATCH_DEPTH": lambda: _nonnegative_int_env("ATOM_DP_DISPATCH_DEPTH", 0), # Place a new agent session on the lightest DP rank, then keep every later # request on that immutable cache owner. Existing sessions never spill; # child correlation ids are independently load-placed rather than diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 93e2940fed..9865c97580 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -18,6 +18,7 @@ This document describes the environment variables used in the ATOM project. | **ATOM_DP_MASTER_IP** | str | 127.0.0.1 | Master IP address for DP ranks coordination. | | **ATOM_DP_MASTER_PORT** | int | 29500 | Master port for DP ranks coordination. | | **ATOM_DP_LB_REQ_EQUIV** | int | 512 | Token-equivalent decode pressure assigned to each in-flight request by `least_tokens` routing. | +| **ATOM_DP_DISPATCH_DEPTH** | int | 0 | Per-rank cap on in-flight load-balanced requests. `0` sprays the offline batch up front. `>0` holds the rest and refills a rank as it finishes, so ranks finish prefilling together and the tail stays all-decode on the CUDA graph. A prefill on any rank is what leaves the graph; an idle rank's dummy batch does not. Keep it `>= max_num_seqs` (about 1.2–1.5× is a starting point). A non-integer or negative value warns and disables the feature. A positive value below `max_num_seqs` warns and still caps every rank's running batch. | | **ATOM_DP_SESSION_AFFINITY** | bool | false | Load-place each new session, then keep later turns on the same prefix-cache owner. Reads `X-Dynamo-Session-ID`, falling back to `X-Correlation-ID`. | ## Prefill delayer (TP/DCP and DP attention) diff --git a/tests/test_dp_backpressure.py b/tests/test_dp_backpressure.py index c163a53d40..bc7520418b 100644 --- a/tests/test_dp_backpressure.py +++ b/tests/test_dp_backpressure.py @@ -67,7 +67,6 @@ def test_cap_and_pending(): assert mgr._rank_reqs == [3, 3, 3, 3], mgr._rank_reqs assert sum(len(s) for s in mgr.sent) == 12 assert len(mgr._bp_pending) == 88 - print("OK cap+pending: dispatched=12 pending=88 reqs=", mgr._rank_reqs) def test_refill_feeds_fast_rank_more(): @@ -95,16 +94,6 @@ def finish_one_on_rank(rank): assert all(r <= 3 for r in mgr._rank_reqs), mgr._rank_reqs # Pending shrank by the 10 refills. assert len(mgr._bp_pending) == 40 - 12 - 10, len(mgr._bp_pending) - print( - "OK refill: rank0 processed", - len(mgr.sent[0]), - "vs peers", - [len(mgr.sent[i]) for i in (1, 2, 3)], - "| in-flight", - mgr._rank_reqs, - "| pending", - len(mgr._bp_pending), - ) def test_drains_everything(): @@ -124,11 +113,3 @@ def test_drains_everything(): dispatched = [sid for rank_ids in mgr.sent for sid in rank_ids] assert len(dispatched) == 20 # every seq dispatched exactly once assert len(set(dispatched)) == 20 # no duplicates - print("OK drain: all 20 dispatched exactly once, nothing stuck") - - -if __name__ == "__main__": - test_cap_and_pending() - test_refill_feeds_fast_rank_more() - test_drains_everything() - print("\nALL BACKPRESSURE TESTS PASSED") From 90b0a7cd847933a56abc820b3ab9146f6a81fc88 Mon Sep 17 00:00:00 2001 From: karverma-amd Date: Wed, 14 Oct 2026 23:37:10 +0000 Subject: [PATCH 4/6] [AMD][DSR1] warn when backpressure overrides the DP load-balance strategy ATOM_DP_DISPATCH_DEPTH balances on in-flight request count, so round_robin and least_tokens do not apply. Say so at init when depth is positive and the configured strategy is not least_requests. Co-authored-by: Cursor --- atom/model_engine/engine_core_mgr.py | 23 ++++++++++++++---- atom/utils/envs.py | 4 +++- docs/distributed_guide.md | 1 + docs/environment_variables.md | 2 +- tests/test_dp_backpressure.py | 36 ++++++++++++++++++++++++++++ 5 files changed, 59 insertions(+), 7 deletions(-) diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 5873700700..e7c99d7638 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -312,7 +312,19 @@ def _init_shared_state( # dummy batch does not. Finishing prefills together lets the tail stay # all-decode and on the graph. 0 keeps the legacy up-front spray. Depth # should stay >= max_num_seqs so each rank can fill a full running batch. + # The pending drain always picks the rank with the fewest in-flight + # requests. That is what lockstep needs. _select_dp_rank_locked cannot + # skip ranks already at the cap, so round_robin and least_tokens do + # not apply on this path. self._bp_depth = envs.ATOM_DP_DISPATCH_DEPTH + if self._bp_depth > 0 and self._dp_lb_strategy != "least_requests": + logger.warning( + "%s: ATOM_DP_DISPATCH_DEPTH=%d balances on in-flight request " + "count and ignores dp_load_balance=%s", + label, + self._bp_depth, + self._dp_lb_strategy, + ) if 0 < self._bp_depth < config.max_num_seqs: logger.warning( "%s: ATOM_DP_DISPATCH_DEPTH=%d is below max_num_seqs=%d; " @@ -1123,11 +1135,12 @@ def _drain_pending_locked(self) -> list[tuple[int, list]]: Fills the lightest rank first (fewest in-flight requests), charging each seq as it goes, until every rank is at the cap or the pending pool is - empty. Balancing on request count keeps the lockstep DP ranks in phase; - the cap is what leaves work queued so a fast rank can be re-fed on its - next completion instead of draining to empty. Returns per-rank - sub-batches to send after the lock is dropped (ZMQ sends must not hold - _lb_lock). + empty. Balancing on request count keeps the lockstep DP ranks in phase. + ``round_robin`` and ``least_tokens`` are not consulted here; init warns + when one of those is configured alongside a positive depth. The cap + leaves work queued so a fast rank can be re-fed on its next completion + instead of draining to empty. Returns per-rank sub-batches to send + after the lock is dropped (ZMQ sends must not hold ``_lb_lock``). """ n = self._routable_engine_count batches: list[list] = [[] for _ in range(n)] diff --git a/atom/utils/envs.py b/atom/utils/envs.py index 2ed3eef072..b421f1e633 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -182,7 +182,9 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: # Per-rank cap on in-flight load-balanced requests. 0 sprays the whole # offline batch up front. >0 holds the rest and refills a rank only as it # finishes, so ranks finish prefilling together and the tail stays - # all-decode (CUDA-graph replay). A non-integer or negative value warns + # all-decode (CUDA-graph replay). While set, dispatch balances on in-flight + # request count and ignores dp_load_balance; CoreManager warns when that + # strategy is not least_requests. A non-integer or negative value warns # and reads as 0. Keep it >= max_num_seqs; a smaller cap truncates every # rank's running batch. CoreManager warns when that happens. "ATOM_DP_DISPATCH_DEPTH": lambda: _nonnegative_int_env("ATOM_DP_DISPATCH_DEPTH", 0), diff --git a/docs/distributed_guide.md b/docs/distributed_guide.md index 23088368c6..5060feb45e 100644 --- a/docs/distributed_guide.md +++ b/docs/distributed_guide.md @@ -260,6 +260,7 @@ drained and warns if it is invoked while requests are still charged. - `Config.dp_load_balance` (str, default `"least_requests"`): DP routing strategy. - CLI: `--data-parallel-size N` or `-dp N`; `--dp-load-balance {round_robin,least_requests,least_tokens}` - Env: `ATOM_DP_LB_REQ_EQUIV` (int, default `512`): token-equivalent cost of one in-flight request for `least_tokens`. +- Env: `ATOM_DP_DISPATCH_DEPTH` (int, default `0`): per-rank in-flight cap for lazy dispatch. While it is positive, load-balanced routing balances on in-flight request count and ignores `round_robin` and `least_tokens`; init warns when `dp_load_balance` is not `least_requests`. - Env: `ATOM_DP_SESSION_AFFINITY` (bool, default `0`): load-place each new session once, then keep all later turns on its cache owner without spill. diff --git a/docs/environment_variables.md b/docs/environment_variables.md index b65cdc03bc..2097023e3b 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -18,7 +18,7 @@ This document describes the environment variables used in the ATOM project. | **ATOM_DP_MASTER_IP** | str | 127.0.0.1 | Master IP address for DP ranks coordination. | | **ATOM_DP_MASTER_PORT** | int | 29500 | Master port for DP ranks coordination. | | **ATOM_DP_LB_REQ_EQUIV** | int | 512 | Token-equivalent decode pressure assigned to each in-flight request by `least_tokens` routing. | -| **ATOM_DP_DISPATCH_DEPTH** | int | 0 | Per-rank cap on in-flight load-balanced requests. `0` sprays the offline batch up front. `>0` holds the rest and refills a rank as it finishes, so ranks finish prefilling together and the tail stays all-decode on the CUDA graph. A prefill on any rank is what leaves the graph; an idle rank's dummy batch does not. Keep it `>= max_num_seqs` (about 1.2–1.5× is a starting point). A non-integer or negative value warns and disables the feature. A positive value below `max_num_seqs` warns and still caps every rank's running batch. | +| **ATOM_DP_DISPATCH_DEPTH** | int | 0 | Per-rank cap on in-flight load-balanced requests. `0` sprays the offline batch up front. `>0` holds the rest and refills a rank as it finishes, so ranks finish prefilling together and the tail stays all-decode on the CUDA graph. A prefill on any rank is what leaves the graph; an idle rank's dummy batch does not. While set, dispatch balances on in-flight request count and ignores `--dp-load-balance` (`round_robin` and `least_tokens`); init warns when the configured strategy is not `least_requests`. Keep it `>= max_num_seqs` (about 1.2–1.5× is a starting point). A non-integer or negative value warns and disables the feature. A positive value below `max_num_seqs` warns and still caps every rank's running batch. | | **ATOM_DP_SESSION_AFFINITY** | bool | false | Load-place each new session, then keep later turns on the same prefix-cache owner. Reads `X-Dynamo-Session-ID`, falling back to `X-Correlation-ID`. | ## Prefill delayer (TP/DCP and DP attention) diff --git a/tests/test_dp_backpressure.py b/tests/test_dp_backpressure.py index bc7520418b..d0ad515bfc 100644 --- a/tests/test_dp_backpressure.py +++ b/tests/test_dp_backpressure.py @@ -6,9 +6,11 @@ ends up processing MORE total requests than a slow one (the whole point of Fix A). """ +import logging import pickle from collections import deque from threading import Lock +from types import SimpleNamespace from atom.model_engine.engine_core_mgr import CoreManager from atom.model_engine.engine_core_protocol import EngineCoreRequestType @@ -96,6 +98,40 @@ def finish_one_on_rank(rank): assert len(mgr._bp_pending) == 40 - 12 - 10, len(mgr._bp_pending) +def _init_with_strategy(strategy): + mgr = CoreManager.__new__(CoreManager) + mgr._init_shared_state( + SimpleNamespace(dp_load_balance=strategy, max_num_seqs=1536), + label="test", + local_engine_count=2, + ) + return mgr + + +def test_init_warns_when_backpressure_overrides_strategy(monkeypatch, caplog): + def check(strategy, depth, overridden): + monkeypatch.setenv("ATOM_DP_DISPATCH_DEPTH", str(depth)) + caplog.clear() + mgr = None + try: + with caplog.at_level(logging.WARNING, logger="atom"): + mgr = _init_with_strategy(strategy) + assert mgr._bp_depth == depth + hit = any( + f"ignores dp_load_balance={strategy}" in r.getMessage() + for r in caplog.records + ) + assert hit is overridden + finally: + if mgr is not None: + mgr.ctx.term() + + check("least_tokens", 8, True) + check("round_robin", 8, True) + check("least_requests", 8, False) + check("least_tokens", 0, False) + + def test_drains_everything(): mgr = make_mgr(n_ranks=2, depth=2) seqs = [Seq() for _ in range(20)] From 8fa12da3043b9613f7332c33a549f394efaf649d Mon Sep 17 00:00:00 2001 From: karverma-amd Date: Fri, 16 Oct 2026 00:57:23 +0000 Subject: [PATCH 5/6] [AMD][DSR1] keep failed DP backpressure sends from leaking or hanging A partial send charged later ranks and then raised, and a refill failure dropped seqs so offline generate spun forever. Re-queue unsent seqs, pause refills after a failed add, and fail the run when nothing is left in flight. Admit the pending pool only from offline generate, and abort a request that is still waiting there. Co-authored-by: Cursor --- atom/model_engine/engine_core_mgr.py | 190 +++++++++++++++++--- atom/model_engine/llm_engine.py | 16 +- atom/rollout/async_engine.py | 3 +- atom/utils/envs.py | 4 +- docs/distributed_guide.md | 2 +- docs/environment_variables.md | 2 +- tests/test_dp_backpressure.py | 164 ++++++++++++++++- tests/test_llm_engine_add_request_fanout.py | 2 + 8 files changed, 344 insertions(+), 39 deletions(-) diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index e7c99d7638..5fef0d9030 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -180,6 +180,8 @@ class CoreManager: # _input_send_lock with their own in _init_shared_state; _bp_depth defaults # to 0 (backpressure disabled). _bp_depth = 0 + _bp_offline_batch = False + _bp_dispatch_paused = False _input_send_lock = Lock() def _init_shared_state( @@ -315,8 +317,16 @@ def _init_shared_state( # The pending drain always picks the rank with the fewest in-flight # requests. That is what lockstep needs. _select_dp_rank_locked cannot # skip ranks already at the cap, so round_robin and least_tokens do - # not apply on this path. + # not apply on this path. Admission is offline-only: online requests + # have no rank hint either, and a seq waiting here is invisible to the + # scheduler until ``offline_backpressure_batch`` lets it in. self._bp_depth = envs.ATOM_DP_DISPATCH_DEPTH + self._bp_offline_batch = False + # Set when an add-path send fails, or a refill fails with nothing left + # in flight. Refills then stop, so a failed add cannot keep dispatching + # after the caller has already seen the error. ``reset_dp_router`` + # clears it. + self._bp_dispatch_paused = False if self._bp_depth > 0 and self._dp_lb_strategy != "least_requests": logger.warning( "%s: ATOM_DP_DISPATCH_DEPTH=%d balances on in-flight request " @@ -1048,13 +1058,17 @@ def _dispatch_to_dp_ranks(self, seqs: list[Sequence]) -> None: # getattr/int pass per seq. hints = self._resolve_and_validate_hints(seqs) - # Backpressure (lazy) dispatch. Only plain load-balanced batches (no - # explicit rank hints, no sticky sessions), which is the offline path. - # Holding work back makes every rank finish prefilling together, so the - # tail can stay all-decode and on CUDA-graph replay. A prefill on any - # rank is what takes the group off the graph; an idle dummy batch is not. + # Backpressure (lazy) dispatch. Only an offline batch (see + # ``offline_backpressure_batch``) of plain load-balanced seqs: no + # explicit rank hints, no sticky sessions. Online API requests carry + # no rank hint either, and must not wait in ``_bp_pending`` — abort + # and the scheduler cannot see them there. Holding the offline batch + # back makes every rank finish prefilling together, so the tail can + # stay all-decode and on CUDA-graph replay. A prefill on any rank is + # what takes the group off the graph; an idle dummy batch is not. if ( getattr(self, "_bp_depth", 0) > 0 + and getattr(self, "_bp_offline_batch", False) and not self._dp_session_affinity_enabled and all(h is None for h in hints) ): @@ -1160,43 +1174,55 @@ def _bp_dispatch_or_refill(self, new_seqs=None, raise_on_error: bool = False): Called both on add (``new_seqs`` set, request thread) and after each completion (``new_seqs`` None, per-rank output thread) to re-feed the rank that just freed a slot. Charging happens under ``_lb_lock``; the - actual ZMQ sends happen after the lock is dropped. On a send failure the - just-charged seqs are released so the per-rank counters never leak. - - ``raise_on_error`` propagates on the add path (the offline ``generate`` - caller fails the run cleanly). The completion path swallows + logs so a - broken socket cannot silently kill an output thread. + actual ZMQ sends happen after the lock is dropped. + + Every rank in this batch is attempted. A rank whose send fails has its + charge released and its seqs put back at the head of ``_bp_pending``, + so a later rank is not left charged-but-unsent and a seq is never + dropped. On the add path the error is re-raised and refills are paused, + so those seqs are not dispatched after the caller has already failed + the run. On the completion path the error is logged and the next + completion retries; if nothing is left in flight to retry, the run + fails through ``outputs_queue`` instead of spinning in ``generate``. """ with self._lb_lock: + # A failed add paused refills. A new add (new_seqs) is a later + # admission and may proceed; reset_dp_router clears the pause. + if new_seqs is None and getattr(self, "_bp_dispatch_paused", False): + return if new_seqs: self._bp_pending.extend(new_seqs) sends = self._drain_pending_locked() if not sends: return added = [] + failed: list[tuple[int, list]] = [] + send_error: Exception | None = None for dp_rank, rank_seqs in sends: try: self._send_request( dp_rank, pickle.dumps((EngineCoreRequestType.ADD, rank_seqs)) ) added.append(f"rank{dp_rank}: {len(rank_seqs)} req") - except Exception: - # Roll back this rank's charge; it will never finish to release. - for seq in rank_seqs: - self._release_seq_load(seq.id) - if raise_on_error: - raise + except Exception as exc: # noqa: BLE001 - re-queued, then re-raised logger.exception( - "%s: backpressure send to rank%d failed; dropped %d seq(s)", + "%s: backpressure send to rank%d failed; re-queued %d seq(s)", self.label, dp_rank, len(rank_seqs), ) + failed.append((dp_rank, rank_seqs)) + send_error = exc + stalled = self._bp_requeue_unsent(failed, pause_always=raise_on_error) + if send_error is not None and raise_on_error: + raise send_error + if stalled: + self._bp_fail_stalled_run() if added: with self._lb_lock: reqs_snapshot = list(self._rank_reqs) pending = len(self._bp_pending) - logger.info( + logger.debug( "%s: bp-dispatch %s | in-flight reqs=%s pending=%d", self.label, ", ".join(added), @@ -1204,6 +1230,56 @@ def _bp_dispatch_or_refill(self, new_seqs=None, raise_on_error: bool = False): pending, ) + def _bp_requeue_unsent( + self, failed: list[tuple[int, list]], *, pause_always: bool + ) -> bool: + """Release unsent charges and put those seqs back at the head. + + ``extendleft(reversed(...))`` restores each rank's seq order. Failed + ranks are put back from last to first so the earliest rank ends up at + the head, ahead of anything the drain had not popped yet. Returns + whether the pool is non-empty and no rank is in flight. An add-path + failure pauses refills even when some ranks were sent, so their later + completions cannot drain a batch the caller already saw fail. A refill + failure pauses only when nothing is left in flight to retry. Caller + must not hold ``_lb_lock``. + """ + if not failed: + return False + with self._lb_lock: + for _rank, rank_seqs in reversed(failed): + for seq in rank_seqs: + self._release_seq_load_locked(seq.id) + self._bp_pending.extendleft(reversed(rank_seqs)) + stalled = sum(self._rank_reqs) == 0 and bool(self._bp_pending) + if pause_always or stalled: + self._bp_dispatch_paused = True + return stalled + + def _bp_fail_stalled_run(self) -> None: + """Fail ``generate`` when a refill cannot send and nothing will retry. + + ``LLMEngine.generate`` waits while the engine is alive and any request + is unfinished. A dropped seq never finishes, so that wait never ends. + The seqs are still in ``_bp_pending``; this only surfaces the error. + """ + pending = len(getattr(self, "_bp_pending", ())) + logger.error( + "%s: backpressure send failed with nothing in flight; " + "%d request(s) still pending", + self.label, + pending, + ) + outputs = getattr(self, "outputs_queue", None) + if outputs is None: + return + outputs.put( + RuntimeError( + f"{self.label}: backpressure send failed and no rank is in " + f"flight; {pending} request(s) were not dispatched" + ) + ) + def _select_dp_rank_locked(self) -> int: """Pick a DP engine rank for a new request. Caller must hold _lb_lock. @@ -1444,12 +1520,16 @@ def _release_seq_load(self, seq_id) -> None: call (e.g. finish followed by abort) is a no-op. """ with self._lb_lock: - entry = self._seq_load.pop(seq_id, None) - if entry is None: - return - dp_rank, req_cost, tok_cost = entry - self._rank_reqs[dp_rank] -= req_cost - self._rank_tokens[dp_rank] -= tok_cost + self._release_seq_load_locked(seq_id) + + def _release_seq_load_locked(self, seq_id) -> None: + """``_release_seq_load`` for a caller that already holds ``_lb_lock``.""" + entry = self._seq_load.pop(seq_id, None) + if entry is None: + return + dp_rank, req_cost, tok_cost = entry + self._rank_reqs[dp_rank] -= req_cost + self._rank_tokens[dp_rank] -= tok_cost def _mark_seq_prefill_complete(self, seq_id) -> None: """Release only a sequence's prefill-token charge, once.""" @@ -1510,6 +1590,7 @@ def reset_dp_router(self) -> None: pending.clear() self._dp_session_owners.clear() self._dp_session_prompt_tokens.clear() + self._bp_dispatch_paused = False def send_utility_command(self, cmd: str, dp_rank: int | None = None): self._refuse_unowned_weight_sync_cmd(cmd, None) @@ -1530,17 +1611,72 @@ def send_utility_command(self, cmd: str, dp_rank: int | None = None): dp_rank, pickle.dumps((EngineCoreRequestType.UTILITY, {"cmd": cmd})) ) + @contextmanager + def offline_backpressure_batch(self): + """Admit the enclosed ``add_request`` through lazy DP dispatch. + + Online serving calls ``add_request`` with no rank hint, same as an + offline batch. Only this context enables the pending pool, and only + for that admission. Refills of seqs already in the pool do not consult + the flag. ``LLMEngine.generate`` is the entry point that sets it. + """ + previous = getattr(self, "_bp_offline_batch", False) + self._bp_offline_batch = True + try: + yield + finally: + self._bp_offline_batch = previous + + def _bp_take_pending(self, req_id): + """Remove one still-pending seq, or None. Holds ``_lb_lock``.""" + pending = getattr(self, "_bp_pending", None) + if not pending: + return None + with self._lb_lock: + found = None + kept = [] + while pending: + seq = pending.popleft() + if found is None and seq.id == req_id: + found = seq + else: + kept.append(seq) + pending.extend(kept) + return found + def abort_request(self, req_id): """Tell the engine core(s) to drop a request (client disconnected). Broadcast to every DP rank (only the one holding ``req_id`` acts). The scheduler finishes the seq at its next step via the normal stop path, freeing its KV blocks. Fire-and-forget; safe if the seq already finished. + + A seq still in the backpressure pool has not reached any engine, so the + broadcast would not find it and a later refill would run it anyway. + Pull it out of the pool and, when the client is waiting on a callback, + deliver the terminal output here. """ + pending_seq = self._bp_take_pending(req_id) # Release DP load bookkeeping now: an aborted seq may never emit a # finished STREAM output, so relying on the finish path alone would leak - # its in-flight count. _release_seq_load is idempotent. + # its in-flight count. _release_seq_load is idempotent. A pending seq + # was never charged, so this is a no-op for it. self._release_seq_load(req_id) + if pending_seq is not None: + if not getattr(pending_seq, "leave_reason", None): + pending_seq.leave_reason = "aborted" + if self._deliver_terminal_output(pending_seq): + flush = getattr(self, "_flush_stream_batch_fn", None) + if flush is not None: + try: + flush() + except Exception as e: # noqa: BLE001 - abort must not raise + logger.warning( + "%s: flush_stream_batch failed during abort: %s", + self.label, + e, + ) + return try: self.broadcast_utility_command("abort_request", req_id=req_id) except Exception as e: # noqa: BLE001 - disconnect cleanup must not raise diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index 9364430b6f..7647df8515 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -302,7 +302,10 @@ def generate( # fresh batch gets deterministic DP assignment and no leaked counts. self.core_mgr.reset_dp_router() - self.add_request(prompts, sampling_params, request_ids=request_ids) + # Backpressure admission is offline-only. The flag covers this add, + # not the collect loop; refills of the pool do not consult it. + with self.core_mgr.offline_backpressure_batch(): + self.add_request(prompts, sampling_params, request_ids=request_ids) outputs = {} while not self.is_finished() and ( self.core_mgr.is_alive() or self.core_mgr.is_rest() @@ -322,11 +325,12 @@ def generate_multimodal( ) -> list[dict]: """Generate completions for multimodal inputs (token IDs + vision data).""" self.core_mgr.reset_dp_router() - self.add_request( - token_ids_list, - sampling_params, - multimodal_data_list=multimodal_data_list, - ) + with self.core_mgr.offline_backpressure_batch(): + self.add_request( + token_ids_list, + sampling_params, + multimodal_data_list=multimodal_data_list, + ) outputs = {} while not self.is_finished() and ( self.core_mgr.is_alive() or self.core_mgr.is_rest() diff --git a/atom/rollout/async_engine.py b/atom/rollout/async_engine.py index 570badafd1..681757d51a 100644 --- a/atom/rollout/async_engine.py +++ b/atom/rollout/async_engine.py @@ -201,7 +201,8 @@ def generate_hidden_states(self, input_ids_list, data_ids): ] self.core_mgr.reset_dp_router() - self.add_request(prompts, sampling_params, request_ids=data_ids) + with self.core_mgr.offline_backpressure_batch(): + self.add_request(prompts, sampling_params, request_ids=data_ids) outputs = {} while not self.is_finished() and ( diff --git a/atom/utils/envs.py b/atom/utils/envs.py index b421f1e633..14ef3e659c 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -182,7 +182,9 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: # Per-rank cap on in-flight load-balanced requests. 0 sprays the whole # offline batch up front. >0 holds the rest and refills a rank only as it # finishes, so ranks finish prefilling together and the tail stays - # all-decode (CUDA-graph replay). While set, dispatch balances on in-flight + # all-decode (CUDA-graph replay). Only offline batch admission + # (LLMEngine.generate / generate_multimodal) uses it; online API requests + # keep immediate dispatch. While set, that path balances on in-flight # request count and ignores dp_load_balance; CoreManager warns when that # strategy is not least_requests. A non-integer or negative value warns # and reads as 0. Keep it >= max_num_seqs; a smaller cap truncates every diff --git a/docs/distributed_guide.md b/docs/distributed_guide.md index 5060feb45e..d27e7fb98e 100644 --- a/docs/distributed_guide.md +++ b/docs/distributed_guide.md @@ -260,7 +260,7 @@ drained and warns if it is invoked while requests are still charged. - `Config.dp_load_balance` (str, default `"least_requests"`): DP routing strategy. - CLI: `--data-parallel-size N` or `-dp N`; `--dp-load-balance {round_robin,least_requests,least_tokens}` - Env: `ATOM_DP_LB_REQ_EQUIV` (int, default `512`): token-equivalent cost of one in-flight request for `least_tokens`. -- Env: `ATOM_DP_DISPATCH_DEPTH` (int, default `0`): per-rank in-flight cap for lazy dispatch. While it is positive, load-balanced routing balances on in-flight request count and ignores `round_robin` and `least_tokens`; init warns when `dp_load_balance` is not `least_requests`. +- Env: `ATOM_DP_DISPATCH_DEPTH` (int, default `0`): per-rank in-flight cap for lazy dispatch of an offline batch (`LLMEngine.generate` / `generate_multimodal`). Online API admission is unchanged. While it is positive, that offline routing balances on in-flight request count and ignores `round_robin` and `least_tokens`; init warns when `dp_load_balance` is not `least_requests`. A failed send returns the requests to the pending pool. `abort_request` removes a request that has not been sent yet. - Env: `ATOM_DP_SESSION_AFFINITY` (bool, default `0`): load-place each new session once, then keep all later turns on its cache owner without spill. diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 2097023e3b..d3f2c953ee 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -18,7 +18,7 @@ This document describes the environment variables used in the ATOM project. | **ATOM_DP_MASTER_IP** | str | 127.0.0.1 | Master IP address for DP ranks coordination. | | **ATOM_DP_MASTER_PORT** | int | 29500 | Master port for DP ranks coordination. | | **ATOM_DP_LB_REQ_EQUIV** | int | 512 | Token-equivalent decode pressure assigned to each in-flight request by `least_tokens` routing. | -| **ATOM_DP_DISPATCH_DEPTH** | int | 0 | Per-rank cap on in-flight load-balanced requests. `0` sprays the offline batch up front. `>0` holds the rest and refills a rank as it finishes, so ranks finish prefilling together and the tail stays all-decode on the CUDA graph. A prefill on any rank is what leaves the graph; an idle rank's dummy batch does not. While set, dispatch balances on in-flight request count and ignores `--dp-load-balance` (`round_robin` and `least_tokens`); init warns when the configured strategy is not `least_requests`. Keep it `>= max_num_seqs` (about 1.2–1.5× is a starting point). A non-integer or negative value warns and disables the feature. A positive value below `max_num_seqs` warns and still caps every rank's running batch. | +| **ATOM_DP_DISPATCH_DEPTH** | int | 0 | Per-rank cap on in-flight load-balanced requests for an offline batch (`LLMEngine.generate` / `generate_multimodal`). `0` sprays that batch up front. `>0` holds the rest and refills a rank as it finishes, so ranks finish prefilling together and the tail stays all-decode on the CUDA graph. A prefill on any rank is what leaves the graph; an idle rank's dummy batch does not. Online API requests are dispatched immediately even when this is set. While set, offline dispatch balances on in-flight request count and ignores `--dp-load-balance` (`round_robin` and `least_tokens`); init warns when the configured strategy is not `least_requests`. A failed send puts the requests back on the pending pool instead of dropping them. `abort_request` removes a request that is still pending. Keep it `>= max_num_seqs` (about 1.2–1.5× is a starting point). A non-integer or negative value warns and disables the feature. A positive value below `max_num_seqs` warns and still caps every rank's running batch. | | **ATOM_DP_SESSION_AFFINITY** | bool | false | Load-place each new session, then keep later turns on the same prefix-cache owner. Reads `X-Dynamo-Session-ID`, falling back to `X-Correlation-ID`. | ## Prefill delayer (TP/DCP and DP attention) diff --git a/tests/test_dp_backpressure.py b/tests/test_dp_backpressure.py index d0ad515bfc..6fd503af3f 100644 --- a/tests/test_dp_backpressure.py +++ b/tests/test_dp_backpressure.py @@ -8,10 +8,15 @@ import logging import pickle +import queue +import threading +import time from collections import deque from threading import Lock from types import SimpleNamespace +import pytest + from atom.model_engine.engine_core_mgr import CoreManager from atom.model_engine.engine_core_protocol import EngineCoreRequestType @@ -29,7 +34,7 @@ def __init__(self, prompt=10): self.stream_callback = None -def make_mgr(n_ranks=4, depth=3): +def make_mgr(n_ranks=4, depth=3, fail_ranks=None): mgr = CoreManager.__new__(CoreManager) mgr.label = "test" mgr._dp_lb_strategy = "least_requests" @@ -48,19 +53,34 @@ def make_mgr(n_ranks=4, depth=3): mgr._input_send_lock = Lock() mgr._bp_depth = depth mgr._bp_pending = deque() + mgr._bp_offline_batch = True + mgr._bp_dispatch_paused = False mgr._seq_id_to_callback = {} + mgr._flush_stream_batch_fn = None + mgr.fail_ranks = set(fail_ranks or ()) # capture what got sent to each rank mgr.sent = [[] for _ in range(n_ranks)] + mgr._send_lock = Lock() def fake_send(dp_rank, payload): + if dp_rank in mgr.fail_ranks: + raise RuntimeError(f"send failed rank {dp_rank}") _type, seqs = pickle.loads(payload) assert _type == EngineCoreRequestType.ADD - mgr.sent[dp_rank].extend(s.id for s in seqs) + with mgr._send_lock: + mgr.sent[dp_rank].extend(s.id for s in seqs) mgr._send_request = fake_send return mgr +def _accounted(mgr): + with mgr._send_lock: + sent = [sid for rank_ids in mgr.sent for sid in rank_ids] + pending = [seq.id for seq in mgr._bp_pending] + return sent, pending + + def test_cap_and_pending(): mgr = make_mgr(n_ranks=4, depth=3) seqs = [Seq() for _ in range(100)] @@ -149,3 +169,143 @@ def test_drains_everything(): dispatched = [sid for rank_ids in mgr.sent for sid in rank_ids] assert len(dispatched) == 20 # every seq dispatched exactly once assert len(set(dispatched)) == 20 # no duplicates + + +def test_online_add_does_not_enter_pending_pool(): + mgr = make_mgr(n_ranks=2, depth=1) + mgr._bp_offline_batch = False + seqs = [Seq() for _ in range(6)] + mgr.add_request(seqs) + assert len(mgr._bp_pending) == 0 + sent, pending = _accounted(mgr) + assert pending == [] + assert sorted(sent) == sorted(seq.id for seq in seqs) + + +def test_partial_send_failure_requeues_later_ranks_and_pauses(): + # Rank 1 fails in the middle of the send loop. Rank 2 must still be + # attempted. The failed seq goes back to the head of the pool, its charge + # returns to zero, and a later refill does not dispatch the failed add. + mgr = make_mgr(n_ranks=3, depth=1, fail_ranks={1}) + seqs = [Seq() for _ in range(6)] + with pytest.raises(RuntimeError, match="send failed rank 1"): + mgr.add_request(seqs) + assert mgr._rank_reqs == [1, 0, 1] + assert [seq.id for seq in mgr._bp_pending][0] == seqs[1].id + sent, pending = _accounted(mgr) + assert sent == [seqs[0].id, seqs[2].id] + assert sorted(sent + pending) == sorted(seq.id for seq in seqs) + assert len(sent + pending) == len(set(sent + pending)) + assert mgr._bp_dispatch_paused + mgr._bp_dispatch_or_refill(raise_on_error=False) + sent_after, _pending_after = _accounted(mgr) + assert sent_after == sent + + +def test_every_send_failing_returns_counters_to_zero(): + mgr = make_mgr(n_ranks=2, depth=1, fail_ranks={0, 1}) + seqs = [Seq() for _ in range(4)] + with pytest.raises(RuntimeError): + mgr.add_request(seqs) + assert mgr._rank_reqs == [0, 0] + assert not mgr._seq_load + sent, pending = _accounted(mgr) + assert sent == [] + assert sorted(pending) == sorted(seq.id for seq in seqs) + assert len(pending) == len(set(pending)) + + +def test_refill_send_failure_requeues_and_fails_a_stalled_run(): + mgr = make_mgr(n_ranks=2, depth=1) + mgr.outputs_queue = queue.Queue() + seqs = [Seq() for _ in range(4)] + mgr.add_request(seqs) + assert sum(mgr._rank_reqs) == 2 + mgr.fail_ranks.update({0, 1}) + for sid in list(mgr._seq_load): + mgr._release_seq_load(sid) + mgr._bp_dispatch_or_refill(raise_on_error=False) + assert mgr._rank_reqs == [0, 0] + sent, pending = _accounted(mgr) + # The two originally sent seqs stay sent; the two that failed the refill + # are back in the pool, once each. + assert len(sent) == 2 + assert len(pending) == 2 + assert len(set(sent + pending)) == 4 + assert mgr._bp_dispatch_paused + err = mgr.outputs_queue.get_nowait() + assert isinstance(err, RuntimeError) + + +def test_abort_pending_seq_is_not_dispatched(): + mgr = make_mgr(n_ranks=2, depth=1) + seqs = [Seq() for _ in range(4)] + delivered = {} + + def callback(output): + delivered[output.request_id] = output.finish_reason + + seqs[-1].stream_callback = callback + mgr.add_request(seqs) + target = seqs[-1] + assert any(seq.id == target.id for seq in mgr._bp_pending) + mgr.abort_request(target.id) + assert all(seq.id != target.id for seq in mgr._bp_pending) + assert delivered[target.id] == "aborted" + guard = 0 + while mgr._seq_load or mgr._bp_pending: + if mgr._seq_load: + sid = next(iter(mgr._seq_load)) + mgr._release_seq_load(sid) + if mgr._bp_pending: + mgr._bp_dispatch_or_refill(raise_on_error=False) + guard += 1 + assert guard < 1000 + sent, pending = _accounted(mgr) + assert target.id not in sent + assert pending == [] + assert len(sent) == 3 + + +def test_concurrent_add_and_refill_dispatches_once(): + mgr = make_mgr(n_ranks=4, depth=2) + seqs = [Seq() for _ in range(40)] + errors = [] + + def add_chunks(): + for i in range(0, len(seqs), 4): + mgr.add_request(seqs[i : i + 4]) + + def release_loop(): + try: + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + sid = None + with mgr._lb_lock: + if mgr._seq_load: + sid = next(iter(mgr._seq_load)) + if sid is None: + if not adder.is_alive() and not mgr._bp_pending: + return + time.sleep(0.001) + continue + mgr._release_seq_load(sid) + mgr._bp_dispatch_or_refill(raise_on_error=False) + raise AssertionError("timed out draining concurrent backpressure") + except Exception as exc: # noqa: BLE001 - surfaced to the test thread + errors.append(exc) + + adder = threading.Thread(target=add_chunks) + releaser = threading.Thread(target=release_loop) + adder.start() + releaser.start() + adder.join() + releaser.join() + assert not errors, errors + assert mgr._rank_reqs == [0, 0, 0, 0] + assert not mgr._seq_load + assert not mgr._bp_pending + sent, pending = _accounted(mgr) + assert pending == [] + assert sorted(sent) == sorted(seq.id for seq in seqs) + assert len(sent) == len(set(sent)) diff --git a/tests/test_llm_engine_add_request_fanout.py b/tests/test_llm_engine_add_request_fanout.py index 9e6ea86b3a..cbbb58f66d 100644 --- a/tests/test_llm_engine_add_request_fanout.py +++ b/tests/test_llm_engine_add_request_fanout.py @@ -14,6 +14,7 @@ still does. """ +from contextlib import nullcontext from types import SimpleNamespace import pytest @@ -117,6 +118,7 @@ def __init__(self): self._pending = True self.core_mgr = SimpleNamespace( reset_dp_router=lambda: None, + offline_backpressure_batch=lambda: nullcontext(), add_request=self._submitted.extend, get_output=self._get_output, is_alive=lambda: True, From abbea70cf19bba43feca33910828f1dce421e811 Mon Sep 17 00:00:00 2001 From: karverma-amd Date: Fri, 16 Oct 2026 01:15:38 +0000 Subject: [PATCH 6/6] [AMD][DSR1] clear the ruff findings in the backpressure diff The send handler re-raises, so the BLE001 suppression was unused, and the pending-head check should use next() instead of a one-element slice. Co-authored-by: Cursor --- atom/model_engine/engine_core_mgr.py | 2 +- tests/test_dp_backpressure.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 5fef0d9030..c54a9dc9cc 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -1204,7 +1204,7 @@ def _bp_dispatch_or_refill(self, new_seqs=None, raise_on_error: bool = False): dp_rank, pickle.dumps((EngineCoreRequestType.ADD, rank_seqs)) ) added.append(f"rank{dp_rank}: {len(rank_seqs)} req") - except Exception as exc: # noqa: BLE001 - re-queued, then re-raised + except Exception as exc: logger.exception( "%s: backpressure send to rank%d failed; re-queued %d seq(s)", self.label, diff --git a/tests/test_dp_backpressure.py b/tests/test_dp_backpressure.py index 6fd503af3f..8abca2f278 100644 --- a/tests/test_dp_backpressure.py +++ b/tests/test_dp_backpressure.py @@ -191,7 +191,7 @@ def test_partial_send_failure_requeues_later_ranks_and_pauses(): with pytest.raises(RuntimeError, match="send failed rank 1"): mgr.add_request(seqs) assert mgr._rank_reqs == [1, 0, 1] - assert [seq.id for seq in mgr._bp_pending][0] == seqs[1].id + assert next(seq.id for seq in mgr._bp_pending) == seqs[1].id sent, pending = _accounted(mgr) assert sent == [seqs[0].id, seqs[2].id] assert sorted(sent + pending) == sorted(seq.id for seq in seqs)