diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 10a284c81b..fc15865fbc 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -12,6 +12,7 @@ import time import uuid import weakref +from collections import deque from contextlib import contextmanager from dataclasses import dataclass from threading import Lock, RLock, Thread, local @@ -92,7 +93,8 @@ def build_internode_dp_socket_plan( selects the IPC path (and, with it, the connect/bind polarity) further down. Three ports per engine: requests, outputs, and control. Control is separate - so the request socket keeps a single writer thread (see _send_request). + so utility, abort, and shutdown never share ``input_sockets`` with add and + backpressure refill (see _send_request). """ base = master_port + INTERNODE_DP_SOCKET_PORT_OFFSET return [ @@ -182,6 +184,16 @@ def _add_frame(seqs: list[Sequence]) -> bytes: class CoreManager: + # Class-level defaults so bare instances built via ``__new__`` (the DP + # load-balance unit tests) and any subclass that skips ``_init_shared_state`` + # still have working backpressure state. Real instances override + # _input_send_lock with their own in _init_shared_state; _bp_depth defaults + # to 0 (backpressure disabled). + _bp_depth = 0 + _bp_offline_batch = False + _bp_dispatch_paused = False + _input_send_lock = Lock() + def _init_shared_state( self, config: Config, @@ -303,6 +315,50 @@ def _init_shared_state( # what dispatch added, and only for ranks that were actually charged. self._seq_load = {} self._lb_lock = Lock() + # --- Backpressure (lazy) DP dispatch -------------------------------- + # ATOM_DP_DISPATCH_DEPTH > 0 holds load-balanced seqs in _bp_pending and + # releases them to the lightest rank only up to that in-flight cap. A + # rank that finishes its share early is re-fed instead of sitting on a + # dummy decode while peers still open prefills. A prefill on any rank + # takes the whole lockstep group off CUDA-graph replay; an idle rank's + # dummy batch does not. Finishing prefills together lets the tail stay + # all-decode and on the graph. 0 keeps the legacy up-front spray. Depth + # should stay >= max_num_seqs so each rank can fill a full running batch. + # The pending drain always picks the rank with the fewest in-flight + # requests. That is what lockstep needs. _select_dp_rank_locked cannot + # skip ranks already at the cap, so round_robin and least_tokens do + # not apply on this path. Admission is offline-only: online requests + # have no rank hint either, and a seq waiting here is invisible to the + # scheduler until ``offline_backpressure_batch`` lets it in. + self._bp_depth = envs.ATOM_DP_DISPATCH_DEPTH + self._bp_offline_batch = False + # Set when an add-path send fails, or a refill fails with nothing left + # in flight. Refills then stop, so a failed add cannot keep dispatching + # after the caller has already seen the error. ``reset_dp_router`` + # clears it. + self._bp_dispatch_paused = False + if self._bp_depth > 0 and self._dp_lb_strategy != "least_requests": + logger.warning( + "%s: ATOM_DP_DISPATCH_DEPTH=%d balances on in-flight request " + "count and ignores dp_load_balance=%s", + label, + self._bp_depth, + self._dp_lb_strategy, + ) + if 0 < self._bp_depth < config.max_num_seqs: + logger.warning( + "%s: ATOM_DP_DISPATCH_DEPTH=%d is below max_num_seqs=%d; " + "each rank's running batch is capped below a full step", + label, + self._bp_depth, + config.max_num_seqs, + ) + self._bp_pending: deque = deque() + # Backpressure refill runs on the per-rank output threads, which then + # also write input_sockets. A ZMQ socket is not thread-safe, so every + # input_sockets send is serialized through this lock (see _send_request). + # Uncontended in the legacy path, so it costs nothing when disabled. + self._input_send_lock = Lock() # Control traffic (utility commands, abort, shutdown) travels on its own # sockets so that input_sockets keeps a single writer -- see # _send_request. These have several writer threads and so do need @@ -776,6 +832,12 @@ def process_outputs_socket(): exc_info=True, ) self.outputs_queue.put_nowait(seqs) + # These completions freed slots on their ranks — re-feed + # from the pending pool so a fast rank never drains to + # empty while peers still have work. No-op when + # ATOM_DP_DISPATCH_DEPTH is unset (legacy spray path). + if getattr(self, "_bp_depth", 0) > 0 and self._bp_pending: + self._bp_dispatch_or_refill(raise_on_error=False) finally: # Close sockets. shutdown_socket.close(linger=0) @@ -900,24 +962,26 @@ def close(self): def _send_request(self, dp_rank: int, payload: bytes) -> None: """Send one already-pickled request to an engine core. Hot path. - Deliberately unsynchronized: ``input_sockets`` carries nothing but - ``add_request``, which runs on a single thread (the API server's event - loop online, the caller's thread offline). Everything that can be sent - from another thread -- utility commands, abort, shutdown -- goes to - :meth:`_send_control` on a separate socket instead. - - That separation is load-bearing, not stylistic. A ZMQ socket is not - thread-safe: two unserialized ``send_multipart`` calls interleave their - frames, the DEALER on the other end then reads a routing identity where - a payload should be, and its input thread dies on ``UnpicklingError``. - Nothing recovers from that -- the engine spins on a forever-empty input - queue, the workers idle, and every client hangs with no error logged - anywhere but that thread's own traceback. So: never send to - ``input_sockets`` from anywhere but here. + Every ``input_sockets`` write goes through here and holds + ``_input_send_lock``. Two caller classes share the socket: the request + thread (``add_request``: the API server's event loop online, the + caller's thread offline) and the per-rank output threads (backpressure + refill when ``ATOM_DP_DISPATCH_DEPTH`` > 0). A ZMQ socket is not + thread-safe: unserialized ``send_multipart`` calls interleave frames, + the DEALER reads a routing identity where a payload should be, and its + input thread dies on ``UnpicklingError``. Nothing recovers from that. + + Refill sends the same ADD payload as the request thread, so it reuses + this socket under the lock. A second ROUTER would need another + engine-side reader for the same queue. Utility commands, abort, and + shutdown stay on :meth:`_send_control`, which is why control has its + own socket. Never send to ``input_sockets`` from anywhere but here. + The lock is uncontended when backpressure is off. """ - self.input_sockets[dp_rank].send_multipart( - [self.engine_core_identities[dp_rank], payload], copy=False - ) + with self._input_send_lock: + self.input_sockets[dp_rank].send_multipart( + [self.engine_core_identities[dp_rank], payload], copy=False + ) def _send_control(self, dp_rank: int, payload: bytes, copy: bool = False) -> None: """Send one already-pickled control message. Serialized, never hot. @@ -1005,6 +1069,23 @@ def _dispatch_to_dp_ranks(self, seqs: list[Sequence]) -> None: # getattr/int pass per seq. hints = self._resolve_and_validate_hints(seqs) + # Backpressure (lazy) dispatch. Only an offline batch (see + # ``offline_backpressure_batch``) of plain load-balanced seqs: no + # explicit rank hints, no sticky sessions. Online API requests carry + # no rank hint either, and must not wait in ``_bp_pending`` — abort + # and the scheduler cannot see them there. Holding the offline batch + # back makes every rank finish prefilling together, so the tail can + # stay all-decode and on CUDA-graph replay. A prefill on any rank is + # what takes the group off the graph; an idle dummy batch is not. + if ( + getattr(self, "_bp_depth", 0) > 0 + and getattr(self, "_bp_offline_batch", False) + and not self._dp_session_affinity_enabled + and all(h is None for h in hints) + ): + self._bp_dispatch_or_refill(new_seqs=seqs, raise_on_error=True) + return + # round_robin normally skips load bookkeeping. Session affinity still # needs queued-prefill counters even if the fallback strategy is RR. track_load = ( @@ -1072,6 +1153,142 @@ def _dispatch_to_dp_ranks(self, seqs: list[Sequence]) -> None: else: logger.info("%s: add %s", self.label, ", ".join(added)) + def _drain_pending_locked(self) -> list[tuple[int, list]]: + """Release pending seqs up to the per-rank depth cap. Hold _lb_lock. + + Fills the lightest rank first (fewest in-flight requests), charging each + seq as it goes, until every rank is at the cap or the pending pool is + empty. Balancing on request count keeps the lockstep DP ranks in phase. + ``round_robin`` and ``least_tokens`` are not consulted here; init warns + when one of those is configured alongside a positive depth. The cap + leaves work queued so a fast rank can be re-fed on its next completion + instead of draining to empty. Returns per-rank sub-batches to send + after the lock is dropped (ZMQ sends must not hold ``_lb_lock``). + """ + n = self._routable_engine_count + batches: list[list] = [[] for _ in range(n)] + while self._bp_pending: + # Lightest rank by (in-flight reqs, rank) for a stable tie-break. + best = min(range(n), key=lambda r: (self._rank_reqs[r], r)) + if self._rank_reqs[best] >= self._bp_depth: + break # every rank is at/above the cap; hold the rest. + seq = self._bp_pending.popleft() + self._charge_seq_load_locked(seq, best) + batches[best].append(seq) + return [(r, s) for r, s in enumerate(batches) if s] + + def _bp_dispatch_or_refill(self, new_seqs=None, raise_on_error: bool = False): + """Backpressure dispatch: enqueue, then top ranks up to the cap. + + Called both on add (``new_seqs`` set, request thread) and after each + completion (``new_seqs`` None, per-rank output thread) to re-feed the + rank that just freed a slot. Charging happens under ``_lb_lock``; the + actual ZMQ sends happen after the lock is dropped. + + Every rank in this batch is attempted. A rank whose send fails has its + charge released and its seqs put back at the head of ``_bp_pending``, + so a later rank is not left charged-but-unsent and a seq is never + dropped. On the add path the error is re-raised and refills are paused, + so those seqs are not dispatched after the caller has already failed + the run. On the completion path the error is logged and the next + completion retries; if nothing is left in flight to retry, the run + fails through ``outputs_queue`` instead of spinning in ``generate``. + """ + with self._lb_lock: + # A failed add paused refills. A new add (new_seqs) is a later + # admission and may proceed; reset_dp_router clears the pause. + if new_seqs is None and getattr(self, "_bp_dispatch_paused", False): + return + if new_seqs: + self._bp_pending.extend(new_seqs) + sends = self._drain_pending_locked() + if not sends: + return + added = [] + failed: list[tuple[int, list]] = [] + send_error: Exception | None = None + for dp_rank, rank_seqs in sends: + try: + self._send_request( + dp_rank, pickle.dumps((EngineCoreRequestType.ADD, rank_seqs)) + ) + added.append(f"rank{dp_rank}: {len(rank_seqs)} req") + except Exception as exc: + logger.exception( + "%s: backpressure send to rank%d failed; re-queued %d seq(s)", + self.label, + dp_rank, + len(rank_seqs), + ) + failed.append((dp_rank, rank_seqs)) + send_error = exc + stalled = self._bp_requeue_unsent(failed, pause_always=raise_on_error) + if send_error is not None and raise_on_error: + raise send_error + if stalled: + self._bp_fail_stalled_run() + if added: + with self._lb_lock: + reqs_snapshot = list(self._rank_reqs) + pending = len(self._bp_pending) + logger.debug( + "%s: bp-dispatch %s | in-flight reqs=%s pending=%d", + self.label, + ", ".join(added), + reqs_snapshot, + pending, + ) + + def _bp_requeue_unsent( + self, failed: list[tuple[int, list]], *, pause_always: bool + ) -> bool: + """Release unsent charges and put those seqs back at the head. + + ``extendleft(reversed(...))`` restores each rank's seq order. Failed + ranks are put back from last to first so the earliest rank ends up at + the head, ahead of anything the drain had not popped yet. Returns + whether the pool is non-empty and no rank is in flight. An add-path + failure pauses refills even when some ranks were sent, so their later + completions cannot drain a batch the caller already saw fail. A refill + failure pauses only when nothing is left in flight to retry. Caller + must not hold ``_lb_lock``. + """ + if not failed: + return False + with self._lb_lock: + for _rank, rank_seqs in reversed(failed): + for seq in rank_seqs: + self._release_seq_load_locked(seq.id) + self._bp_pending.extendleft(reversed(rank_seqs)) + stalled = sum(self._rank_reqs) == 0 and bool(self._bp_pending) + if pause_always or stalled: + self._bp_dispatch_paused = True + return stalled + + def _bp_fail_stalled_run(self) -> None: + """Fail ``generate`` when a refill cannot send and nothing will retry. + + ``LLMEngine.generate`` waits while the engine is alive and any request + is unfinished. A dropped seq never finishes, so that wait never ends. + The seqs are still in ``_bp_pending``; this only surfaces the error. + """ + pending = len(getattr(self, "_bp_pending", ())) + logger.error( + "%s: backpressure send failed with nothing in flight; " + "%d request(s) still pending", + self.label, + pending, + ) + outputs = getattr(self, "outputs_queue", None) + if outputs is None: + return + outputs.put( + RuntimeError( + f"{self.label}: backpressure send failed and no rank is in " + f"flight; {pending} request(s) were not dispatched" + ) + ) + def _select_dp_rank_locked(self) -> int: """Pick a DP engine rank for a new request. Caller must hold _lb_lock. @@ -1312,12 +1529,16 @@ def _release_seq_load(self, seq_id) -> None: call (e.g. finish followed by abort) is a no-op. """ with self._lb_lock: - entry = self._seq_load.pop(seq_id, None) - if entry is None: - return - dp_rank, req_cost, tok_cost = entry - self._rank_reqs[dp_rank] -= req_cost - self._rank_tokens[dp_rank] -= tok_cost + self._release_seq_load_locked(seq_id) + + def _release_seq_load_locked(self, seq_id) -> None: + """``_release_seq_load`` for a caller that already holds ``_lb_lock``.""" + entry = self._seq_load.pop(seq_id, None) + if entry is None: + return + dp_rank, req_cost, tok_cost = entry + self._rank_reqs[dp_rank] -= req_cost + self._rank_tokens[dp_rank] -= tok_cost def _mark_seq_prefill_complete(self, seq_id) -> None: """Release only a sequence's prefill-token charge, once.""" @@ -1367,8 +1588,18 @@ def reset_dp_router(self) -> None: self._rank_reqs = [0] * self._routable_engine_count self._rank_tokens = [0] * self._routable_engine_count self._seq_load.clear() + pending = getattr(self, "_bp_pending", None) + if pending: + logger.warning( + "%s: reset_dp_router() dropping %d undispatched " + "request(s) still in the backpressure pool", + self.label, + len(pending), + ) + pending.clear() self._dp_session_owners.clear() self._dp_session_prompt_tokens.clear() + self._bp_dispatch_paused = False def send_utility_command(self, cmd: str, dp_rank: int | None = None): self._refuse_unowned_weight_sync_cmd(cmd, None) @@ -1389,17 +1620,72 @@ def send_utility_command(self, cmd: str, dp_rank: int | None = None): dp_rank, pickle.dumps((EngineCoreRequestType.UTILITY, {"cmd": cmd})) ) + @contextmanager + def offline_backpressure_batch(self): + """Admit the enclosed ``add_request`` through lazy DP dispatch. + + Online serving calls ``add_request`` with no rank hint, same as an + offline batch. Only this context enables the pending pool, and only + for that admission. Refills of seqs already in the pool do not consult + the flag. ``LLMEngine.generate`` is the entry point that sets it. + """ + previous = getattr(self, "_bp_offline_batch", False) + self._bp_offline_batch = True + try: + yield + finally: + self._bp_offline_batch = previous + + def _bp_take_pending(self, req_id): + """Remove one still-pending seq, or None. Holds ``_lb_lock``.""" + pending = getattr(self, "_bp_pending", None) + if not pending: + return None + with self._lb_lock: + found = None + kept = [] + while pending: + seq = pending.popleft() + if found is None and seq.id == req_id: + found = seq + else: + kept.append(seq) + pending.extend(kept) + return found + def abort_request(self, req_id): """Tell the engine core(s) to drop a request (client disconnected). Broadcast to every DP rank (only the one holding ``req_id`` acts). The scheduler finishes the seq at its next step via the normal stop path, freeing its KV blocks. Fire-and-forget; safe if the seq already finished. + + A seq still in the backpressure pool has not reached any engine, so the + broadcast would not find it and a later refill would run it anyway. + Pull it out of the pool and, when the client is waiting on a callback, + deliver the terminal output here. """ + pending_seq = self._bp_take_pending(req_id) # Release DP load bookkeeping now: an aborted seq may never emit a # finished STREAM output, so relying on the finish path alone would leak - # its in-flight count. _release_seq_load is idempotent. + # its in-flight count. _release_seq_load is idempotent. A pending seq + # was never charged, so this is a no-op for it. self._release_seq_load(req_id) + if pending_seq is not None: + if not getattr(pending_seq, "leave_reason", None): + pending_seq.leave_reason = "aborted" + if self._deliver_terminal_output(pending_seq): + flush = getattr(self, "_flush_stream_batch_fn", None) + if flush is not None: + try: + flush() + except Exception as e: # noqa: BLE001 - abort must not raise + logger.warning( + "%s: flush_stream_batch failed during abort: %s", + self.label, + e, + ) + return try: self.broadcast_utility_command("abort_request", req_id=req_id) except Exception as e: # noqa: BLE001 - disconnect cleanup must not raise diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index 2576d0d8fc..04ff042a6e 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -301,7 +301,10 @@ def generate( # fresh batch gets deterministic DP assignment and no leaked counts. self.core_mgr.reset_dp_router() - self.add_request(prompts, sampling_params, request_ids=request_ids) + # Backpressure admission is offline-only. The flag covers this add, + # not the collect loop; refills of the pool do not consult it. + with self.core_mgr.offline_backpressure_batch(): + self.add_request(prompts, sampling_params, request_ids=request_ids) outputs = {} while not self.is_finished() and ( self.core_mgr.is_alive() or self.core_mgr.is_rest() @@ -321,11 +324,12 @@ def generate_multimodal( ) -> list[dict]: """Generate completions for multimodal inputs (token IDs + vision data).""" self.core_mgr.reset_dp_router() - self.add_request( - token_ids_list, - sampling_params, - multimodal_data_list=multimodal_data_list, - ) + with self.core_mgr.offline_backpressure_batch(): + self.add_request( + token_ids_list, + sampling_params, + multimodal_data_list=multimodal_data_list, + ) outputs = {} while not self.is_finished() and ( self.core_mgr.is_alive() or self.core_mgr.is_rest() diff --git a/atom/rollout/async_engine.py b/atom/rollout/async_engine.py index 570badafd1..681757d51a 100644 --- a/atom/rollout/async_engine.py +++ b/atom/rollout/async_engine.py @@ -201,7 +201,8 @@ def generate_hidden_states(self, input_ids_list, data_ids): ] self.core_mgr.reset_dp_router() - self.add_request(prompts, sampling_params, request_ids=data_ids) + with self.core_mgr.offline_backpressure_batch(): + self.add_request(prompts, sampling_params, request_ids=data_ids) outputs = {} while not self.is_finished() and ( diff --git a/atom/utils/envs.py b/atom/utils/envs.py index 9f81e5a203..3ff130083d 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -206,6 +206,17 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: # pressure) and a smaller value toward prompt-token balance (prefill # pressure). See engine_core_mgr.CoreManager._select_dp_rank_locked. "ATOM_DP_LB_REQ_EQUIV": lambda: int(os.getenv("ATOM_DP_LB_REQ_EQUIV", "512")), + # Per-rank cap on in-flight load-balanced requests. 0 sprays the whole + # offline batch up front. >0 holds the rest and refills a rank only as it + # finishes, so ranks finish prefilling together and the tail stays + # all-decode (CUDA-graph replay). Only offline batch admission + # (LLMEngine.generate / generate_multimodal) uses it; online API requests + # keep immediate dispatch. While set, that path balances on in-flight + # request count and ignores dp_load_balance; CoreManager warns when that + # strategy is not least_requests. A non-integer or negative value warns + # and reads as 0. Keep it >= max_num_seqs; a smaller cap truncates every + # rank's running batch. CoreManager warns when that happens. + "ATOM_DP_DISPATCH_DEPTH": lambda: _nonnegative_int_env("ATOM_DP_DISPATCH_DEPTH", 0), # Place a new agent session on the lightest DP rank, then keep every later # request on that immutable cache owner. Existing sessions never spill; # child correlation ids are independently load-placed rather than diff --git a/docs/distributed_guide.md b/docs/distributed_guide.md index 470649732e..830e9bc51d 100644 --- a/docs/distributed_guide.md +++ b/docs/distributed_guide.md @@ -258,6 +258,7 @@ drained and warns if it is invoked while requests are still charged. - `Config.dp_load_balance` (str, default `"least_requests"`): DP routing strategy. - CLI: `--data-parallel-size N` or `-dp N`; `--dp-load-balance {round_robin,least_requests,least_tokens}` - Env: `ATOM_DP_LB_REQ_EQUIV` (int, default `512`): token-equivalent cost of one in-flight request for `least_tokens`. +- Env: `ATOM_DP_DISPATCH_DEPTH` (int, default `0`): per-rank in-flight cap for lazy dispatch of an offline batch (`LLMEngine.generate` / `generate_multimodal`). Online API admission is unchanged. While it is positive, that offline routing balances on in-flight request count and ignores `round_robin` and `least_tokens`; init warns when `dp_load_balance` is not `least_requests`. A failed send returns the requests to the pending pool. `abort_request` removes a request that has not been sent yet. - Env: `ATOM_DP_SESSION_AFFINITY` (bool, default `0`): load-place each new session once, then keep all later turns on its cache owner without spill. diff --git a/docs/environment_variables.md b/docs/environment_variables.md index d03a018909..9108bd8071 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -29,6 +29,7 @@ limitations](observability/README.md). | **ATOM_DP_MASTER_IP** | str | 127.0.0.1 | Master IP address for DP ranks coordination. | | **ATOM_DP_MASTER_PORT** | int | 29500 | Master port for DP ranks coordination. | | **ATOM_DP_LB_REQ_EQUIV** | int | 512 | Token-equivalent decode pressure assigned to each in-flight request by `least_tokens` routing. | +| **ATOM_DP_DISPATCH_DEPTH** | int | 0 | Per-rank cap on in-flight load-balanced requests for an offline batch (`LLMEngine.generate` / `generate_multimodal`). `0` sprays that batch up front. `>0` holds the rest and refills a rank as it finishes, so ranks finish prefilling together and the tail stays all-decode on the CUDA graph. A prefill on any rank is what leaves the graph; an idle rank's dummy batch does not. Online API requests are dispatched immediately even when this is set. While set, offline dispatch balances on in-flight request count and ignores `--dp-load-balance` (`round_robin` and `least_tokens`); init warns when the configured strategy is not `least_requests`. A failed send puts the requests back on the pending pool instead of dropping them. `abort_request` removes a request that is still pending. Keep it `>= max_num_seqs` (about 1.2–1.5× is a starting point). A non-integer or negative value warns and disables the feature. A positive value below `max_num_seqs` warns and still caps every rank's running batch. | | **ATOM_DP_SESSION_AFFINITY** | bool | false | Load-place each new session, then keep later turns on the same prefix-cache owner. Reads `X-Dynamo-Session-ID`, then `X-Session-ID` (MLPerf Endpoints client), then `X-Correlation-ID`. | | **ATOM_DP_PREFILL_BALANCE_TOKEN_WEIGHT** | int | 360000 | DP prefill cost balancing (always on under DP attention with more than one rank; needs the prefill delayer): each lockstep prefill step aims at min(costliest, mean x (1 + slack)) of the ranks' estimated chunk costs, and a costlier rank takes fewer tokens. This is the per-token term of the chunk cost, in attended-context tokens: cost = new tokens x (weight + prior context + new / 2). | | **ATOM_DP_PREFILL_BALANCE_SLACK** | float | 0.1 | The target cost is at most mean x (1 + slack) over the ranks that prefill. | diff --git a/tests/test_dp_backpressure.py b/tests/test_dp_backpressure.py new file mode 100644 index 0000000000..8abca2f278 --- /dev/null +++ b/tests/test_dp_backpressure.py @@ -0,0 +1,311 @@ +"""Fast CPU-only test of Fix A backpressure dispatch (ATOM_DP_DISPATCH_DEPTH). + +Builds a bare CoreManager via __new__ (same pattern as tests/test_dp_load_balance.py), +stubs _send_request, and checks: (1) per-rank cap is honored, (2) the rest waits in +the pending pool, (3) completing a seq re-feeds the drained rank, (4) a fast rank +ends up processing MORE total requests than a slow one (the whole point of Fix A). +""" + +import logging +import pickle +import queue +import threading +import time +from collections import deque +from threading import Lock +from types import SimpleNamespace + +import pytest + +from atom.model_engine.engine_core_mgr import CoreManager +from atom.model_engine.engine_core_protocol import EngineCoreRequestType + + +class Seq: + _n = 0 + + def __init__(self, prompt=10): + Seq._n += 1 + self.id = f"s{Seq._n}" + self.num_prompt_tokens = prompt + self.data_parallel_rank = None + self.dp_session_id = None + self.dp_parent_session_id = None + self.stream_callback = None + + +def make_mgr(n_ranks=4, depth=3, fail_ranks=None): + mgr = CoreManager.__new__(CoreManager) + mgr.label = "test" + mgr._dp_lb_strategy = "least_requests" + mgr._dp_lb_req_equiv = 512 + mgr._dp_session_affinity_enabled = False + mgr.global_engine_count = n_ranks + mgr.local_engine_count = n_ranks + mgr.pp_size = 1 + mgr._rank_rotation_cursor = 0 + mgr._rank_reqs = [0] * n_ranks + mgr._rank_tokens = [0] * n_ranks + mgr._rank_routed_total = [0] * n_ranks + mgr._dp_route_counters = {"explicit_total": 0, "load_balanced_total": 0} + mgr._seq_load = {} + mgr._lb_lock = Lock() + mgr._input_send_lock = Lock() + mgr._bp_depth = depth + mgr._bp_pending = deque() + mgr._bp_offline_batch = True + mgr._bp_dispatch_paused = False + mgr._seq_id_to_callback = {} + mgr._flush_stream_batch_fn = None + mgr.fail_ranks = set(fail_ranks or ()) + # capture what got sent to each rank + mgr.sent = [[] for _ in range(n_ranks)] + mgr._send_lock = Lock() + + def fake_send(dp_rank, payload): + if dp_rank in mgr.fail_ranks: + raise RuntimeError(f"send failed rank {dp_rank}") + _type, seqs = pickle.loads(payload) + assert _type == EngineCoreRequestType.ADD + with mgr._send_lock: + mgr.sent[dp_rank].extend(s.id for s in seqs) + + mgr._send_request = fake_send + return mgr + + +def _accounted(mgr): + with mgr._send_lock: + sent = [sid for rank_ids in mgr.sent for sid in rank_ids] + pending = [seq.id for seq in mgr._bp_pending] + return sent, pending + + +def test_cap_and_pending(): + mgr = make_mgr(n_ranks=4, depth=3) + seqs = [Seq() for _ in range(100)] + mgr.add_request(seqs) + # 4 ranks * depth 3 = 12 dispatched, rest pending. + assert mgr._rank_reqs == [3, 3, 3, 3], mgr._rank_reqs + assert sum(len(s) for s in mgr.sent) == 12 + assert len(mgr._bp_pending) == 88 + + +def test_refill_feeds_fast_rank_more(): + mgr = make_mgr(n_ranks=4, depth=3) + seqs = [Seq() for _ in range(40)] + mgr.add_request(seqs) + assert mgr._rank_reqs == [3, 3, 3, 3] + + # Simulate rank 0 completing fast: finish 10 of its seqs one at a time, + # each completion triggering a refill (as the output thread does). + def finish_one_on_rank(rank): + # pick a charged seq currently on `rank` + sid = next(s for s, (r, _, _) in mgr._seq_load.items() if r == rank) + mgr._release_seq_load(sid) + if mgr._bp_depth > 0 and mgr._bp_pending: + mgr._bp_dispatch_or_refill(raise_on_error=False) + return sid + + for _ in range(10): + finish_one_on_rank(0) + + # Rank 0 should have been re-fed each time -> processed more than peers. + assert len(mgr.sent[0]) > len(mgr.sent[1]), (len(mgr.sent[0]), mgr.sent[1:]) + # Every rank still capped at depth in-flight (never exceeded). + assert all(r <= 3 for r in mgr._rank_reqs), mgr._rank_reqs + # Pending shrank by the 10 refills. + assert len(mgr._bp_pending) == 40 - 12 - 10, len(mgr._bp_pending) + + +def _init_with_strategy(strategy): + mgr = CoreManager.__new__(CoreManager) + mgr._init_shared_state( + SimpleNamespace(dp_load_balance=strategy, max_num_seqs=1536), + label="test", + local_engine_count=2, + ) + return mgr + + +def test_init_warns_when_backpressure_overrides_strategy(monkeypatch, caplog): + def check(strategy, depth, overridden): + monkeypatch.setenv("ATOM_DP_DISPATCH_DEPTH", str(depth)) + caplog.clear() + mgr = None + try: + with caplog.at_level(logging.WARNING, logger="atom"): + mgr = _init_with_strategy(strategy) + assert mgr._bp_depth == depth + hit = any( + f"ignores dp_load_balance={strategy}" in r.getMessage() + for r in caplog.records + ) + assert hit is overridden + finally: + if mgr is not None: + mgr.ctx.term() + + check("least_tokens", 8, True) + check("round_robin", 8, True) + check("least_requests", 8, False) + check("least_tokens", 0, False) + + +def test_drains_everything(): + mgr = make_mgr(n_ranks=2, depth=2) + seqs = [Seq() for _ in range(20)] + mgr.add_request(seqs) + # finish all in-flight repeatedly until pending + in-flight empty + guard = 0 + while mgr._seq_load: + sid = next(iter(mgr._seq_load)) + mgr._release_seq_load(sid) + if mgr._bp_pending: + mgr._bp_dispatch_or_refill(raise_on_error=False) + guard += 1 + assert guard < 1000 + assert len(mgr._bp_pending) == 0 + dispatched = [sid for rank_ids in mgr.sent for sid in rank_ids] + assert len(dispatched) == 20 # every seq dispatched exactly once + assert len(set(dispatched)) == 20 # no duplicates + + +def test_online_add_does_not_enter_pending_pool(): + mgr = make_mgr(n_ranks=2, depth=1) + mgr._bp_offline_batch = False + seqs = [Seq() for _ in range(6)] + mgr.add_request(seqs) + assert len(mgr._bp_pending) == 0 + sent, pending = _accounted(mgr) + assert pending == [] + assert sorted(sent) == sorted(seq.id for seq in seqs) + + +def test_partial_send_failure_requeues_later_ranks_and_pauses(): + # Rank 1 fails in the middle of the send loop. Rank 2 must still be + # attempted. The failed seq goes back to the head of the pool, its charge + # returns to zero, and a later refill does not dispatch the failed add. + mgr = make_mgr(n_ranks=3, depth=1, fail_ranks={1}) + seqs = [Seq() for _ in range(6)] + with pytest.raises(RuntimeError, match="send failed rank 1"): + mgr.add_request(seqs) + assert mgr._rank_reqs == [1, 0, 1] + assert next(seq.id for seq in mgr._bp_pending) == seqs[1].id + sent, pending = _accounted(mgr) + assert sent == [seqs[0].id, seqs[2].id] + assert sorted(sent + pending) == sorted(seq.id for seq in seqs) + assert len(sent + pending) == len(set(sent + pending)) + assert mgr._bp_dispatch_paused + mgr._bp_dispatch_or_refill(raise_on_error=False) + sent_after, _pending_after = _accounted(mgr) + assert sent_after == sent + + +def test_every_send_failing_returns_counters_to_zero(): + mgr = make_mgr(n_ranks=2, depth=1, fail_ranks={0, 1}) + seqs = [Seq() for _ in range(4)] + with pytest.raises(RuntimeError): + mgr.add_request(seqs) + assert mgr._rank_reqs == [0, 0] + assert not mgr._seq_load + sent, pending = _accounted(mgr) + assert sent == [] + assert sorted(pending) == sorted(seq.id for seq in seqs) + assert len(pending) == len(set(pending)) + + +def test_refill_send_failure_requeues_and_fails_a_stalled_run(): + mgr = make_mgr(n_ranks=2, depth=1) + mgr.outputs_queue = queue.Queue() + seqs = [Seq() for _ in range(4)] + mgr.add_request(seqs) + assert sum(mgr._rank_reqs) == 2 + mgr.fail_ranks.update({0, 1}) + for sid in list(mgr._seq_load): + mgr._release_seq_load(sid) + mgr._bp_dispatch_or_refill(raise_on_error=False) + assert mgr._rank_reqs == [0, 0] + sent, pending = _accounted(mgr) + # The two originally sent seqs stay sent; the two that failed the refill + # are back in the pool, once each. + assert len(sent) == 2 + assert len(pending) == 2 + assert len(set(sent + pending)) == 4 + assert mgr._bp_dispatch_paused + err = mgr.outputs_queue.get_nowait() + assert isinstance(err, RuntimeError) + + +def test_abort_pending_seq_is_not_dispatched(): + mgr = make_mgr(n_ranks=2, depth=1) + seqs = [Seq() for _ in range(4)] + delivered = {} + + def callback(output): + delivered[output.request_id] = output.finish_reason + + seqs[-1].stream_callback = callback + mgr.add_request(seqs) + target = seqs[-1] + assert any(seq.id == target.id for seq in mgr._bp_pending) + mgr.abort_request(target.id) + assert all(seq.id != target.id for seq in mgr._bp_pending) + assert delivered[target.id] == "aborted" + guard = 0 + while mgr._seq_load or mgr._bp_pending: + if mgr._seq_load: + sid = next(iter(mgr._seq_load)) + mgr._release_seq_load(sid) + if mgr._bp_pending: + mgr._bp_dispatch_or_refill(raise_on_error=False) + guard += 1 + assert guard < 1000 + sent, pending = _accounted(mgr) + assert target.id not in sent + assert pending == [] + assert len(sent) == 3 + + +def test_concurrent_add_and_refill_dispatches_once(): + mgr = make_mgr(n_ranks=4, depth=2) + seqs = [Seq() for _ in range(40)] + errors = [] + + def add_chunks(): + for i in range(0, len(seqs), 4): + mgr.add_request(seqs[i : i + 4]) + + def release_loop(): + try: + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + sid = None + with mgr._lb_lock: + if mgr._seq_load: + sid = next(iter(mgr._seq_load)) + if sid is None: + if not adder.is_alive() and not mgr._bp_pending: + return + time.sleep(0.001) + continue + mgr._release_seq_load(sid) + mgr._bp_dispatch_or_refill(raise_on_error=False) + raise AssertionError("timed out draining concurrent backpressure") + except Exception as exc: # noqa: BLE001 - surfaced to the test thread + errors.append(exc) + + adder = threading.Thread(target=add_chunks) + releaser = threading.Thread(target=release_loop) + adder.start() + releaser.start() + adder.join() + releaser.join() + assert not errors, errors + assert mgr._rank_reqs == [0, 0, 0, 0] + assert not mgr._seq_load + assert not mgr._bp_pending + sent, pending = _accounted(mgr) + assert pending == [] + assert sorted(sent) == sorted(seq.id for seq in seqs) + assert len(sent) == len(set(sent)) diff --git a/tests/test_llm_engine_add_request_fanout.py b/tests/test_llm_engine_add_request_fanout.py index 9e6ea86b3a..cbbb58f66d 100644 --- a/tests/test_llm_engine_add_request_fanout.py +++ b/tests/test_llm_engine_add_request_fanout.py @@ -14,6 +14,7 @@ still does. """ +from contextlib import nullcontext from types import SimpleNamespace import pytest @@ -117,6 +118,7 @@ def __init__(self): self._pending = True self.core_mgr = SimpleNamespace( reset_dp_router=lambda: None, + offline_backpressure_batch=lambda: nullcontext(), add_request=self._submitted.extend, get_output=self._get_output, is_alive=lambda: True,