Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 54 additions & 6 deletions atom/model_engine/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,20 @@ def __init__(
self.last_prompt_latency = 0.0
self.delay_factor = config.scheduler_delay_factor

# Reserve projected decode growth before admitting fresh prefills.
# Existing partial prefills are already committed and bypass this gate.
self.reserve_decode_tokens = envs.ATOM_RESERVE_DECODE_TOKENS
self._reserve_decode_blocks = (
self.reserve_decode_tokens + self.block_manager.block_size - 1
) // self.block_manager.block_size
if self.reserve_decode_tokens > 0:
logger.info(
"Projected-decode KV admission enabled: reserving %d tokens "
"(%d KV blocks) per running sequence",
self.reserve_decode_tokens,
self._reserve_decode_blocks,
)

# Speculative decoding
self.use_spec = config.speculative_config is not None
self.mtp_k: int = (
Expand Down Expand Up @@ -796,7 +810,11 @@ def _local_prefill_pending_work(self) -> tuple[bool, int]:
return prefillable, min(pending, budget)
probed_tokens = 0
slots = self.max_num_seqs - len(self.running)
for seq in self._waiting_prefills():
for hypothetical_new_prefills, seq in enumerate(self._waiting_prefills()):
if not self._kv_reservation_allows_new_prefill(
hypothetical_new_prefills=hypothetical_new_prefills
):
break
offload_resume = self._is_offload_prefill_resume(seq)
if offload_resume:
cached = self._offload_prefill_start(seq)
Expand Down Expand Up @@ -890,6 +908,8 @@ def _can_admit_head_prefill(self) -> bool:
return True
if not self.waiting:
return False
if not self._kv_reservation_allows_new_prefill():
return False
for i, seq in enumerate(self.waiting):
if i >= 4:
break
Expand Down Expand Up @@ -926,6 +946,23 @@ def _kv_usage(self) -> float:
return 0.0
return bm.kv.num_used / total

def _kv_reservation_allows_new_prefill(
self, *, hypothetical_new_prefills: int = 0
) -> bool:
"""Whether one more fresh prefill fits the projected decode reserve.

``hypothetical_new_prefills`` is only for non-mutating queue probes. It
counts candidates that are not in ``running`` yet. Real admissions are
appended to ``running`` immediately and must not also be passed here.
"""
if self._reserve_decode_blocks <= 0:
return True
total_blocks = self.block_manager.kv.num_blocks
if total_blocks <= 0:
return True
committed = len(self.running) + hypothetical_new_prefills
return (committed + 1) * self._reserve_decode_blocks <= total_blocks

def _record_throughput(
self, num_prompt_tokens: int = 0, num_generation_tokens: int = 0
) -> None:
Expand Down Expand Up @@ -980,6 +1017,7 @@ def _waiting_new_token_count(self) -> int:
"""
cap = self.max_num_batched_tokens
total = 0
hypothetical_new_prefills = 0
for seq in self.waiting:
if self._unschedulable_reason(seq) is not None:
continue
Expand All @@ -994,7 +1032,12 @@ def _waiting_new_token_count(self) -> int:
and num_new_tokens > self.max_num_batched_tokens
):
continue
if not self._kv_reservation_allows_new_prefill(
hypothetical_new_prefills=hypothetical_new_prefills
):
break
total += max(0, num_new_tokens)
hypothetical_new_prefills += 1
if total >= cap:
return cap
return total
Expand Down Expand Up @@ -1042,6 +1085,8 @@ def _oldest_waiting_prefill_age_ms(self) -> float:
or checkpoint waits. ABORTED, remote-loading and statically rejected
requests do not contribute.
"""
if not self._kv_reservation_allows_new_prefill():
return 0.0
oldest_arrive = None
for seq in self.waiting:
if self._unschedulable_reason(seq) is not None:
Expand Down Expand Up @@ -1564,11 +1609,13 @@ def _schedule(self) -> tuple[ScheduledBatch, dict[int, Sequence]] | None:
self._local_prefill_coalescing and not running_decode_batch
):
# Existence is sufficient during the hard protection window:
# fit probes cannot change its decision. Coalescer hold bounds
# start afterwards. With no decode, admission proceeds directly.
prefillable = (
self._partial_prefill_count > 0
or next(self._waiting_prefills(), None) is not None
# fit probes cannot change its decision. The reservation gate
# still applies to fresh work; partials are already committed.
# Coalescer hold bounds start afterwards. With no decode,
# admission proceeds directly.
prefillable = self._partial_prefill_count > 0 or (
self._kv_reservation_allows_new_prefill()
and next(self._waiting_prefills(), None) is not None
)
pending_tokens = 0
elif self._local_prefill_coalescing:
Expand Down Expand Up @@ -1637,6 +1684,7 @@ def _schedule(self) -> tuple[ScheduledBatch, dict[int, Sequence]] | None:
and self.waiting
and num_seqs_prefill < self.max_num_seqs
and num_batched_tokens < self.max_num_batched_tokens
and self._kv_reservation_allows_new_prefill()
):
if prefix_waiters:
if prefix_bypass_left == 0:
Expand Down
5 changes: 5 additions & 0 deletions atom/utils/envs.py
Original file line number Diff line number Diff line change
Expand Up @@ -815,6 +815,11 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float:
# They are configured via --dspark-config (JSON dict) and carried in
# config.dspark (see atom/config.py DSparkConfig). See
# recipes/DSpark.md.
# Project this many decode tokens per running sequence when deciding whether
# another fresh prefill may enter the running queue. Nonpositive disables it.
"ATOM_RESERVE_DECODE_TOKENS": lambda: max(
0, _int_env_or_default("ATOM_RESERVE_DECODE_TOKENS", 0)
),
# --- PrefillDelayer (cross-DP prefill alignment) ---
# Master switch; default on. Set "0" to disable construction.
# The delayer is a prefill COALESCER: it holds back prefill admission under
Expand Down
6 changes: 6 additions & 0 deletions docs/environment_variables.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@ This document describes the environment variables used in the ATOM project.
| **ATOM_DP_LB_REQ_EQUIV** | int | 512 | Token-equivalent decode pressure assigned to each in-flight request by `least_tokens` routing. |
| **ATOM_DP_SESSION_AFFINITY** | bool | false | Load-place each new session, then keep later turns on the same prefix-cache owner. Reads `X-Dynamo-Session-ID`, falling back to `X-Correlation-ID`. |

## Scheduler admission

| Variable | Type | Default | Description |
|----------|------|---------|-------------|
| **ATOM_RESERVE_DECODE_TOKENS** | int | 0 | Reserve this many projected decode tokens per running sequence when admitting a fresh prefill. The value is rounded up to whole KV blocks; admission requires room for one reservation per sequence after adding the candidate. Already-committed partial prefills may resume. Unset, invalid, zero, or negative values disable the gate and preserve normal scheduling. |

## Prefill delayer (TP/DCP and DP attention)

Coalesces waiting prefills while decode continues. DP attention enables it by
Expand Down
154 changes: 154 additions & 0 deletions tests/test_scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1085,6 +1085,160 @@ def test_offload_partial_handoff_keeps_resumed_deferred_output(self, seq_factory
assert list(seq.output_tokens) == [999, sched.eos_token_id]


class TestReserveDecodeAdmission:
@pytest.mark.parametrize("value", [None, "", "0", "-1", "bad", "1.5", "nan", "inf"])
def test_unset_invalid_or_nonpositive_values_disable_gate(
self, monkeypatch, seq_factory, value
):
if value is None:
monkeypatch.delenv("ATOM_RESERVE_DECODE_TOKENS", raising=False)
else:
monkeypatch.setenv("ATOM_RESERVE_DECODE_TOKENS", value)
sched = Scheduler(
MockConfig(
max_num_seqs=4,
max_num_batched_tokens=64,
num_kvcache_blocks=4,
kv_cache_block_size=4,
)
)
for token in range(4):
sched.add(seq_factory([token]))

batch, _ = sched.schedule()

assert sched.reserve_decode_tokens == 0
assert batch.total_seqs_num_prefill == 4

def test_new_prefills_already_in_running_are_counted_once(
self, monkeypatch, seq_factory
):
# Five reserved tokens round up to two 4-token blocks. Six blocks
# therefore allow three committed sequences.
monkeypatch.setenv("ATOM_RESERVE_DECODE_TOKENS", "5")
sched = Scheduler(
MockConfig(
max_num_seqs=8,
max_num_batched_tokens=64,
num_kvcache_blocks=6,
kv_cache_block_size=4,
)
)
sched.add(seq_factory([0]))
sched.schedule()
for token in range(1, 5):
sched.add(seq_factory([token]))

batch, _ = sched.schedule()

assert batch.total_seqs_num_prefill == 2
assert len(sched.running) == 3
assert len(sched.waiting) == 2

@pytest.mark.parametrize(
("is_local", "protects_decode"),
[(False, False), (True, False), (True, True)],
)
def test_delayer_signals_exclude_reservation_blocked_waiting(
self, monkeypatch, seq_factory, is_local, protects_decode
):
class DecisionCaptured(Exception):
pass

class RecordingDelayer:
max_queue_ms = None

def __init__(self):
self.is_local = is_local
self.decision = None

def protects_decode(self, running_decode_batch):
assert running_decode_batch == 1
return protects_decode

def should_allow_prefill(self, **decision):
self.decision = decision
raise DecisionCaptured

monkeypatch.setenv("ATOM_RESERVE_DECODE_TOKENS", "8")
sched = Scheduler(
MockConfig(
max_num_seqs=4,
max_num_batched_tokens=64,
num_kvcache_blocks=2,
kv_cache_block_size=4,
)
)
sched.add(seq_factory([0]))
sched.schedule()
sched.add(seq_factory([1]))
delayer = RecordingDelayer()
sched.set_prefill_delayer(delayer)

with pytest.raises(DecisionCaptured):
sched.schedule()

assert delayer.decision["prefillable"] is False
assert delayer.decision["pending_tokens"] == 0

def test_committed_partial_prefill_resumes_when_gate_is_full(
self, monkeypatch, seq_factory
):
class RecordingDelayer:
is_local = False
max_queue_ms = None

def __init__(self):
self.decision = None

def protects_decode(self, running_decode_batch):
assert running_decode_batch == 0
return False

def should_allow_prefill(self, **decision):
self.decision = decision
return True

monkeypatch.setenv("ATOM_RESERVE_DECODE_TOKENS", "16")
sched = Scheduler(
MockConfig(
max_num_seqs=4,
max_num_batched_tokens=8,
long_prefill_token_threshold=4,
max_model_len=32,
num_kvcache_blocks=4,
kv_cache_block_size=4,
enable_chunked_prefill=True,
)
)
partial = seq_factory(list(range(8)))
sched.add(partial)
first_batch, _ = sched.schedule()
sched.postprocess(
list(sched.running),
ScheduledBatchOutput(
req_ids=[],
token_ids=[],
num_rejected=None,
num_bonus=None,
draft_token_ids=None,
),
batch=first_batch,
)
assert partial.is_partial_prefill is True
fresh = seq_factory([20, 21, 22, 23])
sched.add(fresh)
delayer = RecordingDelayer()
sched.set_prefill_delayer(delayer)

second_batch, _ = sched.schedule()

assert delayer.decision["prefillable"] is True
assert delayer.decision["pending_tokens"] == 4
assert list(second_batch.req_ids) == [partial.id]
assert fresh in sched.waiting


# ── _waiting_new_token_count (PrefillDelayer queue signal) ─────────────────


Expand Down