Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 48 additions & 11 deletions components/src/dynamo/planner/core/perf_model/engine_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,20 +251,36 @@ def diagnostics(self) -> dict[str, Any]:
def get_queued_prefill_time(
self, metrics_by_rank: list[ForwardPassMetrics]
) -> Optional[float]:
"""Estimate queued-prefill drain time in seconds."""
"""Estimate queued-prefill drain time, or None when decode leaves no room."""
if self._worker_type == "decode":
return None
self._validate_metrics_by_rank(metrics_by_rank)

plans = [
_PrefillChunkPlan.build(
metrics.queued_requests.sum_prefill_tokens,
self._limits.max_num_batched_tokens,
if all(
metrics.queued_requests.sum_prefill_tokens == 0
for metrics in metrics_by_rank
):
return 0.0

chunk_limits = [
self._limits.max_num_batched_tokens
- (
metrics.scheduled_requests.num_decode_requests
if self._worker_type == "aggregated"
else 0
)
for metrics in metrics_by_rank
]
if all(plan.full_chunks == 0 and plan.tail_tokens == 0 for plan in plans):
return 0.0
plans: list[_PrefillChunkPlan] = []
for metrics, max_chunk in zip(metrics_by_rank, chunk_limits, strict=True):
queued_tokens = metrics.queued_requests.sum_prefill_tokens
if max_chunk < 0 or (max_chunk == 0 and queued_tokens > 0):
return None
plans.append(
_PrefillChunkPlan.build(queued_tokens, max_chunk)
if max_chunk > 0
else _PrefillChunkPlan(full_chunks=0, tail_tokens=0)
)

breakpoints = {0}
for plan in plans:
Expand All @@ -277,8 +293,10 @@ def get_queued_prefill_time(
for start, end in pairwise(sorted(breakpoints)):
repeat_count = end - start
chunk_metrics: list[ForwardPassMetrics] = []
for snapshot, plan in zip(metrics_by_rank, plans, strict=True):
chunk = plan.chunk_at(start, self._limits.max_num_batched_tokens)
for snapshot, plan, max_chunk in zip(
metrics_by_rank, plans, chunk_limits, strict=True
):
chunk = plan.chunk_at(start, max_chunk)
scheduled = ScheduledRequestMetrics(
num_prefill_requests=_estimate_prefill_request_count(
snapshot.queued_requests.num_prefill_requests, chunk
Expand Down Expand Up @@ -538,22 +556,41 @@ def _aggregated_ttft(
current_decode_requests: int,
current_decode_kv: int,
) -> Optional[float]:
# Synthetic work is split evenly across DP ranks. The busiest decode
# rank bounds the total prefill chunk under that same distribution.
decode_per_rank = _split_total(
current_decode_requests, self._attention_dp_size, 0
)
max_chunk = min(
self._limits.max_num_batched_tokens,
(self._limits.max_num_batched_tokens - decode_per_rank)
* self._attention_dp_size,
)
return self._prefill_chunk_time(
queued_prefill_tokens,
lambda chunk: self._mixed_time(
chunk, current_decode_requests, current_decode_kv
),
max_chunk=max_chunk,
)

def _prefill_chunk_time(
self,
tokens: int,
estimate_chunk: Callable[[int], Optional[float]],
*,
max_chunk: Optional[int] = None,
) -> Optional[float]:
plan = _PrefillChunkPlan.build(tokens, self._limits.max_num_batched_tokens)
if tokens == 0:
return 0.0
if max_chunk is None:
max_chunk = self._limits.max_num_batched_tokens
if max_chunk <= 0:
return None
plan = _PrefillChunkPlan.build(tokens, max_chunk)
total_s = 0.0
if plan.full_chunks > 0:
duration_s = estimate_chunk(self._limits.max_num_batched_tokens)
duration_s = estimate_chunk(max_chunk)
if duration_s is None:
return None
total_s = _checked_duration_add(
Expand Down
172 changes: 172 additions & 0 deletions components/src/dynamo/planner/tests/unit/test_engine_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,178 @@ def test_queued_prefill_uses_full_chunks_plus_tail():
] == [100, 50]


@pytest.mark.parametrize(
"worker_type,decode_requests,expected_chunks,expected_s",
[
("aggregated", 6, (2, 1), 0.05),
("aggregated", 0, (8, 1), 0.02),
("prefill", 6, (8, 1), 0.02),
],
)
def test_queue_drain_reserves_decode_tokens(
worker_type, decode_requests, expected_chunks, expected_s
):
model, forward_pass_model = _model(
worker_type=worker_type,
estimate=lambda _metrics: 10.0,
max_num_batched_tokens=8,
max_num_seqs=8,
)
snapshot = ForwardPassMetrics(
scheduled_requests=ScheduledRequestMetrics(
num_decode_requests=decode_requests,
sum_decode_kv_tokens=decode_requests * 10,
),
queued_requests=QueuedRequestMetrics(
num_prefill_requests=1, sum_prefill_tokens=9
),
)

estimate_s = model.get_queued_prefill_time([snapshot])

assert estimate_s == pytest.approx(expected_s)
assert (
tuple(
call[0]["scheduled_requests"]["sum_prefill_tokens"]
for call in forward_pass_model.estimate_calls
)
== expected_chunks
)
for call in forward_pass_model.estimate_calls:
scheduled = call[0]["scheduled_requests"]
assert scheduled["sum_prefill_tokens"] + scheduled["num_decode_requests"] <= 8


@pytest.mark.parametrize(
"queued_tokens,decode_requests,expected_s,expected_chunks",
[
((9, 9), (6, 2), 0.05, ((2, 6), (2, 3), (2, 0), (1, 0))),
((0, 9), (8, 2), 0.02, ((0, 6), (0, 3))),
((1, 9), (8, 2), None, ()),
((0, 9), (9, 2), None, ()),
((0, 0), (8, 8), 0.0, ()),
],
)
def test_aggregate_queue_drain_uses_each_ranks_remaining_budget(
queued_tokens, decode_requests, expected_s, expected_chunks
):
model, forward_pass_model = _model(
worker_type="aggregated",
estimate=lambda _metrics: 10.0,
max_num_batched_tokens=8,
max_num_seqs=8,
attention_dp_size=2,
)
snapshots = [
ForwardPassMetrics(
dp_rank=rank,
scheduled_requests=ScheduledRequestMetrics(
num_decode_requests=decode_requests[rank],
sum_decode_kv_tokens=decode_requests[rank] * 10,
),
queued_requests=QueuedRequestMetrics(
num_prefill_requests=int(tokens > 0), sum_prefill_tokens=tokens
),
)
for rank, tokens in enumerate(queued_tokens)
]

estimate_s = model.get_queued_prefill_time(snapshots)

if expected_s is None:
assert estimate_s is None
else:
assert estimate_s == pytest.approx(expected_s)
assert (
tuple(
tuple(rank["scheduled_requests"]["sum_prefill_tokens"] for rank in call)
for call in forward_pass_model.estimate_calls
)
== expected_chunks
)
for call in forward_pass_model.estimate_calls:
for rank in call:
scheduled = rank["scheduled_requests"]
assert (
scheduled["sum_prefill_tokens"] + scheduled["num_decode_requests"] <= 8
)


@pytest.mark.parametrize(
"ranks,decode_requests,tokens,expected_s",
[
(1, 6, 9, 0.05),
(1, 8, 9, None),
(1, 8, 0, 0.0),
(2, 6, 9, 0.02),
(2, 13, 5, 0.03),
(2, 16, 1, None),
(2, 0, 9, 0.02),
],
)
def test_aggregate_capacity_ttft_keeps_each_synthetic_rank_within_budget(
ranks, decode_requests, tokens, expected_s
):
model, forward_pass_model = _model(
worker_type="aggregated",
estimate=lambda _metrics: 10.0,
max_num_batched_tokens=8,
max_num_seqs=8,
attention_dp_size=ranks,
)

estimate_s = model._aggregated_ttft(tokens, decode_requests, decode_requests * 10)

if expected_s is None:
assert estimate_s is None
else:
assert estimate_s == pytest.approx(expected_s)
for call in forward_pass_model.estimate_calls:
for rank in call:
scheduled = rank["scheduled_requests"]
assert (
scheduled["sum_prefill_tokens"] + scheduled["num_decode_requests"] <= 8
)


def test_aggregate_queue_drain_groups_identical_rounds():
model, forward_pass_model = _model(
worker_type="aggregated",
estimate=lambda _metrics: 1.0,
max_num_batched_tokens=8,
)
snapshot = ForwardPassMetrics(
scheduled_requests=ScheduledRequestMetrics(
num_decode_requests=7, sum_decode_kv_tokens=70
),
queued_requests=QueuedRequestMetrics(
num_prefill_requests=1, sum_prefill_tokens=1_000_000
),
)

assert model.get_queued_prefill_time([snapshot]) == pytest.approx(1000.0)
assert len(forward_pass_model.estimate_calls) == 1


def test_aggregate_capacity_applies_ttft_after_reserving_decode_tokens():
model, forward_pass_model = _model(
worker_type="aggregated",
estimate=lambda _metrics: 10.0,
max_num_batched_tokens=8,
max_num_seqs=8,
)

capacity = model.find_engine_capacity_rps(
EngineCapacityRequest(isl=6, osl=32, ttft_sla_s=0.015, itl_sla_s=0.1)
)

assert capacity is not None and capacity.eligible
assert capacity.rps == pytest.approx(3.125)
for call in forward_pass_model.estimate_calls:
scheduled = call[0]["scheduled_requests"]
assert scheduled["sum_prefill_tokens"] + scheduled["num_decode_requests"] <= 8


def test_aggregated_decode_uses_learned_prefill_mix():
model, forward_pass_model = _model(
worker_type="aggregated",
Expand Down
Loading