Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions components/src/dynamo/planner/connectors/mdc.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,12 @@ def worker_info_from_mdc(
# TODO(rank-aware-kv-capacity): propagate capacity provenance into WorkerInfo. Only an exact
# or conservative scalar is safe for Planner's scale-down feasibility check; an aggregate
# mean must remain explicitly approximate rather than masquerade as a worker minimum.
runtime_data = runtime_cfg.get("runtime_data")
scheduled_limit = (
runtime_data.get("max_num_scheduled_tokens")
if isinstance(runtime_data, dict)
else None
)
return WorkerInfo(
k8s_name=k8s_name,
component_name=component_name,
Expand All @@ -163,4 +169,5 @@ def worker_info_from_mdc(
max_num_batched_tokens=runtime_cfg.get("max_num_batched_tokens"),
context_length=context_length,
speculative_nextn=_parse_speculative_nextn(runtime_cfg),
max_num_scheduled_tokens=scheduled_limit,
)
3 changes: 3 additions & 0 deletions components/src/dynamo/planner/core/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,9 @@ def _engine_caps(
kv_cache_block_size=worker_info.kv_cache_block_size if worker_info else None,
speculative_nextn=worker_info.speculative_nextn if worker_info else None,
power_watts_per_replica=power_watts_per_replica,
max_num_scheduled_tokens=(
worker_info.max_num_scheduled_tokens if worker_info else None
),
)


Expand Down
23 changes: 20 additions & 3 deletions components/src/dynamo/planner/core/perf_model/ais_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,11 @@ def _limit_values(self) -> Optional[tuple[int, int, int]]:
return None
if caps.max_kv_tokens is not None and caps.max_kv_tokens <= 0:
return None
scheduled = caps.max_num_scheduled_tokens
if scheduled is not None and (
type(scheduled) is not int or not 0 < scheduled < (1 << 32)
):
return None

# Match the historical engine-query defaults when the planner has a
# capability object but an individual runtime limit has not been
Expand All @@ -221,6 +226,11 @@ def _build_limits(self) -> Optional[Any]:
max_num_batched_tokens=max_num_batched_tokens,
max_num_seqs=max_num_seqs,
max_kv_tokens=max_kv_tokens,
max_num_scheduled_tokens=(
self._capabilities.max_num_scheduled_tokens
if self._capabilities is not None
else None
),
)

def _build_options(self) -> dict[str, int]:
Expand All @@ -245,9 +255,11 @@ def _build_ais_config(self) -> dict[str, Any]:
model=self._config.model_name
or f"dynamo-planner:{self._config.namespace}",
system="unprofiled",
backend="vllm"
if self._config.backend == "mocker"
else self._config.backend,
backend=(
"vllm"
if self._config.backend == "mocker"
else self._config.backend
),
worker_type=self._worker_type,
estimation_mode="fpm_regression",
)
Expand Down Expand Up @@ -301,6 +313,11 @@ def _model_key(self) -> Optional[tuple[Any, ...]]:
return (
self._worker_type,
values,
(
self._capabilities.max_num_scheduled_tokens
if self._capabilities is not None
else None
),
json.dumps(self._build_ais_config(), sort_keys=True, separators=(",", ":")),
)

Expand Down
56 changes: 45 additions & 11 deletions components/src/dynamo/planner/core/perf_model/engine_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ class EnginePerfLimits:
max_num_batched_tokens: int
max_num_seqs: int
max_kv_tokens: int
max_num_scheduled_tokens: Optional[int] = None

def __post_init__(self) -> None:
for name, value in (
Expand All @@ -54,6 +55,18 @@ def __post_init__(self) -> None:
raise ValueError(f"{name} must be positive, got {value}")
if value > _U32_MAX:
raise ValueError(f"{name} exceeds u32::MAX")
scheduled = self.max_num_scheduled_tokens
if scheduled is not None and (
type(scheduled) is not int or not 0 < scheduled <= _U32_MAX
):
raise ValueError("max_num_scheduled_tokens must be a positive u32")

@property
def scheduling_token_budget(self) -> int:
"""Issued-token ceiling, bounded by the model's input capacity."""
if self.max_num_scheduled_tokens is None:
return self.max_num_batched_tokens
return min(self.max_num_batched_tokens, self.max_num_scheduled_tokens)


class OptimizationTarget(str, Enum):
Expand Down Expand Up @@ -259,7 +272,7 @@ def get_queued_prefill_time(
plans = [
_PrefillChunkPlan.build(
metrics.queued_requests.sum_prefill_tokens,
self._limits.max_num_batched_tokens,
self._limits.scheduling_token_budget,
)
for metrics in metrics_by_rank
]
Expand All @@ -278,7 +291,7 @@ def get_queued_prefill_time(
repeat_count = end - start
chunk_metrics: list[ForwardPassMetrics] = []
for snapshot, plan in zip(metrics_by_rank, plans, strict=True):
chunk = plan.chunk_at(start, self._limits.max_num_batched_tokens)
chunk = plan.chunk_at(start, self._limits.scheduling_token_budget)
scheduled = ScheduledRequestMetrics(
num_prefill_requests=_estimate_prefill_request_count(
snapshot.queued_requests.num_prefill_requests, chunk
Expand Down Expand Up @@ -431,15 +444,15 @@ def _find_decode_capacity(
def _find_aggregated_capacity(
self, request: EngineCapacityRequest
) -> Optional[EngineCapacity]:
if request.osl == 0 or self._limits.max_num_batched_tokens <= 1:
if request.osl == 0 or self._limits.scheduling_token_budget <= 1:
return None

prefill_isl = _effective_prefill_isl(request)
accept_length = _normalized_accept_length(request.accept_length)
context_length = _decode_context_length(request)
hard_cap = min(
self._decode_max_batch(context_length),
max(1, self._limits.max_num_batched_tokens - 1),
max(1, self._limits.scheduling_token_budget - 1),
)
if hard_cap == 0:
return None
Expand All @@ -450,14 +463,14 @@ def _find_aggregated_capacity(
prefill_isl,
request.osl,
batch_size,
self._limits.max_num_batched_tokens,
self._limits.scheduling_token_budget,
accept_length,
):
continue

decode_kv = _saturating_mul_u32(batch_size, context_length)
prefill_per_iteration = min(
self._limits.max_num_batched_tokens,
self._limits.scheduling_token_budget,
_aggregate_prefill_per_iteration(
prefill_isl,
request.osl,
Expand Down Expand Up @@ -501,12 +514,22 @@ def _find_aggregated_capacity(
best = _select_capacity(best, capacity, request.optimization_target)
return best

def _synthetic_prefill_budget(self) -> int:
# Preserve the existing total ceiling when splitting work across ranks.
return min(
self._limits.max_num_batched_tokens,
_saturating_mul_u32(
self._limits.scheduling_token_budget, self._attention_dp_size
),
)

def _prefill_time_for_tokens(self, tokens: int) -> Optional[float]:
return self._prefill_chunk_time(
tokens,
lambda chunk: self._estimate_forward_pass_seconds(
_synthetic_prefill_by_rank(chunk, self._attention_dp_size)
),
max_chunk=self._synthetic_prefill_budget(),
)

def _decode_time_for_batch(
Expand Down Expand Up @@ -549,11 +572,15 @@ def _prefill_chunk_time(
self,
tokens: int,
estimate_chunk: Callable[[int], Optional[float]],
*,
max_chunk: Optional[int] = None,
) -> Optional[float]:
plan = _PrefillChunkPlan.build(tokens, self._limits.max_num_batched_tokens)
if max_chunk is None:
max_chunk = self._limits.scheduling_token_budget
plan = _PrefillChunkPlan.build(tokens, max_chunk)
total_s = 0.0
if plan.full_chunks > 0:
duration_s = estimate_chunk(self._limits.max_num_batched_tokens)
duration_s = estimate_chunk(max_chunk)
if duration_s is None:
return None
total_s = _checked_duration_add(
Expand All @@ -571,20 +598,27 @@ def _prefill_chunk_time(
return total_s

def _prefill_max_batch(self, isl: int) -> int:
if isl > self._limits.max_num_batched_tokens:
budget = self._synthetic_prefill_budget()
if isl > budget:
return 1
return min(
self._limits.max_num_seqs,
self._limits.max_num_batched_tokens // max(1, isl),
budget // max(1, isl),
)

def _decode_max_batch(self, context_length: int) -> int:
if context_length == 0:
return 0
return min(
max_batch = min(
self._limits.max_num_seqs,
self._limits.max_kv_tokens // context_length,
)
if self._limits.max_num_scheduled_tokens is not None:
max_batch = min(
max_batch,
self._limits.scheduling_token_budget * self._attention_dp_size,
)
return max_batch

def _validate_metrics_by_rank(
self, metrics_by_rank: list[ForwardPassMetrics]
Expand Down
3 changes: 3 additions & 0 deletions components/src/dynamo/planner/core/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -614,6 +614,9 @@ class EngineCapabilities:
# cap has not been resolved. The final budget clamp reads this.
power_watts_per_replica: Optional[int] = None

# Optional issued-token ceiling, separate from model-input capacity.
max_num_scheduled_tokens: Optional[int] = None

@property
def resolved_gpu_cost_per_replica(self) -> Optional[int]:
"""GPU budget cost, falling back to the legacy engine width."""
Expand Down
1 change: 1 addition & 0 deletions components/src/dynamo/planner/core/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"kv_cache_block_size",
"max_num_seqs",
"max_num_batched_tokens",
"max_num_scheduled_tokens",
"context_length",
"speculative_nextn",
)
Expand Down
1 change: 1 addition & 0 deletions components/src/dynamo/planner/environment/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
"kv_cache_block_size",
"max_num_seqs",
"max_num_batched_tokens",
"max_num_scheduled_tokens",
"context_length",
"speculative_nextn",
)
Expand Down
3 changes: 3 additions & 0 deletions components/src/dynamo/planner/monitoring/worker_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ class WorkerInfo:
max_num_batched_tokens: Optional[int] = None
context_length: Optional[int] = None
speculative_nextn: Optional[int] = None
max_num_scheduled_tokens: Optional[int] = None

@property
def max_kv_tokens(self) -> Optional[int]:
Expand All @@ -63,6 +64,8 @@ def summary(self) -> str:
parts.append(f"max_num_seqs(max_bs)={self.max_num_seqs}")
if self.max_num_batched_tokens is not None:
parts.append(f"max_num_batched_tokens={self.max_num_batched_tokens}")
if self.max_num_scheduled_tokens is not None:
parts.append(f"max_num_scheduled_tokens={self.max_num_scheduled_tokens}")
if self.context_length is not None:
parts.append(f"context_length={self.context_length}")
if self.speculative_nextn is not None:
Expand Down
Loading
Loading