Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 98 additions & 46 deletions components/src/dynamo/planner/config/planner_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -1043,6 +1043,10 @@ def _lease_fits_two_renewals(cls, value: float) -> float:
# DGDR v1beta2 ``spec.recommendation.maxCandidates`` maximum: one Request (a
# grid shard) carries at most this many router points.
DGDR_MAX_SHARD_POINTS = 100
# History buckets per throughput interval one DGDR Request carries as its
# inner planners' load-predictor warmup (Scenario 2), and each Candidate
# again as its replayable config: it bounds the objects' size.
DGDR_MAX_WARMUP_BUCKETS = 1024
# Router knobs a DGDR v1beta2 Request can search, by public name
# (``contracts.DGDR_ROUTER_KNOBS``); the Request's router section has no
# place for the retune extension knobs.
Expand Down Expand Up @@ -1086,19 +1090,31 @@ def dgdr_rounds(points: int, parallelism: int) -> int:


def dgdr_need_seconds(
points: int, dgdr: DgdrExecutorConfig, *, needs_cut: bool
points: int,
dgdr: DgdrExecutorConfig,
*,
needs_cut: bool,
needs_history: bool = False,
) -> float:
"""Wall-clock seconds one DGDR shard of ``points`` router points needs.
"""Wall-clock seconds one DGDR shard of ``points`` candidate points needs.

``cut_budget`` (when the window file must still be cut) + Run Job
start-up + ``dgdr_rounds(points, run.parallelism)`` rounds of
``cut_budget`` (when the window file must still be cut) + another
``cut_budget`` (when the history must still be read and the predictor
pre-sweep run, Scenario 2) + Run Job start-up +
``dgdr_rounds(points, run.parallelism)`` rounds of
``candidate_timeout``. ``parallelism`` is the configured value; Job
resources may lower it, so this is optimistic.
"""

rounds = dgdr_rounds(points, dgdr.run.parallelism)
cut = dgdr.cut_budget_seconds if needs_cut else 0.0
return cut + dgdr.job_startup_seconds + rounds * dgdr.run.candidate_timeout_seconds
history = dgdr.cut_budget_seconds if needs_history else 0.0
return (
cut
+ history
+ dgdr.job_startup_seconds
+ rounds * dgdr.run.candidate_timeout_seconds
)


def balanced_shard_parts(
Expand Down Expand Up @@ -2451,12 +2467,12 @@ def _validate_retune(self, retune: RetuneConfig) -> None:
planner_domain = expand_planner_domain(self)
router_domain = expand_router_domain(retune)

if self.enable_throughput_scaling and retune.history_seconds is not None:
if retune.history_seconds is not None:
required = required_history_seconds(self, planner_domain)
if retune.history_seconds < required:
raise ValueError(
f"retune.history_seconds must be >= {required:g}s "
"(throughput_adjustment_interval_seconds x "
"(the largest throughput_adjustment_interval_seconds x "
"max(prophet_window_size, kalman_min_points) over the "
"current predictor and the load_predictor candidates) to "
"warm throughput predictors"
Expand All @@ -2466,17 +2482,19 @@ def _validate_retune(self, retune: RetuneConfig) -> None:
raise ValueError(
"retune requires a non-empty router or planner search domain"
)
size = grid_size(router_domain, planner_domain.grid)
size = grid_size(router_domain) * len(planner_domain.points)
if size > retune.max_candidates:
description = describe_grid(
router_domain, planner_domain.grid, len(planner_domain.points)
)
raise ValueError(
f"retune grid has {size} candidates, exceeding "
f"retune.max_candidates={retune.max_candidates} "
f"({describe_grid(router_domain, planner_domain.grid)}); an "
"omitted AISimulate knob searches its default domain, so pin "
"knobs with a scalar to shrink the grid"
f"({description}); an omitted AISimulate knob searches its "
"default domain, so pin knobs with a scalar to shrink the grid"
)
if retune.executor == "dgdr":
self._validate_live_retune_budget(retune, router_domain)
self._validate_live_retune_budget(retune, router_domain, planner_domain)

def _validate_live_retune(self, retune: RetuneConfig) -> None:
"""Configuration premises P1, P2, P3 and P5 of live retune.
Expand Down Expand Up @@ -2523,15 +2541,12 @@ def _validate_live_retune(self, retune: RetuneConfig) -> None:
f"{DGDR_TIMEOUT_MARGIN_SECONDS:g} = {limit:g} with executor "
f"'dgdr' (P1), got {retune.timeout_seconds:g}"
)
if self.scaling_enabled():
raise ValueError(
"retune.executor 'dgdr' supports retune-only mode (P2): set "
"enable_load_scaling and enable_throughput_scaling both to false"
)
if retune.planner is not None:
if retune.router is None:
raise ValueError(
"retune.executor 'dgdr' retunes the router only (P2); remove "
"retune.planner"
"retune.executor 'dgdr' requires retune.router (P2): every DGDR "
"Request sets the three router knobs; to retune the Planner "
"alone, pin overlap_score_credit, prefill_load_scale and "
"temperature to the frontend's startup values with scalars"
)
self._validate_live_retune_router(retune)
_validate_dgdr_engine(dgdr.engine, self.mode, self.backend)
Expand Down Expand Up @@ -2601,41 +2616,75 @@ def _validate_live_retune_router(retune: RetuneConfig) -> None:
f"executor 'dgdr' (P3), got {written}"
)

@staticmethod
def _validate_live_retune_budget(
retune: RetuneConfig, router_domain: Mapping[str, Sequence[float]]
self,
retune: RetuneConfig,
router_domain: Mapping[str, Sequence[float]],
planner_domain: Any,
) -> None:
"""P13: the largest grid shard can finish within the sweep deadline.

The baseline can fall in any grid shard, and a window whose baseline
shard cannot finish fails with ``launch_too_late``, so the largest
shard must fit. The domain is the widest one: a zero startup credit
only narrows it at runtime.
only narrows it at runtime, and folding inactive fields to the
baseline's values only merges planner points. With scaling, the
planner points are one more shard dimension, the history read and
the predictor pre-sweep take another ``cut_budget_seconds``, and the
warmup a Request carries is bounded by ``DGDR_MAX_WARMUP_BUCKETS``
buckets per throughput interval.
"""

dgdr = retune.dgdr
if dgdr is None or retune.timeout_seconds is None:
# _validate_live_retune rejects both first.
raise ValueError("retune.executor 'dgdr' requires dgdr and timeout_seconds")
scaling = self.scaling_enabled()
if scaling:
# Imported on use: the module imports this one at module scope.
from dynamo.planner.retune.domains import (
required_history_seconds,
throughput_intervals,
)

intervals = throughput_intervals(self, planner_domain.points)
history_s = retune.history_seconds
if history_s is None:
history_s = required_history_seconds(self, planner_domain)
if intervals and history_s / min(intervals) > DGDR_MAX_WARMUP_BUCKETS:
raise ValueError(
f"retune history of {history_s:g}s at the shortest throughput "
f"interval {min(intervals)}s is "
f"{math.ceil(history_s / min(intervals))} warmup buckets, more "
f"than the {DGDR_MAX_WARMUP_BUCKETS} a DGDR Request carries "
"(P13); lower retune.history_seconds or the predictor window"
)
sizes = [len(values) for values in router_domain.values()]
if scaling:
sizes.append(len(planner_domain.points))
parts = balanced_shard_parts(
sizes, dgdr.run.max_shards, parallelism=dgdr.run.parallelism
)
points = largest_shard_points(sizes, parts)
if points > DGDR_MAX_SHARD_POINTS:
raise ValueError(
f"the retune router grid of {math.prod(sizes)} points cannot be "
f"the retune grid of {math.prod(sizes)} points cannot be "
f"split into at most retune.dgdr.run.max_shards="
f"{dgdr.run.max_shards} Cartesian shards of at most "
f"{DGDR_MAX_SHARD_POINTS} points (P13); the largest would have "
f"{points}. Raise max_shards or pin router knobs with a scalar"
)
need = dgdr_need_seconds(points, dgdr, needs_cut=True)
need = dgdr_need_seconds(points, dgdr, needs_cut=True, needs_history=scaling)
if need > retune.timeout_seconds:
rounds = dgdr_rounds(points, dgdr.run.parallelism)
history = (
f"cut_budget_seconds {dgdr.cut_budget_seconds:g} for the history + "
if scaling
else ""
)
raise ValueError(
f"the largest retune grid shard ({points} points) needs "
f"{need:g}s: cut_budget_seconds {dgdr.cut_budget_seconds:g} + "
f"{need:g}s: {history}cut_budget_seconds {dgdr.cut_budget_seconds:g} + "
f"job_startup_seconds {dgdr.job_startup_seconds:g} + {rounds} "
f"rounds x candidate_timeout_seconds "
f"{dgdr.run.candidate_timeout_seconds:g}, more than "
Expand All @@ -2648,36 +2697,33 @@ def _validate_retune_planner_grid(
self,
grid: Mapping[str, Sequence[Any]],
public_names: Mapping[str, str],
*,
load_scaling: Optional[bool] = None,
throughput_scaling: Optional[bool] = None,
) -> dict[str, list[Any]]:
"""Reject planner grid fields that are inert in this configuration.

``grid`` is keyed by ``PlannerConfig`` field; ``public_names`` gives
the ``retune.planner`` name to report. Each value must pass the
field's strict validation, as a runtime reconfiguration would.
Returns the grid with validated values (``80`` becomes ``80.0`` for a
float field), so candidates compare equal to the live config by key.
the ``retune.planner`` name to report. ``load_scaling`` and
``throughput_scaling`` say whether some grid point scales that way
(default: this config does). Each value must pass the field's strict
validation, as a runtime reconfiguration would. Returns the grid with
validated values (``80`` becomes ``80.0`` for a float field), so
candidates compare equal to the live config by key.
"""

active_endpoints = (
{"min_endpoint"}
if self.mode == "agg"
else {"prefill_min_endpoint", "decode_min_endpoint"}
)
if load_scaling is None:
load_scaling = self.enable_load_scaling
if throughput_scaling is None:
throughput_scaling = self.enable_throughput_scaling
validated: dict[str, list[Any]] = {}
for key, values in grid.items():
name = f"retune.planner.{public_names.get(key, key)}"
if key in ("min_endpoint", "prefill_min_endpoint", "decode_min_endpoint"):
relevant = key in active_endpoints
requirement = (
"mode='agg' (use prefill_min_workers/decode_min_workers in disagg)"
if key == "min_endpoint"
else "mode='disagg' (use min_workers in agg)"
)
elif key == "max_throughput_scaling_replicas":
relevant = self.enable_throughput_scaling
if key == "max_throughput_scaling_replicas":
relevant = throughput_scaling
requirement = "enable_throughput_scaling"
elif key == "load_scaling_down_sensitivity":
relevant = self.enable_load_scaling
relevant = load_scaling
requirement = "enable_load_scaling"
elif key in (
"prefill_scale_up_queue_tokens",
Expand All @@ -2689,7 +2735,8 @@ def _validate_retune_planner_grid(
relevant = self.optimization_target == "load"
requirement = "optimization_target='load'"
else:
# min_gpu_budget / max_gpu_budget apply to any scaling mode.
# The scaling-policy fields: expand_planner_domain keeps only
# the policies that apply.
relevant = True
requirement = ""
if not relevant:
Expand Down Expand Up @@ -2736,6 +2783,11 @@ def from_config_arg(cls, config_arg: str) -> "PlannerConfig":

return cls.model_validate(data)

@property
def fpm_max_age_seconds(self) -> float:
"""Freshness window of live FPM samples: three load intervals."""
return 3.0 * self.load_adjustment_interval_seconds

@classmethod
def _load_from_file(cls, path: Path) -> "PlannerConfig":
suffix = path.suffix.lower()
Expand Down
Loading
Loading