diff --git a/components/src/dynamo/planner/config/planner_config.py b/components/src/dynamo/planner/config/planner_config.py index 43620aeddcc7..a709634432f1 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/components/src/dynamo/planner/config/planner_config.py @@ -1043,6 +1043,10 @@ def _lease_fits_two_renewals(cls, value: float) -> float: # DGDR v1beta2 ``spec.recommendation.maxCandidates`` maximum: one Request (a # grid shard) carries at most this many router points. DGDR_MAX_SHARD_POINTS = 100 +# History buckets per throughput interval one DGDR Request carries as its +# inner planners' load-predictor warmup (Scenario 2), and each Candidate +# again as its replayable config: it bounds the objects' size. +DGDR_MAX_WARMUP_BUCKETS = 1024 # Router knobs a DGDR v1beta2 Request can search, by public name # (``contracts.DGDR_ROUTER_KNOBS``); the Request's router section has no # place for the retune extension knobs. @@ -1086,19 +1090,31 @@ def dgdr_rounds(points: int, parallelism: int) -> int: def dgdr_need_seconds( - points: int, dgdr: DgdrExecutorConfig, *, needs_cut: bool + points: int, + dgdr: DgdrExecutorConfig, + *, + needs_cut: bool, + needs_history: bool = False, ) -> float: - """Wall-clock seconds one DGDR shard of ``points`` router points needs. + """Wall-clock seconds one DGDR shard of ``points`` candidate points needs. - ``cut_budget`` (when the window file must still be cut) + Run Job - start-up + ``dgdr_rounds(points, run.parallelism)`` rounds of + ``cut_budget`` (when the window file must still be cut) + another + ``cut_budget`` (when the history must still be read and the predictor + pre-sweep run, Scenario 2) + Run Job start-up + + ``dgdr_rounds(points, run.parallelism)`` rounds of ``candidate_timeout``. ``parallelism`` is the configured value; Job resources may lower it, so this is optimistic. """ rounds = dgdr_rounds(points, dgdr.run.parallelism) cut = dgdr.cut_budget_seconds if needs_cut else 0.0 - return cut + dgdr.job_startup_seconds + rounds * dgdr.run.candidate_timeout_seconds + history = dgdr.cut_budget_seconds if needs_history else 0.0 + return ( + cut + + history + + dgdr.job_startup_seconds + + rounds * dgdr.run.candidate_timeout_seconds + ) def balanced_shard_parts( @@ -2493,7 +2509,7 @@ def _validate_retune(self, retune: RetuneConfig) -> None: "default domain, so pin knobs with a scalar to shrink the grid" ) if retune.executor == "dgdr": - self._validate_live_retune_budget(retune, router_domain) + self._validate_live_retune_budget(retune, router_domain, planner_domain) def _validate_live_retune(self, retune: RetuneConfig) -> None: """Configuration premises P1, P2, P3 and P5 of live retune. @@ -2540,15 +2556,12 @@ def _validate_live_retune(self, retune: RetuneConfig) -> None: f"{DGDR_TIMEOUT_MARGIN_SECONDS:g} = {limit:g} with executor " f"'dgdr' (P1), got {retune.timeout_seconds:g}" ) - if self.scaling_enabled(): - raise ValueError( - "retune.executor 'dgdr' supports retune-only mode (P2): set " - "enable_load_scaling and enable_throughput_scaling both to false" - ) - if retune.planner is not None: + if retune.router is None: raise ValueError( - "retune.executor 'dgdr' retunes the router only (P2); remove " - "retune.planner" + "retune.executor 'dgdr' requires retune.router (P2): every DGDR " + "Request sets the three router knobs; to retune the Planner " + "alone, pin overlap_score_credit, prefill_load_scale and " + "temperature to the frontend's startup values with scalars" ) self._validate_live_retune_router(retune) _validate_dgdr_engine(dgdr.engine, self.mode, self.backend) @@ -2618,41 +2631,75 @@ def _validate_live_retune_router(retune: RetuneConfig) -> None: f"executor 'dgdr' (P3), got {written}" ) - @staticmethod def _validate_live_retune_budget( - retune: RetuneConfig, router_domain: Mapping[str, Sequence[float]] + self, + retune: RetuneConfig, + router_domain: Mapping[str, Sequence[float]], + planner_domain: Any, ) -> None: """P13: the largest grid shard can finish within the sweep deadline. The baseline can fall in any grid shard, and a window whose baseline shard cannot finish fails with ``launch_too_late``, so the largest shard must fit. The domain is the widest one: a zero startup credit - only narrows it at runtime. + only narrows it at runtime, and folding inactive fields to the + baseline's values only merges planner points. With scaling, the + planner points are one more shard dimension, the history read and + the predictor pre-sweep take another ``cut_budget_seconds``, and the + warmup a Request carries is bounded by ``DGDR_MAX_WARMUP_BUCKETS`` + buckets per throughput interval. """ dgdr = retune.dgdr if dgdr is None or retune.timeout_seconds is None: # _validate_live_retune rejects both first. raise ValueError("retune.executor 'dgdr' requires dgdr and timeout_seconds") + scaling = self.scaling_enabled() + if scaling: + # Imported on use: the module imports this one at module scope. + from dynamo.planner.retune.domains import ( + required_history_seconds, + throughput_intervals, + ) + + intervals = throughput_intervals(self, planner_domain.points) + history_s = retune.history_seconds + if history_s is None: + history_s = required_history_seconds(self, planner_domain) + if intervals and history_s / min(intervals) > DGDR_MAX_WARMUP_BUCKETS: + raise ValueError( + f"retune history of {history_s:g}s at the shortest throughput " + f"interval {min(intervals)}s is " + f"{math.ceil(history_s / min(intervals))} warmup buckets, more " + f"than the {DGDR_MAX_WARMUP_BUCKETS} a DGDR Request carries " + "(P13); lower retune.history_seconds or the predictor window" + ) sizes = [len(values) for values in router_domain.values()] + if scaling: + sizes.append(len(planner_domain.points)) parts = balanced_shard_parts( sizes, dgdr.run.max_shards, parallelism=dgdr.run.parallelism ) points = largest_shard_points(sizes, parts) if points > DGDR_MAX_SHARD_POINTS: raise ValueError( - f"the retune router grid of {math.prod(sizes)} points cannot be " + f"the retune grid of {math.prod(sizes)} points cannot be " f"split into at most retune.dgdr.run.max_shards=" f"{dgdr.run.max_shards} Cartesian shards of at most " f"{DGDR_MAX_SHARD_POINTS} points (P13); the largest would have " f"{points}. Raise max_shards or pin router knobs with a scalar" ) - need = dgdr_need_seconds(points, dgdr, needs_cut=True) + need = dgdr_need_seconds(points, dgdr, needs_cut=True, needs_history=scaling) if need > retune.timeout_seconds: rounds = dgdr_rounds(points, dgdr.run.parallelism) + history = ( + f"cut_budget_seconds {dgdr.cut_budget_seconds:g} for the history + " + if scaling + else "" + ) raise ValueError( f"the largest retune grid shard ({points} points) needs " - f"{need:g}s: cut_budget_seconds {dgdr.cut_budget_seconds:g} + " + f"{need:g}s: {history}cut_budget_seconds {dgdr.cut_budget_seconds:g} + " f"job_startup_seconds {dgdr.job_startup_seconds:g} + {rounds} " f"rounds x candidate_timeout_seconds " f"{dgdr.run.candidate_timeout_seconds:g}, more than " diff --git a/components/src/dynamo/planner/core/base.py b/components/src/dynamo/planner/core/base.py index ef69de072611..fc1d67aa22ec 100644 --- a/components/src/dynamo/planner/core/base.py +++ b/components/src/dynamo/planner/core/base.py @@ -15,7 +15,7 @@ import importlib import logging import time -from collections.abc import AsyncIterator, Iterable +from collections.abc import AsyncIterator, Iterable, Sequence from contextlib import asynccontextmanager from typing import TYPE_CHECKING, Any, Optional, Protocol @@ -39,6 +39,7 @@ EngineCapabilities, FpmObservations, PlannerEffects, + RetuneDecision, ScheduledTick, TickDiagnostics, TickInput, @@ -115,7 +116,10 @@ async def start(self, host: NativePlannerBase) -> None: async def stop(self) -> None: ... - def on_effects(self, effects: PlannerEffects) -> None: + def on_effects(self, effects: PlannerEffects) -> Sequence[RetuneDecision]: + """Take the tick's decisions; return the ones it queued to persist. + The Planner fields of each apply once it is persisted + (``retune_decision_persisted``).""" ... @@ -263,6 +267,8 @@ def __init__( self._legacy_retune_task: Optional[asyncio.Task[None]] = None self._legacy_retune_cleanup: Optional[RetuneCleanupHandle] = None self._retune_lease_found = asyncio.Event() + # Background applications of live retune Planner changes. + self._retune_apply_tasks: set[asyncio.Task[None]] = set() async def _async_init(self) -> None: # Shutdown is safe for a partially initialized environment and is @@ -442,6 +448,11 @@ async def _shutdown_runtime(self) -> None: """Release initialized planner resources after normal or failed startup.""" await self.reconfiguration.close() + applying = list(self._retune_apply_tasks) + for task in applying: + task.cancel() + if applying: + await asyncio.gather(*applying, return_exceptions=True) if not self._diagnostics_finalized: self._diagnostics_finalized = True @@ -625,6 +636,46 @@ def _sync_fpm_max_age(self) -> None: setter(age) logger.info("FPM freshness window is now %.1fs", age) + def retune_decision_persisted(self, decision: RetuneDecision) -> None: + """Apply a persisted live retune decision's Planner fields + (``LiveRetuneHost``). + + In the background, through ``PlannerReconfiguration.apply_decision``, + the control API's two-phase path. The supervisor calls this only once + the decision is written under this process's term. + """ + + if decision.outcome != "applied" or not decision.planner_changes: + return + task = asyncio.create_task( + self._apply_retune_planner_changes(decision), + name=f"planner-retune-apply-{decision.decision_id}", + ) + self._retune_apply_tasks.add(task) + task.add_done_callback(self._retune_apply_tasks.discard) + + async def _apply_retune_planner_changes(self, decision: RetuneDecision) -> None: + changes = dict(decision.planner_changes) + try: + await self.reconfiguration.apply_decision( + changes, decision.predictor_warmup + ) + logger.info( + "Retune %s applied planner changes %s", + decision.decision_id, + changes, + ) + except Exception: + logger.exception( + "Retune %s could not apply planner changes %s; the Planner keeps " + "its configuration", + decision.decision_id, + changes, + ) + controller = getattr(self._engine, "retune_controller", None) + if controller is not None: + controller.discard_planner_changes(changes) + @asynccontextmanager async def _bounded_config_lock(self) -> AsyncIterator[None]: """Acquire the tick decision lock or ask the caller to retry.""" @@ -1297,8 +1348,9 @@ async def _run_one_tick( effects = await engine.tick(tick, tick_input) if self._live_retune is not None: # Retune decisions go to the live supervisor's outbox, which - # persists them before actuating routers; _apply_effects never - # sees router parameters. + # persists them before actuating routers and before calling back + # retune_decision_persisted for the Planner fields; _apply_effects + # never sees router parameters. try: self._live_retune.on_effects(effects) except Exception: diff --git a/components/src/dynamo/planner/reconfiguration.py b/components/src/dynamo/planner/reconfiguration.py index 547525ae99ac..6c86b22a8eb4 100644 --- a/components/src/dynamo/planner/reconfiguration.py +++ b/components/src/dynamo/planner/reconfiguration.py @@ -16,7 +16,7 @@ import uuid import weakref from collections import OrderedDict -from collections.abc import Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from pathlib import Path from typing import TYPE_CHECKING, Any @@ -412,7 +412,8 @@ def __init__(self, planner: NativePlannerBase) -> None: self.planner = planner self.instance_id = str(uuid.uuid4()) self._operations: OrderedDict[str, dict[str, Any]] = OrderedDict() - self._task: asyncio.Task[None] | None = None + # The update preparing or activating: a control API one or a decision's. + self._task: asyncio.Task[Any] | None = None self._preparation: asyncio.Task[BuiltinLoadPredict] | None = None self._closed = False self._registered_runtime: DistributedRuntime | None = None @@ -508,27 +509,107 @@ async def _apply(self, request, candidate, changed_predictor, operation) -> None operation["state"] = ( "warming" if request.warmup_trace is not None else "preparing" ) - capabilities = self.planner._build_worker_capabilities() - self._preparation = asyncio.create_task( - asyncio.to_thread( - _prepare_predictor, candidate, capabilities, request.warmup_trace - ) - ) - # A timed-out thread cannot be killed. Keep tracking it and reject a - # second preparation until it exits; it never owns active state. - self._preparation.add_done_callback(_consume_preparation_result) - predictor = await asyncio.wait_for( - asyncio.shield(self._preparation), PREPARATION_TIMEOUT_SECONDS + predictor = await self._prepare( + _prepare_predictor, + candidate, + self.planner._build_worker_capabilities(), + request.warmup_trace, ) + operation["applied_revision"] = await self._activate( + candidate, + request.config.keys(), + predictor, + lambda: self._check_revision(request), + ) + operation["state"] = "applied" + + async def apply_decision( + self, + changes: Mapping[str, Any], + observations: Sequence[TrafficObservation] | None, + ) -> int: + """Activate a persisted live retune decision's Planner ``changes``; + the new revision. + + The same two-phase path as a control API update: one update at a + time, a new load predictor prepared off the event loop on the + decision's ``observations`` and tracked past its deadline, and the + decision dropped (``ConfigurationConflict``) when another update + activates first. Raises on any failure; nothing changes then. + """ + + async with self.planner._bounded_config_lock(): + if self._closed: + raise _MinimumEndpointUnavailableError("planner is shutting down") + if self._task is not None and not self._task.done(): + raise ConfigurationConflict( + "another configuration update is still preparing" + ) + candidate = candidate_config(self.planner.config, dict(changes)) + self.planner.validate_runtime_candidate(candidate) + rebuild = predictor_rebuild_required(self.planner.config, candidate) + if ( + rebuild + and self._preparation is not None + and not self._preparation.done() + ): + raise ConfigurationConflict("another predictor is still preparing") + revision = self.planner._config_generation + + def unchanged() -> None: + if self.planner._config_generation != revision: + raise ConfigurationConflict( + "the configuration changed while the predictor was " + f"prepared (revision {revision} -> " + f"{self.planner._config_generation})" + ) + + async def apply() -> int: + predictor = None + if rebuild: + predictor = await self._prepare( + prepare_predictor, + candidate, + self.planner._build_worker_capabilities(), + observations, + ) + return await self._activate( + candidate, list(changes), predictor, unchanged + ) + + task = asyncio.create_task(apply()) + self._task = task + return await task + + async def _prepare( + self, prepare: Callable[..., BuiltinLoadPredict], *args: Any + ) -> BuiltinLoadPredict: + """Run ``prepare(*args)`` off the event loop within the deadline.""" + + self._preparation = asyncio.create_task(asyncio.to_thread(prepare, *args)) + # A timed-out thread cannot be killed. Keep tracking it and reject a + # second preparation until it exits; it never owns active state. + self._preparation.add_done_callback(_consume_preparation_result) + return await asyncio.wait_for( + asyncio.shield(self._preparation), PREPARATION_TIMEOUT_SECONDS + ) + + async def _activate( + self, + candidate: PlannerConfig, + fields: Iterable[str], + predictor: BuiltinLoadPredict | None, + check: Callable[[], None], + ) -> int: + """Activate ``candidate`` between decisions if ``check()`` passes; + the new revision.""" + async with self.planner._effect_admission_lock: async with self.planner._bounded_config_lock(): - self._check_revision(request) + check() self.planner.validate_runtime_candidate(candidate) - self.planner.apply_runtime_candidate( - candidate, request.config.keys(), predictor - ) - operation["state"] = "applied" - operation["applied_revision"] = self.planner._config_generation + self.planner.apply_runtime_candidate(candidate, fields, predictor) + return self.planner._config_generation @staticmethod def _finish(task: asyncio.Task[None], operation: dict[str, Any]) -> None: diff --git a/components/src/dynamo/planner/retune/controller.py b/components/src/dynamo/planner/retune/controller.py index 9b85e52f3d68..0814a967407e 100644 --- a/components/src/dynamo/planner/retune/controller.py +++ b/components/src/dynamo/planner/retune/controller.py @@ -414,6 +414,17 @@ def rewind(self, last_decided_k: int) -> None: if self._pending is None: self._last_k = min(self._last_k, int(last_decided_k)) + def discard_planner_changes(self, changes: Mapping[str, Any]) -> None: + """Forget ``changes`` of an ``applied`` decision the host could not + apply (a live Planner's predictor warmup failed, say): the current + point is the running configuration again.""" + + for key, value in changes.items(): + if key in self._pending_planner and _same_value( + self._pending_planner[key], value + ): + del self._pending_planner[key] + def restore_desired(self, flat: Mapping[str, float]) -> None: """Re-install persisted desired router values (live re-election). @@ -602,6 +613,16 @@ def _replicas_at(self, time_s: float) -> dict[str, int] | None: return None return dict(chosen) + def _covering_counts(self, time_s: float) -> dict[str, int] | None: + """The ready counts recorded last at or before ``time_s``.""" + + covering: dict[str, int] | None = None + for at_s, counts in self._history: + if at_s > time_s + _EPSILON: + break + covering = counts + return covering + def _due_window(self, now_s: float) -> int: """Latest window ``k`` whose launch time ``L_k`` is at or before now.""" @@ -692,16 +713,25 @@ def _replicas_for_launch( """``(replicas, verified, mismatch)`` for a window launch. Without ``spec_replicas`` (replay), the ready counts at window start, - verified. With them (live), every ready count recorded over the - window, including the one in effect at its start, must equal the - spec; a mismatch is reported. The spec replicas are used either way, - unverified when no recorded count covers the window start (the - window began before this controller started). + verified. With them (live), a planner that scales starts each + candidate's inner planner from the ready counts recorded at the + window start, as replay does, and the spec replicas stand in, + unverified, when no recorded count covers it. With fixed replicas, + every ready count recorded over the window, including the one in + effect at its start, must equal the spec; a mismatch is reported. + The spec replicas are used either way, unverified when no recorded + count covers the window start (the window began before this + controller started). """ if spec_replicas is None: return self._replicas_at(start_s), True, None spec = {key: int(value) for key, value in spec_replicas.items()} + if self._config.scaling_enabled(): + at_start = self._covering_counts(start_s) + if at_start is None or any(value < 1 for value in at_start.values()): + return spec, False, None + return dict(at_start), True, None covering: dict[str, int] | None = None observed: list[dict[str, int]] = [] for at_s, counts in self._history: @@ -831,8 +861,25 @@ def _live_request_fields( for key, values in self._router_domain_values.items() }, "deadline_s": as_of_s + self._retune.effective_timeout_seconds, - "spec_replicas": None if spec_replicas is None else dict(spec_replicas), + # A scaling planner's sweep starts from SweepRequest.replicas. + "spec_replicas": ( + None + if spec_replicas is None or self._config.scaling_enabled() + else dict(spec_replicas) + ), "replicas_verified": verified, + # The planner axis a DGDR sweep shards: the grid's planner + # points, without the baseline's or the startup point's. A grid + # that does not search the planner has the one point {}: the + # running planner, with the pre-sweep winner when the predictor + # is searched. + "grid_planners": tuple( + dict(planner) + for planner in { + RetuneParams(planner=point.planner).key(): point.planner + for point in sorted(self._grid, key=RetuneParams.key) + }.values() + ), } def _submit( diff --git a/components/src/dynamo/planner/retune/domains.py b/components/src/dynamo/planner/retune/domains.py index 2b7d8c258fe6..80ddc4349c18 100644 --- a/components/src/dynamo/planner/retune/domains.py +++ b/components/src/dynamo/planner/retune/domains.py @@ -884,3 +884,16 @@ def required_history_seconds(config: PlannerConfig, domain: PlannerDomain) -> fl ) min_points.append(fields.get("kalman_min_points", config.kalman_min_points)) return float(max(intervals) * max(*window_sizes, *min_points)) + + +def retune_history_seconds(config: PlannerConfig) -> float: + """History each sweep reads before its window: ``retune.history_seconds``, + else ``required_history_seconds`` (0 without throughput scaling, or with + fixed replicas, whose sweeps run no inner planner).""" + + retune = config.retune + if retune is None or not config.scaling_enabled(): + return 0.0 + if retune.history_seconds is not None: + return float(retune.history_seconds) + return required_history_seconds(config, expand_planner_domain(config)) diff --git a/components/src/dynamo/planner/retune/live/bootstrap.py b/components/src/dynamo/planner/retune/live/bootstrap.py index 274a73123f18..9d6305529885 100644 --- a/components/src/dynamo/planner/retune/live/bootstrap.py +++ b/components/src/dynamo/planner/retune/live/bootstrap.py @@ -47,7 +47,7 @@ from dynamo.planner.core.types import RetuneInputs from dynamo.planner.errors import ComponentError from dynamo.planner.monitoring.dgd_services import get_component_from_type_or_name -from dynamo.planner.retune.domains import expand_router_domain +from dynamo.planner.retune.domains import expand_router_domain, retune_history_seconds from dynamo.planner.retune.live import frontend_config, window_cut from dynamo.planner.retune.live.actuator import ( AiohttpRouterPolicyClient, @@ -75,6 +75,7 @@ SupervisorSettings, worker_types_for, ) +from dynamo.planner.simulation.config import retune_sweep_base logger = logging.getLogger(__name__) @@ -294,11 +295,14 @@ def canary(supervisor: LiveRetuneSupervisor) -> bool: if not replicas or not point: return False sla_ttft_ms, sla_itl_ms = retune.scoring_sla(config.ttft_ms, config.itl_ms) + # A scaling Planner's Requests carry the planner section too. + scaling = config.scaling_enabled() shard = Shard( id=CANARY_SHARD_ID, kind="point", choices={key: (value,) for key, value in point.items()}, points=(dict(point),), + planners=({},) if scaling else (), ) body = build_request( dgd=supervisor.dgd, @@ -313,6 +317,10 @@ def canary(supervisor: LiveRetuneSupervisor) -> bool: spec_replicas=replicas, backend=str(config.backend), attempt=0, + planner_base=( + retune_sweep_base(config.model_dump(mode="json")) if scaling else None + ), + planner_warmup={}, ) try: supervisor.kube.create_request(body, dry_run=True, timeout_s=timeout_s) @@ -426,7 +434,13 @@ def evaluate(obj: dict) -> SpecEvaluation: evaluation = evaluate(dgd_obj) if reconciler is None: reconciler = Reconciler( - kube, evaluation.dgd, dgdr, retune, dgdr.mount_path, metrics + kube, + evaluation.dgd, + dgdr, + retune, + dgdr.mount_path, + metrics, + history_s=retune_history_seconds(config), ) worker_types = worker_types_for(config.mode) settings = SupervisorSettings.from_config( diff --git a/components/src/dynamo/planner/retune/live/dgdr.py b/components/src/dynamo/planner/retune/live/dgdr.py index 24468412dcf9..ac272ea2666c 100644 --- a/components/src/dynamo/planner/retune/live/dgdr.py +++ b/components/src/dynamo/planner/retune/live/dgdr.py @@ -42,6 +42,7 @@ from __future__ import annotations import copy +import dataclasses import functools import hashlib import itertools @@ -50,14 +51,17 @@ import math import os import re +import tempfile import threading import time from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass, field +from pathlib import Path from typing import Any, Literal, NoReturn, Optional from dynamo.planner.config.planner_config import ( DGDR_MAX_SHARD_POINTS, + DGDR_MAX_WARMUP_BUCKETS, DgdrExecutorConfig, RetuneConfig, balanced_shard_parts, @@ -88,12 +92,24 @@ WindowCutter, ) from dynamo.planner.retune.live.reconciler import trace_pvc_source +from dynamo.planner.retune.live.window_cut import bucket_history from dynamo.planner.retune.sweep import ( + SweepInterrupted, SweepRequest, SweepResult, SweepStatus, + WorkerPool, + evaluated_params, + evaluated_planner, + predictor_swap_errors, + predictor_swap_tasks, + run_presweeps, + scales_on_throughput, score_summary, + sweep_throughput_intervals, + write_observations, ) +from dynamo.planner.simulation.config import inner_planner_config, retune_sweep_base logger = logging.getLogger(__name__) @@ -111,6 +127,11 @@ REQUEST_GENERATION_ANNOTATION = "nvidia.com/dgdr-request-generation" TRACE_MOUNT_PATH = "/var/run/aisimulate/trace" ROUTER_ADAPTER = "dynamo.router" +PLANNER_ADAPTER = "dynamo.planner" +# The planner adapter's live retune sweep section +# (``simulation.provider.RETUNE_SWEEP``). A Candidate's planner parameters +# are the section narrowed to its one point. +RETUNE_SWEEP = "retune_sweep" _ROLES = ("aggregated", "prefill", "decode") _TEMPERATURE = DGDR_ROUTER_KNOBS["temperature"] # spec.objective.metric -> AISimulate optimization.target (operator lowering). @@ -142,20 +163,43 @@ class Shard: ``choices`` maps each live router knob to the values the shard spans; ``points`` is their Cartesian product (live names). A ``point`` or - ``probe`` shard has one value per knob. + ``probe`` shard has one value per knob. ``planners`` are the planner + points (``PlannerConfig`` fields) the shard crosses with them when its + candidates replay an inner planner (Scenario 2; ``({},)`` when no + planner field is searched), and ``()`` with fixed replicas. """ id: str kind: ShardKind choices: dict[str, tuple[float, ...]] points: tuple[dict[str, float], ...] + planners: tuple[dict[str, Any], ...] = () @property def size(self) -> int: - return len(self.points) + return len(self.points) * max(1, len(self.planners)) + + def contains( + self, point: Mapping[str, float], planner: Optional[Mapping[str, Any]] = None + ) -> bool: + if not all(point.get(key) in values for key, values in self.choices.items()): + return False + return ( + planner is None + or not self.planners + or any(_same_planner(planner, option) for option in self.planners) + ) + + @property + def searches_planner(self) -> bool: + """Whether the shard's candidates replay an inner planner.""" + return bool(self.planners) - def contains(self, point: Mapping[str, float]) -> bool: - return all(point.get(key) in values for key, values in self.choices.items()) + +def _same_planner(left: Mapping[str, Any], right: Mapping[str, Any]) -> bool: + return set(left) == set(right) and all( + _same_json(left[key], right[key]) for key in left + ) def plan_shards( @@ -166,6 +210,9 @@ def plan_shards( max_shards: int, parallelism: int, probe: bool = True, + planner_points: Sequence[Mapping[str, Any]] = (), + baseline_planner: Optional[Mapping[str, Any]] = None, + startup_planner: Optional[Mapping[str, Any]] = None, ) -> list[Shard]: """Split ``router_domain`` (live knob -> values) into Request shards. @@ -182,6 +229,13 @@ def plan_shards( middle value of each knob, a positive temperature when the grid has one) to check common random numbers across Runs. Raises ``ValueError`` when no split fits or a point does not cover the domain's knobs. + + ``planner_points`` (distinct planner points, in order) add one more + dimension, cut into contiguous chunks like a router knob, so a shard + crosses its router sub-grid with a chunk of planner points. The + baseline and startup points then also carry their planner parts + (``baseline_planner``, ``startup_planner``), and the probe takes the + middle planner point. """ if not router_domain: @@ -204,7 +258,10 @@ def plan_shards( f"router domain {list(keys)}" ) + planners = tuple(dict(point) for point in planner_points) sizes = [len(axes[key]) for key in keys] + if planners: + sizes.append(len(planners)) parts = balanced_shard_parts(sizes, max_shards, parallelism=parallelism) largest = largest_shard_points(sizes, parts) if largest > DGDR_MAX_SHARD_POINTS: @@ -216,7 +273,11 @@ def plan_shards( ) shards: list[Shard] = [] - chunked = [_chunks(axes[key], part) for key, part in zip(keys, parts)] + chunked: list[list[Any]] = [ + _chunks(axes[key], part) for key, part in zip(keys, parts) + ] + if planners: + chunked.append(_chunks(planners, parts[-1])) for index, combo in enumerate(itertools.product(*chunked)): choices = dict(zip(keys, combo)) shards.append( @@ -225,41 +286,56 @@ def plan_shards( kind="grid", choices=choices, points=_product(choices), + planners=tuple(combo[len(keys)]) if planners else (), ) ) - def in_grid(point: Mapping[str, float]) -> bool: - return all(float(point[key]) in axes[key] for key in keys) + def in_grid(point: Mapping[str, float], planner: Mapping[str, Any]) -> bool: + return all(float(point[key]) in axes[key] for key in keys) and ( + not planners or any(_same_planner(planner, p) for p in planners) + ) - def single(shard_id: str, kind: ShardKind, point: Mapping[str, float]) -> Shard: + def single( + shard_id: str, + kind: ShardKind, + point: Mapping[str, float], + planner: Mapping[str, Any], + ) -> Shard: values = {key: float(point[key]) for key in keys} return Shard( id=shard_id, kind=kind, choices={key: (value,) for key, value in values.items()}, points=(values,), + planners=(dict(planner),) if planners else (), ) baseline_point = {key: float(baseline[key]) for key in keys} startup_point = {key: float(startup[key]) for key in keys} - if not in_grid(baseline_point): - shards.append(single("point-baseline", "point", baseline_point)) - if not in_grid(startup_point) and startup_point != baseline_point: - shards.append(single("point-startup", "point", startup_point)) + base_planner = dict(baseline_planner or {}) + start_planner = dict(startup_planner or {}) + if not in_grid(baseline_point, base_planner): + shards.append(single("point-baseline", "point", baseline_point, base_planner)) + if not in_grid(startup_point, start_planner) and ( + startup_point != baseline_point + or not _same_planner(start_planner, base_planner) + ): + shards.append(single("point-startup", "point", startup_point, start_planner)) if probe: - shards.append(single("probe", "probe", _probe_point(axes))) + middle = planners[len(planners) // 2] if planners else {} + shards.append(single("probe", "probe", _probe_point(axes), middle)) return shards -def _chunks(values: tuple[float, ...], count: int) -> list[tuple[float, ...]]: +def _chunks(values: Sequence[Any], count: int) -> list[tuple[Any, ...]]: """``values`` cut into ``count`` contiguous chunks whose sizes differ by <= 1.""" base, extra = divmod(len(values), count) - chunks: list[tuple[float, ...]] = [] + chunks: list[tuple[Any, ...]] = [] start = 0 for index in range(count): end = start + base + (1 if index < extra else 0) - chunks.append(values[start:end]) + chunks.append(tuple(values[start:end])) start = end return chunks @@ -365,6 +441,8 @@ def build_request( spec_replicas: Mapping[str, int], backend: str, attempt: int, + planner_base: Optional[Mapping[str, Any]] = None, + planner_warmup: Optional[Mapping[str, Any]] = None, ) -> dict: """The DGDR v1beta2 Request that replays one shard of a window. @@ -379,6 +457,13 @@ def build_request( is the one the configuration check (P5) accepted. Raises ``ValueError`` for ``spec_replicas`` or a shard the Request cannot express. + + A shard with planner points (Scenario 2) adds the planner adapter's + ``retune_sweep`` section: ``planner_base`` (the live config's + ``retune_sweep_base``), the shard's points, and ``planner_warmup`` + (history rows by throughput interval). ``spec_replicas`` are then the + ready counts at the window start, where each candidate's inner planner + starts scaling. """ engine = _pinned_engine(cfg.engine, spec_replicas, backend) @@ -428,6 +513,19 @@ def build_request( "candidate_timeout_seconds": cfg.run.candidate_timeout_seconds, }, "optimization": {"strict_sla": False}, + **( + { + "planner": { + RETUNE_SWEEP: { + "base": copy.deepcopy(dict(planner_base or {})), + "points": [dict(point) for point in shard.planners], + "warmup": copy.deepcopy(dict(planner_warmup or {})), + } + } + } + if shard.searches_planner + else {} + ), # The planner reads only each Candidate's parameters and # score, never its DGD. The direct generator keeps retune # Runs independent of the AIC generator, whose rendering can @@ -696,6 +794,10 @@ class DgdrContext: now: Callable[[], float] = time.time sleep: Optional[Callable[[float], None]] = None poll_interval_s: float = 5.0 + # Scenario 2: the history reader and the pre-sweep's worker pool + # (window_cut.bucket_history and sweep.WorkerPool; tests replace them). + bucket_history: Callable[..., Any] = field(default=bucket_history) + worker_pool: Callable[..., Any] = field(default=WorkerPool) def run_dgdr_sweep( @@ -883,6 +985,17 @@ def __init__( } self.k8s_errors = 0 self.runtime: dict[str, Any] = {} + # Scenario 2 (set by _prepare_planner): what each candidate replays + # as, the planner points and warmup the Requests carry, and the + # pre-sweep and swap inputs the decision reads. + self.evaluated: dict[str, RetuneParams] = {} + self.planner_points: list[dict[str, Any]] = [] + self.planner_base: Optional[dict[str, Any]] = None + self.planner_warmup: Optional[dict[str, Any]] = None + self.presweep: Optional[dict[str, Any]] = None + self.interval_observations: dict[int, tuple[Any, ...]] = {} + self.warmup_errors: dict[int, str] = {} + self.base_interval: Optional[int] = None # ------------------------------------------------------------ driver @@ -948,16 +1061,28 @@ def _result( details["shards"] = {key: run.summary() for key, run in self.shards.items()} details["k8s_errors"] = self.k8s_errors details.update(self.runtime) + base_interval = self.base_interval return SweepResult( status=status, num_requests=self.num_requests, row_counts=dict(self.row_counts), scores=scores or {}, metrics=metrics or {}, + predictor_presweep=self.presweep, + warmup_observations=( + None + if base_interval is None + else self.interval_observations.get(base_interval) + ), + predictor_warmup_error=( + None if base_interval is None else self.warmup_errors.get(base_interval) + ), wall_s=max(0.0, self._now() - self.started), error=error, failure_reason=failure_reason, details=details, + interval_observations=dict(self.interval_observations), + predictor_warmup_errors=dict(self.warmup_errors), ) # ----------------------------------------------------------- helpers @@ -1040,24 +1165,39 @@ def _prepare(self) -> None: domain = request.router_domain if not domain: self._fail("invalid_request", "the sweep request has no router domain") - if any(candidate.planner for candidate in request.candidates): + if request.base_planner is None and any( + candidate.planner for candidate in request.candidates + ): self._fail( "invalid_request", - "a DGDR sweep searches router parameters only (Scenario 1)", + "planner candidates need the inner planner's base configuration", ) keys = set(domain) - startup = self._startup_point(domain) + if request.base_planner is not None: + self._prepare_planner() + startup = self._startup_candidate(domain) + baseline = self._evaluated(request.baseline) + planner_points = self.planner_points try: shards = plan_shards( domain, - baseline=request.baseline.router, - startup=startup, + baseline=baseline.router, + startup=startup.router, max_shards=self.cfg.run.max_shards, parallelism=self.cfg.run.parallelism, + planner_points=planner_points, + baseline_planner=baseline.planner, + startup_planner=startup.planner, ) except (KeyError, ValueError) as error: self._fail("invalid_domain", str(error)) - replicas = request.spec_replicas or request.replicas + # A scaling planner starts each candidate from the ready counts at + # the window start (SweepRequest.replicas); a fixed one from the spec. + replicas = ( + request.replicas + if request.base_planner is not None + else request.spec_replicas or request.replicas + ) self.details["replicas"] = dict(replicas) probe = next((shard for shard in shards if shard.kind == "probe"), None) for shard in shards: @@ -1075,6 +1215,8 @@ def _prepare(self) -> None: spec_replicas=replicas, backend=self.ctx.backend, attempt=1, + planner_base=self.planner_base, + planner_warmup=self.planner_warmup, ) except (KeyError, TypeError, ValueError) as error: self._fail("invalid_engine", f"shard {shard.id}: {error}") @@ -1082,8 +1224,13 @@ def _prepare(self) -> None: # the probe is compared with. critical = ( shard.kind == "probe" - or shard.contains(request.baseline.router) - or (probe is not None and shard.contains(probe.points[0])) + or shard.contains(baseline.router, baseline.planner) + or ( + probe is not None + and shard.contains( + probe.points[0], probe.planners[0] if probe.planners else None + ) + ) ) self.shards[shard.id] = _ShardRun( shard=shard, @@ -1092,32 +1239,199 @@ def _prepare(self) -> None: digest=body["metadata"]["labels"][LABEL_SPEC], ) uncovered = [ - candidate.router + candidate.to_dict() for candidate in request.candidates if set(candidate.router) != keys - or not any( - run.shard.kind != "probe" and run.shard.contains(candidate.router) - for run in self.shards.values() - ) + or not self._covered(self._evaluated(candidate)) ] if uncovered: self.details["uncovered_candidates"] = uncovered[:10] - def _startup_point(self, domain: Mapping[str, Sequence[float]]) -> dict[str, float]: + def _covered(self, point: RetuneParams) -> bool: + return any( + run.shard.kind != "probe" + and run.shard.contains(point.router, point.planner) + for run in self.shards.values() + ) + + def _evaluated(self, candidate: RetuneParams) -> RetuneParams: + """The point a candidate replays as (``sweep.evaluated_params``).""" + + return self.evaluated.get(candidate.key(), candidate) + + def _startup_candidate(self, domain: Mapping[str, Sequence[float]]) -> RetuneParams: """The off-grid candidate other than the baseline, else the baseline. The controller's candidates are the grid plus the startup point and - the baseline, so an off-grid candidate is one of those two. + the baseline, so an off-grid candidate is one of those two. A + candidate is off the grid when its router point is, or when its + evaluated planner part is not on the planner axis. """ - baseline = self.request.baseline + baseline = self._evaluated(self.request.baseline) + axis = self.planner_points for candidate in self.request.candidates: router = candidate.router - if candidate == baseline or set(router) != set(domain): + if candidate == self.request.baseline or set(router) != set(domain): continue - if not all(float(router[key]) in domain[key] for key in domain): - return dict(router) - return dict(baseline.router) + evaluated = self._evaluated(candidate) + if not all(float(router[key]) in domain[key] for key in domain) or ( + axis and not any(_same_planner(evaluated.planner, p) for p in axis) + ): + return evaluated + return baseline + + def _prepare_planner(self) -> None: + """Bucket the history, run the predictor pre-sweep and fix the + planner points each Request carries (Scenario 2). + + Within ``cut_budget_seconds`` of the deadline, before any Request is + planned, because the points and the warmup are part of every + Request's spec: the history before the window (and the window + itself, for the pre-sweep) is read from the trace segments at each + throughput interval a candidate scales at; the pre-sweep picks a + predictor per interval; each candidate is mapped to the point it + replays as (``evaluated_params``); and every predictor a decision + could swap in is prepared on the same observations, as replay + retune does, so the decision can check it. + """ + + request = self.request + budget = min(self.cfg.cut_budget_seconds, self.deadline - self._now()) + if budget <= 0: + self._fail("launch_too_late", "no time left to read the history", "timeout") + deadline_s = self._now() + budget + intervals = sweep_throughput_intervals(request) + observations: dict[int, tuple[Any, ...]] = {} + warmups: dict[int, tuple[Any, ...]] = {} + if intervals: + history = self._bucket_history(intervals, deadline_s) + observations = dict(history.observations) + warmups = dict(history.warmups) + self.details["history"] = { + "start_ms": int(round(request.history_start_ms)), + "requests": history.history_requests, + "dropped_members": history.dropped_members, + "intervals": list(intervals), + } + presweep = None + warmup_errors: dict[int, str] = {} + # The configuration the Runs replay (only its scaling fields): the + # live dump may hold settings, such as non-advisory batch + # scheduling, that need what a dump leaves out. + inner_base = inner_planner_config(retune_sweep_base(request.base_planner or {})) + with tempfile.TemporaryDirectory(prefix="retune-dgdr-planner-") as work: + work_dir = Path(work) + paths = { + interval: write_observations( + work_dir / f"observations-{interval}.json", rows + ) + for interval, rows in observations.items() + } + pool = self.ctx.worker_pool( + self.ctx.retune.max_parallel, + time.monotonic() + (deadline_s - self._now()), + self.stop, + work, + ) + try: + if request.predictor_candidates and intervals: + presweep = run_presweeps( + request, pool, observations, paths, intervals + ) + for candidate in request.candidates: + self.evaluated[candidate.key()] = evaluated_params( + candidate, request.baseline, presweep, request.base_planner + ) + tasks = predictor_swap_tasks( + request, + list(self.evaluated.values()), + inner_base, + observations, + paths, + work_dir, + ) + if tasks: + warmup_errors = predictor_swap_errors(pool.run(tasks)) + except SweepInterrupted as interrupted: + self._check_interrupt() + self._fail( + "window_cut_timeout", + f"history and predictor pre-sweep: {interrupted}", + "timeout", + ) + finally: + pool.close() + # The planner axis: the grid's planner points as evaluated (what + # P13 counts). The baseline, evaluated as is, and the startup point + # are off the grid when their planner part is not among them, and + # plan_shards gives each a single-point shard. + if request.grid_planners is None: + sources = [point.planner for point in self.evaluated.values()] + else: + sources = [ + evaluated_planner( + planner, request.baseline, presweep, request.base_planner + ) + for planner in request.grid_planners + ] + planners: dict[str, dict[str, Any]] = {} + for planner in sources: + planners.setdefault(RetuneParams(planner=planner).key(), planner) + self.planner_points = [planners[key] for key in sorted(planners)] + self.planner_base = retune_sweep_base(request.base_planner or {}) + self.planner_warmup = { + str(interval): [list(dataclasses.astuple(row)) for row in rows] + for interval, rows in warmups.items() + } + oversized = { + interval: len(rows) + for interval, rows in warmups.items() + if len(rows) > DGDR_MAX_WARMUP_BUCKETS + } + if oversized: + self._fail( + "invalid_request", + f"the history warmup has {oversized} buckets by interval, more " + f"than {DGDR_MAX_WARMUP_BUCKETS} per interval a Request carries", + ) + self.presweep = presweep + self.interval_observations = observations + self.warmup_errors = warmup_errors + running = {**(request.base_planner or {}), **request.baseline.planner} + self.base_interval = scales_on_throughput({}, running) + self.details["planner"] = { + "points": len(self.planner_points), + "presweep": presweep, + "warmup_errors": {str(k): v for k, v in warmup_errors.items()}, + } + + def _bucket_history(self, intervals: Sequence[int], deadline_s: float) -> Any: + request = self.request + try: + return self.ctx.bucket_history( + self._segments_root(), + int(round(request.history_start_ms)), + self.window_start_ms, + self.window_end_ms, + self.as_of_ms, + intervals, + deadline_s=deadline_s, + cancel=self.stop, + ) + except Exception as error: + self._check_interrupt() + reason = getattr(error, "reason", None) + if reason not in ("window_cut_timeout", "trace_corrupt"): + reason = "window_cut_failed" + self._fail(reason, f"history read: {type(error).__name__}: {error}") + + def _segments_root(self) -> str: + if self.request.trace_paths: + return self.request.trace_paths[0] + return os.path.join( + self.cfg.mount_path, self.cfg.segments_dir, self.ctx.dgd.name + ) def _build(self, run: _ShardRun) -> dict: body = build_request( @@ -1133,6 +1447,8 @@ def _build(self, run: _ShardRun) -> dict: spec_replicas=self.details["replicas"], backend=self.ctx.backend, attempt=run.attempt, + planner_base=self.planner_base, + planner_warmup=self.planner_warmup, ) rows = {"num_requests": self.num_requests, "row_counts": self.row_counts} body["metadata"]["annotations"] = { @@ -1342,13 +1658,7 @@ def _cut(self) -> None: """Cut the window file; an empty window ends the sweep as ``empty``.""" self._check_interrupt() - segments_root = ( - self.request.trace_paths[0] - if self.request.trace_paths - else os.path.join( - self.cfg.mount_path, self.cfg.segments_dir, self.ctx.dgd.name - ) - ) + segments_root = self._segments_root() budget = min(self.cfg.cut_budget_seconds, self.deadline - self._now()) if budget <= 0: self._fail("launch_too_late", "no time left to cut the window", "timeout") @@ -1712,7 +2022,7 @@ def _harvest(self) -> SweepResult: name = (candidate.get("metadata") or {}).get("name", "") parameters = _get(candidate, ("status", "parameters")) try: - point, signature = _map_parameters(parameters, run.shard) + point, planner, signature = _map_parameters(parameters, run.shard) except _Unmapped as error: if error.fatal: self._fail("engine_not_pinned", f"Candidate {name}: {error}") @@ -1726,13 +2036,14 @@ def _harvest(self) -> SweepResult: f"Candidates {signature_source} and {name} differ in " "non-router parameters", ) - key = RetuneParams(router=point).key() + params = RetuneParams(router=point, planner=planner) + key = params.key() target = probe_results if run.shard.kind == "probe" else results if key in target: continue metrics = _get(candidate, ("status", "metrics"), {}) or {} entry: dict[str, Any] = { - "params": RetuneParams(router=point).to_dict(), + "params": params.to_dict(), "shard": run.shard.id, "candidate": name, } @@ -1755,15 +2066,16 @@ def _harvest(self) -> SweepResult: missing = 0 for candidate in request.candidates: key = candidate.key() - found = results.get(key) + evaluated = self._evaluated(candidate) + found = results.get(evaluated.key()) if found is not None and set(candidate.router) == domain_keys: scores[key] = found.score metrics_out[key] = found.entry else: scores[key] = None metrics_out[key] = { - "params": candidate.to_dict(), - "error": self._missing_reason(candidate.router), + "params": evaluated.to_dict(), + "error": self._missing_reason(evaluated), } if scores[key] is None: missing += 1 @@ -1780,9 +2092,11 @@ def _harvest(self) -> SweepResult: def _list_candidates(self, selector: str) -> list[dict]: return self.ctx.kube.list_candidates(selector, timeout_s=self._timeout()) - def _missing_reason(self, point: Mapping[str, float]) -> str: + def _missing_reason(self, point: RetuneParams) -> str: for run in self.shards.values(): - if run.shard.kind != "probe" and run.shard.contains(point): + if run.shard.kind != "probe" and run.shard.contains( + point.router, point.planner + ): if run.state == "missing": return f"shard {run.shard.id} {run.reason}: {run.message}" return f"not published by Run {run.run_name} (infeasible or failed)" @@ -1795,7 +2109,8 @@ def _check_probe( if probe is None: return point = probe.shard.points[0] - key = RetuneParams(router=point).key() + planner = probe.shard.planners[0] if probe.shard.planners else {} + key = RetuneParams(router=point, planner=planner).key() grid = results.get(key) repeat = probe_results.get(key) grid_score = None if grid is None else grid.score @@ -1831,13 +2146,17 @@ def __init__(self, message: str, *, fatal: bool = False) -> None: self.fatal = fatal -def _map_parameters(parameters: Any, shard: Shard) -> tuple[dict[str, float], str]: - """A Candidate's live router point and its non-router signature. +def _map_parameters( + parameters: Any, shard: Shard +) -> tuple[dict[str, float], dict[str, Any], str]: + """A Candidate's live router point, planner point and other signature. The signature is the canonical JSON of the parameters without the three - searched knobs; every Candidate of a window must share it. Raises - ``_Unmapped`` for parameters that name no point of ``shard``, fatally - for a router other than the pinned ``kv_router`` without a load model. + searched router knobs and the planner point; every Candidate of a + window must share it. The planner point is ``{}`` for a shard without + planner points. Raises ``_Unmapped`` for parameters that name no point + of ``shard``, fatally for a router other than the pinned ``kv_router`` + without a load model. """ if not isinstance(parameters, Mapping): @@ -1869,15 +2188,42 @@ def _map_parameters(parameters: Any, shard: Shard) -> tuple[dict[str, float], st f"router knob {public}={value!r} is not one of the shard's values" ) point[live] = snapped - rest = dict(parameters) - rest["adapters"] = { + planner: dict[str, Any] = {} + rest_adapters = { **adapters, ROUTER_ADAPTER: { key: value for key, value in router.items() if key not in DGDR_ROUTER_KNOBS }, } + if shard.searches_planner: + section = adapters.get(PLANNER_ADAPTER) + if not isinstance(section, Mapping): + raise _Unmapped( + f"status.parameters.adapters[{PLANNER_ADAPTER!r}] is missing" + ) + sweep = section.get(RETUNE_SWEEP) + points = sweep.get("points") if isinstance(sweep, Mapping) else None + chosen = points[0] if isinstance(points, list) and len(points) == 1 else None + if not isinstance(chosen, Mapping): + raise _Unmapped( + f"status.parameters.adapters[{PLANNER_ADAPTER!r}].{RETUNE_SWEEP} " + "does not name one planner point" + ) + match = next( + (option for option in shard.planners if _same_planner(chosen, option)), + None, + ) + if match is None: + raise _Unmapped(f"planner point {dict(chosen)} is not one of the shard's") + planner = dict(match) + # The base and warmup are the Request's; only the point varies. + rest_adapters[PLANNER_ADAPTER] = { + key: value for key, value in section.items() if key != RETUNE_SWEEP + } + rest = dict(parameters) + rest["adapters"] = rest_adapters signature = json.dumps(rest, sort_keys=True, separators=(",", ":"), default=str) - return point, signature + return point, planner, signature def _snap(value: float, values: Sequence[float]) -> Optional[float]: diff --git a/components/src/dynamo/planner/retune/live/reconciler.py b/components/src/dynamo/planner/retune/live/reconciler.py index a3ce986b1b10..7b0f49149726 100644 --- a/components/src/dynamo/planner/retune/live/reconciler.py +++ b/components/src/dynamo/planner/retune/live/reconciler.py @@ -177,6 +177,7 @@ def __init__( fs_root: str, metrics: RetuneMetricsSink, now: Callable[[], float] = time.time, + history_s: float = 0.0, ) -> None: self._kube = kube self._cfg = cfg @@ -185,8 +186,11 @@ def __init__( self._now = now interval_s = float(retune.interval_seconds) sweep_timeout_s = float(retune.effective_timeout_seconds) + # A scaling Planner's sweep also reads history_s before its window + # (the inner planners' predictor warmup). self._segment_max_age_s = ( - interval_s + float(history_s) + + interval_s + float(retune.grace_seconds) + sweep_timeout_s + VISIBILITY_MARGIN_S diff --git a/components/src/dynamo/planner/retune/live/supervisor.py b/components/src/dynamo/planner/retune/live/supervisor.py index 2fb9089fa78b..e1b33cac23a5 100644 --- a/components/src/dynamo/planner/retune/live/supervisor.py +++ b/components/src/dynamo/planner/retune/live/supervisor.py @@ -43,7 +43,9 @@ leader of the round's epoch or the budget has passed. - outbox: write each decision, conditional on the state's ``leader_term`` being this process's term and on the resourceVersion read; N consecutive - failures re-initialize. + failures re-initialize. Once an applied decision is written, its Planner + fields go to ``host.retune_decision_persisted``; a decision dropped + before that has its Planner fields discarded from the controller. Local leadership loss, a failed outbox, and a controller rebuild share one path: clear ``initialized``, stop the outbox writer and the in-flight @@ -221,6 +223,11 @@ async def replace_live_retune( ) -> None: ... + def retune_decision_persisted(self, decision: RetuneDecision) -> None: + """Apply a persisted ``applied`` decision's Planner fields (in the + background; must not block).""" + ... + @dataclass(frozen=True) class SupervisorSettings: @@ -922,20 +929,23 @@ def sweep_fn( failure_reason="sweep_error", ) - def on_effects(self, effects: PlannerEffects) -> None: - """Enqueue the tick's decisions; never raises into the tick.""" + def on_effects(self, effects: PlannerEffects) -> list[RetuneDecision]: + """Enqueue the tick's decisions and return them, or ``[]`` when they + are dropped; never raises into the tick. Their Planner fields apply + once written (``outbox_round``).""" decisions = list(effects.retune_decisions or ()) if not decisions: - return + return [] if not (self.is_leader() and self._initialized): logger.warning( "retune: dropping %d decision(s) %s: not an initialized leader", len(decisions), [decision.decision_id for decision in decisions], ) + self._discard_planner_changes(decisions) self._rewind_now() - return + return [] router_parameters = effects.router_parameters for decision in decisions: self._outbox.append( @@ -953,6 +963,7 @@ def on_effects(self, effects: PlannerEffects) -> None: ) ) self._outbox_wake.set() + return decisions # ------------------------------------------------------------ lifecycle @@ -1083,12 +1094,17 @@ async def outbox_round(self) -> None: "retune outbox: dropped %s of an earlier term/generation", entry.decision.decision_id, ) + self._discard_planner_changes([entry.decision]) continue outcome = await self._write_decision(entry) if outcome in ("ok", "already", "discarded"): if self._outbox and self._outbox[0] is entry: self._outbox.popleft() self._write_failures = 0 + if outcome == "discarded": + self._discard_planner_changes([entry.decision]) + else: + self._decision_persisted(entry.decision) continue if outcome == "lost_term": self._request_reinit("the state's leader_term is not ours") @@ -1305,6 +1321,7 @@ def current() -> bool: return False self._initialized = True self._rollback_since = None + self._warn_planner_reset(plan.state) if self._background: self._outbox_task = asyncio.create_task( self._outbox_loop(), name="planner-retune-outbox" @@ -1312,6 +1329,40 @@ def current() -> bool: self._outbox_wake.set() return True + def _warn_planner_reset(self, state: RetuneState) -> None: + """Report applied Planner changes a restart did not restore. + + Router parameters live in the frontends and survive a Planner + restart; Planner fields live in this process, which starts from its + configuration. Retune does not restore them (restoring a predictor + needs a warmup): the next window's baseline is the running + configuration, and a later sweep may choose the change again. + """ + + config = getattr(self._host, "config", None) + if config is None: + return + last = next( + ( + item + for item in reversed(state.history) + if item.get("outcome") == "applied" and item.get("planner") + ), + None, + ) + if last is None: + return + planner = last["planner"] + running = {key: getattr(config, key, None) for key in planner} + if running != planner: + logger.warning( + "retune: Planner fields %s of decision %s are not restored after " + "the restart; the Planner runs %s", + planner, + last.get("decision_id"), + running, + ) + async def _read_own_snapshot(self) -> Optional[StateSnapshot]: """Read the state; drop a snapshot of another DGD uid. @@ -1385,6 +1436,7 @@ async def _local_loss_locked(self, reason: str) -> None: await self._cancel(self._reconcile_task) self._reconcile_task = None dropped = len(self._outbox) + unwritten = [entry.decision for entry in self._outbox] self._outbox.clear() self._write_failures = 0 controller = self._controller() @@ -1394,6 +1446,7 @@ async def _local_loss_locked(self, reason: str) -> None: controller.drop_pending() except Exception: logger.exception("retune: drop_pending failed") + self._discard_planner_changes(unwritten) self._rewind(controller) if was: logger.info( @@ -1562,6 +1615,10 @@ def _history_entry(self, entry: OutboxEntry) -> dict[str, Any]: item["by_wt"] = entry.router_parameters elif decision.outcome == "advisory": item["by_wt"] = self._resolve_by_wt(flat) + if decision.planner_changes and decision.outcome in ("applied", "advisory"): + # The Planner fields the decision changes (or recommends). A + # restarted Planner does not restore them (_warn_planner_reset). + item["planner"] = dict(decision.planner_changes) details = decision.details image = { key: details[key] @@ -2071,6 +2128,42 @@ def _rewind_now(self) -> None: if controller is not None: self._rewind(controller) + def _decision_persisted(self, decision: RetuneDecision) -> None: + """Hand a written ``applied`` decision's Planner fields to the host.""" + + if decision.outcome != "applied" or not decision.planner_changes: + return + host = self._host + try: + if host is None: + raise RuntimeError("no planner host") + host.retune_decision_persisted(decision) + except Exception: + logger.exception( + "retune: applying the Planner fields of %s failed", + decision.decision_id, + ) + self._discard_planner_changes([decision]) + + def _discard_planner_changes(self, decisions: Sequence[RetuneDecision]) -> None: + """Forget the Planner fields of ``applied`` decisions that will not be + applied, so the controller's current point is the running + configuration again.""" + + controller = self._controller() + if controller is None: + return + for decision in decisions: + if decision.outcome != "applied" or not decision.planner_changes: + continue + try: + controller.discard_planner_changes(decision.planner_changes) + except Exception: + logger.exception( + "retune: discarding the Planner fields of %s failed", + decision.decision_id, + ) + def _engine(self) -> Any: """The host's engine adapter; ``None`` before ``start`` or without one.""" diff --git a/components/src/dynamo/planner/retune/live/window_cut.py b/components/src/dynamo/planner/retune/live/window_cut.py index a44006201487..ef089f73504d 100644 --- a/components/src/dynamo/planner/retune/live/window_cut.py +++ b/components/src/dynamo/planner/retune/live/window_cut.py @@ -53,8 +53,10 @@ import numpy as np +from dynamo.planner.core.types import TrafficObservation from dynamo.planner.retune.live.contracts import SupportsIsSet, WindowCutResult from dynamo.planner.retune.window import ( + ObservationBuckets, _dumps, _new_counts, _parse_request_end, @@ -287,6 +289,166 @@ def cut_window_inprocess( ) from error +@dataclass(frozen=True) +class TraceHistory: + """Traffic before and in a window, bucketed per throughput interval. + + Keys are throughput intervals in seconds. ``observations`` are the + buckets of history and window that end at the window end, for the load + predictor pre-sweep and a predictor swap; ``warmups`` the buckets of + history alone that end at the window start, the warmup of a candidate's + inner planner. ``history_requests`` counts the history rows. + """ + + observations: dict[int, tuple[TrafficObservation, ...]] + warmups: dict[int, tuple[TrafficObservation, ...]] + history_requests: int + dropped_members: int + + +def bucket_history( + segments_root: StrPath, + history_start_ms: float, + window_start_ms: float, + window_end_ms: float, + as_of_ms: float, + intervals_s: Sequence[int], + *, + deadline_s: float, + cancel: Optional[SupportsIsSet] = None, +) -> TraceHistory: + """``bucket_history_inprocess`` in a spawned child process. + + The child is killed at ``deadline_s`` (absolute wall-clock seconds) or + on ``cancel``, as ``cut_window``'s is; failures raise ``WindowCutError``. + """ + + if not math.isfinite(deadline_s): + raise ValueError("deadline_s must be finite") + deadline = time.monotonic() + (deadline_s - time.time()) + started = time.monotonic() + result: TraceHistory = _run_in_child( + bucket_history_inprocess, + ( + os.fspath(segments_root), + history_start_ms, + window_start_ms, + window_end_ms, + as_of_ms, + tuple(int(interval) for interval in intervals_s), + ), + {}, + deadline=deadline, + cancel=cancel, + name=f"retune-history-{int(round(window_end_ms))}", + ) + logger.info( + "retune history [%s, %s) of window ending %s: %d requests bucketed at " + "%s s in %.1fs; dropped %d gzip members", + history_start_ms, + window_start_ms, + window_end_ms, + result.history_requests, + sorted(result.observations), + time.monotonic() - started, + result.dropped_members, + ) + return result + + +def bucket_history_inprocess( + segments_root: StrPath, + history_start_ms: float, + window_start_ms: float, + window_end_ms: float, + as_of_ms: float, + intervals_s: Sequence[int], +) -> TraceHistory: + """Bucket the requests of ``[history_start_ms, window_end_ms)`` that + completed by ``as_of_ms``, at each interval of ``intervals_s``. + + Row selection and censoring are the window cut's; as in + ``retune.window.read_trace_history``, rows are streamed and not + de-duplicated. A gzip member that fails validation is skipped and + counted, without the cut's corruption limit: a few lost history rows + only perturb a predictor's warmup. Failures raise ``WindowCutError``. + """ + + if not history_start_ms <= window_start_ms < window_end_ms <= as_of_ms: + raise ValueError( + "need history_start_ms <= window_start_ms < window_end_ms <= as_of_ms" + ) + observations = { + int(interval): ObservationBuckets( + end_ms=window_end_ms, + interval_ms=int(interval) * 1000.0, + duration_s=float(interval), + ) + for interval in intervals_s + } + warmups = { + interval: ObservationBuckets( + end_ms=window_start_ms, + interval_ms=buckets.interval_ms, + duration_s=buckets.duration_s, + ) + for interval, buckets in observations.items() + } + targets = (*observations.values(), *warmups.values()) + history_requests = 0 + dropped = 0 + try: + files = _segment_files(os.fspath(segments_root)) + items = _trace_items(files, history_start_ms / 1000.0 - SEGMENT_MTIME_MARGIN_S) + try: + for _, path, item in items: + if not isinstance(item, _Member): + dropped += isinstance(item, _Garbage) + continue + for index, raw in enumerate(item.payload.split(b"\n")): + if not raw.strip(): + continue + location = f"{path}@{item.offset}:{index + 1}" + event = _event(raw, location, "trace_corrupt") + # Buckets read arrival, ISL and OSL only, as the replay + # read task's history does: no hashes needed. + row = ( + None + if event is None + else _request_row( + event, location, "trace_corrupt", need_hashes=False + ) + ) + if row is None or not ( + history_start_ms <= row.arrival_ms < window_end_ms + ): + continue + if row.completion_ms > as_of_ms: + continue + if row.arrival_ms < window_start_ms: + history_requests += 1 + for target in targets: + target.add(row.arrival_ms, row.input_length, row.output_tokens) + finally: + items.close() + except OSError as error: + raise WindowCutError( + "window_cut_failed", f"{type(error).__name__}: {error}" + ) from error + return TraceHistory( + observations={ + interval: tuple(buckets.observations()) + for interval, buckets in observations.items() + }, + warmups={ + interval: tuple(buckets.observations()) + for interval, buckets in warmups.items() + }, + history_requests=history_requests, + dropped_members=dropped, + ) + + def _validate( start_ms: float, end_ms: float, @@ -978,10 +1140,14 @@ def _event( def _request_row( - event: dict[str, Any], location: str, failure: WindowCutFailure + event: dict[str, Any], + location: str, + failure: WindowCutFailure, + *, + need_hashes: bool = True, ) -> Optional[_RequestRow]: try: - return _parse_request_end(event, location, need_hashes=True) + return _parse_request_end(event, location, need_hashes=need_hashes) except ValueError as error: raise WindowCutError(failure, str(error)) from error diff --git a/components/src/dynamo/planner/retune/sweep.py b/components/src/dynamo/planner/retune/sweep.py index 5aaf57f9734a..01d32217e240 100644 --- a/components/src/dynamo/planner/retune/sweep.py +++ b/components/src/dynamo/planner/retune/sweep.py @@ -73,6 +73,7 @@ read_trace_history, read_trace_window, ) +from dynamo.planner.simulation.config import inner_planner_config from dynamo.planner.simulation.load_predictor import ( Window, evaluate_preset, @@ -215,6 +216,10 @@ class SweepRequest: spec_replicas: dict[str, int] | None = None replicas_verified: bool = True generation: int = 0 + # Live only: the grid's distinct planner points (searched fields), the + # planner axis a DGDR sweep shards; ``None`` derives it from every + # candidate. + grid_planners: tuple[dict[str, Any], ...] | None = None def __post_init__(self) -> None: keys = [candidate.key() for candidate in self.candidates] @@ -376,7 +381,21 @@ def evaluated_params( if candidate == baseline: return candidate - planner = {**baseline.planner, **candidate.planner} + planner = evaluated_planner(candidate.planner, baseline, presweep, base_planner) + evaluated = RetuneParams(router=dict(candidate.router), planner=planner) + return candidate if evaluated == candidate else evaluated + + +def evaluated_planner( + planner_point: Mapping[str, Any], + baseline: RetuneParams, + presweep: Mapping[str, Any] | None, + base_planner: Mapping[str, Any] | None, +) -> dict[str, Any]: + """The planner part ``evaluated_params`` gives any candidate other than + the baseline whose planner point is ``planner_point``.""" + + planner = {**baseline.planner, **planner_point} if planner and base_planner is not None: running = _running_planner(baseline, base_planner) planner = canonical_planner_point(planner, running) @@ -392,8 +411,7 @@ def evaluated_params( if key in planner and key in running } planner.update(fields) - evaluated = RetuneParams(router=dict(candidate.router), planner=planner) - return candidate if evaluated == candidate else evaluated + return planner def score_summary( @@ -804,7 +822,7 @@ def _run_sweep( ) -> SweepResult: deadline = started + request.timeout_s running = _running_planner(request.baseline, request.base_planner) - intervals = _throughput_intervals(request) + intervals = sweep_throughput_intervals(request) buckets = { interval: ObservationBuckets( end_ms=request.window_end_ms, @@ -859,24 +877,16 @@ def _run_sweep( inner_base = None if request.base_planner is not None: # The inner planner is the current one, warmed from the recorded - # history (the read task's warmup buckets) instead of the outer - # run's own warmup trace, never retuned, and without diagnostics - # reports (one HTML file per candidate). It always scales, even - # under an advisory outer planner: a candidate is scored on the - # replicas it would drive. Replay never runs batch scheduling, so - # it is off, which also drops the Redis settings an actuating - # planner would require. - inner_base = { - **request.base_planner, - "retune": None, - "load_predictor_warmup_trace": None, - "report_interval_hours": None, - "live_dashboard_port": 0, - "advisory": False, - "batch_scheduling": {"enabled": False}, - } + # history (the read task's warmup buckets). + inner_base = inner_planner_config(request.base_planner) if presweep_intervals: - presweep = _run_presweeps(request, pool, inputs, presweep_intervals) + presweep = run_presweeps( + request, + pool, + inputs.observations, + inputs.observations_paths, + presweep_intervals, + ) by_effective: dict[str, tuple[RetuneParams, list[str]]] = {} for candidate in request.candidates: effective = evaluated_params( @@ -909,26 +919,14 @@ def _run_sweep( # rather than on the planner tick that decides to swap it in. # They go first, so an ARIMA or Prophet fit on days of history # overlaps the replays instead of trailing them. - swaps = _predictor_swaps( - request.baseline, - running, + tasks[:0] = predictor_swap_tasks( + request, [effective for effective, _ in by_effective.values()], + inner_base, + inputs.observations, + inputs.observations_paths, + work_dir, ) - for interval, planner in swaps.items(): - path = inputs.observations_paths.get(interval) - if path is None: - path = _write_observations( - work_dir / f"{_OBSERVATIONS_FILE}-{interval}.json", - inputs.observations.get(interval, ()), - ) - tasks.insert( - 0, - ( - f"{_PREDICTOR_WARMUP_TASK}:{interval}", - _predictor_warmup_task, - ({**inner_base, **planner}, path), - ), - ) outcomes = pool.run(tasks) except _SweepInterrupted as interrupted: logger.warning( @@ -947,10 +945,7 @@ def _run_sweep( finally: pool.close() - warmup_errors: dict[int, str] = {} - for task_id, (ok, value) in outcomes.items(): - if task_id.startswith(f"{_PREDICTOR_WARMUP_TASK}:") and not ok: - warmup_errors[int(task_id.rsplit(":", 1)[1])] = value + warmup_errors = predictor_swap_errors(outcomes) for effective_key, (effective, original_keys) in by_effective.items(): ok, value = outcomes[effective_key] if ok: @@ -1006,6 +1001,51 @@ def _warmup_path( return None if interval is None else inputs.warmup_paths.get(interval) +def predictor_swap_tasks( + request: SweepRequest, + evaluated: Sequence[RetuneParams], + inner_base: Mapping[str, Any], + observations: Mapping[int, Sequence[TrafficObservation]], + observations_paths: Mapping[int, str], + work_dir: Path, +) -> list[tuple[str, Callable[..., Any], tuple[Any, ...]]]: + """Worker tasks that prepare, per throughput interval, the predictor a + host would swap in for the ``evaluated`` candidates at that interval + (``_predictor_swaps``), on the observations of that interval; a missing + observations file is written to ``work_dir``. ``predictor_swap_errors`` + reads their outcomes.""" + + running = _running_planner(request.baseline, request.base_planner) + tasks: list[tuple[str, Callable[..., Any], tuple[Any, ...]]] = [] + for interval, planner in _predictor_swaps( + request.baseline, running, evaluated + ).items(): + path = observations_paths.get(interval) + if path is None: + path = _write_observations( + work_dir / f"{_OBSERVATIONS_FILE}-{interval}.json", + observations.get(interval, ()), + ) + tasks.append( + ( + f"{_PREDICTOR_WARMUP_TASK}:{interval}", + _predictor_warmup_task, + ({**inner_base, **planner}, path), + ) + ) + return tasks + + +def predictor_swap_errors(outcomes: Mapping[str, tuple[bool, Any]]) -> dict[int, str]: + """The failed ``predictor_swap_tasks`` outcomes, by throughput interval.""" + + errors: dict[int, str] = {} + for task_id, (ok, value) in outcomes.items(): + if task_id.startswith(f"{_PREDICTOR_WARMUP_TASK}:") and not ok: + errors[int(task_id.rsplit(":", 1)[1])] = value + return errors + + def _predictor_swaps( baseline: RetuneParams, running: Mapping[str, Any], @@ -1043,7 +1083,7 @@ def _nonzero(counts: Mapping[str, int]) -> dict[str, int]: return {key: value for key, value in counts.items() if value} -def _throughput_intervals(request: SweepRequest) -> list[int]: +def sweep_throughput_intervals(request: SweepRequest) -> list[int]: """The throughput intervals an inner planner scales at: the base planner's and every candidate's (``[]`` without an inner planner).""" @@ -1064,21 +1104,23 @@ def _throughput_intervals(request: SweepRequest) -> list[int]: return sorted(intervals) -def _run_presweeps( +def run_presweeps( request: SweepRequest, pool: _WorkerPool, - inputs: _SweepInputs, + observations: Mapping[int, Sequence[TrafficObservation]], + observations_paths: Mapping[int, str], intervals: Sequence[int], ) -> dict[str, Any]: """Score every predictor candidate at every throughput interval in its own worker task, then pick one per interval. - ``inputs.observations`` are history and window buckets ending at the - window end; only the buckets wholly inside ``W_k`` are forecast and - scored, and the ones before them only warm the predictors. Each task - carries the path of the observations file, not the observations. One - interval gives its summary (``SweepResult.predictor_presweep``); several - give ``{"by_interval": {...}}``. + ``observations`` (by interval) are history and window buckets ending at + the window end, in the files ``observations_paths``; only the buckets + wholly inside ``W_k`` are forecast and scored, and the ones before them + only warm the predictors. Each task carries the path of the + observations file, not the observations. One interval gives its + summary (``SweepResult.predictor_presweep``); several give + ``{"by_interval": {...}}``. """ started = time.monotonic() @@ -1086,16 +1128,16 @@ def _run_presweeps( tasks: list[tuple[str, Callable[..., Any], tuple[Any, ...]]] = [] steps: dict[int, tuple[int, int]] = {} for interval in intervals: - path = inputs.observations_paths.get(interval) - observations = inputs.observations.get(interval) - if path is None or observations is None: + path = observations_paths.get(interval) + rows = observations.get(interval) + if path is None or rows is None: raise RuntimeError("the pre-sweep needs the read task's observations file") interval_ms = interval * 1000.0 * request.arrival_speedup_ratio window_steps = int( (request.window_end_ms - request.window_start_ms) / interval_ms + 1e-9 ) - eval_steps = min(len(observations), window_steps) - steps[interval] = (eval_steps, len(observations) - eval_steps) + eval_steps = min(len(rows), window_steps) + steps[interval] = (eval_steps, len(rows) - eval_steps) tasks.extend( ( f"{_PRESWEEP_TASK}:{interval}:{label}", @@ -1333,3 +1375,10 @@ def _worker_main(conn: Connection, temp_dir: str) -> None: except Exception as error: # An unpicklable result is reported as this task's failure. conn.send((False, f"unsendable result: {error}")) + + +# Public names for the live DGDR executor, which prepares inner planners +# with the same worker pool before it creates its Requests. +WorkerPool = _WorkerPool +SweepInterrupted = _SweepInterrupted +write_observations = _write_observations diff --git a/components/src/dynamo/planner/simulation/config.py b/components/src/dynamo/planner/simulation/config.py index 828f5068a7fe..641fce8b8ad8 100644 --- a/components/src/dynamo/planner/simulation/config.py +++ b/components/src/dynamo/planner/simulation/config.py @@ -7,6 +7,7 @@ import math from collections.abc import Mapping +from copy import deepcopy from typing import Annotated, Any, Literal, TypeAlias from aisimulate.config.common import Choices, IntegerRange, NumericRange @@ -382,6 +383,103 @@ def _validate_preset_conflicts(self) -> PlannerRecommendationConfig: } +# ``PlannerConfig`` fields a live retune sweep (``retune.executor: dgdr``) +# sends as the base of each candidate's inner planner, in the +# ``planner.retune_sweep`` section of a DGDR Request: the fields that decide +# scaling. Connections, credentials, file paths, ports, reports, the +# Kubernetes wiring and the performance-model sources stay in the live +# Planner; the Run's replay brings its own performance model. +PLANNER_RETUNE_SWEEP_BASE_FIELDS = frozenset( + { + "mode", + "optimization_target", + "ttft_ms", + "itl_ms", + "enable_throughput_scaling", + "enable_load_scaling", + "throughput_adjustment_interval_seconds", + "load_adjustment_interval_seconds", + "max_throughput_scaling_replicas", + "max_gpu_budget", + "min_gpu_budget", + "min_endpoint", + "prefill_min_endpoint", + "decode_min_endpoint", + "load_predictor", + "load_predictor_log1p", + "prophet_window_size", + "kalman_q_level", + "kalman_q_trend", + "kalman_r", + "kalman_min_points", + "max_num_fpm_samples", + "fpm_sample_bucket_size", + "load_scaling_down_sensitivity", + "load_min_observations", + "prefill_scale_up_queue_tokens", + "prefill_scale_down_queue_tokens", + "decode_scale_up_kv_rate", + "decode_scale_down_kv_rate", + "speculative_nextn", + "throughput_metrics_source", + "scheduling", + } +) +# ``scheduling`` keys a retune sweep base keeps: the tick cadence. Plugins, +# the registration gateway and the live tick deadline stay in the Planner. +PLANNER_RETUNE_SWEEP_SCHEDULING_FIELDS = frozenset({"scale_interval_seconds"}) + + +def retune_sweep_base(planner: Mapping[str, Any]) -> dict[str, Any]: + """The ``planner.retune_sweep.base`` of a live Planner config dump.""" + + base = { + key: value + for key, value in planner.items() + if key in PLANNER_RETUNE_SWEEP_BASE_FIELDS and key != "scheduling" + } + scheduling = planner.get("scheduling") + if isinstance(scheduling, Mapping): + base["scheduling"] = { + key: value + for key, value in scheduling.items() + if key in PLANNER_RETUNE_SWEEP_SCHEDULING_FIELDS + } + return base + + +# What a retune candidate's inner planner overrides in the running planner's +# configuration. It is never retuned and always actuates, even under an +# advisory outer planner: a candidate is scored on the replicas it would +# drive. It writes no reports and opens no listener, warms from the sweep's +# history instead of the outer planner's warmup trace, and runs no batch +# scheduling, which replay never runs and which would need the Redis settings +# an actuating planner requires. +INNER_PLANNER_OVERRIDES: dict[str, Any] = { + "retune": None, + "advisory": False, + "report_interval_hours": None, + "live_dashboard_port": 0, + "metric_reporting_prometheus_port": 0, + "load_predictor_warmup_trace": None, + "batch_scheduling": {"enabled": False}, +} + + +def inner_planner_config( + base: Mapping[str, Any], point: Mapping[str, Any] | None = None +) -> dict[str, Any]: + """A retune candidate's inner planner: ``base`` with ``point`` applied and + ``INNER_PLANNER_OVERRIDES`` over both. Every retune executor (the replay + sweep, the DGDR executor and the AISimulate adapter) builds it here.""" + + return { + **deepcopy(dict(base)), + **deepcopy(dict(point or {})), + **deepcopy(INNER_PLANNER_OVERRIDES), + } + + class PlannerRetuneConfig(PlannerRecommendationConfig): """``retune.planner``: the recommendation schema, restricted to what a running planner can change. diff --git a/components/src/dynamo/planner/simulation/provider.py b/components/src/dynamo/planner/simulation/provider.py index 69cf47cdebf6..48d0258ebff3 100644 --- a/components/src/dynamo/planner/simulation/provider.py +++ b/components/src/dynamo/planner/simulation/provider.py @@ -49,6 +49,7 @@ PLANNER_PUBLIC_TO_CONFIG, PlannerRecommendationConfig, Policy, + inner_planner_config, ) from .load_predictor import ( LOAD_PREDICTOR_PRESETS, @@ -96,6 +97,13 @@ _WARMUP_TRACE = "load_predictor_warmup_trace" _ADVISORY = "advisory" _PREDICTION_ONLY_FIELDS = (_WARMUP_TRACE, _RETUNE, _ADVISORY) +# Alone in the planner section: a live retune sweep's candidates +# (``RetuneSweepSpec``). Each candidate replays the live Planner's +# configuration with one point's fields applied. ``aisimulate recommend`` +# searches its points; each candidate's concrete config is the section with +# its one point, which ``aisimulate predict`` replays as that candidate. +RETUNE_SWEEP = "retune_sweep" +RETUNE_POINT = "retune_point" _HOOK = RuntimeHookSpec( @@ -403,6 +411,167 @@ def _validate_preset(self) -> LoadPredictorSearch: return self +class RetuneSweepSpec(BaseModel): + """``planner.retune_sweep``: the candidates of a live retune sweep. + + ``base`` is the live Planner configuration (only + ``PLANNER_RETUNE_SWEEP_BASE_FIELDS``), ``points`` the Planner fields each + candidate sets (the searchable runtime fields), and ``warmup`` the + recorded history before the window, keyed by throughput interval in + seconds, as ``TrafficObservation`` rows in field order: the load + predictor warmup of a candidate that scales on throughput at that + interval. + """ + + model_config = ConfigDict(extra="forbid") + + base: dict[str, Any] + points: list[dict[str, Any]] = Field(min_length=1) + warmup: dict[str, list[list[float | None]]] = Field(default_factory=dict) + + @model_validator(mode="after") + def _validate_fields(self) -> RetuneSweepSpec: + from dynamo.planner.reconfiguration import ( + PREDICTOR_FIELDS, + SCALING_POLICY_FIELDS, + ) + + from .config import ( + PLANNER_RETUNE_KNOBS, + PLANNER_RETUNE_SWEEP_BASE_FIELDS, + PLANNER_RETUNE_SWEEP_SCHEDULING_FIELDS, + ) + + extra = sorted(set(self.base) - PLANNER_RETUNE_SWEEP_BASE_FIELDS) + if extra: + raise ValueError(f"retune_sweep.base cannot set {extra}") + scheduling = self.base.get("scheduling", {}) + if not isinstance(scheduling, dict) or set(scheduling) - ( + PLANNER_RETUNE_SWEEP_SCHEDULING_FIELDS + ): + raise ValueError( + "retune_sweep.base.scheduling may set only " + f"{sorted(PLANNER_RETUNE_SWEEP_SCHEDULING_FIELDS)}" + ) + searchable = ( + set(PLANNER_RETUNE_KNOBS.values()) + | {"load_scaling_down_sensitivity"} + | PREDICTOR_FIELDS + | SCALING_POLICY_FIELDS + ) + for point in self.points: + unknown = sorted(set(point) - searchable) + if unknown: + raise ValueError(f"retune_sweep point {point} sets {unknown}") + for interval, rows in self.warmup.items(): + if not interval.isdigit() or int(interval) < 1: + raise ValueError( + f"retune_sweep.warmup keys are intervals in seconds, got {interval!r}" + ) + if any(len(row) != len(_OBSERVATION_FIELDS) for row in rows): + raise ValueError( + f"retune_sweep.warmup rows have {len(_OBSERVATION_FIELDS)} " + f"values ({', '.join(_OBSERVATION_FIELDS)})" + ) + return self + + +_OBSERVATION_FIELDS = ( + "duration_s", + "num_req", + "isl", + "osl", + "kv_hit_rate", + "accept_length", +) + + +def _retune_sweep_planner_config( + spec: RetuneSweepSpec, + point: Mapping[str, JSONValue], + sample: Mapping[str, JSONValue] | None, +) -> dict[str, JSONValue]: + """The inner planner configuration of one retune sweep candidate. + + ``base`` with ``point`` applied, retune off, not advisory (a candidate is + scored on the replicas it would drive), without reports or listeners, + and with the GPUs per worker of the candidate's pinned engine + (``sample``), validated as a whole. Only the fields it sets are + serialized, never environment-derived connection settings. + """ + + payload: dict[str, Any] = inner_planner_config(spec.base, point) + if sample is not None: + mode = str(sample["deployment_mode"]) + if mode == "disagg": + payload["prefill_engine_num_gpu"] = _int_value( + sample, "prefill_tp" + ) * _int_value(sample, "prefill_attention_dp") + payload["decode_engine_num_gpu"] = _int_value( + sample, "decode_tp" + ) * _int_value(sample, "decode_attention_dp") + else: + payload["decode_engine_num_gpu"] = _int_value(sample, "tp") * _int_value( + sample, "attention_dp" + ) + resolved = PlannerConfig.model_validate( + { + **payload, + "metric_pulling_prometheus_endpoint": "", + "metric_pulling_prometheus_token": None, + "metric_pulling_prometheus_token_file": None, + "metric_pulling_prometheus_ssl_verify": False, + "metric_pulling_prometheus_ca_bundle": None, + "metric_pulling_prometheus_request_timeout_seconds": ( + SLAPlannerDefaults.metric_pulling_prometheus_request_timeout_seconds + ), + } + ) + # Retune is off, so PlannerConfig requires a scaling mode: a candidate + # always replays a planner that scales. + return resolved.model_dump(mode="json", include=set(payload)) + + +def _retune_sweep_warmup( + spec: RetuneSweepSpec, planner_config: Mapping[str, JSONValue] +) -> tuple[str, list[list[float | None]]] | None: + """The candidate's throughput interval and its warmup rows, if any.""" + + if not planner_config.get("enable_throughput_scaling"): + return None + interval = str(planner_config.get("throughput_adjustment_interval_seconds")) + rows = spec.warmup.get(interval) + return None if rows is None else (interval, rows) + + +def _retune_sweep_replay( + spec: RetuneSweepSpec, + point: Mapping[str, JSONValue], + sample: Mapping[str, JSONValue], +) -> AdapterReplaySpec: + """The replay of one retune sweep candidate. + + Its config is the section narrowed to the point and its own warmup: a + Candidate's ``adapters["dynamo.planner"]`` names the point, and the + config alone replays it again. + """ + + planner_config = _retune_sweep_planner_config(spec, point, sample) + hook_config: dict[str, JSONValue] = {"planner_config": planner_config} + warmup = _retune_sweep_warmup(spec, planner_config) + if warmup is not None: + hook_config["warmup_observations"] = cast(JSONValue, warmup[1]) + section = { + "base": deepcopy(spec.base), + "points": [deepcopy(dict(point))], + "warmup": {} if warmup is None else {warmup[0]: deepcopy(warmup[1])}, + } + return AdapterReplaySpec( + config={RETUNE_SWEEP: cast(JSONValue, section)}, + runtime_hooks=(replace(_HOOK, config=hook_config),), + ) + + _LEGACY_PRESET_REMOVAL_NOTE = ( "Legacy flat Dynamo Planner Sweeper preset fields are deprecated and will be " "removed after the 1.5 release. Nest choices under each sub-item's 'preset' field." @@ -669,6 +838,8 @@ def compile_prediction( config: Mapping[str, JSONValue], context: PredictionAdapterContext, ) -> AdapterReplaySpec: + if RETUNE_SWEEP in config: + return self._compile_retune_sweep_point(config, context) unknown = ( set(config) - PLANNER_PUBLIC_TO_CONFIG.keys() @@ -722,6 +893,8 @@ def compile_recommendation( config: Mapping[str, JSONValue], context: RecommendationAdapterContext, ) -> AdapterSearchPlan: + if RETUNE_SWEEP in config: + return self._compile_retune_sweep(config, context) if _RETUNE in config: raise ValueError( "planner.retune is not supported by aisimulate recommend: each " @@ -804,12 +977,87 @@ def compile_recommendation( state=state, ) + def _compile_retune_sweep( + self, + config: Mapping[str, JSONValue], + context: RecommendationAdapterContext, + ) -> AdapterSearchPlan: + """A live retune sweep: one choice per candidate point. + + Every point is validated on the base before any candidate runs, so a + Request the live Planner built wrong fails as a whole instead of + publishing no candidates. + """ + + extra = sorted(set(config) - {RETUNE_SWEEP}) + if extra: + raise ValueError(f"planner.{RETUNE_SWEEP} cannot be combined with {extra}") + spec = RetuneSweepSpec.model_validate(config[RETUNE_SWEEP]) + for point in spec.points: + try: + _retune_sweep_planner_config(spec, point, None) + except ValueError as error: + raise ValueError( + f"planner.{RETUNE_SWEEP} point {point} is invalid: {error}" + ) from None + points = cast(list[JSONValue], deepcopy(spec.points)) + return AdapterSearchPlan( + fragment=SearchSpaceFragment( + choices_by_branch={ + mode: {RETUNE_POINT: list(points)} + for mode in _deployment_modes(context.sweep.core_search_space) + } + ), + state={RETUNE_SWEEP: spec.model_dump(mode="json")}, + potential_runtime_hooks=(_HOOK,), + ) + + @staticmethod + def _compile_retune_sweep_point( + config: Mapping[str, JSONValue], + context: PredictionAdapterContext, + ) -> AdapterReplaySpec: + """One live retune sweep candidate (``retune_sweep`` with one point), + as a recommendation emits it.""" + + extra = sorted(set(config) - {RETUNE_SWEEP}) + if extra: + raise ValueError(f"planner.{RETUNE_SWEEP} cannot be combined with {extra}") + spec = RetuneSweepSpec.model_validate(config[RETUNE_SWEEP]) + if len(spec.points) != 1: + raise ValueError( + f"aisimulate predict replays one planner.{RETUNE_SWEEP} point, got " + f"{len(spec.points)}" + ) + return _retune_sweep_replay( + spec, spec.points[0], _prediction_sample(context.engine) + ) + + @staticmethod + def _materialize_retune_sweep( + raw: JSONValue, + selection: Mapping[str, JSONValue], + context: CandidateContext, + ) -> AdapterReplaySpec: + spec = RetuneSweepSpec.model_validate(raw) + point = selection.get(RETUNE_POINT) + if not isinstance(point, dict): + raise TypeError(f"Planner {RETUNE_POINT} selection must be a mapping") + try: + return _retune_sweep_replay(spec, point, context.sample) + except ValueError as error: + raise InfeasibleCandidate(f"Planner candidate: {error}") from error + def materialize_candidate( self, plan: AdapterSearchPlan, selection: Mapping[str, JSONValue], context: CandidateContext, ) -> AdapterReplaySpec: + if isinstance(plan.state, dict) and RETUNE_SWEEP in plan.state: + return self._materialize_retune_sweep( + plan.state[RETUNE_SWEEP], selection, context + ) if isinstance(plan.state, dict) and plan.state.get("forced_disabled") is True: return AdapterReplaySpec(config={"policy": "disabled"}) materialized = dict(selection) diff --git a/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py b/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py index daee3d4da40a..97c075cd5150 100644 --- a/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py +++ b/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py @@ -573,6 +573,11 @@ async def replace_live_retune(self, inputs, *, generation): inputs, generation=generation, now_s=self._clock.time() ) + def retune_decision_persisted(self, decision): + # Router-only retune: no decision carries Planner fields. + if decision.planner_changes: + raise AssertionError(f"unexpected planner changes {decision}") + class Planner: """One planner process: its supervisor, engine, actuator and reconciler.""" diff --git a/components/src/dynamo/planner/tests/live_retune_fakes.py b/components/src/dynamo/planner/tests/live_retune_fakes.py index 1ad273badc11..f5d29403715d 100644 --- a/components/src/dynamo/planner/tests/live_retune_fakes.py +++ b/components/src/dynamo/planner/tests/live_retune_fakes.py @@ -290,6 +290,32 @@ def router_points(effective_input: Mapping[str, Any]) -> list[dict[str, float]]: ] +def planner_points( + effective_input: Mapping[str, Any] +) -> list[Optional[dict[str, Any]]]: + """The planner points of an input's ``planner.retune_sweep`` section; + ``[None]`` without one (fixed replicas).""" + + section = nested_get(effective_input, "planner", "retune_sweep") + if not isinstance(section, Mapping): + return [None] + return [dict(point) for point in section["points"]] + + +def planner_metrics( + point_public: Mapping[str, float], planner: Optional[Mapping[str, Any]] +) -> dict[str, float]: + """``default_metrics`` with a planner point's load sensitivity added to + its goodput (for tests: a higher sensitivity scores better).""" + + metrics = default_metrics(point_public) + if planner: + metrics["goodput_output_throughput_tok_s"] += float( + planner.get("load_scaling_down_sensitivity", 0) + ) + return metrics + + def engine_parameters(effective_input: Mapping[str, Any]) -> dict[str, Any]: """The non-router parameters every Candidate of a pinned engine shares.""" @@ -309,23 +335,34 @@ def engine_parameters(effective_input: Mapping[str, Any]) -> dict[str, Any]: def candidate_parameters( - effective_input: Mapping[str, Any], point_public: Mapping[str, float] + effective_input: Mapping[str, Any], + point_public: Mapping[str, float], + planner: Optional[Mapping[str, Any]] = None, ) -> dict[str, Any]: - """``Candidate.status.parameters`` of one evaluated router point.""" + """``Candidate.status.parameters`` of one evaluated router (and planner) + point.""" router = effective_input.get("router") or {} - return { - **engine_parameters(effective_input), - "adapters": { - ROUTER_ADAPTER: { - "policy": router.get("policy", "kv_router"), - "prefill_load_model": dict( - router.get("prefill_load_model") or {"type": "none"} - ), - **{name: float(value) for name, value in point_public.items()}, - } - }, + adapters: dict[str, Any] = { + ROUTER_ADAPTER: { + "policy": router.get("policy", "kv_router"), + "prefill_load_model": dict( + router.get("prefill_load_model") or {"type": "none"} + ), + **{name: float(value) for name, value in point_public.items()}, + } } + if planner is not None: + # The adapter's concrete config: the section narrowed to the point. + section = nested_get(effective_input, "planner", "retune_sweep") + adapters["dynamo.planner"] = { + "retune_sweep": { + "base": dict(section["base"]), + "points": [dict(planner)], + "warmup": {}, + } + } + return {**engine_parameters(effective_input), "adapters": adapters} def default_metrics(point_public: Mapping[str, float]) -> dict[str, float]: @@ -1087,6 +1124,10 @@ def __init__( self.evaluate: Evaluator = evaluate or ( lambda _request, point: default_metrics(point) ) + # Scenario 2: metrics of a router point crossed with a planner point. + self.evaluate_planner: Callable[ + ..., Optional[dict[str, float]] + ] = lambda _request, point, planner: planner_metrics(point, planner) self.runtime_image = runtime_image self.dynamo_version = dynamo_version self.mutate_input = mutate_input @@ -1145,10 +1186,18 @@ def step(self) -> int: effective = run["status"]["aisim"]["input"] request = self.kube.requests[name] published = [] - for point in router_points(effective): - metrics = self.evaluate(copy.deepcopy(request), point) + for point, planner in itertools.product( + router_points(effective), planner_points(effective) + ): + metrics = ( + self.evaluate(copy.deepcopy(request), point) + if planner is None + else self.evaluate_planner( + copy.deepcopy(request), point, planner + ) + ) if metrics is not None: - parameters = candidate_parameters(effective, point) + parameters = candidate_parameters(effective, point, planner) if self.mutate_parameters is not None: self.mutate_parameters(name, parameters) published.append((parameters, metrics)) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py b/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py index 34cbd95a6f71..1b913f426faa 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py @@ -375,6 +375,7 @@ def on_effects(self, effects): self.effects.append(effects) if self.on_effects_error is not None: raise self.on_effects_error + return list(effects.retune_decisions or ()) def _live_config(executor: str = "dgdr") -> PlannerConfig: diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py index ef2cbddf43d4..8527434e55c0 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py @@ -6,7 +6,7 @@ """Configuration premises of live retune (``retune.executor: dgdr``). P1 (Kubernetes, decision latency, explicit timeout within the interval), P2 -(retune-only, router only), P3 (the DGDR router knobs), P5 (a pinned engine +(a router domain; scaling and a planner domain are Scenario 2), P3 (the DGDR router knobs), P5 (a pinned engine section) and P13 (the largest grid shard fits the sweep deadline), plus the shard-split and need-seconds helpers the executor shares with validation. """ @@ -225,37 +225,56 @@ def test_p1_rejects_a_timeout_within_ten_minutes_of_the_interval(): # --------------------------------------------------------------------- P2 +# Two credits, the rest pinned: a 2-point router grid. +PINNED_ROUTER = { + "overlap_score_credit": {"choices": [0.5, 1.0]}, + "prefill_load_scale": 1.0, + "temperature": 0.0, +} +LOAD_ONLY = {"enable_load_scaling": True, "enable_throughput_scaling": False} +SLA_THROUGHPUT = { + "enable_load_scaling": False, + "enable_throughput_scaling": True, + "optimization_target": "sla", +} + + @pytest.mark.parametrize( - "scaling", + ("scaling", "planner"), [ - {"enable_load_scaling": True, "enable_throughput_scaling": False}, - { - "enable_load_scaling": False, - "enable_throughput_scaling": True, - "optimization_target": "sla", - }, + (LOAD_ONLY, {"load_sensitivity": {"preset": ["default", "conservative"]}}), + (SLA_THROUGHPUT, "default"), + # Easy mode keeps load scaling on without the flags written. + ({"enable_throughput_scaling": False}, None), + ( + {**SLA_THROUGHPUT, "enable_load_scaling": True}, + {"scaling_policy": {"preset": ["hybrid_180_5", "hybrid_600_5"]}}, + ), ], + ids=["load", "sla-throughput", "easy-mode", "scaling-policy"], ) -def test_p2_requires_retune_only(scaling): - _rejects(r"retune-only mode \(P2\)", **scaling) - - -def test_p2_requires_both_scaling_flags_explicitly(): - # Without both flags written, easy mode keeps load scaling on. - with pytest.raises(ValidationError, match=r"retune-only mode \(P2\)"): - PlannerConfig( - namespace="test-ns", - environment="kubernetes", - mode="agg", - enable_throughput_scaling=False, - retune=_retune(), - ) +def test_p2_accepts_scaling_and_a_planner_domain(scaling, planner): + # Scenario 2: candidates replay an inner planner from the window start. + retune = _retune(router=PINNED_ROUTER) + if planner is not None: + retune["planner"] = planner + config = PlannerConfig( + namespace="test-ns", + environment="kubernetes", + mode="agg", + retune=retune, + **scaling, + ) + assert config.scaling_enabled() -def test_p2_rejects_a_planner_domain(): +def test_p2_requires_a_router_domain(): + # Every DGDR Request sets the three router knobs; pin them to retune the + # Planner alone. _rejects( - r"retunes the router only \(P2\)", - _retune(planner={"load_sensitivity": {"preset": ["default"]}}), + r"requires retune.router \(P2\)", + _retune(router=None, planner="default"), + **LOAD_ONLY, ) @@ -617,13 +636,58 @@ def test_p13_counts_the_largest_shard_of_fewer_max_shards(): def test_p13_rejects_a_grid_that_needs_more_shards(): _rejects( - r"router grid of 108 points cannot be split into at most " + r"retune grid of 108 points cannot be split into at most " r"retune.dgdr.run.max_shards=1 Cartesian shards of at most 100 points " r"\(P13\); the largest would have 108", _retune(dgdr=_dgdr(run={"max_shards": 1})), ) +def test_p13_shards_the_planner_points_too(): + # The 108 router points x the 3 default sensitivities: 324 points need a + # fourth shard, which cuts the temperature axis. + retune = _retune(planner="default") + _rejects( + r"retune grid of 324 points cannot be split into at most " + r"retune.dgdr.run.max_shards=3 Cartesian shards", + retune, + **LOAD_ONLY, + ) + dgdr = _dgdr(run={"max_shards": 4, "parallelism": 16}) + config = _config(_retune(planner="default", dgdr=dgdr), **LOAD_ONLY) + assert config.retune is not None + + +def test_p13_budgets_the_history_read_of_a_scaling_planner(): + dgdr = _dgdr( + cut_budget_seconds=60, + job_startup_seconds=0, + run={"parallelism": 36, "candidate_timeout_seconds": 100}, + ) + retune = _retune(router=PINNED_ROUTER, planner=None, dgdr=dgdr) + # Two one-point shards: cut 60 + history 60 + 0 + 1 round x 100 = 220. + assert _config({**retune, "timeout_seconds": 220}, **LOAD_ONLY).retune is not None + _rejects( + r"needs 220s: cut_budget_seconds 60 for the history \+ cut_budget_seconds 60", + {**retune, "timeout_seconds": 219}, + **LOAD_ONLY, + ) + # Fixed replicas read no history. + assert _config({**retune, "timeout_seconds": 160}).retune is not None + + +def test_p13_bounds_the_warmup_a_request_carries(): + # 180-s buckets: 1024 fit, 1025 do not. + retune = _retune(router=PINNED_ROUTER, planner=None) + accepted = {**retune, "history_seconds": 180.0 * 1024} + assert _config(accepted, **SLA_THROUGHPUT).retune is not None + _rejects( + r"is 1025 warmup buckets, more than the 1024 a DGDR Request carries \(P13\)", + {**retune, "history_seconds": 180.0 * 1025}, + **SLA_THROUGHPUT, + ) + + def test_p13_uses_the_configured_run_parallelism_and_timeouts(): dgdr = _dgdr( cut_budget_seconds=60, diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py index a0f52a33e496..fac68ca8c99b 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py @@ -571,6 +571,107 @@ async def test_window_started_before_the_controller_launches_unverified(): assert request.replicas == SPEC +def _scaling_config(planner="default", **overrides): + # Scenario 2: load scaling (easy mode) with the default sensitivities. + retune = { + "interval_seconds": INTERVAL, + "grace_seconds": GRACE, + "router": ROUTER, + } + if planner is not None: + retune["planner"] = planner + return PlannerConfig( + namespace="test-ns", + mode="agg", + **{"enable_throughput_scaling": False, **overrides}, + retune=retune, + ) + + +@pytest.mark.parametrize( + ("start", "replicas", "verified"), + [ + # Ready counts recorded from before the window: its start's count. + (T0 - INTERVAL - 2 * STEP, {"aggregated": 3}, True), + # The window began before the controller: the spec stands in. + (T0 - INTERVAL / 2, SPEC, False), + ], + ids=["recorded", "unrecorded"], +) +async def test_a_scaling_planner_sweeps_from_the_ready_counts_at_window_start( + start, replicas, verified +): + # The planner scales within the window: 3 workers at its start, 1 in + # the middle, 2 (the spec) at its end. That is not replicas_changed. + sweeps = FakeSweeps() + sweeps.default = _flat() + + def counts(now): + if now < T0 - INTERVAL / 2: + return 3 + return 1 if now < T0 - STEP else 2 + + controller = _live(sweeps, start=start, config=_scaling_config()) + try: + decisions = await _run_window(controller, counts, start=start, until=L_K) + finally: + controller.close() + + assert decisions == [] + (request,) = _requests_for_k(sweeps) + assert request.replicas == replicas + assert request.replicas_verified is verified + # The executor starts each inner planner from SweepRequest.replicas. + assert request.spec_replicas is None + assert request.base_planner is not None + assert {c.planner["load_scaling_down_sensitivity"] for c in request.candidates} == { + 70, + 80, + 90, + } + # The planner axis the DGDR sweep shards: the grid's planner points. + assert sorted( + planner["load_scaling_down_sensitivity"] for planner in request.grid_planners + ) == [70, 80, 90] + + +@pytest.mark.parametrize( + ("planner", "overrides"), + [ + (None, {}), + ( + {"load_predictor": {"preset": ["constant_last", "kalman_default_raw"]}}, + { + "optimization_target": "sla", + "enable_throughput_scaling": True, + "load_predictor": "kalman", + }, + ), + ], + ids=["router_only", "predictor_only"], +) +async def test_a_scaling_planner_without_planner_points_keeps_one_planner_point( + planner, overrides +): + # The grid searches no planner field: its one planner point is {}, the + # running planner (with the pre-sweep winner), so every Run still + # replays an inner planner. + sweeps = FakeSweeps() + sweeps.default = _flat() + config = _scaling_config(planner, **overrides) + controller = _live(sweeps, start=T0 - INTERVAL - 2 * STEP, config=config) + try: + await _run_window( + controller, lambda _now: 2, start=T0 - INTERVAL - 2 * STEP, until=L_K + ) + finally: + controller.close() + + (request,) = _requests_for_k(sweeps) + assert request.base_planner is not None + assert request.grid_planners == ({},) + + # --------------------------------------------------------------------------- # restore_desired # --------------------------------------------------------------------------- diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py index 0759fdb4858f..b24c613b9000 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py @@ -34,11 +34,14 @@ from dynamo.planner.config.planner_config import ( DgdrExecutorConfig, + PlannerConfig, RetuneConfig, RetuneEngineSpec, dgdr_need_seconds, ) -from dynamo.planner.core.types import RetuneParams +from dynamo.planner.core.types import RetuneParams, TrafficObservation +from dynamo.planner.retune import sweep as sweep_module +from dynamo.planner.retune.domains import PREDICTOR_KEYS from dynamo.planner.retune.live import dgdr from dynamo.planner.retune.live.contracts import ( LABEL_ATTEMPT, @@ -68,6 +71,7 @@ window_selection, ) from dynamo.planner.retune.live.reconciler import trace_pvc_source +from dynamo.planner.retune.live.window_cut import TraceHistory from dynamo.planner.retune.sweep import SweepRequest, score_summary from dynamo.planner.tests.live_retune_fakes import ( FakeDgdrOperator, @@ -2116,3 +2120,403 @@ def real_cut(*args, deadline_s, **kwargs): _, claim_path = trace_pvc_source(h.created()[0]["spec"]) assert tmp_path / "trace" / claim_path == window_file assert_scores(result, request) + + +# ------------------------------------------------------------- Scenario 2 + +LSD = "load_scaling_down_sensitivity" +THROUGHPUT_INTERVAL = "throughput_adjustment_interval_seconds" + + +class InlinePool: + """``sweep.WorkerPool`` that runs each task in this process.""" + + def __init__(self, *args: Any) -> None: + self.closed = False + + def run(self, tasks): + results = {} + for task_id, function, args in tasks: + try: + results[task_id] = (True, function(*args)) + except Exception as error: + results[task_id] = (False, f"{type(error).__name__}: {error}") + return results + + def close(self) -> None: + self.closed = True + + +class FakeHistory: + """``window_cut.bucket_history``: records calls, returns 4 buckets per + interval.""" + + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + + def __call__( + self, + segments_root, + history_start_ms, + window_start_ms, + window_end_ms, + as_of_ms, + intervals_s, + *, + deadline_s, + cancel=None, + ): + self.calls.append( + { + "segments_root": segments_root, + "history_start_ms": history_start_ms, + "window": (window_start_ms, window_end_ms, as_of_ms), + "intervals": list(intervals_s), + } + ) + + def rows(interval: int, count: int): + return tuple( + TrafficObservation(float(interval), 10.0 + i, 100.0, 20.0) + for i in range(count) + ) + + return TraceHistory( + observations={interval: rows(interval, 6) for interval in intervals_s}, + warmups={interval: rows(interval, 4) for interval in intervals_s}, + history_requests=40, + dropped_members=0, + ) + + +def _base_planner(**overrides: Any) -> dict[str, Any]: + values: dict[str, Any] = { + "namespace": "test-ns", + "mode": "agg", + "enable_load_scaling": True, + "enable_throughput_scaling": False, + "metric_pulling_prometheus_endpoint": "http://prometheus:9090", + **overrides, + } + return PlannerConfig(**values).model_dump(mode="json") + + +def scenario_2_request( + base_planner: dict[str, Any], sensitivities=(70, 90), running=80 +) -> SweepRequest: + """SMALL_DOMAIN x load sensitivities, from the running sensitivity, with + two ready workers at the window start and three in the spec.""" + + request = make_request(spec_replicas={"aggregated": 3}) + baseline = RetuneParams(router=request.baseline.router, planner={LSD: running}) + candidates = { + RetuneParams(router=point, planner={LSD: value}) + for point in grid(SMALL_DOMAIN) + for value in sensitivities + } + candidates.add(baseline) + return dataclasses.replace( + request, + candidates=tuple(sorted(candidates, key=RetuneParams.key)), + baseline=baseline, + replicas={"aggregated": 2}, + base_planner=base_planner, + history_start_ms=request.window_start_ms - 600_000.0, + ) + + +def test_plan_shards_crosses_router_chunks_with_planner_points(): + planners = [{LSD: 70}, {LSD: 80}, {LSD: 90}] + baseline = grid(SMALL_DOMAIN)[-1] + + shards = plan_shards( + SMALL_DOMAIN, + baseline=baseline, + startup=baseline, + max_shards=3, + parallelism=8, + planner_points=planners, + baseline_planner={LSD: 80}, + startup_planner={LSD: 80}, + ) + + grids = [shard for shard in shards if shard.kind == "grid"] + covered = [ + (json.dumps(point, sort_keys=True), planner[LSD]) + for shard in grids + for point in shard.points + for planner in shard.planners + ] + # Every router point with every planner point, once; 24 points in 3 + # shards of 8. + assert sorted(covered) == sorted( + (json.dumps(point, sort_keys=True), value) + for point in grid(SMALL_DOMAIN) + for value in (70, 80, 90) + ) + assert [shard.size for shard in grids] == [8, 8, 8] + (probe,) = [shard for shard in shards if shard.kind == "probe"] + assert probe.planners == ({LSD: 80},) + # The baseline is in the grid: no point shard. + assert [shard.kind for shard in shards].count("point") == 0 + + +def test_scenario_2_window_replays_the_inner_planner_from_the_window_start(tmp_path): + h = Harness(tmp_path) + history = FakeHistory() + request = scenario_2_request(_base_planner()) + + result = h.sweep(request, bucket_history=history, worker_pool=InlinePool) + + assert result.status == "ok", (result.error, result.details) + # Load scaling only: no predictor to warm, so no history is read. + assert history.calls == [] + bodies = h.created() + assert bodies + for body in bodies: + overrides = body["spec"]["aisim"]["inputOverrides"] + section = overrides["planner"]["retune_sweep"] + # The live config's scaling fields only: no connection settings. + assert section["base"]["enable_load_scaling"] is True + assert "metric_pulling_prometheus_endpoint" not in section["base"] + assert "namespace" not in section["base"] + assert section["warmup"] == {} + assert {point[LSD] for point in section["points"]} <= {70, 80, 90} + # Each inner planner starts from the ready counts at the window + # start, not the spec. + preset = overrides["engine"]["workers"]["aggregated"]["parallelism"]["preset"] + assert preset[0]["replicas"] == 2 + for candidate in request.candidates: + metrics = default_metrics(public(candidate.router)) + metrics["goodput_output_throughput_tok_s"] += candidate.planner[LSD] + expected, _ = score_summary(metrics, request.window_s, request.objective) + assert result.scores[candidate.key()] == pytest.approx(expected, rel=1e-12) + assert result.metrics[candidate.key()]["params"] == candidate.to_dict() + assert result.details["planner"]["points"] == 3 + + +def test_scenario_2_throughput_planner_carries_its_history_warmup(tmp_path): + h = Harness(tmp_path) + history = FakeHistory() + base = _base_planner( + optimization_target="sla", + enable_throughput_scaling=True, + enable_load_scaling=True, + ) + request = scenario_2_request(base) + + result = h.sweep(request, bucket_history=history, worker_pool=InlinePool) + + assert result.status == "ok", (result.error, result.details) + (call,) = history.calls + assert call["intervals"] == [base[THROUGHPUT_INTERVAL]] + assert call["history_start_ms"] == int(request.history_start_ms) + assert call["window"] == (WINDOW_START_MS, WINDOW_END_MS, AS_OF_MS) + interval = base[THROUGHPUT_INTERVAL] + for body in h.created(): + warmup = body["spec"]["aisim"]["inputOverrides"]["planner"]["retune_sweep"][ + "warmup" + ] + # Four history buckets, as TrafficObservation rows. + assert warmup == { + str(interval): [ + [float(interval), 10.0 + i, 100.0, 20.0, None, None] for i in range(4) + ] + } + # The decision's swap inputs: history and window at the interval. + assert len(result.observations_at(interval)) == 6 + assert result.predictor_warmup_errors == {} + + +def test_scenario_2_off_grid_running_planner_gets_its_own_shard(tmp_path): + # Grid {70, 90}, running 80: the planner axis is the grid's two points, + # as P13 counts it, and the baseline replays in a single-point shard + # instead of adding 80 to the axis of every grid shard. + h = Harness(tmp_path) + request = dataclasses.replace( + scenario_2_request(_base_planner()), + grid_planners=({LSD: 70}, {LSD: 90}), + ) + + result = h.sweep(request, bucket_history=FakeHistory(), worker_pool=InlinePool) + + assert result.status == "ok", (result.error, result.details) + assert result.details["planner"]["points"] == 2 + assert "point-baseline" in result.details["shards"] + grid_points = { + point[LSD] + for body in h.created() + if body["metadata"]["labels"][LABEL_SHARD].startswith("grid-") + for point in body["spec"]["aisim"]["inputOverrides"]["planner"]["retune_sweep"][ + "points" + ] + } + assert grid_points == {70, 90} + assert all(result.scores[c.key()] is not None for c in request.candidates) + + +def test_scenario_2_presweep_winner_keeps_the_planner_axis(tmp_path, monkeypatch): + # The pre-sweep picks kalman while the planner runs arima: the grid's + # points carry kalman and the baseline keeps arima, so the baseline is + # off the grid instead of doubling the planner axis. + h = Harness(tmp_path) + base = _base_planner( + optimization_target="sla", + enable_throughput_scaling=True, + enable_load_scaling=True, + load_predictor="arima", + ) + request = dataclasses.replace( + scenario_2_request(base), + grid_planners=({LSD: 70}, {LSD: 90}), + predictor_candidates=(("kalman_default", {"load_predictor": "kalman"}),), + ) + interval = str(base[THROUGHPUT_INTERVAL]) + monkeypatch.setattr( + dgdr, + "run_presweeps", + lambda *args: { + "by_interval": {interval: {"fields": {"load_predictor": "kalman"}}} + }, + ) + monkeypatch.setattr(sweep_module, "prepare_predictor", lambda *args: None) + + result = h.sweep(request, bucket_history=FakeHistory(), worker_pool=InlinePool) + + assert result.status == "ok", (result.error, result.details) + assert result.details["planner"]["points"] == 2 + assert "point-baseline" in result.details["shards"] + assert all(result.scores[c.key()] is not None for c in request.candidates) + + +@pytest.mark.parametrize( + "predictor_only", [False, True], ids=["router_only", "predictor_only"] +) +def test_scenario_2_without_planner_points_replays_the_running_planner( + tmp_path, monkeypatch, predictor_only +): + # The grid searches no planner field, so its one planner point is {}: + # every Run still replays the running planner (with the pre-sweep winner + # when the predictor is searched), and every candidate gets a score. + h = Harness(tmp_path) + running: dict[str, Any] = {} + if predictor_only: + base = _base_planner( + optimization_target="sla", + enable_throughput_scaling=True, + enable_load_scaling=True, + load_predictor="arima", + ) + running = {key: base[key] for key in PREDICTOR_KEYS} + else: + base = _base_planner() + request = make_request(planner=running) + request = dataclasses.replace( + request, + replicas={"aggregated": 2}, + base_planner=base, + history_start_ms=request.window_start_ms - 600_000.0, + grid_planners=({},), + ) + if predictor_only: + request = dataclasses.replace( + request, + predictor_candidates=(("kalman_default", {"load_predictor": "kalman"}),), + ) + interval = str(base[THROUGHPUT_INTERVAL]) + monkeypatch.setattr( + dgdr, + "run_presweeps", + lambda *args: { + "by_interval": {interval: {"fields": {"load_predictor": "kalman"}}} + }, + ) + monkeypatch.setattr(sweep_module, "prepare_predictor", lambda *args: None) + + result = h.sweep(request, bucket_history=FakeHistory(), worker_pool=InlinePool) + + assert result.status == "ok", (result.error, result.details) + assert result.details["planner"]["points"] == 1 + grid_points = [ + point + for body in h.created() + if body["metadata"]["labels"][LABEL_SHARD].startswith("grid-") + for point in body["spec"]["aisim"]["inputOverrides"]["planner"]["retune_sweep"][ + "points" + ] + ] + assert grid_points + if predictor_only: + assert {point["load_predictor"] for point in grid_points} == {"kalman"} + assert all(result.scores[c.key()] is not None for c in request.candidates) + + +def test_scenario_2_candidate_with_a_foreign_planner_point_is_unmapped(tmp_path): + h = Harness(tmp_path) + + def foreign(_name: str, parameters: dict) -> None: + parameters["adapters"]["dynamo.planner"]["retune_sweep"]["points"] = [{LSD: 55}] + + h.operator.mutate_parameters = foreign + result = h.sweep( + scenario_2_request(_base_planner()), + bucket_history=FakeHistory(), + worker_pool=InlinePool, + ) + + # The baseline is unscored, so the window cannot be decided. + assert result.details["unmapped_candidates"] + assert all(score is None for score in result.scores.values()) + + +def test_scenario_2_prepares_the_predictor_of_a_new_throughput_interval( + tmp_path, monkeypatch +): + # Candidates at 360 s next to the running 180 s: the history is read at + # both intervals, and the 360-s predictor a decision would swap in is + # prepared on that interval's observations. + h = Harness(tmp_path) + history = FakeHistory() + base = _base_planner( + optimization_target="sla", + enable_throughput_scaling=True, + enable_load_scaling=True, + load_predictor="constant", + ) + request = scenario_2_request(base) + slower = { + RetuneParams( + router=candidate.router, + planner={**candidate.planner, THROUGHPUT_INTERVAL: 360}, + ) + for candidate in request.candidates + if candidate != request.baseline + } + request = dataclasses.replace( + request, + candidates=tuple(sorted({*request.candidates, *slower}, key=RetuneParams.key)), + ) + + prepared = [] + monkeypatch.setattr( + sweep_module, + "prepare_predictor", + lambda config, _caps, observations: prepared.append( + (config.throughput_adjustment_interval_seconds, len(observations)) + ), + ) + + result = h.sweep(request, bucket_history=history, worker_pool=InlinePool) + + assert result.status == "ok", (result.error, result.details) + assert history.calls[0]["intervals"] == [180, 360] + # Only the new interval needs a predictor, on its 6 observations. + assert prepared == [(360, 6)] + warmup = h.created()[0]["spec"]["aisim"]["inputOverrides"]["planner"][ + "retune_sweep" + ]["warmup"] + assert sorted(warmup) == ["180", "360"] + assert sorted(result.interval_observations) == [180, 360] + assert result.predictor_warmup_errors == {} + scored = [key for key, score in result.scores.items() if score is not None] + assert len(scored) == len(request.candidates) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py index 58988605bdfd..a19d113a0272 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py @@ -217,7 +217,7 @@ def __getattr__(self, name: str): def make( - tmp_path: Path, *, keep_requests: int = 2 + tmp_path: Path, *, keep_requests: int = 2, history_s: float = 0.0 ) -> tuple[Reconciler, FakeKube, RecordingMetrics]: cfg = DgdrExecutorConfig( claim_name="traces", @@ -237,7 +237,14 @@ def make( kube = FakeKube() metrics = RecordingMetrics() reconciler = Reconciler( - kube, DGD, cfg, retune, str(tmp_path), metrics, now=lambda: NOW + kube, + DGD, + cfg, + retune, + str(tmp_path), + metrics, + now=lambda: NOW, + history_s=history_s, ) return reconciler, kube, metrics @@ -300,6 +307,21 @@ def test_segments_of_a_live_pod_keep_the_newest_and_recent_ones(tmp_path: Path) assert report.errors == {} +def test_a_scaling_planner_keeps_the_segments_of_its_history(tmp_path: Path) -> None: + # Sweeps also read history_s before the window (the inner planners' + # predictor warmup), so segments live that much longer. + reconciler, _, _ = make(tmp_path, history_s=3600.0) + pod_dir = Path(reconciler.segments_root) / "fe-0" + history = touch(pod_dir / "req.000000.jsonl.gz", SEGMENT_AGE + 1800) + older = touch(pod_dir / "req.000001.jsonl.gz", SEGMENT_AGE + 3601) + touch(pod_dir / "req.000002.jsonl.gz", 10) + + report = run_once(reconciler, pods=[frontend("fe-0")]) + + assert report.deleted_segments == 1 + assert history.exists() and not older.exists() + + def test_directory_of_a_gone_pod_goes_only_when_everything_is_old( tmp_path: Path, ) -> None: diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py index c8dcbdbf14a0..180c781dfd89 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py @@ -583,6 +583,8 @@ def __init__( retune_inputs=inputs, retune_hooks=hooks, ) + # The Planner's live configuration, as NativePlannerBase.config. + self.config = self._engine._config if self.controller is not None: # What the adapter's initial_tick does. self.controller.start(world.clock.time(), hooks.phase_s) @@ -590,11 +592,16 @@ def __init__( self.variant = "a" if inputs is None else variant_of(inputs) self.replacements: list[tuple[Any, int]] = [] self.fail_replace = False + # Decisions whose Planner fields the supervisor handed over. + self.persisted: list[Any] = [] @property def controller(self) -> Optional[TracedController]: return self._engine.retune_controller + def retune_decision_persisted(self, decision): + self.persisted.append(decision) + async def replace_live_retune(self, inputs, *, generation): async with self._config_lock: self.replacements.append((inputs, generation)) @@ -794,8 +801,9 @@ async def settle(processes: list[Process], world: World, rounds: int = 6) -> Non await process.sup.control_round() -def enqueue(process: Process, *decisions: RetuneDecision) -> None: - """Hand the supervisor a tick's decisions with the controller's routers.""" +def enqueue(process: Process, *decisions: RetuneDecision) -> list[RetuneDecision]: + """Hand the supervisor a tick's decisions with the controller's routers; + the ones it accepted.""" controller = process.controller desired = ( @@ -803,7 +811,7 @@ def enqueue(process: Process, *decisions: RetuneDecision) -> None: if controller is not None else startup_flat("a") ) - process.sup.on_effects( + return process.sup.on_effects( SimpleNamespace( retune_decisions=list(decisions), router_parameters=by_wt(process.host.variant, desired), @@ -1110,6 +1118,51 @@ async def test_applied_decision_is_persisted_then_deployed() -> None: assert sup.launch_gate(K0 + 1, window(K0 + 1)).reason == "already_decided" +def _with_planner_changes(process: Process, k: int) -> RetuneDecision: + """An applied decision with a Planner field, recorded in the controller + as its decide step does.""" + + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + process.controller.restore_desired(chosen) + changes = {"load_scaling_down_sensitivity": 70} + process.controller._pending_planner.update(changes) + return dataclasses.replace( + make_decision(k, "applied", chosen=chosen), planner_changes=changes + ) + + +async def test_planner_fields_apply_only_once_their_decision_is_persisted() -> None: + world = World() + process = await leading(world) + decision = _with_planner_changes(process, K0 + 1) + + enqueue(process, decision) + assert process.host.persisted == [] # queued, not written yet + await process.sup.outbox_round() + + assert world.shared_state().history[-1]["decision_id"] == decision.decision_id + assert process.host.persisted == [decision] + + +async def test_planner_fields_of_an_unwritten_decision_are_discarded() -> None: + world = World() + process = await leading(world) + sup = process.sup + store = StateStore(world.kube, sup.dgd) + snapshot = store.read() + store.write(dataclasses.replace(snapshot.state, leader_term="9/x/1"), snapshot) + decision = _with_planner_changes(process, K0 + 1) + + enqueue(process, decision) + await sup.outbox_round() # lost_term: not written + await sup.lease_round() # re-initializes and drops the outbox + + assert sup.outbox_size == 0 + assert process.host.persisted == [] + # The controller's current point is the running configuration again. + assert process.controller._pending_planner == {} + + async def test_advisory_decision_records_the_recommendation_only() -> None: world = World() process = await leading(world, advisory=True) @@ -1247,7 +1300,8 @@ def failures() -> list[str]: store = StateStore(world.kube, sup.dgd) snapshot = store.read() store.write(dataclasses.replace(snapshot.state, leader_term="9/x/1"), snapshot) - enqueue(process, make_decision(K0 + 1, "kept")) + decision = make_decision(K0 + 1, "kept") + assert enqueue(process, decision) == [decision] await sup.outbox_round() assert not sup.initialized and failures() == ["state_write"] * 3 @@ -1257,9 +1311,14 @@ async def test_decisions_are_dropped_and_rewound_without_leadership() -> None: process = await started(world) controller = process.controller controller._last_k = K0 + 1 - enqueue(process, make_decision(K0 + 1, "kept")) + # Dropped decisions are not handed back: the planner applies none of + # their planner changes, and the controller forgets them. + decision = _with_planner_changes(process, K0 + 1) + assert enqueue(process, make_decision(K0 + 1, "kept"), decision) == [] assert process.sup.outbox_size == 0 assert controller._last_k == -1 # no state: every window is due again + assert controller._pending_planner == {} + assert process.host.persisted == [] async def test_a_foreign_term_forces_reinitialization() -> None: @@ -1357,6 +1416,34 @@ async def test_restart_restores_the_persisted_point() -> None: assert second.sup.target().desired_by_wt == by_wt("a", chosen) +async def test_planner_changes_are_recorded_but_not_restored_after_a_restart( + caplog, +) -> None: + world = World() + first = await leading(world) + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + first.controller.restore_desired(chosen) + decision = dataclasses.replace( + make_decision(K0 + 1, "applied", chosen=chosen), + planner_changes={"load_scaling_down_sensitivity": 70}, + ) + enqueue(first, decision) + await first.sup.outbox_round() + assert world.shared_state().history[-1]["planner"] == { + "load_scaling_down_sensitivity": 70 + } + await first.sup.stop() + + with caplog.at_level(logging.WARNING, logger="dynamo.planner.retune.live"): + second = await leading(world, "p2") + + # The router point is restored; the Planner field runs its configured + # value, and the restart says so. + assert second.controller.current_point().router == chosen + assert "load_scaling_down_sensitivity': 70} of decision" in caplog.text + assert "are not restored after the restart" in caplog.text + + async def test_fingerprint_change_resets_with_a_two_phase_rollback() -> None: world = World() first = await leading(world) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py index c686a1b42be9..ab0b2d6f0351 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py @@ -1242,3 +1242,73 @@ def test_interrupt_kills_child_and_removes_its_temp_file(tmp_path, interrupt): assert caught.value.reason == expected assert time.monotonic() - started < 20 assert not temp.exists() + + +# ---------------------------------------------------------------- history + + +def test_history_is_bucketed_at_each_interval_from_the_segments(tmp_path): + # One row a second from 0 to 15 s, plus one in the window that completes + # after as_of; a damaged member between two good ones is skipped. + root = tmp_path / "segments" + rows = [_envelope(_row(f"r{i}", 1000 * i)) for i in range(15)] + censored = _envelope(_row("late", 14_900, total_ms=1000.0)) + _write( + root / "pod-a" / "req.000001.jsonl.gz", + _member(rows[:8]), + b"\x1f\x8b\x08 not a member", + _member([*rows[8:], censored]), + ) + + history = window_cut.bucket_history_inprocess( + str(root), 1000, START, END, AS_OF, [2, 5] + ) + + # 1..4 s of history; the window's rows count only toward observations. + assert history.history_requests == 4 + assert history.dropped_members == 1 + num_req = { + interval: [obs.num_req for obs in rows] + for interval, rows in history.observations.items() + } + assert num_req == {2: [2.0] * 7, 5: [4.0, 5.0, 5.0]} + warmups = { + interval: [obs.num_req for obs in rows] + for interval, rows in history.warmups.items() + } + # Buckets end at the window start: no window request is warmup. + assert warmups == {2: [2.0, 2.0], 5: [4.0]} + assert {obs.duration_s for obs in history.observations[5]} == {5.0} + + +def test_history_rows_need_no_block_hashes(tmp_path): + # History buckets read arrival, ISL and OSL only: a row without its + # replay hashes is counted, not a corrupt trace. + root = tmp_path / "segments" + record = _row("r1", 1000) + del record["request"]["replay"]["input_sequence_hashes"] + del record["request"]["replay"]["trace_block_size"] + _write(root / "pod-a" / "req.000001.jsonl.gz", _member([_envelope(record)])) + + history = window_cut.bucket_history_inprocess( + str(root), 1000, START, END, AS_OF, [5] + ) + + assert history.history_requests == 1 + + +def test_history_runs_in_a_child_that_the_deadline_stops(tmp_path): + root = tmp_path / "segments" + _write( + root / "pod-a" / "req.000001.jsonl.gz", _member([_envelope(_row("r", 6000))]) + ) + + history = window_cut.bucket_history( + str(root), START, START, END, AS_OF, [5], deadline_s=time.time() + 60 + ) + assert [obs.num_req for obs in history.observations[5]] == [1.0, 0.0] + with pytest.raises(WindowCutError) as caught: + window_cut.bucket_history( + str(root), START, START, END, AS_OF, [5], deadline_s=time.time() - 1 + ) + assert caught.value.reason == "window_cut_timeout" diff --git a/components/src/dynamo/planner/tests/unit/test_planner_provider_retune.py b/components/src/dynamo/planner/tests/unit/test_planner_provider_retune.py index 0ca6983fb3d0..24b6bfc063c5 100644 --- a/components/src/dynamo/planner/tests/unit/test_planner_provider_retune.py +++ b/components/src/dynamo/planner/tests/unit/test_planner_provider_retune.py @@ -261,3 +261,131 @@ def test_prediction_only_fields_need_an_enabled_policy_and_predict() -> None: for field in ({"retune": _RETUNE}, {"advisory": False}): with pytest.raises(ValueError, match="not supported by aisimulate recommend"): provider.compile_recommendation({"policy": "enabled", **field}, context) + + +def _recommendation_context() -> RecommendationAdapterContext: + return RecommendationAdapterContext( + engine={}, + traffic={}, + evaluation={}, + optimization={}, + sweep=SweepContext( + core_search_space={"deployment_mode": ["agg"]}, + workload={"trace_path": None, "isl": 512, "osl": 128}, + goal={"target": "goodput_per_gpu"}, + show_progress=False, + ), + ) + + +def _live_base(**overrides: Any) -> dict[str, Any]: + from dynamo.planner.simulation.config import retune_sweep_base + + config = PlannerConfig( + namespace="test-ns", + mode="agg", + optimization_target="sla", + enable_load_scaling=True, + enable_throughput_scaling=True, + ttft_ms=60.0, + itl_ms=100.0, + metric_pulling_prometheus_endpoint="http://prometheus:9090", + **overrides, + ) + return retune_sweep_base(config.model_dump(mode="json")) + + +def test_retune_sweep_replays_the_live_base_with_each_point() -> None: + from aisimulate.sweeper.provider import CandidateContext + from aisimulate.sweeper.replay import BackendDeploymentSpec + + provider = create_provider() + base = _live_base() + points = [ + {"load_scaling_down_sensitivity": 70}, + { + "load_scaling_down_sensitivity": 90, + "throughput_adjustment_interval_seconds": 360, + }, + ] + warmup = { + "180": [[180.0, 5.0, 100.0, 20.0, None, None]], + "360": [[360.0, 10.0, 100.0, 20.0, None, None]], + } + plan = provider.compile_recommendation( + {"retune_sweep": {"base": base, "points": points, "warmup": warmup}}, + _recommendation_context(), + ) + + assert plan.fragment.choices_by_branch == {"agg": {"retune_point": points}} + context = CandidateContext( + sample={"deployment_mode": "agg", "tp": 2, "attention_dp": 1}, + backend_deployment=BackendDeploymentSpec( + deployment_mode="agg", + backend="vllm", + backend_version="0.11.0", + agg_engine_args={"engine_type": "vllm"}, + num_workers=2, + ), + ) + spec = provider.materialize_candidate(plan, {"retune_point": points[1]}, context) + + # Candidate parameters are the section narrowed to the point and its + # own warmup; the hook carries the inner planner. + assert spec.config == { + "retune_sweep": { + "base": base, + "points": [points[1]], + "warmup": {"360": warmup["360"]}, + } + } + (hook,) = spec.runtime_hooks + planner = hook.config["planner_config"] + assert planner["load_scaling_down_sensitivity"] == 90 + assert planner["throughput_adjustment_interval_seconds"] == 360 + assert planner["advisory"] is False and planner.get("retune") is None + assert planner["decode_engine_num_gpu"] == 2 + assert "metric_pulling_prometheus_endpoint" not in planner + # The warmup of the point's own throughput interval. + assert hook.config["warmup_observations"] == warmup["360"] + config = PlannerConfig.from_config_arg(json.dumps(planner)) + assert config.ttft_ms == 60.0 and config.scaling_enabled() + # aisimulate predict replays the candidate from its config alone. + again = provider.compile_prediction(spec.config, _context(sla=_SLA)) + assert again.config == spec.config + assert again.runtime_hooks[0].config["warmup_observations"] == warmup["360"] + assert again.runtime_hooks[0].config["planner_config"]["decode_engine_num_gpu"] == 1 + with pytest.raises(ValueError, match="replays one planner.retune_sweep point"): + provider.compile_prediction( + {"retune_sweep": {"base": base, "points": points}}, _context(sla=_SLA) + ) + + +@pytest.mark.parametrize( + ("section", "match"), + [ + ({"base": {"namespace": "x"}, "points": [{}]}, r"cannot set \['namespace'\]"), + ( + {"base": {}, "points": [{"max_gpu_budget": 4}]}, + "sets \\['max_gpu_budget'\\]", + ), + ({"base": {}, "points": []}, "at least 1 item"), + ( + { + "base": { + "optimization_target": "sla", + "enable_load_scaling": False, + "enable_throughput_scaling": False, + }, + "points": [{}], + }, + "At least one scaling mode must be enabled", + ), + ], + ids=["base-field", "point-field", "no-points", "no-scaling"], +) +def test_retune_sweep_rejects_what_a_live_planner_never_sends(section, match) -> None: + with pytest.raises(ValueError, match=match): + create_provider().compile_recommendation( + {"retune_sweep": section}, _recommendation_context() + ) diff --git a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py index 7bd51b744aa1..b5f54d4bc6c5 100644 --- a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py +++ b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio +import dataclasses import gzip import json import threading @@ -22,6 +23,8 @@ from dynamo.planner.core.types import ( EngineCapabilities, FpmObservations, + RetuneDecision, + RetuneParams, ScheduledTick, TickInput, TrafficObservation, @@ -847,6 +850,169 @@ async def sleep(seconds): assert sleeps == [1.0, 2.0] +def _applied(changes, warmup=None): + return RetuneDecision( + decision_id="retune-w1", + outcome="applied", + window_index=0, + window_start_s=0.0, + window_end_s=60.0, + as_of_s=65.0, + launched_at_s=65.0, + decided_at_s=70.0, + num_requests=10, + num_candidates=2, + previous=RetuneParams(), + chosen=RetuneParams(planner=dict(changes)), + baseline_score=1.0, + best_score=2.0, + planner_changes=dict(changes), + predictor_warmup=warmup, + ) + + +async def _apply_decisions(planner, decisions): + # What the live supervisor does once each decision is persisted. + for decision in decisions: + planner.retune_decision_persisted(decision) + await asyncio.gather(*planner._retune_apply_tasks) + + +async def test_live_retune_applies_planner_changes_through_runtime_reconfiguration(): + planner = sla_planner(enable_load_scaling=True) + engine = planner._ensure_engine() + advisory = dataclasses.replace( + _applied({"load_scaling_down_sensitivity": 70}), outcome="advisory" + ) + + await _apply_decisions( + planner, [advisory, _applied({"load_scaling_down_sensitivity": 90})] + ) + + # The advisory decision changes nothing; the applied one is a new revision. + assert planner.config.load_scaling_down_sensitivity == 90 + assert planner._config_generation == 1 + assert "load_propose" in engine._builtins + await planner._shutdown_runtime() + + +async def test_live_retune_drops_a_planner_change_overtaken_while_preparing( + monkeypatch, +): + # A decision needs a new predictor for 360 s; while it is prepared, a + # control API update activates another revision. The decision is dropped + # instead of installing a predictor warmed for another configuration. + planner = _planner("disagg", optimization_target="sla", load_predictor="constant") + engine = planner._ensure_engine() + discarded = [] + + class Controller: + def discard_planner_changes(self, changes): + discarded.append(dict(changes)) + + engine._retune = Controller() + real_prepare = reconfiguration.prepare_predictor + + def prepare(config, capabilities, observations): + predictor = real_prepare(config, capabilities, observations) + # The control API's activation, between the two locked sections. + planner._config_generation += 1 + planner.config.load_scaling_down_sensitivity = 70 + return predictor + + monkeypatch.setattr(reconfiguration, "prepare_predictor", prepare) + changes = {"throughput_adjustment_interval_seconds": 360} + warmup = tuple(TrafficObservation(360.0, 10.0, 100.0, 10.0) for _ in range(8)) + + await _apply_decisions(planner, [_applied(changes, warmup)]) + + assert planner.config.throughput_adjustment_interval_seconds == 180 + assert planner.config.load_scaling_down_sensitivity == 70 + assert planner._config_generation == 1 + assert discarded == [changes] + engine._retune = None + await planner._shutdown_runtime() + + +async def test_live_retune_and_the_control_api_take_turns(monkeypatch): + # One update prepares at a time, a control API one or a live decision's: + # the other is rejected instead of fitting a second predictor. + planner = _planner("disagg", optimization_target="sla", load_predictor="constant") + engine = planner._ensure_engine() + discarded = [] + + class Controller: + def discard_planner_changes(self, changes): + discarded.append(dict(changes)) + + engine._retune = Controller() + release = threading.Event() + real_prepare = reconfiguration.prepare_predictor + + def slow_prepare(config, capabilities, observations): + release.wait(5) + return real_prepare(config, capabilities, observations) + + monkeypatch.setattr(reconfiguration, "prepare_predictor", slow_prepare) + changes = {"throughput_adjustment_interval_seconds": 360} + warmup = tuple(TrafficObservation(360.0, 10.0, 100.0, 10.0) for _ in range(8)) + planner.retune_decision_persisted(_applied(changes, warmup)) + while planner.reconfiguration._preparation is None: + await asyncio.sleep(0) + + # The decision prepares its predictor: a control API update waits. + with pytest.raises(ConfigurationConflict, match="still preparing"): + await planner.reconfiguration.submit(request(planner, {"ttft_ms": 250.0})) + release.set() + await asyncio.gather(*planner._retune_apply_tasks) + assert planner.config.throughput_adjustment_interval_seconds == 360 + assert planner._config_generation == 1 + + # A control API update prepares: the decision is dropped. + release.clear() + await planner.reconfiguration.submit( + request(planner, {"load_predictor": "kalman"}, cold_start=True) + ) + other = {"load_scaling_down_sensitivity": 70} + await _apply_decisions(planner, [_applied(other)]) + assert discarded == [other] + release.set() + assert (await finish(planner))["state"] == "applied" + assert planner.config.load_scaling_down_sensitivity != 70 + engine._retune = None + await planner._shutdown_runtime() + + +async def test_live_retune_keeps_the_config_when_a_planner_change_cannot_apply(): + planner = _planner( + "disagg", + optimization_target="sla", + load_predictor="kalman", + kalman_min_points=20, + ) + engine = planner._ensure_engine() + discarded = [] + + class Controller: + def discard_planner_changes(self, changes): + discarded.append(dict(changes)) + + engine._retune = Controller() + # A new throughput interval needs a warmed predictor; one bucket is too + # few for a Kalman filter that needs 20. + changes = {"throughput_adjustment_interval_seconds": 360} + warmup = (TrafficObservation(360.0, 10.0, 100.0, 10.0),) + + await _apply_decisions(planner, [_applied(changes, warmup)]) + + assert planner.config.throughput_adjustment_interval_seconds == 180 + assert planner._config_generation == 0 + # The controller's current point is the running configuration again. + assert discarded == [changes] + engine._retune = None + await planner._shutdown_runtime() + + async def test_run_loop_replans_the_pending_tick_after_an_activation(monkeypatch): class Engine: def __init__(self): diff --git a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py index d69e29db7ab3..afffedeff021 100644 --- a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py +++ b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py @@ -586,7 +586,7 @@ def evaluator(*args): def run_with_winner(fields): presweep = {"winner": "w", "fields": fields, "losses": {"w": 0.1}} - monkeypatch.setattr(sweep_module, "_run_presweeps", lambda *_: presweep) + monkeypatch.setattr(sweep_module, "run_presweeps", lambda *_: presweep) tasks.clear() return sweep_module.run_sweep(request, evaluator=evaluator) @@ -804,7 +804,9 @@ def importing_worker(): timer.start() try: with pytest.raises(sweep_module._SweepInterrupted, match="cancelled"): - sweep_module._run_presweeps(request, pool, inputs, [60]) + sweep_module.run_presweeps( + request, pool, inputs.observations, inputs.observations_paths, [60] + ) finally: timer.cancel() pool.close() diff --git a/components/src/dynamo/replay/simulation.py b/components/src/dynamo/replay/simulation.py index 251e033199b4..ce7db7f9e3fd 100644 --- a/components/src/dynamo/replay/simulation.py +++ b/components/src/dynamo/replay/simulation.py @@ -33,6 +33,7 @@ from dynamo.llm import AisPerfConfig, KvRouterConfig from dynamo.mocker.config import normalize_mocker_config +from dynamo.planner.core.types import TrafficObservation from dynamo.replay.api import ( TelemetryOptions, run_synthetic_trace_replay, @@ -145,6 +146,7 @@ def run( ais_perf_config, conversation_affinity, ) = self._resolve_hooks(spec.runtime_hooks) + planner_warmup = self._planner_warmup(spec.runtime_hooks) common: dict[str, Any] = { "router_mode": router_mode, "router_config": router_config, @@ -170,6 +172,12 @@ def run( **self._goodput_sla_kwargs(spec), } + if planner_warmup is not None: + # A live retune sweep's candidate warms its load predictor with + # the recorded history before the window. + if not self._is_trace(spec): + raise ValueError("planner warmup observations require a trace replay") + common["_planner_warmup_observations"] = planner_warmup if conversation_affinity is not None: if not self._is_trace(spec): raise ValueError("conversation_affinity requires an agentic trace") @@ -203,6 +211,27 @@ def close(self) -> None: between candidates. """ + @staticmethod + def _planner_warmup( + hooks: tuple[RuntimeHookSpec, ...], + ) -> list[TrafficObservation] | None: + """The Planner hook's ``warmup_observations`` rows, if it has any.""" + + for hook in hooks: + if not _PLANNER_HOOK.supports(hook): + continue + rows = hook.config.get("warmup_observations") + if rows is None: + return None + if not isinstance(rows, list) or not all( + isinstance(row, list) for row in rows + ): + raise TypeError( + "Dynamo Planner hook warmup_observations must be a list of rows" + ) + return [TrafficObservation(*row) for row in rows] + return None + @staticmethod def _resolve_hooks( hooks: tuple[RuntimeHookSpec, ...], diff --git a/components/src/dynamo/replay/tests/e2e/configs/unified_cli/fixtures/traces/dynamo-retune-window.jsonl b/components/src/dynamo/replay/tests/e2e/configs/unified_cli/fixtures/traces/dynamo-retune-window.jsonl new file mode 100644 index 000000000000..276d67304e8c --- /dev/null +++ b/components/src/dynamo/replay/tests/e2e/configs/unified_cli/fixtures/traces/dynamo-retune-window.jsonl @@ -0,0 +1,60 @@ +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":1010,"request":{"request_id":"window-0","output_tokens":4,"request_received_ms":1000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[0,1]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":2010,"request":{"request_id":"window-1","output_tokens":4,"request_received_ms":2000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[10,11]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":3010,"request":{"request_id":"window-2","output_tokens":4,"request_received_ms":3000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[20,21]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":4010,"request":{"request_id":"window-3","output_tokens":4,"request_received_ms":4000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[30,31]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":5010,"request":{"request_id":"window-4","output_tokens":4,"request_received_ms":5000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[40,41]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":6010,"request":{"request_id":"window-5","output_tokens":4,"request_received_ms":6000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[50,51]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":7010,"request":{"request_id":"window-6","output_tokens":4,"request_received_ms":7000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[60,61]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":8010,"request":{"request_id":"window-7","output_tokens":4,"request_received_ms":8000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[70,71]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":9010,"request":{"request_id":"window-8","output_tokens":4,"request_received_ms":9000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[80,81]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":10010,"request":{"request_id":"window-9","output_tokens":4,"request_received_ms":10000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[90,91]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":11010,"request":{"request_id":"window-10","output_tokens":4,"request_received_ms":11000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[100,101]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":12010,"request":{"request_id":"window-11","output_tokens":4,"request_received_ms":12000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[110,111]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":13010,"request":{"request_id":"window-12","output_tokens":4,"request_received_ms":13000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[120,121]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":14010,"request":{"request_id":"window-13","output_tokens":4,"request_received_ms":14000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[130,131]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":15010,"request":{"request_id":"window-14","output_tokens":4,"request_received_ms":15000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[140,141]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":16010,"request":{"request_id":"window-15","output_tokens":4,"request_received_ms":16000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[150,151]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":17010,"request":{"request_id":"window-16","output_tokens":4,"request_received_ms":17000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[160,161]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":18010,"request":{"request_id":"window-17","output_tokens":4,"request_received_ms":18000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[170,171]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":19010,"request":{"request_id":"window-18","output_tokens":4,"request_received_ms":19000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[180,181]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":20010,"request":{"request_id":"window-19","output_tokens":4,"request_received_ms":20000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[190,191]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":21010,"request":{"request_id":"window-20","output_tokens":4,"request_received_ms":21000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[200,201]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":22010,"request":{"request_id":"window-21","output_tokens":4,"request_received_ms":22000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[210,211]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":23010,"request":{"request_id":"window-22","output_tokens":4,"request_received_ms":23000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[220,221]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":24010,"request":{"request_id":"window-23","output_tokens":4,"request_received_ms":24000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[230,231]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":25010,"request":{"request_id":"window-24","output_tokens":4,"request_received_ms":25000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[240,241]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":26010,"request":{"request_id":"window-25","output_tokens":4,"request_received_ms":26000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[250,251]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":27010,"request":{"request_id":"window-26","output_tokens":4,"request_received_ms":27000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[260,261]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":28010,"request":{"request_id":"window-27","output_tokens":4,"request_received_ms":28000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[270,271]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":29010,"request":{"request_id":"window-28","output_tokens":4,"request_received_ms":29000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[280,281]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":30010,"request":{"request_id":"window-29","output_tokens":4,"request_received_ms":30000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[290,291]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":31010,"request":{"request_id":"window-30","output_tokens":4,"request_received_ms":31000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[300,301]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":32010,"request":{"request_id":"window-31","output_tokens":4,"request_received_ms":32000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[310,311]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":33010,"request":{"request_id":"window-32","output_tokens":4,"request_received_ms":33000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[320,321]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":34010,"request":{"request_id":"window-33","output_tokens":4,"request_received_ms":34000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[330,331]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":35010,"request":{"request_id":"window-34","output_tokens":4,"request_received_ms":35000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[340,341]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":36010,"request":{"request_id":"window-35","output_tokens":4,"request_received_ms":36000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[350,351]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":37010,"request":{"request_id":"window-36","output_tokens":4,"request_received_ms":37000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[360,361]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":38010,"request":{"request_id":"window-37","output_tokens":4,"request_received_ms":38000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[370,371]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":39010,"request":{"request_id":"window-38","output_tokens":4,"request_received_ms":39000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[380,381]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":40010,"request":{"request_id":"window-39","output_tokens":4,"request_received_ms":40000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[390,391]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":41010,"request":{"request_id":"window-40","output_tokens":4,"request_received_ms":41000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[400,401]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":42010,"request":{"request_id":"window-41","output_tokens":4,"request_received_ms":42000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[410,411]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":43010,"request":{"request_id":"window-42","output_tokens":4,"request_received_ms":43000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[420,421]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":44010,"request":{"request_id":"window-43","output_tokens":4,"request_received_ms":44000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[430,431]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":45010,"request":{"request_id":"window-44","output_tokens":4,"request_received_ms":45000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[440,441]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":46010,"request":{"request_id":"window-45","output_tokens":4,"request_received_ms":46000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[450,451]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":47010,"request":{"request_id":"window-46","output_tokens":4,"request_received_ms":47000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[460,461]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":48010,"request":{"request_id":"window-47","output_tokens":4,"request_received_ms":48000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[470,471]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":49010,"request":{"request_id":"window-48","output_tokens":4,"request_received_ms":49000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[480,481]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":50010,"request":{"request_id":"window-49","output_tokens":4,"request_received_ms":50000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[490,491]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":51010,"request":{"request_id":"window-50","output_tokens":4,"request_received_ms":51000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[500,501]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":52010,"request":{"request_id":"window-51","output_tokens":4,"request_received_ms":52000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[510,511]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":53010,"request":{"request_id":"window-52","output_tokens":4,"request_received_ms":53000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[520,521]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":54010,"request":{"request_id":"window-53","output_tokens":4,"request_received_ms":54000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[530,531]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":55010,"request":{"request_id":"window-54","output_tokens":4,"request_received_ms":55000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[540,541]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":56010,"request":{"request_id":"window-55","output_tokens":4,"request_received_ms":56000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[550,551]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":57010,"request":{"request_id":"window-56","output_tokens":4,"request_received_ms":57000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[560,561]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":58010,"request":{"request_id":"window-57","output_tokens":4,"request_received_ms":58000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[570,571]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":59010,"request":{"request_id":"window-58","output_tokens":4,"request_received_ms":59000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[580,581]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":60010,"request":{"request_id":"window-59","output_tokens":4,"request_received_ms":60000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[590,591]}}} diff --git a/components/src/dynamo/replay/tests/e2e/configs/unified_cli/recommend/dynamo/07-live-retune-sweep-goodput.yaml b/components/src/dynamo/replay/tests/e2e/configs/unified_cli/recommend/dynamo/07-live-retune-sweep-goodput.yaml new file mode 100644 index 000000000000..4ceebda5af1b --- /dev/null +++ b/components/src/dynamo/replay/tests/e2e/configs/unified_cli/recommend/dynamo/07-live-retune-sweep-goodput.yaml @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# A live retune sweep (retune.executor: dgdr) as its DGDR Request lowers it: +# the window trace, two router temperatures, and the Planner's configuration +# with two load sensitivities. Each candidate's inner planner scales from the +# window start, warmed with the 20-s history buckets. +traffic: + source: + type: trace + format: dynamo + paths: [components/src/dynamo/replay/tests/e2e/configs/unified_cli/fixtures/traces/dynamo-retune-window.jsonl] + load: {type: trace_timestamps} +engine: + mode: aggregated + model: components/src/dynamo/replay/tests/e2e/configs/unified_cli/fixtures/tiny-model + hardware: h200_sxm + backend: vllm + context_length: 2048 + workers: + aggregated: + parallelism: + preset: + - {replicas: 2, tensor: 1, pipeline: 1, attention_data: 1, moe_tensor: 1, moe_expert: 1} + scheduler: {max_batched_tokens: 8192, max_sequences: 256} + kv_cache: {block_size: 64, prefix_caching: true, capacity: {type: fixed, blocks: 256}} + timing: {type: fixed, prefill_ms: 1, decode_ms: 1} +router: + policy: kv_router + prefill_load_model: {type: none} + overlap_score_credit: 1.0 + prefill_load_scale: 1.0 + temperature: {choices: [0.0, 0.5]} +planner: + retune_sweep: + base: + mode: agg + optimization_target: sla + ttft_ms: 10.0 + itl_ms: 5.0 + enable_throughput_scaling: true + enable_load_scaling: true + throughput_adjustment_interval_seconds: 20 + load_adjustment_interval_seconds: 5 + max_gpu_budget: 4 + min_endpoint: 1 + load_predictor: constant + scheduling: {scale_interval_seconds: 5} + points: + - {load_scaling_down_sensitivity: 70} + - {load_scaling_down_sensitivity: 90} + warmup: + "20": + - [20.0, 18.0, 128.0, 4.0, null, null] + - [20.0, 19.0, 128.0, 4.0, null, null] + - [20.0, 20.0, 128.0, 4.0, null, null] + - [20.0, 21.0, 128.0, 4.0, null, null] +evaluation: + sla: {ttft_ms: 10.0, itl_ms: 5.0} +optimization: + target: goodput_per_gpu + constraints: {max_candidate_gpus: 2} + strict_sla: false +optimizer: {algorithm: random, max_trials: 8, parallelism: 1, candidate_timeout_seconds: 60.0, seed: 23} diff --git a/components/src/dynamo/replay/tests/test_simulation.py b/components/src/dynamo/replay/tests/test_simulation.py index 23725992c6d5..b32a5e756029 100644 --- a/components/src/dynamo/replay/tests/test_simulation.py +++ b/components/src/dynamo/replay/tests/test_simulation.py @@ -204,6 +204,47 @@ def fake_run_trace_replay(**kwargs): assert report.metadata["planner_total_ticks"] == 7 +def test_planner_hook_warmup_reaches_the_inner_planner(monkeypatch) -> None: + # A live retune sweep's candidate warms its load predictor with the + # history rows its planner hook carries. + seen = {} + + def fake_run_trace_replay(**kwargs): + seen.update(kwargs) + return _report({"goodput_output_throughput_tok_s": 40.0}, total_ticks=1) + + monkeypatch.setattr(simulation, "run_trace_replay", fake_run_trace_replay) + rows = [[180.0, 12.0, 100.0, 20.0, None, None], [180.0, 9.0, 90.0, 25.0, 0.5, None]] + spec = ReplaySpec( + backend_deployment=_agg_deployment(), + workload={"trace_path": "window.jsonl.gz", "trace_format": "dynamo"}, + goal={"target": "goodput", "sla": {"ttft_ms": 100.0, "itl_ms": 20.0}}, + adapters={ + "dynamo.planner": AdapterReplaySpec( + runtime_hooks=( + RuntimeHookSpec( + provider="dynamo.planner", + kind="scaling_policy", + api_version=1, + config={ + "planner_config": {"mode": "agg"}, + "warmup_observations": rows, + }, + ), + ) + ) + }, + ) + + simulation.DynamoReplayRunnerFactory().create(0).run(spec) + + warmup = seen["_planner_warmup_observations"] + assert [(obs.duration_s, obs.num_req, obs.kv_hit_rate) for obs in warmup] == [ + (180.0, 12.0, None), + (180.0, 9.0, 0.5), + ] + + def test_trace_paths_only_workload_routes_to_trace_replay(monkeypatch) -> None: seen = {} diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md index 4d4318bbfd27..4fbb4fb8b105 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md @@ -296,8 +296,8 @@ In aggregated mode (`mode: agg`), engines handle both prefill and decode via chu > [!WARNING] > **Experimental.** With `retune.executor: local`, the default, retune runs only in > planner-in-the-loop offline replay, and a live Planner rejects it at startup. With -> `executor: dgdr`, a live Planner on Kubernetes retunes three KV router knobs with fixed replicas -> through DGDR v1beta2 Requests; it is for development clusters only. See +> `executor: dgdr`, a live Planner on Kubernetes retunes three KV router knobs, and with scaling +> its own Planner fields, through DGDR v1beta2 Requests; it is for development clusters only. See > [Planner Live Retune](../../../../kubernetes/auto-deployment/planner-live-retune.md). Retune periodically replays the last window of recorded traffic against a grid of router and @@ -562,7 +562,12 @@ trace that ends at or before the first replayed request; without throughput scal `runtime_decision.router_parameters`. The AISimulate Planner adapter (`planner/simulation/provider.py`) passes `planner.retune` through to -`PlannerConfig` and fills the scoring SLA from `evaluation.sla`. `dynamo.replay` (`run_trace_replay`, +`PlannerConfig` and fills the scoring SLA from `evaluation.sla`. For live retune with scaling, it +also reads `planner.retune_sweep` in `aisimulate recommend`: the live Planner's scaling fields +(`base`), the Planner points to search (`points`) and the history warm-up by throughput interval +(`warmup`). Each candidate replays `base` with its point applied, warmed with the rows of its own +interval, which `DynamoReplayRunner` hands to `run_trace_replay`; its concrete config is the +section narrowed to that point, which `aisimulate predict` replays again. `dynamo.replay` (`run_trace_replay`, and `prepare_planner_replay` for synthetic traffic) rejects traffic that is not an open-loop Dynamo request trace, and its error names the AISimulate setting. diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md index 341aa7f03ee5..2f009120c332 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md @@ -331,8 +331,8 @@ spec: > **Experimental.** `retune.executor` selects where retune runs. With `local`, the default, it runs > only in offline simulation: `aisimulate predict --stack dynamo` and > `dynamo.replay.run_trace_replay`, and a live Planner rejects it at startup. With `dgdr`, a live -> Planner on Kubernetes retunes three KV router knobs with fixed replicas through DGDR v1beta2 -> Requests; it is for development clusters only. See +> Planner on Kubernetes retunes three KV router knobs, and with scaling its own Planner fields, +> through DGDR v1beta2 Requests; it is for development clusters only. See > [Planner Live Retune](../../../../kubernetes/auto-deployment/planner-live-retune.md). Retune lets the Planner tune its own runtime parameters from the traffic it served. Every diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md index 762e5f76b20a..4a80c45e0484 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: Planner Live Retune -subtitle: Let the Planner tune the KV router of a running DynamoGraphDeployment from its own recorded traffic, through DGDR v1beta2 search runs. +subtitle: Let the Planner tune the KV router and its own scaling parameters on a running DynamoGraphDeployment from its recorded traffic, through DGDR v1beta2 search runs. --- > [!CAUTION] @@ -14,36 +14,64 @@ subtitle: Let the Planner tune the KV router of a running DynamoGraphDeployment Live retune is the Kubernetes form of [Periodic Retune](../../developer-guide/knowledge-base/modular-components/planner/planner-guide.md#periodic-retune-experimental). Every `retune.interval_seconds`, the Planner cuts the window of request traces that just ended, creates DynamoGraphDeploymentRequests (DGDR) v1beta2 that replay the window for a grid of KV -router parameters, scores the candidates the Runs report on `retune.objective`, and keeps the -current parameters unless a candidate beats them by more than `retune.min_improvement_percentage`. -Outside advisory mode it then sets the winner on every frontend router with -`PATCH /router_policy`. - -Live retune searches the three router knobs a DGDR Request can express: -`overlap_score_credit`, `prefill_load_scale` and `temperature`. Replicas stay fixed: the Planner -runs in retune-only mode and never scales. +router and Planner parameters, scores the candidates the Runs report on `retune.objective`, and +keeps the current parameters unless a candidate beats them by more than +`retune.min_improvement_percentage`. Outside advisory mode it then sets the winning router +parameters on every frontend router with `PATCH /router_policy` and applies the winning Planner +parameters to itself. + +Live retune searches the three router knobs a DGDR Request can express, +`overlap_score_credit`, `prefill_load_scale` and `temperature`, in one of two setups: + +- **Fixed replicas.** The Planner runs in retune-only mode (`enable_load_scaling` and + `enable_throughput_scaling` both `false`) and never scales. Each candidate replays with the + replica counts of the DGD spec. +- **Scaling.** The Planner scales on load, throughput or both, and `retune.planner` may also + search its runtime parameters, the scaling policy included (see + [Search domains](../../reference/components/planner-configuration.mdx#search-domains)). Each + candidate replays the Planner's own configuration with the candidate's fields, starting from + the replica counts that were ready at the window start, as + [replay retune](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#candidates-and-the-sweep) + does. This needs an operator whose configured planner image includes this release: the Runs' + AISimulate Planner adapter must read the Requests' `planner.retune_sweep` section. ## How it works 1. **Record.** Each frontend writes a `request_end` record per request to its own directory on a shared ReadWriteMany claim (`///`). -2. **Cut a window.** At `L_k = window end + grace_seconds`, the Planner merges the records of the +2. **Read the history** (scaling only). At `L_k`, before any Request is created, the Planner + reads the records of the `history_seconds` before the window and buckets them at each + throughput interval a candidate scales at. With throughput scaling it runs the load-predictor + pre-sweep on them and prepares every predictor a decision could swap in. This takes another + `cut_budget_seconds` at most. +3. **Cut a window.** At `L_k = window end + grace_seconds`, the Planner merges the records of the window into one canonical trace file, `//-.jsonl.gz`, in a subprocess bounded by `cut_budget_seconds`. `` is a digest of the window's start, end and cutoff (`window end + grace_seconds`), so a Planner restarted with another `interval_seconds` or `grace_seconds` cuts its own file and creates its own Requests instead of reusing those of a window with the same end. -3. **Search.** The Planner splits the router grid into at most `run.max_shards` Cartesian shards of - at most 100 points and creates one DGDR v1beta2 Request per shard. Each Request replays the - window file on the pinned `retune.dgdr.engine` with the replica counts of the DGD spec. One more - single-point Request replays a grid point again as a common-random-numbers probe: its score - must match the grid's within a relative error of 1e-9. -4. **Decide.** The Planner reads each Run's Candidates, scores them on `retune.objective`, and +4. **Search.** The Planner splits the grid into at most `run.max_shards` Cartesian shards of at + most 100 points and creates one DGDR v1beta2 Request per shard. With scaling, the Planner + points are one more shard dimension. Each Request replays the window file on the pinned + `retune.dgdr.engine`, with the replica counts of the DGD spec (fixed replicas) or those ready + at the window start (scaling). With scaling, it also carries the Planner's scaling + configuration, the shard's Planner points and the history warm-up in + `spec.aisim.inputOverrides.planner.retune_sweep`. One more single-point Request replays a grid + point again as a common-random-numbers probe: its score must match the grid's within a + relative error of 1e-9. +5. **Decide.** The Planner reads each Run's Candidates, scores them on `retune.objective`, and decides with the same rules as replay retune. A decision is persisted in a Lease before it reaches any router. -5. **Apply.** The Planner sets all eight router policy parameters of every frontend router to - the decided values, and keeps them there: a manual change to them is reverted. +6. **Apply.** The Planner sets all eight router policy parameters of every frontend router to + the decided values, and keeps them there: a manual change to them is reverted. Decided Planner + parameters take effect in the Planner itself once the decision is persisted, through its + [runtime reconfiguration](../../reference/components/planner-configuration.mdx#system-port-reconfiguration-api), + with a new load predictor warmed on the window's history when the decision needs one. They + share the control API's one-update-at-a-time rule: the decision's Planner parameters are + dropped when another update is still preparing, or activates while their predictor is warmed. + A Planner that restarts runs its configured values again: retune does not restore applied + Planner parameters, and logs the ones it lost. Windows are aligned to `retune.dgdr.phase_seconds + k × interval_seconds` in epoch time, so a restarted Planner continues the same windows. Leadership and state live in two Leases owned by @@ -149,7 +177,7 @@ environment: kubernetes backend: vllm mode: agg advisory: true # start advisory; see Roll out -enable_load_scaling: false # retune-only: both flags false +enable_load_scaling: false # fixed replicas: both flags false enable_throughput_scaling: false retune: interval_seconds: 21600 # T: window length and decision cadence @@ -217,6 +245,28 @@ retune: For the other `retune` fields and the limits that `executor: dgdr` adds to them, see the [Retune field reference](../../reference/components/planner-configuration.mdx#retune-experimental). +To let the Planner scale and retune its own parameters, enable the scaling modes and add a +`planner` domain. A `router` domain is still required, since every Request sets the three router +knobs; pin them with scalars to the frontend's startup values to retune the Planner alone: + +```yaml +optimization_target: sla +ttft_ms: 500 +itl_ms: 30 +enable_load_scaling: true +enable_throughput_scaling: true +retune: + # interval_seconds, timeout_seconds, executor and dgdr as above + router: + overlap_score_credit: 1.0 # pinned to the frontend's startup values + prefill_load_scale: 1.0 + temperature: 0.0 + planner: + scaling_policy: {preset: [hybrid_180_5, hybrid_600_5]} + load_sensitivity: {preset: false} + load_scaling_down_sensitivity: {choices: [70, 90]} +``` + Live retune takes its scoring SLA when it starts: `retune.sla_ttft_ms` and `retune.sla_itl_ms`, or the Planner's `ttft_ms` and `itl_ms` where they are unset. An update of `ttft_ms` or `itl_ms` through the @@ -237,8 +287,9 @@ AISimulate searches by default, would make the Run search engine settings as wel `scheduler.max_sequences`. - The preset entry may also set `prefill_context`, but only to `1`, and must not set `decode_context`: the pinned AISimulate recommendation does not run context parallelism. -- Do not set `replicas` anywhere: the Planner fills them in from the DGD spec at each launch and - sets `spec.hardware.gpu.budget` to their GPU total. +- Do not set `replicas` anywhere: the Planner fills them in at each launch, from the DGD spec with + fixed replicas or from the ready counts at the window start with scaling, and sets + `spec.hardware.gpu.budget` to their GPU total. - Write roles the mode does not use as `null`, or leave them out. - Set the model and GPU with `retune.dgdr.model` and `retune.dgdr.sku`, not in the engine. - Leave `backend_version` out to let the pinned AISimulate pick its version for @@ -255,7 +306,8 @@ need(n) = cut_budget_seconds + job_startup_seconds + (1 + ceil((n - 1) / run.parallelism)) x run.candidate_timeout_seconds ``` -AISimulate evaluates the first candidate alone, hence the `1 +`. The Planner rejects a +AISimulate evaluates the first candidate alone, hence the `1 +`. With scaling, a shard also needs +another `cut_budget_seconds` for the history read and the pre-sweep. The Planner rejects a configuration whose largest grid shard needs more than `timeout_seconds`, because the baseline can fall in any shard and a window without a baseline score fails. With the defaults, the 108-point grid splits into 3 shards of 36 points and needs 900 + 300 + 6 × 1800 = 12000 seconds. Measure @@ -271,10 +323,10 @@ configuration error at startup. | Rule | Requirement | |---|---| | P1 | `environment: kubernetes`; `decision_latency_seconds` is `0`, or `measured` with `decision_latency_scale: 1`; `timeout_seconds` is set and at most `interval_seconds - 600` | -| P2 | `enable_load_scaling` and `enable_throughput_scaling` are both `false`; `retune.planner` is not set | +| P2 | `retune.router` is set; to retune only the Planner, pin its three knobs with scalars | | P3 | `retune.router` writes only `overlap_score_credit`, `prefill_load_scale`, `temperature`, `policy: kv_router` and `prefill_load_model: {type: none}` | | P5 | `retune.dgdr.engine` is all scalars, as in [Engine section](#engine-section) | -| P13 | The largest grid shard has at most 100 points and fits the [time budget](#time-budget) | +| P13 | The largest grid shard has at most 100 points and fits the [time budget](#time-budget). With scaling, the Planner points count as one more shard dimension, and the history before a window holds at most 1024 buckets of the shortest throughput interval | **Deployment**, checked at startup, when the DGD changes and before every router round. While one fails, the Planner launches no windows, resets the routers to their startup parameters, and @@ -455,7 +507,9 @@ The Planner exports these on `metric_reporting_prometheus_port`: | Not supported | Reason | |---|---| -| Retune with load or throughput scaling, or of Planner parameters | A DGDR Run has no base Planner configuration, warm-up or predictor pre-sweep | +| Retune with scaling on an operator whose planner image predates this release | The Run's AISimulate Planner adapter rejects `planner.retune_sweep`, so every Run fails | +| Restoring applied Planner parameters after a Planner restart | They live in the Planner process, and a load predictor needs a history warm-up; the next window starts from the configured values | +| Load-predictor history beyond 1024 buckets of the shortest throughput interval | The warm-up travels inside every Request and Candidate | | Frontends with non-default routing fields beyond the three knobs, or `kv_event_lag_ms` other than 0 | The DGDR router section expresses only the three knobs, and the Run does not pass a lag | | Workers that set their own router mode (`--router-mode` or `DYN_ROUTER_MODE`) | A worker's advertised router configuration replaces the frontend's for its workers, and the Run replays only the frontend's | | Worker arguments that pass a ConfigMap, Secret, annotation or `envFrom` value where an option could stand, including an unquoted `$VAR` of one in a worker shell script or one that `eval` or a shell given `-c` parses again (P4 fails) | The Planner cannot read the value, so it cannot rule out a router mode the worker would advertise | diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index c52382980132..7635b401611e 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -665,7 +665,7 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p **Experimental.** Where retune runs depends on `executor`: - `local`, the default, runs only inside planner-in-the-loop offline replay: `dynamo.replay.run_trace_replay` with `trace_format="dynamo"`, or `aisimulate predict --stack dynamo` on a Dynamo request trace. A live Planner rejects it at startup. -- `dgdr` runs on a live Planner on Kubernetes, through DGDR v1beta2 Requests. It retunes three KV router knobs with fixed replicas and is meant for development clusters only; see [Planner Live Retune](../../kubernetes/auto-deployment/planner-live-retune.md). +- `dgdr` runs on a live Planner on Kubernetes, through DGDR v1beta2 Requests. It retunes three KV router knobs, with fixed replicas or with scaling, and with scaling the Planner fields of `retune.planner` too. It is meant for development clusters only; see [Planner Live Retune](../../kubernetes/auto-deployment/planner-live-retune.md). @@ -726,7 +726,7 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p Where sweeps run. `local` replays the candidates in worker processes of the Planner and runs only inside planner-in-the-loop replay; a live Planner rejects it at startup with `planner.retune is replay-only with executor 'local'; use executor: dgdr`. `dgdr` runs each sweep as DGDR v1beta2 Requests from a live Planner on Kubernetes and requires the `dgdr` section below. **Experimental**, development clusters only: see [Planner Live Retune](../../kubernetes/auto-deployment/planner-live-retune.md). - With `dgdr`, the Planner also rejects at startup a config that breaks a [configuration rule](../../kubernetes/auto-deployment/planner-live-retune.md#deployment-rules) of live retune: it requires `environment: kubernetes`, retune-only mode with no `planner` domain, a `router` domain that writes only `overlap_score_credit`, `prefill_load_scale`, `temperature`, `policy: kv_router` and `prefill_load_model: {type: none}`, the `timeout_seconds` and `decision_latency_seconds` limits above, a `dgdr.engine` that follows the rules below, and a largest grid shard of at most 100 points that fits the [time budget](../../kubernetes/auto-deployment/planner-live-retune.md#time-budget). + With `dgdr`, the Planner also rejects at startup a config that breaks a [configuration rule](../../kubernetes/auto-deployment/planner-live-retune.md#deployment-rules) of live retune: it requires `environment: kubernetes`, a `router` domain (pin its knobs with scalars to retune only the Planner) that writes only `overlap_score_credit`, `prefill_load_scale`, `temperature`, `policy: kv_router` and `prefill_load_model: {type: none}`, the `timeout_seconds` and `decision_latency_seconds` limits above, a `dgdr.engine` that follows the rules below, and a largest grid shard of at most 100 points that fits the [time budget](../../kubernetes/auto-deployment/planner-live-retune.md#time-budget). With scaling, the Planner points are one more shard dimension, the history read takes another `cut_budget_seconds`, and the history before a window may hold at most 1024 buckets of the shortest throughput interval. Allowed values: local dgdr