From 1c1ed551b94baf28a7910f6dcccebbe308b98b75 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Wed, 4 Nov 2026 11:51:27 -0800 Subject: [PATCH 1/8] feat(planner): search the scaling policy in retune, not user constraints Retune no longer searches worker minimums and GPU budgets: they are user constraints, rejected in retune.planner like the SLA and still changeable through the Planner config and the runtime control API. The scaling_policy group (both scaling flags and both intervals) becomes a retune knob and a runtime field: - retune.planner expands to a list of canonical points: invalid policies (both modes off, load interval not below the throughput interval, an interval the scale interval does not divide) are dropped, fields a point's modes make inactive fold to the running value, and equal points are kept once. Retune-only mode still rejects retune.planner. - The replay sweep buckets history at every throughput interval in use, runs the predictor pre-sweep per interval, warms each inner planner at its own interval, and prepares a swap for each interval whose candidate needs a new predictor. - A running engine re-registers its builtin plugins when the policy changes, resets the throughput lower bounds when throughput scaling turns off, and requires a prepared predictor when the throughput interval changes or throughput scaling turns on. - The run loop re-reads its poll interval, and the FPM freshness window follows the load interval through FpmEventSubscriber.set_max_age, a shared atomic the running cleanup task reads too. Refs DYN-5400 Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/config/planner_config.py | 57 +- components/src/dynamo/planner/core/base.py | 26 +- .../dynamo/planner/core/planner_factory.py | 4 +- .../src/dynamo/planner/core/state_machine.py | 9 + .../metrics_provider/runtime_provider.py | 11 + .../dynamo/planner/offline/replay_adapter.py | 9 +- .../plugins/orchestrator/engine_adapter.py | 176 +++++-- .../plugins/orchestrator/orchestrator.py | 5 + .../dynamo/planner/plugins/registry/server.py | 34 ++ .../src/dynamo/planner/reconfiguration.py | 74 ++- .../src/dynamo/planner/retune/controller.py | 86 +-- .../src/dynamo/planner/retune/domains.py | 409 +++++++++++++-- components/src/dynamo/planner/retune/sweep.py | 493 ++++++++++++------ .../src/dynamo/planner/retune/window.py | 16 +- .../src/dynamo/planner/simulation/config.py | 89 ++-- .../orchestrator/test_engine_adapter.py | 105 ++++ .../unit/test_planner_provider_retune.py | 23 + .../tests/unit/test_planner_retune_config.py | 23 +- .../tests/unit/test_reconfiguration.py | 129 ++++- .../planner/tests/unit/test_retune_config.py | 40 +- .../tests/unit/test_retune_controller.py | 121 ++++- .../planner/tests/unit/test_retune_domains.py | 233 ++++++++- .../planner/tests/unit/test_retune_sweep.py | 117 ++++- .../dynamo/replay/tests/test_retune_replay.py | 70 +++ .../planner/planner-design.md | 65 ++- .../planner/planner-guide.md | 14 +- .../components/planner-configuration.mdx | 62 ++- lib/bindings/python/rust/llm/fpm.rs | 101 +++- lib/bindings/python/src/dynamo/_core.pyi | 18 +- 29 files changed, 2154 insertions(+), 465 deletions(-) diff --git a/components/src/dynamo/planner/config/planner_config.py b/components/src/dynamo/planner/config/planner_config.py index 784b198460f7..5d296d1d53ef 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/components/src/dynamo/planner/config/planner_config.py @@ -2451,12 +2451,12 @@ def _validate_retune(self, retune: RetuneConfig) -> None: planner_domain = expand_planner_domain(self) router_domain = expand_router_domain(retune) - if self.enable_throughput_scaling and retune.history_seconds is not None: + if retune.history_seconds is not None: required = required_history_seconds(self, planner_domain) if retune.history_seconds < required: raise ValueError( f"retune.history_seconds must be >= {required:g}s " - "(throughput_adjustment_interval_seconds x " + "(the largest throughput_adjustment_interval_seconds x " "max(prophet_window_size, kalman_min_points) over the " "current predictor and the load_predictor candidates) to " "warm throughput predictors" @@ -2466,14 +2466,16 @@ def _validate_retune(self, retune: RetuneConfig) -> None: raise ValueError( "retune requires a non-empty router or planner search domain" ) - size = grid_size(router_domain, planner_domain.grid) + size = grid_size(router_domain) * len(planner_domain.points) if size > retune.max_candidates: + description = describe_grid( + router_domain, planner_domain.grid, len(planner_domain.points) + ) raise ValueError( f"retune grid has {size} candidates, exceeding " f"retune.max_candidates={retune.max_candidates} " - f"({describe_grid(router_domain, planner_domain.grid)}); an " - "omitted AISimulate knob searches its default domain, so pin " - "knobs with a scalar to shrink the grid" + f"({description}); an omitted AISimulate knob searches its " + "default domain, so pin knobs with a scalar to shrink the grid" ) if retune.executor == "dgdr": self._validate_live_retune_budget(retune, router_domain) @@ -2648,36 +2650,33 @@ def _validate_retune_planner_grid( self, grid: Mapping[str, Sequence[Any]], public_names: Mapping[str, str], + *, + load_scaling: Optional[bool] = None, + throughput_scaling: Optional[bool] = None, ) -> dict[str, list[Any]]: """Reject planner grid fields that are inert in this configuration. ``grid`` is keyed by ``PlannerConfig`` field; ``public_names`` gives - the ``retune.planner`` name to report. Each value must pass the - field's strict validation, as a runtime reconfiguration would. - Returns the grid with validated values (``80`` becomes ``80.0`` for a - float field), so candidates compare equal to the live config by key. + the ``retune.planner`` name to report. ``load_scaling`` and + ``throughput_scaling`` say whether some grid point scales that way + (default: this config does). Each value must pass the field's strict + validation, as a runtime reconfiguration would. Returns the grid with + validated values (``80`` becomes ``80.0`` for a float field), so + candidates compare equal to the live config by key. """ - active_endpoints = ( - {"min_endpoint"} - if self.mode == "agg" - else {"prefill_min_endpoint", "decode_min_endpoint"} - ) + if load_scaling is None: + load_scaling = self.enable_load_scaling + if throughput_scaling is None: + throughput_scaling = self.enable_throughput_scaling validated: dict[str, list[Any]] = {} for key, values in grid.items(): name = f"retune.planner.{public_names.get(key, key)}" - if key in ("min_endpoint", "prefill_min_endpoint", "decode_min_endpoint"): - relevant = key in active_endpoints - requirement = ( - "mode='agg' (use prefill_min_workers/decode_min_workers in disagg)" - if key == "min_endpoint" - else "mode='disagg' (use min_workers in agg)" - ) - elif key == "max_throughput_scaling_replicas": - relevant = self.enable_throughput_scaling + if key == "max_throughput_scaling_replicas": + relevant = throughput_scaling requirement = "enable_throughput_scaling" elif key == "load_scaling_down_sensitivity": - relevant = self.enable_load_scaling + relevant = load_scaling requirement = "enable_load_scaling" elif key in ( "prefill_scale_up_queue_tokens", @@ -2689,7 +2688,8 @@ def _validate_retune_planner_grid( relevant = self.optimization_target == "load" requirement = "optimization_target='load'" else: - # min_gpu_budget / max_gpu_budget apply to any scaling mode. + # The scaling-policy fields: expand_planner_domain keeps only + # the policies that apply. relevant = True requirement = "" if not relevant: @@ -2736,6 +2736,11 @@ def from_config_arg(cls, config_arg: str) -> "PlannerConfig": return cls.model_validate(data) + @property + def fpm_max_age_seconds(self) -> float: + """Freshness window of live FPM samples: three load intervals.""" + return 3.0 * self.load_adjustment_interval_seconds + @classmethod def _load_from_file(cls, path: Path) -> "PlannerConfig": suffix = path.suffix.lower() diff --git a/components/src/dynamo/planner/core/base.py b/components/src/dynamo/planner/core/base.py index 258b7dbbc861..ce185810ae0e 100644 --- a/components/src/dynamo/planner/core/base.py +++ b/components/src/dynamo/planner/core/base.py @@ -607,10 +607,24 @@ def apply_runtime_candidate( self._config_generation += 1 self.prometheus_metrics.sla_target_ttft_ms.set(self.config.ttft_ms) self.prometheus_metrics.sla_target_itl_ms.set(self.config.itl_ms) + self._sync_fpm_max_age() logger.info( "Activated planner configuration revision %d", self._config_generation ) + def _sync_fpm_max_age(self) -> None: + """Keep the live FPM freshness window at three load intervals after a + runtime change of ``load_adjustment_interval_seconds``; the + subscribers keep their samples.""" + provider = getattr(self.environment, "fpm_provider", None) + setter = getattr(provider, "set_max_fpm_age_seconds", None) + if not callable(setter): + return + age = self.config.fpm_max_age_seconds + if getattr(provider, "max_fpm_age_seconds", None) != age: + setter(age) + logger.info("FPM freshness window is now %.1fs", age) + @asynccontextmanager async def _bounded_config_lock(self) -> AsyncIterator[None]: """Acquire the tick decision lock or ask the caller to retry.""" @@ -1314,14 +1328,16 @@ async def _run_one_tick( async def run(self) -> None: engine = self._ensure_engine() next_tick = engine.initial_tick(time.time()) - poll_interval = self.config.load_adjustment_interval_seconds / 10 - gpu_shape_retry_interval = min( - max(poll_interval, _GPU_SHAPE_RETRY_MIN_SECONDS), - _GPU_SHAPE_RETRY_MAX_SECONDS, - ) try: while True: + # Read every iteration: a runtime reconfiguration may change + # the load interval. + poll_interval = self.config.load_adjustment_interval_seconds / 10 + gpu_shape_retry_interval = min( + max(poll_interval, _GPU_SHAPE_RETRY_MIN_SECONDS), + _GPU_SHAPE_RETRY_MAX_SECONDS, + ) now = time.time() if now < next_tick.at_s: await asyncio.sleep(min(next_tick.at_s - now, poll_interval)) diff --git a/components/src/dynamo/planner/core/planner_factory.py b/components/src/dynamo/planner/core/planner_factory.py index 6e95053103a0..01ba0e3970de 100644 --- a/components/src/dynamo/planner/core/planner_factory.py +++ b/components/src/dynamo/planner/core/planner_factory.py @@ -86,7 +86,7 @@ def construct_environment( backend=config.backend, model_name=config.model_name, runtime=runtime, - max_fpm_age_seconds=3.0 * config.load_adjustment_interval_seconds, + max_fpm_age_seconds=config.fpm_max_age_seconds, ) connector = construct_connector( config, @@ -132,7 +132,7 @@ def construct_environment( backend=config.backend, model_name=config.model_name, runtime=runtime, - max_fpm_age_seconds=3.0 * config.load_adjustment_interval_seconds, + max_fpm_age_seconds=config.fpm_max_age_seconds, state_source=environment, namespace_source=namespace_binding, ) diff --git a/components/src/dynamo/planner/core/state_machine.py b/components/src/dynamo/planner/core/state_machine.py index 7b04701e9f5e..5b4bfb503af3 100644 --- a/components/src/dynamo/planner/core/state_machine.py +++ b/components/src/dynamo/planner/core/state_machine.py @@ -209,6 +209,15 @@ def begin_tick(self) -> None: """Reset per-tick diagnostics before builtin plugins run.""" self._reset_diag() + def reset_throughput_lower_bounds(self) -> None: + """Forget the last throughput targets that floor load scaling. + + For a planner whose throughput scaling turns off at runtime: load + scaling would otherwise stay floored at a target nothing updates. + """ + self._throughput_lower_bound_p = 1 + self._throughput_lower_bound_d = 1 + def observe_worker_counts(self, counts: WorkerCounts) -> None: self._update_inventory(counts) diff --git a/components/src/dynamo/planner/environment/metrics_provider/runtime_provider.py b/components/src/dynamo/planner/environment/metrics_provider/runtime_provider.py index fda0a3415470..df5eda16986d 100644 --- a/components/src/dynamo/planner/environment/metrics_provider/runtime_provider.py +++ b/components/src/dynamo/planner/environment/metrics_provider/runtime_provider.py @@ -62,6 +62,17 @@ def bind_sources( self.state_source = state_source self.namespace_source = namespace_source + def set_max_fpm_age_seconds(self, max_fpm_age_seconds: float) -> None: + """Change the FPM freshness window of both subscribers in place. + + Tracked samples are kept; reads and the subscribers' clean-up use the + new age from now on. Subscribers created later start with it. + """ + for subscriber in (self._prefill_fpm_sub, self._decode_fpm_sub): + if subscriber is not None: + subscriber.set_max_age(max_age_seconds=max_fpm_age_seconds) + self.max_fpm_age_seconds = max_fpm_age_seconds + async def async_init(self, namespace: Optional[str] = None) -> None: del namespace await self._rebind_subscribers() diff --git a/components/src/dynamo/planner/offline/replay_adapter.py b/components/src/dynamo/planner/offline/replay_adapter.py index a3c979cabc2c..d0cd0f51d548 100644 --- a/components/src/dynamo/planner/offline/replay_adapter.py +++ b/components/src/dynamo/planner/offline/replay_adapter.py @@ -79,8 +79,8 @@ from dynamo.planner.plugins.clock import VirtualClock from dynamo.planner.plugins.orchestrator.engine_adapter import OrchestratorEngineAdapter from dynamo.planner.reconfiguration import ( - PREDICTOR_FIELDS, candidate_config, + predictor_rebuild_required, prepare_predictor, runtime_candidate_errors, ) @@ -494,10 +494,9 @@ def _apply_retune_decisions(self, decisions: Sequence[RetuneDecision]) -> None: f"planner configuration: {'; '.join(errors)}" ) predictor = None - if any( - getattr(candidate, field) != getattr(self._config, field) - for field in PREDICTOR_FIELDS - ): + if predictor_rebuild_required(self._config, candidate): + # The sweep bucketed the warmup at the candidate's throughput + # interval, as the swapped-in predictor steps. predictor = prepare_predictor( candidate, capabilities, decision.predictor_warmup ) diff --git a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py index f39f06b1d479..5d46277edb1c 100644 --- a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py +++ b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py @@ -517,55 +517,115 @@ def retune_sweeps_done(self) -> Optional[Future[None]]: def plugins_bootstrapped(self) -> bool: return self._plugins_bootstrapped - def _register_builtin_plugins(self) -> None: - """Register the in-process builtins that implement local planning.""" + def _builtin_registrations(self) -> dict[str, dict[str, Any]]: + """``register_internal`` arguments (without the instance) of every + builtin the current scaling policy runs, keyed by builtin name.""" cfg = self._config throughput_interval = float(cfg.throughput_adjustment_interval_seconds) load_interval = float(cfg.load_adjustment_interval_seconds) - + registrations: dict[str, dict[str, Any]] = {} if cfg.enable_throughput_scaling: - predict = BuiltinLoadPredict(cfg, self._capabilities) - self._builtins["load_predict"] = predict - self._orchestrator.register_internal( - plugin_id=predict.plugin_id, - plugin_type="predict", - priority=0, - instance=predict, - execution_interval_seconds=throughput_interval, - needs=["observations.traffic"], - observation_window_seconds=throughput_interval, - ) - - throughput = BuiltinThroughputPropose(cfg, self._scaling_state) - self._builtins["throughput_propose"] = throughput - self._orchestrator.register_internal( - plugin_id=throughput.plugin_id, - plugin_type="propose", - priority=20, - instance=throughput, - execution_interval_seconds=throughput_interval, - needs=["observations.traffic", "predictions"], - requires_produced_fields=["predictions"], - observation_window_seconds=throughput_interval, - ) - + registrations["load_predict"] = { + "plugin_type": "predict", + "priority": 0, + "execution_interval_seconds": throughput_interval, + "needs": ["observations.traffic"], + "requires_produced_fields": [], + "observation_window_seconds": throughput_interval, + } + registrations["throughput_propose"] = { + "plugin_type": "propose", + "priority": 20, + "execution_interval_seconds": throughput_interval, + "needs": ["observations.traffic", "predictions"], + "requires_produced_fields": ["predictions"], + "observation_window_seconds": throughput_interval, + } if cfg.enable_load_scaling: - load = BuiltinLoadPropose(cfg, self._scaling_state) - self._builtins["load_propose"] = load needs = ["observations.fpm", "observations.workers"] observation_window = 0.0 if cfg.optimization_target == "sla" and not cfg.enable_throughput_scaling: needs.append("observations.traffic") observation_window = load_interval + registrations["load_propose"] = { + "plugin_type": "propose", + "priority": 10, + "execution_interval_seconds": load_interval, + "needs": needs, + "requires_produced_fields": [], + "observation_window_seconds": observation_window, + } + return registrations + + def _new_builtin(self, name: str) -> Any: + if name == "load_predict": + return BuiltinLoadPredict(self._config, self._capabilities) + if name == "throughput_propose": + return BuiltinThroughputPropose(self._config, self._scaling_state) + return BuiltinLoadPropose(self._config, self._scaling_state) + + def _register_builtin_plugins(self) -> None: + """Register the in-process builtins that implement local planning.""" + for name, registration in self._builtin_registrations().items(): + instance = self._new_builtin(name) + self._builtins[name] = instance self._orchestrator.register_internal( - plugin_id=load.plugin_id, - plugin_type="propose", - priority=10, - instance=load, - execution_interval_seconds=load_interval, - needs=needs, - observation_window_seconds=observation_window, + plugin_id=instance.plugin_id, instance=instance, **registration + ) + + def _reconcile_builtin_plugins( + self, predictor: Optional[BuiltinLoadPredict] + ) -> None: + """Make the registered builtins match the current scaling policy. + + A builtin the policy no longer runs is unregistered; one it newly + runs is registered as at startup, so it first fires one interval + after registration; one it keeps is updated in place (interval, + observation window and needs), keeping its state and phase. A new + or kept load predictor takes ``predictor``'s prepared forecasts when + given. Turning throughput scaling off also drops the throughput + lower bounds, which would otherwise keep flooring load scaling at + the last throughput target. + """ + desired = self._builtin_registrations() + for name in [name for name in self._builtins if name not in desired]: + removed: Any = self._builtins.pop(name) + self._orchestrator.unregister_internal( + removed.plugin_id, reason="scaling_policy" + ) + for name, registration in desired.items(): + instance: Any = self._builtins.get(name) + if instance is None: + instance = self._new_builtin(name) + if name == "load_predict" and predictor is not None: + instance.install_prepared_predictors(predictor) + self._builtins[name] = instance + self._orchestrator.register_internal( + plugin_id=instance.plugin_id, instance=instance, **registration + ) + continue + if name == "load_predict" and predictor is not None: + instance.install_prepared_predictors(predictor) + plugin = self._orchestrator._registry.get_plugin(instance.plugin_id) + if plugin is None: # pragma: no cover - registered above or at startup + raise RuntimeError(f"builtin {name} is not registered") + plugin.execution_interval_seconds = registration[ + "execution_interval_seconds" + ] + plugin.observation_window_seconds = registration[ + "observation_window_seconds" + ] + plugin.needs = list(registration["needs"]) + plugin.requires_produced_fields = list( + registration["requires_produced_fields"] ) + if not self._config.enable_throughput_scaling: + self._scaling_state.reset_throughput_lower_bounds() + for component in ("prefill", "decode"): + self._orchestrator.set_throughput_lower_bound(component, 1) + # The loop pointers restart with the policy, as at startup. + self._last_load_loop_monotonic = self._last_tick_monotonic + self._last_throughput_loop_monotonic = self._last_tick_monotonic # ------------------------------------------------------------------ # Bootstrap API (delegates to orchestrator) @@ -771,14 +831,42 @@ def apply_runtime_configuration( fields: Iterable[str], predictor: BuiltinLoadPredict | None, ) -> None: - active_predictor: BuiltinLoadPredict | None = None + """Activate ``candidate``'s ``fields`` in the running engine. + + A scaling-policy change re-registers the builtins + (``_reconcile_builtin_plugins``). ``predictor`` is the prepared load + predictor; it is required when the candidate needs a new one + (``reconfiguration.predictor_rebuild_required``: throughput scaling + turning on, or a predictor field or the throughput interval + changing). Everything is checked before anything changes. + """ + # Imported on use: reconfiguration imports the planner core. + from dynamo.planner.reconfiguration import ( + SCALING_POLICY_FIELDS, + predictor_rebuild_required, + ) + + fields = list(fields) if predictor is not None: - active = self._builtins["load_predict"] - if not isinstance(active, BuiltinLoadPredict): + if not isinstance(predictor, BuiltinLoadPredict): raise TypeError( "runtime prediction updates require the builtin load predictor" ) - active_predictor = active + if not candidate.enable_throughput_scaling: + raise ValueError( + "a prepared load predictor requires throughput scaling" + ) + elif predictor_rebuild_required(self._config, candidate): + raise ValueError( + "this configuration change needs a new load predictor " + "(throughput scaling turned on, or a predictor field or " + "throughput_adjustment_interval_seconds changed); prepare one" + ) + active = self._builtins.get("load_predict") + if active is not None and not isinstance(active, BuiltinLoadPredict): + raise TypeError( + "runtime prediction updates require the builtin load predictor" + ) changed = { field for field in fields @@ -786,8 +874,10 @@ def apply_runtime_configuration( } for field in changed: setattr(self._config, field, getattr(candidate, field)) - if active_predictor is not None and predictor is not None: - active_predictor.install_prepared_predictors(predictor) + if changed & SCALING_POLICY_FIELDS: + self._reconcile_builtin_plugins(predictor) + elif active is not None and predictor is not None: + active.install_prepared_predictors(predictor) if changed or predictor is not None: self._orchestrator.invalidate_runtime_configuration() self._startup_down_candidate = None diff --git a/components/src/dynamo/planner/plugins/orchestrator/orchestrator.py b/components/src/dynamo/planner/plugins/orchestrator/orchestrator.py index 34dda632c986..6d931e8e452c 100644 --- a/components/src/dynamo/planner/plugins/orchestrator/orchestrator.py +++ b/components/src/dynamo/planner/plugins/orchestrator/orchestrator.py @@ -193,6 +193,11 @@ def register_internal( observation_window_seconds=observation_window_seconds, ) + def unregister_internal(self, plugin_id: str, reason: str = "") -> bool: + """Synchronously unregister an in-process plugin (see + ``PluginRegistryServer.unregister_internal``).""" + return self._registry.unregister_internal(plugin_id, reason=reason) + def list_plugins( self, request: Optional[ListPluginsRequest] = None ) -> list[PluginInfo]: diff --git a/components/src/dynamo/planner/plugins/registry/server.py b/components/src/dynamo/planner/plugins/registry/server.py index 35ab301a930c..330945fc0d0f 100644 --- a/components/src/dynamo/planner/plugins/registry/server.py +++ b/components/src/dynamo/planner/plugins/registry/server.py @@ -381,6 +381,40 @@ async def unregister(self, plugin_id: str, reason: str = "") -> bool: ) return True + def unregister_internal(self, plugin_id: str, reason: str = "") -> bool: + """Synchronously unregister an in-process plugin. + + For the engine's runtime reconfiguration, which runs without an + await point. An in-process transport holds nothing to close, so this + is ``unregister`` without the transport close; any other transport + raises (use ``unregister``). + """ + + plugin = self._plugins.get(plugin_id) + if plugin is None: + return False + if plugin.transport_type != "in_process": + raise ValueError( + f"unregister_internal: plugin_id={plugin_id!r} is not in-process" + ) + del self._plugins[plugin_id] + self._circuit_breaker.reset(plugin_id) + for cb in list(self._unregister_callbacks): + try: + cb(plugin_id, reason) + except Exception as exc: # noqa: BLE001 + log.warning( + "unregister: on_unregister callback failed plugin_id=%s detail=%s", + plugin_id, + exc, + ) + log.info( + "unregister_internal plugin_id=%s reason=%s", + plugin_id, + reason or "", + ) + return True + def list_plugins(self, req: ListPluginsRequest) -> list[PluginInfo]: """Return plugin metadata filtered by ``stage_filter`` and ``include_disabled``. Full observability fields diff --git a/components/src/dynamo/planner/reconfiguration.py b/components/src/dynamo/planner/reconfiguration.py index a8874a0d243f..e4bf01d85bc0 100644 --- a/components/src/dynamo/planner/reconfiguration.py +++ b/components/src/dynamo/planner/reconfiguration.py @@ -46,21 +46,36 @@ "kalman_min_points", } ) -RUNTIME_FIELDS = PREDICTOR_FIELDS | { - "min_endpoint", - "prefill_min_endpoint", - "decode_min_endpoint", - "min_gpu_budget", - "max_gpu_budget", - "max_throughput_scaling_replicas", - "ttft_ms", - "itl_ms", - "load_scaling_down_sensitivity", - "prefill_scale_up_queue_tokens", - "prefill_scale_down_queue_tokens", - "decode_scale_up_kv_rate", - "decode_scale_down_kv_rate", -} +# The scaling policy. A change re-registers the builtin plugins of the running +# engine; see ``predictor_rebuild_required`` for when it also needs a new +# load predictor. +SCALING_POLICY_FIELDS = frozenset( + { + "enable_throughput_scaling", + "enable_load_scaling", + "throughput_adjustment_interval_seconds", + "load_adjustment_interval_seconds", + } +) +RUNTIME_FIELDS = ( + PREDICTOR_FIELDS + | SCALING_POLICY_FIELDS + | { + "min_endpoint", + "prefill_min_endpoint", + "decode_min_endpoint", + "min_gpu_budget", + "max_gpu_budget", + "max_throughput_scaling_replicas", + "ttft_ms", + "itl_ms", + "load_scaling_down_sensitivity", + "prefill_scale_up_queue_tokens", + "prefill_scale_down_queue_tokens", + "decode_scale_up_kv_rate", + "decode_scale_down_kv_rate", + } +) PREPARATION_TIMEOUT_SECONDS = 120.0 OPERATION_HISTORY_SIZE = 32 MAX_WARMUP_TRACE_BYTES = 16 * 1024 * 1024 @@ -278,6 +293,27 @@ def live_retune_sla_errors( ] +def predictor_rebuild_required( + current: PlannerConfig, candidate: PlannerConfig +) -> bool: + """Whether activating ``candidate`` needs a newly prepared load predictor. + + Only a candidate with throughput scaling has a predictor. It needs a new + one when throughput scaling turns on, when a predictor field changes, or + when ``throughput_adjustment_interval_seconds`` changes: the interval is + the predictor's step and the bucket length of its history. + """ + + if not candidate.enable_throughput_scaling: + return False + if not current.enable_throughput_scaling: + return True + return any( + getattr(candidate, field) != getattr(current, field) + for field in (*PREDICTOR_FIELDS, "throughput_adjustment_interval_seconds") + ) + + def prepare_predictor( config: PlannerConfig, capabilities: WorkerCapabilities, @@ -432,7 +468,8 @@ async def submit(self, request: ReconfigureRequest) -> dict[str, Any]: candidate = candidate_config(self.planner.config, request.config) self.planner.validate_runtime_candidate(candidate) changed_predictor = ( - any( + predictor_rebuild_required(self.planner.config, candidate) + or any( getattr(candidate, field) != getattr(self.planner.config, field) for field in PREDICTOR_FIELDS ) @@ -453,7 +490,10 @@ async def submit(self, request: ReconfigureRequest) -> dict[str, Any]: ) if request.warmup_trace is None and not request.cold_start: raise ValueError( - "predictor replacement requires warmup_trace or explicit cold_start: true" + "predictor replacement requires warmup_trace or explicit " + "cold_start: true (a predictor field or " + "throughput_adjustment_interval_seconds changed, or " + "throughput scaling turned on)" ) operation = { "operation_id": request.request_id, diff --git a/components/src/dynamo/planner/retune/controller.py b/components/src/dynamo/planner/retune/controller.py index 1f0b263e85df..9b85e52f3d68 100644 --- a/components/src/dynamo/planner/retune/controller.py +++ b/components/src/dynamo/planner/retune/controller.py @@ -72,11 +72,14 @@ from dynamo.planner.reconfiguration import ( PREDICTOR_FIELDS, candidate_config, + predictor_rebuild_required, runtime_candidate_errors, ) from dynamo.planner.retune.domains import ( LOAD_SCALING_DOWN_SENSITIVITY, OVERLAP_SCORE_CREDIT, + SCALING_POLICY_KEYS, + PlannerDomain, PredictorCandidate, expand_planner_domain, expand_router_domain, @@ -92,9 +95,9 @@ from dynamo.planner.retune.sweep import ( SweepRequest, SweepResult, + evaluated_params, objective_maximized, run_sweep, - with_predictor_fields, ) from dynamo.planner.simulation.load_predictor import predictor_fields @@ -227,11 +230,7 @@ def __init__( # the minimum the current and candidate predictors need. history_s = retune.history_seconds if history_s is None: - history_s = ( - required_history_seconds(config, planner_domain) - if config.enable_throughput_scaling - else 0.0 - ) + history_s = required_history_seconds(config, planner_domain) self._history_s = history_s planner_keys = set(planner_domain.grid) if self._predictor_candidates: @@ -244,7 +243,7 @@ def __init__( "set it explicitly in the planner config" ) - self._grid = self._feasible_grid(router_domain, planner_domain.grid) + self._grid = self._feasible_grid(router_domain, planner_domain) self._desired_router: dict[str, float] = self._startup_router_values() self._pending_planner: dict[str, Any] = {} @@ -484,17 +483,19 @@ def _router_domain(self, retune: RetuneConfig) -> dict[str, list[float]]: def _feasible_grid( self, router_domain: Mapping[str, Sequence[float]], - planner_grid: Mapping[str, Sequence[Any]], + planner_domain: PlannerDomain, ) -> list[RetuneParams]: """The grid without the planner points that cannot run here. Grid points and predictor candidates are checked apart, each on the startup config: no ``PlannerConfig`` validator relates a predictor field to another field, and ``_decide`` checks the combination it - applies. A point that fails validation or does not fit the - deployment is dropped, as recommend drops an infeasible candidate. - An invalid predictor candidate, or a grid value that no point keeps - (so also a grid without a valid point), raises. + applies. A point that fails the full ``PlannerConfig`` validation + (for instance a throughput-scaling point the configuration cannot + run) or does not fit the deployment is dropped, as recommend drops an + infeasible candidate. An invalid predictor candidate, or a value of a + written knob that no point keeps (so also a grid without a valid + point), raises; a scaling-policy value may be dropped entirely. """ # The retune checks never depend on a candidate; skip them. @@ -507,7 +508,7 @@ def _feasible_grid( error = _candidate_error(base, updates, self._capabilities) if error is not None: raise ValueError(f"retune.planner candidate {updates} {error}") - points = grid_points({}, planner_grid) + points = grid_points({}, planner_domain.points) rejected: dict[str, str] = {} for point in points: if point.planner: @@ -515,18 +516,30 @@ def _feasible_grid( if error is not None: rejected[point.key()] = error if not rejected: - return grid_points(router_domain, planner_grid) + return grid_points(router_domain, planner_domain.points) kept = [point for point in points if point.key() not in rejected] - for key, values in planner_grid.items(): + if not kept: + first = points[0] + raise ValueError( + f"no retune.planner candidate fits; candidate {first.planner} " + f"{rejected[first.key()]}" + ) + for key, values in planner_domain.grid.items(): + if key in SCALING_POLICY_KEYS: + continue for value in values: if not any(_same_value(p.planner[key], value) for p in kept): - point = next( - p for p in points if _same_value(p.planner[key], value) + holder = next( + (p for p in points if _same_value(p.planner[key], value)), + None, ) + if holder is None: + # Folded away in every point (an inactive mode). + continue raise ValueError( f"no retune.planner candidate with {key}={value!r} fits; " - f"candidate {point.planner} {rejected[point.key()]}" + f"candidate {holder.planner} {rejected[holder.key()]}" ) first = next(p for p in points if p.key() in rejected) logger.warning( @@ -539,7 +552,7 @@ def _feasible_grid( ) return [ point - for point in grid_points(router_domain, planner_grid) + for point in grid_points(router_domain, planner_domain.points) if RetuneParams(planner=point.planner).key() not in rejected ] @@ -876,11 +889,7 @@ def _decide( ) -> RetuneDecision: request = pending.request baseline = request.baseline - presweep_fields = ( - None - if result.predictor_presweep is None - else result.predictor_presweep.get("fields") - ) + presweep = result.predictor_presweep def record(outcome: RetuneOutcome, **overrides: Any) -> RetuneDecision: values: dict[str, Any] = { @@ -943,7 +952,9 @@ def record(outcome: RetuneOutcome, **overrides: Any) -> RetuneDecision: top: list[tuple[str, float]] = [] for score, candidate in ranked: # Equivalent candidates share one evaluation; list it once. - key = with_predictor_fields(candidate, baseline, presweep_fields).key() + key = evaluated_params( + candidate, baseline, presweep, request.base_planner + ).key() if len(top) < _TOP_SCORES and all(key != seen for seen, _ in top): top.append((key, score)) best_score, best = ranked[0] @@ -955,7 +966,7 @@ def record(outcome: RetuneOutcome, **overrides: Any) -> RetuneDecision: if best == baseline or not self._improves(best_score, baseline_score): return record("kept", **scores) - chosen = with_predictor_fields(best, baseline, presweep_fields) + chosen = evaluated_params(best, baseline, presweep, request.base_planner) current = self.current_point() planner_changes = { key: value @@ -973,6 +984,7 @@ def record(outcome: RetuneOutcome, **overrides: Any) -> RetuneDecision: ) predictor_warmup = None + rebuild_interval: int | None = None if planner_changes: # _feasible_grid checked grid points and predictors apart; check the # combination the host will apply, so it never raises. @@ -982,21 +994,31 @@ def record(outcome: RetuneOutcome, **overrides: Any) -> RetuneDecision: return record( "failed", reason=f"planner candidate {updates} {error}", **scores ) - if planner_changes.keys() & PREDICTOR_FIELDS: + applied = candidate_config(self._config, updates) + current_config = ( + candidate_config(self._config, dict(self._pending_planner)) + if self._pending_planner + else self._config + ) + if predictor_rebuild_required(current_config, applied): + rebuild_interval = applied.throughput_adjustment_interval_seconds + if rebuild_interval is not None: # Hosts swap a predictor in only warmed; the sweep prepared the - # winner on the same warmup. - if not result.warmup_observations: + # candidate's predictor on the same warmup, bucketed at its + # throughput interval. + observations = result.observations_at(rebuild_interval) + if not observations: return record( "failed", reason="predictor change without warmup observations", **scores, ) - error = result.predictor_warmup_error + error = result.warmup_error_at(rebuild_interval) if error is not None: return record( "failed", reason=f"predictor warmup rejected: {error}", **scores ) - predictor_warmup = result.warmup_observations + predictor_warmup = observations if self._config.advisory: # The decision an active planner would apply, after the same # feasibility checks; nothing is applied, so the desired router @@ -1192,7 +1214,7 @@ def router_maps(router: Mapping[str, float]) -> dict[str, dict[str, float]]: # The schema admits only PolicyParameters names, with the bounds # KvRouterConfig checks; resolving every candidate also guards against # the two drifting apart. - for point in grid_points(domain, {}): + for point in grid_points(domain): try: maps = router_maps(point.router) except ValueError as error: diff --git a/components/src/dynamo/planner/retune/domains.py b/components/src/dynamo/planner/retune/domains.py index 785418f670cc..2e20b2d42f76 100644 --- a/components/src/dynamo/planner/retune/domains.py +++ b/components/src/dynamo/planner/retune/domains.py @@ -14,11 +14,18 @@ - An omitted AISimulate knob searches its AISimulate default domain; a scalar pins it. ``default`` omits every knob. -- Knobs fixed at startup keep their running value. -- Retune extension knobs (no AISimulate default domain) keep their current - value when omitted. +- Knobs fixed at startup keep their running value; worker minimums and GPU + budgets are user constraints and are never searched. +- Retune extension knobs (no AISimulate default domain) and the + ``scaling_policy`` group keep their current value when omitted. - The load predictor is never a grid dimension: its candidates go to the - pre-sweep, and every non-baseline candidate uses the winner. + pre-sweep, and every non-baseline candidate uses the winner of its + throughput interval. +- The planner grid is a list of points, not a full Cartesian product: a + scaling policy is one correlated choice of its four fields, points with an + invalid scaling policy are dropped, and fields a point's scaling modes make + inactive are folded to their running value, so equivalent points appear + once (``canonical_planner_point``). The adapter schemas and providers are imported on use: they import ``PlannerConfig``, whose validation imports this module, and the providers @@ -43,6 +50,7 @@ from dynamo.planner.simulation.provider import ( LoadPredictorSearch, LoadSensitivitySearch, + ScalingPolicySearch, ) from dynamo.router.simulation.config import RouterRetuneConfig @@ -53,6 +61,24 @@ OVERLAP_SCORE_CREDIT = "overlap_score_credit" LOAD_SCALING_DOWN_SENSITIVITY = "load_scaling_down_sensitivity" +ENABLE_THROUGHPUT = "enable_throughput_scaling" +ENABLE_LOAD = "enable_load_scaling" +THROUGHPUT_INTERVAL = "throughput_adjustment_interval_seconds" +LOAD_INTERVAL = "load_adjustment_interval_seconds" +MAX_THROUGHPUT_REPLICAS = "max_throughput_scaling_replicas" +# The scaling-policy fields, in the order a grid lists them. +SCALING_POLICY_KEYS: tuple[str, ...] = ( + ENABLE_THROUGHPUT, + ENABLE_LOAD, + THROUGHPUT_INTERVAL, + LOAD_INTERVAL, +) +# Fields that act only with load or only with throughput scaling. Without +# their mode a point folds them to the running value. The load interval is +# not among them: it is also the cadence at which live FPM observations +# refit the performance model that throughput scaling uses. +LOAD_ONLY_KEYS: tuple[str, ...] = (LOAD_SCALING_DOWN_SENSITIVITY,) +THROUGHPUT_ONLY_KEYS: tuple[str, ...] = (THROUGHPUT_INTERVAL, MAX_THROUGHPUT_REPLICAS) # Combinations a ``preset: false`` group may expand to before deduplication, # which bounds the expansion's memory. The AISimulate default load-predictor # domain is 256 combinations. @@ -64,13 +90,20 @@ class PlannerDomain: """Expanded planner search domain. ``grid`` holds the goodput-grid dimensions (``PlannerConfig`` field -> - candidate values). ``predictor_candidates`` is set when the load + every candidate value), the four scaling-policy fields included when the + scaling policy is searched. ``points`` are the distinct grid points, each + with every ``grid`` key: the scaling policies times the other dimensions, + canonical (``canonical_planner_point``) and valid + (``scaling_point_error``). Without dimensions there is one empty point, + the current configuration. ``predictor_candidates`` is set when the load predictor is searched (``optimization_target='sla'`` with throughput - scaling): the forecast-loss pre-sweep picks one of these ``(label, - mapping)`` candidates per sweep, and it is not a grid dimension. + scaling in some point): the forecast-loss pre-sweep picks one of these + ``(label, mapping)`` candidates per throughput interval and sweep, and it + is not a grid dimension. """ grid: dict[str, list[Any]] = field(default_factory=dict) + points: tuple[dict[str, Any], ...] = ({},) predictor_candidates: list[PredictorCandidate] | None = None @property @@ -80,6 +113,88 @@ def empty(self) -> bool: return not self.grid and not self.predictor_candidates +_ABSENT = object() + + +def _running_value(running: Mapping[str, Any] | PlannerConfig, key: str) -> Any: + """``running``'s ``key``; ``_ABSENT`` when a config dump lacks it.""" + + if isinstance(running, Mapping): + return running.get(key, _ABSENT) + return getattr(running, key) + + +def _point_value( + point: Mapping[str, Any], running: Mapping[str, Any] | PlannerConfig, key: str +) -> Any: + return point[key] if key in point else _running_value(running, key) + + +def canonical_planner_point( + point: Mapping[str, Any], running: Mapping[str, Any] | PlannerConfig +) -> dict[str, Any]: + """``point`` with the fields its scaling modes make inactive folded. + + A field that acts only with load scaling (``LOAD_ONLY_KEYS``) or only + with throughput scaling (``THROUGHPUT_ONLY_KEYS``) takes its ``running`` + value when the point's own mode (its scaling-policy field, else the + running one) is off. Only keys already in ``point`` are folded, so the + key set is kept. ``running`` is the planner config or a config dump; a + dump without a mode flag or a field leaves it unfolded. + """ + + folded = dict(point) + for flag, keys in ( + (ENABLE_LOAD, LOAD_ONLY_KEYS), + (ENABLE_THROUGHPUT, THROUGHPUT_ONLY_KEYS), + ): + active = _point_value(point, running, flag) + if active is _ABSENT or active: + continue + for key in keys: + value = _running_value(running, key) + if key in folded and value is not _ABSENT: + folded[key] = value + return folded + + +def scaling_point_error(point: Mapping[str, Any], config: PlannerConfig) -> str | None: + """Why the scaling policy of ``point`` (over ``config``) cannot run. + + Checks only what ``PlannerConfig`` validation would reject and the point + can be dropped for without validating a full candidate: both modes off, + a load interval not shorter than the throughput interval with both on, + and an interval that ``scheduling.scale_interval_seconds`` (fixed at + startup) does not divide. ``None`` when it can run. + """ + + throughput = bool(_point_value(point, config, ENABLE_THROUGHPUT)) + load = bool(_point_value(point, config, ENABLE_LOAD)) + load_interval = _point_value(point, config, LOAD_INTERVAL) + throughput_interval = _point_value(point, config, THROUGHPUT_INTERVAL) + if not throughput and not load: + return "turns both scaling modes off" + if throughput and load and load_interval >= throughput_interval: + return ( + f"has load_adjustment_interval_seconds {load_interval} not shorter " + f"than throughput_adjustment_interval_seconds {throughput_interval}" + ) + # Imported on use: planner_config imports this module on use too. + from dynamo.planner.config.planner_config import _is_multiple_of + + scale = config.scheduling.scale_interval_seconds + intervals = [(LOAD_INTERVAL, load_interval)] + if throughput: + intervals.append((THROUGHPUT_INTERVAL, throughput_interval)) + for name, interval in intervals: + if not _is_multiple_of(float(interval), scale): + return ( + f"has {name} {interval}, not a multiple of the planner's " + f"scheduling.scale_interval_seconds {scale:g} (fixed at startup)" + ) + return None + + @overload def parse_retune_section( section: Literal["router"], spec: str | Mapping[str, Any] @@ -257,19 +372,30 @@ def expand_planner_domain(config: PlannerConfig) -> PlannerDomain: Only what a running planner can change and what applies to ``config`` is searched: - - ``load_sensitivity``, with load scaling: the ``load_scaling_down_sensitivity`` - of its entries (``load_min_observations`` is fixed at startup). Omitted, - the three AISimulate presets give {70, 80, 90}. + - ``scaling_policy``, when written: its preset entries (``default`` is + every AISimulate preset that scales; ``preset: false`` every + combination of its four knobs). With ``optimization_target`` other + than ``sla`` the planner always scales on load only, so only the load + interval can vary. Points that turn both modes off, order the + intervals wrongly or use an interval the scale interval does not + divide are dropped; an entry written in a preset list must be valid. + - ``load_sensitivity``, with load scaling in some point: the + ``load_scaling_down_sensitivity`` of its entries + (``load_min_observations`` is fixed at startup). Omitted, the three + AISimulate presets give {70, 80, 90}. - ``load_predictor``, with ``optimization_target='sla'`` and throughput - scaling: the distinct predictor candidates of the pre-sweep. Omitted, - the 11 AISimulate presets. + scaling in some point: the distinct predictor candidates of the + pre-sweep. Omitted, the 11 AISimulate presets. - Any written ``PLANNER_RETUNE_KNOBS`` knob, under its ``PlannerConfig`` name. An omitted group that does not apply is skipped; a written one raises, as does any written knob without load or throughput scaling. Grid values - must affect this configuration and pass strict field validation - (``PlannerConfig._validate_retune_planner_grid``). + must affect some point and pass strict field validation + (``PlannerConfig._validate_retune_planner_grid``). Points are made + canonical and deduplicated; the full ``PlannerConfig`` validation of each + point, which can drop more, is the retune controller's + (``RetuneController._feasible_grid``). """ retune = config.retune @@ -293,9 +419,21 @@ def expand_planner_domain(config: PlannerConfig) -> PlannerDomain: require_enumerable(public, prefix="retune.planner.") limit = retune.max_candidates + policies = _scaling_policies(config, public, limit) + modes = ( + [(config.enable_throughput_scaling, config.enable_load_scaling)] + if policies is None + else [(p[ENABLE_THROUGHPUT], p[ENABLE_LOAD]) for p in policies] + ) + any_throughput = any(throughput for throughput, _ in modes) + any_load = any(load for _, load in modes) + grid: dict[str, list[Any]] = {} public_names: dict[str, str] = {} - sensitivities = _load_sensitivities(config, public, limit) + if policies is not None: + for key in SCALING_POLICY_KEYS: + grid[key] = list(dict.fromkeys(policy[key] for policy in policies)) + sensitivities = _load_sensitivities(config, public, limit, any_load) if sensitivities is not None: grid[LOAD_SCALING_DOWN_SENSITIVITY] = sensitivities for name, live in PLANNER_RETUNE_KNOBS.items(): @@ -304,17 +442,179 @@ def expand_planner_domain(config: PlannerConfig) -> PlannerDomain: f"retune.planner.{name}", getattr(public, name), None, limit=limit ) public_names[live] = name - grid = config._validate_retune_planner_grid(grid, public_names) + grid = config._validate_retune_planner_grid( + grid, + public_names, + load_scaling=any_load, + throughput_scaling=any_throughput, + ) + if policies is not None: + # The validated values (strict field types) back the policies too. + policies = [ + {key: _validated(grid[key], policy[key]) for key in SCALING_POLICY_KEYS} + for policy in policies + ] return PlannerDomain( - grid=grid, predictor_candidates=_predictor_candidates(config, public, limit) + grid=grid, + points=_planner_points(config, grid, policies), + predictor_candidates=_predictor_candidates( + config, public, limit, any_throughput + ), ) -def _load_sensitivities( +def _validated(values: Sequence[Any], value: Any) -> Any: + """The validated grid value that equals ``value`` (``5`` -> ``5``).""" + + for candidate in values: + if (isinstance(candidate, bool), candidate) == (isinstance(value, bool), value): + return candidate + raise AssertionError(f"{value!r} is not a grid value") # pragma: no cover + + +def _planner_points( + config: PlannerConfig, + grid: Mapping[str, Sequence[Any]], + policies: Sequence[Mapping[str, Any]] | None, +) -> tuple[dict[str, Any], ...]: + """Every distinct valid canonical point, in canonical key order.""" + + independent = { + key: values for key, values in grid.items() if key not in SCALING_POLICY_KEYS + } + combinations = math.prod(len(values) for values in independent.values()) * ( + len(policies) if policies is not None else 1 + ) + if combinations > MAX_PRESET_COMBINATIONS: + raise ValueError( + f"retune.planner expands to {combinations} points before " + f"deduplication, more than {MAX_PRESET_COMBINATIONS}; narrow its " + "knob domains" + ) + points: dict[str, dict[str, Any]] = {} + dropped: str | None = None + for policy in policies if policies is not None else [{}]: + for values in itertools.product(*independent.values()): + point = canonical_planner_point( + {**dict(zip(independent, values)), **policy}, config + ) + error = scaling_point_error(point, config) + if error is not None: + dropped = dropped or f"{point} {error}" + continue + points.setdefault(RetuneParams(planner=point).key(), point) + if not points: + raise ValueError( + "retune.planner has no valid point: every scaling policy is " + f"invalid here, for example {dropped}" + ) + return tuple(points[key] for key in sorted(points)) + + +def _scaling_policies( config: PlannerConfig, public: PlannerRetuneConfig, limit: int +) -> list[dict[str, Any]] | None: + """The searched scaling policies, each a mapping of its four fields. + + ``None`` when ``scaling_policy`` is omitted: the running policy is kept. + A preset's null interval is the running value, except for an enabled + mode, which raises. Policies a planner cannot run are dropped from a + default or ``preset: false`` expansion (``scaling_point_error`` reports + them again for a written entry). Without ``optimization_target='sla'`` + every policy must scale on load only. + """ + + control = public.scaling_policy + if control is None: + return None + from dynamo.planner.simulation.presets import SCALING_POLICIES, scaling_fields + from dynamo.planner.simulation.provider import ScalingPolicySearch + + sla = config.optimization_target == "sla" + written: list[str | dict[str, Any]] | None = None + entries: list[str | dict[str, Any]] + if control.preset == "default": + # Every AISimulate preset that scales: retune never turns scaling off. + entries = [ + name + for name, policy in SCALING_POLICIES.items() + if policy.enable_throughput or policy.enable_load + ] + elif control.preset in (False, {}): + if not sla: + knobs = sorted( + {ENABLE_THROUGHPUT, ENABLE_LOAD, THROUGHPUT_INTERVAL} + & public.model_fields_set + ) + if knobs: + raise ValueError( + f"retune.planner {knobs} have no effect here; they require " + "optimization_target='sla' (other targets always scale on " + "load only)" + ) + entries = list(_preset_off_entries("scaling_policy", public, limit)) + else: + written = _preset_entries(ScalingPolicySearch, "scaling_policy", control) + entries = list(written) + + policies: list[dict[str, Any]] = [] + for entry in entries: + fields = scaling_fields(entry) + for key, enabled in ( + (THROUGHPUT_INTERVAL, fields[ENABLE_THROUGHPUT]), + (LOAD_INTERVAL, fields[ENABLE_LOAD]), + ): + if fields[key] is None: + if enabled and written is not None: + raise ValueError( + f"retune.planner.scaling_policy entry {entry!r}: {key} " + "cannot be null while its scaling mode is enabled" + ) + fields[key] = getattr(config, key) + policy = {key: fields[key] for key in SCALING_POLICY_KEYS} + load_only = policy[ENABLE_LOAD] and not policy[ENABLE_THROUGHPUT] + if not sla and not load_only: + if written is not None: + raise ValueError( + f"retune.planner.scaling_policy entry {entry!r} has no effect " + "here; with optimization_target " + f"{config.optimization_target!r} the planner always scales " + "on load only, so only load-only entries apply" + ) + continue + if written is not None: + error = scaling_point_error(policy, config) + if error is not None: + hint = ( + "" + if policy[ENABLE_THROUGHPUT] or policy[ENABLE_LOAD] + else "; retune never turns scaling off" + ) + raise ValueError( + f"retune.planner.scaling_policy entry {entry!r} {error}{hint}" + ) + # An interval of a mode the policy turns off is inert: fold it so + # policies that differ only there are one choice. + policies.append(canonical_planner_point(policy, config)) + if not policies: + raise ValueError( + "retune.planner.scaling_policy has no policy that applies here " + f"(optimization_target {config.optimization_target!r})" + ) + unique = {RetuneParams(planner=policy).key(): policy for policy in policies} + if len(unique) > limit: + raise ValueError( + f"retune.planner.scaling_policy has {len(unique)} distinct policies, " + f"exceeding retune.max_candidates={limit}" + ) + return list(unique.values()) + + +def _load_sensitivities( + config: PlannerConfig, public: PlannerRetuneConfig, limit: int, load: bool ) -> list[Any] | None: explicit = "load_sensitivity" in public.model_fields_set - if not config.enable_load_scaling: + if not load: if explicit: raise ValueError( "retune.planner.load_sensitivity has no effect here; it requires " @@ -354,10 +654,10 @@ def _load_sensitivities( def _predictor_candidates( - config: PlannerConfig, public: PlannerRetuneConfig, limit: int + config: PlannerConfig, public: PlannerRetuneConfig, limit: int, throughput: bool ) -> list[PredictorCandidate] | None: explicit = "load_predictor" in public.model_fields_set - if not (config.optimization_target == "sla" and config.enable_throughput_scaling): + if not (config.optimization_target == "sla" and throughput): if explicit: raise ValueError( "retune.planner.load_predictor has no effect here; it requires " @@ -388,7 +688,9 @@ def _predictor_candidates( def _preset_entries( - search: type[LoadSensitivitySearch] | type[LoadPredictorSearch], + search: type[ScalingPolicySearch] + | type[LoadSensitivitySearch] + | type[LoadPredictorSearch], group: str, control: Any, ) -> list[str | dict[str, Any]]: @@ -403,11 +705,15 @@ def _preset_entries( from dynamo.planner.simulation.config import ( LoadPredictorMapping, LoadSensitivityMapping, + ScalingPolicyMapping, ) - mapping_model: type[BaseModel] = ( - LoadPredictorMapping if group == "load_predictor" else LoadSensitivityMapping - ) + mapping_models: dict[str, type[BaseModel]] = { + "scaling_policy": ScalingPolicyMapping, + "load_sensitivity": LoadSensitivityMapping, + "load_predictor": LoadPredictorMapping, + } + mapping_model = mapping_models[group] try: entries = search.model_validate({"preset": control.preset}).preset return [ @@ -462,10 +768,14 @@ def _preset_off_entries( def describe_grid( - router: Mapping[str, Sequence[Any]], planner: Mapping[str, Sequence[Any]] + router: Mapping[str, Sequence[Any]], + planner: Mapping[str, Sequence[Any]], + planner_points: int | None = None, ) -> str: """Each grid dimension's size under its public name, for the error of a - grid larger than ``retune.max_candidates``.""" + grid larger than ``retune.max_candidates``. ``planner_points`` (the + number of distinct planner points) is reported when it differs from the + product of the planner dimensions.""" from dynamo.planner.simulation.config import PLANNER_RETUNE_KNOBS from dynamo.router.simulation.config import ROUTER_PUBLIC_TO_LIVE @@ -481,6 +791,12 @@ def describe_grid( f"{public.get(key, key)} {len(values)}" for key, values in domain.items() ) + if ( + section == "planner" + and planner_points is not None + and planner_points != grid_size(planner) + ): + sizes += f" = {planner_points} distinct points" parts.append(f"{section} {sizes}") return "; ".join(parts) @@ -495,31 +811,50 @@ def grid_size(*domains: Mapping[str, Sequence[Any]]) -> int: def grid_points( - router: Mapping[str, Sequence[float]], planner: Mapping[str, Sequence[Any]] + router: Mapping[str, Sequence[float]], + planner_points: Sequence[Mapping[str, Any]] = ({},), ) -> list[RetuneParams]: - """Enumerate the Cartesian grid in canonical ``RetuneParams.key`` order.""" + """The router Cartesian grid times ``planner_points``, in canonical + ``RetuneParams.key`` order.""" router_keys = sorted(router) - planner_keys = sorted(planner) points = [ RetuneParams( - router=dict(zip(router_keys, router_values)), - planner=dict(zip(planner_keys, planner_values)), + router=dict(zip(router_keys, router_values)), planner=dict(planner) ) for router_values in itertools.product(*(router[k] for k in router_keys)) - for planner_values in itertools.product(*(planner[k] for k in planner_keys)) + for planner in planner_points ] return sorted(points, key=RetuneParams.key) +def throughput_intervals( + config: PlannerConfig, points: Sequence[Mapping[str, Any]] +) -> list[int]: + """The throughput intervals of the points that scale on throughput, and + of ``config`` when it does.""" + + intervals = set() + if config.enable_throughput_scaling: + intervals.add(int(config.throughput_adjustment_interval_seconds)) + for point in points: + if _point_value(point, config, ENABLE_THROUGHPUT): + intervals.add(int(_point_value(point, config, THROUGHPUT_INTERVAL))) + return sorted(intervals) + + def required_history_seconds(config: PlannerConfig, domain: PlannerDomain) -> float: """Trace history needed to warm every predictor the sweep may run. - ``throughput_adjustment_interval_seconds`` x the largest + The largest throughput interval (``throughput_intervals``) x the largest ``prophet_window_size`` or ``kalman_min_points`` over the current config - and the pre-sweep candidates' predictor fields. + and the pre-sweep candidates' predictor fields; 0 when no point scales on + throughput. """ + intervals = throughput_intervals(config, domain.points) + if not intervals: + return 0.0 window_sizes = [config.prophet_window_size] min_points = [config.kalman_min_points] if domain.predictor_candidates: @@ -532,6 +867,4 @@ def required_history_seconds(config: PlannerConfig, domain: PlannerDomain) -> fl fields.get("prophet_window_size", config.prophet_window_size) ) min_points.append(fields.get("kalman_min_points", config.kalman_min_points)) - return float( - config.throughput_adjustment_interval_seconds * max(*window_sizes, *min_points) - ) + return float(max(intervals) * max(*window_sizes, *min_points)) diff --git a/components/src/dynamo/planner/retune/sweep.py b/components/src/dynamo/planner/retune/sweep.py index d8cf1521f201..55f8acf595c6 100644 --- a/components/src/dynamo/planner/retune/sweep.py +++ b/components/src/dynamo/planner/retune/sweep.py @@ -60,8 +60,13 @@ TrafficObservation, WorkerCapabilities, ) -from dynamo.planner.reconfiguration import prepare_predictor -from dynamo.planner.retune.domains import PredictorCandidate +from dynamo.planner.reconfiguration import PREDICTOR_FIELDS, prepare_predictor +from dynamo.planner.retune.domains import ( + ENABLE_THROUGHPUT, + THROUGHPUT_INTERVAL, + PredictorCandidate, + canonical_planner_point, +) from dynamo.planner.retune.window import ( ObservationBuckets, read_trace_history, @@ -88,8 +93,9 @@ _PRESWEEP_TASK = "predictor-presweep" _PREDICTOR_WARMUP_TASK = "predictor-warmup" _WINDOW_FILE = "window.jsonl" -_OBSERVATIONS_FILE = "observations.json" -_WARMUP_FILE = "warmup.json" +# One file each per throughput interval: ``observations-.json``. +_OBSERVATIONS_FILE = "observations" +_WARMUP_FILE = "warmup" _WORKER_TEMP_DIR = "tmp" @@ -248,13 +254,19 @@ class SweepResult: carry the evaluated ``params`` (the candidate with any pre-sweep predictor fields) and either the evaluator's metrics or ``error``. ``predictor_presweep`` is ``{winner, fields, losses, eval_steps, - warmup_steps, wall_s}`` (plus ``errors`` when a candidate's task failed) - when the pre-sweep ran; ``winner`` and the ``losses`` keys are candidate - labels. ``warmup_observations`` covers history - and window, bucketed at the inner planner's throughput interval, for a - predictor swap. When the winner changes the predictor, the sweep - prepares it on ``warmup_observations`` as a host does before a swap; - ``predictor_warmup_error`` is that preparation's error, if any. + warmup_steps, wall_s, interval_seconds}`` (plus ``errors`` when a + candidate's task failed) when the pre-sweep ran at one throughput + interval; ``winner`` and the ``losses`` keys are candidate labels. With + candidates at several intervals it is ``{"by_interval": {"": + }}`` (``presweep_fields`` reads either). + ``interval_observations`` maps each throughput interval a candidate (or + the inner planner) scales at to the history and window bucketed at that + interval, for a predictor swap; ``warmup_observations`` is the entry of + the inner planner's own interval. When the candidate at an interval + changes the predictor, the sweep prepares it on those observations as a + host does before a swap; ``predictor_warmup_errors`` holds each + preparation's error, and ``predictor_warmup_error`` the one at the inner + planner's interval. ``row_counts`` holds the window reader's counts and, when the history was read, ``history_requests``. ``failure_reason`` is a stable, machine-readable reason a live executor sets next to the free-text @@ -275,30 +287,123 @@ class SweepResult: error: str | None = None failure_reason: str | None = None details: dict[str, Any] = field(default_factory=dict) + interval_observations: dict[int, tuple[TrafficObservation, ...]] = field( + default_factory=dict + ) + predictor_warmup_errors: dict[int, str] = field(default_factory=dict) + def observations_at(self, interval: int) -> tuple[TrafficObservation, ...] | None: + """The swap warmup at throughput ``interval`` (``None``: not read). -def with_predictor_fields( + A result without per-interval observations (one interval) has only + ``warmup_observations``. + """ + + if self.interval_observations: + return self.interval_observations.get(int(interval)) + return self.warmup_observations + + def warmup_error_at(self, interval: int) -> str | None: + """The predictor preparation error at throughput ``interval``.""" + + if self.predictor_warmup_errors: + return self.predictor_warmup_errors.get(int(interval)) + return self.predictor_warmup_error + + +def presweep_fields( + presweep: Mapping[str, Any] | None, interval: int +) -> dict[str, Any] | None: + """The winner's predictor fields of the pre-sweep at ``interval``. + + ``None`` when the pre-sweep did not run at that interval. A single + summary without ``interval_seconds`` (written before intervals were + searched) applies at any interval. + """ + + if presweep is None: + return None + if "by_interval" in presweep: + summary = presweep["by_interval"].get(str(int(interval))) + return None if summary is None else summary.get("fields") + recorded = presweep.get("interval_seconds") + if recorded is not None and int(recorded) != int(interval): + return None + return presweep.get("fields") + + +def _running_planner( + baseline: RetuneParams, base_planner: Mapping[str, Any] | None +) -> dict[str, Any]: + return {**(base_planner or {}), **baseline.planner} + + +def scales_on_throughput( + planner: Mapping[str, Any], running: Mapping[str, Any] +) -> int | None: + """The throughput interval ``planner`` (over ``running``) scales at, or + ``None`` without throughput scaling.""" + + if not planner.get(ENABLE_THROUGHPUT, running.get(ENABLE_THROUGHPUT)): + return None + return int(planner.get(THROUGHPUT_INTERVAL, running[THROUGHPUT_INTERVAL])) + + +def evaluated_params( candidate: RetuneParams, baseline: RetuneParams, - fields: Mapping[str, Any] | None, + presweep: Mapping[str, Any] | None, + base_planner: Mapping[str, Any] | None, ) -> RetuneParams: - """The params a candidate is evaluated with after the predictor pre-sweep. - - The baseline keeps the current predictor; every other candidate carries - the pre-sweep winner's predictor fields. Searched planner keys that a - candidate does not set (the predictor fields of a grid point) take the - baseline's values first. That leaves the inner planner unchanged, because - the base planner already holds the current values, but gives a candidate - equivalent to the baseline the baseline's key, so it is replayed once. + """The params a candidate is evaluated (and applied) with. + + The baseline is evaluated as is. Every other candidate: + + - takes the baseline's value for a searched planner key it does not set + (the predictor fields of a grid point); + - folds the fields its scaling modes make inactive to the baseline's + values (``canonical_planner_point``), so an inactive difference is + neither replayed nor applied; + - when it scales on throughput, carries the pre-sweep winner of its + throughput interval (``presweep_fields``). + + That leaves the inner planner unchanged where the base planner already + holds the current values, and gives a candidate equivalent to the + baseline the baseline's key, so it is replayed once. """ if candidate == baseline: return candidate - planner = {**baseline.planner, **candidate.planner, **(fields or {})} + planner = {**baseline.planner, **candidate.planner} + if planner and base_planner is not None: + running = _running_planner(baseline, base_planner) + planner = canonical_planner_point(planner, running) + interval = scales_on_throughput(planner, running) + fields = None if interval is None else presweep_fields(presweep, interval) + planner.update(fields or {}) + elif presweep is not None: + # No inner planner to read the interval from: one pre-sweep summary. + planner.update(presweep.get("fields") or {}) evaluated = RetuneParams(router=dict(candidate.router), planner=planner) return candidate if evaluated == candidate else evaluated +def with_predictor_fields( + candidate: RetuneParams, + baseline: RetuneParams, + fields: Mapping[str, Any] | None, +) -> RetuneParams: + """``evaluated_params`` for one pre-sweep winner's ``fields``, without an + inner planner to fold against (kept for single-interval callers).""" + + return evaluated_params( + candidate, + baseline, + None if fields is None else {"fields": dict(fields)}, + None, + ) + + def score_summary( summary: Mapping[str, Any], window_s: float, @@ -529,19 +634,21 @@ def _changes_predictor(baseline: RetuneParams, fields: Mapping[str, Any]) -> boo class _SweepInputs(NamedTuple): """What the read task hands back; the files live in the sweep directory. + Each mapping is keyed by throughput interval (seconds), one entry per + interval the read spec bucketed (none without throughput scaling). ``observations`` are the history and window buckets ending at the window - end (``None`` without throughput scaling); ``observations_path`` is their - file, written only for the pre-sweep. ``warmup_path`` holds the history - buckets ending at the window start, the inner planner's warmup; it is - ``None`` when no history row was selected. + end; ``observations_paths`` are their files, written only for the + pre-sweep. ``warmup_paths`` hold the history buckets ending at the + window start, the inner planner's warmup; an interval is missing when no + history row was selected. """ window_path: str num_requests: int row_counts: dict[str, int] - observations: tuple[TrafficObservation, ...] | None = None - observations_path: str | None = None - warmup_path: str | None = None + observations: dict[int, tuple[TrafficObservation, ...]] = {} + observations_paths: dict[int, str] = {} + warmup_paths: dict[int, str] = {} @dataclass(frozen=True) @@ -550,8 +657,9 @@ class _ReadSpec: Only what the readers need: the request also carries every candidate and the engine, which would make the task message large. ``buckets`` (empty) - is set for an inner planner that scales on throughput, the only one with - a load predictor to warm; the history is read only then. + holds one bucket set per throughput interval an inner planner or a + candidate scales at, the only ones with a load predictor to warm; the + history is read only then. """ trace_paths: tuple[str, ...] @@ -560,14 +668,14 @@ class _ReadSpec: window_end_ms: float history_start_ms: float as_of_ms: float - buckets: ObservationBuckets | None + buckets: dict[int, ObservationBuckets] write_observations: bool @classmethod def from_request( cls, request: SweepRequest, - buckets: ObservationBuckets | None, + buckets: Mapping[int, ObservationBuckets], *, write_observations: bool, ) -> _ReadSpec: @@ -578,7 +686,7 @@ def from_request( window_end_ms=request.window_end_ms, history_start_ms=request.history_start_ms, as_of_ms=request.as_of_ms, - buckets=buckets, + buckets=dict(buckets), write_observations=write_observations, ) @@ -588,57 +696,68 @@ def _read_sweep_inputs(spec: _ReadSpec, work_dir: str) -> _SweepInputs: Runs as a sweep worker task, so the deadline and cancel kill a hung read. An empty window stops here: the history is not read. A history row goes - into ``spec.buckets``, as the window's rows do, and into buckets that end - at the window start, written as the inner planner's warmup: those never - take in a window request, wherever the window start falls on the grid of - ``spec.buckets``. Raises what the readers raise. + into each interval's ``spec.buckets``, as the window's rows do, and into + buckets that end at the window start, written as the inner planner's + warmup at that interval: those never take in a window request, wherever + the window start falls on the grid of ``spec.buckets``. Raises what the + readers raise. """ directory = Path(work_dir) - buckets = spec.buckets window = read_trace_window( spec.trace_paths, spec.window_start_ms, spec.window_end_ms, spec.as_of_ms, directory / _WINDOW_FILE, - buckets=buckets, + buckets=tuple(spec.buckets.values()), ) row_counts = dict(window.row_counts) - if window.num_requests == 0 or buckets is None: + if window.num_requests == 0 or not spec.buckets: return _SweepInputs(window.path, window.num_requests, row_counts) - warmup_path = None + warmup_paths: dict[int, str] = {} if spec.history_start_ms < spec.window_start_ms: - warmup = ObservationBuckets( - end_ms=spec.window_start_ms, - interval_ms=buckets.interval_ms, - duration_s=buckets.duration_s, - ) + warmups = { + interval: ObservationBuckets( + end_ms=spec.window_start_ms, + interval_ms=buckets.interval_ms, + duration_s=buckets.duration_s, + ) + for interval, buckets in spec.buckets.items() + } row_counts["history_requests"] = read_trace_history( (*spec.trace_paths, *spec.history_paths), spec.history_start_ms, spec.window_start_ms, spec.as_of_ms, - buckets=(buckets, warmup), + buckets=(*spec.buckets.values(), *warmups.values()), ) - if warmup.bucket_count(): - warmup_path = _write_observations( - directory / _WARMUP_FILE, warmup.observations() - ) - observations = tuple(buckets.observations()) - observations_path = None + for interval, warmup in warmups.items(): + if warmup.bucket_count(): + warmup_paths[interval] = _write_observations( + directory / f"{_WARMUP_FILE}-{interval}.json", + warmup.observations(), + ) + observations = { + interval: tuple(buckets.observations()) + for interval, buckets in spec.buckets.items() + } + observations_paths: dict[int, str] = {} if spec.write_observations: - observations_path = _write_observations( - directory / _OBSERVATIONS_FILE, observations - ) + observations_paths = { + interval: _write_observations( + directory / f"{_OBSERVATIONS_FILE}-{interval}.json", rows + ) + for interval, rows in observations.items() + } return _SweepInputs( window_path=window.path, num_requests=window.num_requests, row_counts=row_counts, observations=observations, - observations_path=observations_path, - warmup_path=warmup_path, + observations_paths=observations_paths, + warmup_paths=warmup_paths, ) @@ -692,25 +811,20 @@ def _run_sweep( evaluator: Evaluator, ) -> SweepResult: deadline = started + request.timeout_s - interval_s = _inner_throughput_interval(request.base_planner) - buckets = ( - None - if interval_s is None - else ObservationBuckets( + running = _running_planner(request.baseline, request.base_planner) + intervals = _throughput_intervals(request) + buckets = { + interval: ObservationBuckets( end_ms=request.window_end_ms, - interval_ms=interval_s * 1000.0 * request.arrival_speedup_ratio, - duration_s=float(interval_s), + interval_ms=interval * 1000.0 * request.arrival_speedup_ratio, + duration_s=float(interval), ) - ) + for interval in intervals + } predictors = tuple(request.predictor_candidates or ()) - # (bucket length in trace ms, planner interval in s) when the pre-sweep runs - presweep_interval = ( - (buckets.interval_ms, interval_s) - if predictors and buckets is not None and interval_s - else None - ) + presweep_intervals = intervals if predictors else [] spec = _ReadSpec.from_request( - request, buckets, write_observations=presweep_interval is not None + request, buckets, write_observations=bool(presweep_intervals) ) temp_dir = work_dir / _WORKER_TEMP_DIR @@ -724,7 +838,7 @@ def _run_sweep( # Start every worker the later rounds can use now, so they import # while the first one reads instead of after it. pool.start( - max(len(request.candidates), len(predictors) if presweep_interval else 0) + max(len(request.candidates), len(predictors) * len(presweep_intervals)) ) read = pool.run([(_READ_TASK, _read_sweep_inputs, (spec, str(work_dir)))]) ok, value = read[_READ_TASK] @@ -769,12 +883,13 @@ def _run_sweep( "advisory": False, "batch_scheduling": {"enabled": False}, } - if presweep_interval is not None: - presweep = _run_presweep(request, pool, inputs, *presweep_interval) - fields = None if presweep is None else presweep["fields"] + if presweep_intervals: + presweep = _run_presweeps(request, pool, inputs, presweep_intervals) by_effective: dict[str, tuple[RetuneParams, list[str]]] = {} for candidate in request.candidates: - effective = with_predictor_fields(candidate, request.baseline, fields) + effective = evaluated_params( + candidate, request.baseline, presweep, request.base_planner + ) entry = by_effective.setdefault(effective.key(), (effective, [])) entry[1].append(candidate.key()) tasks: list[tuple[str, Callable[..., Any], tuple[Any, ...]]] = [ @@ -788,7 +903,7 @@ def _run_sweep( effective, dict(request.replicas), inner_base, - inputs.warmup_path, + _warmup_path(inputs, effective, running), request.sla_ttft_ms, request.sla_itl_ms, request.window_s, @@ -797,23 +912,31 @@ def _run_sweep( ) for effective_key, (effective, _) in by_effective.items() ] - if ( - fields - and inner_base is not None - and _changes_predictor(request.baseline, fields) - ): - # Prepare the winner's predictor here, beside the replays, rather - # than on the planner tick that decides to swap it in. It goes - # first, so an ARIMA or Prophet fit on days of history overlaps - # the replays instead of trailing them. - tasks.insert( - 0, - ( - _PREDICTOR_WARMUP_TASK, - _predictor_warmup_task, - ({**inner_base, **fields}, inputs.observations_path), - ), + if inner_base is not None: + # Prepare each interval's new predictor here, beside the replays, + # rather than on the planner tick that decides to swap it in. + # They go first, so an ARIMA or Prophet fit on days of history + # overlaps the replays instead of trailing them. + swaps = _predictor_swaps( + request.baseline, + running, + [effective for effective, _ in by_effective.values()], ) + for interval, planner in swaps.items(): + path = inputs.observations_paths.get(interval) + if path is None: + path = _write_observations( + work_dir / f"{_OBSERVATIONS_FILE}-{interval}.json", + inputs.observations.get(interval, ()), + ) + tasks.insert( + 0, + ( + f"{_PREDICTOR_WARMUP_TASK}:{interval}", + _predictor_warmup_task, + ({**inner_base, **planner}, path), + ), + ) outcomes = pool.run(tasks) except _SweepInterrupted as interrupted: logger.warning( @@ -832,8 +955,10 @@ def _run_sweep( finally: pool.close() - ok, value = outcomes.get(_PREDICTOR_WARMUP_TASK, (True, None)) - predictor_warmup_error = None if ok else value + warmup_errors: dict[int, str] = {} + for task_id, (ok, value) in outcomes.items(): + if task_id.startswith(f"{_PREDICTOR_WARMUP_TASK}:") and not ok: + warmup_errors[int(task_id.rsplit(":", 1)[1])] = value for effective_key, (effective, original_keys) in by_effective.items(): ok, value = outcomes[effective_key] if ok: @@ -858,6 +983,7 @@ def _run_sweep( wall_s, _nonzero(inputs.row_counts), ) + base_interval = scales_on_throughput({}, running) if running else None return SweepResult( status="ok", num_requests=inputs.num_requests, @@ -865,78 +991,155 @@ def _run_sweep( scores=scores, metrics=metrics, predictor_presweep=presweep, - warmup_observations=inputs.observations, - predictor_warmup_error=predictor_warmup_error, + warmup_observations=( + None if base_interval is None else inputs.observations.get(base_interval) + ), + predictor_warmup_error=( + None if base_interval is None else warmup_errors.get(base_interval) + ), wall_s=wall_s, + interval_observations=dict(inputs.observations), + predictor_warmup_errors=warmup_errors, ) -def _nonzero(counts: Mapping[str, int]) -> dict[str, int]: - return {key: value for key, value in counts.items() if value} +def _warmup_path( + inputs: _SweepInputs, effective: RetuneParams, running: Mapping[str, Any] +) -> str | None: + """The inner planner warmup of ``effective``'s throughput interval.""" + if not running: + return None + interval = scales_on_throughput(effective.planner, running) + return None if interval is None else inputs.warmup_paths.get(interval) -def _inner_throughput_interval(base_planner: Mapping[str, Any] | None) -> int | None: - """The inner planner's predictor interval, when it scales on throughput.""" - if base_planner is None or not base_planner.get("enable_throughput_scaling"): - return None - return int(base_planner["throughput_adjustment_interval_seconds"]) +def _predictor_swaps( + baseline: RetuneParams, + running: Mapping[str, Any], + evaluated: Sequence[RetuneParams], +) -> dict[int, dict[str, Any]]: + """Per throughput interval, the planner fields of one candidate whose + predictor a host would have to prepare to swap it in. + + A candidate at an interval needs a new predictor when the baseline does + not scale on throughput, scales at another interval, or has other + predictor fields (``reconfiguration.predictor_rebuild_required``). Every + candidate at an interval carries that interval's pre-sweep winner, so + one of them stands for all; its own planner fields keep the prepared + config valid (a load interval shorter than the throughput interval). + """ + + base_interval = scales_on_throughput({}, running) + swaps: dict[int, dict[str, Any]] = {} + for candidate in evaluated: + interval = scales_on_throughput(candidate.planner, running) + if interval is None or interval in swaps: + continue + predictor = { + key: value + for key, value in candidate.planner.items() + if key in PREDICTOR_FIELDS + } + if interval == base_interval and not _changes_predictor(baseline, predictor): + continue + swaps[interval] = {**candidate.planner, ENABLE_THROUGHPUT: True} + return swaps -def _run_presweep( +def _nonzero(counts: Mapping[str, int]) -> dict[str, int]: + return {key: value for key, value in counts.items() if value} + + +def _throughput_intervals(request: SweepRequest) -> list[int]: + """The throughput intervals an inner planner scales at: the base + planner's and every candidate's (``[]`` without an inner planner).""" + + if request.base_planner is None: + return [] + running = _running_planner(request.baseline, request.base_planner) + intervals = set() + for planner in ( + {}, + *( + {**request.baseline.planner, **candidate.planner} + for candidate in request.candidates + ), + ): + interval = scales_on_throughput(planner, running) + if interval is not None: + intervals.add(interval) + return sorted(intervals) + + +def _run_presweeps( request: SweepRequest, pool: _WorkerPool, inputs: _SweepInputs, - interval_ms: float, - interval_s: int, + intervals: Sequence[int], ) -> dict[str, Any]: - """Score every predictor candidate in its own worker task, then pick one. + """Score every predictor candidate at every throughput interval in its + own worker task, then pick one per interval. ``inputs.observations`` are history and window buckets ending at the window end; only the buckets wholly inside ``W_k`` are forecast and scored, and the ones before them only warm the predictors. Each task - carries the path of the observations file, not the observations. + carries the path of the observations file, not the observations. One + interval gives its summary (``SweepResult.predictor_presweep``); several + give ``{"by_interval": {...}}``. """ - if inputs.observations is None or inputs.observations_path is None: - raise RuntimeError("the pre-sweep needs the read task's observations file") started = time.monotonic() - window_steps = int( - (request.window_end_ms - request.window_start_ms) / interval_ms + 1e-9 - ) - eval_steps = min(len(inputs.observations), window_steps) - warmup_steps = len(inputs.observations) - eval_steps candidates = tuple(request.predictor_candidates or ()) - outcomes = pool.run( - [ + tasks: list[tuple[str, Callable[..., Any], tuple[Any, ...]]] = [] + steps: dict[int, tuple[int, int]] = {} + for interval in intervals: + path = inputs.observations_paths.get(interval) + observations = inputs.observations.get(interval) + if path is None or observations is None: + raise RuntimeError("the pre-sweep needs the read task's observations file") + interval_ms = interval * 1000.0 * request.arrival_speedup_ratio + window_steps = int( + (request.window_end_ms - request.window_start_ms) / interval_ms + 1e-9 + ) + eval_steps = min(len(observations), window_steps) + steps[interval] = (eval_steps, len(observations) - eval_steps) + tasks.extend( ( - f"{_PRESWEEP_TASK}:{label}", + f"{_PRESWEEP_TASK}:{interval}:{label}", _presweep_task, - (inputs.observations_path, entry, interval_s, eval_steps), + (path, entry, interval, eval_steps), ) for label, entry in candidates - ] - ) - losses: dict[str, float | None] = {} - errors: dict[str, str] = {} - for label, _ in candidates: - ok, value = outcomes[f"{_PRESWEEP_TASK}:{label}"] - if ok: - losses[label] = value - else: - # A failed candidate cannot win; the others still compete. - losses[label] = None - errors[label] = value - summary = _presweep_summary( - losses, - dict(candidates), - eval_steps, - warmup_steps, - time.monotonic() - started, - ) - if errors: - summary["errors"] = errors - return summary + ) + outcomes = pool.run(tasks) + summaries: dict[str, dict[str, Any]] = {} + for interval in intervals: + losses: dict[str, float | None] = {} + errors: dict[str, str] = {} + for label, _ in candidates: + ok, value = outcomes[f"{_PRESWEEP_TASK}:{interval}:{label}"] + if ok: + losses[label] = value + else: + # A failed candidate cannot win; the others still compete. + losses[label] = None + errors[label] = value + eval_steps, warmup_steps = steps[interval] + summary = _presweep_summary( + losses, + dict(candidates), + eval_steps, + warmup_steps, + time.monotonic() - started, + ) + summary["interval_seconds"] = interval + if errors: + summary["errors"] = errors + summaries[str(interval)] = summary + if len(summaries) == 1: + return next(iter(summaries.values())) + return {"by_interval": summaries} @dataclass diff --git a/components/src/dynamo/planner/retune/window.py b/components/src/dynamo/planner/retune/window.py index ca2dc132b93d..fa81fe327449 100644 --- a/components/src/dynamo/planner/retune/window.py +++ b/components/src/dynamo/planner/retune/window.py @@ -187,7 +187,7 @@ def read_trace_window( as_of_ms: float, out_path: str | os.PathLike[str], *, - buckets: ObservationBuckets | None = None, + buckets: ObservationBuckets | Sequence[ObservationBuckets] | None = None, ) -> WindowFile: """Write the requests that arrived in ``[start_ms, end_ms)`` to ``out_path``. @@ -195,9 +195,17 @@ def read_trace_window( bare-row ``dynamo.request.trace.v1`` file sorted by (arrival, completion, request_id) with ``request_received_ms`` set to the normalized arrival, so AISimulate replays it as is. ``buckets``, when given, receives every kept - row. + row; a sequence of bucket sets receives it in each. """ + targets: tuple[ObservationBuckets, ...] = ( + () + if buckets is None + else (buckets,) + if isinstance(buckets, ObservationBuckets) + else tuple(buckets) + ) + counts = _new_counts() selected: dict[str, _RequestRow] = {} for row in _iter_selected_rows( @@ -221,8 +229,8 @@ def read_trace_window( with _open_output(out_path) as sink: for row in rows: _count_kept(row, counts) - if buckets is not None: - buckets.add(row.arrival_ms, row.input_length, row.output_tokens) + for target in targets: + target.add(row.arrival_ms, row.input_length, row.output_tokens) sink.write(_dumps(_window_record(row))) sink.write("\n") return WindowFile( diff --git a/components/src/dynamo/planner/simulation/config.py b/components/src/dynamo/planner/simulation/config.py index 4811299c6292..828f5068a7fe 100644 --- a/components/src/dynamo/planner/simulation/config.py +++ b/components/src/dynamo/planner/simulation/config.py @@ -329,17 +329,31 @@ def _validate_preset_conflicts(self) -> PlannerRecommendationConfig: PLANNER_RETUNE_STARTUP_ONLY = frozenset( { "policy", - "scaling_policy", "fpm_sampling", - "enable_throughput_scaling", - "enable_load_scaling", - "throughput_adjustment_interval_seconds", - "load_adjustment_interval_seconds", "max_num_fpm_samples", "fpm_sample_bucket_size", "load_min_observations", } ) +# Public recommendation knobs that are deployment constraints the user owns, +# not tuning knobs. Retune never searches them; the Planner config and the +# runtime control API change them. +PLANNER_RETUNE_USER_CONSTRAINTS = frozenset( + { + "min_workers", + "prefill_min_workers", + "decode_min_workers", + "max_num_gpus", + "min_num_gpus", + } +) +_PLANNER_USER_CONSTRAINT_FIELDS = { + "min_endpoint": "min_workers", + "prefill_min_endpoint": "prefill_min_workers", + "decode_min_endpoint": "decode_min_workers", + "max_gpu_budget": "max_num_gpus", + "min_gpu_budget": "min_num_gpus", +} # Runtime ``PlannerConfig`` fields without an AISimulate public name. Retune # accepts them under their own names. PLANNER_RETUNE_EXTENSIONS: tuple[str, ...] = ( @@ -351,23 +365,17 @@ def _validate_preset_conflicts(self) -> PlannerRecommendationConfig: # Public retune grid knob -> ``PlannerConfig`` field. Retune searches these # only when written; omitted, each keeps its current value. PLANNER_RETUNE_KNOBS: dict[str, str] = { - **{ - name: PLANNER_PUBLIC_TO_CONFIG[name] - for name in ( - "min_workers", - "prefill_min_workers", - "decode_min_workers", - "max_num_gpus", - "min_num_gpus", - "max_throughput_scaling_replicas", - ) - }, + "max_throughput_scaling_replicas": "max_throughput_scaling_replicas", **{name: name for name in PLANNER_RETUNE_EXTENSIONS}, } -# Preset groups retune searches the runtime part of: -# ``load_scaling_down_sensitivity`` in the goodput grid, the load predictor in -# the forecast-loss pre-sweep. -PLANNER_RETUNE_GROUPS: tuple[str, ...] = ("load_sensitivity", "load_predictor") +# Preset groups retune searches the runtime part of: the scaling policy +# (when written) and ``load_scaling_down_sensitivity`` in the goodput grid, +# the load predictor in the forecast-loss pre-sweep. +PLANNER_RETUNE_GROUPS: tuple[str, ...] = ( + "scaling_policy", + "load_sensitivity", + "load_predictor", +) _PLANNER_FIXED_SLA = ("ttft_ms", "itl_ms") _PLANNER_LIVE_NAME_HINTS = { live: public for public, live in PLANNER_RETUNE_KNOBS.items() if live != public @@ -379,19 +387,18 @@ class PlannerRetuneConfig(PlannerRecommendationConfig): running planner can change. Syntax, preset groups and the preset-conflict rules are the recommendation - ones. ``PLANNER_RETUNE_STARTUP_ONLY`` knobs are rejected; ``max_num_gpus`` - and ``max_throughput_scaling_replicas``, scalars in recommend, accept a - domain here, and like ``min_num_gpus`` and ``PLANNER_RETUNE_EXTENSIONS`` - keep their current value when omitted. The SLA (``ttft_ms``/``itl_ms``) is - not a knob. Retune knobs do not accept ``null``, and role minimums do not - list ``null`` (inherit ``min_workers``) yet. + ones. ``PLANNER_RETUNE_STARTUP_ONLY`` knobs are rejected, and so are the + ``PLANNER_RETUNE_USER_CONSTRAINTS`` (worker minimums and GPU budgets): + the user owns them. ``max_throughput_scaling_replicas``, a scalar in + recommend, accepts a domain here, and like ``PLANNER_RETUNE_EXTENSIONS`` + and the ``scaling_policy`` group keeps its current value when omitted. + The SLA (``ttft_ms``/``itl_ms``) is not a knob. Retune knobs do not + accept ``null``. """ policy: PolicyDomain | None = None - # Recommend pins these two to one value; retune searches them, so the - # override widens the field type on purpose. - max_num_gpus: StrictIntDomain | None = None # type: ignore[assignment] - min_num_gpus: StrictIntDomain | None = None + # Recommend pins it to one value; retune searches it, so the override + # widens the field type on purpose. max_throughput_scaling_replicas: IntDomain | None = None # type: ignore[assignment] prefill_scale_up_queue_tokens: StrictIntDomain | None = None prefill_scale_down_queue_tokens: StrictIntDomain | None = None @@ -410,6 +417,21 @@ def _reject_fixed_knobs_and_live_names(cls, value: Any) -> Any: f"during retune; set retune.sla_{name} to change the SLA " "that retune scores against" ) + constraints = sorted( + { + str(_PLANNER_USER_CONSTRAINT_FIELDS.get(name, name)) + for name in value + if name in PLANNER_RETUNE_USER_CONSTRAINTS + or name in _PLANNER_USER_CONSTRAINT_FIELDS + } + ) + if constraints: + raise ValueError( + f"retune.planner cannot search {constraints}: worker minimums " + "and GPU budgets are user constraints, not tuning knobs; " + "change them through the Planner config or the runtime " + "control API" + ) for live, public in _PLANNER_LIVE_NAME_HINTS.items(): if live in value: raise ValueError( @@ -432,11 +454,4 @@ def _reject_null_knob_values(self) -> PlannerRetuneConfig: f"retune.planner.{name} does not accept null; omit it to " "keep the current value" ) - for name in ("prefill_min_workers", "decode_min_workers"): - domain = getattr(self, name) - if isinstance(domain, Choices) and None in domain.choices: - raise ValueError( - f"retune.planner.{name} does not accept null (inherit " - "min_workers) yet; list explicit worker counts" - ) return self diff --git a/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py b/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py index facad9626818..79a763b85b32 100644 --- a/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py +++ b/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py @@ -2169,6 +2169,111 @@ def test_throughput_only_sla_still_pulls_fpm_for_live_regression(): assert not load_interval_tick.run_throughput_scaling +@pytest.mark.asyncio +async def test_runtime_scaling_policy_change_reregisters_builtins(): + # Imported here: reconfiguration pulls in the planner core. + from dynamo.planner.reconfiguration import candidate_config, prepare_predictor + + cfg = PlannerConfig.model_validate( + { + "mode": "agg", + "enable_load_scaling": True, + "enable_throughput_scaling": True, + "optimization_target": "sla", + "served_model_name": "test", + "load_adjustment_interval_seconds": 5, + "throughput_adjustment_interval_seconds": 60, + "load_predictor": "constant", + } + ) + adapter = OrchestratorEngineAdapter(cfg, _caps(), clock=VirtualClock()) + registry = adapter._orchestrator.registry + predict = adapter._builtins["load_predict"] + adapter._scaling_state._throughput_lower_bound_d = 4 + + def apply(updates, predictor=None): + adapter.apply_runtime_configuration( + candidate_config(cfg, updates), list(updates), predictor + ) + + adapter.initial_tick(start_s=0.0) + # Load scaling off: its plugin goes; the predictor and the throughput + # bound stay. + apply({"enable_load_scaling": False}) + assert registry.get_plugin("builtin_load_propose") is None + assert adapter._builtins["load_predict"] is predict + assert adapter._scaling_state._throughput_lower_bound_d == 4 + + # A throughput-only SLA planner still pulls FPM at the load interval and + # refits the performance model, so turning load scaling back on later + # does not start it cold. + fpm_calls: list[FpmObservations] = [] + adapter._scaling_state.observe_fpm = fpm_calls.append # type: ignore[method-assign] + tick = adapter._compute_next_scheduled_tick() + assert tick.need_worker_fpm and tick.run_load_scaling + observations = FpmObservations(decode={("w1", 0): _make_fpm("w1")}) + await adapter.tick(tick, TickInput(now_s=tick.at_s, fpm_observations=observations)) + assert fpm_calls == [observations] + + # Load scaling back on, a new load interval and throughput scaling off: + # the load plugin is registered again, alone, and takes the traffic it + # needs without a predictor; the throughput bound no longer floors it. + apply( + { + "enable_load_scaling": True, + "enable_throughput_scaling": False, + "load_adjustment_interval_seconds": 10, + } + ) + assert set(adapter._builtins) == {"load_propose"} + assert registry.get_plugin("builtin_load_predict") is None + assert registry.get_plugin("builtin_throughput_propose") is None + load = registry.get_plugin("builtin_load_propose") + assert load.execution_interval_seconds == 10.0 + assert "observations.traffic" in load.needs + assert load.observation_window_seconds == 10.0 + assert adapter._scaling_state._throughput_lower_bound_d == 1 + + # Throughput scaling back on needs a prepared predictor; nothing changes + # without one. + with pytest.raises(ValueError, match="needs a new load predictor"): + apply( + { + "enable_throughput_scaling": True, + "throughput_adjustment_interval_seconds": 120, + } + ) + assert cfg.enable_throughput_scaling is False + target = candidate_config( + cfg, + { + "enable_throughput_scaling": True, + "throughput_adjustment_interval_seconds": 120, + }, + ) + prepared = prepare_predictor( + target, _caps(), [TrafficObservation(120.0, 10.0, 100.0, 10.0)] * 3 + ) + apply( + { + "enable_throughput_scaling": True, + "throughput_adjustment_interval_seconds": 120, + }, + prepared, + ) + new_predict = adapter._builtins["load_predict"] + assert new_predict is not predict + assert new_predict._num_req_predictor is prepared._num_req_predictor + for plugin_id in ("builtin_load_predict", "builtin_throughput_propose"): + plugin = registry.get_plugin(plugin_id) + assert plugin.execution_interval_seconds == 120.0 + assert plugin.observation_window_seconds == 120.0 + # The kept load plugin is updated in place: no traffic of its own now. + assert registry.get_plugin("builtin_load_propose") is load + assert "observations.traffic" not in load.needs + assert load.observation_window_seconds == 0.0 + + def test_builtin_first_fire_is_anchored_to_initial_tick_not_construction(): cfg = _agg_config_custom_intervals(load_interval=7.0, throughput_interval=60.0) vc = VirtualClock() diff --git a/components/src/dynamo/planner/tests/unit/test_planner_provider_retune.py b/components/src/dynamo/planner/tests/unit/test_planner_provider_retune.py index 89b0781e8b2b..0ca6983fb3d0 100644 --- a/components/src/dynamo/planner/tests/unit/test_planner_provider_retune.py +++ b/components/src/dynamo/planner/tests/unit/test_planner_provider_retune.py @@ -108,6 +108,29 @@ def test_retune_reaches_the_replay_host_as_written() -> None: assert _hook_config(kept)["retune"] == explicit +def test_predict_retune_searches_the_scaling_policy_not_user_constraints() -> None: + # Imported here: the domains module imports the adapter schemas on use. + from dynamo.planner.retune.domains import expand_planner_domain + + provider = create_provider() + planner = {"scaling_policy": {"preset": ["load_180_5", "load_180_10"]}} + spec = provider.compile_prediction( + {"policy": "enabled", "retune": {**_RETUNE, "planner": planner}}, + _context(sla=_SLA), + ) + + domain = expand_planner_domain(_host_config(spec)) + assert domain.grid["load_adjustment_interval_seconds"] == [5, 10] + with pytest.raises(ValueError, match="user constraints, not tuning knobs"): + provider.compile_prediction( + { + "policy": "enabled", + "retune": {**_RETUNE, "planner": {"max_num_gpus": {"choices": [4, 8]}}}, + }, + _context(sla=_SLA), + ) + + def _write_trace(path: Path, *, count: int, start_ms: int, prefix: str) -> None: rows = [ { diff --git a/components/src/dynamo/planner/tests/unit/test_planner_retune_config.py b/components/src/dynamo/planner/tests/unit/test_planner_retune_config.py index 9d186cf1cff4..be2fb38e21dc 100644 --- a/components/src/dynamo/planner/tests/unit/test_planner_retune_config.py +++ b/components/src/dynamo/planner/tests/unit/test_planner_retune_config.py @@ -14,13 +14,18 @@ reason="AI Simulate is an optional Dynamo simulation dependency", ) -from dynamo.planner.reconfiguration import PREDICTOR_FIELDS, RUNTIME_FIELDS +from dynamo.planner.reconfiguration import ( + PREDICTOR_FIELDS, + RUNTIME_FIELDS, + SCALING_POLICY_FIELDS, +) from dynamo.planner.simulation.config import ( PLANNER_PRESET_GROUPS, PLANNER_PUBLIC_TO_CONFIG, PLANNER_RETUNE_GROUPS, PLANNER_RETUNE_KNOBS, PLANNER_RETUNE_STARTUP_ONLY, + PLANNER_RETUNE_USER_CONSTRAINTS, PlannerRetuneConfig, ) @@ -42,22 +47,34 @@ def test_every_retune_field_is_classified_and_reaches_a_runtime_field() -> None: for knob in PLANNER_PRESET_GROUPS[group] if knob in RUNTIME_FIELDS } + # The user constraints are recommendation fields retune rejects; + # min_num_gpus has no recommendation field. + constraints = PLANNER_RETUNE_USER_CONSTRAINTS & set( + PlannerRetuneConfig.model_fields + ) classes = [ PLANNER_RETUNE_STARTUP_ONLY, set(PLANNER_RETUNE_KNOBS), set(PLANNER_RETUNE_GROUPS), runtime_group_knobs, + constraints, ] fields = set(PlannerRetuneConfig.model_fields) assert set().union(*classes) == fields assert sum(len(names) for names in classes) == len(fields) - # Retune reaches every runtime field except the SLA. + # Retune reaches every runtime field except the SLA and the user + # constraints, which the control API changes. reachable = ( set(PLANNER_RETUNE_KNOBS.values()) | {"load_scaling_down_sensitivity"} | PREDICTOR_FIELDS + | SCALING_POLICY_FIELDS ) - assert reachable == RUNTIME_FIELDS - {"ttft_ms", "itl_ms"} + user_constraint_fields = { + PLANNER_PUBLIC_TO_CONFIG[name] for name in PLANNER_RETUNE_USER_CONSTRAINTS + } + assert reachable == RUNTIME_FIELDS - {"ttft_ms", "itl_ms"} - user_constraint_fields + assert user_constraint_fields <= RUNTIME_FIELDS assert all( PLANNER_PUBLIC_TO_CONFIG.get(public, public) == live for public, live in PLANNER_RETUNE_KNOBS.items() diff --git a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py index d41db54c0851..7c760d4f5f2a 100644 --- a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py +++ b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py @@ -585,7 +585,7 @@ def blocked(config, capabilities, _trace): @pytest.mark.parametrize( "changes", [ - {"throughput_adjustment_interval_seconds": 60}, + {"max_num_fpm_samples": 32}, {"ais_perf_model": None}, {"ttft_ms": -1.0}, {"ttft_ms": True}, @@ -718,3 +718,130 @@ def test_native_planner_rejects_retune_config(): "router": {"overlap_score_credit": {"choices": [0.0, 1.0]}}, }, ) + + +class _FakeFpmProvider: + def __init__(self, age): + self.max_fpm_age_seconds = age + self.ages = [] + + def set_max_fpm_age_seconds(self, age): + self.ages.append(age) + self.max_fpm_age_seconds = age + + +async def test_scaling_policy_is_a_runtime_field(tmp_path): + # SLA, throughput scaling at 180 s, load scaling off, load interval 5 s. + planner = sla_planner(runtime_warmup_trace_dir=str(tmp_path)) + provider = _FakeFpmProvider(planner.config.fpm_max_age_seconds) + planner.environment.fpm_provider = provider + engine = planner._ensure_engine() + predictor = engine._builtins["load_predict"] + assert reconfiguration.SCALING_POLICY_FIELDS <= set( + (await planner.reconfiguration.get_config())["supported_fields"] + ) + + # Load scaling on at 10 s: the predictor is kept, the FPM window follows. + await planner.reconfiguration.submit( + request( + planner, + {"enable_load_scaling": True, "load_adjustment_interval_seconds": 10}, + ) + ) + assert (await finish(planner))["state"] == "applied" + assert engine._builtins["load_predict"] is predictor + assert "load_propose" in engine._builtins + assert provider.ages == [30.0] + + # Both scaling modes off is not a runtime configuration. + off = await planner.reconfiguration.handle_request( + { + "method": "PATCH", + "path": "config", + "body": request( + planner, + {"enable_load_scaling": False, "enable_throughput_scaling": False}, + ) + .model_copy(update={"request_id": "off", "expected_revision": 1}) + .model_dump(), + } + ) + assert off["status"] == 422 + assert "At least one scaling mode must be enabled" in off["body"]["error"] + + # A new throughput interval is a new predictor step: it needs a warmup. + second = request( + planner, {"throughput_adjustment_interval_seconds": 360} + ).model_copy(update={"request_id": "update-2", "expected_revision": 1}) + with pytest.raises(ValueError, match="predictor replacement requires"): + await planner.reconfiguration.submit(second) + await planner.reconfiguration.submit( + second.model_copy(update={"warmup_trace": trace(tmp_path)}) + ) + await asyncio.gather(planner.reconfiguration._task, return_exceptions=True) + operation = planner.reconfiguration.get_operation("update-2") + assert operation["state"] == "applied", operation + assert planner.config.throughput_adjustment_interval_seconds == 360 + plugin = engine._orchestrator.registry.get_plugin("builtin_load_predict") + assert plugin.execution_interval_seconds == 360.0 + await planner._shutdown_runtime() + + +def test_runtime_fpm_provider_updates_both_subscribers_in_place(): + from dynamo.planner.environment.metrics_provider.runtime_provider import ( + RuntimeFpmProvider, + ) + + class Subscriber: + def __init__(self): + self.ages = [] + + def set_max_age(self, *, max_age_seconds): + self.ages.append(max_age_seconds) + + provider = RuntimeFpmProvider( + require_prefill=True, + require_decode=True, + backend="vllm", + model_name=None, + runtime=None, + max_fpm_age_seconds=15.0, + ) + prefill, decode = Subscriber(), Subscriber() + provider._prefill_fpm_sub, provider._decode_fpm_sub = prefill, decode + + provider.set_max_fpm_age_seconds(30.0) + + assert (prefill.ages, decode.ages) == ([30.0], [30.0]) + # A subscriber created later starts with the new age. + assert provider.max_fpm_age_seconds == 30.0 + + +async def test_run_loop_polls_at_the_current_load_interval(monkeypatch): + # The loop sleeps a tenth of the load interval between tick checks; a + # runtime change of the interval takes effect at the next check. + class FutureTickEngine: + def initial_tick(self, start_s): + return ScheduledTick(at_s=start_s + 1_000.0) + + async def tick(self, scheduled_tick, tick_input): + raise AssertionError("no tick is due") + + async def shutdown(self): + pass + + planner = _planner("agg", load_adjustment_interval_seconds=10) + planner._engine = FutureTickEngine() + sleeps = [] + + async def sleep(seconds): + sleeps.append(seconds) + if len(sleeps) == 1: + planner.config.load_adjustment_interval_seconds = 20 + else: + raise asyncio.CancelledError + + monkeypatch.setattr("dynamo.planner.core.base.asyncio.sleep", sleep) + with pytest.raises(asyncio.CancelledError): + await planner.run() + assert sleeps == [1.0, 2.0] diff --git a/components/src/dynamo/planner/tests/unit/test_retune_config.py b/components/src/dynamo/planner/tests/unit/test_retune_config.py index 063644070ebf..e861f39fc5c4 100644 --- a/components/src/dynamo/planner/tests/unit/test_retune_config.py +++ b/components/src/dynamo/planner/tests/unit/test_retune_config.py @@ -108,18 +108,19 @@ def _config(mode="disagg", retune=None, **overrides): planner={ "prefill_scale_up_queue_tokens": {"choices": [800, 1200]}, "decode_scale_down_kv_rate": {"choices": [30, 50.0]}, - "decode_min_workers": 2, - "min_num_gpus": {"choices": [-1, 4]}, + "scaling_policy": {"preset": ["load_180_5", "load_180_10"]}, } ), {"decode_scale_down_kv_rate": 50.0, LSD: 70}, ROUTER_GRID, { + "enable_throughput_scaling": [False], + "enable_load_scaling": [True], + "throughput_adjustment_interval_seconds": [180], + "load_adjustment_interval_seconds": [5, 10], LSD: [70, 80, 90], "prefill_scale_up_queue_tokens": [800, 1200], "decode_scale_down_kv_rate": [30.0, 50.0], - "decode_min_endpoint": [2], - "min_gpu_budget": [-1, 4], }, ), # The load predictor goes to the pre-sweep, not the grid. @@ -175,8 +176,8 @@ def test_retune_round_trips_through_strict_runtime_reconfiguration( ("retune", "match"), [ ( - {"planner": {"enable_load_scaling": {"choices": [True, False]}}}, - r"cannot change \['enable_load_scaling'\]: they are fixed when the " + {"planner": {"max_num_fpm_samples": {"choices": [32, 64]}}}, + r"cannot change \['max_num_fpm_samples'\]: they are fixed when the " "planner starts", ), ({"router": {"policy": "round_robin"}}, "policy is fixed at startup"), @@ -200,6 +201,33 @@ def test_startup_only_knobs_are_rejected(retune, match): _config(retune=_retune(**retune)) +@pytest.mark.parametrize( + ("name", "public"), + [ + ("min_workers", "min_workers"), + ("prefill_min_workers", "prefill_min_workers"), + ("decode_min_workers", "decode_min_workers"), + ("max_num_gpus", "max_num_gpus"), + ("min_num_gpus", "min_num_gpus"), + # A live name is reported under its public name. + ("max_gpu_budget", "max_num_gpus"), + ("decode_min_endpoint", "decode_min_workers"), + ], +) +def test_user_constraints_are_never_searched(name, public): + # Worker minimums and GPU budgets belong to the user: retune rejects + # them like the SLA, and the control API still changes them. + with pytest.raises( + ValidationError, + match=rf"retune.planner cannot search \['{public}'\]: worker minimums and " + "GPU budgets are user constraints, not tuning knobs; change them through " + "the Planner config or the runtime control API", + ): + _config(retune=_retune(planner={name: {"choices": [1, 2]}}), **LOAD_TARGET) + config = _config(retune=_retune(), **LOAD_TARGET) + assert candidate_config(config, {"max_gpu_budget": 4}).max_gpu_budget == 4 + + @pytest.mark.parametrize( ("overrides", "retune", "match"), [ diff --git a/components/src/dynamo/planner/tests/unit/test_retune_controller.py b/components/src/dynamo/planner/tests/unit/test_retune_controller.py index ab08e18646b7..24244b8895e8 100644 --- a/components/src/dynamo/planner/tests/unit/test_retune_controller.py +++ b/components/src/dynamo/planner/tests/unit/test_retune_controller.py @@ -20,7 +20,6 @@ from dynamo.llm import KvRouterConfig from dynamo.planner.config.planner_config import PlannerConfig, RetuneEngineSpec from dynamo.planner.core.types import ( - EngineCapabilities, RetuneInputs, RetuneParams, TrafficObservation, @@ -806,6 +805,56 @@ async def test_predictor_presweep_winner_is_applied_with_warmup( assert decision.predictor_presweep == presweep +@pytest.mark.parametrize("warmup_error", [None, "ValueError: too few points"]) +async def test_a_new_throughput_interval_is_applied_with_its_own_warmup( + sweeps, warmup_error +): + # Throughput scaling every 10 s; retune may move it to 20 s. + policies = [ + { + "enable_throughput_scaling": True, + "enable_load_scaling": False, + "throughput_adjustment_interval_seconds": interval, + "load_adjustment_interval_seconds": 5, + } + for interval in (10, 20) + ] + config = _config( + "agg", + {"router": None, "planner": {"scaling_policy": {"preset": policies}}}, + optimization_target="sla", + enable_load_scaling=False, + enable_throughput_scaling=True, + throughput_adjustment_interval_seconds=10, + ) + + def buckets(interval): + return tuple( + TrafficObservation(float(interval), float(i), 1.0, 1.0) for i in range(5) + ) + + sweeps.default = _scored( + lambda c, _r: 2.0 + if c.planner["throughput_adjustment_interval_seconds"] == 20 + else 1.0, + interval_observations={10: buckets(10), 20: buckets(20)}, + predictor_warmup_errors={} if warmup_error is None else {20: warmup_error}, + ) + controller = _controller(config) + + ticks = await _run(controller, 70) + + ((_, decision),) = _decisions(ticks) + if warmup_error is not None: + assert decision.outcome == "failed" + assert decision.reason == f"predictor warmup rejected: {warmup_error}" + return + assert decision.outcome == "applied" + assert decision.planner_changes == {"throughput_adjustment_interval_seconds": 20} + # The predictor steps every 20 s now: warmed on 20-s buckets. + assert decision.predictor_warmup == buckets(20) + + def _sla_predictor_config(planner, **overrides): return _config( "agg", @@ -1047,51 +1096,77 @@ def test_planner_grid_points_that_do_not_fit_are_dropped(caplog): assert "retune drops 3 of 12 planner grid points" in record.getMessage() -def test_planner_values_that_never_fit_the_deployment_raise(): +LOAD_THRESHOLDS = { + "optimization_target": "load", + "decode_scale_up_kv_rate": 80.0, + "decode_scale_down_kv_rate": 50.0, +} + + +def test_planner_values_that_never_fit_raise(): + # Each point is validated as a full PlannerConfig: an up rate must exceed + # the running down rate, so 40 fits no point. config = _config( "agg", - {"router": None, "planner": {"min_workers": {"choices": [1, 4]}}}, - optimization_target="throughput", - max_gpu_budget=2, + {"router": None, "planner": {"decode_scale_up_kv_rate": {"choices": [40, 90]}}}, + **LOAD_THRESHOLDS, ) - capabilities = WorkerCapabilities(decode=EngineCapabilities(num_gpu=1)) with pytest.raises( - ValueError, match="no retune.planner candidate with min_endpoint=4 fits" + ValueError, match="no retune.planner candidate with decode_scale_up_kv_rate=40" ): - _controller(config, capabilities=capabilities) + _controller(config) + + +def test_points_that_fail_validation_are_dropped(): + # Up 60 fails with down 70 only: the point is dropped, both values stay. + config = _config( + "agg", + { + "router": None, + "planner": { + "decode_scale_up_kv_rate": {"choices": [60, 90]}, + "decode_scale_down_kv_rate": {"choices": [40, 70]}, + }, + }, + **LOAD_THRESHOLDS, + ) + + grid = { + (p.planner["decode_scale_up_kv_rate"], p.planner["decode_scale_down_kv_rate"]) + for p in _controller(config)._grid + } + + assert grid == {(60.0, 40.0), (90.0, 40.0), (90.0, 70.0)} async def test_planner_changes_that_no_longer_fit_fail_the_decision(sweeps): - # A grid point fit at startup, but the budget dropped before the decision. + # A grid point was valid at startup, but the down rate rose before the + # decision. config = _config( "agg", - {"router": None, "planner": {"min_workers": {"choices": [1, 2]}}}, - optimization_target="throughput", - max_gpu_budget=4, + {"router": None, "planner": {"decode_scale_up_kv_rate": {"choices": [60, 90]}}}, + **LOAD_THRESHOLDS, ) sweeps.default = _scored( - lambda c, _r: 2.0 if c.planner["min_endpoint"] == 2 else 1.0 - ) - controller = _controller( - config, capabilities=WorkerCapabilities(decode=EngineCapabilities(num_gpu=1)) + lambda c, _r: 2.0 if c.planner["decode_scale_up_kv_rate"] == 60.0 else 1.0 ) + controller = _controller(config) - def lower_budget(now, _result): + def raise_down_rate(now, _result): if now == 65.0: - config.max_gpu_budget = 1 + config.decode_scale_down_kv_rate = 70.0 - ticks = await _run(controller, 70, after_tick=lower_budget) + ticks = await _run(controller, 70, after_tick=raise_down_rate) ((_, decision),) = _decisions(ticks) assert decision.outcome == "failed" assert decision.reason.startswith("planner candidate ") - assert decision.reason.endswith( - "cannot run on this deployment: minimum endpoint footprint requires " - "2 GPUs, exceeding max_gpu_budget=1" + assert "decode_scale_up_kv_rate must be greater than decode_scale_down_kv_rate" in ( + decision.reason ) assert decision.planner_changes == {} - assert controller.current_point().planner["min_endpoint"] == 1 + assert controller.current_point().planner["decode_scale_up_kv_rate"] == 80.0 def test_scaling_retune_rejects_external_plugin_gateways(): diff --git a/components/src/dynamo/planner/tests/unit/test_retune_domains.py b/components/src/dynamo/planner/tests/unit/test_retune_domains.py index 72a37f4b2087..6b2244c4f215 100644 --- a/components/src/dynamo/planner/tests/unit/test_retune_domains.py +++ b/components/src/dynamo/planner/tests/unit/test_retune_domains.py @@ -20,8 +20,10 @@ expand_planner_domain, expand_router_domain, grid_size, + required_history_seconds, + throughput_intervals, ) -from dynamo.planner.retune.sweep import with_predictor_fields +from dynamo.planner.retune.sweep import evaluated_params, with_predictor_fields from dynamo.planner.simulation.load_predictor import ( LOAD_PREDICTOR_PRESETS, predictor_fields, @@ -38,11 +40,19 @@ LSD = "load_scaling_down_sensitivity" -def _config(planner="default", **overrides): +# One router point, so the planner grid alone sizes the search. +PINNED_ROUTER = { + "overlap_score_credit": 1.0, + "prefill_load_scale": 1.0, + "temperature": 0.0, +} + + +def _config(planner="default", router="default", **overrides): retune = { "interval_seconds": 600, "history_seconds": 30000, - "router": "default", + "router": router, "planner": planner, } return PlannerConfig(namespace="test-ns", mode="disagg", retune=retune, **overrides) @@ -117,3 +127,220 @@ def test_custom_predictor_mapping_values_take_their_field_types(): with_predictor_fields(RetuneParams(), baseline, predictor_fields(entry)) == baseline ) + + +# --- scaling policy ------------------------------------------------------- + +THROUGHPUT = "enable_throughput_scaling" +LOAD = "enable_load_scaling" +TI = "throughput_adjustment_interval_seconds" +LI = "load_adjustment_interval_seconds" +SLA_HYBRID = {"enable_load_scaling": True, **SLA_THROUGHPUT} + + +def _points(domain, *keys): + return sorted(tuple(point[key] for key in keys) for point in domain.points) + + +def test_scaling_policy_is_searched_only_when_written(): + domain = expand_planner_domain(_config(router=PINNED_ROUTER, **SLA_HYBRID)) + + assert not set(domain.grid) & {THROUGHPUT, LOAD, TI, LI} + assert _points(domain, LSD) == [(70,), (80,), (90,)] + + +def test_default_scaling_policies_fold_inactive_fields_and_deduplicate(): + # The six AISimulate presets that scale; "disabled" is never a candidate. + domain = expand_planner_domain( + _config( + router=PINNED_ROUTER, + planner={"scaling_policy": {"preset": "default"}}, + **SLA_HYBRID, + ) + ) + + assert domain.grid[THROUGHPUT] == [True, False] + # Throughput-only points fold the load sensitivity to the running 80 and + # load-only points the throughput interval to the running 180, so each + # appears once per distinct active value: 2 + 2 x 3 + 2 x 3 points. + assert _points(domain, THROUGHPUT, LOAD, TI, LI, LSD) == sorted( + [ + (True, False, 180, 5, 80), + (True, False, 600, 5, 80), + *((False, True, 180, li, lsd) for li in (5, 10) for lsd in (70, 80, 90)), + *((True, True, ti, 5, lsd) for ti in (180, 600) for lsd in (70, 80, 90)), + ] + ) + # The pre-sweep runs: some point scales on throughput. + assert len(domain.predictor_candidates) == len(LOAD_PREDICTOR_PRESETS) + + +def test_independent_scaling_knobs_combine_and_drop_invalid_points(): + domain = expand_planner_domain( + _config( + router=PINNED_ROUTER, + planner={ + "scaling_policy": {"preset": False}, + THROUGHPUT: True, + LOAD: {"choices": [False, True]}, + TI: {"choices": [10, 180]}, + LI: {"choices": [5, 10]}, + "load_sensitivity": {"preset": False}, + LSD: 80, + }, + **SLA_THROUGHPUT, + ) + ) + + # Hybrid points need the load interval below the throughput interval: + # (10, 10) is dropped. + assert _points(domain, LOAD, TI, LI) == sorted( + [ + (False, 10, 5), + (False, 10, 10), + (False, 180, 5), + (False, 180, 10), + (True, 10, 5), + (True, 180, 5), + (True, 180, 10), + ] + ) + + +@pytest.mark.parametrize( + ("overrides", "policy", "match"), + [ + ( + SLA_HYBRID, + ["disabled"], + "'disabled' turns both scaling modes off; retune never turns " + "scaling off", + ), + ( + SLA_HYBRID, + [{THROUGHPUT: True, LOAD: True, TI: 10, LI: 10}], + "load_adjustment_interval_seconds 10 not shorter than " + "throughput_adjustment_interval_seconds 10", + ), + ( + {**SLA_HYBRID, LI: 10, "scheduling": {"scale_interval_seconds": 10}}, + ["load_180_5"], + "load_adjustment_interval_seconds 5, not a multiple of the planner's " + r"scheduling.scale_interval_seconds 10 \(fixed at startup\)", + ), + # Other targets always scale on load only. + ({}, ["hybrid_180_5"], "has no effect here"), + ], + ids=["disabled", "interval-order", "scale-interval", "easy-mode"], +) +def test_written_scaling_policies_that_cannot_run_are_rejected( + overrides, policy, match +): + with pytest.raises(ValueError, match=match): + expand_planner_domain( + _config( + router=PINNED_ROUTER, + planner={"scaling_policy": {"preset": policy}}, + **overrides, + ) + ) + + +def test_easy_mode_searches_the_load_interval_only(): + domain = expand_planner_domain( + _config(router=PINNED_ROUTER, planner={"scaling_policy": {"preset": "default"}}) + ) + + # load_180_5 and load_180_10, each with the three sensitivities; the + # throughput interval folds away. + assert _points(domain, THROUGHPUT, LOAD, TI, LI, LSD) == [ + (False, True, 180, li, lsd) for li in (5, 10) for lsd in (70, 80, 90) + ] + assert domain.predictor_candidates is None + with pytest.raises(ValueError, match="require optimization_target='sla'"): + expand_planner_domain( + _config( + router=PINNED_ROUTER, + planner={"scaling_policy": {"preset": False}, THROUGHPUT: True}, + ) + ) + + +def test_a_policy_that_adds_throughput_scaling_brings_the_presweep_and_history(): + # A load-only planner whose grid can turn throughput scaling on at 600 s. + config = _config( + router=PINNED_ROUTER, + planner={"scaling_policy": {"preset": ["load_180_5", "hybrid_600_5"]}}, + optimization_target="sla", + enable_throughput_scaling=False, + enable_load_scaling=True, + ) + + domain = expand_planner_domain(config) + + assert len(domain.predictor_candidates) == len(LOAD_PREDICTOR_PRESETS) + assert throughput_intervals(config, domain.points) == [600] + # 600 s x the largest prophet_window_size / kalman_min_points. + assert required_history_seconds(config, domain) % 600 == 0 + assert required_history_seconds(config, domain) >= 600 * config.prophet_window_size + + +def test_max_throughput_scaling_replicas_needs_a_throughput_point(): + planner = {"max_throughput_scaling_replicas": {"choices": [4, 8]}} + with pytest.raises(ValueError, match="requires enable_throughput_scaling"): + expand_planner_domain( + _config( + router=PINNED_ROUTER, + planner=planner, + optimization_target="sla", + enable_throughput_scaling=False, + enable_load_scaling=True, + ) + ) + + domain = expand_planner_domain( + _config( + router=PINNED_ROUTER, + planner={ + **planner, + "scaling_policy": {"preset": ["load_180_5", "hybrid_180_5"]}, + }, + optimization_target="sla", + enable_throughput_scaling=False, + enable_load_scaling=True, + ) + ) + # Load-only points fold the replicas to the running 8. + assert _points( + domain, THROUGHPUT, "max_throughput_scaling_replicas", LSD + ) == sorted( + [ + *((False, 8, lsd) for lsd in (70, 80, 90)), + *((True, replicas, lsd) for replicas in (4, 8) for lsd in (70, 80, 90)), + ] + ) + + +def test_evaluated_params_fold_inactive_fields_to_the_baseline(): + base = PlannerConfig(namespace="test-ns", mode="disagg", **SLA_HYBRID).model_dump( + mode="json" + ) + baseline = RetuneParams(planner={THROUGHPUT: True, LSD: 70}) + # Throughput-only: its sensitivity is inert, so it is the baseline's. + candidate = RetuneParams(planner={THROUGHPUT: True, LOAD: False, LSD: 90}) + + evaluated = evaluated_params(candidate, baseline, None, base) + + assert evaluated.planner == {THROUGHPUT: True, LOAD: False, LSD: 70} + # The pre-sweep winner of the candidate's interval only. + presweep = { + "by_interval": { + "180": {"fields": {"load_predictor": "kalman"}}, + "600": {"fields": {"load_predictor": "arima"}}, + } + } + at_600 = RetuneParams(planner={THROUGHPUT: True, TI: 600, LSD: 70}) + assert ( + evaluated_params(at_600, baseline, presweep, base).planner["load_predictor"] + == "arima" + ) diff --git a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py index 4098ea8bfe79..8263e7167ee4 100644 --- a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py +++ b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py @@ -50,6 +50,8 @@ write_trace, ) +THROUGHPUT_INTERVAL = "throughput_adjustment_interval_seconds" + pytestmark = [ pytest.mark.gpu_0, pytest.mark.pre_merge, @@ -257,19 +259,21 @@ def test_read_task_writes_the_window_and_buckets_the_history(tmp_path): ) def read(out, buckets_interval_ms=10_000.0, **changes): + interval = int(buckets_interval_ms / 1000) buckets = ObservationBuckets( request.window_end_ms, buckets_interval_ms, buckets_interval_ms / 1000 ) spec = sweep_module._ReadSpec.from_request( - request, buckets, write_observations=True + request, {interval: buckets}, write_observations=True ) return sweep_module._read_sweep_inputs( dataclasses.replace(spec, **changes), str(tmp_path / out) ) - def warmup(inputs): + def warmup(inputs, interval=10): return [ - obs.num_req for obs in sweep_module._read_observations(inputs.warmup_path) + obs.num_req + for obs in sweep_module._read_observations(inputs.warmup_paths[interval]) ] inputs = read("out") @@ -279,25 +283,45 @@ def warmup(inputs): assert inputs.row_counts["history_requests"] == 20 # History and window share the pre-sweep buckets, which end at the # window end; the pre-sweep workers load them from a file. - assert [obs.num_req for obs in inputs.observations] == [10.0, 10.0, 10.0] - assert sweep_module._read_observations(inputs.observations_path) == list( - inputs.observations + assert [obs.num_req for obs in inputs.observations[10]] == [10.0, 10.0, 10.0] + assert sweep_module._read_observations(inputs.observations_paths[10]) == list( + inputs.observations[10] ) # The inner planner's warmup is the history alone, in buckets that end at # the window start: off the grid of the window end, a pre-sweep bucket # straddles the window start, but no warmup bucket does. assert warmup(inputs) == [10.0, 10.0] - assert warmup(read("off-grid", 7_000.0)) == [6.0, 7.0, 7.0] + assert warmup(read("off-grid", 7_000.0), 7) == [6.0, 7.0, 7.0] + + # Candidates at several throughput intervals: one read buckets the + # history and the window at each of them. + both = sweep_module._read_sweep_inputs( + sweep_module._ReadSpec.from_request( + request, + { + interval: ObservationBuckets( + request.window_end_ms, interval * 1000.0, float(interval) + ) + for interval in (5, 10) + }, + write_observations=False, + ), + str(tmp_path / "both"), + ) + assert [obs.num_req for obs in both.observations[5]] == [5.0] * 6 + assert [obs.num_req for obs in both.observations[10]] == [10.0] * 3 + assert sorted(both.warmup_paths) == [5, 10] + assert both.observations_paths == {} # The history is read only for a planner that scales on throughput (the # only one with a load predictor), and not for an empty window. - no_scaling = read("no-scaling", buckets=None) + no_scaling = read("no-scaling", buckets={}) empty = read( "empty", window_start_ms=ORIGIN_MS + 60_000, window_end_ms=ORIGIN_MS + 70_000 ) for unread, num_requests in [(no_scaling, 10), (empty, 0)]: assert unread.num_requests == num_requests - assert (unread.observations, unread.warmup_path) == (None, None) + assert (unread.observations, unread.warmup_paths) == ({}, {}) assert "history_requests" not in unread.row_counts # A read error propagates: the worker reports it as the task's failure. with pytest.raises(ValueError, match="No warmup trace files matched"): @@ -551,7 +575,7 @@ def evaluator(*args): def run_with_winner(fields): presweep = {"winner": "w", "fields": fields, "losses": {"w": 0.1}} - monkeypatch.setattr(sweep_module, "_run_presweep", lambda *_: presweep) + monkeypatch.setattr(sweep_module, "_run_presweeps", lambda *_: presweep) tasks.clear() return sweep_module.run_sweep(request, evaluator=evaluator) @@ -570,6 +594,71 @@ def run_with_winner(fields): assert tasks == ["prepare", "replay", "replay"] +def test_candidates_at_other_throughput_intervals_warm_at_their_own( + tmp_path, monkeypatch +): + # The baseline scales on throughput every 5 s; one candidate every 10 s + # and one on load only. Each inner planner is warmed with history at its + # own interval, and a swap to the 10-s predictor is prepared on buckets + # of that interval. + monkeypatch.setattr(sweep_module, "_WorkerPool", _InlinePool) + prepared = [] + warmups = {} + + def prepare_predictor(config, capabilities, observations): + prepared.append( + (config.throughput_adjustment_interval_seconds, list(observations)) + ) + + def evaluator(*args): + params, warmup_path = args[3], args[6] + # The sweep directory goes with the sweep: read the warmup now. + warmups[params.router["overlap_score_credit"]] = ( + None + if warmup_path is None + else [obs.num_req for obs in sweep_module._read_observations(warmup_path)] + ) + return fake_evaluator(*args) + + monkeypatch.setattr(sweep_module, "prepare_predictor", prepare_predictor) + trace = write_trace( + tmp_path / "trace.jsonl", + trace_rows([ORIGIN_MS + 1_000 * i for i in range(-20, 10)]), + ) + baseline = retune_params(1.0) + slow = retune_params(0.0, **{THROUGHPUT_INTERVAL: 10}) + load_only = retune_params( + 0.5, enable_throughput_scaling=False, enable_load_scaling=True + ) + request = sweep_request( + [trace], + [baseline, slow, load_only], + baseline, + history_start_ms=ORIGIN_MS - 20_000, + base_planner={ + "mode": "agg", + "optimization_target": "sla", + "enable_throughput_scaling": True, + "enable_load_scaling": False, + THROUGHPUT_INTERVAL: 5, + "load_adjustment_interval_seconds": 5, + }, + ) + + result = sweep_module.run_sweep(request, evaluator=evaluator) + + assert result.status == "ok", result.error + assert sorted(result.interval_observations) == [5, 10] + assert [obs.num_req for obs in result.observations_at(5)] == [5.0] * 6 + assert [obs.num_req for obs in result.observations_at(10)] == [10.0] * 3 + assert result.warmup_observations == result.observations_at(5) + # The 20 s of history before the window, at each candidate's interval; + # a load-only inner planner has no predictor to warm. + assert warmups == {1.0: [5.0] * 4, 0.0: [10.0] * 2, 0.5: None} + assert prepared == [(10, list(result.observations_at(10)))] + assert result.predictor_warmup_errors == {} + + # --------------------------------------------------------------------------- # Predictor pre-sweep # --------------------------------------------------------------------------- @@ -672,8 +761,8 @@ def test_presweep_dispatch_does_not_wait_for_workers_that_are_not_reading(tmp_pa str(tmp_path / "window.jsonl"), 1, {}, - observations=tuple(observations), - observations_path=path, + observations={60: tuple(observations)}, + observations_paths={60: path}, ) presets = ("constant_last", "kalman_default_raw") request = sweep_request( @@ -703,7 +792,7 @@ def importing_worker(): timer.start() try: with pytest.raises(sweep_module._SweepInterrupted, match="cancelled"): - sweep_module._run_presweep(request, pool, inputs, 60_000.0, 60) + sweep_module._run_presweeps(request, pool, inputs, [60]) finally: timer.cancel() pool.close() @@ -712,7 +801,7 @@ def importing_worker(): # Every preset was dispatched, as a message that carries only the path. messages = [conn.recv() for conn in unread] assert sorted(message[0] for message in messages) == [ - f"predictor-presweep:{name}" for name in presets + f"predictor-presweep:60:{name}" for name in presets ] for task_id, function, args in messages: assert function is sweep_module._presweep_task diff --git a/components/src/dynamo/replay/tests/test_retune_replay.py b/components/src/dynamo/replay/tests/test_retune_replay.py index 791292404e9a..4e2a81586e2b 100644 --- a/components/src/dynamo/replay/tests/test_retune_replay.py +++ b/components/src/dynamo/replay/tests/test_retune_replay.py @@ -245,6 +245,76 @@ async def recording_tick(self, scheduled_tick, tick_input): assert {policy["parameters"]["router_temperature"] for policy in later} == {0.0} +@pytest.mark.integration +@pytest.mark.timeout(300) +def test_retune_lands_a_scaling_policy_change_mid_run(tmp_path, monkeypatch): + # The router temperature wins the window; the load interval rides along. + # Neither grid interval is the running 5 s, so the decision changes it, + # and the running engine re-registers its load plugin at the new one. + traces = _write_workload(tmp_path, blocks=16, output_tokens=128) + seen: list[tuple[float, int, float]] = [] + original_tick = OrchestratorEngineAdapter.tick + + async def recording_tick(self, scheduled_tick, tick_input): + plugin = self._orchestrator.registry.get_plugin("builtin_load_propose") + seen.append( + ( + tick_input.now_s, + self._config.load_adjustment_interval_seconds, + plugin.execution_interval_seconds, + ) + ) + return await original_tick(self, scheduled_tick, tick_input) + + monkeypatch.setattr(OrchestratorEngineAdapter, "tick", recording_tick) + planner_config = { + "mode": "agg", + "optimization_target": "throughput", + "report_interval_hours": None, + "live_dashboard_port": 0, + "min_endpoint": 4, + "retune": { + "interval_seconds": _INTERVAL_S, + "grace_seconds": 1, + "max_parallel": 2, + "sla_ttft_ms": 60.0, + "sla_itl_ms": 100.0, + "router": _ROUTER_DOMAIN, + "planner": { + "scaling_policy": {"preset": False}, + "load_adjustment_interval_seconds": {"choices": [10, 15]}, + "load_sensitivity": {"preset": False}, + "load_scaling_down_sensitivity": 80, + }, + }, + } + + report = run_trace_replay( + traces, + trace_format="dynamo", + router_mode="kv_router", + router_config=KvRouterConfig(router_temperature=100.0), + num_workers=4, + extra_engine_args=_engine(), + planner_config=planner_config, + ) + + assert report.summary["completed_requests"] == _HEAD_REQUESTS + _TAIL_REQUESTS + first = report.planner.retune_events[0] + assert first["outcome"] == "applied", first + assert first["chosen"]["router"] == _router_point(0.0) + changes = first["planner_changes"] + assert set(changes) == {"load_adjustment_interval_seconds"} + new_interval = changes["load_adjustment_interval_seconds"] + assert new_interval in (10, 15) + # Easy mode scales on load only: no predictor to swap. + assert first["predictor_warmup"] is None + before = {(value, plugin) for now_s, value, plugin in seen if now_s <= _DECIDE_S} + after = {(value, plugin) for now_s, value, plugin in seen if now_s > _DECIDE_S} + assert before == {(5, 5.0)} + assert after == {(new_interval, float(new_interval))} + + @pytest.mark.integration @pytest.mark.timeout(120) def test_candidate_replays_share_common_random_numbers(tmp_path): diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md index f7871aae831d..4d4318bbfd27 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md @@ -370,17 +370,22 @@ was recorded or a role had no ready worker then, the decision is `failed`. ### Candidates and the Sweep `retune.router` and `retune.planner` use the AISimulate recommendation syntax of the Dynamo router -and Planner adapters. `planner/retune/domains.py` expands them into a Cartesian grid keyed by live -names: PATCH `/router_policy` parameters and `PlannerConfig` fields. The candidate set is the grid -plus the startup point plus the current point, at most `max_candidates` + 2 points in a canonical -order. +and Planner adapters. `planner/retune/domains.py` expands them into a grid keyed by live names: +PATCH `/router_policy` parameters and `PlannerConfig` fields. The router grid is a Cartesian product; +the Planner grid is a list of points, because a scaling policy is one correlated choice of its four +fields. Each Planner point is made canonical: fields that its own scaling modes make inactive, such +as `load_scaling_down_sensitivity` without load scaling or `max_throughput_scaling_replicas` without +throughput scaling, take the running value, and equal points are kept once. Points that turn both +modes off, order the intervals wrongly, or use an interval `scheduling.scale_interval_seconds` does +not divide are dropped. The candidate set is the router grid times the Planner points, plus the +startup point and the current point, at most `max_candidates` + 2 points in a canonical order. When the Planner starts, the controller validates each Planner grid point and each load-predictor candidate once against the startup configuration, not their product. A grid point that fails `PlannerConfig` validation or does not fit the deployment's worker capabilities is dropped, and one warning reports how many points were dropped, with an example and its error. Engine construction -fails only when some value of a searched Planner knob fits in no remaining point, or when a -load-predictor candidate is invalid. +fails only when some value of a searched Planner knob other than a scaling-policy field fits in no +remaining point, or when a load-predictor candidate is invalid. Before an `applied` decision changes Planner fields, the controller checks the combination the host will apply against the live configuration again; if it no longer validates or fits, the decision is `failed`. @@ -389,18 +394,20 @@ will apply against the live configuration again; if it no longer validates or fi up to `max_parallel` worker processes with the `spawn` method, at niceness 10, and feeds each one task at a time over its own pipe: -1. A read task writes the window to a temporary sweep directory. For an inner Planner that scales - on throughput, it also reads the history before the window once and buckets it twice at the - inner Planner's throughput interval: ending at the window end, with the window, for the - pre-sweep and a predictor swap, and ending at the window start as the inner Planner's predictor - warmup. +1. A read task writes the window to a temporary sweep directory. When the inner Planner or a + candidate scales on throughput, it also reads the history before the window once and buckets it + twice at each throughput interval in use: ending at the window end, with the window, for the + pre-sweep and a predictor swap, and ending at the window start as the predictor warmup of an + inner Planner at that interval. 2. With a searched load predictor, one pre-sweep task per predictor candidate runs, as described below. 3. One `dynamo.replay.run_trace_replay` per distinct candidate replays the window file with `trace_format="dynamo"`, the pinned engine, and the candidate's router parameters. Candidates that resolve to the same configuration share one replay. -4. When the pre-sweep winner changes the load predictor, one more task prepares that predictor on - the swap's warmup observations, as the host will (see [Predictor Swap](#predictor-swap)). It is +4. For each throughput interval at which a candidate would need a new load predictor (the + pre-sweep winner changes it, the interval differs from the current one, or the candidate turns + throughput scaling on), one more task prepares that predictor on the swap's warmup observations + at that interval, as the host will (see [Predictor Swap](#predictor-swap)). These tasks are queued ahead of the candidate replays, so a long ARIMA or Prophet fit overlaps them. Every candidate replay, the current point's included, runs with the canonical router seed and @@ -412,9 +419,12 @@ depend on which sweep worker process replays the candidate, including with an in uses `optimization_target: sla`. The outer replay keeps its default, non-deterministic routing. Without scaling, a candidate replay keeps the replica counts fixed. With scaling, it runs an inner -Planner: the current configuration with the candidate's Planner fields, retune disabled, no -diagnostics report, and, with throughput scaling, the history buckets that end at the window start -as its predictor warmup. A worker that dies fails only its task and is replaced. At +Planner: the current configuration with the candidate's Planner fields, scaling policy included, +retune disabled, no diagnostics report, and, with throughput scaling, the history buckets of its +own throughput interval that end at the window start as its predictor warmup. Every inner Planner, +the current point's included, starts at the window start with a freshly warmed predictor, so a +candidate is not credited for re-warming the predictor; the host re-warms the running one only when +the applied point needs a new predictor. A worker that dies fails only its task and is replaced. At `timeout_seconds` of wall-clock time every worker is killed and the decision is `timeout`; closing the controller kills them the same way. Workers keep their temporary files, such as Prophet's model files, in the sweep directory, which is removed when the sweep ends. @@ -422,15 +432,17 @@ files, in the sweep directory, which is removed when the sweep ends. ### Load-Predictor Pre-Sweep The load predictor is never a grid dimension. With `optimization_target: sla` and throughput -scaling, the sweep buckets the history and window traffic at the inner Planner's throughput -interval. Each predictor candidate observes the history buckets, then forecasts each bucket of the +scaling in some candidate, the sweep buckets the history and window traffic at each throughput +interval in use, and runs the pre-sweep once per interval. Each predictor candidate observes the history buckets, then forecasts each bucket of the window one step ahead and observes it. Its loss is the mean over window buckets of a weighted forecast error: input tokens and output tokens 40% each, average input and output length 10% each. A candidate therefore costs three forecasts per window bucket however long the history is. The lowest loss wins, the earliest candidate on ties, and a candidate whose forecast fails cannot win. -Every candidate except the current point then uses the winner's predictor fields; the current point -keeps the current predictor. The decision records the result in `predictor_presweep`. +Every candidate that scales on throughput, except the current point, then uses the predictor +fields of the winner at its own interval; the current point keeps the current predictor. The +decision records the result in `predictor_presweep`: one summary with `interval_seconds`, or +`{"by_interval": {...}}` with candidates at several intervals. ### Scoring and Outcomes @@ -504,10 +516,15 @@ after the tick, so Planner fields take effect from the next tick. ### Predictor Swap -A decision that changes the load predictor carries the history and window observations that warm -the new predictor (`predictor_warmup`). The sweep prepares the winning predictor on those -observations in a worker process, beside the candidate replays, the way the host will, so the -Planner tick never fits a predictor. If that traffic is too short for it, for example in the first +A decision needs a new load predictor when it changes a predictor field, changes +`throughput_adjustment_interval_seconds` (the predictor's step and the bucket length of its +history), or turns throughput scaling on. It then carries the history and window observations, +bucketed at the new interval, that warm the new predictor (`predictor_warmup`). The sweep prepares +that predictor on those observations in a worker process, beside the candidate replays, the way the +host will, so the Planner tick never fits a predictor. A scaling-policy change re-registers the +builtin plugins of the running engine: a mode turned on starts its plugins one interval later, as at +startup, and turning throughput scaling off drops the last throughput targets that floor load +scaling. If that traffic is too short for it, for example in the first window of a replay without earlier history, the decision is `failed` and nothing changes. The warmup ends at the window end, and the new predictor takes over after the decision tick, so the traffic of the grace period and the decision latency, `G + max(D, Δ)`, is not in its history. diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md index 58a34a71f76f..341aa7f03ee5 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md @@ -349,12 +349,14 @@ Retune searches parameters a running deployment can change without a restart: - **Router parameters** of the KV router, in `retune.router`: `overlap_score_credit`, `prefill_load_scale`, `temperature`, and four more `/router_policy` parameters. -- **Planner runtime fields**, in `retune.planner`: load-scaling sensitivity, the load predictor, - minimum worker counts, GPU budgets, `max_throughput_scaling_replicas`, and the thresholds of - `optimization_target: load`. - -The engine, the topology, the router mode, and Planner settings fixed at startup, such as the -scaling intervals and `ttft_ms` / `itl_ms`, stay as configured. +- **Planner runtime fields**, in `retune.planner`: the scaling policy (which scaling modes run, and + their intervals), load-scaling sensitivity, the load predictor, + `max_throughput_scaling_replicas`, and the thresholds of `optimization_target: load`. + +The engine, the topology, the router mode, Planner settings fixed at startup such as +`scheduling.scale_interval_seconds` and the FPM sampling, and `ttft_ms` / `itl_ms` stay as +configured. Minimum worker counts and GPU budgets are user constraints: retune never searches +them, and every candidate replays within them. Retune supports two scenarios: diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index 5f0c0f46b9cc..5b69042874c8 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -469,13 +469,32 @@ The supported configuration groups are: | --- | --- | | Endpoint floors and GPU budgets | Mode-specific minimum endpoint fields from the table above, `min_gpu_budget`, `max_gpu_budget`. | | SLA targets | `ttft_ms`, `itl_ms`. | +| Scaling modes and intervals | `enable_throughput_scaling`, `enable_load_scaling`, `throughput_adjustment_interval_seconds`, `load_adjustment_interval_seconds`. | | Scaling policy | `max_throughput_scaling_replicas`, `load_scaling_down_sensitivity`, `prefill_scale_up_queue_tokens`, `prefill_scale_down_queue_tokens`, `decode_scale_up_kv_rate`, `decode_scale_down_kv_rate`. | | Load prediction | `load_predictor`, `load_predictor_log1p`, `prophet_window_size`, `kalman_q_level`, `kalman_q_trend`, `kalman_r`, `kalman_min_points`. | Fields retain their normal mode-specific meaning. For example, the explicit queue/KV -thresholds are used by `optimization_target: load`. Predictor replacement requires -`optimization_target: sla` with throughput scaling enabled. Scheduling intervals, -AIS performance models, deployment topology, and other fields are rejected by this API. +thresholds are used by `optimization_target: load`, and under any other target than `sla` +the Planner keeps scaling on load only, whatever the flags say. An update is validated as a +whole configuration, so it cannot turn both scaling modes off, and with both on the load +interval must stay shorter than the throughput interval. Both intervals must remain +multiples of `scheduling.scale_interval_seconds`, which is fixed at startup. + +A new load predictor needs `optimization_target: sla` with throughput scaling enabled, +and `warmup_trace` or `cold_start: true`. An update needs one when it changes a predictor +field, changes `throughput_adjustment_interval_seconds` (the predictor's step and the +bucket length of its history), or turns throughput scaling on. Other scaling-mode changes +keep the running predictor and the live FPM samples: + +- Turning a mode on registers its builtin plugins as at startup, so they first run one + interval later; turning it off unregisters them. Turning throughput scaling off also + drops the last throughput targets, which would otherwise keep flooring load scaling. +- A new load interval also sets the FPM freshness window to three load intervals. +- A throughput-only SLA Planner keeps pulling FPM samples at the load interval to refit its + performance model, so turning load scaling on later does not start that model cold. + +`scheduling.scale_interval_seconds`, AIS performance models, deployment topology, and +other fields are rejected by this API. With [live retune](../../kubernetes/auto-deployment/planner-live-retune.md) (`retune.executor: dgdr`), an update of `ttft_ms` or `itl_ms` that changes the retune scoring SLA, because `retune.sla_ttft_ms` or `retune.sla_itl_ms` is unset, is rejected with HTTP `422`. To change the SLA, restart the @@ -790,20 +809,21 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p - **SLA.** Only `Goodput` and `GoodputPerGPU` count SLA-good tokens, against `sla_ttft_ms` and `sla_itl_ms`. The other objectives ignore both fields. - **Time base.** A candidate replay runs until every request of the window completes, so its duration is the window plus a drain that differs by candidate. `Goodput` and `GoodputPerGPU` are rescaled to the window length; the throughput objectives are not, so they rank the candidate that finishes the window's tokens sooner. [Scoring and Outcomes](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#scoring-and-outcomes) gives the formulas and the reasons. - **Missing inputs.** A replay that lacks an input of the objective fails that candidate instead of scoring it, for example `TTFT` with no first-token samples, or a per-GPU objective without GPU-hours. -- **Scaling.** With load or throughput scaling enabled, each candidate replay runs a Planner that sizes its own replicas, and `retune.planner` can search `min_workers` and the GPU budgets. A metric that is not divided by GPUs improves with more GPUs and would apply the candidate that provisions the most, so only `GoodputPerGPU` and `ThroughputPerGPU` are accepted. `ThroughputPerGPU` ignores the SLA, so with scaling it favors the candidate that serves the window on the fewest GPUs, whatever its latency. +- **Scaling.** With load or throughput scaling enabled, each candidate replay runs a Planner that sizes its own replicas, and `retune.planner` can search the scaling policy. A metric that is not divided by GPUs improves with more GPUs and would apply the candidate that provisions the most, so only `GoodputPerGPU` and `ThroughputPerGPU` are accepted. `ThroughputPerGPU` ignores the SLA, so with scaling it favors the candidate that serves the window on the fewest GPUs, whatever its latency. `GoodputPerGPU` counts only SLA-good tokens, but a candidate that provisions fewer GPUs can still score higher while it meets the SLA for fewer requests; retune has no SLA-attainment floor, so check `completed_requests` and `goodput_completed_requests` in the decision metrics when the scaling policy is searched. ### Search domains `retune.router` and `retune.planner` use the knob names, value syntax, and default domains of the Dynamo router and Planner adapters of `aisimulate recommend --stack dynamo`: -- **Values.** A knob takes the [domain syntax](dynosim-sweep-reference.mdx#domain-syntax) of `aisimulate recommend`: a scalar, `{choices: [...]}`, `{range: {min, max, step}}`, or, for the `load_sensitivity` and `load_predictor` groups, `{preset: ...}`. Retune adds these rules: +- **Values.** A knob takes the [domain syntax](dynosim-sweep-reference.mdx#domain-syntax) of `aisimulate recommend`: a scalar, `{choices: [...]}`, `{range: {min, max, step}}`, or, for the `scaling_policy`, `load_sensitivity` and `load_predictor` groups, `{preset: ...}`. Retune adds these rules: - A range must be enumerable: a range without `step` and a `scale: log` range are rejected. A range with `min` equal to `max` is one value. Stepped ranges are computed in decimal, so `{min: 0.0, max: 0.3, step: 0.1}` ends at 0.3. - Each knob may expand to at most `max_candidates` values. - A bare list (`[0.5, 1.0]`) is rejected; write `{choices: [0.5, 1.0]}`. - In YAML, write floats with a decimal point and a signed exponent (`1.0e-3`): YAML 1.1 reads `1e-3` as a string, which the schema rejects. -- **Infeasible points.** When the Planner starts, a Planner grid point that fails `PlannerConfig` validation or does not fit the deployment, such as a `min_workers` above what `max_num_gpus` allows, is dropped, and one warning reports how many points were dropped. Startup fails only when some value of a searched Planner knob fits in no grid point, or when a load-predictor candidate is invalid. A decision that would apply Planner fields that no longer validate or fit, for example after a budget change, is `failed`. -- **Omitted knobs.** A knob that has an AISimulate default domain searches it when omitted, as in `aisimulate recommend`. The retune-only knobs in the tables below keep their current value unless written. Knobs fixed at startup always keep their running value. -- **Names.** Write the public names below. A live name, such as `router_temperature`, `min_endpoint`, or `max_gpu_budget`, is rejected with the public name to use. Decisions and `report.planner.retune_events` record the live names: the PATCH `/router_policy` parameter names and the `PlannerConfig` field names. +- **Infeasible points.** When the Planner starts, a Planner grid point that fails `PlannerConfig` validation or does not fit the deployment, such as a `decode_scale_up_kv_rate` not above the `decode_scale_down_kv_rate` it is combined with, or a throughput-scaling policy the configuration cannot run, is dropped, and one warning reports how many points were dropped. Startup fails only when some value of a searched Planner knob other than a scaling-policy field fits in no grid point, or when a load-predictor candidate is invalid. A decision that would apply Planner fields that no longer validate or fit, for example after a runtime threshold change, is `failed`. +- **Omitted knobs.** A knob that has an AISimulate default domain searches it when omitted, as in `aisimulate recommend`. The retune-only knobs in the tables below and the `scaling_policy` group keep their current value unless written. Knobs fixed at startup always keep their running value. +- **User constraints.** Worker minimums and GPU budgets (`min_workers`, `prefill_min_workers`, `decode_min_workers`, `max_num_gpus`, `min_num_gpus`, and their live names) are never searched: they bound what the Planner may provision, and every candidate replays within them. Change them through the Planner config or the [runtime configuration API](#system-port-reconfiguration-api). +- **Names.** Write the public names below. A live name, such as `router_temperature`, is rejected with the public name to use. Decisions and `report.planner.retune_events` record the live names: the PATCH `/router_policy` parameter names and the `PlannerConfig` field names. Router knobs: @@ -827,24 +847,30 @@ Planner knobs: | Knob | Live name | Omitted | Written | |---|---|---|---| -| `load_sensitivity` group | `load_scaling_down_sensitivity` | With load scaling: `{70, 80, 90}`, from the `aggressive`, `default`, and `conservative` presets | `{preset: [...]}` with preset names or mappings, or `{preset: false}` with `load_scaling_down_sensitivity` as a domain | -| `load_predictor` group | Predictor fields | With `optimization_target: sla` and throughput scaling: the 11 simulation predictor presets | `{preset: [...]}` with preset names or complete mappings, or `{preset: false}` with `type` and the predictor knobs as domains | -| `min_workers` | `min_endpoint` | Current value | Domain; `mode: agg` only | -| `prefill_min_workers`, `decode_min_workers` | `prefill_min_endpoint`, `decode_min_endpoint` | Current value | Domain; `mode: disagg` only; `null` is rejected | -| `max_num_gpus`, `min_num_gpus` | `max_gpu_budget`, `min_gpu_budget` | Current value | Domain; values follow `max_gpu_budget` and `min_gpu_budget` (`min_num_gpus: -1` disables the floor) | -| `max_throughput_scaling_replicas` | Same | Current value | Domain; requires throughput scaling | +| `scaling_policy` group | `enable_throughput_scaling`, `enable_load_scaling`, `throughput_adjustment_interval_seconds`, `load_adjustment_interval_seconds` | Running value | `{preset: [...]}` with preset names or complete mappings, `{preset: default}` for the six presets that scale, or `{preset: false}` with the four knobs as domains. See [Scaling policy](#scaling-policy) | +| `load_sensitivity` group | `load_scaling_down_sensitivity` | With load scaling in some point: `{70, 80, 90}`, from the `aggressive`, `default`, and `conservative` presets | `{preset: [...]}` with preset names or mappings, or `{preset: false}` with `load_scaling_down_sensitivity` as a domain | +| `load_predictor` group | Predictor fields | With `optimization_target: sla` and throughput scaling in some point: the 11 simulation predictor presets | `{preset: [...]}` with preset names or complete mappings, or `{preset: false}` with `type` and the predictor knobs as domains | +| `max_throughput_scaling_replicas` | Same | Current value | Domain; requires throughput scaling in some point | | `prefill_scale_up_queue_tokens`, `prefill_scale_down_queue_tokens` (retune only) | Same | Current value | Domain; requires `optimization_target: load` and `mode: disagg` | | `decode_scale_up_kv_rate`, `decode_scale_down_kv_rate` (retune only) | Same | Current value | Domain; requires `optimization_target: load` | -| `policy`, `scaling_policy`, `fpm_sampling` and their knobs, `load_min_observations` | — | Running value | Rejected: fixed when the Planner starts | +| `min_workers`, `prefill_min_workers`, `decode_min_workers`, `max_num_gpus`, `min_num_gpus` | — | Running value | Rejected: user constraints, changed through the Planner config or the runtime configuration API | +| `policy`, `fpm_sampling` and its knobs, `load_min_observations` | — | Running value | Rejected: fixed when the Planner starts | | `ttft_ms`, `itl_ms` | — | Fixed | Rejected: set `sla_ttft_ms` and `sla_itl_ms` to change the scoring SLA | The group rules are those of `aisimulate recommend`: to write a knob of a group, set the group to `{preset: false}`; a group that is omitted or `{preset: default}` cannot be combined with its knobs. Within these groups: +- **Scaling policy.** Each policy is one correlated choice of its four fields, so the Planner grid is a list of points rather than a full product. Retune turns a mode on or off and moves its interval at runtime ([runtime configuration](#system-port-reconfiguration-api)), with these rules: + - Retune never turns scaling off: a point with both modes off is dropped, and a written `disabled` preset is rejected. A retune-only Planner (both flags `false`) rejects `retune.planner` altogether. + - With both modes on, the load interval must be shorter than the throughput interval, and both intervals must be multiples of `scheduling.scale_interval_seconds`, which is fixed at startup. A written preset entry that breaks either rule is rejected; a point of `{preset: default}` or `{preset: false}` that breaks one is dropped. A null interval in a preset is the running value, except for an enabled mode. + - Under any `optimization_target` other than `sla`, the Planner always scales on load only: `{preset: default}` keeps the two `load_*` presets, and a written entry or knob that enables throughput scaling is rejected. + - Fields that a point's own modes make inactive take the running value, so equivalent points are replayed once: `load_scaling_down_sensitivity` without load scaling, and the throughput interval, `max_throughput_scaling_replicas` and the predictor without throughput scaling. The load interval stays active either way: it is also the cadence at which live FPM samples refit the performance model. + - The predictor pre-sweep runs per throughput interval, and a point carries the winner of its own interval. `history_seconds` must cover the largest throughput interval of any point. + - Applying a point that changes the throughput interval or turns throughput scaling on swaps in a new predictor, warmed on the sweep's history bucketed at the new interval. Other scaling-policy changes keep the running predictor. - **Load sensitivity.** Retune searches only `load_scaling_down_sensitivity`; `load_min_observations` keeps its running value. A preset name contributes its sensitivity alone, so with a running `load_min_observations` other than 5 the `default` preset is applied only in part. A preset mapping must set `load_min_observations` to the running value. - **Load predictor.** The predictor is never a grid dimension: a [pre-sweep](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#load-predictor-pre-sweep) picks one candidate per sweep. Candidates that give the same predictor are evaluated once (`{preset: false}` with the default knob domains is 256 combinations and 40 predictors). Records label a candidate with its preset name, or `custom_` for the `i`-th entry. The number of candidates must not exceed `max_candidates`. -- **Planner defaults.** `planner: default` searches `load_scaling_down_sensitivity` ∈ `{70, 80, 90}` with load scaling and the 11 predictor presets with `optimization_target: sla` and throughput scaling. Without scaling it expands to nothing. +- **Planner defaults.** `planner: default` searches `load_scaling_down_sensitivity` ∈ `{70, 80, 90}` with load scaling and the 11 predictor presets with `optimization_target: sla` and throughput scaling. It never searches the scaling policy. Without scaling it expands to nothing. -An explicit domain for a disaggregated SLA Planner with load and throughput scaling. The router grid is 3 × 1 × 3 × 2 = 18 points and the Planner grid 3 × 2 = 6, so the sweep replays 108 grid points, after a pre-sweep over three predictors: +An explicit domain for a disaggregated SLA Planner with load and throughput scaling. The router grid is 3 × 1 × 3 × 2 = 18 points and the Planner grid 2 scaling policies × 3 sensitivities = 6 points, so the sweep replays 108 grid points, after a pre-sweep over three predictors at each of the two throughput intervals: ```yaml retune: @@ -855,9 +881,9 @@ retune: temperature: {range: {min: 0.0, max: 0.2, step: 0.1}} # 0, 0.1, 0.2 decode_active_request_weight: {choices: [0.5, 1.0]} # retune-only knob planner: + scaling_policy: {preset: [hybrid_180_5, hybrid_600_5]} load_sensitivity: {preset: false} load_scaling_down_sensitivity: {choices: [70, 80, 90]} - decode_min_workers: {choices: [1, 2]} load_predictor: {preset: [arima_log1p, kalman_default_raw, kalman_reactive_log1p]} ``` diff --git a/lib/bindings/python/rust/llm/fpm.rs b/lib/bindings/python/rust/llm/fpm.rs index 473b2a1f4fb8..748cc73be1f5 100644 --- a/lib/bindings/python/rust/llm/fpm.rs +++ b/lib/bindings/python/rust/llm/fpm.rs @@ -21,7 +21,7 @@ use pyo3::prelude::*; use std::collections::HashMap; use std::ops::Range; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; @@ -132,6 +132,36 @@ fn record_fpm( ); } +/// Freshness window shared by snapshot reads and the tracking task's cleanup, +/// so `set_max_age` reaches a task that already runs. +#[derive(Clone, Debug)] +struct SharedMaxAge(Arc); + +impl SharedMaxAge { + fn new(age: Duration) -> Self { + Self(Arc::new(AtomicU64::new(duration_nanos(age)))) + } + + fn get(&self) -> Duration { + Duration::from_nanos(self.0.load(Ordering::Relaxed)) + } + + fn set(&self, age: Duration) { + self.0.store(duration_nanos(age), Ordering::Relaxed); + } +} + +fn duration_nanos(age: Duration) -> u64 { + u64::try_from(age.as_nanos()).unwrap_or(u64::MAX) +} + +fn parse_max_age(max_age_seconds: f64) -> PyResult { + Duration::try_from_secs_f64(max_age_seconds) + .ok() + .filter(|age| !age.is_zero()) + .ok_or_else(|| PyValueError::new_err("max_age_seconds must be finite and positive")) +} + fn prune_inactive_workers(workers: &DashMap, now: Instant, max_age: Duration) { workers.retain(|_, worker| match worker.lifecycle { FpmWorkerLifecycle::Active { .. } => true, @@ -568,7 +598,7 @@ pub(crate) struct FpmEventSubscriber { rx: Arc>>>>, tracking_started: Arc, workers: Arc>, - max_age: Duration, + max_age: SharedMaxAge, } #[derive(Default)] @@ -748,7 +778,7 @@ impl FpmEventSubscriber { rx: Arc::new(std::sync::Mutex::new(None)), tracking_started: Arc::new(AtomicBool::new(false)), workers: Arc::new(DashMap::new()), - max_age: DEFAULT_FPM_MAX_AGE, + max_age: SharedMaxAge::new(DEFAULT_FPM_MAX_AGE), }) } @@ -832,14 +862,12 @@ impl FpmEventSubscriber { }) } - /// Track samples and discovery state with a fixed freshness and cleanup window. + /// Track samples and discovery state with a freshness and cleanup window. /// Legacy snapshots retain samples until worker removal; health snapshots filter by age. + /// `set_max_age` changes the window later. #[pyo3(signature = (max_age_seconds = DEFAULT_FPM_MAX_AGE.as_secs_f64()))] fn start_tracking(&mut self, max_age_seconds: f64) -> PyResult<()> { - let max_age = Duration::try_from_secs_f64(max_age_seconds) - .ok() - .filter(|age| !age.is_zero()) - .ok_or_else(|| PyValueError::new_err("max_age_seconds must be finite and positive"))?; + let max_age = parse_max_age(max_age_seconds)?; if self.recv_started.load(Ordering::SeqCst) { return Err(PyRuntimeError::new_err( "Cannot call start_tracking() after recv()", @@ -852,7 +880,8 @@ impl FpmEventSubscriber { let endpoint = self.endpoint.clone(); let rt = endpoint.component().drt().runtime().secondary(); let cancel = self.cancel.clone(); - self.max_age = max_age; + self.max_age.set(max_age); + let max_age = self.max_age.clone(); let workers = self.workers.clone(); rt.spawn({ @@ -881,7 +910,7 @@ impl FpmEventSubscriber { break; } _ = cleanup_tick.tick() => { - prune_inactive_workers(&workers, Instant::now(), max_age); + prune_inactive_workers(&workers, Instant::now(), max_age.get()); } event = subscriber.next() => { match event { @@ -1055,10 +1084,24 @@ impl FpmEventSubscriber { Ok(fresh_fpm_snapshot( &self.workers, Instant::now(), - self.max_age, + self.max_age.get(), )) } + /// Change the freshness and cleanup window, before or after `start_tracking`. + /// Tracked samples are kept; reads and the next cleanup use the new age. + #[pyo3(signature = (max_age_seconds,))] + fn set_max_age(&self, max_age_seconds: f64) -> PyResult<()> { + self.max_age.set(parse_max_age(max_age_seconds)?); + Ok(()) + } + + /// The current freshness and cleanup window, in seconds. + #[getter] + fn max_age_seconds(&self) -> f64 { + self.max_age.get().as_secs_f64() + } + /// Selected model deployment cards for active workers, as serialized JSON. fn get_model_cards(&self) -> PyResult> { if !self.tracking_started.load(Ordering::SeqCst) { @@ -1095,6 +1138,42 @@ impl Drop for FpmEventSubscriber { mod tests { use super::*; + #[test] + fn shared_max_age_reaches_a_captured_clone() { + let max_age = SharedMaxAge::new(Duration::from_secs(10)); + // What the tracking task captures when it starts. + let captured = max_age.clone(); + let workers = DashMap::new(); + let now = Instant::now(); + record_fpm(&workers, ("worker".to_string(), 0), vec![1], now); + let later = now + Duration::from_secs(5); + prune_inactive_workers(&workers, later, captured.get()); + assert!(workers.contains_key("worker")); + assert_eq!( + fresh_fpm_snapshot(&workers, later, max_age.get()).unjoined_worker_ids, + vec!["worker".to_string()] + ); + + max_age.set(Duration::from_secs(2)); + assert_eq!(captured.get(), Duration::from_secs(2)); + assert!( + fresh_fpm_snapshot(&workers, later, max_age.get()) + .unjoined_worker_ids + .is_empty() + ); + prune_inactive_workers(&workers, later, captured.get()); + assert!(!workers.contains_key("worker")); + } + + #[test] + fn parse_max_age_rejects_non_positive_and_non_finite() { + assert_eq!(parse_max_age(1.5).unwrap(), Duration::from_millis(1500)); + assert!(parse_max_age(0.0).is_err()); + assert!(parse_max_age(-1.0).is_err()); + assert!(parse_max_age(f64::NAN).is_err()); + assert!(parse_max_age(f64::INFINITY).is_err()); + } + fn active_worker(ranks: Range, now: Instant) -> WorkerFpm { let mut worker = WorkerFpm::default(); worker.activate(now); diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi index f4dcc23d3d59..c0933852e010 100644 --- a/lib/bindings/python/src/dynamo/_core.pyi +++ b/lib/bindings/python/src/dynamo/_core.pyi @@ -1481,8 +1481,9 @@ class FpmEventSubscriber: """ Start background tracking of the latest FPM per (worker_id, dp_rank). - ``max_age_seconds`` fixes the freshness and cleanup window for this - subscription. It must be finite and positive. + ``max_age_seconds`` sets the freshness and cleanup window for this + subscription; ``set_max_age`` changes it later. It must be finite and + positive. Spawns two background tasks: @@ -1497,6 +1498,19 @@ class FpmEventSubscriber: """ ... + def set_max_age(self, max_age_seconds: float) -> None: + """ + Change the freshness and cleanup window, before or after + ``start_tracking``. Tracked samples are kept; snapshot reads and the + next cleanup use the new age. It must be finite and positive. + """ + ... + + @property + def max_age_seconds(self) -> float: + """The current freshness and cleanup window, in seconds.""" + ... + def get_recent_stats(self) -> dict[tuple[str, int], bytes]: """ Return the latest FPM bytes for every tracked (worker_id, dp_rank). From ddd7cc021cd802975a816241a8ffe183916ba2ee Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Wed, 4 Nov 2026 12:31:23 -0800 Subject: [PATCH 2/8] feat(planner): live retune of planner parameters with scaling (Scenario 2) retune.executor 'dgdr' no longer requires retune-only mode: a scaling Planner retunes the router and its retune.planner fields through DGDR v1beta2 Requests, and each candidate replays the Planner as replay retune does. - Before planning its Requests, the executor reads the history before the window from the trace segments (window_cut.bucket_history), runs the load-predictor pre-sweep per throughput interval, maps each candidate to the point it replays as, and prepares every predictor a decision could swap in. - Requests carry inputOverrides.planner.retune_sweep: the live config's scaling fields, the shard's planner points (one more shard dimension) and the warmup rows by interval. Engine replicas are the ready counts at the window start, so a replica change in the window is no longer replicas_changed. - The Dynamo planner adapter replays each point on the base with its interval's warmup (DynamoReplayRunner passes it on); a candidate's concrete config is the section narrowed to its point, which aisimulate predict replays again. Candidates map back through it. - An applied decision's planner fields activate locally through the runtime reconfiguration path, a new predictor warmed on the decision's observations; a failure keeps the config and drops the controller's pending change. History entries record the fields; a restart runs the configured values and logs the lost ones. - P2 now requires a router domain; P13 counts the planner points, a second cut_budget_seconds for the history, and at most 1024 warmup buckets per interval. Segment GC keeps the history window. Refs DYN-5400 Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/config/planner_config.py | 87 +++- components/src/dynamo/planner/core/base.py | 80 ++- .../src/dynamo/planner/retune/controller.py | 47 +- .../src/dynamo/planner/retune/domains.py | 13 + .../dynamo/planner/retune/live/bootstrap.py | 18 +- .../src/dynamo/planner/retune/live/dgdr.py | 454 +++++++++++++++--- .../dynamo/planner/retune/live/reconciler.py | 6 +- .../dynamo/planner/retune/live/supervisor.py | 39 ++ .../dynamo/planner/retune/live/window_cut.py | 158 ++++++ components/src/dynamo/planner/retune/sweep.py | 121 +++-- .../src/dynamo/planner/simulation/config.py | 65 +++ .../src/dynamo/planner/simulation/provider.py | 255 ++++++++++ .../dynamo/planner/tests/live_retune_fakes.py | 81 +++- .../tests/unit/test_live_retune_config.py | 116 ++++- .../unit/test_live_retune_controller_hooks.py | 58 +++ .../tests/unit/test_live_retune_dgdr.py | 284 +++++++++++ .../tests/unit/test_live_retune_reconciler.py | 26 +- .../tests/unit/test_live_retune_supervisor.py | 30 ++ .../tests/unit/test_live_retune_window_cut.py | 54 +++ .../unit/test_planner_provider_retune.py | 128 +++++ .../tests/unit/test_reconfiguration.py | 77 +++ .../planner/tests/unit/test_retune_sweep.py | 6 +- components/src/dynamo/replay/simulation.py | 29 ++ .../traces/dynamo-retune-window.jsonl | 60 +++ .../dynamo/07-live-retune-sweep-goodput.yaml | 64 +++ .../dynamo/replay/tests/test_simulation.py | 41 ++ .../planner/planner-design.md | 11 +- .../planner/planner-guide.md | 4 +- .../auto-deployment/planner-live-retune.md | 102 +++- .../components/planner-configuration.mdx | 4 +- 30 files changed, 2313 insertions(+), 205 deletions(-) create mode 100644 components/src/dynamo/replay/tests/e2e/configs/unified_cli/fixtures/traces/dynamo-retune-window.jsonl create mode 100644 components/src/dynamo/replay/tests/e2e/configs/unified_cli/recommend/dynamo/07-live-retune-sweep-goodput.yaml diff --git a/components/src/dynamo/planner/config/planner_config.py b/components/src/dynamo/planner/config/planner_config.py index 5d296d1d53ef..62513df3e310 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/components/src/dynamo/planner/config/planner_config.py @@ -1043,6 +1043,10 @@ def _lease_fits_two_renewals(cls, value: float) -> float: # DGDR v1beta2 ``spec.recommendation.maxCandidates`` maximum: one Request (a # grid shard) carries at most this many router points. DGDR_MAX_SHARD_POINTS = 100 +# History buckets per throughput interval one DGDR Request carries as its +# inner planners' load-predictor warmup (Scenario 2), and each Candidate +# again as its replayable config: it bounds the objects' size. +DGDR_MAX_WARMUP_BUCKETS = 1024 # Router knobs a DGDR v1beta2 Request can search, by public name # (``contracts.DGDR_ROUTER_KNOBS``); the Request's router section has no # place for the retune extension knobs. @@ -1086,19 +1090,31 @@ def dgdr_rounds(points: int, parallelism: int) -> int: def dgdr_need_seconds( - points: int, dgdr: DgdrExecutorConfig, *, needs_cut: bool + points: int, + dgdr: DgdrExecutorConfig, + *, + needs_cut: bool, + needs_history: bool = False, ) -> float: - """Wall-clock seconds one DGDR shard of ``points`` router points needs. + """Wall-clock seconds one DGDR shard of ``points`` candidate points needs. - ``cut_budget`` (when the window file must still be cut) + Run Job - start-up + ``dgdr_rounds(points, run.parallelism)`` rounds of + ``cut_budget`` (when the window file must still be cut) + another + ``cut_budget`` (when the history must still be read and the predictor + pre-sweep run, Scenario 2) + Run Job start-up + + ``dgdr_rounds(points, run.parallelism)`` rounds of ``candidate_timeout``. ``parallelism`` is the configured value; Job resources may lower it, so this is optimistic. """ rounds = dgdr_rounds(points, dgdr.run.parallelism) cut = dgdr.cut_budget_seconds if needs_cut else 0.0 - return cut + dgdr.job_startup_seconds + rounds * dgdr.run.candidate_timeout_seconds + history = dgdr.cut_budget_seconds if needs_history else 0.0 + return ( + cut + + history + + dgdr.job_startup_seconds + + rounds * dgdr.run.candidate_timeout_seconds + ) def balanced_shard_parts( @@ -2478,7 +2494,7 @@ def _validate_retune(self, retune: RetuneConfig) -> None: "default domain, so pin knobs with a scalar to shrink the grid" ) if retune.executor == "dgdr": - self._validate_live_retune_budget(retune, router_domain) + self._validate_live_retune_budget(retune, router_domain, planner_domain) def _validate_live_retune(self, retune: RetuneConfig) -> None: """Configuration premises P1, P2, P3 and P5 of live retune. @@ -2525,15 +2541,12 @@ def _validate_live_retune(self, retune: RetuneConfig) -> None: f"{DGDR_TIMEOUT_MARGIN_SECONDS:g} = {limit:g} with executor " f"'dgdr' (P1), got {retune.timeout_seconds:g}" ) - if self.scaling_enabled(): - raise ValueError( - "retune.executor 'dgdr' supports retune-only mode (P2): set " - "enable_load_scaling and enable_throughput_scaling both to false" - ) - if retune.planner is not None: + if retune.router is None: raise ValueError( - "retune.executor 'dgdr' retunes the router only (P2); remove " - "retune.planner" + "retune.executor 'dgdr' requires retune.router (P2): every DGDR " + "Request sets the three router knobs; to retune the Planner " + "alone, pin overlap_score_credit, prefill_load_scale and " + "temperature to the frontend's startup values with scalars" ) self._validate_live_retune_router(retune) _validate_dgdr_engine(dgdr.engine, self.mode, self.backend) @@ -2603,41 +2616,75 @@ def _validate_live_retune_router(retune: RetuneConfig) -> None: f"executor 'dgdr' (P3), got {written}" ) - @staticmethod def _validate_live_retune_budget( - retune: RetuneConfig, router_domain: Mapping[str, Sequence[float]] + self, + retune: RetuneConfig, + router_domain: Mapping[str, Sequence[float]], + planner_domain: Any, ) -> None: """P13: the largest grid shard can finish within the sweep deadline. The baseline can fall in any grid shard, and a window whose baseline shard cannot finish fails with ``launch_too_late``, so the largest shard must fit. The domain is the widest one: a zero startup credit - only narrows it at runtime. + only narrows it at runtime, and folding inactive fields to the + baseline's values only merges planner points. With scaling, the + planner points are one more shard dimension, the history read and + the predictor pre-sweep take another ``cut_budget_seconds``, and the + warmup a Request carries is bounded by ``DGDR_MAX_WARMUP_BUCKETS`` + buckets per throughput interval. """ dgdr = retune.dgdr if dgdr is None or retune.timeout_seconds is None: # _validate_live_retune rejects both first. raise ValueError("retune.executor 'dgdr' requires dgdr and timeout_seconds") + scaling = self.scaling_enabled() + if scaling: + # Imported on use: the module imports this one at module scope. + from dynamo.planner.retune.domains import ( + required_history_seconds, + throughput_intervals, + ) + + intervals = throughput_intervals(self, planner_domain.points) + history_s = retune.history_seconds + if history_s is None: + history_s = required_history_seconds(self, planner_domain) + if intervals and history_s / min(intervals) > DGDR_MAX_WARMUP_BUCKETS: + raise ValueError( + f"retune history of {history_s:g}s at the shortest throughput " + f"interval {min(intervals)}s is " + f"{math.ceil(history_s / min(intervals))} warmup buckets, more " + f"than the {DGDR_MAX_WARMUP_BUCKETS} a DGDR Request carries " + "(P13); lower retune.history_seconds or the predictor window" + ) sizes = [len(values) for values in router_domain.values()] + if scaling: + sizes.append(len(planner_domain.points)) parts = balanced_shard_parts( sizes, dgdr.run.max_shards, parallelism=dgdr.run.parallelism ) points = largest_shard_points(sizes, parts) if points > DGDR_MAX_SHARD_POINTS: raise ValueError( - f"the retune router grid of {math.prod(sizes)} points cannot be " + f"the retune grid of {math.prod(sizes)} points cannot be " f"split into at most retune.dgdr.run.max_shards=" f"{dgdr.run.max_shards} Cartesian shards of at most " f"{DGDR_MAX_SHARD_POINTS} points (P13); the largest would have " f"{points}. Raise max_shards or pin router knobs with a scalar" ) - need = dgdr_need_seconds(points, dgdr, needs_cut=True) + need = dgdr_need_seconds(points, dgdr, needs_cut=True, needs_history=scaling) if need > retune.timeout_seconds: rounds = dgdr_rounds(points, dgdr.run.parallelism) + history = ( + f"cut_budget_seconds {dgdr.cut_budget_seconds:g} for the history + " + if scaling + else "" + ) raise ValueError( f"the largest retune grid shard ({points} points) needs " - f"{need:g}s: cut_budget_seconds {dgdr.cut_budget_seconds:g} + " + f"{need:g}s: {history}cut_budget_seconds {dgdr.cut_budget_seconds:g} + " f"job_startup_seconds {dgdr.job_startup_seconds:g} + {rounds} " f"rounds x candidate_timeout_seconds " f"{dgdr.run.candidate_timeout_seconds:g}, more than " diff --git a/components/src/dynamo/planner/core/base.py b/components/src/dynamo/planner/core/base.py index ce185810ae0e..fbb514f720ff 100644 --- a/components/src/dynamo/planner/core/base.py +++ b/components/src/dynamo/planner/core/base.py @@ -15,7 +15,7 @@ import importlib import logging import time -from collections.abc import AsyncIterator, Iterable +from collections.abc import AsyncIterator, Iterable, Sequence from contextlib import asynccontextmanager from typing import TYPE_CHECKING, Any, Optional, Protocol @@ -39,6 +39,7 @@ EngineCapabilities, FpmObservations, PlannerEffects, + RetuneDecision, ScheduledTick, TickDiagnostics, TickInput, @@ -54,8 +55,12 @@ from dynamo.planner.monitoring.planner_metrics import PlannerPrometheusMetrics from dynamo.planner.offline.trace_data import extract_traffic_observations_from_trace from dynamo.planner.reconfiguration import ( + PREPARATION_TIMEOUT_SECONDS, PlannerReconfiguration, + candidate_config, live_retune_sla_errors, + predictor_rebuild_required, + prepare_predictor, runtime_candidate_errors, runtime_configuration_errors, ) @@ -263,6 +268,8 @@ def __init__( self._legacy_retune_task: Optional[asyncio.Task[None]] = None self._legacy_retune_cleanup: Optional[RetuneCleanupHandle] = None self._retune_lease_found = asyncio.Event() + # Background applications of live retune Planner changes. + self._retune_apply_tasks: set[asyncio.Task[None]] = set() async def _async_init(self) -> None: # Shutdown is safe for a partially initialized environment and is @@ -442,6 +449,11 @@ async def _shutdown_runtime(self) -> None: """Release initialized planner resources after normal or failed startup.""" await self.reconfiguration.close() + applying = list(self._retune_apply_tasks) + for task in applying: + task.cancel() + if applying: + await asyncio.gather(*applying, return_exceptions=True) if not self._diagnostics_finalized: self._diagnostics_finalized = True @@ -625,6 +637,71 @@ def _sync_fpm_max_age(self) -> None: setter(age) logger.info("FPM freshness window is now %.1fs", age) + def _schedule_retune_planner_changes( + self, decisions: Sequence[RetuneDecision] + ) -> None: + """Apply each ``applied`` live retune decision's Planner fields. + + In the background, through the runtime reconfiguration path: a new + load predictor is prepared off the event loop on the decision's + warmup, then the candidate activates under the effect admission and + configuration locks, as a control API update does. + """ + + for decision in decisions: + if decision.outcome != "applied" or not decision.planner_changes: + continue + task = asyncio.create_task( + self._apply_retune_planner_changes(decision), + name=f"planner-retune-apply-{decision.decision_id}", + ) + self._retune_apply_tasks.add(task) + task.add_done_callback(self._retune_apply_tasks.discard) + + async def _apply_retune_planner_changes(self, decision: RetuneDecision) -> None: + changes = dict(decision.planner_changes) + try: + async with self._config_lock: + candidate = candidate_config(self.config, changes) + self.validate_runtime_candidate(candidate) + rebuild = predictor_rebuild_required(self.config, candidate) + predictor = None + if rebuild: + predictor = await asyncio.wait_for( + asyncio.to_thread( + prepare_predictor, + candidate, + self._build_worker_capabilities(), + decision.predictor_warmup, + ), + PREPARATION_TIMEOUT_SECONDS, + ) + async with self._effect_admission_lock: + async with self._config_lock: + candidate = candidate_config(self.config, changes) + self.validate_runtime_candidate(candidate) + if predictor_rebuild_required(self.config, candidate) != rebuild: + raise ValueError( + "the configuration changed while the predictor was " + "prepared" + ) + self.apply_runtime_candidate(candidate, changes.keys(), predictor) + logger.info( + "Retune %s applied planner changes %s", + decision.decision_id, + changes, + ) + except Exception: + logger.exception( + "Retune %s could not apply planner changes %s; the Planner keeps " + "its configuration", + decision.decision_id, + changes, + ) + controller = getattr(self._engine, "retune_controller", None) + if controller is not None: + controller.discard_planner_changes(changes) + @asynccontextmanager async def _bounded_config_lock(self) -> AsyncIterator[None]: """Acquire the tick decision lock or ask the caller to retry.""" @@ -1303,6 +1380,7 @@ async def _run_one_tick( self._live_retune.on_effects(effects) except Exception: logger.exception("Live retune failed to take the tick's effects") + self._schedule_retune_planner_changes(effects.retune_decisions) await self._submit_effects(effects, config_generation) emit_diagnostics = self._should_emit_tick_diagnostics(tick, effects) if emit_diagnostics: diff --git a/components/src/dynamo/planner/retune/controller.py b/components/src/dynamo/planner/retune/controller.py index 9b85e52f3d68..34d0fa737d78 100644 --- a/components/src/dynamo/planner/retune/controller.py +++ b/components/src/dynamo/planner/retune/controller.py @@ -414,6 +414,17 @@ def rewind(self, last_decided_k: int) -> None: if self._pending is None: self._last_k = min(self._last_k, int(last_decided_k)) + def discard_planner_changes(self, changes: Mapping[str, Any]) -> None: + """Forget ``changes`` of an ``applied`` decision the host could not + apply (a live Planner's predictor warmup failed, say): the current + point is the running configuration again.""" + + for key, value in changes.items(): + if key in self._pending_planner and _same_value( + self._pending_planner[key], value + ): + del self._pending_planner[key] + def restore_desired(self, flat: Mapping[str, float]) -> None: """Re-install persisted desired router values (live re-election). @@ -602,6 +613,16 @@ def _replicas_at(self, time_s: float) -> dict[str, int] | None: return None return dict(chosen) + def _covering_counts(self, time_s: float) -> dict[str, int] | None: + """The ready counts recorded last at or before ``time_s``.""" + + covering: dict[str, int] | None = None + for at_s, counts in self._history: + if at_s > time_s + _EPSILON: + break + covering = counts + return covering + def _due_window(self, now_s: float) -> int: """Latest window ``k`` whose launch time ``L_k`` is at or before now.""" @@ -692,16 +713,25 @@ def _replicas_for_launch( """``(replicas, verified, mismatch)`` for a window launch. Without ``spec_replicas`` (replay), the ready counts at window start, - verified. With them (live), every ready count recorded over the - window, including the one in effect at its start, must equal the - spec; a mismatch is reported. The spec replicas are used either way, - unverified when no recorded count covers the window start (the - window began before this controller started). + verified. With them (live), a planner that scales starts each + candidate's inner planner from the ready counts recorded at the + window start, as replay does, and the spec replicas stand in, + unverified, when no recorded count covers it. With fixed replicas, + every ready count recorded over the window, including the one in + effect at its start, must equal the spec; a mismatch is reported. + The spec replicas are used either way, unverified when no recorded + count covers the window start (the window began before this + controller started). """ if spec_replicas is None: return self._replicas_at(start_s), True, None spec = {key: int(value) for key, value in spec_replicas.items()} + if self._config.scaling_enabled(): + at_start = self._covering_counts(start_s) + if at_start is None or any(value < 1 for value in at_start.values()): + return spec, False, None + return dict(at_start), True, None covering: dict[str, int] | None = None observed: list[dict[str, int]] = [] for at_s, counts in self._history: @@ -831,7 +861,12 @@ def _live_request_fields( for key, values in self._router_domain_values.items() }, "deadline_s": as_of_s + self._retune.effective_timeout_seconds, - "spec_replicas": None if spec_replicas is None else dict(spec_replicas), + # A scaling planner's sweep starts from SweepRequest.replicas. + "spec_replicas": ( + None + if spec_replicas is None or self._config.scaling_enabled() + else dict(spec_replicas) + ), "replicas_verified": verified, } diff --git a/components/src/dynamo/planner/retune/domains.py b/components/src/dynamo/planner/retune/domains.py index 2e20b2d42f76..65a400852600 100644 --- a/components/src/dynamo/planner/retune/domains.py +++ b/components/src/dynamo/planner/retune/domains.py @@ -868,3 +868,16 @@ def required_history_seconds(config: PlannerConfig, domain: PlannerDomain) -> fl ) min_points.append(fields.get("kalman_min_points", config.kalman_min_points)) return float(max(intervals) * max(*window_sizes, *min_points)) + + +def retune_history_seconds(config: PlannerConfig) -> float: + """History each sweep reads before its window: ``retune.history_seconds``, + else ``required_history_seconds`` (0 without throughput scaling, or with + fixed replicas, whose sweeps run no inner planner).""" + + retune = config.retune + if retune is None or not config.scaling_enabled(): + return 0.0 + if retune.history_seconds is not None: + return float(retune.history_seconds) + return required_history_seconds(config, expand_planner_domain(config)) diff --git a/components/src/dynamo/planner/retune/live/bootstrap.py b/components/src/dynamo/planner/retune/live/bootstrap.py index 274a73123f18..9d6305529885 100644 --- a/components/src/dynamo/planner/retune/live/bootstrap.py +++ b/components/src/dynamo/planner/retune/live/bootstrap.py @@ -47,7 +47,7 @@ from dynamo.planner.core.types import RetuneInputs from dynamo.planner.errors import ComponentError from dynamo.planner.monitoring.dgd_services import get_component_from_type_or_name -from dynamo.planner.retune.domains import expand_router_domain +from dynamo.planner.retune.domains import expand_router_domain, retune_history_seconds from dynamo.planner.retune.live import frontend_config, window_cut from dynamo.planner.retune.live.actuator import ( AiohttpRouterPolicyClient, @@ -75,6 +75,7 @@ SupervisorSettings, worker_types_for, ) +from dynamo.planner.simulation.config import retune_sweep_base logger = logging.getLogger(__name__) @@ -294,11 +295,14 @@ def canary(supervisor: LiveRetuneSupervisor) -> bool: if not replicas or not point: return False sla_ttft_ms, sla_itl_ms = retune.scoring_sla(config.ttft_ms, config.itl_ms) + # A scaling Planner's Requests carry the planner section too. + scaling = config.scaling_enabled() shard = Shard( id=CANARY_SHARD_ID, kind="point", choices={key: (value,) for key, value in point.items()}, points=(dict(point),), + planners=({},) if scaling else (), ) body = build_request( dgd=supervisor.dgd, @@ -313,6 +317,10 @@ def canary(supervisor: LiveRetuneSupervisor) -> bool: spec_replicas=replicas, backend=str(config.backend), attempt=0, + planner_base=( + retune_sweep_base(config.model_dump(mode="json")) if scaling else None + ), + planner_warmup={}, ) try: supervisor.kube.create_request(body, dry_run=True, timeout_s=timeout_s) @@ -426,7 +434,13 @@ def evaluate(obj: dict) -> SpecEvaluation: evaluation = evaluate(dgd_obj) if reconciler is None: reconciler = Reconciler( - kube, evaluation.dgd, dgdr, retune, dgdr.mount_path, metrics + kube, + evaluation.dgd, + dgdr, + retune, + dgdr.mount_path, + metrics, + history_s=retune_history_seconds(config), ) worker_types = worker_types_for(config.mode) settings = SupervisorSettings.from_config( diff --git a/components/src/dynamo/planner/retune/live/dgdr.py b/components/src/dynamo/planner/retune/live/dgdr.py index ef3e9c123064..56a99cf7e9d6 100644 --- a/components/src/dynamo/planner/retune/live/dgdr.py +++ b/components/src/dynamo/planner/retune/live/dgdr.py @@ -42,6 +42,7 @@ from __future__ import annotations import copy +import dataclasses import functools import hashlib import itertools @@ -50,14 +51,17 @@ import math import os import re +import tempfile import threading import time from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass, field +from pathlib import Path from typing import Any, Literal, NoReturn, Optional from dynamo.planner.config.planner_config import ( DGDR_MAX_SHARD_POINTS, + DGDR_MAX_WARMUP_BUCKETS, DgdrExecutorConfig, RetuneConfig, balanced_shard_parts, @@ -88,12 +92,23 @@ WindowCutter, ) from dynamo.planner.retune.live.reconciler import trace_pvc_source +from dynamo.planner.retune.live.window_cut import bucket_history from dynamo.planner.retune.sweep import ( + SweepInterrupted, SweepRequest, SweepResult, SweepStatus, + WorkerPool, + evaluated_params, + predictor_swap_errors, + predictor_swap_tasks, + run_presweeps, + scales_on_throughput, score_summary, + sweep_throughput_intervals, + write_observations, ) +from dynamo.planner.simulation.config import retune_sweep_base logger = logging.getLogger(__name__) @@ -111,6 +126,13 @@ REQUEST_GENERATION_ANNOTATION = "nvidia.com/dgdr-request-generation" TRACE_MOUNT_PATH = "/var/run/aisimulate/trace" ROUTER_ADAPTER = "dynamo.router" +PLANNER_ADAPTER = "dynamo.planner" +# The planner adapter's live retune sweep section +# (``simulation.provider.RETUNE_SWEEP``). A Candidate's planner parameters +# are the section narrowed to its one point. +RETUNE_SWEEP = "retune_sweep" +# The shard axis of the planner points (sorts after every router knob). +_PLANNER_AXIS = "~planner" _ROLES = ("aggregated", "prefill", "decode") _TEMPERATURE = DGDR_ROUTER_KNOBS["temperature"] # spec.objective.metric -> AISimulate optimization.target (operator lowering). @@ -142,20 +164,43 @@ class Shard: ``choices`` maps each live router knob to the values the shard spans; ``points`` is their Cartesian product (live names). A ``point`` or - ``probe`` shard has one value per knob. + ``probe`` shard has one value per knob. ``planners`` are the planner + points (``PlannerConfig`` fields) the shard crosses with them when its + candidates replay an inner planner (Scenario 2; ``({},)`` when no + planner field is searched), and ``()`` with fixed replicas. """ id: str kind: ShardKind choices: dict[str, tuple[float, ...]] points: tuple[dict[str, float], ...] + planners: tuple[dict[str, Any], ...] = () @property def size(self) -> int: - return len(self.points) + return len(self.points) * max(1, len(self.planners)) + + def contains( + self, point: Mapping[str, float], planner: Optional[Mapping[str, Any]] = None + ) -> bool: + if not all(point.get(key) in values for key, values in self.choices.items()): + return False + return ( + planner is None + or not self.planners + or any(_same_planner(planner, option) for option in self.planners) + ) + + @property + def searches_planner(self) -> bool: + """Whether the shard's candidates replay an inner planner.""" + return bool(self.planners) - def contains(self, point: Mapping[str, float]) -> bool: - return all(point.get(key) in values for key, values in self.choices.items()) + +def _same_planner(left: Mapping[str, Any], right: Mapping[str, Any]) -> bool: + return set(left) == set(right) and all( + _same_json(left[key], right[key]) for key in left + ) def plan_shards( @@ -166,6 +211,9 @@ def plan_shards( max_shards: int, parallelism: int, probe: bool = True, + planner_points: Sequence[Mapping[str, Any]] = (), + baseline_planner: Optional[Mapping[str, Any]] = None, + startup_planner: Optional[Mapping[str, Any]] = None, ) -> list[Shard]: """Split ``router_domain`` (live knob -> values) into Request shards. @@ -182,6 +230,13 @@ def plan_shards( middle value of each knob, a positive temperature when the grid has one) to check common random numbers across Runs. Raises ``ValueError`` when no split fits or a point does not cover the domain's knobs. + + ``planner_points`` (distinct planner points, in order) add one more + dimension, cut into contiguous chunks like a router knob, so a shard + crosses its router sub-grid with a chunk of planner points. The + baseline and startup points then also carry their planner parts + (``baseline_planner``, ``startup_planner``), and the probe takes the + middle planner point. """ if not router_domain: @@ -204,7 +259,10 @@ def plan_shards( f"router domain {list(keys)}" ) + planners = tuple(dict(point) for point in planner_points) sizes = [len(axes[key]) for key in keys] + if planners: + sizes.append(len(planners)) parts = balanced_shard_parts(sizes, max_shards, parallelism=parallelism) largest = largest_shard_points(sizes, parts) if largest > DGDR_MAX_SHARD_POINTS: @@ -216,7 +274,11 @@ def plan_shards( ) shards: list[Shard] = [] - chunked = [_chunks(axes[key], part) for key, part in zip(keys, parts)] + chunked: list[list[Any]] = [ + _chunks(axes[key], part) for key, part in zip(keys, parts) + ] + if planners: + chunked.append(_chunks(planners, parts[-1])) for index, combo in enumerate(itertools.product(*chunked)): choices = dict(zip(keys, combo)) shards.append( @@ -225,41 +287,56 @@ def plan_shards( kind="grid", choices=choices, points=_product(choices), + planners=tuple(combo[len(keys)]) if planners else (), ) ) - def in_grid(point: Mapping[str, float]) -> bool: - return all(float(point[key]) in axes[key] for key in keys) + def in_grid(point: Mapping[str, float], planner: Mapping[str, Any]) -> bool: + return all(float(point[key]) in axes[key] for key in keys) and ( + not planners or any(_same_planner(planner, p) for p in planners) + ) - def single(shard_id: str, kind: ShardKind, point: Mapping[str, float]) -> Shard: + def single( + shard_id: str, + kind: ShardKind, + point: Mapping[str, float], + planner: Mapping[str, Any], + ) -> Shard: values = {key: float(point[key]) for key in keys} return Shard( id=shard_id, kind=kind, choices={key: (value,) for key, value in values.items()}, points=(values,), + planners=(dict(planner),) if planners else (), ) baseline_point = {key: float(baseline[key]) for key in keys} startup_point = {key: float(startup[key]) for key in keys} - if not in_grid(baseline_point): - shards.append(single("point-baseline", "point", baseline_point)) - if not in_grid(startup_point) and startup_point != baseline_point: - shards.append(single("point-startup", "point", startup_point)) + base_planner = dict(baseline_planner or {}) + start_planner = dict(startup_planner or {}) + if not in_grid(baseline_point, base_planner): + shards.append(single("point-baseline", "point", baseline_point, base_planner)) + if not in_grid(startup_point, start_planner) and ( + startup_point != baseline_point + or not _same_planner(start_planner, base_planner) + ): + shards.append(single("point-startup", "point", startup_point, start_planner)) if probe: - shards.append(single("probe", "probe", _probe_point(axes))) + middle = planners[len(planners) // 2] if planners else {} + shards.append(single("probe", "probe", _probe_point(axes), middle)) return shards -def _chunks(values: tuple[float, ...], count: int) -> list[tuple[float, ...]]: +def _chunks(values: Sequence[Any], count: int) -> list[tuple[Any, ...]]: """``values`` cut into ``count`` contiguous chunks whose sizes differ by <= 1.""" base, extra = divmod(len(values), count) - chunks: list[tuple[float, ...]] = [] + chunks: list[tuple[Any, ...]] = [] start = 0 for index in range(count): end = start + base + (1 if index < extra else 0) - chunks.append(values[start:end]) + chunks.append(tuple(values[start:end])) start = end return chunks @@ -365,6 +442,8 @@ def build_request( spec_replicas: Mapping[str, int], backend: str, attempt: int, + planner_base: Optional[Mapping[str, Any]] = None, + planner_warmup: Optional[Mapping[str, Any]] = None, ) -> dict: """The DGDR v1beta2 Request that replays one shard of a window. @@ -379,6 +458,13 @@ def build_request( is the one the configuration check (P5) accepted. Raises ``ValueError`` for ``spec_replicas`` or a shard the Request cannot express. + + A shard with planner points (Scenario 2) adds the planner adapter's + ``retune_sweep`` section: ``planner_base`` (the live config's + ``retune_sweep_base``), the shard's points, and ``planner_warmup`` + (history rows by throughput interval). ``spec_replicas`` are then the + ready counts at the window start, where each candidate's inner planner + starts scaling. """ engine = _pinned_engine(cfg.engine, spec_replicas, backend) @@ -428,6 +514,19 @@ def build_request( "candidate_timeout_seconds": cfg.run.candidate_timeout_seconds, }, "optimization": {"strict_sla": False}, + **( + { + "planner": { + RETUNE_SWEEP: { + "base": copy.deepcopy(dict(planner_base or {})), + "points": [dict(point) for point in shard.planners], + "warmup": copy.deepcopy(dict(planner_warmup or {})), + } + } + } + if shard.searches_planner + else {} + ), # The planner reads only each Candidate's parameters and # score, never its DGD. The direct generator keeps retune # Runs independent of the AIC generator, whose rendering can @@ -696,6 +795,10 @@ class DgdrContext: now: Callable[[], float] = time.time sleep: Optional[Callable[[float], None]] = None poll_interval_s: float = 5.0 + # Scenario 2: the history reader and the pre-sweep's worker pool + # (window_cut.bucket_history and sweep.WorkerPool; tests replace them). + bucket_history: Callable[..., Any] = field(default=bucket_history) + worker_pool: Callable[..., Any] = field(default=WorkerPool) def run_dgdr_sweep( @@ -883,6 +986,17 @@ def __init__( } self.k8s_errors = 0 self.runtime: dict[str, Any] = {} + # Scenario 2 (set by _prepare_planner): what each candidate replays + # as, the planner points and warmup the Requests carry, and the + # pre-sweep and swap inputs the decision reads. + self.evaluated: dict[str, RetuneParams] = {} + self.planner_points: list[dict[str, Any]] = [] + self.planner_base: Optional[dict[str, Any]] = None + self.planner_warmup: Optional[dict[str, Any]] = None + self.presweep: Optional[dict[str, Any]] = None + self.interval_observations: dict[int, tuple[Any, ...]] = {} + self.warmup_errors: dict[int, str] = {} + self.base_interval: Optional[int] = None # ------------------------------------------------------------ driver @@ -948,16 +1062,28 @@ def _result( details["shards"] = {key: run.summary() for key, run in self.shards.items()} details["k8s_errors"] = self.k8s_errors details.update(self.runtime) + base_interval = self.base_interval return SweepResult( status=status, num_requests=self.num_requests, row_counts=dict(self.row_counts), scores=scores or {}, metrics=metrics or {}, + predictor_presweep=self.presweep, + warmup_observations=( + None + if base_interval is None + else self.interval_observations.get(base_interval) + ), + predictor_warmup_error=( + None if base_interval is None else self.warmup_errors.get(base_interval) + ), wall_s=max(0.0, self._now() - self.started), error=error, failure_reason=failure_reason, details=details, + interval_observations=dict(self.interval_observations), + predictor_warmup_errors=dict(self.warmup_errors), ) # ----------------------------------------------------------- helpers @@ -1040,24 +1166,39 @@ def _prepare(self) -> None: domain = request.router_domain if not domain: self._fail("invalid_request", "the sweep request has no router domain") - if any(candidate.planner for candidate in request.candidates): + if request.base_planner is None and any( + candidate.planner for candidate in request.candidates + ): self._fail( "invalid_request", - "a DGDR sweep searches router parameters only (Scenario 1)", + "planner candidates need the inner planner's base configuration", ) keys = set(domain) - startup = self._startup_point(domain) + if request.base_planner is not None: + self._prepare_planner() + startup = self._startup_candidate(domain) + baseline = self._evaluated(request.baseline) + planner_points = self.planner_points try: shards = plan_shards( domain, - baseline=request.baseline.router, - startup=startup, + baseline=baseline.router, + startup=startup.router, max_shards=self.cfg.run.max_shards, parallelism=self.cfg.run.parallelism, + planner_points=planner_points, + baseline_planner=baseline.planner, + startup_planner=startup.planner, ) except (KeyError, ValueError) as error: self._fail("invalid_domain", str(error)) - replicas = request.spec_replicas or request.replicas + # A scaling planner starts each candidate from the ready counts at + # the window start (SweepRequest.replicas); a fixed one from the spec. + replicas = ( + request.replicas + if request.base_planner is not None + else request.spec_replicas or request.replicas + ) self.details["replicas"] = dict(replicas) probe = next((shard for shard in shards if shard.kind == "probe"), None) for shard in shards: @@ -1075,6 +1216,8 @@ def _prepare(self) -> None: spec_replicas=replicas, backend=self.ctx.backend, attempt=1, + planner_base=self.planner_base, + planner_warmup=self.planner_warmup, ) except (KeyError, TypeError, ValueError) as error: self._fail("invalid_engine", f"shard {shard.id}: {error}") @@ -1082,8 +1225,13 @@ def _prepare(self) -> None: # the probe is compared with. critical = ( shard.kind == "probe" - or shard.contains(request.baseline.router) - or (probe is not None and shard.contains(probe.points[0])) + or shard.contains(baseline.router, baseline.planner) + or ( + probe is not None + and shard.contains( + probe.points[0], probe.planners[0] if probe.planners else None + ) + ) ) self.shards[shard.id] = _ShardRun( shard=shard, @@ -1092,32 +1240,190 @@ def _prepare(self) -> None: digest=body["metadata"]["labels"][LABEL_SPEC], ) uncovered = [ - candidate.router + candidate.to_dict() for candidate in request.candidates if set(candidate.router) != keys - or not any( - run.shard.kind != "probe" and run.shard.contains(candidate.router) - for run in self.shards.values() - ) + or not self._covered(self._evaluated(candidate)) ] if uncovered: self.details["uncovered_candidates"] = uncovered[:10] - def _startup_point(self, domain: Mapping[str, Sequence[float]]) -> dict[str, float]: + def _covered(self, point: RetuneParams) -> bool: + return any( + run.shard.kind != "probe" + and run.shard.contains(point.router, point.planner) + for run in self.shards.values() + ) + + def _evaluated(self, candidate: RetuneParams) -> RetuneParams: + """The point a candidate replays as (``sweep.evaluated_params``).""" + + return self.evaluated.get(candidate.key(), candidate) + + def _startup_candidate(self, domain: Mapping[str, Sequence[float]]) -> RetuneParams: """The off-grid candidate other than the baseline, else the baseline. The controller's candidates are the grid plus the startup point and - the baseline, so an off-grid candidate is one of those two. + the baseline, so an off-grid candidate is one of those two. A + candidate is off the grid when its router point is; planner points + are the distinct points of every candidate, so they never are. """ - baseline = self.request.baseline + baseline = self._evaluated(self.request.baseline) for candidate in self.request.candidates: router = candidate.router - if candidate == baseline or set(router) != set(domain): + if candidate == self.request.baseline or set(router) != set(domain): continue if not all(float(router[key]) in domain[key] for key in domain): - return dict(router) - return dict(baseline.router) + return self._evaluated(candidate) + return baseline + + def _prepare_planner(self) -> None: + """Bucket the history, run the predictor pre-sweep and fix the + planner points each Request carries (Scenario 2). + + Within ``cut_budget_seconds`` of the deadline, before any Request is + planned, because the points and the warmup are part of every + Request's spec: the history before the window (and the window + itself, for the pre-sweep) is read from the trace segments at each + throughput interval a candidate scales at; the pre-sweep picks a + predictor per interval; each candidate is mapped to the point it + replays as (``evaluated_params``); and every predictor a decision + could swap in is prepared on the same observations, as replay + retune does, so the decision can check it. + """ + + request = self.request + budget = min(self.cfg.cut_budget_seconds, self.deadline - self._now()) + if budget <= 0: + self._fail("launch_too_late", "no time left to read the history", "timeout") + deadline_s = self._now() + budget + intervals = sweep_throughput_intervals(request) + observations: dict[int, tuple[Any, ...]] = {} + warmups: dict[int, tuple[Any, ...]] = {} + if intervals: + history = self._bucket_history(intervals, deadline_s) + observations = dict(history.observations) + warmups = dict(history.warmups) + self.details["history"] = { + "start_ms": int(round(request.history_start_ms)), + "requests": history.history_requests, + "dropped_members": history.dropped_members, + "intervals": list(intervals), + } + presweep = None + warmup_errors: dict[int, str] = {} + # The configuration the Runs replay (only its scaling fields): the + # live dump may hold settings, such as non-advisory batch + # scheduling, that need what a dump leaves out. + inner_base = { + **retune_sweep_base(request.base_planner or {}), + "retune": None, + "advisory": False, + "report_interval_hours": None, + "live_dashboard_port": 0, + } + with tempfile.TemporaryDirectory(prefix="retune-dgdr-planner-") as work: + work_dir = Path(work) + paths = { + interval: write_observations( + work_dir / f"observations-{interval}.json", rows + ) + for interval, rows in observations.items() + } + pool = self.ctx.worker_pool( + self.ctx.retune.max_parallel, + time.monotonic() + (deadline_s - self._now()), + self.stop, + work, + ) + try: + if request.predictor_candidates and intervals: + presweep = run_presweeps( + request, pool, observations, paths, intervals + ) + for candidate in request.candidates: + self.evaluated[candidate.key()] = evaluated_params( + candidate, request.baseline, presweep, request.base_planner + ) + tasks = predictor_swap_tasks( + request, + list(self.evaluated.values()), + inner_base, + observations, + paths, + work_dir, + ) + if tasks: + warmup_errors = predictor_swap_errors(pool.run(tasks)) + except SweepInterrupted as interrupted: + self._check_interrupt() + self._fail( + "window_cut_timeout", + f"history and predictor pre-sweep: {interrupted}", + "timeout", + ) + finally: + pool.close() + planners: dict[str, dict[str, Any]] = {} + for point in self.evaluated.values(): + planners.setdefault( + RetuneParams(planner=point.planner).key(), point.planner + ) + self.planner_points = [planners[key] for key in sorted(planners)] + self.planner_base = retune_sweep_base(request.base_planner or {}) + self.planner_warmup = { + str(interval): [list(dataclasses.astuple(row)) for row in rows] + for interval, rows in warmups.items() + } + oversized = { + interval: len(rows) + for interval, rows in warmups.items() + if len(rows) > DGDR_MAX_WARMUP_BUCKETS + } + if oversized: + self._fail( + "invalid_request", + f"the history warmup has {oversized} buckets by interval, more " + f"than {DGDR_MAX_WARMUP_BUCKETS} per interval a Request carries", + ) + self.presweep = presweep + self.interval_observations = observations + self.warmup_errors = warmup_errors + running = {**(request.base_planner or {}), **request.baseline.planner} + self.base_interval = scales_on_throughput({}, running) + self.details["planner"] = { + "points": len(self.planner_points), + "presweep": presweep, + "warmup_errors": {str(k): v for k, v in warmup_errors.items()}, + } + + def _bucket_history(self, intervals: Sequence[int], deadline_s: float) -> Any: + request = self.request + try: + return self.ctx.bucket_history( + self._segments_root(), + int(round(request.history_start_ms)), + self.window_start_ms, + self.window_end_ms, + self.as_of_ms, + intervals, + deadline_s=deadline_s, + cancel=self.stop, + ) + except Exception as error: + self._check_interrupt() + reason = getattr(error, "reason", None) + if reason not in ("window_cut_timeout", "trace_corrupt"): + reason = "window_cut_failed" + self._fail(reason, f"history read: {type(error).__name__}: {error}") + + def _segments_root(self) -> str: + if self.request.trace_paths: + return self.request.trace_paths[0] + return os.path.join( + self.cfg.mount_path, self.cfg.segments_dir, self.ctx.dgd.name + ) def _build(self, run: _ShardRun) -> dict: body = build_request( @@ -1133,6 +1439,8 @@ def _build(self, run: _ShardRun) -> dict: spec_replicas=self.details["replicas"], backend=self.ctx.backend, attempt=run.attempt, + planner_base=self.planner_base, + planner_warmup=self.planner_warmup, ) rows = {"num_requests": self.num_requests, "row_counts": self.row_counts} body["metadata"]["annotations"] = { @@ -1342,13 +1650,7 @@ def _cut(self) -> None: """Cut the window file; an empty window ends the sweep as ``empty``.""" self._check_interrupt() - segments_root = ( - self.request.trace_paths[0] - if self.request.trace_paths - else os.path.join( - self.cfg.mount_path, self.cfg.segments_dir, self.ctx.dgd.name - ) - ) + segments_root = self._segments_root() budget = min(self.cfg.cut_budget_seconds, self.deadline - self._now()) if budget <= 0: self._fail("launch_too_late", "no time left to cut the window", "timeout") @@ -1712,7 +2014,7 @@ def _harvest(self) -> SweepResult: name = (candidate.get("metadata") or {}).get("name", "") parameters = _get(candidate, ("spec", "parameters")) try: - point, signature = _map_parameters(parameters, run.shard) + point, planner, signature = _map_parameters(parameters, run.shard) except _Unmapped as error: if error.fatal: self._fail("engine_not_pinned", f"Candidate {name}: {error}") @@ -1726,13 +2028,14 @@ def _harvest(self) -> SweepResult: f"Candidates {signature_source} and {name} differ in " "non-router parameters", ) - key = RetuneParams(router=point).key() + params = RetuneParams(router=point, planner=planner) + key = params.key() target = probe_results if run.shard.kind == "probe" else results if key in target: continue metrics = _get(candidate, ("status", "metrics"), {}) or {} entry: dict[str, Any] = { - "params": RetuneParams(router=point).to_dict(), + "params": params.to_dict(), "shard": run.shard.id, "candidate": name, } @@ -1755,15 +2058,16 @@ def _harvest(self) -> SweepResult: missing = 0 for candidate in request.candidates: key = candidate.key() - found = results.get(key) + evaluated = self._evaluated(candidate) + found = results.get(evaluated.key()) if found is not None and set(candidate.router) == domain_keys: scores[key] = found.score metrics_out[key] = found.entry else: scores[key] = None metrics_out[key] = { - "params": candidate.to_dict(), - "error": self._missing_reason(candidate.router), + "params": evaluated.to_dict(), + "error": self._missing_reason(evaluated), } if scores[key] is None: missing += 1 @@ -1780,9 +2084,11 @@ def _harvest(self) -> SweepResult: def _list_candidates(self, selector: str) -> list[dict]: return self.ctx.kube.list_candidates(selector, timeout_s=self._timeout()) - def _missing_reason(self, point: Mapping[str, float]) -> str: + def _missing_reason(self, point: RetuneParams) -> str: for run in self.shards.values(): - if run.shard.kind != "probe" and run.shard.contains(point): + if run.shard.kind != "probe" and run.shard.contains( + point.router, point.planner + ): if run.state == "missing": return f"shard {run.shard.id} {run.reason}: {run.message}" return f"not published by Run {run.run_name} (infeasible or failed)" @@ -1795,7 +2101,8 @@ def _check_probe( if probe is None: return point = probe.shard.points[0] - key = RetuneParams(router=point).key() + planner = probe.shard.planners[0] if probe.shard.planners else {} + key = RetuneParams(router=point, planner=planner).key() grid = results.get(key) repeat = probe_results.get(key) grid_score = None if grid is None else grid.score @@ -1831,13 +2138,17 @@ def __init__(self, message: str, *, fatal: bool = False) -> None: self.fatal = fatal -def _map_parameters(parameters: Any, shard: Shard) -> tuple[dict[str, float], str]: - """A Candidate's live router point and its non-router signature. +def _map_parameters( + parameters: Any, shard: Shard +) -> tuple[dict[str, float], dict[str, Any], str]: + """A Candidate's live router point, planner point and other signature. The signature is the canonical JSON of the parameters without the three - searched knobs; every Candidate of a window must share it. Raises - ``_Unmapped`` for parameters that name no point of ``shard``, fatally - for a router other than the pinned ``kv_router`` without a load model. + searched router knobs and the planner point; every Candidate of a + window must share it. The planner point is ``{}`` for a shard without + planner points. Raises ``_Unmapped`` for parameters that name no point + of ``shard``, fatally for a router other than the pinned ``kv_router`` + without a load model. """ if not isinstance(parameters, Mapping): @@ -1869,15 +2180,40 @@ def _map_parameters(parameters: Any, shard: Shard) -> tuple[dict[str, float], st f"router knob {public}={value!r} is not one of the shard's values" ) point[live] = snapped - rest = dict(parameters) - rest["adapters"] = { + planner: dict[str, Any] = {} + rest_adapters = { **adapters, ROUTER_ADAPTER: { key: value for key, value in router.items() if key not in DGDR_ROUTER_KNOBS }, } + if shard.searches_planner: + section = adapters.get(PLANNER_ADAPTER) + if not isinstance(section, Mapping): + raise _Unmapped(f"spec.parameters.adapters[{PLANNER_ADAPTER!r}] is missing") + sweep = section.get(RETUNE_SWEEP) + points = sweep.get("points") if isinstance(sweep, Mapping) else None + chosen = points[0] if isinstance(points, list) and len(points) == 1 else None + if not isinstance(chosen, Mapping): + raise _Unmapped( + f"spec.parameters.adapters[{PLANNER_ADAPTER!r}].{RETUNE_SWEEP} " + "does not name one planner point" + ) + match = next( + (option for option in shard.planners if _same_planner(chosen, option)), + None, + ) + if match is None: + raise _Unmapped(f"planner point {dict(chosen)} is not one of the shard's") + planner = dict(match) + # The base and warmup are the Request's; only the point varies. + rest_adapters[PLANNER_ADAPTER] = { + key: value for key, value in section.items() if key != RETUNE_SWEEP + } + rest = dict(parameters) + rest["adapters"] = rest_adapters signature = json.dumps(rest, sort_keys=True, separators=(",", ":"), default=str) - return point, signature + return point, planner, signature def _snap(value: float, values: Sequence[float]) -> Optional[float]: diff --git a/components/src/dynamo/planner/retune/live/reconciler.py b/components/src/dynamo/planner/retune/live/reconciler.py index a3ce986b1b10..7b0f49149726 100644 --- a/components/src/dynamo/planner/retune/live/reconciler.py +++ b/components/src/dynamo/planner/retune/live/reconciler.py @@ -177,6 +177,7 @@ def __init__( fs_root: str, metrics: RetuneMetricsSink, now: Callable[[], float] = time.time, + history_s: float = 0.0, ) -> None: self._kube = kube self._cfg = cfg @@ -185,8 +186,11 @@ def __init__( self._now = now interval_s = float(retune.interval_seconds) sweep_timeout_s = float(retune.effective_timeout_seconds) + # A scaling Planner's sweep also reads history_s before its window + # (the inner planners' predictor warmup). self._segment_max_age_s = ( - interval_s + float(history_s) + + interval_s + float(retune.grace_seconds) + sweep_timeout_s + VISIBILITY_MARGIN_S diff --git a/components/src/dynamo/planner/retune/live/supervisor.py b/components/src/dynamo/planner/retune/live/supervisor.py index d69ece31fbaa..59a435cabea0 100644 --- a/components/src/dynamo/planner/retune/live/supervisor.py +++ b/components/src/dynamo/planner/retune/live/supervisor.py @@ -1281,6 +1281,7 @@ def current() -> bool: return False self._initialized = True self._rollback_since = None + self._warn_planner_reset(plan.state) if self._background: self._outbox_task = asyncio.create_task( self._outbox_loop(), name="planner-retune-outbox" @@ -1288,6 +1289,40 @@ def current() -> bool: self._outbox_wake.set() return True + def _warn_planner_reset(self, state: RetuneState) -> None: + """Report applied Planner changes a restart did not restore. + + Router parameters live in the frontends and survive a Planner + restart; Planner fields live in this process, which starts from its + configuration. Retune does not restore them (restoring a predictor + needs a warmup): the next window's baseline is the running + configuration, and a later sweep may choose the change again. + """ + + config = getattr(self._host, "config", None) + if config is None: + return + last = next( + ( + item + for item in reversed(state.history) + if item.get("outcome") == "applied" and item.get("planner") + ), + None, + ) + if last is None: + return + planner = last["planner"] + running = {key: getattr(config, key, None) for key in planner} + if running != planner: + logger.warning( + "retune: Planner fields %s of decision %s are not restored after " + "the restart; the Planner runs %s", + planner, + last.get("decision_id"), + running, + ) + async def _read_own_snapshot(self) -> Optional[StateSnapshot]: """Read the state; drop a snapshot of another DGD uid. @@ -1538,6 +1573,10 @@ def _history_entry(self, entry: OutboxEntry) -> dict[str, Any]: item["by_wt"] = entry.router_parameters elif decision.outcome == "advisory": item["by_wt"] = self._resolve_by_wt(flat) + if decision.planner_changes and decision.outcome in ("applied", "advisory"): + # The Planner fields the decision changes (or recommends). A + # restarted Planner does not restore them (_warn_planner_reset). + item["planner"] = dict(decision.planner_changes) details = decision.details image = { key: details[key] diff --git a/components/src/dynamo/planner/retune/live/window_cut.py b/components/src/dynamo/planner/retune/live/window_cut.py index a44006201487..e2617eec4080 100644 --- a/components/src/dynamo/planner/retune/live/window_cut.py +++ b/components/src/dynamo/planner/retune/live/window_cut.py @@ -53,8 +53,10 @@ import numpy as np +from dynamo.planner.core.types import TrafficObservation from dynamo.planner.retune.live.contracts import SupportsIsSet, WindowCutResult from dynamo.planner.retune.window import ( + ObservationBuckets, _dumps, _new_counts, _parse_request_end, @@ -287,6 +289,162 @@ def cut_window_inprocess( ) from error +@dataclass(frozen=True) +class TraceHistory: + """Traffic before and in a window, bucketed per throughput interval. + + Keys are throughput intervals in seconds. ``observations`` are the + buckets of history and window that end at the window end, for the load + predictor pre-sweep and a predictor swap; ``warmups`` the buckets of + history alone that end at the window start, the warmup of a candidate's + inner planner. ``history_requests`` counts the history rows. + """ + + observations: dict[int, tuple[TrafficObservation, ...]] + warmups: dict[int, tuple[TrafficObservation, ...]] + history_requests: int + dropped_members: int + + +def bucket_history( + segments_root: StrPath, + history_start_ms: float, + window_start_ms: float, + window_end_ms: float, + as_of_ms: float, + intervals_s: Sequence[int], + *, + deadline_s: float, + cancel: Optional[SupportsIsSet] = None, +) -> TraceHistory: + """``bucket_history_inprocess`` in a spawned child process. + + The child is killed at ``deadline_s`` (absolute wall-clock seconds) or + on ``cancel``, as ``cut_window``'s is; failures raise ``WindowCutError``. + """ + + if not math.isfinite(deadline_s): + raise ValueError("deadline_s must be finite") + deadline = time.monotonic() + (deadline_s - time.time()) + started = time.monotonic() + result: TraceHistory = _run_in_child( + bucket_history_inprocess, + ( + os.fspath(segments_root), + history_start_ms, + window_start_ms, + window_end_ms, + as_of_ms, + tuple(int(interval) for interval in intervals_s), + ), + {}, + deadline=deadline, + cancel=cancel, + name=f"retune-history-{int(round(window_end_ms))}", + ) + logger.info( + "retune history [%s, %s) of window ending %s: %d requests bucketed at " + "%s s in %.1fs; dropped %d gzip members", + history_start_ms, + window_start_ms, + window_end_ms, + result.history_requests, + sorted(result.observations), + time.monotonic() - started, + result.dropped_members, + ) + return result + + +def bucket_history_inprocess( + segments_root: StrPath, + history_start_ms: float, + window_start_ms: float, + window_end_ms: float, + as_of_ms: float, + intervals_s: Sequence[int], +) -> TraceHistory: + """Bucket the requests of ``[history_start_ms, window_end_ms)`` that + completed by ``as_of_ms``, at each interval of ``intervals_s``. + + Row selection and censoring are the window cut's; as in + ``retune.window.read_trace_history``, rows are streamed and not + de-duplicated. A gzip member that fails validation is skipped and + counted, without the cut's corruption limit: a few lost history rows + only perturb a predictor's warmup. Failures raise ``WindowCutError``. + """ + + if not history_start_ms <= window_start_ms < window_end_ms <= as_of_ms: + raise ValueError( + "need history_start_ms <= window_start_ms < window_end_ms <= as_of_ms" + ) + observations = { + int(interval): ObservationBuckets( + end_ms=window_end_ms, + interval_ms=int(interval) * 1000.0, + duration_s=float(interval), + ) + for interval in intervals_s + } + warmups = { + interval: ObservationBuckets( + end_ms=window_start_ms, + interval_ms=buckets.interval_ms, + duration_s=buckets.duration_s, + ) + for interval, buckets in observations.items() + } + targets = (*observations.values(), *warmups.values()) + history_requests = 0 + dropped = 0 + try: + files = _segment_files(os.fspath(segments_root)) + items = _trace_items(files, history_start_ms / 1000.0 - SEGMENT_MTIME_MARGIN_S) + try: + for _, path, item in items: + if not isinstance(item, _Member): + dropped += isinstance(item, _Garbage) + continue + for index, raw in enumerate(item.payload.split(b"\n")): + if not raw.strip(): + continue + location = f"{path}@{item.offset}:{index + 1}" + event = _event(raw, location, "trace_corrupt") + row = ( + None + if event is None + else _request_row(event, location, "trace_corrupt") + ) + if row is None or not ( + history_start_ms <= row.arrival_ms < window_end_ms + ): + continue + if row.completion_ms > as_of_ms: + continue + if row.arrival_ms < window_start_ms: + history_requests += 1 + for target in targets: + target.add(row.arrival_ms, row.input_length, row.output_tokens) + finally: + items.close() + except OSError as error: + raise WindowCutError( + "window_cut_failed", f"{type(error).__name__}: {error}" + ) from error + return TraceHistory( + observations={ + interval: tuple(buckets.observations()) + for interval, buckets in observations.items() + }, + warmups={ + interval: tuple(buckets.observations()) + for interval, buckets in warmups.items() + }, + history_requests=history_requests, + dropped_members=dropped, + ) + + def _validate( start_ms: float, end_ms: float, diff --git a/components/src/dynamo/planner/retune/sweep.py b/components/src/dynamo/planner/retune/sweep.py index 55f8acf595c6..b236efb4660a 100644 --- a/components/src/dynamo/planner/retune/sweep.py +++ b/components/src/dynamo/planner/retune/sweep.py @@ -812,7 +812,7 @@ def _run_sweep( ) -> SweepResult: deadline = started + request.timeout_s running = _running_planner(request.baseline, request.base_planner) - intervals = _throughput_intervals(request) + intervals = sweep_throughput_intervals(request) buckets = { interval: ObservationBuckets( end_ms=request.window_end_ms, @@ -884,7 +884,13 @@ def _run_sweep( "batch_scheduling": {"enabled": False}, } if presweep_intervals: - presweep = _run_presweeps(request, pool, inputs, presweep_intervals) + presweep = run_presweeps( + request, + pool, + inputs.observations, + inputs.observations_paths, + presweep_intervals, + ) by_effective: dict[str, tuple[RetuneParams, list[str]]] = {} for candidate in request.candidates: effective = evaluated_params( @@ -917,26 +923,14 @@ def _run_sweep( # rather than on the planner tick that decides to swap it in. # They go first, so an ARIMA or Prophet fit on days of history # overlaps the replays instead of trailing them. - swaps = _predictor_swaps( - request.baseline, - running, + tasks[:0] = predictor_swap_tasks( + request, [effective for effective, _ in by_effective.values()], + inner_base, + inputs.observations, + inputs.observations_paths, + work_dir, ) - for interval, planner in swaps.items(): - path = inputs.observations_paths.get(interval) - if path is None: - path = _write_observations( - work_dir / f"{_OBSERVATIONS_FILE}-{interval}.json", - inputs.observations.get(interval, ()), - ) - tasks.insert( - 0, - ( - f"{_PREDICTOR_WARMUP_TASK}:{interval}", - _predictor_warmup_task, - ({**inner_base, **planner}, path), - ), - ) outcomes = pool.run(tasks) except _SweepInterrupted as interrupted: logger.warning( @@ -955,10 +949,7 @@ def _run_sweep( finally: pool.close() - warmup_errors: dict[int, str] = {} - for task_id, (ok, value) in outcomes.items(): - if task_id.startswith(f"{_PREDICTOR_WARMUP_TASK}:") and not ok: - warmup_errors[int(task_id.rsplit(":", 1)[1])] = value + warmup_errors = predictor_swap_errors(outcomes) for effective_key, (effective, original_keys) in by_effective.items(): ok, value = outcomes[effective_key] if ok: @@ -1014,6 +1005,51 @@ def _warmup_path( return None if interval is None else inputs.warmup_paths.get(interval) +def predictor_swap_tasks( + request: SweepRequest, + evaluated: Sequence[RetuneParams], + inner_base: Mapping[str, Any], + observations: Mapping[int, Sequence[TrafficObservation]], + observations_paths: Mapping[int, str], + work_dir: Path, +) -> list[tuple[str, Callable[..., Any], tuple[Any, ...]]]: + """Worker tasks that prepare, per throughput interval, the predictor a + host would swap in for the ``evaluated`` candidates at that interval + (``_predictor_swaps``), on the observations of that interval; a missing + observations file is written to ``work_dir``. ``predictor_swap_errors`` + reads their outcomes.""" + + running = _running_planner(request.baseline, request.base_planner) + tasks: list[tuple[str, Callable[..., Any], tuple[Any, ...]]] = [] + for interval, planner in _predictor_swaps( + request.baseline, running, evaluated + ).items(): + path = observations_paths.get(interval) + if path is None: + path = _write_observations( + work_dir / f"{_OBSERVATIONS_FILE}-{interval}.json", + observations.get(interval, ()), + ) + tasks.append( + ( + f"{_PREDICTOR_WARMUP_TASK}:{interval}", + _predictor_warmup_task, + ({**inner_base, **planner}, path), + ) + ) + return tasks + + +def predictor_swap_errors(outcomes: Mapping[str, tuple[bool, Any]]) -> dict[int, str]: + """The failed ``predictor_swap_tasks`` outcomes, by throughput interval.""" + + errors: dict[int, str] = {} + for task_id, (ok, value) in outcomes.items(): + if task_id.startswith(f"{_PREDICTOR_WARMUP_TASK}:") and not ok: + errors[int(task_id.rsplit(":", 1)[1])] = value + return errors + + def _predictor_swaps( baseline: RetuneParams, running: Mapping[str, Any], @@ -1051,7 +1087,7 @@ def _nonzero(counts: Mapping[str, int]) -> dict[str, int]: return {key: value for key, value in counts.items() if value} -def _throughput_intervals(request: SweepRequest) -> list[int]: +def sweep_throughput_intervals(request: SweepRequest) -> list[int]: """The throughput intervals an inner planner scales at: the base planner's and every candidate's (``[]`` without an inner planner).""" @@ -1072,21 +1108,23 @@ def _throughput_intervals(request: SweepRequest) -> list[int]: return sorted(intervals) -def _run_presweeps( +def run_presweeps( request: SweepRequest, pool: _WorkerPool, - inputs: _SweepInputs, + observations: Mapping[int, Sequence[TrafficObservation]], + observations_paths: Mapping[int, str], intervals: Sequence[int], ) -> dict[str, Any]: """Score every predictor candidate at every throughput interval in its own worker task, then pick one per interval. - ``inputs.observations`` are history and window buckets ending at the - window end; only the buckets wholly inside ``W_k`` are forecast and - scored, and the ones before them only warm the predictors. Each task - carries the path of the observations file, not the observations. One - interval gives its summary (``SweepResult.predictor_presweep``); several - give ``{"by_interval": {...}}``. + ``observations`` (by interval) are history and window buckets ending at + the window end, in the files ``observations_paths``; only the buckets + wholly inside ``W_k`` are forecast and scored, and the ones before them + only warm the predictors. Each task carries the path of the + observations file, not the observations. One interval gives its + summary (``SweepResult.predictor_presweep``); several give + ``{"by_interval": {...}}``. """ started = time.monotonic() @@ -1094,16 +1132,16 @@ def _run_presweeps( tasks: list[tuple[str, Callable[..., Any], tuple[Any, ...]]] = [] steps: dict[int, tuple[int, int]] = {} for interval in intervals: - path = inputs.observations_paths.get(interval) - observations = inputs.observations.get(interval) - if path is None or observations is None: + path = observations_paths.get(interval) + rows = observations.get(interval) + if path is None or rows is None: raise RuntimeError("the pre-sweep needs the read task's observations file") interval_ms = interval * 1000.0 * request.arrival_speedup_ratio window_steps = int( (request.window_end_ms - request.window_start_ms) / interval_ms + 1e-9 ) - eval_steps = min(len(observations), window_steps) - steps[interval] = (eval_steps, len(observations) - eval_steps) + eval_steps = min(len(rows), window_steps) + steps[interval] = (eval_steps, len(rows) - eval_steps) tasks.extend( ( f"{_PRESWEEP_TASK}:{interval}:{label}", @@ -1341,3 +1379,10 @@ def _worker_main(conn: Connection, temp_dir: str) -> None: except Exception as error: # An unpicklable result is reported as this task's failure. conn.send((False, f"unsendable result: {error}")) + + +# Public names for the live DGDR executor, which prepares inner planners +# with the same worker pool before it creates its Requests. +WorkerPool = _WorkerPool +SweepInterrupted = _SweepInterrupted +write_observations = _write_observations diff --git a/components/src/dynamo/planner/simulation/config.py b/components/src/dynamo/planner/simulation/config.py index 828f5068a7fe..86b12d96e32a 100644 --- a/components/src/dynamo/planner/simulation/config.py +++ b/components/src/dynamo/planner/simulation/config.py @@ -382,6 +382,71 @@ def _validate_preset_conflicts(self) -> PlannerRecommendationConfig: } +# ``PlannerConfig`` fields a live retune sweep (``retune.executor: dgdr``) +# sends as the base of each candidate's inner planner, in the +# ``planner.retune_sweep`` section of a DGDR Request: the fields that decide +# scaling. Connections, credentials, file paths, ports, reports, the +# Kubernetes wiring and the performance-model sources stay in the live +# Planner; the Run's replay brings its own performance model. +PLANNER_RETUNE_SWEEP_BASE_FIELDS = frozenset( + { + "mode", + "optimization_target", + "ttft_ms", + "itl_ms", + "enable_throughput_scaling", + "enable_load_scaling", + "throughput_adjustment_interval_seconds", + "load_adjustment_interval_seconds", + "max_throughput_scaling_replicas", + "max_gpu_budget", + "min_gpu_budget", + "min_endpoint", + "prefill_min_endpoint", + "decode_min_endpoint", + "load_predictor", + "load_predictor_log1p", + "prophet_window_size", + "kalman_q_level", + "kalman_q_trend", + "kalman_r", + "kalman_min_points", + "max_num_fpm_samples", + "fpm_sample_bucket_size", + "load_scaling_down_sensitivity", + "load_min_observations", + "prefill_scale_up_queue_tokens", + "prefill_scale_down_queue_tokens", + "decode_scale_up_kv_rate", + "decode_scale_down_kv_rate", + "speculative_nextn", + "throughput_metrics_source", + "scheduling", + } +) +# ``scheduling`` keys a retune sweep base keeps: the tick cadence. Plugins, +# the registration gateway and the live tick deadline stay in the Planner. +PLANNER_RETUNE_SWEEP_SCHEDULING_FIELDS = frozenset({"scale_interval_seconds"}) + + +def retune_sweep_base(planner: Mapping[str, Any]) -> dict[str, Any]: + """The ``planner.retune_sweep.base`` of a live Planner config dump.""" + + base = { + key: value + for key, value in planner.items() + if key in PLANNER_RETUNE_SWEEP_BASE_FIELDS and key != "scheduling" + } + scheduling = planner.get("scheduling") + if isinstance(scheduling, Mapping): + base["scheduling"] = { + key: value + for key, value in scheduling.items() + if key in PLANNER_RETUNE_SWEEP_SCHEDULING_FIELDS + } + return base + + class PlannerRetuneConfig(PlannerRecommendationConfig): """``retune.planner``: the recommendation schema, restricted to what a running planner can change. diff --git a/components/src/dynamo/planner/simulation/provider.py b/components/src/dynamo/planner/simulation/provider.py index 69cf47cdebf6..6836d7e73e6e 100644 --- a/components/src/dynamo/planner/simulation/provider.py +++ b/components/src/dynamo/planner/simulation/provider.py @@ -96,6 +96,13 @@ _WARMUP_TRACE = "load_predictor_warmup_trace" _ADVISORY = "advisory" _PREDICTION_ONLY_FIELDS = (_WARMUP_TRACE, _RETUNE, _ADVISORY) +# Alone in the planner section: a live retune sweep's candidates +# (``RetuneSweepSpec``). Each candidate replays the live Planner's +# configuration with one point's fields applied. ``aisimulate recommend`` +# searches its points; each candidate's concrete config is the section with +# its one point, which ``aisimulate predict`` replays as that candidate. +RETUNE_SWEEP = "retune_sweep" +RETUNE_POINT = "retune_point" _HOOK = RuntimeHookSpec( @@ -403,6 +410,175 @@ def _validate_preset(self) -> LoadPredictorSearch: return self +class RetuneSweepSpec(BaseModel): + """``planner.retune_sweep``: the candidates of a live retune sweep. + + ``base`` is the live Planner configuration (only + ``PLANNER_RETUNE_SWEEP_BASE_FIELDS``), ``points`` the Planner fields each + candidate sets (the searchable runtime fields), and ``warmup`` the + recorded history before the window, keyed by throughput interval in + seconds, as ``TrafficObservation`` rows in field order: the load + predictor warmup of a candidate that scales on throughput at that + interval. + """ + + model_config = ConfigDict(extra="forbid") + + base: dict[str, Any] + points: list[dict[str, Any]] = Field(min_length=1) + warmup: dict[str, list[list[float | None]]] = Field(default_factory=dict) + + @model_validator(mode="after") + def _validate_fields(self) -> RetuneSweepSpec: + from dynamo.planner.reconfiguration import ( + PREDICTOR_FIELDS, + SCALING_POLICY_FIELDS, + ) + + from .config import ( + PLANNER_RETUNE_KNOBS, + PLANNER_RETUNE_SWEEP_BASE_FIELDS, + PLANNER_RETUNE_SWEEP_SCHEDULING_FIELDS, + ) + + extra = sorted(set(self.base) - PLANNER_RETUNE_SWEEP_BASE_FIELDS) + if extra: + raise ValueError(f"retune_sweep.base cannot set {extra}") + scheduling = self.base.get("scheduling", {}) + if not isinstance(scheduling, dict) or set(scheduling) - ( + PLANNER_RETUNE_SWEEP_SCHEDULING_FIELDS + ): + raise ValueError( + "retune_sweep.base.scheduling may set only " + f"{sorted(PLANNER_RETUNE_SWEEP_SCHEDULING_FIELDS)}" + ) + searchable = ( + set(PLANNER_RETUNE_KNOBS.values()) + | {"load_scaling_down_sensitivity"} + | PREDICTOR_FIELDS + | SCALING_POLICY_FIELDS + ) + for point in self.points: + unknown = sorted(set(point) - searchable) + if unknown: + raise ValueError(f"retune_sweep point {point} sets {unknown}") + for interval, rows in self.warmup.items(): + if not interval.isdigit() or int(interval) < 1: + raise ValueError( + f"retune_sweep.warmup keys are intervals in seconds, got {interval!r}" + ) + if any(len(row) != len(_OBSERVATION_FIELDS) for row in rows): + raise ValueError( + f"retune_sweep.warmup rows have {len(_OBSERVATION_FIELDS)} " + f"values ({', '.join(_OBSERVATION_FIELDS)})" + ) + return self + + +_OBSERVATION_FIELDS = ( + "duration_s", + "num_req", + "isl", + "osl", + "kv_hit_rate", + "accept_length", +) + + +def _retune_sweep_planner_config( + spec: RetuneSweepSpec, + point: Mapping[str, JSONValue], + sample: Mapping[str, JSONValue] | None, +) -> dict[str, JSONValue]: + """The inner planner configuration of one retune sweep candidate. + + ``base`` with ``point`` applied, retune off, not advisory (a candidate is + scored on the replicas it would drive), without reports or listeners, + and with the GPUs per worker of the candidate's pinned engine + (``sample``), validated as a whole. Only the fields it sets are + serialized, never environment-derived connection settings. + """ + + payload: dict[str, Any] = { + **deepcopy(spec.base), + **deepcopy(dict(point)), + "retune": None, + "advisory": False, + "report_interval_hours": None, + "live_dashboard_port": 0, + "metric_reporting_prometheus_port": 0, + } + if sample is not None: + mode = str(sample["deployment_mode"]) + if mode == "disagg": + payload["prefill_engine_num_gpu"] = _int_value( + sample, "prefill_tp" + ) * _int_value(sample, "prefill_attention_dp") + payload["decode_engine_num_gpu"] = _int_value( + sample, "decode_tp" + ) * _int_value(sample, "decode_attention_dp") + else: + payload["decode_engine_num_gpu"] = _int_value(sample, "tp") * _int_value( + sample, "attention_dp" + ) + resolved = PlannerConfig.model_validate( + { + **payload, + "metric_pulling_prometheus_endpoint": "", + "metric_pulling_prometheus_token": None, + "metric_pulling_prometheus_token_file": None, + "metric_pulling_prometheus_ssl_verify": False, + "metric_pulling_prometheus_ca_bundle": None, + "metric_pulling_prometheus_request_timeout_seconds": ( + SLAPlannerDefaults.metric_pulling_prometheus_request_timeout_seconds + ), + } + ) + # Retune is off, so PlannerConfig requires a scaling mode: a candidate + # always replays a planner that scales. + return resolved.model_dump(mode="json", include=set(payload)) + + +def _retune_sweep_warmup( + spec: RetuneSweepSpec, planner_config: Mapping[str, JSONValue] +) -> tuple[str, list[list[float | None]]] | None: + """The candidate's throughput interval and its warmup rows, if any.""" + + if not planner_config.get("enable_throughput_scaling"): + return None + interval = str(planner_config.get("throughput_adjustment_interval_seconds")) + rows = spec.warmup.get(interval) + return None if rows is None else (interval, rows) + + +def _retune_sweep_replay( + spec: RetuneSweepSpec, + point: Mapping[str, JSONValue], + sample: Mapping[str, JSONValue], +) -> AdapterReplaySpec: + """The replay of one retune sweep candidate. + + Its config is the section narrowed to the point and its own warmup: a + Candidate's ``adapters["dynamo.planner"]`` names the point, and the + config alone replays it again. + """ + + planner_config = _retune_sweep_planner_config(spec, point, sample) + hook_config: dict[str, JSONValue] = {"planner_config": planner_config} + warmup = _retune_sweep_warmup(spec, planner_config) + if warmup is not None: + hook_config["warmup_observations"] = cast(JSONValue, warmup[1]) + section = { + "base": deepcopy(spec.base), + "points": [deepcopy(dict(point))], + "warmup": {} if warmup is None else {warmup[0]: deepcopy(warmup[1])}, + } + return AdapterReplaySpec( + config={RETUNE_SWEEP: cast(JSONValue, section)}, + runtime_hooks=(replace(_HOOK, config=hook_config),), + ) + + _LEGACY_PRESET_REMOVAL_NOTE = ( "Legacy flat Dynamo Planner Sweeper preset fields are deprecated and will be " "removed after the 1.5 release. Nest choices under each sub-item's 'preset' field." @@ -669,6 +845,8 @@ def compile_prediction( config: Mapping[str, JSONValue], context: PredictionAdapterContext, ) -> AdapterReplaySpec: + if RETUNE_SWEEP in config: + return self._compile_retune_sweep_point(config, context) unknown = ( set(config) - PLANNER_PUBLIC_TO_CONFIG.keys() @@ -722,6 +900,8 @@ def compile_recommendation( config: Mapping[str, JSONValue], context: RecommendationAdapterContext, ) -> AdapterSearchPlan: + if RETUNE_SWEEP in config: + return self._compile_retune_sweep(config, context) if _RETUNE in config: raise ValueError( "planner.retune is not supported by aisimulate recommend: each " @@ -804,12 +984,87 @@ def compile_recommendation( state=state, ) + def _compile_retune_sweep( + self, + config: Mapping[str, JSONValue], + context: RecommendationAdapterContext, + ) -> AdapterSearchPlan: + """A live retune sweep: one choice per candidate point. + + Every point is validated on the base before any candidate runs, so a + Request the live Planner built wrong fails as a whole instead of + publishing no candidates. + """ + + extra = sorted(set(config) - {RETUNE_SWEEP}) + if extra: + raise ValueError(f"planner.{RETUNE_SWEEP} cannot be combined with {extra}") + spec = RetuneSweepSpec.model_validate(config[RETUNE_SWEEP]) + for point in spec.points: + try: + _retune_sweep_planner_config(spec, point, None) + except ValueError as error: + raise ValueError( + f"planner.{RETUNE_SWEEP} point {point} is invalid: {error}" + ) from None + points = cast(list[JSONValue], deepcopy(spec.points)) + return AdapterSearchPlan( + fragment=SearchSpaceFragment( + choices_by_branch={ + mode: {RETUNE_POINT: list(points)} + for mode in _deployment_modes(context.sweep.core_search_space) + } + ), + state={RETUNE_SWEEP: spec.model_dump(mode="json")}, + potential_runtime_hooks=(_HOOK,), + ) + + @staticmethod + def _compile_retune_sweep_point( + config: Mapping[str, JSONValue], + context: PredictionAdapterContext, + ) -> AdapterReplaySpec: + """One live retune sweep candidate (``retune_sweep`` with one point), + as a recommendation emits it.""" + + extra = sorted(set(config) - {RETUNE_SWEEP}) + if extra: + raise ValueError(f"planner.{RETUNE_SWEEP} cannot be combined with {extra}") + spec = RetuneSweepSpec.model_validate(config[RETUNE_SWEEP]) + if len(spec.points) != 1: + raise ValueError( + f"aisimulate predict replays one planner.{RETUNE_SWEEP} point, got " + f"{len(spec.points)}" + ) + return _retune_sweep_replay( + spec, spec.points[0], _prediction_sample(context.engine) + ) + + @staticmethod + def _materialize_retune_sweep( + raw: JSONValue, + selection: Mapping[str, JSONValue], + context: CandidateContext, + ) -> AdapterReplaySpec: + spec = RetuneSweepSpec.model_validate(raw) + point = selection.get(RETUNE_POINT) + if not isinstance(point, dict): + raise TypeError(f"Planner {RETUNE_POINT} selection must be a mapping") + try: + return _retune_sweep_replay(spec, point, context.sample) + except ValueError as error: + raise InfeasibleCandidate(f"Planner candidate: {error}") from error + def materialize_candidate( self, plan: AdapterSearchPlan, selection: Mapping[str, JSONValue], context: CandidateContext, ) -> AdapterReplaySpec: + if isinstance(plan.state, dict) and RETUNE_SWEEP in plan.state: + return self._materialize_retune_sweep( + plan.state[RETUNE_SWEEP], selection, context + ) if isinstance(plan.state, dict) and plan.state.get("forced_disabled") is True: return AdapterReplaySpec(config={"policy": "disabled"}) materialized = dict(selection) diff --git a/components/src/dynamo/planner/tests/live_retune_fakes.py b/components/src/dynamo/planner/tests/live_retune_fakes.py index cf6d4e4a1024..16a8b9247da6 100644 --- a/components/src/dynamo/planner/tests/live_retune_fakes.py +++ b/components/src/dynamo/planner/tests/live_retune_fakes.py @@ -290,6 +290,32 @@ def router_points(effective_input: Mapping[str, Any]) -> list[dict[str, float]]: ] +def planner_points( + effective_input: Mapping[str, Any] +) -> list[Optional[dict[str, Any]]]: + """The planner points of an input's ``planner.retune_sweep`` section; + ``[None]`` without one (fixed replicas).""" + + section = nested_get(effective_input, "planner", "retune_sweep") + if not isinstance(section, Mapping): + return [None] + return [dict(point) for point in section["points"]] + + +def planner_metrics( + point_public: Mapping[str, float], planner: Optional[Mapping[str, Any]] +) -> dict[str, float]: + """``default_metrics`` with a planner point's load sensitivity added to + its goodput (for tests: a higher sensitivity scores better).""" + + metrics = default_metrics(point_public) + if planner: + metrics["goodput_output_throughput_tok_s"] += float( + planner.get("load_scaling_down_sensitivity", 0) + ) + return metrics + + def engine_parameters(effective_input: Mapping[str, Any]) -> dict[str, Any]: """The non-router parameters every Candidate of a pinned engine shares.""" @@ -309,23 +335,34 @@ def engine_parameters(effective_input: Mapping[str, Any]) -> dict[str, Any]: def candidate_parameters( - effective_input: Mapping[str, Any], point_public: Mapping[str, float] + effective_input: Mapping[str, Any], + point_public: Mapping[str, float], + planner: Optional[Mapping[str, Any]] = None, ) -> dict[str, Any]: - """``Candidate.spec.parameters`` of one evaluated router point.""" + """``Candidate.spec.parameters`` of one evaluated router (and planner) + point.""" router = effective_input.get("router") or {} - return { - **engine_parameters(effective_input), - "adapters": { - ROUTER_ADAPTER: { - "policy": router.get("policy", "kv_router"), - "prefill_load_model": dict( - router.get("prefill_load_model") or {"type": "none"} - ), - **{name: float(value) for name, value in point_public.items()}, - } - }, + adapters: dict[str, Any] = { + ROUTER_ADAPTER: { + "policy": router.get("policy", "kv_router"), + "prefill_load_model": dict( + router.get("prefill_load_model") or {"type": "none"} + ), + **{name: float(value) for name, value in point_public.items()}, + } } + if planner is not None: + # The adapter's concrete config: the section narrowed to the point. + section = nested_get(effective_input, "planner", "retune_sweep") + adapters["dynamo.planner"] = { + "retune_sweep": { + "base": dict(section["base"]), + "points": [dict(planner)], + "warmup": {}, + } + } + return {**engine_parameters(effective_input), "adapters": adapters} def default_metrics(point_public: Mapping[str, float]) -> dict[str, float]: @@ -1072,6 +1109,10 @@ def __init__( self.evaluate: Evaluator = evaluate or ( lambda _request, point: default_metrics(point) ) + # Scenario 2: metrics of a router point crossed with a planner point. + self.evaluate_planner: Callable[ + ..., Optional[dict[str, float]] + ] = lambda _request, point, planner: planner_metrics(point, planner) self.runtime_image = runtime_image self.dynamo_version = dynamo_version self.mutate_input = mutate_input @@ -1130,10 +1171,18 @@ def step(self) -> int: effective = run["status"]["aisim"]["input"] request = self.kube.requests[name] published = [] - for point in router_points(effective): - metrics = self.evaluate(copy.deepcopy(request), point) + for point, planner in itertools.product( + router_points(effective), planner_points(effective) + ): + metrics = ( + self.evaluate(copy.deepcopy(request), point) + if planner is None + else self.evaluate_planner( + copy.deepcopy(request), point, planner + ) + ) if metrics is not None: - parameters = candidate_parameters(effective, point) + parameters = candidate_parameters(effective, point, planner) if self.mutate_parameters is not None: self.mutate_parameters(name, parameters) published.append((parameters, metrics)) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py index ef2cbddf43d4..8527434e55c0 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py @@ -6,7 +6,7 @@ """Configuration premises of live retune (``retune.executor: dgdr``). P1 (Kubernetes, decision latency, explicit timeout within the interval), P2 -(retune-only, router only), P3 (the DGDR router knobs), P5 (a pinned engine +(a router domain; scaling and a planner domain are Scenario 2), P3 (the DGDR router knobs), P5 (a pinned engine section) and P13 (the largest grid shard fits the sweep deadline), plus the shard-split and need-seconds helpers the executor shares with validation. """ @@ -225,37 +225,56 @@ def test_p1_rejects_a_timeout_within_ten_minutes_of_the_interval(): # --------------------------------------------------------------------- P2 +# Two credits, the rest pinned: a 2-point router grid. +PINNED_ROUTER = { + "overlap_score_credit": {"choices": [0.5, 1.0]}, + "prefill_load_scale": 1.0, + "temperature": 0.0, +} +LOAD_ONLY = {"enable_load_scaling": True, "enable_throughput_scaling": False} +SLA_THROUGHPUT = { + "enable_load_scaling": False, + "enable_throughput_scaling": True, + "optimization_target": "sla", +} + + @pytest.mark.parametrize( - "scaling", + ("scaling", "planner"), [ - {"enable_load_scaling": True, "enable_throughput_scaling": False}, - { - "enable_load_scaling": False, - "enable_throughput_scaling": True, - "optimization_target": "sla", - }, + (LOAD_ONLY, {"load_sensitivity": {"preset": ["default", "conservative"]}}), + (SLA_THROUGHPUT, "default"), + # Easy mode keeps load scaling on without the flags written. + ({"enable_throughput_scaling": False}, None), + ( + {**SLA_THROUGHPUT, "enable_load_scaling": True}, + {"scaling_policy": {"preset": ["hybrid_180_5", "hybrid_600_5"]}}, + ), ], + ids=["load", "sla-throughput", "easy-mode", "scaling-policy"], ) -def test_p2_requires_retune_only(scaling): - _rejects(r"retune-only mode \(P2\)", **scaling) - - -def test_p2_requires_both_scaling_flags_explicitly(): - # Without both flags written, easy mode keeps load scaling on. - with pytest.raises(ValidationError, match=r"retune-only mode \(P2\)"): - PlannerConfig( - namespace="test-ns", - environment="kubernetes", - mode="agg", - enable_throughput_scaling=False, - retune=_retune(), - ) +def test_p2_accepts_scaling_and_a_planner_domain(scaling, planner): + # Scenario 2: candidates replay an inner planner from the window start. + retune = _retune(router=PINNED_ROUTER) + if planner is not None: + retune["planner"] = planner + config = PlannerConfig( + namespace="test-ns", + environment="kubernetes", + mode="agg", + retune=retune, + **scaling, + ) + assert config.scaling_enabled() -def test_p2_rejects_a_planner_domain(): +def test_p2_requires_a_router_domain(): + # Every DGDR Request sets the three router knobs; pin them to retune the + # Planner alone. _rejects( - r"retunes the router only \(P2\)", - _retune(planner={"load_sensitivity": {"preset": ["default"]}}), + r"requires retune.router \(P2\)", + _retune(router=None, planner="default"), + **LOAD_ONLY, ) @@ -617,13 +636,58 @@ def test_p13_counts_the_largest_shard_of_fewer_max_shards(): def test_p13_rejects_a_grid_that_needs_more_shards(): _rejects( - r"router grid of 108 points cannot be split into at most " + r"retune grid of 108 points cannot be split into at most " r"retune.dgdr.run.max_shards=1 Cartesian shards of at most 100 points " r"\(P13\); the largest would have 108", _retune(dgdr=_dgdr(run={"max_shards": 1})), ) +def test_p13_shards_the_planner_points_too(): + # The 108 router points x the 3 default sensitivities: 324 points need a + # fourth shard, which cuts the temperature axis. + retune = _retune(planner="default") + _rejects( + r"retune grid of 324 points cannot be split into at most " + r"retune.dgdr.run.max_shards=3 Cartesian shards", + retune, + **LOAD_ONLY, + ) + dgdr = _dgdr(run={"max_shards": 4, "parallelism": 16}) + config = _config(_retune(planner="default", dgdr=dgdr), **LOAD_ONLY) + assert config.retune is not None + + +def test_p13_budgets_the_history_read_of_a_scaling_planner(): + dgdr = _dgdr( + cut_budget_seconds=60, + job_startup_seconds=0, + run={"parallelism": 36, "candidate_timeout_seconds": 100}, + ) + retune = _retune(router=PINNED_ROUTER, planner=None, dgdr=dgdr) + # Two one-point shards: cut 60 + history 60 + 0 + 1 round x 100 = 220. + assert _config({**retune, "timeout_seconds": 220}, **LOAD_ONLY).retune is not None + _rejects( + r"needs 220s: cut_budget_seconds 60 for the history \+ cut_budget_seconds 60", + {**retune, "timeout_seconds": 219}, + **LOAD_ONLY, + ) + # Fixed replicas read no history. + assert _config({**retune, "timeout_seconds": 160}).retune is not None + + +def test_p13_bounds_the_warmup_a_request_carries(): + # 180-s buckets: 1024 fit, 1025 do not. + retune = _retune(router=PINNED_ROUTER, planner=None) + accepted = {**retune, "history_seconds": 180.0 * 1024} + assert _config(accepted, **SLA_THROUGHPUT).retune is not None + _rejects( + r"is 1025 warmup buckets, more than the 1024 a DGDR Request carries \(P13\)", + {**retune, "history_seconds": 180.0 * 1025}, + **SLA_THROUGHPUT, + ) + + def test_p13_uses_the_configured_run_parallelism_and_timeouts(): dgdr = _dgdr( cut_budget_seconds=60, diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py index a0f52a33e496..ed0497ed1b78 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py @@ -571,6 +571,64 @@ async def test_window_started_before_the_controller_launches_unverified(): assert request.replicas == SPEC +def _scaling_config(): + # Scenario 2: load scaling (easy mode) with the default sensitivities. + return PlannerConfig( + namespace="test-ns", + mode="agg", + enable_throughput_scaling=False, + retune={ + "interval_seconds": INTERVAL, + "grace_seconds": GRACE, + "router": ROUTER, + "planner": "default", + }, + ) + + +@pytest.mark.parametrize( + ("start", "replicas", "verified"), + [ + # Ready counts recorded from before the window: its start's count. + (T0 - INTERVAL - 2 * STEP, {"aggregated": 3}, True), + # The window began before the controller: the spec stands in. + (T0 - INTERVAL / 2, SPEC, False), + ], + ids=["recorded", "unrecorded"], +) +async def test_a_scaling_planner_sweeps_from_the_ready_counts_at_window_start( + start, replicas, verified +): + # The planner scales within the window: 3 workers at its start, 1 in + # the middle, 2 (the spec) at its end. That is not replicas_changed. + sweeps = FakeSweeps() + sweeps.default = _flat() + + def counts(now): + if now < T0 - INTERVAL / 2: + return 3 + return 1 if now < T0 - STEP else 2 + + controller = _live(sweeps, start=start, config=_scaling_config()) + try: + decisions = await _run_window(controller, counts, start=start, until=L_K) + finally: + controller.close() + + assert decisions == [] + (request,) = _requests_for_k(sweeps) + assert request.replicas == replicas + assert request.replicas_verified is verified + # The executor starts each inner planner from SweepRequest.replicas. + assert request.spec_replicas is None + assert request.base_planner is not None + assert {c.planner["load_scaling_down_sensitivity"] for c in request.candidates} == { + 70, + 80, + 90, + } + + # --------------------------------------------------------------------------- # restore_desired # --------------------------------------------------------------------------- diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py index 0759fdb4858f..8be7986100df 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py @@ -2116,3 +2116,287 @@ def real_cut(*args, deadline_s, **kwargs): _, claim_path = trace_pvc_source(h.created()[0]["spec"]) assert tmp_path / "trace" / claim_path == window_file assert_scores(result, request) + + +# ------------------------------------------------------------- Scenario 2 + +LSD = "load_scaling_down_sensitivity" +THROUGHPUT_INTERVAL = "throughput_adjustment_interval_seconds" + + +class InlinePool: + """``sweep.WorkerPool`` that runs each task in this process.""" + + def __init__(self, *args: Any) -> None: + self.closed = False + + def run(self, tasks): + results = {} + for task_id, function, args in tasks: + try: + results[task_id] = (True, function(*args)) + except Exception as error: + results[task_id] = (False, f"{type(error).__name__}: {error}") + return results + + def close(self) -> None: + self.closed = True + + +class FakeHistory: + """``window_cut.bucket_history``: records calls, returns 4 buckets per + interval.""" + + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + + def __call__( + self, + segments_root, + history_start_ms, + window_start_ms, + window_end_ms, + as_of_ms, + intervals_s, + *, + deadline_s, + cancel=None, + ): + from dynamo.planner.core.types import TrafficObservation + from dynamo.planner.retune.live.window_cut import TraceHistory + + self.calls.append( + { + "segments_root": segments_root, + "history_start_ms": history_start_ms, + "window": (window_start_ms, window_end_ms, as_of_ms), + "intervals": list(intervals_s), + } + ) + + def rows(interval: int, count: int): + return tuple( + TrafficObservation(float(interval), 10.0 + i, 100.0, 20.0) + for i in range(count) + ) + + return TraceHistory( + observations={interval: rows(interval, 6) for interval in intervals_s}, + warmups={interval: rows(interval, 4) for interval in intervals_s}, + history_requests=40, + dropped_members=0, + ) + + +def _base_planner(**overrides: Any) -> dict[str, Any]: + from dynamo.planner.config.planner_config import PlannerConfig + + values: dict[str, Any] = { + "namespace": "test-ns", + "mode": "agg", + "enable_load_scaling": True, + "enable_throughput_scaling": False, + "metric_pulling_prometheus_endpoint": "http://prometheus:9090", + **overrides, + } + return PlannerConfig(**values).model_dump(mode="json") + + +def scenario_2_request( + base_planner: dict[str, Any], sensitivities=(70, 90), running=80 +) -> SweepRequest: + """SMALL_DOMAIN x load sensitivities, from the running sensitivity, with + two ready workers at the window start and three in the spec.""" + + request = make_request(spec_replicas={"aggregated": 3}) + baseline = RetuneParams(router=request.baseline.router, planner={LSD: running}) + candidates = { + RetuneParams(router=point, planner={LSD: value}) + for point in grid(SMALL_DOMAIN) + for value in sensitivities + } + candidates.add(baseline) + return dataclasses.replace( + request, + candidates=tuple(sorted(candidates, key=RetuneParams.key)), + baseline=baseline, + replicas={"aggregated": 2}, + base_planner=base_planner, + history_start_ms=request.window_start_ms - 600_000.0, + ) + + +def test_plan_shards_crosses_router_chunks_with_planner_points(): + planners = [{LSD: 70}, {LSD: 80}, {LSD: 90}] + baseline = grid(SMALL_DOMAIN)[-1] + + shards = plan_shards( + SMALL_DOMAIN, + baseline=baseline, + startup=baseline, + max_shards=3, + parallelism=8, + planner_points=planners, + baseline_planner={LSD: 80}, + startup_planner={LSD: 80}, + ) + + grids = [shard for shard in shards if shard.kind == "grid"] + covered = [ + (json.dumps(point, sort_keys=True), planner[LSD]) + for shard in grids + for point in shard.points + for planner in shard.planners + ] + # Every router point with every planner point, once; 24 points in 3 + # shards of 8. + assert sorted(covered) == sorted( + (json.dumps(point, sort_keys=True), value) + for point in grid(SMALL_DOMAIN) + for value in (70, 80, 90) + ) + assert [shard.size for shard in grids] == [8, 8, 8] + (probe,) = [shard for shard in shards if shard.kind == "probe"] + assert probe.planners == ({LSD: 80},) + # The baseline is in the grid: no point shard. + assert [shard.kind for shard in shards].count("point") == 0 + + +def test_scenario_2_window_replays_the_inner_planner_from_the_window_start(tmp_path): + h = Harness(tmp_path) + history = FakeHistory() + request = scenario_2_request(_base_planner()) + + result = h.sweep(request, bucket_history=history, worker_pool=InlinePool) + + assert result.status == "ok", (result.error, result.details) + # Load scaling only: no predictor to warm, so no history is read. + assert history.calls == [] + bodies = h.created() + assert bodies + for body in bodies: + overrides = body["spec"]["aisim"]["inputOverrides"] + section = overrides["planner"]["retune_sweep"] + # The live config's scaling fields only: no connection settings. + assert section["base"]["enable_load_scaling"] is True + assert "metric_pulling_prometheus_endpoint" not in section["base"] + assert "namespace" not in section["base"] + assert section["warmup"] == {} + assert {point[LSD] for point in section["points"]} <= {70, 80, 90} + # Each inner planner starts from the ready counts at the window + # start, not the spec. + preset = overrides["engine"]["workers"]["aggregated"]["parallelism"]["preset"] + assert preset[0]["replicas"] == 2 + for candidate in request.candidates: + metrics = default_metrics(public(candidate.router)) + metrics["goodput_output_throughput_tok_s"] += candidate.planner[LSD] + expected, _ = score_summary(metrics, request.window_s, request.objective) + assert result.scores[candidate.key()] == pytest.approx(expected, rel=1e-12) + assert result.metrics[candidate.key()]["params"] == candidate.to_dict() + assert result.details["planner"]["points"] == 3 + + +def test_scenario_2_throughput_planner_carries_its_history_warmup(tmp_path): + h = Harness(tmp_path) + history = FakeHistory() + base = _base_planner( + optimization_target="sla", + enable_throughput_scaling=True, + enable_load_scaling=True, + ) + request = scenario_2_request(base) + + result = h.sweep(request, bucket_history=history, worker_pool=InlinePool) + + assert result.status == "ok", (result.error, result.details) + (call,) = history.calls + assert call["intervals"] == [base[THROUGHPUT_INTERVAL]] + assert call["history_start_ms"] == int(request.history_start_ms) + assert call["window"] == (WINDOW_START_MS, WINDOW_END_MS, AS_OF_MS) + interval = base[THROUGHPUT_INTERVAL] + for body in h.created(): + warmup = body["spec"]["aisim"]["inputOverrides"]["planner"]["retune_sweep"][ + "warmup" + ] + # Four history buckets, as TrafficObservation rows. + assert warmup == { + str(interval): [ + [float(interval), 10.0 + i, 100.0, 20.0, None, None] for i in range(4) + ] + } + # The decision's swap inputs: history and window at the interval. + assert len(result.observations_at(interval)) == 6 + assert result.predictor_warmup_errors == {} + + +def test_scenario_2_candidate_with_a_foreign_planner_point_is_unmapped(tmp_path): + h = Harness(tmp_path) + + def foreign(_name: str, parameters: dict) -> None: + parameters["adapters"]["dynamo.planner"]["retune_sweep"]["points"] = [{LSD: 55}] + + h.operator.mutate_parameters = foreign + result = h.sweep( + scenario_2_request(_base_planner()), + bucket_history=FakeHistory(), + worker_pool=InlinePool, + ) + + # The baseline is unscored, so the window cannot be decided. + assert result.details["unmapped_candidates"] + assert all(score is None for score in result.scores.values()) + + +def test_scenario_2_prepares_the_predictor_of_a_new_throughput_interval( + tmp_path, monkeypatch +): + # Candidates at 360 s next to the running 180 s: the history is read at + # both intervals, and the 360-s predictor a decision would swap in is + # prepared on that interval's observations. + h = Harness(tmp_path) + history = FakeHistory() + base = _base_planner( + optimization_target="sla", + enable_throughput_scaling=True, + enable_load_scaling=True, + load_predictor="constant", + ) + request = scenario_2_request(base) + slower = { + RetuneParams( + router=candidate.router, + planner={**candidate.planner, THROUGHPUT_INTERVAL: 360}, + ) + for candidate in request.candidates + if candidate != request.baseline + } + request = dataclasses.replace( + request, + candidates=tuple(sorted({*request.candidates, *slower}, key=RetuneParams.key)), + ) + + from dynamo.planner.retune import sweep as sweep_module + + prepared = [] + monkeypatch.setattr( + sweep_module, + "prepare_predictor", + lambda config, _caps, observations: prepared.append( + (config.throughput_adjustment_interval_seconds, len(observations)) + ), + ) + + result = h.sweep(request, bucket_history=history, worker_pool=InlinePool) + + assert result.status == "ok", (result.error, result.details) + assert history.calls[0]["intervals"] == [180, 360] + # Only the new interval needs a predictor, on its 6 observations. + assert prepared == [(360, 6)] + warmup = h.created()[0]["spec"]["aisim"]["inputOverrides"]["planner"][ + "retune_sweep" + ]["warmup"] + assert sorted(warmup) == ["180", "360"] + assert sorted(result.interval_observations) == [180, 360] + assert result.predictor_warmup_errors == {} + scored = [key for key, score in result.scores.items() if score is not None] + assert len(scored) == len(request.candidates) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py index 58988605bdfd..a19d113a0272 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py @@ -217,7 +217,7 @@ def __getattr__(self, name: str): def make( - tmp_path: Path, *, keep_requests: int = 2 + tmp_path: Path, *, keep_requests: int = 2, history_s: float = 0.0 ) -> tuple[Reconciler, FakeKube, RecordingMetrics]: cfg = DgdrExecutorConfig( claim_name="traces", @@ -237,7 +237,14 @@ def make( kube = FakeKube() metrics = RecordingMetrics() reconciler = Reconciler( - kube, DGD, cfg, retune, str(tmp_path), metrics, now=lambda: NOW + kube, + DGD, + cfg, + retune, + str(tmp_path), + metrics, + now=lambda: NOW, + history_s=history_s, ) return reconciler, kube, metrics @@ -300,6 +307,21 @@ def test_segments_of_a_live_pod_keep_the_newest_and_recent_ones(tmp_path: Path) assert report.errors == {} +def test_a_scaling_planner_keeps_the_segments_of_its_history(tmp_path: Path) -> None: + # Sweeps also read history_s before the window (the inner planners' + # predictor warmup), so segments live that much longer. + reconciler, _, _ = make(tmp_path, history_s=3600.0) + pod_dir = Path(reconciler.segments_root) / "fe-0" + history = touch(pod_dir / "req.000000.jsonl.gz", SEGMENT_AGE + 1800) + older = touch(pod_dir / "req.000001.jsonl.gz", SEGMENT_AGE + 3601) + touch(pod_dir / "req.000002.jsonl.gz", 10) + + report = run_once(reconciler, pods=[frontend("fe-0")]) + + assert report.deleted_segments == 1 + assert history.exists() and not older.exists() + + def test_directory_of_a_gone_pod_goes_only_when_everything_is_old( tmp_path: Path, ) -> None: diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py index e9864dddfd15..8326d3289a4c 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py @@ -579,6 +579,8 @@ def __init__( retune_inputs=inputs, retune_hooks=hooks, ) + # The Planner's live configuration, as NativePlannerBase.config. + self.config = self._engine._config if self.controller is not None: # What the adapter's initial_tick does. self.controller.start(world.clock.time(), hooks.phase_s) @@ -1311,6 +1313,34 @@ async def test_restart_restores_the_persisted_point() -> None: assert second.sup.target().desired_by_wt == by_wt("a", chosen) +async def test_planner_changes_are_recorded_but_not_restored_after_a_restart( + caplog, +) -> None: + world = World() + first = await leading(world) + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + first.controller.restore_desired(chosen) + decision = dataclasses.replace( + make_decision(K0 + 1, "applied", chosen=chosen), + planner_changes={"load_scaling_down_sensitivity": 70}, + ) + enqueue(first, decision) + await first.sup.outbox_round() + assert world.shared_state().history[-1]["planner"] == { + "load_scaling_down_sensitivity": 70 + } + await first.sup.stop() + + with caplog.at_level(logging.WARNING, logger="dynamo.planner.retune.live"): + second = await leading(world, "p2") + + # The router point is restored; the Planner field runs its configured + # value, and the restart says so. + assert second.controller.current_point().router == chosen + assert "load_scaling_down_sensitivity': 70} of decision" in caplog.text + assert "are not restored after the restart" in caplog.text + + async def test_fingerprint_change_resets_with_a_two_phase_rollback() -> None: world = World() first = await leading(world) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py index c686a1b42be9..66536f031444 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py @@ -1242,3 +1242,57 @@ def test_interrupt_kills_child_and_removes_its_temp_file(tmp_path, interrupt): assert caught.value.reason == expected assert time.monotonic() - started < 20 assert not temp.exists() + + +# ---------------------------------------------------------------- history + + +def test_history_is_bucketed_at_each_interval_from_the_segments(tmp_path): + # One row a second from 0 to 15 s, plus one in the window that completes + # after as_of; a damaged member between two good ones is skipped. + root = tmp_path / "segments" + rows = [_envelope(_row(f"r{i}", 1000 * i)) for i in range(15)] + censored = _envelope(_row("late", 14_900, total_ms=1000.0)) + _write( + root / "pod-a" / "req.000001.jsonl.gz", + _member(rows[:8]), + b"\x1f\x8b\x08 not a member", + _member([*rows[8:], censored]), + ) + + history = window_cut.bucket_history_inprocess( + str(root), 1000, START, END, AS_OF, [2, 5] + ) + + # 1..4 s of history; the window's rows count only toward observations. + assert history.history_requests == 4 + assert history.dropped_members == 1 + num_req = { + interval: [obs.num_req for obs in rows] + for interval, rows in history.observations.items() + } + assert num_req == {2: [2.0] * 7, 5: [4.0, 5.0, 5.0]} + warmups = { + interval: [obs.num_req for obs in rows] + for interval, rows in history.warmups.items() + } + # Buckets end at the window start: no window request is warmup. + assert warmups == {2: [2.0, 2.0], 5: [4.0]} + assert {obs.duration_s for obs in history.observations[5]} == {5.0} + + +def test_history_runs_in_a_child_that_the_deadline_stops(tmp_path): + root = tmp_path / "segments" + _write( + root / "pod-a" / "req.000001.jsonl.gz", _member([_envelope(_row("r", 6000))]) + ) + + history = window_cut.bucket_history( + str(root), START, START, END, AS_OF, [5], deadline_s=time.time() + 60 + ) + assert [obs.num_req for obs in history.observations[5]] == [1.0, 0.0] + with pytest.raises(WindowCutError) as caught: + window_cut.bucket_history( + str(root), START, START, END, AS_OF, [5], deadline_s=time.time() - 1 + ) + assert caught.value.reason == "window_cut_timeout" diff --git a/components/src/dynamo/planner/tests/unit/test_planner_provider_retune.py b/components/src/dynamo/planner/tests/unit/test_planner_provider_retune.py index 0ca6983fb3d0..24b6bfc063c5 100644 --- a/components/src/dynamo/planner/tests/unit/test_planner_provider_retune.py +++ b/components/src/dynamo/planner/tests/unit/test_planner_provider_retune.py @@ -261,3 +261,131 @@ def test_prediction_only_fields_need_an_enabled_policy_and_predict() -> None: for field in ({"retune": _RETUNE}, {"advisory": False}): with pytest.raises(ValueError, match="not supported by aisimulate recommend"): provider.compile_recommendation({"policy": "enabled", **field}, context) + + +def _recommendation_context() -> RecommendationAdapterContext: + return RecommendationAdapterContext( + engine={}, + traffic={}, + evaluation={}, + optimization={}, + sweep=SweepContext( + core_search_space={"deployment_mode": ["agg"]}, + workload={"trace_path": None, "isl": 512, "osl": 128}, + goal={"target": "goodput_per_gpu"}, + show_progress=False, + ), + ) + + +def _live_base(**overrides: Any) -> dict[str, Any]: + from dynamo.planner.simulation.config import retune_sweep_base + + config = PlannerConfig( + namespace="test-ns", + mode="agg", + optimization_target="sla", + enable_load_scaling=True, + enable_throughput_scaling=True, + ttft_ms=60.0, + itl_ms=100.0, + metric_pulling_prometheus_endpoint="http://prometheus:9090", + **overrides, + ) + return retune_sweep_base(config.model_dump(mode="json")) + + +def test_retune_sweep_replays_the_live_base_with_each_point() -> None: + from aisimulate.sweeper.provider import CandidateContext + from aisimulate.sweeper.replay import BackendDeploymentSpec + + provider = create_provider() + base = _live_base() + points = [ + {"load_scaling_down_sensitivity": 70}, + { + "load_scaling_down_sensitivity": 90, + "throughput_adjustment_interval_seconds": 360, + }, + ] + warmup = { + "180": [[180.0, 5.0, 100.0, 20.0, None, None]], + "360": [[360.0, 10.0, 100.0, 20.0, None, None]], + } + plan = provider.compile_recommendation( + {"retune_sweep": {"base": base, "points": points, "warmup": warmup}}, + _recommendation_context(), + ) + + assert plan.fragment.choices_by_branch == {"agg": {"retune_point": points}} + context = CandidateContext( + sample={"deployment_mode": "agg", "tp": 2, "attention_dp": 1}, + backend_deployment=BackendDeploymentSpec( + deployment_mode="agg", + backend="vllm", + backend_version="0.11.0", + agg_engine_args={"engine_type": "vllm"}, + num_workers=2, + ), + ) + spec = provider.materialize_candidate(plan, {"retune_point": points[1]}, context) + + # Candidate parameters are the section narrowed to the point and its + # own warmup; the hook carries the inner planner. + assert spec.config == { + "retune_sweep": { + "base": base, + "points": [points[1]], + "warmup": {"360": warmup["360"]}, + } + } + (hook,) = spec.runtime_hooks + planner = hook.config["planner_config"] + assert planner["load_scaling_down_sensitivity"] == 90 + assert planner["throughput_adjustment_interval_seconds"] == 360 + assert planner["advisory"] is False and planner.get("retune") is None + assert planner["decode_engine_num_gpu"] == 2 + assert "metric_pulling_prometheus_endpoint" not in planner + # The warmup of the point's own throughput interval. + assert hook.config["warmup_observations"] == warmup["360"] + config = PlannerConfig.from_config_arg(json.dumps(planner)) + assert config.ttft_ms == 60.0 and config.scaling_enabled() + # aisimulate predict replays the candidate from its config alone. + again = provider.compile_prediction(spec.config, _context(sla=_SLA)) + assert again.config == spec.config + assert again.runtime_hooks[0].config["warmup_observations"] == warmup["360"] + assert again.runtime_hooks[0].config["planner_config"]["decode_engine_num_gpu"] == 1 + with pytest.raises(ValueError, match="replays one planner.retune_sweep point"): + provider.compile_prediction( + {"retune_sweep": {"base": base, "points": points}}, _context(sla=_SLA) + ) + + +@pytest.mark.parametrize( + ("section", "match"), + [ + ({"base": {"namespace": "x"}, "points": [{}]}, r"cannot set \['namespace'\]"), + ( + {"base": {}, "points": [{"max_gpu_budget": 4}]}, + "sets \\['max_gpu_budget'\\]", + ), + ({"base": {}, "points": []}, "at least 1 item"), + ( + { + "base": { + "optimization_target": "sla", + "enable_load_scaling": False, + "enable_throughput_scaling": False, + }, + "points": [{}], + }, + "At least one scaling mode must be enabled", + ), + ], + ids=["base-field", "point-field", "no-points", "no-scaling"], +) +def test_retune_sweep_rejects_what_a_live_planner_never_sends(section, match) -> None: + with pytest.raises(ValueError, match=match): + create_provider().compile_recommendation( + {"retune_sweep": section}, _recommendation_context() + ) diff --git a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py index 7c760d4f5f2a..dcfc51dcdabf 100644 --- a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py +++ b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio +import dataclasses import gzip import json import threading @@ -845,3 +846,79 @@ async def sleep(seconds): with pytest.raises(asyncio.CancelledError): await planner.run() assert sleeps == [1.0, 2.0] + + +def _applied(changes, warmup=None): + from dynamo.planner.core.types import RetuneDecision, RetuneParams + + return RetuneDecision( + decision_id="retune-w1", + outcome="applied", + window_index=0, + window_start_s=0.0, + window_end_s=60.0, + as_of_s=65.0, + launched_at_s=65.0, + decided_at_s=70.0, + num_requests=10, + num_candidates=2, + previous=RetuneParams(), + chosen=RetuneParams(planner=dict(changes)), + baseline_score=1.0, + best_score=2.0, + planner_changes=dict(changes), + predictor_warmup=warmup, + ) + + +async def _apply_decisions(planner, decisions): + planner._schedule_retune_planner_changes(decisions) + await asyncio.gather(*planner._retune_apply_tasks) + + +async def test_live_retune_applies_planner_changes_through_runtime_reconfiguration(): + planner = sla_planner(enable_load_scaling=True) + engine = planner._ensure_engine() + advisory = dataclasses.replace( + _applied({"load_scaling_down_sensitivity": 70}), outcome="advisory" + ) + + await _apply_decisions( + planner, [advisory, _applied({"load_scaling_down_sensitivity": 90})] + ) + + # The advisory decision changes nothing; the applied one is a new revision. + assert planner.config.load_scaling_down_sensitivity == 90 + assert planner._config_generation == 1 + assert "load_propose" in engine._builtins + await planner._shutdown_runtime() + + +async def test_live_retune_keeps_the_config_when_a_planner_change_cannot_apply(): + planner = _planner( + "disagg", + optimization_target="sla", + load_predictor="kalman", + kalman_min_points=20, + ) + engine = planner._ensure_engine() + discarded = [] + + class Controller: + def discard_planner_changes(self, changes): + discarded.append(dict(changes)) + + engine._retune = Controller() + # A new throughput interval needs a warmed predictor; one bucket is too + # few for a Kalman filter that needs 20. + changes = {"throughput_adjustment_interval_seconds": 360} + warmup = (TrafficObservation(360.0, 10.0, 100.0, 10.0),) + + await _apply_decisions(planner, [_applied(changes, warmup)]) + + assert planner.config.throughput_adjustment_interval_seconds == 180 + assert planner._config_generation == 0 + # The controller's current point is the running configuration again. + assert discarded == [changes] + engine._retune = None + await planner._shutdown_runtime() diff --git a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py index 8263e7167ee4..9eb7c58c63c2 100644 --- a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py +++ b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py @@ -575,7 +575,7 @@ def evaluator(*args): def run_with_winner(fields): presweep = {"winner": "w", "fields": fields, "losses": {"w": 0.1}} - monkeypatch.setattr(sweep_module, "_run_presweeps", lambda *_: presweep) + monkeypatch.setattr(sweep_module, "run_presweeps", lambda *_: presweep) tasks.clear() return sweep_module.run_sweep(request, evaluator=evaluator) @@ -792,7 +792,9 @@ def importing_worker(): timer.start() try: with pytest.raises(sweep_module._SweepInterrupted, match="cancelled"): - sweep_module._run_presweeps(request, pool, inputs, [60]) + sweep_module.run_presweeps( + request, pool, inputs.observations, inputs.observations_paths, [60] + ) finally: timer.cancel() pool.close() diff --git a/components/src/dynamo/replay/simulation.py b/components/src/dynamo/replay/simulation.py index 251e033199b4..ce7db7f9e3fd 100644 --- a/components/src/dynamo/replay/simulation.py +++ b/components/src/dynamo/replay/simulation.py @@ -33,6 +33,7 @@ from dynamo.llm import AisPerfConfig, KvRouterConfig from dynamo.mocker.config import normalize_mocker_config +from dynamo.planner.core.types import TrafficObservation from dynamo.replay.api import ( TelemetryOptions, run_synthetic_trace_replay, @@ -145,6 +146,7 @@ def run( ais_perf_config, conversation_affinity, ) = self._resolve_hooks(spec.runtime_hooks) + planner_warmup = self._planner_warmup(spec.runtime_hooks) common: dict[str, Any] = { "router_mode": router_mode, "router_config": router_config, @@ -170,6 +172,12 @@ def run( **self._goodput_sla_kwargs(spec), } + if planner_warmup is not None: + # A live retune sweep's candidate warms its load predictor with + # the recorded history before the window. + if not self._is_trace(spec): + raise ValueError("planner warmup observations require a trace replay") + common["_planner_warmup_observations"] = planner_warmup if conversation_affinity is not None: if not self._is_trace(spec): raise ValueError("conversation_affinity requires an agentic trace") @@ -203,6 +211,27 @@ def close(self) -> None: between candidates. """ + @staticmethod + def _planner_warmup( + hooks: tuple[RuntimeHookSpec, ...], + ) -> list[TrafficObservation] | None: + """The Planner hook's ``warmup_observations`` rows, if it has any.""" + + for hook in hooks: + if not _PLANNER_HOOK.supports(hook): + continue + rows = hook.config.get("warmup_observations") + if rows is None: + return None + if not isinstance(rows, list) or not all( + isinstance(row, list) for row in rows + ): + raise TypeError( + "Dynamo Planner hook warmup_observations must be a list of rows" + ) + return [TrafficObservation(*row) for row in rows] + return None + @staticmethod def _resolve_hooks( hooks: tuple[RuntimeHookSpec, ...], diff --git a/components/src/dynamo/replay/tests/e2e/configs/unified_cli/fixtures/traces/dynamo-retune-window.jsonl b/components/src/dynamo/replay/tests/e2e/configs/unified_cli/fixtures/traces/dynamo-retune-window.jsonl new file mode 100644 index 000000000000..276d67304e8c --- /dev/null +++ b/components/src/dynamo/replay/tests/e2e/configs/unified_cli/fixtures/traces/dynamo-retune-window.jsonl @@ -0,0 +1,60 @@ +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":1010,"request":{"request_id":"window-0","output_tokens":4,"request_received_ms":1000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[0,1]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":2010,"request":{"request_id":"window-1","output_tokens":4,"request_received_ms":2000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[10,11]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":3010,"request":{"request_id":"window-2","output_tokens":4,"request_received_ms":3000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[20,21]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":4010,"request":{"request_id":"window-3","output_tokens":4,"request_received_ms":4000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[30,31]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":5010,"request":{"request_id":"window-4","output_tokens":4,"request_received_ms":5000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[40,41]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":6010,"request":{"request_id":"window-5","output_tokens":4,"request_received_ms":6000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[50,51]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":7010,"request":{"request_id":"window-6","output_tokens":4,"request_received_ms":7000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[60,61]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":8010,"request":{"request_id":"window-7","output_tokens":4,"request_received_ms":8000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[70,71]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":9010,"request":{"request_id":"window-8","output_tokens":4,"request_received_ms":9000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[80,81]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":10010,"request":{"request_id":"window-9","output_tokens":4,"request_received_ms":10000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[90,91]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":11010,"request":{"request_id":"window-10","output_tokens":4,"request_received_ms":11000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[100,101]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":12010,"request":{"request_id":"window-11","output_tokens":4,"request_received_ms":12000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[110,111]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":13010,"request":{"request_id":"window-12","output_tokens":4,"request_received_ms":13000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[120,121]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":14010,"request":{"request_id":"window-13","output_tokens":4,"request_received_ms":14000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[130,131]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":15010,"request":{"request_id":"window-14","output_tokens":4,"request_received_ms":15000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[140,141]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":16010,"request":{"request_id":"window-15","output_tokens":4,"request_received_ms":16000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[150,151]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":17010,"request":{"request_id":"window-16","output_tokens":4,"request_received_ms":17000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[160,161]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":18010,"request":{"request_id":"window-17","output_tokens":4,"request_received_ms":18000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[170,171]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":19010,"request":{"request_id":"window-18","output_tokens":4,"request_received_ms":19000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[180,181]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":20010,"request":{"request_id":"window-19","output_tokens":4,"request_received_ms":20000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[190,191]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":21010,"request":{"request_id":"window-20","output_tokens":4,"request_received_ms":21000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[200,201]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":22010,"request":{"request_id":"window-21","output_tokens":4,"request_received_ms":22000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[210,211]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":23010,"request":{"request_id":"window-22","output_tokens":4,"request_received_ms":23000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[220,221]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":24010,"request":{"request_id":"window-23","output_tokens":4,"request_received_ms":24000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[230,231]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":25010,"request":{"request_id":"window-24","output_tokens":4,"request_received_ms":25000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[240,241]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":26010,"request":{"request_id":"window-25","output_tokens":4,"request_received_ms":26000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[250,251]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":27010,"request":{"request_id":"window-26","output_tokens":4,"request_received_ms":27000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[260,261]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":28010,"request":{"request_id":"window-27","output_tokens":4,"request_received_ms":28000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[270,271]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":29010,"request":{"request_id":"window-28","output_tokens":4,"request_received_ms":29000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[280,281]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":30010,"request":{"request_id":"window-29","output_tokens":4,"request_received_ms":30000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[290,291]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":31010,"request":{"request_id":"window-30","output_tokens":4,"request_received_ms":31000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[300,301]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":32010,"request":{"request_id":"window-31","output_tokens":4,"request_received_ms":32000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[310,311]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":33010,"request":{"request_id":"window-32","output_tokens":4,"request_received_ms":33000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[320,321]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":34010,"request":{"request_id":"window-33","output_tokens":4,"request_received_ms":34000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[330,331]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":35010,"request":{"request_id":"window-34","output_tokens":4,"request_received_ms":35000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[340,341]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":36010,"request":{"request_id":"window-35","output_tokens":4,"request_received_ms":36000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[350,351]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":37010,"request":{"request_id":"window-36","output_tokens":4,"request_received_ms":37000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[360,361]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":38010,"request":{"request_id":"window-37","output_tokens":4,"request_received_ms":38000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[370,371]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":39010,"request":{"request_id":"window-38","output_tokens":4,"request_received_ms":39000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[380,381]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":40010,"request":{"request_id":"window-39","output_tokens":4,"request_received_ms":40000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[390,391]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":41010,"request":{"request_id":"window-40","output_tokens":4,"request_received_ms":41000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[400,401]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":42010,"request":{"request_id":"window-41","output_tokens":4,"request_received_ms":42000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[410,411]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":43010,"request":{"request_id":"window-42","output_tokens":4,"request_received_ms":43000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[420,421]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":44010,"request":{"request_id":"window-43","output_tokens":4,"request_received_ms":44000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[430,431]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":45010,"request":{"request_id":"window-44","output_tokens":4,"request_received_ms":45000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[440,441]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":46010,"request":{"request_id":"window-45","output_tokens":4,"request_received_ms":46000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[450,451]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":47010,"request":{"request_id":"window-46","output_tokens":4,"request_received_ms":47000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[460,461]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":48010,"request":{"request_id":"window-47","output_tokens":4,"request_received_ms":48000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[470,471]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":49010,"request":{"request_id":"window-48","output_tokens":4,"request_received_ms":49000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[480,481]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":50010,"request":{"request_id":"window-49","output_tokens":4,"request_received_ms":50000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[490,491]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":51010,"request":{"request_id":"window-50","output_tokens":4,"request_received_ms":51000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[500,501]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":52010,"request":{"request_id":"window-51","output_tokens":4,"request_received_ms":52000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[510,511]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":53010,"request":{"request_id":"window-52","output_tokens":4,"request_received_ms":53000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[520,521]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":54010,"request":{"request_id":"window-53","output_tokens":4,"request_received_ms":54000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[530,531]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":55010,"request":{"request_id":"window-54","output_tokens":4,"request_received_ms":55000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[540,541]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":56010,"request":{"request_id":"window-55","output_tokens":4,"request_received_ms":56000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[550,551]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":57010,"request":{"request_id":"window-56","output_tokens":4,"request_received_ms":57000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[560,561]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":58010,"request":{"request_id":"window-57","output_tokens":4,"request_received_ms":58000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[570,571]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":59010,"request":{"request_id":"window-58","output_tokens":4,"request_received_ms":59000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[580,581]}}} +{"schema":"dynamo.request.trace.v1","event_type":"request_end","event_time_unix_ms":60010,"request":{"request_id":"window-59","output_tokens":4,"request_received_ms":60000,"total_time_ms":10,"replay":{"trace_block_size":64,"input_length":128,"input_sequence_hashes":[590,591]}}} diff --git a/components/src/dynamo/replay/tests/e2e/configs/unified_cli/recommend/dynamo/07-live-retune-sweep-goodput.yaml b/components/src/dynamo/replay/tests/e2e/configs/unified_cli/recommend/dynamo/07-live-retune-sweep-goodput.yaml new file mode 100644 index 000000000000..4ceebda5af1b --- /dev/null +++ b/components/src/dynamo/replay/tests/e2e/configs/unified_cli/recommend/dynamo/07-live-retune-sweep-goodput.yaml @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# A live retune sweep (retune.executor: dgdr) as its DGDR Request lowers it: +# the window trace, two router temperatures, and the Planner's configuration +# with two load sensitivities. Each candidate's inner planner scales from the +# window start, warmed with the 20-s history buckets. +traffic: + source: + type: trace + format: dynamo + paths: [components/src/dynamo/replay/tests/e2e/configs/unified_cli/fixtures/traces/dynamo-retune-window.jsonl] + load: {type: trace_timestamps} +engine: + mode: aggregated + model: components/src/dynamo/replay/tests/e2e/configs/unified_cli/fixtures/tiny-model + hardware: h200_sxm + backend: vllm + context_length: 2048 + workers: + aggregated: + parallelism: + preset: + - {replicas: 2, tensor: 1, pipeline: 1, attention_data: 1, moe_tensor: 1, moe_expert: 1} + scheduler: {max_batched_tokens: 8192, max_sequences: 256} + kv_cache: {block_size: 64, prefix_caching: true, capacity: {type: fixed, blocks: 256}} + timing: {type: fixed, prefill_ms: 1, decode_ms: 1} +router: + policy: kv_router + prefill_load_model: {type: none} + overlap_score_credit: 1.0 + prefill_load_scale: 1.0 + temperature: {choices: [0.0, 0.5]} +planner: + retune_sweep: + base: + mode: agg + optimization_target: sla + ttft_ms: 10.0 + itl_ms: 5.0 + enable_throughput_scaling: true + enable_load_scaling: true + throughput_adjustment_interval_seconds: 20 + load_adjustment_interval_seconds: 5 + max_gpu_budget: 4 + min_endpoint: 1 + load_predictor: constant + scheduling: {scale_interval_seconds: 5} + points: + - {load_scaling_down_sensitivity: 70} + - {load_scaling_down_sensitivity: 90} + warmup: + "20": + - [20.0, 18.0, 128.0, 4.0, null, null] + - [20.0, 19.0, 128.0, 4.0, null, null] + - [20.0, 20.0, 128.0, 4.0, null, null] + - [20.0, 21.0, 128.0, 4.0, null, null] +evaluation: + sla: {ttft_ms: 10.0, itl_ms: 5.0} +optimization: + target: goodput_per_gpu + constraints: {max_candidate_gpus: 2} + strict_sla: false +optimizer: {algorithm: random, max_trials: 8, parallelism: 1, candidate_timeout_seconds: 60.0, seed: 23} diff --git a/components/src/dynamo/replay/tests/test_simulation.py b/components/src/dynamo/replay/tests/test_simulation.py index 23725992c6d5..b32a5e756029 100644 --- a/components/src/dynamo/replay/tests/test_simulation.py +++ b/components/src/dynamo/replay/tests/test_simulation.py @@ -204,6 +204,47 @@ def fake_run_trace_replay(**kwargs): assert report.metadata["planner_total_ticks"] == 7 +def test_planner_hook_warmup_reaches_the_inner_planner(monkeypatch) -> None: + # A live retune sweep's candidate warms its load predictor with the + # history rows its planner hook carries. + seen = {} + + def fake_run_trace_replay(**kwargs): + seen.update(kwargs) + return _report({"goodput_output_throughput_tok_s": 40.0}, total_ticks=1) + + monkeypatch.setattr(simulation, "run_trace_replay", fake_run_trace_replay) + rows = [[180.0, 12.0, 100.0, 20.0, None, None], [180.0, 9.0, 90.0, 25.0, 0.5, None]] + spec = ReplaySpec( + backend_deployment=_agg_deployment(), + workload={"trace_path": "window.jsonl.gz", "trace_format": "dynamo"}, + goal={"target": "goodput", "sla": {"ttft_ms": 100.0, "itl_ms": 20.0}}, + adapters={ + "dynamo.planner": AdapterReplaySpec( + runtime_hooks=( + RuntimeHookSpec( + provider="dynamo.planner", + kind="scaling_policy", + api_version=1, + config={ + "planner_config": {"mode": "agg"}, + "warmup_observations": rows, + }, + ), + ) + ) + }, + ) + + simulation.DynamoReplayRunnerFactory().create(0).run(spec) + + warmup = seen["_planner_warmup_observations"] + assert [(obs.duration_s, obs.num_req, obs.kv_hit_rate) for obs in warmup] == [ + (180.0, 12.0, None), + (180.0, 9.0, 0.5), + ] + + def test_trace_paths_only_workload_routes_to_trace_replay(monkeypatch) -> None: seen = {} diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md index 4d4318bbfd27..4fbb4fb8b105 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md @@ -296,8 +296,8 @@ In aggregated mode (`mode: agg`), engines handle both prefill and decode via chu > [!WARNING] > **Experimental.** With `retune.executor: local`, the default, retune runs only in > planner-in-the-loop offline replay, and a live Planner rejects it at startup. With -> `executor: dgdr`, a live Planner on Kubernetes retunes three KV router knobs with fixed replicas -> through DGDR v1beta2 Requests; it is for development clusters only. See +> `executor: dgdr`, a live Planner on Kubernetes retunes three KV router knobs, and with scaling +> its own Planner fields, through DGDR v1beta2 Requests; it is for development clusters only. See > [Planner Live Retune](../../../../kubernetes/auto-deployment/planner-live-retune.md). Retune periodically replays the last window of recorded traffic against a grid of router and @@ -562,7 +562,12 @@ trace that ends at or before the first replayed request; without throughput scal `runtime_decision.router_parameters`. The AISimulate Planner adapter (`planner/simulation/provider.py`) passes `planner.retune` through to -`PlannerConfig` and fills the scoring SLA from `evaluation.sla`. `dynamo.replay` (`run_trace_replay`, +`PlannerConfig` and fills the scoring SLA from `evaluation.sla`. For live retune with scaling, it +also reads `planner.retune_sweep` in `aisimulate recommend`: the live Planner's scaling fields +(`base`), the Planner points to search (`points`) and the history warm-up by throughput interval +(`warmup`). Each candidate replays `base` with its point applied, warmed with the rows of its own +interval, which `DynamoReplayRunner` hands to `run_trace_replay`; its concrete config is the +section narrowed to that point, which `aisimulate predict` replays again. `dynamo.replay` (`run_trace_replay`, and `prepare_planner_replay` for synthetic traffic) rejects traffic that is not an open-loop Dynamo request trace, and its error names the AISimulate setting. diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md index 341aa7f03ee5..2f009120c332 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md @@ -331,8 +331,8 @@ spec: > **Experimental.** `retune.executor` selects where retune runs. With `local`, the default, it runs > only in offline simulation: `aisimulate predict --stack dynamo` and > `dynamo.replay.run_trace_replay`, and a live Planner rejects it at startup. With `dgdr`, a live -> Planner on Kubernetes retunes three KV router knobs with fixed replicas through DGDR v1beta2 -> Requests; it is for development clusters only. See +> Planner on Kubernetes retunes three KV router knobs, and with scaling its own Planner fields, +> through DGDR v1beta2 Requests; it is for development clusters only. See > [Planner Live Retune](../../../../kubernetes/auto-deployment/planner-live-retune.md). Retune lets the Planner tune its own runtime parameters from the traffic it served. Every diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md index cdfd95af21d9..5492bb2745ad 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: Planner Live Retune -subtitle: Let the Planner tune the KV router of a running DynamoGraphDeployment from its own recorded traffic, through DGDR v1beta2 search runs. +subtitle: Let the Planner tune the KV router and its own scaling parameters on a running DynamoGraphDeployment from its recorded traffic, through DGDR v1beta2 search runs. --- > [!CAUTION] @@ -14,36 +14,62 @@ subtitle: Let the Planner tune the KV router of a running DynamoGraphDeployment Live retune is the Kubernetes form of [Periodic Retune](../../developer-guide/knowledge-base/modular-components/planner/planner-guide.md#periodic-retune-experimental). Every `retune.interval_seconds`, the Planner cuts the window of request traces that just ended, creates DynamoGraphDeploymentRequests (DGDR) v1beta2 that replay the window for a grid of KV -router parameters, scores the candidates the Runs report on `retune.objective`, and keeps the -current parameters unless a candidate beats them by more than `retune.min_improvement_percentage`. -Outside advisory mode it then sets the winner on every frontend router with -`PATCH /router_policy`. - -Live retune searches the three router knobs a DGDR Request can express: -`overlap_score_credit`, `prefill_load_scale` and `temperature`. Replicas stay fixed: the Planner -runs in retune-only mode and never scales. +router and Planner parameters, scores the candidates the Runs report on `retune.objective`, and +keeps the current parameters unless a candidate beats them by more than +`retune.min_improvement_percentage`. Outside advisory mode it then sets the winning router +parameters on every frontend router with `PATCH /router_policy` and applies the winning Planner +parameters to itself. + +Live retune searches the three router knobs a DGDR Request can express, +`overlap_score_credit`, `prefill_load_scale` and `temperature`, in one of two setups: + +- **Fixed replicas.** The Planner runs in retune-only mode (`enable_load_scaling` and + `enable_throughput_scaling` both `false`) and never scales. Each candidate replays with the + replica counts of the DGD spec. +- **Scaling.** The Planner scales on load, throughput or both, and `retune.planner` may also + search its runtime parameters, the scaling policy included (see + [Search domains](../../reference/components/planner-configuration.mdx#search-domains)). Each + candidate replays the Planner's own configuration with the candidate's fields, starting from + the replica counts that were ready at the window start, as + [replay retune](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#candidates-and-the-sweep) + does. This needs an operator whose configured planner image includes this release: the Runs' + AISimulate Planner adapter must read the Requests' `planner.retune_sweep` section. ## How it works 1. **Record.** Each frontend writes a `request_end` record per request to its own directory on a shared ReadWriteMany claim (`///`). -2. **Cut a window.** At `L_k = window end + grace_seconds`, the Planner merges the records of the +2. **Read the history** (scaling only). At `L_k`, before any Request is created, the Planner + reads the records of the `history_seconds` before the window and buckets them at each + throughput interval a candidate scales at. With throughput scaling it runs the load-predictor + pre-sweep on them and prepares every predictor a decision could swap in. This takes another + `cut_budget_seconds` at most. +3. **Cut a window.** At `L_k = window end + grace_seconds`, the Planner merges the records of the window into one canonical trace file, `//-.jsonl.gz`, in a subprocess bounded by `cut_budget_seconds`. `` is a digest of the window's start, end and cutoff (`window end + grace_seconds`), so a Planner restarted with another `interval_seconds` or `grace_seconds` cuts its own file and creates its own Requests instead of reusing those of a window with the same end. -3. **Search.** The Planner splits the router grid into at most `run.max_shards` Cartesian shards of - at most 100 points and creates one DGDR v1beta2 Request per shard. Each Request replays the - window file on the pinned `retune.dgdr.engine` with the replica counts of the DGD spec. One more - single-point Request replays a grid point again as a common-random-numbers probe: its score - must match the grid's within a relative error of 1e-9. -4. **Decide.** The Planner reads each Run's Candidates, scores them on `retune.objective`, and +4. **Search.** The Planner splits the grid into at most `run.max_shards` Cartesian shards of at + most 100 points and creates one DGDR v1beta2 Request per shard. With scaling, the Planner + points are one more shard dimension. Each Request replays the window file on the pinned + `retune.dgdr.engine`, with the replica counts of the DGD spec (fixed replicas) or those ready + at the window start (scaling). With scaling, it also carries the Planner's scaling + configuration, the shard's Planner points and the history warm-up in + `spec.aisim.inputOverrides.planner.retune_sweep`. One more single-point Request replays a grid + point again as a common-random-numbers probe: its score must match the grid's within a + relative error of 1e-9. +5. **Decide.** The Planner reads each Run's Candidates, scores them on `retune.objective`, and decides with the same rules as replay retune. A decision is persisted in a Lease before it reaches any router. -5. **Apply.** The Planner sets all eight router policy parameters of every frontend router to - the decided values, and keeps them there: a manual change to them is reverted. +6. **Apply.** The Planner sets all eight router policy parameters of every frontend router to + the decided values, and keeps them there: a manual change to them is reverted. Decided Planner + parameters take effect in the Planner itself through its + [runtime reconfiguration](../../reference/components/planner-configuration.mdx#system-port-reconfiguration-api), + with a new load predictor warmed on the window's history when the decision needs one. A + Planner that restarts runs its configured values again: retune does not restore applied + Planner parameters, and logs the ones it lost. Windows are aligned to `retune.dgdr.phase_seconds + k × interval_seconds` in epoch time, so a restarted Planner continues the same windows. Leadership and state live in two Leases owned by @@ -133,7 +159,7 @@ environment: kubernetes backend: vllm mode: agg advisory: true # start advisory; see Roll out -enable_load_scaling: false # retune-only: both flags false +enable_load_scaling: false # fixed replicas: both flags false enable_throughput_scaling: false retune: interval_seconds: 21600 # T: window length and decision cadence @@ -201,6 +227,28 @@ retune: For the other `retune` fields and the limits that `executor: dgdr` adds to them, see the [Retune field reference](../../reference/components/planner-configuration.mdx#retune-experimental). +To let the Planner scale and retune its own parameters, enable the scaling modes and add a +`planner` domain. A `router` domain is still required, since every Request sets the three router +knobs; pin them with scalars to the frontend's startup values to retune the Planner alone: + +```yaml +optimization_target: sla +ttft_ms: 500 +itl_ms: 30 +enable_load_scaling: true +enable_throughput_scaling: true +retune: + # interval_seconds, timeout_seconds, executor and dgdr as above + router: + overlap_score_credit: 1.0 # pinned to the frontend's startup values + prefill_load_scale: 1.0 + temperature: 0.0 + planner: + scaling_policy: {preset: [hybrid_180_5, hybrid_600_5]} + load_sensitivity: {preset: false} + load_scaling_down_sensitivity: {choices: [70, 90]} +``` + Live retune takes its scoring SLA when it starts: `retune.sla_ttft_ms` and `retune.sla_itl_ms`, or the Planner's `ttft_ms` and `itl_ms` where they are unset. An update of `ttft_ms` or `itl_ms` through the @@ -221,8 +269,9 @@ AISimulate searches by default, would make the Run search engine settings as wel `scheduler.max_sequences`. - The preset entry may also set `prefill_context`, but only to `1`, and must not set `decode_context`: the pinned AISimulate recommendation does not run context parallelism. -- Do not set `replicas` anywhere: the Planner fills them in from the DGD spec at each launch and - sets `spec.hardware.gpu.budget` to their GPU total. +- Do not set `replicas` anywhere: the Planner fills them in at each launch, from the DGD spec with + fixed replicas or from the ready counts at the window start with scaling, and sets + `spec.hardware.gpu.budget` to their GPU total. - Write roles the mode does not use as `null`, or leave them out. - Set the model and GPU with `retune.dgdr.model` and `retune.dgdr.sku`, not in the engine. - Leave `backend_version` out to let the pinned AISimulate pick its version for @@ -239,7 +288,8 @@ need(n) = cut_budget_seconds + job_startup_seconds + (1 + ceil((n - 1) / run.parallelism)) x run.candidate_timeout_seconds ``` -AISimulate evaluates the first candidate alone, hence the `1 +`. The Planner rejects a +AISimulate evaluates the first candidate alone, hence the `1 +`. With scaling, a shard also needs +another `cut_budget_seconds` for the history read and the pre-sweep. The Planner rejects a configuration whose largest grid shard needs more than `timeout_seconds`, because the baseline can fall in any shard and a window without a baseline score fails. With the defaults, the 108-point grid splits into 3 shards of 36 points and needs 900 + 300 + 6 × 1800 = 12000 seconds. Measure @@ -255,10 +305,10 @@ configuration error at startup. | Rule | Requirement | |---|---| | P1 | `environment: kubernetes`; `decision_latency_seconds` is `0`, or `measured` with `decision_latency_scale: 1`; `timeout_seconds` is set and at most `interval_seconds - 600` | -| P2 | `enable_load_scaling` and `enable_throughput_scaling` are both `false`; `retune.planner` is not set | +| P2 | `retune.router` is set; to retune only the Planner, pin its three knobs with scalars | | P3 | `retune.router` writes only `overlap_score_credit`, `prefill_load_scale`, `temperature`, `policy: kv_router` and `prefill_load_model: {type: none}` | | P5 | `retune.dgdr.engine` is all scalars, as in [Engine section](#engine-section) | -| P13 | The largest grid shard has at most 100 points and fits the [time budget](#time-budget) | +| P13 | The largest grid shard has at most 100 points and fits the [time budget](#time-budget). With scaling, the Planner points count as one more shard dimension, and the history before a window holds at most 1024 buckets of the shortest throughput interval | **Deployment**, checked at startup, when the DGD changes and before every router round. While one fails, the Planner launches no windows, resets the routers to their startup parameters, and @@ -394,7 +444,9 @@ The Planner exports these on `metric_reporting_prometheus_port`: | Not supported | Reason | |---|---| -| Retune with load or throughput scaling, or of Planner parameters | A DGDR Run has no base Planner configuration, warm-up or predictor pre-sweep | +| Retune with scaling on an operator whose planner image predates this release | The Run's AISimulate Planner adapter rejects `planner.retune_sweep`, so every Run fails | +| Restoring applied Planner parameters after a Planner restart | They live in the Planner process, and a load predictor needs a history warm-up; the next window starts from the configured values | +| Load-predictor history beyond 1024 buckets of the shortest throughput interval | The warm-up travels inside every Request and Candidate | | Frontends with non-default routing fields beyond the three knobs, or `kv_event_lag_ms` other than 0 | The DGDR router section expresses only the three knobs, and the Run does not pass a lag | | Workers that set their own router mode (`--router-mode` or `DYN_ROUTER_MODE`) | A worker's advertised router configuration replaces the frontend's for its workers, and the Run replays only the frontend's | | Worker arguments that pass a ConfigMap, Secret, annotation or `envFrom` value where an option could stand, including an unquoted `$VAR` of one in a worker shell script or one that `eval` or a shell given `-c` parses again (P4 fails) | The Planner cannot read the value, so it cannot rule out a router mode the worker would advertise | diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index 5b69042874c8..883ee6d6bc95 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -661,7 +661,7 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p **Experimental.** Where retune runs depends on `executor`: - `local`, the default, runs only inside planner-in-the-loop offline replay: `dynamo.replay.run_trace_replay` with `trace_format="dynamo"`, or `aisimulate predict --stack dynamo` on a Dynamo request trace. A live Planner rejects it at startup. -- `dgdr` runs on a live Planner on Kubernetes, through DGDR v1beta2 Requests. It retunes three KV router knobs with fixed replicas and is meant for development clusters only; see [Planner Live Retune](../../kubernetes/auto-deployment/planner-live-retune.md). +- `dgdr` runs on a live Planner on Kubernetes, through DGDR v1beta2 Requests. It retunes three KV router knobs, with fixed replicas or with scaling, and with scaling the Planner fields of `retune.planner` too. It is meant for development clusters only; see [Planner Live Retune](../../kubernetes/auto-deployment/planner-live-retune.md). @@ -722,7 +722,7 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p Where sweeps run. `local` replays the candidates in worker processes of the Planner and runs only inside planner-in-the-loop replay; a live Planner rejects it at startup with `planner.retune is replay-only with executor 'local'; use executor: dgdr`. `dgdr` runs each sweep as DGDR v1beta2 Requests from a live Planner on Kubernetes and requires the `dgdr` section below. **Experimental**, development clusters only: see [Planner Live Retune](../../kubernetes/auto-deployment/planner-live-retune.md). - With `dgdr`, the Planner also rejects at startup a config that breaks a [configuration rule](../../kubernetes/auto-deployment/planner-live-retune.md#deployment-rules) of live retune: it requires `environment: kubernetes`, retune-only mode with no `planner` domain, a `router` domain that writes only `overlap_score_credit`, `prefill_load_scale`, `temperature`, `policy: kv_router` and `prefill_load_model: {type: none}`, the `timeout_seconds` and `decision_latency_seconds` limits above, a `dgdr.engine` that follows the rules below, and a largest grid shard of at most 100 points that fits the [time budget](../../kubernetes/auto-deployment/planner-live-retune.md#time-budget). + With `dgdr`, the Planner also rejects at startup a config that breaks a [configuration rule](../../kubernetes/auto-deployment/planner-live-retune.md#deployment-rules) of live retune: it requires `environment: kubernetes`, a `router` domain (pin its knobs with scalars to retune only the Planner) that writes only `overlap_score_credit`, `prefill_load_scale`, `temperature`, `policy: kv_router` and `prefill_load_model: {type: none}`, the `timeout_seconds` and `decision_latency_seconds` limits above, a `dgdr.engine` that follows the rules below, and a largest grid shard of at most 100 points that fits the [time budget](../../kubernetes/auto-deployment/planner-live-retune.md#time-budget). With scaling, the Planner points are one more shard dimension, the history read takes another `cut_budget_seconds`, and the history before a window may hold at most 1024 buckets of the shortest throughput interval. Allowed values: local dgdr From 0b11c13f0d4f32b4619457a6571aace3bbdf102b Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Wed, 4 Nov 2026 12:37:40 -0800 Subject: [PATCH 3/8] fix(planner): keep PyErr out of the FPM max-age test The bindings' Rust tests link without libpython (extension-module), so a test that reaches a PyErr cannot link. Test the pure max_age_from_secs; parse_max_age only maps its None to PyValueError. Refs DYN-5400 Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- lib/bindings/python/rust/llm/fpm.rs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/bindings/python/rust/llm/fpm.rs b/lib/bindings/python/rust/llm/fpm.rs index 748cc73be1f5..975261044e37 100644 --- a/lib/bindings/python/rust/llm/fpm.rs +++ b/lib/bindings/python/rust/llm/fpm.rs @@ -155,10 +155,15 @@ fn duration_nanos(age: Duration) -> u64 { u64::try_from(age.as_nanos()).unwrap_or(u64::MAX) } -fn parse_max_age(max_age_seconds: f64) -> PyResult { +/// A freshness window of ``max_age_seconds``; ``None`` unless finite and positive. +fn max_age_from_secs(max_age_seconds: f64) -> Option { Duration::try_from_secs_f64(max_age_seconds) .ok() .filter(|age| !age.is_zero()) +} + +fn parse_max_age(max_age_seconds: f64) -> PyResult { + max_age_from_secs(max_age_seconds) .ok_or_else(|| PyValueError::new_err("max_age_seconds must be finite and positive")) } @@ -1166,12 +1171,14 @@ mod tests { } #[test] - fn parse_max_age_rejects_non_positive_and_non_finite() { - assert_eq!(parse_max_age(1.5).unwrap(), Duration::from_millis(1500)); - assert!(parse_max_age(0.0).is_err()); - assert!(parse_max_age(-1.0).is_err()); - assert!(parse_max_age(f64::NAN).is_err()); - assert!(parse_max_age(f64::INFINITY).is_err()); + fn max_age_rejects_non_positive_and_non_finite() { + // The pure parser: a PyErr in the test binary would need libpython, + // which the extension-module build does not link. + assert_eq!(max_age_from_secs(1.5), Some(Duration::from_millis(1500))); + assert_eq!(max_age_from_secs(0.0), None); + assert_eq!(max_age_from_secs(-1.0), None); + assert_eq!(max_age_from_secs(f64::NAN), None); + assert_eq!(max_age_from_secs(f64::INFINITY), None); } fn active_worker(ranks: Range, now: Instant) -> WorkerFpm { From 07d1512d9ed04fbed3704e662955e698817805cf Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Wed, 4 Nov 2026 16:47:27 -0800 Subject: [PATCH 4/8] fix(planner): keep scaling-policy switches and predictor folding consistent Review fixes for the runtime scaling policy and retune folding: - Turning load scaling back on drops the throughput lower bounds. The throughput loop refreshes them only while load scaling runs, so a hybrid -> throughput-only -> hybrid switch otherwise floored the first load decisions at the first hybrid phase's target. - A policy change restarts only the loops whose builtins it registers. A kept builtin keeps its phase, and resetting its loop pointer made run_throughput_scaling (and the metrics and diagnostics that follow it) fire on other ticks than the kept predictor. - The predictor fields fold to the running values without throughput scaling, as the docs say, so the startup point no longer replays apart from an equivalent grid point or carries an inert predictor change. - A candidate that scales at an interval where the pre-sweep picked no winner keeps the running predictor, so every candidate at an interval shares the predictor its swap is prepared with. Refs DYN-5400 Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../plugins/orchestrator/engine_adapter.py | 29 ++++--- .../src/dynamo/planner/retune/domains.py | 16 +++- components/src/dynamo/planner/retune/sweep.py | 17 +++- .../orchestrator/test_engine_adapter.py | 81 +++++++++++++++++++ .../planner/tests/unit/test_retune_sweep.py | 56 +++++++++++++ .../components/planner-configuration.mdx | 2 +- 6 files changed, 186 insertions(+), 15 deletions(-) diff --git a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py index 5d46277edb1c..a9277b89b0e8 100644 --- a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py +++ b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py @@ -580,13 +580,18 @@ def _reconcile_builtin_plugins( A builtin the policy no longer runs is unregistered; one it newly runs is registered as at startup, so it first fires one interval - after registration; one it keeps is updated in place (interval, - observation window and needs), keeping its state and phase. A new - or kept load predictor takes ``predictor``'s prepared forecasts when - given. Turning throughput scaling off also drops the throughput - lower bounds, which would otherwise keep flooring load scaling at - the last throughput target. + after registration, and its loop restarts with it; one it keeps is + updated in place (interval, observation window and needs), keeping + its state, its phase and its loop's phase. A new or kept load + predictor takes ``predictor``'s prepared forecasts when given. + + The throughput lower bounds are dropped when throughput scaling + turns off, and when load scaling turns on: the throughput loop sets + them only while load scaling runs, so a bound left from before load + scaling went off would floor its first decisions at a stale target. """ + load_was_running = "load_propose" in self._builtins + throughput_was_running = "throughput_propose" in self._builtins desired = self._builtin_registrations() for name in [name for name in self._builtins if name not in desired]: removed: Any = self._builtins.pop(name) @@ -619,13 +624,17 @@ def _reconcile_builtin_plugins( plugin.requires_produced_fields = list( registration["requires_produced_fields"] ) - if not self._config.enable_throughput_scaling: + load_started = self._config.enable_load_scaling and not load_was_running + if not self._config.enable_throughput_scaling or load_started: self._scaling_state.reset_throughput_lower_bounds() for component in ("prefill", "decode"): self._orchestrator.set_throughput_lower_bound(component, 1) - # The loop pointers restart with the policy, as at startup. - self._last_load_loop_monotonic = self._last_tick_monotonic - self._last_throughput_loop_monotonic = self._last_tick_monotonic + # A loop restarts with its newly registered builtins, as at startup; + # a kept loop keeps the phase its kept builtins fire at. + if load_started: + self._last_load_loop_monotonic = self._last_tick_monotonic + if self._config.enable_throughput_scaling and not throughput_was_running: + self._last_throughput_loop_monotonic = self._last_tick_monotonic # ------------------------------------------------------------------ # Bootstrap API (delegates to orchestrator) diff --git a/components/src/dynamo/planner/retune/domains.py b/components/src/dynamo/planner/retune/domains.py index 2e20b2d42f76..37611ac4e640 100644 --- a/components/src/dynamo/planner/retune/domains.py +++ b/components/src/dynamo/planner/retune/domains.py @@ -73,12 +73,26 @@ THROUGHPUT_INTERVAL, LOAD_INTERVAL, ) +# The load predictor's fields (``reconfiguration.PREDICTOR_FIELDS``). +PREDICTOR_KEYS: tuple[str, ...] = ( + "load_predictor", + "load_predictor_log1p", + "prophet_window_size", + "kalman_q_level", + "kalman_q_trend", + "kalman_r", + "kalman_min_points", +) # Fields that act only with load or only with throughput scaling. Without # their mode a point folds them to the running value. The load interval is # not among them: it is also the cadence at which live FPM observations # refit the performance model that throughput scaling uses. LOAD_ONLY_KEYS: tuple[str, ...] = (LOAD_SCALING_DOWN_SENSITIVITY,) -THROUGHPUT_ONLY_KEYS: tuple[str, ...] = (THROUGHPUT_INTERVAL, MAX_THROUGHPUT_REPLICAS) +THROUGHPUT_ONLY_KEYS: tuple[str, ...] = ( + THROUGHPUT_INTERVAL, + MAX_THROUGHPUT_REPLICAS, + *PREDICTOR_KEYS, +) # Combinations a ``preset: false`` group may expand to before deduplication, # which bounds the expansion's memory. The AISimulate default load-predictor # domain is 256 combinations. diff --git a/components/src/dynamo/planner/retune/sweep.py b/components/src/dynamo/planner/retune/sweep.py index 55f8acf595c6..ca946f7f39c4 100644 --- a/components/src/dynamo/planner/retune/sweep.py +++ b/components/src/dynamo/planner/retune/sweep.py @@ -63,6 +63,7 @@ from dynamo.planner.reconfiguration import PREDICTOR_FIELDS, prepare_predictor from dynamo.planner.retune.domains import ( ENABLE_THROUGHPUT, + PREDICTOR_KEYS, THROUGHPUT_INTERVAL, PredictorCandidate, canonical_planner_point, @@ -365,7 +366,8 @@ def evaluated_params( values (``canonical_planner_point``), so an inactive difference is neither replayed nor applied; - when it scales on throughput, carries the pre-sweep winner of its - throughput interval (``presweep_fields``). + throughput interval (``presweep_fields``), else the running + predictor, so candidates at one interval share one predictor. That leaves the inner planner unchanged where the base planner already holds the current values, and gives a candidate equivalent to the @@ -379,8 +381,17 @@ def evaluated_params( running = _running_planner(baseline, base_planner) planner = canonical_planner_point(planner, running) interval = scales_on_throughput(planner, running) - fields = None if interval is None else presweep_fields(presweep, interval) - planner.update(fields or {}) + if interval is not None: + fields = presweep_fields(presweep, interval) + if not fields: + # No winner at this interval: the predictor is not searched + # there, so the candidate keeps the running one. + fields = { + key: running[key] + for key in PREDICTOR_KEYS + if key in planner and key in running + } + planner.update(fields) elif presweep is not None: # No inner planner to read the interval from: one pre-sweep summary. planner.update(presweep.get("fields") or {}) diff --git a/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py b/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py index 79a763b85b32..46f4cc06ba9a 100644 --- a/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py +++ b/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py @@ -2169,6 +2169,87 @@ def test_throughput_only_sla_still_pulls_fpm_for_live_regression(): assert not load_interval_tick.run_throughput_scaling +def test_load_scaling_turned_back_on_drops_a_stale_throughput_floor(): + # Imported here: reconfiguration pulls in the planner core. + from dynamo.planner.reconfiguration import candidate_config + + cfg = PlannerConfig.model_validate( + { + "mode": "disagg", + "enable_load_scaling": True, + "enable_throughput_scaling": True, + "optimization_target": "sla", + "served_model_name": "test", + "load_adjustment_interval_seconds": 5, + "throughput_adjustment_interval_seconds": 60, + "load_predictor": "constant", + } + ) + adapter = OrchestratorEngineAdapter(cfg, _caps(), clock=VirtualClock()) + adapter.initial_tick(start_s=0.0) + state = adapter._scaling_state + + def apply(updates): + adapter.apply_runtime_configuration( + candidate_config(cfg, updates), list(updates), None + ) + + # Hybrid: the throughput loop floored load scaling at (4, 4). + state._throughput_lower_bound_p = 4 + state._throughput_lower_bound_d = 4 + # Throughput-only: the throughput loop scales directly and no longer + # refreshes the floor, which is harmless while load scaling is off. + apply({"enable_load_scaling": False}) + assert (state._throughput_lower_bound_p, state._throughput_lower_bound_d) == (4, 4) + # Load scaling back on: its first decisions, before the next throughput + # tick, must not be floored at the target of the earlier hybrid phase. + apply({"enable_load_scaling": True}) + assert (state._throughput_lower_bound_p, state._throughput_lower_bound_d) == (1, 1) + + +@pytest.mark.asyncio +async def test_a_kept_throughput_loop_keeps_its_phase_on_a_load_interval_change(): + # Imported here: reconfiguration pulls in the planner core. + from dynamo.planner.reconfiguration import candidate_config + + cfg = PlannerConfig.model_validate( + { + "mode": "agg", + "enable_load_scaling": True, + "enable_throughput_scaling": True, + "optimization_target": "sla", + "served_model_name": "test", + "load_adjustment_interval_seconds": 10, + "throughput_adjustment_interval_seconds": 60, + "scheduling": {"scale_interval_seconds": 5.0}, + "load_predictor": "constant", + } + ) + adapter = OrchestratorEngineAdapter(cfg, _caps(), clock=VirtualClock()) + registry = adapter._orchestrator.registry + tick = adapter.initial_tick(start_s=0.0) + observations = FpmObservations(decode={("w1", 0): _make_fpm("w1")}) + traffic = TrafficObservation(60.0, 10.0, 100.0, 10.0) + rows = [] + while tick.at_s <= 240: + before = registry.get_plugin("builtin_load_predict").last_call_at + effects = await adapter.tick( + tick, + TickInput(now_s=tick.at_s, fpm_observations=observations, traffic=traffic), + ) + fired = registry.get_plugin("builtin_load_predict").last_call_at != before + if tick.run_throughput_scaling or fired: + rows.append((tick.at_s, tick.run_throughput_scaling, fired)) + if tick.at_s == 90.0: + updates = {"load_adjustment_interval_seconds": 5} + adapter.apply_runtime_configuration( + candidate_config(cfg, updates), list(updates), None + ) + tick = effects.next_tick + # The throughput loop's flag follows the kept predictor's phase. + assert rows and all(run == fired for _, run, fired in rows), rows + + @pytest.mark.asyncio async def test_runtime_scaling_policy_change_reregisters_builtins(): # Imported here: reconfiguration pulls in the planner core. diff --git a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py index 8263e7167ee4..e7bde14b5132 100644 --- a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py +++ b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py @@ -35,6 +35,7 @@ from dynamo.planner.retune.sweep import ( _OBJECTIVES, GOODPUT_KEY, + evaluated_params, objective_maximized, score_summary, with_predictor_fields, @@ -815,3 +816,58 @@ def importing_worker(): for process, conn in zip(processes, unread): process.release() conn.close() + + +def test_evaluated_params_folds_the_predictor_without_throughput_scaling(): + # The running planner scales on throughput with kalman; the startup + # point (load only) still carries the arima it started with. + policy = { + "enable_load_scaling": True, + "load_adjustment_interval_seconds": 5, + "throughput_adjustment_interval_seconds": 180, + } + running = {**policy, "enable_throughput_scaling": True, "load_predictor": "kalman"} + baseline = RetuneParams(planner=dict(running)) + startup = RetuneParams( + planner={ + **policy, + "enable_throughput_scaling": False, + "load_predictor": "arima", + } + ) + grid = RetuneParams(planner={**policy, "enable_throughput_scaling": False}) + evaluated = [evaluated_params(c, baseline, None, running) for c in (startup, grid)] + # One replay, and no inert predictor change to apply. + assert len({point.key() for point in evaluated}) == 1 + assert evaluated[0].planner["load_predictor"] == "kalman" + + +def test_evaluated_params_keeps_the_running_predictor_without_a_winner(): + def policy(interval: int) -> dict: + return { + "enable_throughput_scaling": True, + "enable_load_scaling": True, + "throughput_adjustment_interval_seconds": interval, + "load_adjustment_interval_seconds": 5, + } + + running = {**policy(180), "load_predictor": "kalman", "kalman_min_points": 5} + baseline = RetuneParams(planner=dict(running)) + startup = RetuneParams( + planner={**policy(600), "load_predictor": "constant", "kalman_min_points": 5} + ) + grid = RetuneParams(planner=policy(600)) + # The pre-sweep found no winner at 600 s (a window shorter than 600 s). + presweep = { + "by_interval": { + "180": {"fields": {"load_predictor": "kalman"}}, + "600": {"fields": {}}, + } + } + at_600 = { + evaluated_params(c, baseline, presweep, running).planner["load_predictor"] + for c in (startup, grid) + } + # Candidates at one interval share one predictor, the one the swap + # for that interval is prepared with. + assert at_600 == {"kalman"} diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index 5b69042874c8..1a4352c5e39e 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -864,7 +864,7 @@ The group rules are those of `aisimulate recommend`: to write a knob of a group, - With both modes on, the load interval must be shorter than the throughput interval, and both intervals must be multiples of `scheduling.scale_interval_seconds`, which is fixed at startup. A written preset entry that breaks either rule is rejected; a point of `{preset: default}` or `{preset: false}` that breaks one is dropped. A null interval in a preset is the running value, except for an enabled mode. - Under any `optimization_target` other than `sla`, the Planner always scales on load only: `{preset: default}` keeps the two `load_*` presets, and a written entry or knob that enables throughput scaling is rejected. - Fields that a point's own modes make inactive take the running value, so equivalent points are replayed once: `load_scaling_down_sensitivity` without load scaling, and the throughput interval, `max_throughput_scaling_replicas` and the predictor without throughput scaling. The load interval stays active either way: it is also the cadence at which live FPM samples refit the performance model. - - The predictor pre-sweep runs per throughput interval, and a point carries the winner of its own interval. `history_seconds` must cover the largest throughput interval of any point. + - The predictor pre-sweep runs per throughput interval, and a point carries the winner of its own interval, or the running predictor where the pre-sweep picks none. `history_seconds` must cover the largest throughput interval of any point. - Applying a point that changes the throughput interval or turns throughput scaling on swaps in a new predictor, warmed on the sweep's history bucketed at the new interval. Other scaling-policy changes keep the running predictor. - **Load sensitivity.** Retune searches only `load_scaling_down_sensitivity`; `load_min_observations` keeps its running value. A preset name contributes its sensitivity alone, so with a running `load_min_observations` other than 5 the `default` preset is applied only in part. A preset mapping must set `load_min_observations` to the running value. - **Load predictor.** The predictor is never a grid dimension: a [pre-sweep](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#load-predictor-pre-sweep) picks one candidate per sweep. Candidates that give the same predictor are evaluated once (`{preset: false}` with the default knob domains is 256 combinations and 40 predictors). Records label a candidate with its preset name, or `custom_` for the `i`-th entry. The number of candidates must not exceed `max_candidates`. From 8efd287638c348d4ce7cdbf29d6332d0605da887 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Wed, 4 Nov 2026 16:56:04 -0800 Subject: [PATCH 5/8] fix(planner): shard the grid's planner points and guard live apply Review fixes for live retune with scaling: - The DGDR sweep's planner axis is the grid's planner points (SweepRequest.grid_planners, set by the live controller), as P13 counts it. It held every candidate's planner point, so a running value off the grid, or a baseline that keeps its predictor while the grid carries the pre-sweep winner, crossed one more planner point with the whole router grid: a configuration P13 accepted could fail every window (invalid_domain, launch_too_late). The baseline and the startup point now get single-point shards when off the grid, as their router points already did. - A decided planner change is dropped when the configuration's revision moves while its predictor is prepared, as the control API's expected_revision does, instead of installing a predictor warmed for another configuration. - Only decisions the supervisor accepted change the planner; a decision it drops (not an initialized leader) is rewound and not applied. Refs DYN-5400 Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- components/src/dynamo/planner/core/base.py | 23 ++++--- .../src/dynamo/planner/retune/controller.py | 10 +++ .../src/dynamo/planner/retune/live/dgdr.py | 34 +++++++--- .../dynamo/planner/retune/live/supervisor.py | 10 +-- components/src/dynamo/planner/retune/sweep.py | 23 ++++++- .../tests/unit/test_live_retune_adapter.py | 1 + .../unit/test_live_retune_controller_hooks.py | 4 ++ .../tests/unit/test_live_retune_dgdr.py | 63 +++++++++++++++++++ .../tests/unit/test_live_retune_supervisor.py | 14 +++-- .../tests/unit/test_reconfiguration.py | 40 ++++++++++++ 10 files changed, 193 insertions(+), 29 deletions(-) diff --git a/components/src/dynamo/planner/core/base.py b/components/src/dynamo/planner/core/base.py index fbb514f720ff..a31c46e58812 100644 --- a/components/src/dynamo/planner/core/base.py +++ b/components/src/dynamo/planner/core/base.py @@ -120,7 +120,8 @@ async def start(self, host: NativePlannerBase) -> None: async def stop(self) -> None: ... - def on_effects(self, effects: PlannerEffects) -> None: + def on_effects(self, effects: PlannerEffects) -> Sequence[RetuneDecision]: + """Take the tick's decisions; return the ones it accepted.""" ... @@ -665,6 +666,10 @@ async def _apply_retune_planner_changes(self, decision: RetuneDecision) -> None: candidate = candidate_config(self.config, changes) self.validate_runtime_candidate(candidate) rebuild = predictor_rebuild_required(self.config, candidate) + # Like the control API's expected_revision: a change made + # while the predictor is prepared (the control API or another + # decision) wins, and this one is dropped. + revision = self._config_generation predictor = None if rebuild: predictor = await asyncio.wait_for( @@ -678,13 +683,14 @@ async def _apply_retune_planner_changes(self, decision: RetuneDecision) -> None: ) async with self._effect_admission_lock: async with self._config_lock: - candidate = candidate_config(self.config, changes) - self.validate_runtime_candidate(candidate) - if predictor_rebuild_required(self.config, candidate) != rebuild: + if self._config_generation != revision: raise ValueError( "the configuration changed while the predictor was " - "prepared" + f"prepared (revision {revision} -> " + f"{self._config_generation})" ) + candidate = candidate_config(self.config, changes) + self.validate_runtime_candidate(candidate) self.apply_runtime_candidate(candidate, changes.keys(), predictor) logger.info( "Retune %s applied planner changes %s", @@ -1376,11 +1382,14 @@ async def _run_one_tick( # Retune decisions go to the live supervisor's outbox, which # persists them before actuating routers; _apply_effects never # sees router parameters. + # Only a decision the supervisor accepted (persisted, then + # actuated) changes the planner: a dropped one is rewound. + accepted: Sequence[RetuneDecision] = () try: - self._live_retune.on_effects(effects) + accepted = self._live_retune.on_effects(effects) except Exception: logger.exception("Live retune failed to take the tick's effects") - self._schedule_retune_planner_changes(effects.retune_decisions) + self._schedule_retune_planner_changes(accepted) await self._submit_effects(effects, config_generation) emit_diagnostics = self._should_emit_tick_diagnostics(tick, effects) if emit_diagnostics: diff --git a/components/src/dynamo/planner/retune/controller.py b/components/src/dynamo/planner/retune/controller.py index 34d0fa737d78..e20081bec0a9 100644 --- a/components/src/dynamo/planner/retune/controller.py +++ b/components/src/dynamo/planner/retune/controller.py @@ -868,6 +868,16 @@ def _live_request_fields( else dict(spec_replicas) ), "replicas_verified": verified, + # The planner axis a DGDR sweep shards: the grid's planner + # points, without the baseline's or the startup point's. + "grid_planners": tuple( + dict(planner) + for planner in { + RetuneParams(planner=point.planner).key(): point.planner + for point in sorted(self._grid, key=RetuneParams.key) + if point.planner + }.values() + ), } def _submit( diff --git a/components/src/dynamo/planner/retune/live/dgdr.py b/components/src/dynamo/planner/retune/live/dgdr.py index 56a99cf7e9d6..100360c4240f 100644 --- a/components/src/dynamo/planner/retune/live/dgdr.py +++ b/components/src/dynamo/planner/retune/live/dgdr.py @@ -100,6 +100,7 @@ SweepStatus, WorkerPool, evaluated_params, + evaluated_planner, predictor_swap_errors, predictor_swap_tasks, run_presweeps, @@ -131,8 +132,6 @@ # (``simulation.provider.RETUNE_SWEEP``). A Candidate's planner parameters # are the section narrowed to its one point. RETUNE_SWEEP = "retune_sweep" -# The shard axis of the planner points (sorts after every router knob). -_PLANNER_AXIS = "~planner" _ROLES = ("aggregated", "prefill", "decode") _TEMPERATURE = DGDR_ROUTER_KNOBS["temperature"] # spec.objective.metric -> AISimulate optimization.target (operator lowering). @@ -1265,17 +1264,21 @@ def _startup_candidate(self, domain: Mapping[str, Sequence[float]]) -> RetunePar The controller's candidates are the grid plus the startup point and the baseline, so an off-grid candidate is one of those two. A - candidate is off the grid when its router point is; planner points - are the distinct points of every candidate, so they never are. + candidate is off the grid when its router point is, or when its + evaluated planner part is not on the planner axis. """ baseline = self._evaluated(self.request.baseline) + axis = self.planner_points for candidate in self.request.candidates: router = candidate.router if candidate == self.request.baseline or set(router) != set(domain): continue - if not all(float(router[key]) in domain[key] for key in domain): - return self._evaluated(candidate) + evaluated = self._evaluated(candidate) + if not all(float(router[key]) in domain[key] for key in domain) or ( + axis and not any(_same_planner(evaluated.planner, p) for p in axis) + ): + return evaluated return baseline def _prepare_planner(self) -> None: @@ -1365,11 +1368,22 @@ def _prepare_planner(self) -> None: ) finally: pool.close() + # The planner axis: the grid's planner points as evaluated (what + # P13 counts). The baseline, evaluated as is, and the startup point + # are off the grid when their planner part is not among them, and + # plan_shards gives each a single-point shard. + if request.grid_planners is None: + sources = [point.planner for point in self.evaluated.values()] + else: + sources = [ + evaluated_planner( + planner, request.baseline, presweep, request.base_planner + ) + for planner in request.grid_planners + ] planners: dict[str, dict[str, Any]] = {} - for point in self.evaluated.values(): - planners.setdefault( - RetuneParams(planner=point.planner).key(), point.planner - ) + for planner in sources: + planners.setdefault(RetuneParams(planner=planner).key(), planner) self.planner_points = [planners[key] for key in sorted(planners)] self.planner_base = retune_sweep_base(request.base_planner or {}) self.planner_warmup = { diff --git a/components/src/dynamo/planner/retune/live/supervisor.py b/components/src/dynamo/planner/retune/live/supervisor.py index 59a435cabea0..810ad95ee19d 100644 --- a/components/src/dynamo/planner/retune/live/supervisor.py +++ b/components/src/dynamo/planner/retune/live/supervisor.py @@ -898,12 +898,13 @@ def sweep_fn( failure_reason="sweep_error", ) - def on_effects(self, effects: PlannerEffects) -> None: - """Enqueue the tick's decisions; never raises into the tick.""" + def on_effects(self, effects: PlannerEffects) -> list[RetuneDecision]: + """Enqueue the tick's decisions and return them, or ``[]`` when they + are dropped; never raises into the tick.""" decisions = list(effects.retune_decisions or ()) if not decisions: - return + return [] if not (self.is_leader() and self._initialized): logger.warning( "retune: dropping %d decision(s) %s: not an initialized leader", @@ -911,7 +912,7 @@ def on_effects(self, effects: PlannerEffects) -> None: [decision.decision_id for decision in decisions], ) self._rewind_now() - return + return [] router_parameters = effects.router_parameters for decision in decisions: self._outbox.append( @@ -929,6 +930,7 @@ def on_effects(self, effects: PlannerEffects) -> None: ) ) self._outbox_wake.set() + return decisions # ------------------------------------------------------------ lifecycle diff --git a/components/src/dynamo/planner/retune/sweep.py b/components/src/dynamo/planner/retune/sweep.py index 4adb11491b83..79aba8231b1b 100644 --- a/components/src/dynamo/planner/retune/sweep.py +++ b/components/src/dynamo/planner/retune/sweep.py @@ -215,6 +215,10 @@ class SweepRequest: spec_replicas: dict[str, int] | None = None replicas_verified: bool = True generation: int = 0 + # Live only: the grid's distinct planner points (searched fields), the + # planner axis a DGDR sweep shards; ``None`` derives it from every + # candidate. + grid_planners: tuple[dict[str, Any], ...] | None = None def __post_init__(self) -> None: keys = [candidate.key() for candidate in self.candidates] @@ -376,7 +380,21 @@ def evaluated_params( if candidate == baseline: return candidate - planner = {**baseline.planner, **candidate.planner} + planner = evaluated_planner(candidate.planner, baseline, presweep, base_planner) + evaluated = RetuneParams(router=dict(candidate.router), planner=planner) + return candidate if evaluated == candidate else evaluated + + +def evaluated_planner( + planner_point: Mapping[str, Any], + baseline: RetuneParams, + presweep: Mapping[str, Any] | None, + base_planner: Mapping[str, Any] | None, +) -> dict[str, Any]: + """The planner part ``evaluated_params`` gives any candidate other than + the baseline whose planner point is ``planner_point``.""" + + planner = {**baseline.planner, **planner_point} if planner and base_planner is not None: running = _running_planner(baseline, base_planner) planner = canonical_planner_point(planner, running) @@ -395,8 +413,7 @@ def evaluated_params( elif presweep is not None: # No inner planner to read the interval from: one pre-sweep summary. planner.update(presweep.get("fields") or {}) - evaluated = RetuneParams(router=dict(candidate.router), planner=planner) - return candidate if evaluated == candidate else evaluated + return planner def with_predictor_fields( diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py b/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py index 34cbd95a6f71..1b913f426faa 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py @@ -375,6 +375,7 @@ def on_effects(self, effects): self.effects.append(effects) if self.on_effects_error is not None: raise self.on_effects_error + return list(effects.retune_decisions or ()) def _live_config(executor: str = "dgdr") -> PlannerConfig: diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py index ed0497ed1b78..0d752ea90e4d 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py @@ -627,6 +627,10 @@ def counts(now): 80, 90, } + # The planner axis the DGDR sweep shards: the grid's planner points. + assert sorted( + planner["load_scaling_down_sensitivity"] for planner in request.grid_planners + ) == [70, 80, 90] # --------------------------------------------------------------------------- diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py index 8be7986100df..db13574cd9fa 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py @@ -2329,6 +2329,69 @@ def test_scenario_2_throughput_planner_carries_its_history_warmup(tmp_path): assert result.predictor_warmup_errors == {} +def test_scenario_2_off_grid_running_planner_gets_its_own_shard(tmp_path): + # Grid {70, 90}, running 80: the planner axis is the grid's two points, + # as P13 counts it, and the baseline replays in a single-point shard + # instead of adding 80 to the axis of every grid shard. + h = Harness(tmp_path) + request = dataclasses.replace( + scenario_2_request(_base_planner()), + grid_planners=({LSD: 70}, {LSD: 90}), + ) + + result = h.sweep(request, bucket_history=FakeHistory(), worker_pool=InlinePool) + + assert result.status == "ok", (result.error, result.details) + assert result.details["planner"]["points"] == 2 + assert "point-baseline" in result.details["shards"] + grid_points = { + point[LSD] + for body in h.created() + if body["metadata"]["labels"][LABEL_SHARD].startswith("grid-") + for point in body["spec"]["aisim"]["inputOverrides"]["planner"]["retune_sweep"][ + "points" + ] + } + assert grid_points == {70, 90} + assert all(result.scores[c.key()] is not None for c in request.candidates) + + +def test_scenario_2_presweep_winner_keeps_the_planner_axis(tmp_path, monkeypatch): + # The pre-sweep picks kalman while the planner runs arima: the grid's + # points carry kalman and the baseline keeps arima, so the baseline is + # off the grid instead of doubling the planner axis. + h = Harness(tmp_path) + base = _base_planner( + optimization_target="sla", + enable_throughput_scaling=True, + enable_load_scaling=True, + load_predictor="arima", + ) + request = dataclasses.replace( + scenario_2_request(base), + grid_planners=({LSD: 70}, {LSD: 90}), + predictor_candidates=(("kalman_default", {"load_predictor": "kalman"}),), + ) + interval = str(base[THROUGHPUT_INTERVAL]) + monkeypatch.setattr( + dgdr, + "run_presweeps", + lambda *args: { + "by_interval": {interval: {"fields": {"load_predictor": "kalman"}}} + }, + ) + from dynamo.planner.retune import sweep as sweep_module + + monkeypatch.setattr(sweep_module, "prepare_predictor", lambda *args: None) + + result = h.sweep(request, bucket_history=FakeHistory(), worker_pool=InlinePool) + + assert result.status == "ok", (result.error, result.details) + assert result.details["planner"]["points"] == 2 + assert "point-baseline" in result.details["shards"] + assert all(result.scores[c.key()] is not None for c in request.candidates) + + def test_scenario_2_candidate_with_a_foreign_planner_point_is_unmapped(tmp_path): h = Harness(tmp_path) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py index 8326d3289a4c..47abbb9e28e0 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py @@ -792,8 +792,9 @@ async def settle(processes: list[Process], world: World, rounds: int = 6) -> Non await process.sup.control_round() -def enqueue(process: Process, *decisions: RetuneDecision) -> None: - """Hand the supervisor a tick's decisions with the controller's routers.""" +def enqueue(process: Process, *decisions: RetuneDecision) -> list[RetuneDecision]: + """Hand the supervisor a tick's decisions with the controller's routers; + the ones it accepted.""" controller = process.controller desired = ( @@ -801,7 +802,7 @@ def enqueue(process: Process, *decisions: RetuneDecision) -> None: if controller is not None else startup_flat("a") ) - process.sup.on_effects( + return process.sup.on_effects( SimpleNamespace( retune_decisions=list(decisions), router_parameters=by_wt(process.host.variant, desired), @@ -1203,7 +1204,8 @@ def failures() -> list[str]: store = StateStore(world.kube, sup.dgd) snapshot = store.read() store.write(dataclasses.replace(snapshot.state, leader_term="9/x/1"), snapshot) - enqueue(process, make_decision(K0 + 1, "kept")) + decision = make_decision(K0 + 1, "kept") + assert enqueue(process, decision) == [decision] await sup.outbox_round() assert not sup.initialized and failures() == ["state_write"] * 3 @@ -1213,7 +1215,9 @@ async def test_decisions_are_dropped_and_rewound_without_leadership() -> None: process = await started(world) controller = process.controller controller._last_k = K0 + 1 - enqueue(process, make_decision(K0 + 1, "kept")) + # Dropped decisions are not handed back: the planner applies none of + # their planner changes. + assert enqueue(process, make_decision(K0 + 1, "kept")) == [] assert process.sup.outbox_size == 0 assert controller._last_k == -1 # no state: every window is due again diff --git a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py index dcfc51dcdabf..4a294faac353 100644 --- a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py +++ b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py @@ -894,6 +894,46 @@ async def test_live_retune_applies_planner_changes_through_runtime_reconfigurati await planner._shutdown_runtime() +async def test_live_retune_drops_a_planner_change_overtaken_while_preparing( + monkeypatch, +): + # A decision needs a new predictor for 360 s; while it is prepared, a + # control API update activates another revision. The decision is dropped + # instead of installing a predictor warmed for another configuration. + import dynamo.planner.core.base as base_module + + planner = _planner("disagg", optimization_target="sla", load_predictor="constant") + engine = planner._ensure_engine() + discarded = [] + + class Controller: + def discard_planner_changes(self, changes): + discarded.append(dict(changes)) + + engine._retune = Controller() + real_prepare = base_module.prepare_predictor + + def prepare(config, capabilities, observations): + predictor = real_prepare(config, capabilities, observations) + # The control API's activation, between the two locked sections. + planner._config_generation += 1 + planner.config.load_scaling_down_sensitivity = 70 + return predictor + + monkeypatch.setattr(base_module, "prepare_predictor", prepare) + changes = {"throughput_adjustment_interval_seconds": 360} + warmup = tuple(TrafficObservation(360.0, 10.0, 100.0, 10.0) for _ in range(8)) + + await _apply_decisions(planner, [_applied(changes, warmup)]) + + assert planner.config.throughput_adjustment_interval_seconds == 180 + assert planner.config.load_scaling_down_sensitivity == 70 + assert planner._config_generation == 1 + assert discarded == [changes] + engine._retune = None + await planner._shutdown_runtime() + + async def test_live_retune_keeps_the_config_when_a_planner_change_cannot_apply(): planner = _planner( "disagg", From 107b9f217c1d9677339a167ecd90926509b4ca76 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Wed, 4 Nov 2026 19:31:20 -0800 Subject: [PATCH 6/8] fix(planner): address CodeRabbit review of the scaling-policy retune - FpmEventSubscriber.set_max_age: document that start_tracking sets the window from its own argument, so set_max_age takes effect after it; get_recent_stats_with_health no longer says the limit is set once. - planner-configuration: runtime warmup is bucketed at the candidate's throughput interval; only updates that keep the scaling modes and intervals keep the cadence; a kept mode runs one new interval after its last run; turning load scaling on also drops the throughput floor. - _SweepInputs: no shared mutable {} defaults; every construction passes its own mappings. - Import reconfiguration at module level in engine_adapter (it does not import the adapter, so there is no cycle) and in the new tests. Refs DYN-5400 Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../plugins/orchestrator/engine_adapter.py | 10 ++++------ components/src/dynamo/planner/retune/sweep.py | 8 ++++---- .../plugins/orchestrator/test_engine_adapter.py | 10 +--------- .../planner/tests/unit/test_reconfiguration.py | 7 +++---- .../planner/tests/unit/test_retune_sweep.py | 1 + .../components/planner-configuration.mdx | 16 ++++++++++------ lib/bindings/python/rust/llm/fpm.rs | 3 ++- lib/bindings/python/src/dynamo/_core.pyi | 9 +++++---- 8 files changed, 30 insertions(+), 34 deletions(-) diff --git a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py index a9277b89b0e8..a1d064fbc46e 100644 --- a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py +++ b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py @@ -134,6 +134,10 @@ ) from dynamo.planner.plugins.types import PoolTrafficDemand as PoolTrafficDemandData from dynamo.planner.plugins.types import TrafficMetrics, WorkerState +from dynamo.planner.reconfiguration import ( + SCALING_POLICY_FIELDS, + predictor_rebuild_required, +) log = logging.getLogger(__name__) @@ -849,12 +853,6 @@ def apply_runtime_configuration( turning on, or a predictor field or the throughput interval changing). Everything is checked before anything changes. """ - # Imported on use: reconfiguration imports the planner core. - from dynamo.planner.reconfiguration import ( - SCALING_POLICY_FIELDS, - predictor_rebuild_required, - ) - fields = list(fields) if predictor is not None: if not isinstance(predictor, BuiltinLoadPredict): diff --git a/components/src/dynamo/planner/retune/sweep.py b/components/src/dynamo/planner/retune/sweep.py index ca946f7f39c4..f43a984529c0 100644 --- a/components/src/dynamo/planner/retune/sweep.py +++ b/components/src/dynamo/planner/retune/sweep.py @@ -657,9 +657,9 @@ class _SweepInputs(NamedTuple): window_path: str num_requests: int row_counts: dict[str, int] - observations: dict[int, tuple[TrafficObservation, ...]] = {} - observations_paths: dict[int, str] = {} - warmup_paths: dict[int, str] = {} + observations: dict[int, tuple[TrafficObservation, ...]] + observations_paths: dict[int, str] + warmup_paths: dict[int, str] @dataclass(frozen=True) @@ -725,7 +725,7 @@ def _read_sweep_inputs(spec: _ReadSpec, work_dir: str) -> _SweepInputs: ) row_counts = dict(window.row_counts) if window.num_requests == 0 or not spec.buckets: - return _SweepInputs(window.path, window.num_requests, row_counts) + return _SweepInputs(window.path, window.num_requests, row_counts, {}, {}, {}) warmup_paths: dict[int, str] = {} if spec.history_start_ms < spec.window_start_ms: diff --git a/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py b/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py index 46f4cc06ba9a..fe930b2f7a90 100644 --- a/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py +++ b/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py @@ -57,6 +57,7 @@ OverrideType, ScalingProposal, ) +from dynamo.planner.reconfiguration import candidate_config, prepare_predictor from dynamo.planner.tests.retune_stubs import ( install_stub_controller, retune_inputs, @@ -2170,9 +2171,6 @@ def test_throughput_only_sla_still_pulls_fpm_for_live_regression(): def test_load_scaling_turned_back_on_drops_a_stale_throughput_floor(): - # Imported here: reconfiguration pulls in the planner core. - from dynamo.planner.reconfiguration import candidate_config - cfg = PlannerConfig.model_validate( { "mode": "disagg", @@ -2209,9 +2207,6 @@ def apply(updates): @pytest.mark.asyncio async def test_a_kept_throughput_loop_keeps_its_phase_on_a_load_interval_change(): - # Imported here: reconfiguration pulls in the planner core. - from dynamo.planner.reconfiguration import candidate_config - cfg = PlannerConfig.model_validate( { "mode": "agg", @@ -2252,9 +2247,6 @@ async def test_a_kept_throughput_loop_keeps_its_phase_on_a_load_interval_change( @pytest.mark.asyncio async def test_runtime_scaling_policy_change_reregisters_builtins(): - # Imported here: reconfiguration pulls in the planner core. - from dynamo.planner.reconfiguration import candidate_config, prepare_predictor - cfg = PlannerConfig.model_validate( { "mode": "agg", diff --git a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py index 7c760d4f5f2a..636871fc7b02 100644 --- a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py +++ b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py @@ -27,6 +27,9 @@ WorkerCapabilities, WorkerCounts, ) +from dynamo.planner.environment.metrics_provider.runtime_provider import ( + RuntimeFpmProvider, +) from dynamo.planner.plugins.builtins.local_planner import BuiltinLoadPredict from dynamo.planner.plugins.clock import VirtualClock from dynamo.planner.plugins.orchestrator.engine_adapter import OrchestratorEngineAdapter @@ -788,10 +791,6 @@ async def test_scaling_policy_is_a_runtime_field(tmp_path): def test_runtime_fpm_provider_updates_both_subscribers_in_place(): - from dynamo.planner.environment.metrics_provider.runtime_provider import ( - RuntimeFpmProvider, - ) - class Subscriber: def __init__(self): self.ages = [] diff --git a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py index e7bde14b5132..471050756af1 100644 --- a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py +++ b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py @@ -764,6 +764,7 @@ def test_presweep_dispatch_does_not_wait_for_workers_that_are_not_reading(tmp_pa {}, observations={60: tuple(observations)}, observations_paths={60: path}, + warmup_paths={}, ) presets = ("constant_last", "kalman_default_raw") request = sweep_request( diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index 1a4352c5e39e..c52382980132 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -487,8 +487,11 @@ bucket length of its history), or turns throughput scaling on. Other scaling-mod keep the running predictor and the live FPM samples: - Turning a mode on registers its builtin plugins as at startup, so they first run one - interval later; turning it off unregisters them. Turning throughput scaling off also - drops the last throughput targets, which would otherwise keep flooring load scaling. + interval later; turning it off unregisters them. Turning throughput scaling off, or + load scaling on, also drops the last throughput targets, which would otherwise floor + load scaling at a stale target. +- A mode that stays on keeps its builtin plugins and their state. After an interval + change, it next runs one new interval after its last run. - A new load interval also sets the FPM freshness window to three load intervals. - A throughput-only SLA Planner keeps pulling FPM samples at the load interval to refit its performance model, so turning load scaling on later does not start that model cold. @@ -524,7 +527,7 @@ that directory; absolute paths must also be inside it. Runtime warmup accepts at 16 MiB of stored data and 16 MiB after decompression, and at most 4,096 aggregation intervals (including idle gaps). Directories and globs remain supported by the startup `load_predictor_warmup_trace` option but are rejected by runtime reconfiguration. -Observations use the unchanged throughput adjustment interval. Warmup must provide enough +Observations are bucketed at the candidate's `throughput_adjustment_interval_seconds`. Warmup must provide enough effective samples for request count, ISL, and OSL and complete a successful forecast. SLA/budget-only updates do not require warmup. Supplying a trace explicitly also allows rewarming an unchanged predictor. @@ -533,9 +536,10 @@ has a 120-second deadline and only one preparation can run at a time. If a backg fit outlives its deadline, its result cannot activate and further preparations are rejected until that work exits; updates that do not replace the predictor remain available. Invalid data, failed fitting, or a changed revision leave the active settings unchanged. -Successful activation occurs between decisions and increments the revision. Updates preserve -the existing prediction and scaling cadence, including changes to SLA targets, -`max_throughput_scaling_replicas`, and predictor settings. The next scheduled throughput +Successful activation occurs between decisions and increments the revision. An update that +leaves the scaling modes and intervals unchanged preserves the existing prediction and scaling +cadence, including changes to SLA targets, `max_throughput_scaling_replicas`, and predictor +settings. A scaling-mode or interval change sets the new cadence as described above. The next scheduled throughput decision uses the active settings and the full trailing traffic window; predictor replacement does not delay that decision. Performance models, worker state, and already-admitted scaling operations are retained. `applied` means the diff --git a/lib/bindings/python/rust/llm/fpm.rs b/lib/bindings/python/rust/llm/fpm.rs index 975261044e37..16a0283638ec 100644 --- a/lib/bindings/python/rust/llm/fpm.rs +++ b/lib/bindings/python/rust/llm/fpm.rs @@ -1093,7 +1093,8 @@ impl FpmEventSubscriber { )) } - /// Change the freshness and cleanup window, before or after `start_tracking`. + /// Change the freshness and cleanup window after `start_tracking`, which sets + /// it from its own `max_age_seconds` and so replaces a value set earlier. /// Tracked samples are kept; reads and the next cleanup use the new age. #[pyo3(signature = (max_age_seconds,))] fn set_max_age(&self, max_age_seconds: f64) -> PyResult<()> { diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi index c0933852e010..0e8966f10ec3 100644 --- a/lib/bindings/python/src/dynamo/_core.pyi +++ b/lib/bindings/python/src/dynamo/_core.pyi @@ -1500,9 +1500,10 @@ class FpmEventSubscriber: def set_max_age(self, max_age_seconds: float) -> None: """ - Change the freshness and cleanup window, before or after - ``start_tracking``. Tracked samples are kept; snapshot reads and the - next cleanup use the new age. It must be finite and positive. + Change the freshness and cleanup window after ``start_tracking``, + which sets it from its own ``max_age_seconds`` and so replaces a value + set earlier. Tracked samples are kept; snapshot reads and the next + cleanup use the new age. It must be finite and positive. """ ... @@ -1530,7 +1531,7 @@ class FpmEventSubscriber: """Return fresh samples, unjoined IDs, stale count, and missing rank count. Age is measured from local receive time, including idle heartbeats. - The age limit is set once by ``start_tracking()``. + The age limit is set by ``start_tracking()`` and changed by ``set_max_age()``. Missing ranks include discovered workers that have not published a sample. """ ... From 20d4c551de5281e2fdd20bcf5ca8a5ef783c411a Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Wed, 4 Nov 2026 21:43:27 -0800 Subject: [PATCH 7/8] fix(planner): replan the pending tick after a scaling-policy change Review fixes from jthomson04 on the runtime scaling-policy switch: - The host's pending tick was planned under the old intervals, so a shorter throughput interval made the predictor fire on a tick without traffic and drift a tick apart from its loop flag. The engine now exposes next_scheduled_tick(); the run loop replaces its pending tick after an activation, and replay after it applies a decision. - A builtin registered by a policy change is anchored at the last tick, as initial_tick anchors the startup builtins, instead of the scale-interval grid a wall-clock tick is not on. - Drop the orchestrator's throughput lower bound reset: nothing reads it. - reconfiguration.PREDICTOR_FIELDS and SCALING_POLICY_FIELDS derive from the retune domain key tuples; sweep uses one name. - Remove with_predictor_fields and the evaluated_params branch only it reached; its tests now use evaluated_params with a base planner. - Remove the unread FpmEventSubscriber.max_age_seconds getter and stub. Refs DYN-5400 Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- components/src/dynamo/planner/core/base.py | 7 ++ .../dynamo/planner/core/engine_protocol.py | 4 + .../dynamo/planner/offline/replay_adapter.py | 22 +++- .../plugins/orchestrator/engine_adapter.py | 24 +++- .../src/dynamo/planner/reconfiguration.py | 22 +--- .../src/dynamo/planner/retune/domains.py | 6 +- components/src/dynamo/planner/retune/sweep.py | 23 +--- .../orchestrator/test_engine_adapter.py | 108 ++++++++++++++++++ .../tests/unit/test_reconfiguration.py | 43 +++++++ .../planner/tests/unit/test_retune_domains.py | 9 +- .../planner/tests/unit/test_retune_sweep.py | 24 ++-- lib/bindings/python/rust/llm/fpm.rs | 6 - lib/bindings/python/src/dynamo/_core.pyi | 5 - 13 files changed, 228 insertions(+), 75 deletions(-) diff --git a/components/src/dynamo/planner/core/base.py b/components/src/dynamo/planner/core/base.py index ce185810ae0e..ef69de072611 100644 --- a/components/src/dynamo/planner/core/base.py +++ b/components/src/dynamo/planner/core/base.py @@ -1328,9 +1328,16 @@ async def _run_one_tick( async def run(self) -> None: engine = self._ensure_engine() next_tick = engine.initial_tick(time.time()) + tick_generation = self._config_generation try: while True: + if tick_generation != self._config_generation: + # An activation since the pending tick was planned may + # change which loops it runs and what it pulls. + tick_generation = self._config_generation + if isinstance(engine, ReconfigurableEngineProtocol): + next_tick = engine.next_scheduled_tick() # Read every iteration: a runtime reconfiguration may change # the load interval. poll_interval = self.config.load_adjustment_interval_seconds / 10 diff --git a/components/src/dynamo/planner/core/engine_protocol.py b/components/src/dynamo/planner/core/engine_protocol.py index 289fc92ea695..39210d46eb2b 100644 --- a/components/src/dynamo/planner/core/engine_protocol.py +++ b/components/src/dynamo/planner/core/engine_protocol.py @@ -55,5 +55,9 @@ def apply_runtime_configuration( ) -> None: """Activate settings while preserving the existing tick cadence.""" + def next_scheduled_tick(self) -> ScheduledTick: + """The pending tick recomputed under the active configuration; the + host replaces its pending tick with it after an activation.""" + __all__ = ["EngineProtocol", "ReconfigurableEngineProtocol"] diff --git a/components/src/dynamo/planner/offline/replay_adapter.py b/components/src/dynamo/planner/offline/replay_adapter.py index d0cd0f51d548..4b9fdcbcedd8 100644 --- a/components/src/dynamo/planner/offline/replay_adapter.py +++ b/components/src/dynamo/planner/offline/replay_adapter.py @@ -376,7 +376,7 @@ def on_tick(self, result: dict[str, Any]) -> dict[str, Any]: tick = self._pending_tick tick_input = self._build_tick_input(tick, result) effects: PlannerEffects = self._run_sync(self._engine.tick(tick, tick_input)) - self._apply_retune_decisions(effects.retune_decisions) + reconfigured = self._apply_retune_decisions(effects.retune_decisions) emit_diagnostics = self._should_emit_tick_diagnostics(tick, effects) self._total_ticks += 1 if self._capture_details: @@ -416,8 +416,14 @@ def on_tick(self, result: dict[str, Any]) -> dict[str, Any]: next_tick_ms: Optional[float] = None if effects.next_tick is not None: - self._pending_tick = effects.next_tick - next_tick_ms = effects.next_tick.at_s * 1000.0 + # The tick's next tick was planned before its decisions applied. + self._pending_tick = ( + self._engine.next_scheduled_tick() + if reconfigured + and isinstance(self._engine, ReconfigurableEngineProtocol) + else effects.next_tick + ) + next_tick_ms = self._pending_tick.at_s * 1000.0 decision = { "target_prefill": target_prefill, @@ -459,15 +465,17 @@ def on_tick(self, result: dict[str, Any]) -> dict[str, Any]: self._ticks.append(tick_record) return decision - def _apply_retune_decisions(self, decisions: Sequence[RetuneDecision]) -> None: - """Record retune decisions and apply their planner changes. + def _apply_retune_decisions(self, decisions: Sequence[RetuneDecision]) -> bool: + """Record retune decisions and apply their planner changes; whether + any were applied. Planner changes take effect from the next tick, as live. The controller validated them when it decided, so any error here is a bug and fails the replay. """ + applied = False if not decisions: - return + return applied # Decisions are rare (at most a few per retune window). self._retune_decisions = (*self._retune_decisions, *decisions) capabilities = self._capabilities or WorkerCapabilities() @@ -501,12 +509,14 @@ def _apply_retune_decisions(self, decisions: Sequence[RetuneDecision]) -> None: candidate, capabilities, decision.predictor_warmup ) engine.apply_runtime_configuration(candidate, changes.keys(), predictor) + applied = True logger.info( "Retune %s (%s) applied planner changes %s", decision.decision_id, decision.outcome, changes, ) + return applied def finalize( self, lifecycle_operations: list[dict[str, Any]] | None = None diff --git a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py index a1d064fbc46e..665c6a0f9e27 100644 --- a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py +++ b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py @@ -584,7 +584,7 @@ def _reconcile_builtin_plugins( A builtin the policy no longer runs is unregistered; one it newly runs is registered as at startup, so it first fires one interval - after registration, and its loop restarts with it; one it keeps is + after the last tick, and its loop restarts with it; one it keeps is updated in place (interval, observation window and needs), keeping its state, its phase and its loop's phase. A new or kept load predictor takes ``predictor``'s prepared forecasts when given. @@ -612,6 +612,13 @@ def _reconcile_builtin_plugins( self._orchestrator.register_internal( plugin_id=instance.plugin_id, instance=instance, **registration ) + # Anchored at the last tick, as initial_tick anchors the + # startup builtins, so it fires on the tick its loop is due + # (register_internal aligns to the scale interval instead, + # which a wall-clock tick is not). + plugin = self._orchestrator._registry.get_plugin(instance.plugin_id) + if plugin is not None: + plugin.registered_at = self._last_tick_monotonic continue if name == "load_predict" and predictor is not None: instance.install_prepared_predictors(predictor) @@ -631,8 +638,6 @@ def _reconcile_builtin_plugins( load_started = self._config.enable_load_scaling and not load_was_running if not self._config.enable_throughput_scaling or load_started: self._scaling_state.reset_throughput_lower_bounds() - for component in ("prefill", "decode"): - self._orchestrator.set_throughput_lower_bound(component, 1) # A loop restarts with its newly registered builtins, as at startup; # a kept loop keeps the phase its kept builtins fire at. if load_started: @@ -838,6 +843,16 @@ def initial_tick(self, start_s: float) -> ScheduledTick: self._retune.start(start_s, self._retune_hooks.phase_s) return self._compute_next_scheduled_tick() + def next_scheduled_tick(self) -> ScheduledTick: + """The pending tick recomputed under the active configuration. + + The host's pending tick was planned under the configuration of the + tick before it: an activation in between can change which loops and + plugins it runs and whether it pulls traffic. Its time is unchanged, + because the scale interval is fixed. + """ + return self._compute_next_scheduled_tick() + def apply_runtime_configuration( self, candidate: PlannerConfig, @@ -851,7 +866,8 @@ def apply_runtime_configuration( predictor; it is required when the candidate needs a new one (``reconfiguration.predictor_rebuild_required``: throughput scaling turning on, or a predictor field or the throughput interval - changing). Everything is checked before anything changes. + changing). Everything is checked before anything changes. The host + then replaces its pending tick with ``next_scheduled_tick()``. """ fields = list(fields) if predictor is not None: diff --git a/components/src/dynamo/planner/reconfiguration.py b/components/src/dynamo/planner/reconfiguration.py index e4bf01d85bc0..547525ae99ac 100644 --- a/components/src/dynamo/planner/reconfiguration.py +++ b/components/src/dynamo/planner/reconfiguration.py @@ -26,6 +26,7 @@ from dynamo.planner.control_api import _MinimumEndpointUnavailableError from dynamo.planner.core.budget import minimum_power_footprint_fits from dynamo.planner.offline.trace_data import extract_traffic_observations_from_bytes +from dynamo.planner.retune.domains import PREDICTOR_KEYS, SCALING_POLICY_KEYS if TYPE_CHECKING: from dynamo.planner.core.base import NativePlannerBase @@ -35,28 +36,11 @@ logger = logging.getLogger(__name__) -PREDICTOR_FIELDS = frozenset( - { - "load_predictor", - "load_predictor_log1p", - "prophet_window_size", - "kalman_q_level", - "kalman_q_trend", - "kalman_r", - "kalman_min_points", - } -) +PREDICTOR_FIELDS = frozenset(PREDICTOR_KEYS) # The scaling policy. A change re-registers the builtin plugins of the running # engine; see ``predictor_rebuild_required`` for when it also needs a new # load predictor. -SCALING_POLICY_FIELDS = frozenset( - { - "enable_throughput_scaling", - "enable_load_scaling", - "throughput_adjustment_interval_seconds", - "load_adjustment_interval_seconds", - } -) +SCALING_POLICY_FIELDS = frozenset(SCALING_POLICY_KEYS) RUNTIME_FIELDS = ( PREDICTOR_FIELDS | SCALING_POLICY_FIELDS diff --git a/components/src/dynamo/planner/retune/domains.py b/components/src/dynamo/planner/retune/domains.py index 37611ac4e640..2b7d8c258fe6 100644 --- a/components/src/dynamo/planner/retune/domains.py +++ b/components/src/dynamo/planner/retune/domains.py @@ -66,14 +66,16 @@ THROUGHPUT_INTERVAL = "throughput_adjustment_interval_seconds" LOAD_INTERVAL = "load_adjustment_interval_seconds" MAX_THROUGHPUT_REPLICAS = "max_throughput_scaling_replicas" -# The scaling-policy fields, in the order a grid lists them. +# The scaling-policy fields, in the order a grid lists them +# (``reconfiguration.SCALING_POLICY_FIELDS`` is this set). SCALING_POLICY_KEYS: tuple[str, ...] = ( ENABLE_THROUGHPUT, ENABLE_LOAD, THROUGHPUT_INTERVAL, LOAD_INTERVAL, ) -# The load predictor's fields (``reconfiguration.PREDICTOR_FIELDS``). +# The load predictor's fields (``reconfiguration.PREDICTOR_FIELDS`` is this +# set). PREDICTOR_KEYS: tuple[str, ...] = ( "load_predictor", "load_predictor_log1p", diff --git a/components/src/dynamo/planner/retune/sweep.py b/components/src/dynamo/planner/retune/sweep.py index f43a984529c0..5aaf57f9734a 100644 --- a/components/src/dynamo/planner/retune/sweep.py +++ b/components/src/dynamo/planner/retune/sweep.py @@ -60,7 +60,7 @@ TrafficObservation, WorkerCapabilities, ) -from dynamo.planner.reconfiguration import PREDICTOR_FIELDS, prepare_predictor +from dynamo.planner.reconfiguration import prepare_predictor from dynamo.planner.retune.domains import ( ENABLE_THROUGHPUT, PREDICTOR_KEYS, @@ -392,29 +392,10 @@ def evaluated_params( if key in planner and key in running } planner.update(fields) - elif presweep is not None: - # No inner planner to read the interval from: one pre-sweep summary. - planner.update(presweep.get("fields") or {}) evaluated = RetuneParams(router=dict(candidate.router), planner=planner) return candidate if evaluated == candidate else evaluated -def with_predictor_fields( - candidate: RetuneParams, - baseline: RetuneParams, - fields: Mapping[str, Any] | None, -) -> RetuneParams: - """``evaluated_params`` for one pre-sweep winner's ``fields``, without an - inner planner to fold against (kept for single-interval callers).""" - - return evaluated_params( - candidate, - baseline, - None if fields is None else {"fields": dict(fields)}, - None, - ) - - def score_summary( summary: Mapping[str, Any], window_s: float, @@ -1050,7 +1031,7 @@ def _predictor_swaps( predictor = { key: value for key, value in candidate.planner.items() - if key in PREDICTOR_FIELDS + if key in PREDICTOR_KEYS } if interval == base_interval and not _changes_predictor(baseline, predictor): continue diff --git a/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py b/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py index fe930b2f7a90..9b132fcd734d 100644 --- a/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py +++ b/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py @@ -2245,6 +2245,114 @@ async def test_a_kept_throughput_loop_keeps_its_phase_on_a_load_interval_change( assert rows and all(run == fired for _, run, fired in rows), rows +async def _predictor_cadence(adapter, until_s, *, at_tick=None, start_s=0.0): + """Drive ticks to ``until_s``; the ticks where the predictor or the + throughput loop flag fired, as ``(at_s, run_throughput, fired, traffic)``. + ``at_tick(adapter, tick, effects)`` may return a replacement next tick.""" + + registry = adapter._orchestrator.registry + tick = adapter.initial_tick(start_s=start_s) + observations = FpmObservations(decode={("w1", 0): _make_fpm("w1")}) + traffic = TrafficObservation(60.0, 10.0, 100.0, 10.0) + rows = [] + while tick.at_s <= until_s: + plugin = registry.get_plugin("builtin_load_predict") + before = None if plugin is None else plugin.last_call_at + effects = await adapter.tick( + tick, + TickInput(now_s=tick.at_s, fpm_observations=observations, traffic=traffic), + ) + plugin = registry.get_plugin("builtin_load_predict") + fired = plugin is not None and plugin.last_call_at != before + if tick.run_throughput_scaling or fired: + rows.append( + ( + tick.at_s, + tick.run_throughput_scaling, + fired, + tick.need_traffic_metrics, + ) + ) + replacement = None if at_tick is None else at_tick(adapter, tick, effects) + tick = replacement if replacement is not None else effects.next_tick + return rows + + +def _sla_config(**overrides): + return PlannerConfig.model_validate( + { + "mode": "agg", + "optimization_target": "sla", + "served_model_name": "test", + "scheduling": {"scale_interval_seconds": 5.0}, + "load_predictor": "constant", + **overrides, + } + ) + + +@pytest.mark.asyncio +async def test_a_shorter_throughput_interval_replans_the_pending_tick(): + cfg = _sla_config( + enable_load_scaling=True, + enable_throughput_scaling=True, + load_adjustment_interval_seconds=10, + throughput_adjustment_interval_seconds=120, + ) + adapter = OrchestratorEngineAdapter(cfg, _caps(), clock=VirtualClock()) + stale = [] + + def shorten(adapter, tick, effects): + if tick.at_s != 90.0: + return None + updates = {"throughput_adjustment_interval_seconds": 60} + candidate = candidate_config(cfg, updates) + adapter.apply_runtime_configuration( + candidate, list(updates), prepare_predictor(candidate, _caps(), None) + ) + stale.append(effects.next_tick) + return adapter.next_scheduled_tick() + + rows = await _predictor_cadence(adapter, 300, at_tick=shorten) + + # The tick planned before the change would run the now-due predictor + # without traffic and without its loop. + assert not stale[0].run_throughput_scaling + assert not stale[0].need_traffic_metrics + assert rows[0][0] == 95.0 + assert all(run == fired and traffic for _, run, fired, traffic in rows), rows + + +@pytest.mark.asyncio +async def test_a_builtin_registered_between_ticks_fires_with_its_loop(): + # Ticks off the scale-interval grid (start 2 s), and the activation + # lands after a grid boundary (11 s, after the tick at 7 s), as on a + # wall clock: the new predictor must keep its loop's phase. + cfg = _sla_config( + enable_load_scaling=True, + enable_throughput_scaling=False, + load_adjustment_interval_seconds=5, + throughput_adjustment_interval_seconds=60, + ) + adapter = OrchestratorEngineAdapter(cfg, _caps(), clock=VirtualClock()) + + def turn_on(adapter, tick, effects): + if tick.at_s != 7.0: + return None + adapter._clock.advance(4.0) + updates = {"enable_throughput_scaling": True} + candidate = candidate_config(cfg, updates) + adapter.apply_runtime_configuration( + candidate, list(updates), prepare_predictor(candidate, _caps(), None) + ) + return adapter.next_scheduled_tick() + + rows = await _predictor_cadence(adapter, 200, at_tick=turn_on, start_s=2.0) + + assert rows[0][0] == 67.0 + assert all(run == fired for _, run, fired, _ in rows), rows + + @pytest.mark.asyncio async def test_runtime_scaling_policy_change_reregisters_builtins(): cfg = PlannerConfig.model_validate( diff --git a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py index 636871fc7b02..7bd51b744aa1 100644 --- a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py +++ b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py @@ -5,6 +5,7 @@ import gzip import json import threading +import time from pathlib import Path import pytest @@ -844,3 +845,45 @@ async def sleep(seconds): with pytest.raises(asyncio.CancelledError): await planner.run() assert sleeps == [1.0, 2.0] + + +async def test_run_loop_replans_the_pending_tick_after_an_activation(monkeypatch): + class Engine: + def __init__(self): + self.replans = 0 + + def initial_tick(self, start_s): + return ScheduledTick(at_s=start_s + 1_000.0) + + def next_scheduled_tick(self): + self.replans += 1 + return ScheduledTick(at_s=time.time() + 1_000.0) + + def apply_runtime_configuration(self, candidate, fields, predictor): + raise AssertionError("not called") + + async def tick(self, scheduled_tick, tick_input): + raise AssertionError("no tick is due") + + async def shutdown(self): + pass + + planner = _planner("agg", load_adjustment_interval_seconds=10) + engine = Engine() + planner._engine = engine + sleeps = 0 + + async def sleep(seconds): + nonlocal sleeps + sleeps += 1 + if sleeps == 1: + # An activation between the loop's checks. + planner._config_generation += 1 + elif sleeps == 3: + raise asyncio.CancelledError + + monkeypatch.setattr("dynamo.planner.core.base.asyncio.sleep", sleep) + with pytest.raises(asyncio.CancelledError): + await planner.run() + # Replanned once, at the first check after the activation. + assert engine.replans == 1 diff --git a/components/src/dynamo/planner/tests/unit/test_retune_domains.py b/components/src/dynamo/planner/tests/unit/test_retune_domains.py index 6b2244c4f215..a80f241f4968 100644 --- a/components/src/dynamo/planner/tests/unit/test_retune_domains.py +++ b/components/src/dynamo/planner/tests/unit/test_retune_domains.py @@ -23,7 +23,7 @@ required_history_seconds, throughput_intervals, ) -from dynamo.planner.retune.sweep import evaluated_params, with_predictor_fields +from dynamo.planner.retune.sweep import evaluated_params from dynamo.planner.simulation.load_predictor import ( LOAD_PREDICTOR_PRESETS, predictor_fields, @@ -123,10 +123,9 @@ def test_custom_predictor_mapping_values_take_their_field_types(): baseline = RetuneParams( planner={name: getattr(config, name) for name in PREDICTOR_FIELDS} ) - assert ( - with_predictor_fields(RetuneParams(), baseline, predictor_fields(entry)) - == baseline - ) + presweep = {"fields": predictor_fields(entry)} + base = config.model_dump(mode="json") + assert evaluated_params(RetuneParams(), baseline, presweep, base) == baseline # --- scaling policy ------------------------------------------------------- diff --git a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py index 471050756af1..d69e29db7ab3 100644 --- a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py +++ b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py @@ -29,7 +29,7 @@ reason="AI Simulate is an optional Dynamo simulation dependency", ) -from dynamo.planner.config.planner_config import RetuneObjective +from dynamo.planner.config.planner_config import PlannerConfig, RetuneObjective from dynamo.planner.core.types import RetuneParams, TrafficObservation from dynamo.planner.retune import sweep as sweep_module from dynamo.planner.retune.sweep import ( @@ -38,7 +38,6 @@ evaluated_params, objective_maximized, score_summary, - with_predictor_fields, ) from dynamo.planner.retune.window import ObservationBuckets from dynamo.planner.tests.retune_sweep_helpers import ( @@ -213,17 +212,28 @@ def test_a_grid_point_equivalent_to_the_baseline_gets_its_key(): "load_predictor_log1p": False, "prophet_window_size": 50, } - - assert with_predictor_fields(grid_point, baseline, winner) == baseline - assert with_predictor_fields(other, baseline, winner).planner == { + # A hybrid SLA planner: the sensitivity and the predictor are active. + base = PlannerConfig( + namespace="test-ns", + mode="disagg", + optimization_target="sla", + enable_throughput_scaling=True, + enable_load_scaling=True, + ).model_dump(mode="json") + + def evaluated(candidate, fields): + return evaluated_params(candidate, baseline, {"fields": fields}, base) + + assert evaluated(grid_point, winner) == baseline + assert evaluated(other, winner).planner == { **current, "load_scaling_down_sensitivity": 70, } # The baseline keeps the current predictor; other candidates take the # winner's fields over the baseline's. kalman = {"load_predictor": "kalman", "kalman_r": 5.0} - assert with_predictor_fields(baseline, baseline, kalman) == baseline - assert with_predictor_fields(other, baseline, kalman).planner == { + assert evaluated(baseline, kalman) == baseline + assert evaluated(other, kalman).planner == { **current, "load_scaling_down_sensitivity": 70, **kalman, diff --git a/lib/bindings/python/rust/llm/fpm.rs b/lib/bindings/python/rust/llm/fpm.rs index 16a0283638ec..92dd3bb57fa9 100644 --- a/lib/bindings/python/rust/llm/fpm.rs +++ b/lib/bindings/python/rust/llm/fpm.rs @@ -1102,12 +1102,6 @@ impl FpmEventSubscriber { Ok(()) } - /// The current freshness and cleanup window, in seconds. - #[getter] - fn max_age_seconds(&self) -> f64 { - self.max_age.get().as_secs_f64() - } - /// Selected model deployment cards for active workers, as serialized JSON. fn get_model_cards(&self) -> PyResult> { if !self.tracking_started.load(Ordering::SeqCst) { diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi index 0e8966f10ec3..1b700aa91cb8 100644 --- a/lib/bindings/python/src/dynamo/_core.pyi +++ b/lib/bindings/python/src/dynamo/_core.pyi @@ -1507,11 +1507,6 @@ class FpmEventSubscriber: """ ... - @property - def max_age_seconds(self) -> float: - """The current freshness and cleanup window, in seconds.""" - ... - def get_recent_stats(self) -> dict[tuple[str, int], bytes]: """ Return the latest FPM bytes for every tracked (worker_id, dp_rank). From 80cb88322591055d606d85c734da8f3d6fb17002 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Wed, 4 Nov 2026 21:56:36 -0800 Subject: [PATCH 8/8] fix(planner): keep the {} planner point and apply decisions once persisted Review fixes from jthomson04 and CodeRabbit on live Scenario 2: - grid_planners kept only nonempty planner points, so a scaling Planner without retune.planner, or one that searches only the predictor, lost its planner axis: Requests carried no retune_sweep and, predictor-only, no candidate was scored. The grid's {} point is now the axis. - Live decisions apply their Planner fields through PlannerReconfiguration.apply_decision, the control API's two-phase path: one update at a time, the predictor prepared off the loop and tracked past its deadline, dropped when another update activates first. - The supervisor hands a decision's Planner fields to the host only after the outbox wrote it under this term (retune_decision_persisted), and discards them from the controller when the decision is dropped. - One inner_planner_config helper builds the inner planner for the replay sweep, the DGDR executor and the AISimulate adapter. - History buckets parse rows without block hashes, as replay does. - Test imports at module level. Refs DYN-5400 Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- components/src/dynamo/planner/core/base.py | 83 ++++-------- .../src/dynamo/planner/reconfiguration.py | 119 +++++++++++++++--- .../src/dynamo/planner/retune/controller.py | 6 +- .../src/dynamo/planner/retune/live/dgdr.py | 10 +- .../dynamo/planner/retune/live/supervisor.py | 56 ++++++++- .../dynamo/planner/retune/live/window_cut.py | 14 ++- components/src/dynamo/planner/retune/sweep.py | 19 +-- .../src/dynamo/planner/simulation/config.py | 33 +++++ .../src/dynamo/planner/simulation/provider.py | 11 +- .../tests/integration/test_live_retune_e2e.py | 5 + .../unit/test_live_retune_controller_hooks.py | 55 ++++++-- .../tests/unit/test_live_retune_dgdr.py | 77 ++++++++++-- .../tests/unit/test_live_retune_supervisor.py | 57 ++++++++- .../tests/unit/test_live_retune_window_cut.py | 16 +++ .../tests/unit/test_reconfiguration.py | 63 ++++++++-- .../auto-deployment/planner-live-retune.md | 8 +- 16 files changed, 484 insertions(+), 148 deletions(-) diff --git a/components/src/dynamo/planner/core/base.py b/components/src/dynamo/planner/core/base.py index c4f33eebbe47..fc1d67aa22ec 100644 --- a/components/src/dynamo/planner/core/base.py +++ b/components/src/dynamo/planner/core/base.py @@ -55,12 +55,8 @@ from dynamo.planner.monitoring.planner_metrics import PlannerPrometheusMetrics from dynamo.planner.offline.trace_data import extract_traffic_observations_from_trace from dynamo.planner.reconfiguration import ( - PREPARATION_TIMEOUT_SECONDS, PlannerReconfiguration, - candidate_config, live_retune_sla_errors, - predictor_rebuild_required, - prepare_predictor, runtime_candidate_errors, runtime_configuration_errors, ) @@ -121,7 +117,9 @@ async def stop(self) -> None: ... def on_effects(self, effects: PlannerEffects) -> Sequence[RetuneDecision]: - """Take the tick's decisions; return the ones it accepted.""" + """Take the tick's decisions; return the ones it queued to persist. + The Planner fields of each apply once it is persisted + (``retune_decision_persisted``).""" ... @@ -638,60 +636,30 @@ def _sync_fpm_max_age(self) -> None: setter(age) logger.info("FPM freshness window is now %.1fs", age) - def _schedule_retune_planner_changes( - self, decisions: Sequence[RetuneDecision] - ) -> None: - """Apply each ``applied`` live retune decision's Planner fields. + def retune_decision_persisted(self, decision: RetuneDecision) -> None: + """Apply a persisted live retune decision's Planner fields + (``LiveRetuneHost``). - In the background, through the runtime reconfiguration path: a new - load predictor is prepared off the event loop on the decision's - warmup, then the candidate activates under the effect admission and - configuration locks, as a control API update does. + In the background, through ``PlannerReconfiguration.apply_decision``, + the control API's two-phase path. The supervisor calls this only once + the decision is written under this process's term. """ - for decision in decisions: - if decision.outcome != "applied" or not decision.planner_changes: - continue - task = asyncio.create_task( - self._apply_retune_planner_changes(decision), - name=f"planner-retune-apply-{decision.decision_id}", - ) - self._retune_apply_tasks.add(task) - task.add_done_callback(self._retune_apply_tasks.discard) + if decision.outcome != "applied" or not decision.planner_changes: + return + task = asyncio.create_task( + self._apply_retune_planner_changes(decision), + name=f"planner-retune-apply-{decision.decision_id}", + ) + self._retune_apply_tasks.add(task) + task.add_done_callback(self._retune_apply_tasks.discard) async def _apply_retune_planner_changes(self, decision: RetuneDecision) -> None: changes = dict(decision.planner_changes) try: - async with self._config_lock: - candidate = candidate_config(self.config, changes) - self.validate_runtime_candidate(candidate) - rebuild = predictor_rebuild_required(self.config, candidate) - # Like the control API's expected_revision: a change made - # while the predictor is prepared (the control API or another - # decision) wins, and this one is dropped. - revision = self._config_generation - predictor = None - if rebuild: - predictor = await asyncio.wait_for( - asyncio.to_thread( - prepare_predictor, - candidate, - self._build_worker_capabilities(), - decision.predictor_warmup, - ), - PREPARATION_TIMEOUT_SECONDS, - ) - async with self._effect_admission_lock: - async with self._config_lock: - if self._config_generation != revision: - raise ValueError( - "the configuration changed while the predictor was " - f"prepared (revision {revision} -> " - f"{self._config_generation})" - ) - candidate = candidate_config(self.config, changes) - self.validate_runtime_candidate(candidate) - self.apply_runtime_candidate(candidate, changes.keys(), predictor) + await self.reconfiguration.apply_decision( + changes, decision.predictor_warmup + ) logger.info( "Retune %s applied planner changes %s", decision.decision_id, @@ -1380,16 +1348,13 @@ async def _run_one_tick( effects = await engine.tick(tick, tick_input) if self._live_retune is not None: # Retune decisions go to the live supervisor's outbox, which - # persists them before actuating routers; _apply_effects never - # sees router parameters. - # Only a decision the supervisor accepted (persisted, then - # actuated) changes the planner: a dropped one is rewound. - accepted: Sequence[RetuneDecision] = () + # persists them before actuating routers and before calling back + # retune_decision_persisted for the Planner fields; _apply_effects + # never sees router parameters. try: - accepted = self._live_retune.on_effects(effects) + self._live_retune.on_effects(effects) except Exception: logger.exception("Live retune failed to take the tick's effects") - self._schedule_retune_planner_changes(accepted) await self._submit_effects(effects, config_generation) emit_diagnostics = self._should_emit_tick_diagnostics(tick, effects) if emit_diagnostics: diff --git a/components/src/dynamo/planner/reconfiguration.py b/components/src/dynamo/planner/reconfiguration.py index 547525ae99ac..6c86b22a8eb4 100644 --- a/components/src/dynamo/planner/reconfiguration.py +++ b/components/src/dynamo/planner/reconfiguration.py @@ -16,7 +16,7 @@ import uuid import weakref from collections import OrderedDict -from collections.abc import Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from pathlib import Path from typing import TYPE_CHECKING, Any @@ -412,7 +412,8 @@ def __init__(self, planner: NativePlannerBase) -> None: self.planner = planner self.instance_id = str(uuid.uuid4()) self._operations: OrderedDict[str, dict[str, Any]] = OrderedDict() - self._task: asyncio.Task[None] | None = None + # The update preparing or activating: a control API one or a decision's. + self._task: asyncio.Task[Any] | None = None self._preparation: asyncio.Task[BuiltinLoadPredict] | None = None self._closed = False self._registered_runtime: DistributedRuntime | None = None @@ -508,27 +509,107 @@ async def _apply(self, request, candidate, changed_predictor, operation) -> None operation["state"] = ( "warming" if request.warmup_trace is not None else "preparing" ) - capabilities = self.planner._build_worker_capabilities() - self._preparation = asyncio.create_task( - asyncio.to_thread( - _prepare_predictor, candidate, capabilities, request.warmup_trace - ) - ) - # A timed-out thread cannot be killed. Keep tracking it and reject a - # second preparation until it exits; it never owns active state. - self._preparation.add_done_callback(_consume_preparation_result) - predictor = await asyncio.wait_for( - asyncio.shield(self._preparation), PREPARATION_TIMEOUT_SECONDS + predictor = await self._prepare( + _prepare_predictor, + candidate, + self.planner._build_worker_capabilities(), + request.warmup_trace, ) + operation["applied_revision"] = await self._activate( + candidate, + request.config.keys(), + predictor, + lambda: self._check_revision(request), + ) + operation["state"] = "applied" + + async def apply_decision( + self, + changes: Mapping[str, Any], + observations: Sequence[TrafficObservation] | None, + ) -> int: + """Activate a persisted live retune decision's Planner ``changes``; + the new revision. + + The same two-phase path as a control API update: one update at a + time, a new load predictor prepared off the event loop on the + decision's ``observations`` and tracked past its deadline, and the + decision dropped (``ConfigurationConflict``) when another update + activates first. Raises on any failure; nothing changes then. + """ + + async with self.planner._bounded_config_lock(): + if self._closed: + raise _MinimumEndpointUnavailableError("planner is shutting down") + if self._task is not None and not self._task.done(): + raise ConfigurationConflict( + "another configuration update is still preparing" + ) + candidate = candidate_config(self.planner.config, dict(changes)) + self.planner.validate_runtime_candidate(candidate) + rebuild = predictor_rebuild_required(self.planner.config, candidate) + if ( + rebuild + and self._preparation is not None + and not self._preparation.done() + ): + raise ConfigurationConflict("another predictor is still preparing") + revision = self.planner._config_generation + + def unchanged() -> None: + if self.planner._config_generation != revision: + raise ConfigurationConflict( + "the configuration changed while the predictor was " + f"prepared (revision {revision} -> " + f"{self.planner._config_generation})" + ) + + async def apply() -> int: + predictor = None + if rebuild: + predictor = await self._prepare( + prepare_predictor, + candidate, + self.planner._build_worker_capabilities(), + observations, + ) + return await self._activate( + candidate, list(changes), predictor, unchanged + ) + + task = asyncio.create_task(apply()) + self._task = task + return await task + + async def _prepare( + self, prepare: Callable[..., BuiltinLoadPredict], *args: Any + ) -> BuiltinLoadPredict: + """Run ``prepare(*args)`` off the event loop within the deadline.""" + + self._preparation = asyncio.create_task(asyncio.to_thread(prepare, *args)) + # A timed-out thread cannot be killed. Keep tracking it and reject a + # second preparation until it exits; it never owns active state. + self._preparation.add_done_callback(_consume_preparation_result) + return await asyncio.wait_for( + asyncio.shield(self._preparation), PREPARATION_TIMEOUT_SECONDS + ) + + async def _activate( + self, + candidate: PlannerConfig, + fields: Iterable[str], + predictor: BuiltinLoadPredict | None, + check: Callable[[], None], + ) -> int: + """Activate ``candidate`` between decisions if ``check()`` passes; + the new revision.""" + async with self.planner._effect_admission_lock: async with self.planner._bounded_config_lock(): - self._check_revision(request) + check() self.planner.validate_runtime_candidate(candidate) - self.planner.apply_runtime_candidate( - candidate, request.config.keys(), predictor - ) - operation["state"] = "applied" - operation["applied_revision"] = self.planner._config_generation + self.planner.apply_runtime_candidate(candidate, fields, predictor) + return self.planner._config_generation @staticmethod def _finish(task: asyncio.Task[None], operation: dict[str, Any]) -> None: diff --git a/components/src/dynamo/planner/retune/controller.py b/components/src/dynamo/planner/retune/controller.py index e20081bec0a9..0814a967407e 100644 --- a/components/src/dynamo/planner/retune/controller.py +++ b/components/src/dynamo/planner/retune/controller.py @@ -869,13 +869,15 @@ def _live_request_fields( ), "replicas_verified": verified, # The planner axis a DGDR sweep shards: the grid's planner - # points, without the baseline's or the startup point's. + # points, without the baseline's or the startup point's. A grid + # that does not search the planner has the one point {}: the + # running planner, with the pre-sweep winner when the predictor + # is searched. "grid_planners": tuple( dict(planner) for planner in { RetuneParams(planner=point.planner).key(): point.planner for point in sorted(self._grid, key=RetuneParams.key) - if point.planner }.values() ), } diff --git a/components/src/dynamo/planner/retune/live/dgdr.py b/components/src/dynamo/planner/retune/live/dgdr.py index 100360c4240f..b2fe02802e27 100644 --- a/components/src/dynamo/planner/retune/live/dgdr.py +++ b/components/src/dynamo/planner/retune/live/dgdr.py @@ -109,7 +109,7 @@ sweep_throughput_intervals, write_observations, ) -from dynamo.planner.simulation.config import retune_sweep_base +from dynamo.planner.simulation.config import inner_planner_config, retune_sweep_base logger = logging.getLogger(__name__) @@ -1319,13 +1319,7 @@ def _prepare_planner(self) -> None: # The configuration the Runs replay (only its scaling fields): the # live dump may hold settings, such as non-advisory batch # scheduling, that need what a dump leaves out. - inner_base = { - **retune_sweep_base(request.base_planner or {}), - "retune": None, - "advisory": False, - "report_interval_hours": None, - "live_dashboard_port": 0, - } + inner_base = inner_planner_config(retune_sweep_base(request.base_planner or {})) with tempfile.TemporaryDirectory(prefix="retune-dgdr-planner-") as work: work_dir = Path(work) paths = { diff --git a/components/src/dynamo/planner/retune/live/supervisor.py b/components/src/dynamo/planner/retune/live/supervisor.py index 810ad95ee19d..86b62ddf6b30 100644 --- a/components/src/dynamo/planner/retune/live/supervisor.py +++ b/components/src/dynamo/planner/retune/live/supervisor.py @@ -43,7 +43,9 @@ leader of the round's epoch or the budget has passed. - outbox: write each decision, conditional on the state's ``leader_term`` being this process's term and on the resourceVersion read; N consecutive - failures re-initialize. + failures re-initialize. Once an applied decision is written, its Planner + fields go to ``host.retune_decision_persisted``; a decision dropped + before that has its Planner fields discarded from the controller. Local leadership loss, a failed outbox, and a controller rebuild share one path: clear ``initialized``, stop the outbox writer and the in-flight @@ -221,6 +223,11 @@ async def replace_live_retune( ) -> None: ... + def retune_decision_persisted(self, decision: RetuneDecision) -> None: + """Apply a persisted ``applied`` decision's Planner fields (in the + background; must not block).""" + ... + @dataclass(frozen=True) class SupervisorSettings: @@ -900,7 +907,8 @@ def sweep_fn( def on_effects(self, effects: PlannerEffects) -> list[RetuneDecision]: """Enqueue the tick's decisions and return them, or ``[]`` when they - are dropped; never raises into the tick.""" + are dropped; never raises into the tick. Their Planner fields apply + once written (``outbox_round``).""" decisions = list(effects.retune_decisions or ()) if not decisions: @@ -911,6 +919,7 @@ def on_effects(self, effects: PlannerEffects) -> list[RetuneDecision]: len(decisions), [decision.decision_id for decision in decisions], ) + self._discard_planner_changes(decisions) self._rewind_now() return [] router_parameters = effects.router_parameters @@ -1061,12 +1070,17 @@ async def outbox_round(self) -> None: "retune outbox: dropped %s of an earlier term/generation", entry.decision.decision_id, ) + self._discard_planner_changes([entry.decision]) continue outcome = await self._write_decision(entry) if outcome in ("ok", "already", "discarded"): if self._outbox and self._outbox[0] is entry: self._outbox.popleft() self._write_failures = 0 + if outcome == "discarded": + self._discard_planner_changes([entry.decision]) + else: + self._decision_persisted(entry.decision) continue if outcome == "lost_term": self._request_reinit("the state's leader_term is not ours") @@ -1398,6 +1412,7 @@ async def _local_loss_locked(self, reason: str) -> None: await self._cancel(self._reconcile_task) self._reconcile_task = None dropped = len(self._outbox) + unwritten = [entry.decision for entry in self._outbox] self._outbox.clear() self._write_failures = 0 controller = self._controller() @@ -1407,6 +1422,7 @@ async def _local_loss_locked(self, reason: str) -> None: controller.drop_pending() except Exception: logger.exception("retune: drop_pending failed") + self._discard_planner_changes(unwritten) self._rewind(controller) if was: logger.info( @@ -2060,6 +2076,42 @@ def _rewind_now(self) -> None: if controller is not None: self._rewind(controller) + def _decision_persisted(self, decision: RetuneDecision) -> None: + """Hand a written ``applied`` decision's Planner fields to the host.""" + + if decision.outcome != "applied" or not decision.planner_changes: + return + host = self._host + try: + if host is None: + raise RuntimeError("no planner host") + host.retune_decision_persisted(decision) + except Exception: + logger.exception( + "retune: applying the Planner fields of %s failed", + decision.decision_id, + ) + self._discard_planner_changes([decision]) + + def _discard_planner_changes(self, decisions: Sequence[RetuneDecision]) -> None: + """Forget the Planner fields of ``applied`` decisions that will not be + applied, so the controller's current point is the running + configuration again.""" + + controller = self._controller() + if controller is None: + return + for decision in decisions: + if decision.outcome != "applied" or not decision.planner_changes: + continue + try: + controller.discard_planner_changes(decision.planner_changes) + except Exception: + logger.exception( + "retune: discarding the Planner fields of %s failed", + decision.decision_id, + ) + def _engine(self) -> Any: """The host's engine adapter; ``None`` before ``start`` or without one.""" diff --git a/components/src/dynamo/planner/retune/live/window_cut.py b/components/src/dynamo/planner/retune/live/window_cut.py index e2617eec4080..ef089f73504d 100644 --- a/components/src/dynamo/planner/retune/live/window_cut.py +++ b/components/src/dynamo/planner/retune/live/window_cut.py @@ -410,10 +410,14 @@ def bucket_history_inprocess( continue location = f"{path}@{item.offset}:{index + 1}" event = _event(raw, location, "trace_corrupt") + # Buckets read arrival, ISL and OSL only, as the replay + # read task's history does: no hashes needed. row = ( None if event is None - else _request_row(event, location, "trace_corrupt") + else _request_row( + event, location, "trace_corrupt", need_hashes=False + ) ) if row is None or not ( history_start_ms <= row.arrival_ms < window_end_ms @@ -1136,10 +1140,14 @@ def _event( def _request_row( - event: dict[str, Any], location: str, failure: WindowCutFailure + event: dict[str, Any], + location: str, + failure: WindowCutFailure, + *, + need_hashes: bool = True, ) -> Optional[_RequestRow]: try: - return _parse_request_end(event, location, need_hashes=True) + return _parse_request_end(event, location, need_hashes=need_hashes) except ValueError as error: raise WindowCutError(failure, str(error)) from error diff --git a/components/src/dynamo/planner/retune/sweep.py b/components/src/dynamo/planner/retune/sweep.py index 63528e71377a..01d32217e240 100644 --- a/components/src/dynamo/planner/retune/sweep.py +++ b/components/src/dynamo/planner/retune/sweep.py @@ -73,6 +73,7 @@ read_trace_history, read_trace_window, ) +from dynamo.planner.simulation.config import inner_planner_config from dynamo.planner.simulation.load_predictor import ( Window, evaluate_preset, @@ -876,22 +877,8 @@ def _run_sweep( inner_base = None if request.base_planner is not None: # The inner planner is the current one, warmed from the recorded - # history (the read task's warmup buckets) instead of the outer - # run's own warmup trace, never retuned, and without diagnostics - # reports (one HTML file per candidate). It always scales, even - # under an advisory outer planner: a candidate is scored on the - # replicas it would drive. Replay never runs batch scheduling, so - # it is off, which also drops the Redis settings an actuating - # planner would require. - inner_base = { - **request.base_planner, - "retune": None, - "load_predictor_warmup_trace": None, - "report_interval_hours": None, - "live_dashboard_port": 0, - "advisory": False, - "batch_scheduling": {"enabled": False}, - } + # history (the read task's warmup buckets). + inner_base = inner_planner_config(request.base_planner) if presweep_intervals: presweep = run_presweeps( request, diff --git a/components/src/dynamo/planner/simulation/config.py b/components/src/dynamo/planner/simulation/config.py index 86b12d96e32a..641fce8b8ad8 100644 --- a/components/src/dynamo/planner/simulation/config.py +++ b/components/src/dynamo/planner/simulation/config.py @@ -7,6 +7,7 @@ import math from collections.abc import Mapping +from copy import deepcopy from typing import Annotated, Any, Literal, TypeAlias from aisimulate.config.common import Choices, IntegerRange, NumericRange @@ -447,6 +448,38 @@ def retune_sweep_base(planner: Mapping[str, Any]) -> dict[str, Any]: return base +# What a retune candidate's inner planner overrides in the running planner's +# configuration. It is never retuned and always actuates, even under an +# advisory outer planner: a candidate is scored on the replicas it would +# drive. It writes no reports and opens no listener, warms from the sweep's +# history instead of the outer planner's warmup trace, and runs no batch +# scheduling, which replay never runs and which would need the Redis settings +# an actuating planner requires. +INNER_PLANNER_OVERRIDES: dict[str, Any] = { + "retune": None, + "advisory": False, + "report_interval_hours": None, + "live_dashboard_port": 0, + "metric_reporting_prometheus_port": 0, + "load_predictor_warmup_trace": None, + "batch_scheduling": {"enabled": False}, +} + + +def inner_planner_config( + base: Mapping[str, Any], point: Mapping[str, Any] | None = None +) -> dict[str, Any]: + """A retune candidate's inner planner: ``base`` with ``point`` applied and + ``INNER_PLANNER_OVERRIDES`` over both. Every retune executor (the replay + sweep, the DGDR executor and the AISimulate adapter) builds it here.""" + + return { + **deepcopy(dict(base)), + **deepcopy(dict(point or {})), + **deepcopy(INNER_PLANNER_OVERRIDES), + } + + class PlannerRetuneConfig(PlannerRecommendationConfig): """``retune.planner``: the recommendation schema, restricted to what a running planner can change. diff --git a/components/src/dynamo/planner/simulation/provider.py b/components/src/dynamo/planner/simulation/provider.py index 6836d7e73e6e..48d0258ebff3 100644 --- a/components/src/dynamo/planner/simulation/provider.py +++ b/components/src/dynamo/planner/simulation/provider.py @@ -49,6 +49,7 @@ PLANNER_PUBLIC_TO_CONFIG, PlannerRecommendationConfig, Policy, + inner_planner_config, ) from .load_predictor import ( LOAD_PREDICTOR_PRESETS, @@ -499,15 +500,7 @@ def _retune_sweep_planner_config( serialized, never environment-derived connection settings. """ - payload: dict[str, Any] = { - **deepcopy(spec.base), - **deepcopy(dict(point)), - "retune": None, - "advisory": False, - "report_interval_hours": None, - "live_dashboard_port": 0, - "metric_reporting_prometheus_port": 0, - } + payload: dict[str, Any] = inner_planner_config(spec.base, point) if sample is not None: mode = str(sample["deployment_mode"]) if mode == "disagg": diff --git a/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py b/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py index e601b19bf81a..0949e9c51759 100644 --- a/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py +++ b/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py @@ -571,6 +571,11 @@ async def replace_live_retune(self, inputs, *, generation): inputs, generation=generation, now_s=self._clock.time() ) + def retune_decision_persisted(self, decision): + # Router-only retune: no decision carries Planner fields. + if decision.planner_changes: + raise AssertionError(f"unexpected planner changes {decision}") + class Planner: """One planner process: its supervisor, engine, actuator and reconciler.""" diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py index 0d752ea90e4d..fac68ca8c99b 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py @@ -571,18 +571,20 @@ async def test_window_started_before_the_controller_launches_unverified(): assert request.replicas == SPEC -def _scaling_config(): +def _scaling_config(planner="default", **overrides): # Scenario 2: load scaling (easy mode) with the default sensitivities. + retune = { + "interval_seconds": INTERVAL, + "grace_seconds": GRACE, + "router": ROUTER, + } + if planner is not None: + retune["planner"] = planner return PlannerConfig( namespace="test-ns", mode="agg", - enable_throughput_scaling=False, - retune={ - "interval_seconds": INTERVAL, - "grace_seconds": GRACE, - "router": ROUTER, - "planner": "default", - }, + **{"enable_throughput_scaling": False, **overrides}, + retune=retune, ) @@ -633,6 +635,43 @@ def counts(now): ) == [70, 80, 90] +@pytest.mark.parametrize( + ("planner", "overrides"), + [ + (None, {}), + ( + {"load_predictor": {"preset": ["constant_last", "kalman_default_raw"]}}, + { + "optimization_target": "sla", + "enable_throughput_scaling": True, + "load_predictor": "kalman", + }, + ), + ], + ids=["router_only", "predictor_only"], +) +async def test_a_scaling_planner_without_planner_points_keeps_one_planner_point( + planner, overrides +): + # The grid searches no planner field: its one planner point is {}, the + # running planner (with the pre-sweep winner), so every Run still + # replays an inner planner. + sweeps = FakeSweeps() + sweeps.default = _flat() + config = _scaling_config(planner, **overrides) + controller = _live(sweeps, start=T0 - INTERVAL - 2 * STEP, config=config) + try: + await _run_window( + controller, lambda _now: 2, start=T0 - INTERVAL - 2 * STEP, until=L_K + ) + finally: + controller.close() + + (request,) = _requests_for_k(sweeps) + assert request.base_planner is not None + assert request.grid_planners == ({},) + + # --------------------------------------------------------------------------- # restore_desired # --------------------------------------------------------------------------- diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py index db13574cd9fa..b24c613b9000 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py @@ -34,11 +34,14 @@ from dynamo.planner.config.planner_config import ( DgdrExecutorConfig, + PlannerConfig, RetuneConfig, RetuneEngineSpec, dgdr_need_seconds, ) -from dynamo.planner.core.types import RetuneParams +from dynamo.planner.core.types import RetuneParams, TrafficObservation +from dynamo.planner.retune import sweep as sweep_module +from dynamo.planner.retune.domains import PREDICTOR_KEYS from dynamo.planner.retune.live import dgdr from dynamo.planner.retune.live.contracts import ( LABEL_ATTEMPT, @@ -68,6 +71,7 @@ window_selection, ) from dynamo.planner.retune.live.reconciler import trace_pvc_source +from dynamo.planner.retune.live.window_cut import TraceHistory from dynamo.planner.retune.sweep import SweepRequest, score_summary from dynamo.planner.tests.live_retune_fakes import ( FakeDgdrOperator, @@ -2162,9 +2166,6 @@ def __call__( deadline_s, cancel=None, ): - from dynamo.planner.core.types import TrafficObservation - from dynamo.planner.retune.live.window_cut import TraceHistory - self.calls.append( { "segments_root": segments_root, @@ -2189,8 +2190,6 @@ def rows(interval: int, count: int): def _base_planner(**overrides: Any) -> dict[str, Any]: - from dynamo.planner.config.planner_config import PlannerConfig - values: dict[str, Any] = { "namespace": "test-ns", "mode": "agg", @@ -2380,8 +2379,6 @@ def test_scenario_2_presweep_winner_keeps_the_planner_axis(tmp_path, monkeypatch "by_interval": {interval: {"fields": {"load_predictor": "kalman"}}} }, ) - from dynamo.planner.retune import sweep as sweep_module - monkeypatch.setattr(sweep_module, "prepare_predictor", lambda *args: None) result = h.sweep(request, bucket_history=FakeHistory(), worker_pool=InlinePool) @@ -2392,6 +2389,68 @@ def test_scenario_2_presweep_winner_keeps_the_planner_axis(tmp_path, monkeypatch assert all(result.scores[c.key()] is not None for c in request.candidates) +@pytest.mark.parametrize( + "predictor_only", [False, True], ids=["router_only", "predictor_only"] +) +def test_scenario_2_without_planner_points_replays_the_running_planner( + tmp_path, monkeypatch, predictor_only +): + # The grid searches no planner field, so its one planner point is {}: + # every Run still replays the running planner (with the pre-sweep winner + # when the predictor is searched), and every candidate gets a score. + h = Harness(tmp_path) + running: dict[str, Any] = {} + if predictor_only: + base = _base_planner( + optimization_target="sla", + enable_throughput_scaling=True, + enable_load_scaling=True, + load_predictor="arima", + ) + running = {key: base[key] for key in PREDICTOR_KEYS} + else: + base = _base_planner() + request = make_request(planner=running) + request = dataclasses.replace( + request, + replicas={"aggregated": 2}, + base_planner=base, + history_start_ms=request.window_start_ms - 600_000.0, + grid_planners=({},), + ) + if predictor_only: + request = dataclasses.replace( + request, + predictor_candidates=(("kalman_default", {"load_predictor": "kalman"}),), + ) + interval = str(base[THROUGHPUT_INTERVAL]) + monkeypatch.setattr( + dgdr, + "run_presweeps", + lambda *args: { + "by_interval": {interval: {"fields": {"load_predictor": "kalman"}}} + }, + ) + monkeypatch.setattr(sweep_module, "prepare_predictor", lambda *args: None) + + result = h.sweep(request, bucket_history=FakeHistory(), worker_pool=InlinePool) + + assert result.status == "ok", (result.error, result.details) + assert result.details["planner"]["points"] == 1 + grid_points = [ + point + for body in h.created() + if body["metadata"]["labels"][LABEL_SHARD].startswith("grid-") + for point in body["spec"]["aisim"]["inputOverrides"]["planner"]["retune_sweep"][ + "points" + ] + ] + assert grid_points + if predictor_only: + assert {point["load_predictor"] for point in grid_points} == {"kalman"} + assert all(result.scores[c.key()] is not None for c in request.candidates) + + def test_scenario_2_candidate_with_a_foreign_planner_point_is_unmapped(tmp_path): h = Harness(tmp_path) @@ -2438,8 +2497,6 @@ def test_scenario_2_prepares_the_predictor_of_a_new_throughput_interval( candidates=tuple(sorted({*request.candidates, *slower}, key=RetuneParams.key)), ) - from dynamo.planner.retune import sweep as sweep_module - prepared = [] monkeypatch.setattr( sweep_module, diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py index 47abbb9e28e0..78cac3dce67e 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py @@ -588,11 +588,16 @@ def __init__( self.variant = "a" if inputs is None else variant_of(inputs) self.replacements: list[tuple[Any, int]] = [] self.fail_replace = False + # Decisions whose Planner fields the supervisor handed over. + self.persisted: list[Any] = [] @property def controller(self) -> Optional[TracedController]: return self._engine.retune_controller + def retune_decision_persisted(self, decision): + self.persisted.append(decision) + async def replace_live_retune(self, inputs, *, generation): async with self._config_lock: self.replacements.append((inputs, generation)) @@ -1067,6 +1072,51 @@ async def test_applied_decision_is_persisted_then_deployed() -> None: assert sup.launch_gate(K0 + 1, window(K0 + 1)).reason == "already_decided" +def _with_planner_changes(process: Process, k: int) -> RetuneDecision: + """An applied decision with a Planner field, recorded in the controller + as its decide step does.""" + + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + process.controller.restore_desired(chosen) + changes = {"load_scaling_down_sensitivity": 70} + process.controller._pending_planner.update(changes) + return dataclasses.replace( + make_decision(k, "applied", chosen=chosen), planner_changes=changes + ) + + +async def test_planner_fields_apply_only_once_their_decision_is_persisted() -> None: + world = World() + process = await leading(world) + decision = _with_planner_changes(process, K0 + 1) + + enqueue(process, decision) + assert process.host.persisted == [] # queued, not written yet + await process.sup.outbox_round() + + assert world.shared_state().history[-1]["decision_id"] == decision.decision_id + assert process.host.persisted == [decision] + + +async def test_planner_fields_of_an_unwritten_decision_are_discarded() -> None: + world = World() + process = await leading(world) + sup = process.sup + store = StateStore(world.kube, sup.dgd) + snapshot = store.read() + store.write(dataclasses.replace(snapshot.state, leader_term="9/x/1"), snapshot) + decision = _with_planner_changes(process, K0 + 1) + + enqueue(process, decision) + await sup.outbox_round() # lost_term: not written + await sup.lease_round() # re-initializes and drops the outbox + + assert sup.outbox_size == 0 + assert process.host.persisted == [] + # The controller's current point is the running configuration again. + assert process.controller._pending_planner == {} + + async def test_advisory_decision_records_the_recommendation_only() -> None: world = World() process = await leading(world, advisory=True) @@ -1216,10 +1266,13 @@ async def test_decisions_are_dropped_and_rewound_without_leadership() -> None: controller = process.controller controller._last_k = K0 + 1 # Dropped decisions are not handed back: the planner applies none of - # their planner changes. - assert enqueue(process, make_decision(K0 + 1, "kept")) == [] + # their planner changes, and the controller forgets them. + decision = _with_planner_changes(process, K0 + 1) + assert enqueue(process, make_decision(K0 + 1, "kept"), decision) == [] assert process.sup.outbox_size == 0 assert controller._last_k == -1 # no state: every window is due again + assert controller._pending_planner == {} + assert process.host.persisted == [] async def test_a_foreign_term_forces_reinitialization() -> None: diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py index 66536f031444..ab0b2d6f0351 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py @@ -1281,6 +1281,22 @@ def test_history_is_bucketed_at_each_interval_from_the_segments(tmp_path): assert {obs.duration_s for obs in history.observations[5]} == {5.0} +def test_history_rows_need_no_block_hashes(tmp_path): + # History buckets read arrival, ISL and OSL only: a row without its + # replay hashes is counted, not a corrupt trace. + root = tmp_path / "segments" + record = _row("r1", 1000) + del record["request"]["replay"]["input_sequence_hashes"] + del record["request"]["replay"]["trace_block_size"] + _write(root / "pod-a" / "req.000001.jsonl.gz", _member([_envelope(record)])) + + history = window_cut.bucket_history_inprocess( + str(root), 1000, START, END, AS_OF, [5] + ) + + assert history.history_requests == 1 + + def test_history_runs_in_a_child_that_the_deadline_stops(tmp_path): root = tmp_path / "segments" _write( diff --git a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py index 5974d0e5c003..b5f54d4bc6c5 100644 --- a/components/src/dynamo/planner/tests/unit/test_reconfiguration.py +++ b/components/src/dynamo/planner/tests/unit/test_reconfiguration.py @@ -23,6 +23,8 @@ from dynamo.planner.core.types import ( EngineCapabilities, FpmObservations, + RetuneDecision, + RetuneParams, ScheduledTick, TickInput, TrafficObservation, @@ -849,8 +851,6 @@ async def sleep(seconds): def _applied(changes, warmup=None): - from dynamo.planner.core.types import RetuneDecision, RetuneParams - return RetuneDecision( decision_id="retune-w1", outcome="applied", @@ -872,7 +872,9 @@ def _applied(changes, warmup=None): async def _apply_decisions(planner, decisions): - planner._schedule_retune_planner_changes(decisions) + # What the live supervisor does once each decision is persisted. + for decision in decisions: + planner.retune_decision_persisted(decision) await asyncio.gather(*planner._retune_apply_tasks) @@ -900,8 +902,6 @@ async def test_live_retune_drops_a_planner_change_overtaken_while_preparing( # A decision needs a new predictor for 360 s; while it is prepared, a # control API update activates another revision. The decision is dropped # instead of installing a predictor warmed for another configuration. - import dynamo.planner.core.base as base_module - planner = _planner("disagg", optimization_target="sla", load_predictor="constant") engine = planner._ensure_engine() discarded = [] @@ -911,7 +911,7 @@ def discard_planner_changes(self, changes): discarded.append(dict(changes)) engine._retune = Controller() - real_prepare = base_module.prepare_predictor + real_prepare = reconfiguration.prepare_predictor def prepare(config, capabilities, observations): predictor = real_prepare(config, capabilities, observations) @@ -920,7 +920,7 @@ def prepare(config, capabilities, observations): planner.config.load_scaling_down_sensitivity = 70 return predictor - monkeypatch.setattr(base_module, "prepare_predictor", prepare) + monkeypatch.setattr(reconfiguration, "prepare_predictor", prepare) changes = {"throughput_adjustment_interval_seconds": 360} warmup = tuple(TrafficObservation(360.0, 10.0, 100.0, 10.0) for _ in range(8)) @@ -934,6 +934,55 @@ def prepare(config, capabilities, observations): await planner._shutdown_runtime() +async def test_live_retune_and_the_control_api_take_turns(monkeypatch): + # One update prepares at a time, a control API one or a live decision's: + # the other is rejected instead of fitting a second predictor. + planner = _planner("disagg", optimization_target="sla", load_predictor="constant") + engine = planner._ensure_engine() + discarded = [] + + class Controller: + def discard_planner_changes(self, changes): + discarded.append(dict(changes)) + + engine._retune = Controller() + release = threading.Event() + real_prepare = reconfiguration.prepare_predictor + + def slow_prepare(config, capabilities, observations): + release.wait(5) + return real_prepare(config, capabilities, observations) + + monkeypatch.setattr(reconfiguration, "prepare_predictor", slow_prepare) + changes = {"throughput_adjustment_interval_seconds": 360} + warmup = tuple(TrafficObservation(360.0, 10.0, 100.0, 10.0) for _ in range(8)) + planner.retune_decision_persisted(_applied(changes, warmup)) + while planner.reconfiguration._preparation is None: + await asyncio.sleep(0) + + # The decision prepares its predictor: a control API update waits. + with pytest.raises(ConfigurationConflict, match="still preparing"): + await planner.reconfiguration.submit(request(planner, {"ttft_ms": 250.0})) + release.set() + await asyncio.gather(*planner._retune_apply_tasks) + assert planner.config.throughput_adjustment_interval_seconds == 360 + assert planner._config_generation == 1 + + # A control API update prepares: the decision is dropped. + release.clear() + await planner.reconfiguration.submit( + request(planner, {"load_predictor": "kalman"}, cold_start=True) + ) + other = {"load_scaling_down_sensitivity": 70} + await _apply_decisions(planner, [_applied(other)]) + assert discarded == [other] + release.set() + assert (await finish(planner))["state"] == "applied" + assert planner.config.load_scaling_down_sensitivity != 70 + engine._retune = None + await planner._shutdown_runtime() + + async def test_live_retune_keeps_the_config_when_a_planner_change_cannot_apply(): planner = _planner( "disagg", diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md index 5492bb2745ad..6faba467b7c6 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -65,10 +65,12 @@ Live retune searches the three router knobs a DGDR Request can express, reaches any router. 6. **Apply.** The Planner sets all eight router policy parameters of every frontend router to the decided values, and keeps them there: a manual change to them is reverted. Decided Planner - parameters take effect in the Planner itself through its + parameters take effect in the Planner itself once the decision is persisted, through its [runtime reconfiguration](../../reference/components/planner-configuration.mdx#system-port-reconfiguration-api), - with a new load predictor warmed on the window's history when the decision needs one. A - Planner that restarts runs its configured values again: retune does not restore applied + with a new load predictor warmed on the window's history when the decision needs one. They + share the control API's one-update-at-a-time rule: the decision's Planner parameters are + dropped when another update is still preparing, or activates while their predictor is warmed. + A Planner that restarts runs its configured values again: retune does not restore applied Planner parameters, and logs the ones it lost. Windows are aligned to `retune.dgdr.phase_seconds + k × interval_seconds` in epoch time, so a