Skip to content
Merged
2 changes: 2 additions & 0 deletions aisimulate/src/aisimulate/sweeper/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,8 @@ def _reject_removed_flat_fields(cls, data: Any) -> Any:
}
planner_fields = {
"min_endpoint",
"prefill_min_endpoint",
"decode_min_endpoint",
"planner_scaling_policy",
"planner_fpm_sampling",
"planner_load_sensitivity",
Expand Down
2 changes: 2 additions & 0 deletions aisimulate/tests/sweeper/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,8 @@ def test_extra_fields_are_forbidden_at_each_boundary():
"field",
[
"min_endpoint",
"prefill_min_endpoint",
"decode_min_endpoint",
"planner_scaling_policy",
"planner_fpm_sampling",
"planner_load_sensitivity",
Expand Down
9 changes: 8 additions & 1 deletion components/src/dynamo/planner/config/defaults.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,14 @@ class BasePlannerDefaults:
# See dynamo.planner.core.budget.proportional_clamp_pair for the
# tolerance band semantics.
min_gpu_budget = -1
min_endpoint = 1 # applies to both decode and prefill
# Minimum endpoints for agg, or the same minimum for prefill and decode
# in disagg. A role-specific value overrides it for that role.
min_endpoint = 1
prefill_min_endpoint = None
decode_min_endpoint = None
# Localhost-only runtime configuration API (0 disables). It is
# unauthenticated by design and trusts processes in the pod namespace.
control_api_port = 9086
decode_engine_num_gpu = 1
prefill_engine_num_gpu = 1
# Port for exposing planner's own metrics (0 means disabled)
Expand Down
99 changes: 97 additions & 2 deletions components/src/dynamo/planner/config/planner_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import os
from enum import Enum
from pathlib import Path
from typing import Dict, Literal, Optional
from typing import Dict, Literal, Optional, Protocol
from urllib.parse import parse_qsl

import yaml
Expand All @@ -41,6 +41,31 @@
logger = logging.getLogger(__name__)


class MinimumEndpointConfig(Protocol):
"""Configuration fields used to resolve component endpoint floors."""

min_endpoint: int
prefill_min_endpoint: Optional[int]
decode_min_endpoint: Optional[int]


def resolve_min_endpoint(
config: MinimumEndpointConfig, component: Literal["prefill", "decode"]
) -> int:
"""Return the configured minimum for one planner role.

``min_endpoint`` supplies both role values unless a role-specific value is
set.
"""

value = (
config.prefill_min_endpoint
if component == "prefill"
else config.decode_min_endpoint
)
return config.min_endpoint if value is None else value


def _prometheus_ssl_verify_default() -> bool:
return os.environ.get("PROMETHEUS_SSL_VERIFY", "false").lower() in (
"1",
Expand Down Expand Up @@ -405,7 +430,40 @@ class PlannerConfig(BaseModel):
``min_total_gpus`` flag for cross-DGD enforcement; the two are
orthogonal and can both be set.
"""
min_endpoint: int = SLAPlannerDefaults.min_endpoint
min_endpoint: int = Field(
Comment thread
tedzhouhk marked this conversation as resolved.
Resolved
default=SLAPlannerDefaults.min_endpoint,
ge=0,
description=(
"Minimum endpoints for aggregated mode. In disaggregated mode, this "
Comment thread
tedzhouhk marked this conversation as resolved.
Resolved
"value applies to both prefill and decode unless a role-specific "
"value is set. Must be nonnegative; 0 permits scale-to-zero."
),
)
prefill_min_endpoint: Optional[int] = Field(
default=SLAPlannerDefaults.prefill_min_endpoint,
ge=1,
description=(
"Minimum prefill endpoints in disagg and prefill modes. When set, "
"replaces the prefill value supplied by min_endpoint."
),
)
decode_min_endpoint: Optional[int] = Field(
default=SLAPlannerDefaults.decode_min_endpoint,
ge=1,
description=(
"Minimum decode endpoints in disagg and decode modes. When set, "
"replaces the decode value supplied by min_endpoint."
),
)
control_api_port: int = Field(
default=SLAPlannerDefaults.control_api_port,
ge=0,
le=65535,
description=(
"Port for the localhost-only runtime minimum-endpoint API. "
"Set to 0 to disable the API."
),
)

decode_engine_num_gpu: Optional[int] = None
prefill_engine_num_gpu: Optional[int] = None
Expand Down Expand Up @@ -774,6 +832,19 @@ def _validate_config(self) -> "PlannerConfig":
if self.ttft_ms <= 0:
raise ValueError(f"ttft_ms must be > 0, got {self.ttft_ms}")

if self.mode == "prefill" and self.decode_min_endpoint is not None:
raise ValueError("decode_min_endpoint is not supported when mode='prefill'")
if self.mode == "decode" and self.prefill_min_endpoint is not None:
raise ValueError("prefill_min_endpoint is not supported when mode='decode'")
if self.mode == "agg" and (
self.prefill_min_endpoint is not None
or self.decode_min_endpoint is not None
):
raise ValueError(
"prefill_min_endpoint and decode_min_endpoint are not supported "
"when mode='agg'; use min_endpoint"
)

if self.report_interval_hours is not None:
if (
not math.isfinite(self.report_interval_hours)
Expand Down Expand Up @@ -1009,6 +1080,30 @@ def _load_from_file(cls, path: Path) -> "PlannerConfig":
def scaling_enabled(self) -> bool:
return self.enable_throughput_scaling or self.enable_load_scaling

@property
def effective_prefill_min_endpoint(self) -> int:
"""Return the effective prefill endpoint minimum."""

return resolve_min_endpoint(self, "prefill")

@property
def effective_decode_min_endpoint(self) -> int:
"""Return the effective decode endpoint minimum."""

return resolve_min_endpoint(self, "decode")

def active_min_endpoints(self) -> tuple[Optional[int], Optional[int]]:
"""Return effective ``(prefill, decode)`` floors for the active mode."""

if self.mode == "prefill":
return self.effective_prefill_min_endpoint, None
if self.mode in ("decode", "agg"):
return None, self.effective_decode_min_endpoint
return (
self.effective_prefill_min_endpoint,
self.effective_decode_min_endpoint,
)


if __name__ == "__main__":
from pathlib import Path
Expand Down
114 changes: 114 additions & 0 deletions components/src/dynamo/planner/control_api.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Local runtime API for planner minimum endpoint configuration."""

from __future__ import annotations

import json
import logging
from typing import Any, Protocol

from aiohttp import web
from pydantic import BaseModel, ConfigDict, Field, ValidationError

logger = logging.getLogger(__name__)


class _MinimumEndpointValidationError(ValueError):
"""Raised when a well-formed runtime update is invalid for the planner."""


class _MinimumEndpointUnavailableError(RuntimeError):
"""Raised when a runtime update cannot safely run before its deadline."""


class _MinimumEndpointController(Protocol):
async def get_min_endpoints(self) -> dict[str, Any]:
...

async def patch_min_endpoints(self, updates: dict[str, int]) -> dict[str, Any]:
...


class _MinimumEndpointsPatch(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)

min_endpoint: int | None = Field(default=None, ge=0)
prefill_min_endpoint: int | None = Field(default=None, ge=1)
decode_min_endpoint: int | None = Field(default=None, ge=1)


def _error(message: str, status: int) -> web.Response:
return web.json_response({"error": message}, status=status)


def _build_app(controller: _MinimumEndpointController) -> web.Application:
async def handle_get(_request: web.Request) -> web.Response:
try:
response = await controller.get_min_endpoints()
except _MinimumEndpointUnavailableError as exc:
return _error(str(exc), 503)
return web.json_response(response)

async def handle_patch(request: web.Request) -> web.Response:
try:
payload = await request.json()
except (json.JSONDecodeError, UnicodeDecodeError):
return _error("request body must be valid JSON", 400)
if not isinstance(payload, dict):
return _error("request body must be a JSON object", 400)

try:
patch = _MinimumEndpointsPatch.model_validate(payload)
except ValidationError as exc:
# Unknown fields make the patch structurally invalid (400). Type,
# range, and null errors on known fields are semantic validation
# failures (422).
status = (
400
if any(error["type"] == "extra_forbidden" for error in exc.errors())
else 422
)
return _error(str(exc), status)

fields = patch.model_fields_set
if not fields:
return _error("request body must include at least one endpoint field", 400)
if any(getattr(patch, field) is None for field in fields):
return _error("minimum endpoint fields cannot be null", 422)

updates = {field: int(getattr(patch, field)) for field in fields}
try:
response = await controller.patch_min_endpoints(updates)
except _MinimumEndpointUnavailableError as exc:
return _error(str(exc), 503)
except _MinimumEndpointValidationError as exc:
return _error(str(exc), 422)
return web.json_response(response)

app = web.Application()
app.router.add_get("/v1/min-endpoints", handle_get)
app.router.add_patch("/v1/min-endpoints", handle_patch)
return app


async def _start_control_api(
controller: _MinimumEndpointController, port: int
) -> web.AppRunner:
"""Start the localhost-only runtime configuration API."""

runner = web.AppRunner(_build_app(controller))
await runner.setup()
site = web.TCPSite(runner, "127.0.0.1", port)
try:
await site.start()
except Exception:
await runner.cleanup()
raise
logger.info(
"Planner runtime configuration API listening on "
"http://127.0.0.1:%s/v1/min-endpoints",
port,
)
return runner
Loading
Loading