Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 61 additions & 6 deletions components/src/dynamo/planner/core/perf_model/ais_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,24 @@

logger = logging.getLogger(__name__)

# AISimulate's spelling of an identity fact the Planner cannot observe.
_UNKNOWN = "unknown"


def _known(value: Any, placeholder: Any) -> Any:
return placeholder if value == _UNKNOWN else value


# Engine controls a worker does not report to the Planner.
_UNOBSERVED_ENGINE_CONTROLS = (
"gemm_quant_mode",
"moe_quant_mode",
"fmha_quant_mode",
"kvcache_quant_mode",
"comm_quant_mode",
"attention_backend",
)

_ENGINE_MODEL_EXCEPTIONS = (RuntimeError, ValueError, TypeError)
_ENGINE_MODEL_INIT_EXCEPTIONS = (ImportError, RuntimeError)

Expand Down Expand Up @@ -280,14 +298,14 @@ def _build_ais_config(self) -> dict[str, Any]:
if spec is None:
config = asdict(
ForwardPassPerfModelConfig(
model=self._config.model_name
or f"dynamo-planner:{self._config.namespace}",
system="unprofiled",
model=self._config.model_name or _UNKNOWN,
backend="vllm"
if self._config.backend == "mocker"
else self._config.backend,
worker_type=self._worker_type,
estimation_mode="fpm_regression",
fallback_policy="deny",
**self._observed_identity(),
)
)
else:
Expand Down Expand Up @@ -318,11 +336,13 @@ def _build_ais_config(self) -> dict[str, Any]:
defaults = ForwardPassPerfModelConfig.from_legacy_engine_config(
{
"schema_version": 1,
# Only estimator controls are taken from this migration, which
# resolves a native identity; unknown facts do not change them.
"model_name": config["model"],
"system_name": config["system"],
"system_name": _known(config["system"], "unprofiled"),
"backend": config["backend"],
"tp_size": config["tp"],
"pp_size": config["pp"],
"tp_size": _known(config["tp"], 1),
"pp_size": _known(config["pp"], 1),
},
self._worker_type,
self._build_options(),
Expand All @@ -332,6 +352,41 @@ def _build_ais_config(self) -> dict[str, Any]:
)
return config

def _observed_identity(self) -> dict[str, Any]:
"""Identity facts of a cold regression model: what the worker reports,
and AIS's ``"unknown"`` for every fact the Planner cannot observe.

A saved snapshot then times a recommendation role without guessing:
AISimulate pins the known facts and asks the caller for the rest.
"""

caps = self._capabilities
num_gpu = caps.num_gpu if caps is not None else None
# One GPU per worker fixes every parallel width; wider workers do not
# report how they split it.
width: int | str = 1 if num_gpu == 1 else _UNKNOWN
limits = {
"num_gpus": num_gpu,
"max_num_batched_tokens": caps.max_num_batched_tokens if caps else None,
"max_num_seqs": caps.max_num_seqs if caps else None,
"max_kv_tokens": caps.max_kv_tokens if caps else None,
"context_length": caps.context_length if caps else None,
}
return {
"system": _UNKNOWN,
"backend_version": _UNKNOWN,
"tp": width,
"pp": width,
"attention_dp": width,
"moe_tp_size": None if num_gpu == 1 else _UNKNOWN,
"moe_ep_size": None if num_gpu == 1 else _UNKNOWN,
**{name: _UNKNOWN for name in _UNOBSERVED_ENGINE_CONTROLS},
"deployment_limits": {
name: value if value is not None and value > 0 else _UNKNOWN
for name, value in limits.items()
},
}

def _model_key(self) -> Optional[tuple[Any, ...]]:
values = self._limit_values()
if values is None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,73 @@ def test_cold_regression_has_traceable_role_identity(role):
assert model._engine_diagnostics()["readiness"] == "insufficient_data"


_UNOBSERVED = (
"system",
"backend_version",
"gemm_quant_mode",
"moe_quant_mode",
"fmha_quant_mode",
"kvcache_quant_mode",
"comm_quant_mode",
"attention_backend",
)


def test_cold_regression_records_what_the_worker_reports_and_unknown_otherwise():
config = PlannerConfig.model_construct(
namespace="worker-group-a", model_name="actual-model", backend="vllm"
)
caps = EngineCapabilities(
num_gpu=1,
max_num_batched_tokens=8192,
max_num_seqs=256,
max_kv_tokens=16384,
context_length=4096,
kv_cache_block_size=16,
)
model = PlannerEnginePerfModel(
worker_type="decode", config=config, capabilities=caps
)
payload = model._build_ais_config()
assert {name: payload[name] for name in _UNOBSERVED} == dict.fromkeys(
_UNOBSERVED, "unknown"
)
# One GPU per worker fixes every parallel width.
assert (payload["tp"], payload["pp"], payload["attention_dp"]) == (1, 1, 1)
assert (payload["moe_tp_size"], payload["moe_ep_size"]) == (None, None)
assert payload["kv_block_size"] == 16
assert payload["deployment_limits"] == {
"num_gpus": 1,
"max_num_batched_tokens": 8192,
"max_num_seqs": 256,
"max_kv_tokens": 16384,
"context_length": 4096,
}
# AIS accepts the identity: a cold regression model is constructed.
assert model._engine_diagnostics()["readiness"] == "insufficient_data"


def test_cold_regression_marks_unobserved_widths_and_limits_unknown():
config = PlannerConfig.model_construct(namespace="worker-group-a", backend="vllm")
caps = EngineCapabilities(num_gpu=4, max_num_seqs=64)
model = PlannerEnginePerfModel(
worker_type="decode", config=config, capabilities=caps
)
payload = model._build_ais_config()
assert payload["model"] == "unknown"
# A wider worker does not report how it splits its GPUs.
for name in ("tp", "pp", "attention_dp", "moe_tp_size", "moe_ep_size"):
assert payload[name] == "unknown", name
assert payload["deployment_limits"] == {
"num_gpus": 4,
"max_num_batched_tokens": "unknown",
"max_num_seqs": 64,
"max_kv_tokens": "unknown",
"context_length": "unknown",
}
assert model._engine_diagnostics()["readiness"] == "insufficient_data"


def test_worker_capabilities_cannot_silently_override_canonical_identity():
config = _planner(_role_config(estimation_mode="fpm_regression", kv_block_size=16))
limits = _limits()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,14 @@ New configurations default to `estimation_mode: auto` and `fallback_policy: deny
Auto searches `op_level`, `fpm_interpolation`, then `fpm_regression`; `deny` prevents
fallback only when you explicitly select a mode. Without `ais_perf_model`, the
Planner creates a cold `fpm_regression` model and trains it from self-benchmark
or live forward-pass metrics (FPM).
or live forward-pass metrics (FPM). Its identity records what the worker
reports: the model, backend, KV block size and MTP depth, plus the GPUs per
worker, scheduler limits, KV capacity and context length as
`deployment_limits`. Facts the Planner cannot observe are recorded as
`"unknown"`: the hardware, backend version, quantization, attention backend,
and the parallel split of a worker with more than one GPU. A state snapshot
saved from this model can then time an AISimulate recommendation role, which
pins the known facts and takes the rest from the recommendation's engine.

Each role accepts the complete upstream schema, including ordered `systems_paths`,
`database_mode`, `transfer_policy`, quantization and speculation settings, and
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ advisory 模式仅提供建议。Planner 会计算建议副本数、记录日志

SLA 模式通过唯一构造接口 `aisimulate_core.sdk.RustForwardPassPerfModel.best_available(config)` 为每个 worker 角色创建模型。`ais_perf_model.roles` 中,分离式部署配置 `prefill` 和 `decode`,聚合式部署配置 `aggregated`;角色键必须与 `worker_type` 一致。

新配置默认使用 `estimation_mode: auto` 和 `fallback_policy: deny`。Auto 依次查找 `op_level`、`fpm_interpolation`、`fpm_regression`;`deny` 只禁止显式指定模式后的回退。没有配置 `ais_perf_model` 时,Planner 创建未训练的 `fpm_regression` 模型,并用自基准测试或在线 FPM 训练。
新配置默认使用 `estimation_mode: auto` 和 `fallback_policy: deny`。Auto 依次查找 `op_level`、`fpm_interpolation`、`fpm_regression`;`deny` 只禁止显式指定模式后的回退。没有配置 `ais_perf_model` 时,Planner 创建未训练的 `fpm_regression` 模型,并用自基准测试或在线 FPM 训练。该模型的身份记录 worker 上报的信息:模型、后端、KV block 大小和 MTP 深度,以及作为 `deployment_limits` 记录的每个 worker 的 GPU 数、调度上限、KV 容量和上下文长度。Planner 无法观测的信息记为 `"unknown"`:硬件、后端版本、量化、attention backend,以及多 GPU worker 的并行拆分方式。由此保存的 state snapshot 可以为 AISimulate 推荐中的一个角色计时;已知信息被固定,其余取自推荐配置中的 engine。

各角色可传入完整上游配置,包括有序 `systems_paths`、`database_mode`、`transfer_policy`、量化和推测解码参数,以及嵌套 `estimator_config`。显式 estimator 设置覆盖 Planner 的采样默认值;`fpm_sample_bucket_size: 16` 对应 `[4, 4]`,regression 和 correction 可以分别配置网格。未知字段及角色身份冲突会报错。

Expand Down
Loading