From 095c34d92da12ec28063c0a7535266e7637b4f70 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Mon, 17 Aug 2026 21:37:39 +0000 Subject: [PATCH 1/2] feat(router): bridge admission terminal events Signed-off-by: Ishan Dhanani --- .../fern/pages/reference/api/python/_core.mdx | 440 +++++++++--------- docs/fern/pages/reference/api/python/llm.mdx | 296 ++++++------ .../pages/reference/api/python/mocker.mdx | 26 +- lib/bindings/python/rust/llm/kv.rs | 15 + lib/bindings/python/src/dynamo/_core.pyi | 4 + lib/bindings/python/tests/test_kv_bindings.py | 4 + .../src/services/selection/core/mod.rs | 293 +++++++++++- .../src/services/selection/server.rs | 14 + .../src/services/selection/service.rs | 4 + lib/kv-router/src/services/selection/tests.rs | 229 ++++++++- lib/llm/src/kv_router.rs | 21 + .../kv_router/push_router/request_guard.rs | 85 +++- lib/llm/src/kv_router/scheduler.rs | 25 + 13 files changed, 1054 insertions(+), 402 deletions(-) diff --git a/docs/fern/pages/reference/api/python/_core.mdx b/docs/fern/pages/reference/api/python/_core.mdx index 610ff6696a36..7ce6865d1c2c 100644 --- a/docs/fern/pages/reference/api/python/_core.mdx +++ b/docs/fern/pages/reference/api/python/_core.mdx @@ -20,7 +20,7 @@ from dynamo._core import AicPerfConfig AicPerfConfig(aic_backend: str, aic_system: str, aic_model_path: str, aic_tp_size: int = 1, aic_backend_version: Optional[str] = None, aic_moe_tp_size: Optional[int] = None, aic_moe_ep_size: Optional[int] = None, aic_attention_dp_size: Optional[int] = None, aic_nextn: Optional[int] = None, aic_nextn_accept_rates: Optional[str] = None, aic_gemm_dtype: Optional[str] = None, aic_moe_dtype: Optional[str] = None, aic_fmha_dtype: Optional[str] = None, aic_kv_cache_dtype: Optional[str] = None, aic_comm_dtype: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1723`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1723) +[`lib/bindings/python/src/dynamo/_core.pyi#L1727`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1727) **Public methods** @@ -32,7 +32,7 @@ __init__(aic_backend: str, aic_system: str, aic_model_path: str, aic_tp_size: in No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1724) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1728) @@ -51,7 +51,7 @@ This is useful when: - You want to reduce event processing overhead - Lower routing accuracy is acceptable -[`lib/bindings/python/src/dynamo/_core.pyi#L1069`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1069) +[`lib/bindings/python/src/dynamo/_core.pyi#L1073`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1073) **Public methods** @@ -75,7 +75,7 @@ The KV cache block size TTL for blocks in seconds (default: 120.0) -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1083) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1087)

find_matches_for_request

@@ -98,7 +98,7 @@ Optional LoRA adapter name for adapter-aware matching - `OverlapScores` — OverlapScores containing worker matching scores and frequencies -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1099) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1103)

block_size

@@ -112,7 +112,7 @@ Return the block size of the ApproxKvIndexer. - `int` — The KV cache block size -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1114) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1118)

process_routing_decision_for_request

@@ -137,7 +137,7 @@ The worker ID the request was routed to The data parallel rank (default: 0) -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1123) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1127)
@@ -147,7 +147,7 @@ A KV cache block from dynamo._core import Block ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2630`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2630) +[`lib/bindings/python/src/dynamo/_core.pyi#L2634`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2634) **Public methods** @@ -159,7 +159,7 @@ to_list() -> List[Layer] Get a list of layers -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2661) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2665) @@ -169,7 +169,7 @@ A list of KV cache blocks from dynamo._core import BlockList ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2680`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2680) +[`lib/bindings/python/src/dynamo/_core.pyi#L2684`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2684) **Public methods** @@ -181,7 +181,7 @@ to_list() -> List[Block] Get a list of blocks -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2711) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2715) @@ -195,7 +195,7 @@ from dynamo._core import BlockManager BlockManager(worker_id: int, num_layer: int, page_size: int, inner_dim: int, dtype: Optional[str] = None, host_num_blocks: Optional[int] = None, device_num_blocks: Optional[int] = None, device_id: int = 0) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2717`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2717) +[`lib/bindings/python/src/dynamo/_core.pyi#L2721`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2721) **Public methods** @@ -226,7 +226,7 @@ Number of device blocks to allocate, None means no device blocks device_id: int CUDA device ID, defaults to 0 -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2722) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2726)

allocate_host_blocks_blocking

@@ -246,7 +246,7 @@ Returns: BlockList List of allocated blocks -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2757) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2761)

allocate_host_blocks

@@ -266,7 +266,7 @@ Returns: BlockList List of allocated blocks -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2773) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2777)

allocate_device_blocks_blocking

@@ -286,7 +286,7 @@ Returns: BlockList List of allocated blocks -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2789) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2793)

allocate_device_blocks

@@ -306,7 +306,7 @@ Returns: BlockList List of allocated blocks -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2805) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2809)
@@ -316,7 +316,7 @@ The request was cancelled. from dynamo._core import Cancelled ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3311`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3311) +[`lib/bindings/python/src/dynamo/_core.pyi#L3315`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3315) @@ -326,7 +326,7 @@ Failed to establish a connection. from dynamo._core import CannotConnect ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3296`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3296) +[`lib/bindings/python/src/dynamo/_core.pyi#L3300`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3300) @@ -445,7 +445,7 @@ A connection or request timed out. from dynamo._core import ConnectionTimeout ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3306`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3306) +[`lib/bindings/python/src/dynamo/_core.pyi#L3310`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3310) @@ -701,7 +701,7 @@ An established connection was lost. from dynamo._core import Disconnected ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3301`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3301) +[`lib/bindings/python/src/dynamo/_core.pyi#L3305`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3305) @@ -806,7 +806,7 @@ Base exception for all Dynamo error types. from dynamo._core import DynamoException ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3273`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3273) +[`lib/bindings/python/src/dynamo/_core.pyi#L3277`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3277) @@ -939,7 +939,7 @@ Holds internal configuration for a Dynamo engine. from dynamo._core import EngineConfig ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2376`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2376) +[`lib/bindings/python/src/dynamo/_core.pyi#L2380`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2380) @@ -949,7 +949,7 @@ The engine process has shut down or crashed. from dynamo._core import EngineShutdown ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3316`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3316) +[`lib/bindings/python/src/dynamo/_core.pyi#L3320`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3320) @@ -959,7 +959,7 @@ Engine type for Dynamo workers from dynamo._core import EngineType ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3130`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3130) +[`lib/bindings/python/src/dynamo/_core.pyi#L3134`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3134) @@ -973,7 +973,7 @@ from dynamo._core import EntrypointArgs EntrypointArgs(engine_type: EngineType, model_path: Optional[str] = None, model_name: Optional[str] = None, endpoint_id: Optional[str] = None, template_file: Optional[str] = None, router_config: Optional[RouterConfig] = None, kv_cache_block_size: Optional[int] = None, http_host: Optional[str] = None, http_port: Optional[int] = None, http_metrics_port: Optional[int] = None, tls_cert_path: Optional[str] = None, tls_key_path: Optional[str] = None, extra_engine_args: Optional[str] = None, mocker_engine_args: Optional[MockEngineArgs] = None, runtime_config: Optional[ModelRuntimeConfig] = None, namespace: Optional[str] = None, namespace_prefix: Optional[str] = None, is_prefill: bool = False, is_decode: bool = False, migration_limit: int = 0, migration_max_seq_len: Optional[int] = None, chat_engine_factory: Optional[Callable] = None, aic_perf_config: Optional[AicPerfConfig] = None, *, metrics_prefix: Optional[str] = None, enable_anthropic_api: Optional[bool] = None, strip_anthropic_preamble: Optional[bool] = None, enable_streaming_tool_dispatch: Optional[bool] = None, enable_streaming_reasoning_dispatch: Optional[bool] = None, tokenizer_backend: Optional[str] = None, tokenizer_fallback: Optional[bool] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3137`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3137) +[`lib/bindings/python/src/dynamo/_core.pyi#L3141`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3141) **Public methods** @@ -1078,7 +1078,7 @@ Optional tokenizer backend override ("default", "fastokens", or "basetenkenizer" Whether alternate tokenizer load failures fall back to HuggingFace -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3143) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3147) @@ -1092,7 +1092,7 @@ from dynamo._core import FpmDirectPublisher FpmDirectPublisher(endpoint: Endpoint, worker_id: str, dp_size: int = 1) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1286`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1286) +[`lib/bindings/python/src/dynamo/_core.pyi#L1290`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1290) **Public methods** @@ -1117,7 +1117,7 @@ Number of DP ranks to allocate channels for. Use ``1`` when attention DP is disabled. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1297) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1301)

publish

@@ -1137,7 +1137,7 @@ var_queued_prefill_length, var_queued_decode_kv_tokens) are defaulted to 0.0 per the MVP scope; a follow-up PR can add Welford-based variance computation. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1314) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1318)

shutdown

@@ -1147,7 +1147,7 @@ shutdown() -> None Shut down the publisher and its per-rank serialization tasks. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1344) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1348)
@@ -1161,7 +1161,7 @@ from dynamo._core import FpmEventRelay FpmEventRelay(endpoint: Endpoint, zmq_endpoint: str) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1259`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1259) +[`lib/bindings/python/src/dynamo/_core.pyi#L1263`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1263) **Public methods** @@ -1183,7 +1183,7 @@ Local ZMQ PUB address to subscribe to (e.g., "tcp://127.0.0.1:20380"). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1266) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1270)

shutdown

@@ -1193,7 +1193,7 @@ shutdown() -> None Shut down the relay task. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1281) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1285)
@@ -1215,7 +1215,7 @@ Two mutually exclusive usage modes: ``(worker_id, dp_rank)``. Stale entries are cleaned up when workers are removed (via discovery watch). -[`lib/bindings/python/src/dynamo/_core.pyi#L1349`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1349) +[`lib/bindings/python/src/dynamo/_core.pyi#L1353`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1353) **Public methods** @@ -1236,7 +1236,7 @@ No background tasks are started until ``recv()`` or Dynamo component endpoint (provides runtime + discovery). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1363) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1367)

recv

@@ -1253,7 +1253,7 @@ Cannot be used after ``start_tracking()``. - `Optional[bytes]` — Raw msgspec payload, or None if the stream is closed. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1375) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1379)

start_tracking

@@ -1274,7 +1274,7 @@ worker_id matches the removed instance_id are purged. After calling this, ``recv()`` will raise RuntimeError. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1388) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1392)

get_recent_stats

@@ -1294,7 +1294,7 @@ Raises RuntimeError if ``start_tracking()`` has not been called. - `dict[tuple[str, int], bytes]` — dict mapping ``(worker_id, dp_rank)`` to raw msgspec bytes. - `dict[tuple[str, int], bytes]` — Decode each value with ``forward_pass_metrics.decode(data)``. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1405) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1409)

get_model_cards

@@ -1316,7 +1316,7 @@ Raises RuntimeError if ``start_tracking()`` has not been called. - `dict[str, str]` — dict mapping ``worker_id`` to ``card_json`` (JSON string). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1420) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1424)

shutdown

@@ -1326,7 +1326,7 @@ shutdown() -> None Shut down the subscriber (all background tasks). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1437) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1441)
@@ -1340,7 +1340,7 @@ Handlers receive this and answer from current state. The surface is intentionally narrow (typed read-only accessors only); it does not expose the internal service state. -[`lib/bindings/python/src/dynamo/_core.pyi#L2384`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2384) +[`lib/bindings/python/src/dynamo/_core.pyi#L2388`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2388) **Public methods** @@ -1352,7 +1352,7 @@ is_ready() -> bool Whether the HTTP service has finished startup and is ready to serve. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2392) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2396)

is_cancelled

@@ -1362,7 +1362,7 @@ is_cancelled() -> bool Whether the frontend is shutting down (draining). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2396) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2400)

has_any_ready_model

@@ -1372,7 +1372,7 @@ has_any_ready_model() -> bool Whether at least one model is registered and ready to serve. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2400) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2404)

is_model_ready_to_serve

@@ -1382,7 +1382,7 @@ is_model_ready_to_serve(model: str) -> bool Whether the named model is registered and ready to serve. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2404) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2408)

model_display_names

@@ -1392,7 +1392,7 @@ model_display_names() -> list[str] Sorted display names of all registered models. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2408) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2412)

serving_ready_display_names

@@ -1402,7 +1402,7 @@ serving_ready_display_names() -> list[str] Sorted display names of models ready to serve. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2412) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2416)
@@ -1419,7 +1419,7 @@ FrontendResponse(status_code: int, body: object) -> None Return this to set a non-200 status (e.g. ``FrontendResponse(503, body)``); return a plain JSON-serializable value for the default 200. -[`lib/bindings/python/src/dynamo/_core.pyi#L2437`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2437) +[`lib/bindings/python/src/dynamo/_core.pyi#L2441`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2441) **Public methods** @@ -1431,7 +1431,7 @@ __init__(status_code: int, body: object) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2444) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2448) @@ -1451,7 +1451,7 @@ returns a JSON-serializable body (implies HTTP 200) or a ``FrontendResponse`` to set the status code. Async handlers and path parameters are rejected at construction. -[`lib/bindings/python/src/dynamo/_core.pyi#L2416`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2416) +[`lib/bindings/python/src/dynamo/_core.pyi#L2420`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2420) **Public methods** @@ -1463,7 +1463,7 @@ __init__(method: str, path: str, handler: Callable[[FrontendExtensionContext], o No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2426) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2430) @@ -1473,7 +1473,7 @@ An async engine for a distributed Dynamo http service. This is an extension of t from dynamo._core import HttpAsyncEngine ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1483`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1483) +[`lib/bindings/python/src/dynamo/_core.pyi#L1487`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1487) @@ -1487,7 +1487,7 @@ from dynamo._core import HttpService HttpService(port: Optional[int] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1442`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1442) +[`lib/bindings/python/src/dynamo/_core.pyi#L1446`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1446) **Public methods** @@ -1505,7 +1505,7 @@ Create a new HTTP service. Optional port number to bind the service to (default: 8080) -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1448) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1452)

run

@@ -1521,7 +1521,7 @@ Run the HTTP service. DistributedRuntime instance for token management -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1457) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1461)

shutdown

@@ -1531,7 +1531,7 @@ shutdown() -> None Shutdown the HTTP service by cancelling its internal token. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1466) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1470)
@@ -1551,7 +1551,7 @@ Invalid input (e.g., prompt exceeds context length). from dynamo._core import InvalidArgument ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3291`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3291) +[`lib/bindings/python/src/dynamo/_core.pyi#L3295`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3295) @@ -1565,7 +1565,7 @@ from dynamo._core import KserveGrpcService KserveGrpcService(port: Optional[int] = None, host: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1492`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1492) +[`lib/bindings/python/src/dynamo/_core.pyi#L1496`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1496) **Public methods** @@ -1586,7 +1586,7 @@ Optional port number to bind the service to Optional host address to bind the service to -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1498) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1502)

add_completions_model

@@ -1608,7 +1608,7 @@ The model checksum The async engine to handle requests -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1508) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1512)

add_chat_completions_model

@@ -1630,7 +1630,7 @@ The model checksum The async engine to handle requests -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1524) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1528)

add_tensor_model

@@ -1658,7 +1658,7 @@ Optional runtime-resolved worker metadata Optional tensor protocol model metadata -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1540) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1544)

remove_completions_model

@@ -1674,7 +1674,7 @@ Remove a completions model from the service. The model name to remove -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1561) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1565)

remove_chat_completions_model

@@ -1690,7 +1690,7 @@ Remove a chat completions model from the service. The model name to remove -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1570) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1574)

remove_tensor_model

@@ -1706,7 +1706,7 @@ Remove a tensor model from the service. The model name to remove -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1579) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1583)

list_chat_completions_models

@@ -1720,7 +1720,7 @@ List all registered chat completions models. - `List[str]` — List of model names -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1588) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1592)

list_completions_models

@@ -1734,7 +1734,7 @@ List all registered completions models. - `List[str]` — List of model names -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1597) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1601)

list_tensor_models

@@ -1748,7 +1748,7 @@ List all registered tensor models. - `List[str]` — List of model names -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1606) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1610)

run

@@ -1764,7 +1764,7 @@ Run the KServe gRPC service. DistributedRuntime instance for token management -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1615) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1619)

shutdown

@@ -1774,7 +1774,7 @@ shutdown() -> None Shutdown the KServe gRPC service by cancelling its internal token. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1624) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1628)
@@ -1788,7 +1788,7 @@ from dynamo._core import KvDcRelay KvDcRelay(endpoint: Endpoint, dc_id: str, namespace_filter: Optional[str] = None, endpoint_prefix: Optional[str] = None, publication_threshold: int = 16, publication_delay_ms: int = 1, recovery_attempt_timeout_ms: int = 30000) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2829`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2829) +[`lib/bindings/python/src/dynamo/_core.pyi#L2833`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2833) **Public methods** @@ -1800,7 +1800,7 @@ __init__(endpoint: Endpoint, dc_id: str, namespace_filter: Optional[str] = None, No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2830) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2834)

start

@@ -1810,7 +1810,7 @@ start() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2842) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2846)

health

@@ -1820,7 +1820,7 @@ health() -> Dict[str, Any] No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2845) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2849)

flush

@@ -1830,7 +1830,7 @@ flush() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2848) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2852)

shutdown

@@ -1840,7 +1840,7 @@ shutdown() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2851) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2855)
@@ -1854,7 +1854,7 @@ from dynamo._core import KvEventPublisher KvEventPublisher(endpoint: Endpoint, worker_id: Optional[int] = None, kv_block_size: int = 0, dp_rank: int = 0, enable_local_indexer: bool = False, zmq_endpoint: Optional[str] = None, zmq_topic: Optional[str] = None, batching_timeout_ms: Optional[int] = None, image_token_id: Optional[int] = None, kv_state_endpoint: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1157`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1157) +[`lib/bindings/python/src/dynamo/_core.pyi#L1161`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1161) **Public methods** @@ -1903,7 +1903,7 @@ flushes at each submitted source-list boundary. KV event ownership endpoint; defaults to endpoint. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1164) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1168)

publish_stored

@@ -1942,7 +1942,7 @@ Optional Eagle mode flag. When true, stored blocks are reconstructed using overlapping `kv_block_size + 1` token windows. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1199) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1203)

publish_removed

@@ -1960,7 +1960,7 @@ Event IDs are managed internally by the publisher using a monotonic counter. List of block hashes to remove (signed 64-bit integers) -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1229) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1233)

publish_batch

@@ -1974,7 +1974,7 @@ The complete list is validated before it is enqueued. Compatible events are coalesced while preserving source order and the processor's existing block-count limits. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1240) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1244)

shutdown

@@ -1984,7 +1984,7 @@ shutdown() -> None Shuts down the event publisher, stopping any background tasks. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1252) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1256)
@@ -1998,7 +1998,7 @@ from dynamo._core import KvIndexer KvIndexer(endpoint: Endpoint, block_size: int) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1031`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1031) +[`lib/bindings/python/src/dynamo/_core.pyi#L1035`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1035) **Public methods** @@ -2010,7 +2010,7 @@ __init__(endpoint: Endpoint, block_size: int) -> None Create a `KvIndexer` object -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1038) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1042)

find_matches

@@ -2030,7 +2030,7 @@ List of block hashes to find matches for - `OverlapScores` — OverlapScores containing worker matching scores and frequencies -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1043) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1047)

find_matches_for_request

@@ -2040,7 +2040,7 @@ find_matches_for_request(token_ids: List[int], lora_name: Optional[str] = None, Return the overlapping scores of workers for the given token ids. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1055) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1059)

block_size

@@ -2050,7 +2050,7 @@ block_size() -> int Return the block size of the KV Indexer. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1063) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1067)
@@ -2060,7 +2060,7 @@ No summary available. from dynamo._core import KvRemovedEventInput ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1152`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1152) +[`lib/bindings/python/src/dynamo/_core.pyi#L1156`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1156) @@ -2074,7 +2074,7 @@ from dynamo._core import KvRouter KvRouter(endpoint: Endpoint, block_size: int, kv_router_config: KvRouterConfig, aic_perf_config: Optional[AicPerfConfig] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2887`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2887) +[`lib/bindings/python/src/dynamo/_core.pyi#L2891`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2891) **Public methods** @@ -2101,7 +2101,7 @@ Configuration for the KV router Optional AIC perf-model config for effective prefill load tracking -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2892) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2896)

generate

@@ -2181,7 +2181,7 @@ multiple replicas (data_parallel_size > 1). - This is different from query_instance_id which doesn't route the request. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2910) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2914)

generate_from_request

@@ -2196,7 +2196,7 @@ Set response_buffer_size to 0 for demand-driven direct Python consumption; negative values are rejected. Returns an async iterator yielding generation responses. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2971) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2975)

best_worker

@@ -2243,7 +2243,7 @@ configured default family before cache-bucket resolution. - `Tuple[int, int, int]` — A tuple of (worker_id, dp_rank, overlap_blocks) where: - worker_id: The ID of the best matching worker - dp_rank: The data parallel rank of the selected worker - overlap_blocks: The number of overlapping blocks found -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2986) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2990)

get_potential_loads

@@ -2276,7 +2276,7 @@ Each (worker_id, dp_rank) pair is returned as a separate entry. If you need aggregated loads per worker_id, sum the values manually. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3028) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3032)

get_overlap_scores

@@ -2312,7 +2312,7 @@ Whether to query the configured shared cache. - `Dict[str, Any]` — workers. Each worker row is keyed by worker_id and dp_rank and - `Dict[str, Any]` — reports device, host-pinned, disk, and shared-cache overlap blocks. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3059) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3063)

dump_events

@@ -2326,7 +2326,7 @@ Dump all events from the KV router's indexer. - `str` — A JSON string containing all indexer events -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3087) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3091)

mark_prefill_complete

@@ -2351,7 +2351,7 @@ This is typically called automatically by the router when using the `best_worker()` with `request_id` for custom routing. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3096) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3100)

free

@@ -2376,7 +2376,7 @@ This is typically called automatically by the router when using the `best_worker()` with `request_id` for custom routing. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3113) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3117)
@@ -2390,7 +2390,7 @@ from dynamo._core import KvRouterConfig KvRouterConfig(overlap_score_weight: Optional[float] = None, host_cache_hit_weight: float = 0.75, disk_cache_hit_weight: float = 0.25, router_temperature: float = 0.0, use_kv_events: bool = True, *, router_replica_sync: bool = False, router_track_active_blocks: bool = True, router_track_output_blocks: bool = False, router_assume_kv_reuse: bool = True, router_track_prefill_tokens: bool = True, router_prefill_load_model: str = 'none', router_ttl_secs: float = 120.0, router_queue_threshold: Optional[float] = None, router_event_threads: int = 4, router_queue_policy: str = 'fcfs', use_remote_indexer: bool = False, serve_indexer: bool = False, shared_cache_multiplier: float = 0.0, shared_cache_type: str = 'none', router_predicted_ttl_secs: Optional[float] = None, conditional_disagg_enabled: bool = False, conditional_disagg_policy: str = 'isl_bounding', conditional_disagg_eff_isl_threshold: int = 2048, conditional_disagg_eff_isl_ratio_threshold: float = 0.7, conditional_disagg_prefill_busy_threshold: Optional[float] = None, conditional_disagg_decode_busy_threshold: Optional[float] = None, overlap_score_credit: float = 1.0, overlap_score_credit_decay: float = 0.0, prefill_load_scale: float = 1.0, decode_active_request_weight: float = 0.0, router_policy_config: Optional[str] = None, router_prefill_policy: Optional[str] = None, router_decode_policy: Optional[str] = None, router_tracking_hash: Literal['public-xxh3-v1', 'keyed-xxh3-v1'] = 'public-xxh3-v1', router_tracking_key_file: Optional[str | os.PathLike[str]] = None, router_tracking_key_id: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1744`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1744) +[`lib/bindings/python/src/dynamo/_core.pyi#L1748`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1748) **Public methods** @@ -2535,7 +2535,7 @@ use_kv_events=True. Set to None to disable. Independent of router_ttl_secs, which covers pure approximate mode. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1747) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1751)

from_json

@@ -2545,7 +2545,7 @@ from_json(config_json: str) -> KvRouterConfig No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1854) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1858)

copy

@@ -2555,7 +2555,7 @@ copy() -> KvRouterConfig No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1858) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1862)

with_overrides

@@ -2565,7 +2565,7 @@ with_overrides(overlap_score_weight: Optional[float] = None, *, overlap_score_cr No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1884) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1888)
@@ -2579,7 +2579,7 @@ from dynamo._core import KvStateAgentHost KvStateAgentHost(endpoint: Endpoint, max_slots: int = 8) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2854`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2854) +[`lib/bindings/python/src/dynamo/_core.pyi#L2858`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2858) **Public methods** @@ -2591,7 +2591,7 @@ __init__(endpoint: Endpoint, max_slots: int = 8) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2855) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2859)

start

@@ -2601,7 +2601,7 @@ start() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2858) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2862)

status

@@ -2611,7 +2611,7 @@ status() -> Dict[str, Any] No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2861) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2865)

shutdown

@@ -2621,7 +2621,7 @@ shutdown() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2864) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2868)

wait_terminated

@@ -2631,7 +2631,7 @@ wait_terminated() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2867) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2871)
@@ -2645,7 +2645,7 @@ from dynamo._core import KvStateAttachmentOwner KvStateAttachmentOwner(endpoint: Endpoint, worker_id: int, descriptors: List[Dict[str, Any]]) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2870`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2870) +[`lib/bindings/python/src/dynamo/_core.pyi#L2874`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2874) **Public methods** @@ -2657,7 +2657,7 @@ __init__(endpoint: Endpoint, worker_id: int, descriptors: List[Dict[str, Any]]) No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2871) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2875)

start

@@ -2667,7 +2667,7 @@ start() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2876) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2880)

set_cache_readable

@@ -2677,7 +2677,7 @@ set_cache_readable(global_dp_rank: int, readable: bool) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2879) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2883)

close

@@ -2687,7 +2687,7 @@ close() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2884) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2888)
@@ -2697,7 +2697,7 @@ No summary available. from dynamo._core import KvStoredEventInput ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1140`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1140) +[`lib/bindings/python/src/dynamo/_core.pyi#L1144`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1144) @@ -2711,7 +2711,7 @@ from dynamo._core import KvbmRequest KvbmRequest(request_id: int, tokens: List[int], block_size: int) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2821`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2821) +[`lib/bindings/python/src/dynamo/_core.pyi#L2825`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2825) **Public methods** @@ -2723,7 +2723,7 @@ __init__(request_id: int, tokens: List[int], block_size: int) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2826) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2830) @@ -2733,7 +2733,7 @@ A KV cache block layer from dynamo._core import Layer ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2611`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2611) +[`lib/bindings/python/src/dynamo/_core.pyi#L2615`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2615) @@ -2747,7 +2747,7 @@ from dynamo._core import LoRADownloader LoRADownloader(cache_path: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2333`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2333) +[`lib/bindings/python/src/dynamo/_core.pyi#L2337`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2337) **Public methods** @@ -2759,7 +2759,7 @@ __init__(cache_path: Optional[str] = None) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2336) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2340)

download_if_needed

@@ -2769,7 +2769,7 @@ download_if_needed(lora_uri: str) -> Awaitable[str] No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2337) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2341)

get_cache_path

@@ -2779,7 +2779,7 @@ get_cache_path(cache_key: str) -> str No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2338) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2342)

is_cached

@@ -2789,7 +2789,7 @@ is_cached(lora_uri: str) -> bool No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2339) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2343)

validate_cached

@@ -2799,7 +2799,7 @@ validate_cached(cache_key: str) -> bool No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2340) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2344)

uri_to_cache_key

@@ -2809,7 +2809,7 @@ uri_to_cache_key(uri: str) -> str No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2342) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2346)
@@ -2823,7 +2823,7 @@ from dynamo._core import MediaDecoder MediaDecoder() -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2346`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2346) +[`lib/bindings/python/src/dynamo/_core.pyi#L2350`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2350) **Public methods** @@ -2835,7 +2835,7 @@ __init__() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2349) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2353)

enable_image

@@ -2845,7 +2845,7 @@ enable_image(decoder_options: Dict[str, Any]) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2350) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2354)
@@ -2859,7 +2859,7 @@ from dynamo._core import MediaFetcher MediaFetcher() -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2353`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2353) +[`lib/bindings/python/src/dynamo/_core.pyi#L2357`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2357) **Public methods** @@ -2871,7 +2871,7 @@ __init__() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2356) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2360)

user_agent

@@ -2881,7 +2881,7 @@ user_agent(user_agent: str) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2357) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2361)

allow_direct_ip

@@ -2891,7 +2891,7 @@ allow_direct_ip(allow: bool) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2358) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2362)

allow_direct_port

@@ -2901,7 +2901,7 @@ allow_direct_port(allow: bool) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2359) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2363)

allowed_media_domains

@@ -2911,7 +2911,7 @@ allowed_media_domains(domains: List[str]) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2360) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2364)

timeout_ms

@@ -2921,7 +2921,7 @@ timeout_ms(timeout_ms: int) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2361) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2365)
@@ -2935,7 +2935,7 @@ from dynamo._core import MockEngineArgs MockEngineArgs(engine_type: str = 'vllm', num_gpu_blocks: Optional[int] = None, block_size: int = 0, max_num_seqs: Optional[int] = 256, max_num_batched_tokens: Optional[int] = 8192, enable_prefix_caching: bool = True, enable_chunked_prefill: bool = True, speedup_ratio: float = 1.0, decode_speedup_ratio: float = 1.0, dp_size: int = 1, startup_time: Optional[float] = None, worker_type: str = 'aggregated', planner_profile_data: Optional[str | os.PathLike[str]] = None, aic_backend: Optional[str] = None, aic_system: Optional[str] = None, aic_backend_version: Optional[str] = None, aic_tp_size: Optional[int] = None, aic_model_path: Optional[str] = None, aic_moe_tp_size: Optional[int] = None, aic_moe_ep_size: Optional[int] = None, aic_attention_dp_size: Optional[int] = None, aic_nextn: Optional[int] = None, aic_nextn_accept_rates: Optional[str] = None, aic_mtp_seed: int = 42, aic_gemm_dtype: Optional[str] = None, aic_moe_dtype: Optional[str] = None, aic_fmha_dtype: Optional[str] = None, aic_kv_cache_dtype: Optional[str] = None, aic_comm_dtype: Optional[str] = None, gpu_memory_utilization: Optional[float] = None, mem_fraction_static: Optional[float] = None, free_gpu_memory_fraction: Optional[float] = None, enable_local_indexer: bool = False, bootstrap_port: Optional[int] = None, handoff_session_timeout_ms: int = 300000, kv_bytes_per_token: Optional[int] = None, kv_transfer_bandwidth: Optional[float] = None, kv_transfer_timing_mode: str = 'full_prompt', reasoning: Optional[ReasoningConfig] = None, response_replay_trace_path: Optional[str | os.PathLike[str]] = None, zmq_kv_events_port: Optional[int] = None, zmq_replay_port: Optional[int] = None, preemption_mode: str = 'lifo', router_queue_policy: Optional[str] = None, sglang: Optional[SglangArgs] = None, trtllm: Optional[TrtllmArgs] = None, num_g2_blocks: Optional[int] = None, num_g3_blocks: Optional[int] = None, offload_batch_size: Optional[int] = None, bandwidth_g1_to_g2_gbps: Optional[float] = None, bandwidth_g2_to_g1_gbps: Optional[float] = None, bandwidth_g2_to_g3_gbps: Optional[float] = None, bandwidth_g3_to_g2_gbps: Optional[float] = None, enable_g4_storage: bool = False, bandwidth_g2_to_g4_gbps: Optional[float] = None, bandwidth_g4_to_g2_gbps: Optional[float] = None, max_model_len: Optional[int] = None, g1_backend: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1922`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1922) +[`lib/bindings/python/src/dynamo/_core.pyi#L1926`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1926) **Public methods** @@ -2947,7 +2947,7 @@ __init__(engine_type: str = 'vllm', num_gpu_blocks: Optional[int] = None, block_ No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1923) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1927)

from_json

@@ -2957,7 +2957,7 @@ from_json(config_json: str) -> MockEngineArgs No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1986) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1990)

copy

@@ -2967,7 +2967,7 @@ copy() -> MockEngineArgs No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1990) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1994)

is_prefill

@@ -2977,7 +2977,7 @@ is_prefill() -> bool No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2190) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2194)

is_decode

@@ -2987,7 +2987,7 @@ is_decode() -> bool No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2192) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2196)

with_overrides

@@ -2997,7 +2997,7 @@ with_overrides(bootstrap_port: Optional[int] = None, zmq_kv_events_port: Optiona No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2194) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2198)
@@ -3029,7 +3029,7 @@ A model deployment card is a collection of model information from dynamo._core import ModelDeploymentCard ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L820`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L820) +[`lib/bindings/python/src/dynamo/_core.pyi#L824`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L824) **Public methods** @@ -3041,7 +3041,7 @@ to_json_str() -> str Serialize the model deployment card to a JSON string. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L825) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L829)

from_json_str

@@ -3051,7 +3051,7 @@ from_json_str(json: str) -> ModelDeploymentCard Deserialize a model deployment card from a JSON string. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L829) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L833)

model_type

@@ -3061,7 +3061,7 @@ model_type() -> ModelType Return the model type of this deployment card. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L834) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L838)

source_path

@@ -3071,7 +3071,7 @@ source_path() -> str Return the source path of this deployment card. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L838) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L842)

local_dir

@@ -3081,7 +3081,7 @@ local_dir() -> str Resolved metadata directory (post-`download_config`). Raises ValueError if the path contains non-UTF-8 bytes. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L842) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L846)

name

@@ -3091,7 +3091,7 @@ name() -> str Return the model name. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L847) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L851)

runtime_config

@@ -3101,7 +3101,7 @@ runtime_config() -> Any Return the runtime configuration as a dict. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L851) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L855)
@@ -3111,7 +3111,7 @@ What type of request this model needs: Text, Tokens or Tensor from dynamo._core import ModelInput ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1630`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1630) +[`lib/bindings/python/src/dynamo/_core.pyi#L1634`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1634) @@ -3125,7 +3125,7 @@ from dynamo._core import ModelRuntimeConfig ModelRuntimeConfig() -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L855`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L855) +[`lib/bindings/python/src/dynamo/_core.pyi#L859`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L859) **Public methods** @@ -3137,7 +3137,7 @@ __init__() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L885) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L889)

set_engine_specific

@@ -3147,7 +3147,7 @@ set_engine_specific(key: str, value: Any) -> None Set an engine-specific runtime configuration value -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L887) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L891)

get_engine_specific

@@ -3157,7 +3157,7 @@ get_engine_specific(key: str) -> Any | None Get an engine-specific runtime configuration value -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L891) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L895)

set_structural_tag_mode

@@ -3167,7 +3167,7 @@ set_structural_tag_mode(mode: str) -> None Set structural tag mode ("off" or "on"). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L895) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L899)

set_structural_tag_scope

@@ -3177,7 +3177,7 @@ set_structural_tag_scope(scope: str) -> None Set structural tag scope ("auto" or "always"). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L899) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L903)

set_structural_tag_schema

@@ -3187,7 +3187,7 @@ set_structural_tag_schema(schema: str) -> None Set structural tag schema mode ("auto" or "strict"). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L903) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L907)

set_disaggregated_endpoint

@@ -3197,7 +3197,7 @@ set_disaggregated_endpoint(bootstrap_host: str | None = None, bootstrap_port: in Set the disaggregated endpoint for the model -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L907) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L911)
@@ -3210,7 +3210,7 @@ from dynamo._core import ModelType Values are Chat, Completions, Embedding, Classify, Pooling, TensorBased, Images, Audios, Videos, Realtime, and Empty (no OpenAI surface). -[`lib/bindings/python/src/dynamo/_core.pyi#L1637`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1637) +[`lib/bindings/python/src/dynamo/_core.pyi#L1641`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1641) **Public methods** @@ -3222,7 +3222,7 @@ supports_chat() -> bool Return True if this model type supports chat. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1667) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1671)

supports_embedding

@@ -3232,7 +3232,7 @@ supports_embedding() -> bool Return True if this model type supports /v1/embeddings. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1671) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1675)

supports_classify

@@ -3242,7 +3242,7 @@ supports_classify() -> bool Return True if this model type supports /v1/classify. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1675) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1679)

supports_pooling

@@ -3252,7 +3252,7 @@ supports_pooling() -> bool Return True if this model type supports /v1/pooling. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1679) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1683)
@@ -3323,7 +3323,7 @@ A collection of prefix matching scores of workers for a given token ids. 'scores from dynamo._core import OverlapScores ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L935`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L935) +[`lib/bindings/python/src/dynamo/_core.pyi#L939`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L939) @@ -3333,7 +3333,7 @@ A request from planner to client to perform a scaling action. Fields: num_prefil from dynamo._core import PlannerDecision ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3214`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3214) +[`lib/bindings/python/src/dynamo/_core.pyi#L3218`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3218) @@ -3396,7 +3396,7 @@ from dynamo._core import PythonAsyncEngine PythonAsyncEngine(generator: Any, event_loop: Any) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1472`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1472) +[`lib/bindings/python/src/dynamo/_core.pyi#L1476`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1476) **Public methods** @@ -3408,7 +3408,7 @@ __init__(generator: Any, event_loop: Any) -> None Wrap a Python generator and event loop for use with Dynamo services. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1477) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1481) @@ -3425,7 +3425,7 @@ RadixTree() -> None Thread-safe: operations route to a dedicated background thread and long calls release the Python GIL. -[`lib/bindings/python/src/dynamo/_core.pyi#L962`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L962) +[`lib/bindings/python/src/dynamo/_core.pyi#L966`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L966) **Public methods** @@ -3437,7 +3437,7 @@ __init__() -> None Create a new RadixTree instance. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L970) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L974)

find_matches

@@ -3460,7 +3460,7 @@ If True, stop searching after finding the first match - `OverlapScores` — OverlapScores containing worker matching scores and frequencies -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L976) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L980)

apply_event

@@ -3483,7 +3483,7 @@ Serialized KV cache event as bytes - `ValueError` — If the event bytes cannot be deserialized -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L991) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L995)

remove_worker

@@ -3499,7 +3499,7 @@ Remove all blocks associated with a specific worker. ID of the worker to remove -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1004) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1008)

clear_all_blocks

@@ -3515,7 +3515,7 @@ Clear all blocks for a specific worker. ID of the worker whose blocks should be cleared -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1013) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1017)

dump_tree_as_events

@@ -3529,7 +3529,7 @@ Dump the current RadixTree state as a list of JSON-serialized KV cache events. - `List[str]` — List of JSON-serialized KV cache events as strings -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1022) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1026)
@@ -3543,7 +3543,7 @@ from dynamo._core import ReasoningConfig ReasoningConfig(start_thinking_token_id: int, end_thinking_token_id: int, thinking_ratio: float) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1894`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1894) +[`lib/bindings/python/src/dynamo/_core.pyi#L1898`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1898) **Public methods** @@ -3555,7 +3555,7 @@ __init__(start_thinking_token_id: int, end_thinking_token_id: int, thinking_rati No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1895) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1899) @@ -3569,7 +3569,7 @@ from dynamo._core import RouterConfig RouterConfig(mode: RouterMode, config: Optional[KvRouterConfig] = None, active_decode_blocks_threshold: Optional[float] = None, active_prefill_tokens_threshold: Optional[int] = None, active_prefill_tokens_threshold_frac: Optional[float] = None, enforce_disagg: bool = False, session_affinity_ttl_secs: Optional[int] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1694`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1694) +[`lib/bindings/python/src/dynamo/_core.pyi#L1698`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1698) **Public methods** @@ -3605,7 +3605,7 @@ Deprecated and ignored. Routing topology and readiness come from registered work Router-local session-affinity idle TTL in seconds. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1699) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1703) @@ -3615,7 +3615,7 @@ Router mode for load balancing requests across workers from dynamo._core import RouterMode ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1683`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1683) +[`lib/bindings/python/src/dynamo/_core.pyi#L1687`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1687) @@ -3625,7 +3625,7 @@ A policy-class queue cap rejected the request. from dynamo._core import RouterQueueLimitExceeded ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3278`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3278) +[`lib/bindings/python/src/dynamo/_core.pyi#L3282`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3282) @@ -3646,7 +3646,7 @@ and ``0.0`` is neutral. Matching weights are summed and squashed with ``tanh``, so opposite preferences cancel before Dynamo converts the bounded bias into a strictly positive score multiplier. -[`lib/bindings/python/src/dynamo/_core.pyi#L915`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L915) +[`lib/bindings/python/src/dynamo/_core.pyi#L919`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L919) **Public methods** @@ -3658,7 +3658,7 @@ __init__(required_taints: Optional[Set[str]] = None, preferred_taints: Optional[ No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L929) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L933) @@ -3841,6 +3841,16 @@ Free a finished reservation, releasing its tracked load. [source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L806) +

abort_reservation

+ +```python +abort_reservation(selection_id: str) -> None +``` + +Abort a failed reservation, releasing its load and reporting failure. + +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L810) +

loads

```python @@ -3849,7 +3859,7 @@ loads(*, model_name: Optional[str] = None, routing_group: Optional[str] = None) Current per-model active load (pending counts + per-worker potential loads). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L810) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L814)

potential_loads

@@ -3859,7 +3869,7 @@ potential_loads(request: JsonLike) -> JsonLike Per-worker potential loads for a prompt, without booking. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L816) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L820)
@@ -3869,7 +3879,7 @@ Raised by `SelectionService` for selector failures that are not malformed input. from dynamo._core import SelectionServiceError ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3326`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3326) +[`lib/bindings/python/src/dynamo/_core.pyi#L3330`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3330) @@ -3883,7 +3893,7 @@ from dynamo._core import SglangArgs SglangArgs(schedule_policy: Optional[str] = None, page_size: Optional[int] = None, max_prefill_tokens: Optional[int] = None, chunked_prefill_size: Optional[int] = None, clip_max_new_tokens: Optional[int] = None, schedule_conservativeness: Optional[float] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1903`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1903) +[`lib/bindings/python/src/dynamo/_core.pyi#L1907`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1907) **Public methods** @@ -3895,7 +3905,7 @@ __init__(schedule_policy: Optional[str] = None, page_size: Optional[int] = None, No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1904) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1908) @@ -3957,7 +3967,7 @@ The response stream was terminated before completion. from dynamo._core import StreamIncomplete ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3321`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3321) +[`lib/bindings/python/src/dynamo/_core.pyi#L3325`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3325) @@ -3981,7 +3991,7 @@ from dynamo._core import TrtllmArgs TrtllmArgs(capacity_scheduler_policy: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1915`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1915) +[`lib/bindings/python/src/dynamo/_core.pyi#L1919`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1919) **Public methods** @@ -3993,7 +4003,7 @@ __init__(capacity_scheduler_policy: Optional[str] = None) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1916) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1920) @@ -4003,7 +4013,7 @@ Uncategorized or unknown error. from dynamo._core import Unknown ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3286`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3286) +[`lib/bindings/python/src/dynamo/_core.pyi#L3290`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3290) @@ -4017,7 +4027,7 @@ from dynamo._core import VirtualConnectorClient VirtualConnectorClient(runtime: DistributedRuntime, dynamo_namespace: str) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3248`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3248) +[`lib/bindings/python/src/dynamo/_core.pyi#L3252`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3252) **Public methods** @@ -4029,7 +4039,7 @@ __init__(runtime: DistributedRuntime, dynamo_namespace: str) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3251) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3255)

get

@@ -4039,7 +4049,7 @@ get() -> PlannerDecision No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3254) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3258)

complete

@@ -4049,7 +4059,7 @@ complete(decision: PlannerDecision) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3257) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3261)

wait

@@ -4059,7 +4069,7 @@ wait() -> None Blocks until there is a new decision to fetch using 'get' -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3260) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3264)
@@ -4073,7 +4083,7 @@ from dynamo._core import VirtualConnectorCoordinator VirtualConnectorCoordinator(runtime: DistributedRuntime, dynamo_namespace: str, check_interval_secs: int, max_wait_time_secs: int, max_retries: int) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3224`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3224) +[`lib/bindings/python/src/dynamo/_core.pyi#L3228`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3228) **Public methods** @@ -4085,7 +4095,7 @@ __init__(runtime: DistributedRuntime, dynamo_namespace: str, check_interval_secs No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3227) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3231)

async_init

@@ -4095,7 +4105,7 @@ async_init() -> None Call this before using the object -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3230) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3234)

read_state

@@ -4105,7 +4115,7 @@ read_state() -> PlannerDecision Get the current values. Most for test / debug. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3234) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3238)

update_scaling_decision

@@ -4115,7 +4125,7 @@ update_scaling_decision(num_prefill: Optional[int] = None, num_decode: Optional[ No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3238) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3242)

wait_for_scaling_completion

@@ -4125,7 +4135,7 @@ wait_for_scaling_completion() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3241) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3245)

is_scaling_ready

@@ -4135,7 +4145,7 @@ is_scaling_ready() -> bool Return whether the client acknowledged the current scaling decision. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3244) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3248)
@@ -4218,7 +4228,7 @@ Each worker has exactly one role; values are not combinable. Use the needs (Prefill AND Decode) OR a single Aggregated peer is expressed as `[[WorkerType.Prefill, WorkerType.Decode], [WorkerType.Aggregated]]`. -[`lib/bindings/python/src/dynamo/_core.pyi#L2224`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2224) +[`lib/bindings/python/src/dynamo/_core.pyi#L2228`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2228) @@ -4228,7 +4238,7 @@ No summary available. from dynamo._core import backend ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3346`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3346) +[`lib/bindings/python/src/dynamo/_core.pyi#L3350`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3350) @@ -4301,7 +4311,7 @@ from dynamo._core import fetch_model fetch_model(remote_name: str, ignore_weights: bool = False) -> str ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2363`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2363) +[`lib/bindings/python/src/dynamo/_core.pyi#L2367`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2367) @@ -4357,7 +4367,7 @@ from dynamo._core import lora_name_to_id lora_name_to_id(lora_name: str) -> int ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2321`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2321) +[`lib/bindings/python/src/dynamo/_core.pyi#L2325`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2325) @@ -4371,7 +4381,7 @@ from dynamo._core import make_engine make_engine(distributed_runtime: DistributedRuntime, args: EntrypointArgs) -> EngineConfig ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2380`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2380) +[`lib/bindings/python/src/dynamo/_core.pyi#L2384`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2384) @@ -4407,7 +4417,7 @@ strategies — for example a prefill tier registered with When `ignore_weights` is true, remote HuggingFace model resolution skips weight files and downloads only the metadata needed for registration. -[`lib/bindings/python/src/dynamo/_core.pyi#L2243`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2243) +[`lib/bindings/python/src/dynamo/_core.pyi#L2247`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2247) @@ -4421,7 +4431,7 @@ from dynamo._core import resolve_routing_image_token_id resolve_routing_image_token_id(model_id: str, model_dir: str) -> Optional[int] ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2325`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2325) +[`lib/bindings/python/src/dynamo/_core.pyi#L2329`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2329) @@ -4438,7 +4448,7 @@ run_input(distributed_runtime: DistributedRuntime, input: str, engine_config: En ``frontend_route_extensions`` supplies additional HTTP routes to the frontend (HTTP input only); see ``FrontendRoute``. -[`lib/bindings/python/src/dynamo/_core.pyi#L2446`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2446) +[`lib/bindings/python/src/dynamo/_core.pyi#L2450`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2450) @@ -4496,7 +4506,7 @@ unregister_model(endpoint: Endpoint, lora_name: Optional[str] = None) -> None If lora_name is provided, unregisters a LoRA adapter instead of a base model. -[`lib/bindings/python/src/dynamo/_core.pyi#L2299`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2299) +[`lib/bindings/python/src/dynamo/_core.pyi#L2303`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2303) @@ -4513,5 +4523,5 @@ update_model_taints(endpoint: Endpoint, taints: Set[str]) -> None Reserved 'dynamo.topology/' taints are derived from the model's topology metadata and cannot be supplied by callers. -[`lib/bindings/python/src/dynamo/_core.pyi#L2310`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2310) +[`lib/bindings/python/src/dynamo/_core.pyi#L2314`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2314) diff --git a/docs/fern/pages/reference/api/python/llm.mdx b/docs/fern/pages/reference/api/python/llm.mdx index df84cfce7b2e..69f79def382f 100644 --- a/docs/fern/pages/reference/api/python/llm.mdx +++ b/docs/fern/pages/reference/api/python/llm.mdx @@ -20,7 +20,7 @@ from dynamo.llm import AicPerfConfig AicPerfConfig(aic_backend: str, aic_system: str, aic_model_path: str, aic_tp_size: int = 1, aic_backend_version: Optional[str] = None, aic_moe_tp_size: Optional[int] = None, aic_moe_ep_size: Optional[int] = None, aic_attention_dp_size: Optional[int] = None, aic_nextn: Optional[int] = None, aic_nextn_accept_rates: Optional[str] = None, aic_gemm_dtype: Optional[str] = None, aic_moe_dtype: Optional[str] = None, aic_fmha_dtype: Optional[str] = None, aic_kv_cache_dtype: Optional[str] = None, aic_comm_dtype: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1723`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1723) +[`lib/bindings/python/src/dynamo/_core.pyi#L1727`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1727) **Public methods** @@ -32,7 +32,7 @@ __init__(aic_backend: str, aic_system: str, aic_model_path: str, aic_tp_size: in No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1724) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1728) @@ -42,7 +42,7 @@ Engine type for Dynamo workers from dynamo.llm import EngineType ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3130`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3130) +[`lib/bindings/python/src/dynamo/_core.pyi#L3134`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3134) @@ -56,7 +56,7 @@ from dynamo.llm import EntrypointArgs EntrypointArgs(engine_type: EngineType, model_path: Optional[str] = None, model_name: Optional[str] = None, endpoint_id: Optional[str] = None, template_file: Optional[str] = None, router_config: Optional[RouterConfig] = None, kv_cache_block_size: Optional[int] = None, http_host: Optional[str] = None, http_port: Optional[int] = None, http_metrics_port: Optional[int] = None, tls_cert_path: Optional[str] = None, tls_key_path: Optional[str] = None, extra_engine_args: Optional[str] = None, mocker_engine_args: Optional[MockEngineArgs] = None, runtime_config: Optional[ModelRuntimeConfig] = None, namespace: Optional[str] = None, namespace_prefix: Optional[str] = None, is_prefill: bool = False, is_decode: bool = False, migration_limit: int = 0, migration_max_seq_len: Optional[int] = None, chat_engine_factory: Optional[Callable] = None, aic_perf_config: Optional[AicPerfConfig] = None, *, metrics_prefix: Optional[str] = None, enable_anthropic_api: Optional[bool] = None, strip_anthropic_preamble: Optional[bool] = None, enable_streaming_tool_dispatch: Optional[bool] = None, enable_streaming_reasoning_dispatch: Optional[bool] = None, tokenizer_backend: Optional[str] = None, tokenizer_fallback: Optional[bool] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3137`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3137) +[`lib/bindings/python/src/dynamo/_core.pyi#L3141`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3141) **Public methods** @@ -161,7 +161,7 @@ Optional tokenizer backend override ("default", "fastokens", or "basetenkenizer" Whether alternate tokenizer load failures fall back to HuggingFace -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3143) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3147) @@ -175,7 +175,7 @@ from dynamo.llm import FpmDirectPublisher FpmDirectPublisher(endpoint: Endpoint, worker_id: str, dp_size: int = 1) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1286`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1286) +[`lib/bindings/python/src/dynamo/_core.pyi#L1290`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1290) **Public methods** @@ -200,7 +200,7 @@ Number of DP ranks to allocate channels for. Use ``1`` when attention DP is disabled. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1297) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1301)

publish

@@ -220,7 +220,7 @@ var_queued_prefill_length, var_queued_decode_kv_tokens) are defaulted to 0.0 per the MVP scope; a follow-up PR can add Welford-based variance computation. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1314) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1318)

shutdown

@@ -230,7 +230,7 @@ shutdown() -> None Shut down the publisher and its per-rank serialization tasks. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1344) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1348)
@@ -244,7 +244,7 @@ from dynamo.llm import FpmEventRelay FpmEventRelay(endpoint: Endpoint, zmq_endpoint: str) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1259`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1259) +[`lib/bindings/python/src/dynamo/_core.pyi#L1263`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1263) **Public methods** @@ -266,7 +266,7 @@ Local ZMQ PUB address to subscribe to (e.g., "tcp://127.0.0.1:20380"). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1266) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1270)

shutdown

@@ -276,7 +276,7 @@ shutdown() -> None Shut down the relay task. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1281) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1285)
@@ -298,7 +298,7 @@ Two mutually exclusive usage modes: ``(worker_id, dp_rank)``. Stale entries are cleaned up when workers are removed (via discovery watch). -[`lib/bindings/python/src/dynamo/_core.pyi#L1349`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1349) +[`lib/bindings/python/src/dynamo/_core.pyi#L1353`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1353) **Public methods** @@ -319,7 +319,7 @@ No background tasks are started until ``recv()`` or Dynamo component endpoint (provides runtime + discovery). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1363) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1367)

recv

@@ -336,7 +336,7 @@ Cannot be used after ``start_tracking()``. - `Optional[bytes]` — Raw msgspec payload, or None if the stream is closed. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1375) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1379)

start_tracking

@@ -357,7 +357,7 @@ worker_id matches the removed instance_id are purged. After calling this, ``recv()`` will raise RuntimeError. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1388) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1392)

get_recent_stats

@@ -377,7 +377,7 @@ Raises RuntimeError if ``start_tracking()`` has not been called. - `dict[tuple[str, int], bytes]` — dict mapping ``(worker_id, dp_rank)`` to raw msgspec bytes. - `dict[tuple[str, int], bytes]` — Decode each value with ``forward_pass_metrics.decode(data)``. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1405) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1409)

get_model_cards

@@ -399,7 +399,7 @@ Raises RuntimeError if ``start_tracking()`` has not been called. - `dict[str, str]` — dict mapping ``worker_id`` to ``card_json`` (JSON string). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1420) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1424)

shutdown

@@ -409,7 +409,7 @@ shutdown() -> None Shut down the subscriber (all background tasks). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1437) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1441)
@@ -423,7 +423,7 @@ Handlers receive this and answer from current state. The surface is intentionally narrow (typed read-only accessors only); it does not expose the internal service state. -[`lib/bindings/python/src/dynamo/_core.pyi#L2384`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2384) +[`lib/bindings/python/src/dynamo/_core.pyi#L2388`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2388) **Public methods** @@ -435,7 +435,7 @@ is_ready() -> bool Whether the HTTP service has finished startup and is ready to serve. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2392) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2396)

is_cancelled

@@ -445,7 +445,7 @@ is_cancelled() -> bool Whether the frontend is shutting down (draining). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2396) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2400)

has_any_ready_model

@@ -455,7 +455,7 @@ has_any_ready_model() -> bool Whether at least one model is registered and ready to serve. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2400) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2404)

is_model_ready_to_serve

@@ -465,7 +465,7 @@ is_model_ready_to_serve(model: str) -> bool Whether the named model is registered and ready to serve. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2404) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2408)

model_display_names

@@ -475,7 +475,7 @@ model_display_names() -> list[str] Sorted display names of all registered models. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2408) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2412)

serving_ready_display_names

@@ -485,7 +485,7 @@ serving_ready_display_names() -> list[str] Sorted display names of models ready to serve. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2412) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2416)
@@ -502,7 +502,7 @@ FrontendResponse(status_code: int, body: object) -> None Return this to set a non-200 status (e.g. ``FrontendResponse(503, body)``); return a plain JSON-serializable value for the default 200. -[`lib/bindings/python/src/dynamo/_core.pyi#L2437`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2437) +[`lib/bindings/python/src/dynamo/_core.pyi#L2441`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2441) **Public methods** @@ -514,7 +514,7 @@ __init__(status_code: int, body: object) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2444) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2448) @@ -534,7 +534,7 @@ returns a JSON-serializable body (implies HTTP 200) or a ``FrontendResponse`` to set the status code. Async handlers and path parameters are rejected at construction. -[`lib/bindings/python/src/dynamo/_core.pyi#L2416`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2416) +[`lib/bindings/python/src/dynamo/_core.pyi#L2420`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2420) **Public methods** @@ -546,7 +546,7 @@ __init__(method: str, path: str, handler: Callable[[FrontendExtensionContext], o No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2426) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2430) @@ -556,7 +556,7 @@ An async engine for a distributed Dynamo http service. This is an extension of t from dynamo.llm import HttpAsyncEngine ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1483`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1483) +[`lib/bindings/python/src/dynamo/_core.pyi#L1487`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1487) @@ -596,7 +596,7 @@ from dynamo.llm import HttpService HttpService(port: Optional[int] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1442`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1442) +[`lib/bindings/python/src/dynamo/_core.pyi#L1446`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1446) **Public methods** @@ -614,7 +614,7 @@ Create a new HTTP service. Optional port number to bind the service to (default: 8080) -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1448) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1452)

run

@@ -630,7 +630,7 @@ Run the HTTP service. DistributedRuntime instance for token management -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1457) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1461)

shutdown

@@ -640,7 +640,7 @@ shutdown() -> None Shutdown the HTTP service by cancelling its internal token. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1466) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1470)
@@ -654,7 +654,7 @@ from dynamo.llm import KserveGrpcService KserveGrpcService(port: Optional[int] = None, host: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1492`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1492) +[`lib/bindings/python/src/dynamo/_core.pyi#L1496`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1496) **Public methods** @@ -675,7 +675,7 @@ Optional port number to bind the service to Optional host address to bind the service to -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1498) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1502)

add_completions_model

@@ -697,7 +697,7 @@ The model checksum The async engine to handle requests -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1508) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1512)

add_chat_completions_model

@@ -719,7 +719,7 @@ The model checksum The async engine to handle requests -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1524) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1528)

add_tensor_model

@@ -747,7 +747,7 @@ Optional runtime-resolved worker metadata Optional tensor protocol model metadata -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1540) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1544)

remove_completions_model

@@ -763,7 +763,7 @@ Remove a completions model from the service. The model name to remove -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1561) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1565)

remove_chat_completions_model

@@ -779,7 +779,7 @@ Remove a chat completions model from the service. The model name to remove -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1570) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1574)

remove_tensor_model

@@ -795,7 +795,7 @@ Remove a tensor model from the service. The model name to remove -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1579) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1583)

list_chat_completions_models

@@ -809,7 +809,7 @@ List all registered chat completions models. - `List[str]` — List of model names -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1588) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1592)

list_completions_models

@@ -823,7 +823,7 @@ List all registered completions models. - `List[str]` — List of model names -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1597) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1601)

list_tensor_models

@@ -837,7 +837,7 @@ List all registered tensor models. - `List[str]` — List of model names -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1606) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1610)

run

@@ -853,7 +853,7 @@ Run the KServe gRPC service. DistributedRuntime instance for token management -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1615) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1619)

shutdown

@@ -863,7 +863,7 @@ shutdown() -> None Shutdown the KServe gRPC service by cancelling its internal token. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1624) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1628)
@@ -877,7 +877,7 @@ from dynamo.llm import KvDcRelay KvDcRelay(endpoint: Endpoint, dc_id: str, namespace_filter: Optional[str] = None, endpoint_prefix: Optional[str] = None, publication_threshold: int = 16, publication_delay_ms: int = 1, recovery_attempt_timeout_ms: int = 30000) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2829`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2829) +[`lib/bindings/python/src/dynamo/_core.pyi#L2833`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2833) **Public methods** @@ -889,7 +889,7 @@ __init__(endpoint: Endpoint, dc_id: str, namespace_filter: Optional[str] = None, No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2830) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2834)

start

@@ -899,7 +899,7 @@ start() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2842) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2846)

health

@@ -909,7 +909,7 @@ health() -> Dict[str, Any] No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2845) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2849)

flush

@@ -919,7 +919,7 @@ flush() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2848) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2852)

shutdown

@@ -929,7 +929,7 @@ shutdown() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2851) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2855)
@@ -943,7 +943,7 @@ from dynamo.llm import KvEventPublisher KvEventPublisher(endpoint: Endpoint, worker_id: Optional[int] = None, kv_block_size: int = 0, dp_rank: int = 0, enable_local_indexer: bool = False, zmq_endpoint: Optional[str] = None, zmq_topic: Optional[str] = None, batching_timeout_ms: Optional[int] = None, image_token_id: Optional[int] = None, kv_state_endpoint: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1157`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1157) +[`lib/bindings/python/src/dynamo/_core.pyi#L1161`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1161) **Public methods** @@ -992,7 +992,7 @@ flushes at each submitted source-list boundary. KV event ownership endpoint; defaults to endpoint. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1164) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1168)

publish_stored

@@ -1031,7 +1031,7 @@ Optional Eagle mode flag. When true, stored blocks are reconstructed using overlapping `kv_block_size + 1` token windows. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1199) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1203)

publish_removed

@@ -1049,7 +1049,7 @@ Event IDs are managed internally by the publisher using a monotonic counter. List of block hashes to remove (signed 64-bit integers) -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1229) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1233)

publish_batch

@@ -1063,7 +1063,7 @@ The complete list is validated before it is enqueued. Compatible events are coalesced while preserving source order and the processor's existing block-count limits. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1240) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1244)

shutdown

@@ -1073,7 +1073,7 @@ shutdown() -> None Shuts down the event publisher, stopping any background tasks. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1252) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1256)
@@ -1087,7 +1087,7 @@ from dynamo.llm import KvRouter KvRouter(endpoint: Endpoint, block_size: int, kv_router_config: KvRouterConfig, aic_perf_config: Optional[AicPerfConfig] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2887`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2887) +[`lib/bindings/python/src/dynamo/_core.pyi#L2891`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2891) **Public methods** @@ -1114,7 +1114,7 @@ Configuration for the KV router Optional AIC perf-model config for effective prefill load tracking -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2892) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2896)

generate

@@ -1194,7 +1194,7 @@ multiple replicas (data_parallel_size > 1). - This is different from query_instance_id which doesn't route the request. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2910) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2914)

generate_from_request

@@ -1209,7 +1209,7 @@ Set response_buffer_size to 0 for demand-driven direct Python consumption; negative values are rejected. Returns an async iterator yielding generation responses. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2971) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2975)

best_worker

@@ -1256,7 +1256,7 @@ configured default family before cache-bucket resolution. - `Tuple[int, int, int]` — A tuple of (worker_id, dp_rank, overlap_blocks) where: - worker_id: The ID of the best matching worker - dp_rank: The data parallel rank of the selected worker - overlap_blocks: The number of overlapping blocks found -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2986) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2990)

get_potential_loads

@@ -1289,7 +1289,7 @@ Each (worker_id, dp_rank) pair is returned as a separate entry. If you need aggregated loads per worker_id, sum the values manually. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3028) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3032)

get_overlap_scores

@@ -1325,7 +1325,7 @@ Whether to query the configured shared cache. - `Dict[str, Any]` — workers. Each worker row is keyed by worker_id and dp_rank and - `Dict[str, Any]` — reports device, host-pinned, disk, and shared-cache overlap blocks. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3059) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3063)

dump_events

@@ -1339,7 +1339,7 @@ Dump all events from the KV router's indexer. - `str` — A JSON string containing all indexer events -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3087) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3091)

mark_prefill_complete

@@ -1364,7 +1364,7 @@ This is typically called automatically by the router when using the `best_worker()` with `request_id` for custom routing. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3096) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3100)

free

@@ -1389,7 +1389,7 @@ This is typically called automatically by the router when using the `best_worker()` with `request_id` for custom routing. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3113) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3117)
@@ -1403,7 +1403,7 @@ from dynamo.llm import KvRouterConfig KvRouterConfig(overlap_score_weight: Optional[float] = None, host_cache_hit_weight: float = 0.75, disk_cache_hit_weight: float = 0.25, router_temperature: float = 0.0, use_kv_events: bool = True, *, router_replica_sync: bool = False, router_track_active_blocks: bool = True, router_track_output_blocks: bool = False, router_assume_kv_reuse: bool = True, router_track_prefill_tokens: bool = True, router_prefill_load_model: str = 'none', router_ttl_secs: float = 120.0, router_queue_threshold: Optional[float] = None, router_event_threads: int = 4, router_queue_policy: str = 'fcfs', use_remote_indexer: bool = False, serve_indexer: bool = False, shared_cache_multiplier: float = 0.0, shared_cache_type: str = 'none', router_predicted_ttl_secs: Optional[float] = None, conditional_disagg_enabled: bool = False, conditional_disagg_policy: str = 'isl_bounding', conditional_disagg_eff_isl_threshold: int = 2048, conditional_disagg_eff_isl_ratio_threshold: float = 0.7, conditional_disagg_prefill_busy_threshold: Optional[float] = None, conditional_disagg_decode_busy_threshold: Optional[float] = None, overlap_score_credit: float = 1.0, overlap_score_credit_decay: float = 0.0, prefill_load_scale: float = 1.0, decode_active_request_weight: float = 0.0, router_policy_config: Optional[str] = None, router_prefill_policy: Optional[str] = None, router_decode_policy: Optional[str] = None, router_tracking_hash: Literal['public-xxh3-v1', 'keyed-xxh3-v1'] = 'public-xxh3-v1', router_tracking_key_file: Optional[str | os.PathLike[str]] = None, router_tracking_key_id: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1744`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1744) +[`lib/bindings/python/src/dynamo/_core.pyi#L1748`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1748) **Public methods** @@ -1548,7 +1548,7 @@ use_kv_events=True. Set to None to disable. Independent of router_ttl_secs, which covers pure approximate mode. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1747) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1751)

from_json

@@ -1558,7 +1558,7 @@ from_json(config_json: str) -> KvRouterConfig No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1854) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1858)

copy

@@ -1568,7 +1568,7 @@ copy() -> KvRouterConfig No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1858) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1862)

with_overrides

@@ -1578,7 +1578,7 @@ with_overrides(overlap_score_weight: Optional[float] = None, *, overlap_score_cr No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1884) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1888)
@@ -1592,7 +1592,7 @@ from dynamo.llm import KvStateAgentHost KvStateAgentHost(endpoint: Endpoint, max_slots: int = 8) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2854`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2854) +[`lib/bindings/python/src/dynamo/_core.pyi#L2858`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2858) **Public methods** @@ -1604,7 +1604,7 @@ __init__(endpoint: Endpoint, max_slots: int = 8) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2855) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2859)

start

@@ -1614,7 +1614,7 @@ start() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2858) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2862)

status

@@ -1624,7 +1624,7 @@ status() -> Dict[str, Any] No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2861) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2865)

shutdown

@@ -1634,7 +1634,7 @@ shutdown() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2864) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2868)

wait_terminated

@@ -1644,7 +1644,7 @@ wait_terminated() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2867) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2871)
@@ -1658,7 +1658,7 @@ from dynamo.llm import KvStateAttachmentOwner KvStateAttachmentOwner(endpoint: Endpoint, worker_id: int, descriptors: List[Dict[str, Any]]) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2870`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2870) +[`lib/bindings/python/src/dynamo/_core.pyi#L2874`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2874) **Public methods** @@ -1670,7 +1670,7 @@ __init__(endpoint: Endpoint, worker_id: int, descriptors: List[Dict[str, Any]]) No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2871) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2875)

start

@@ -1680,7 +1680,7 @@ start() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2876) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2880)

set_cache_readable

@@ -1690,7 +1690,7 @@ set_cache_readable(global_dp_rank: int, readable: bool) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2879) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2883)

close

@@ -1700,7 +1700,7 @@ close() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2884) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2888)
@@ -1714,7 +1714,7 @@ from dynamo.llm import LoRADownloader LoRADownloader(cache_path: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2333`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2333) +[`lib/bindings/python/src/dynamo/_core.pyi#L2337`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2337) **Public methods** @@ -1726,7 +1726,7 @@ __init__(cache_path: Optional[str] = None) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2336) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2340)

download_if_needed

@@ -1736,7 +1736,7 @@ download_if_needed(lora_uri: str) -> Awaitable[str] No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2337) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2341)

get_cache_path

@@ -1746,7 +1746,7 @@ get_cache_path(cache_key: str) -> str No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2338) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2342)

is_cached

@@ -1756,7 +1756,7 @@ is_cached(lora_uri: str) -> bool No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2339) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2343)

validate_cached

@@ -1766,7 +1766,7 @@ validate_cached(cache_key: str) -> bool No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2340) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2344)

uri_to_cache_key

@@ -1776,7 +1776,7 @@ uri_to_cache_key(uri: str) -> str No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2342) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2346)
@@ -1790,7 +1790,7 @@ from dynamo.llm import MediaDecoder MediaDecoder() -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2346`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2346) +[`lib/bindings/python/src/dynamo/_core.pyi#L2350`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2350) **Public methods** @@ -1802,7 +1802,7 @@ __init__() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2349) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2353)

enable_image

@@ -1812,7 +1812,7 @@ enable_image(decoder_options: Dict[str, Any]) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2350) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2354)
@@ -1826,7 +1826,7 @@ from dynamo.llm import MediaFetcher MediaFetcher() -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2353`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2353) +[`lib/bindings/python/src/dynamo/_core.pyi#L2357`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2357) **Public methods** @@ -1838,7 +1838,7 @@ __init__() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2356) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2360)

user_agent

@@ -1848,7 +1848,7 @@ user_agent(user_agent: str) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2357) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2361)

allow_direct_ip

@@ -1858,7 +1858,7 @@ allow_direct_ip(allow: bool) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2358) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2362)

allow_direct_port

@@ -1868,7 +1868,7 @@ allow_direct_port(allow: bool) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2359) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2363)

allowed_media_domains

@@ -1878,7 +1878,7 @@ allowed_media_domains(domains: List[str]) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2360) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2364)

timeout_ms

@@ -1888,7 +1888,7 @@ timeout_ms(timeout_ms: int) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2361) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2365)
@@ -1920,7 +1920,7 @@ What type of request this model needs: Text, Tokens or Tensor from dynamo.llm import ModelInput ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1630`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1630) +[`lib/bindings/python/src/dynamo/_core.pyi#L1634`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1634) @@ -1934,7 +1934,7 @@ from dynamo.llm import ModelRuntimeConfig ModelRuntimeConfig() -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L855`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L855) +[`lib/bindings/python/src/dynamo/_core.pyi#L859`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L859) **Public methods** @@ -1946,7 +1946,7 @@ __init__() -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L885) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L889)

set_engine_specific

@@ -1956,7 +1956,7 @@ set_engine_specific(key: str, value: Any) -> None Set an engine-specific runtime configuration value -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L887) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L891)

get_engine_specific

@@ -1966,7 +1966,7 @@ get_engine_specific(key: str) -> Any | None Get an engine-specific runtime configuration value -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L891) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L895)

set_structural_tag_mode

@@ -1976,7 +1976,7 @@ set_structural_tag_mode(mode: str) -> None Set structural tag mode ("off" or "on"). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L895) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L899)

set_structural_tag_scope

@@ -1986,7 +1986,7 @@ set_structural_tag_scope(scope: str) -> None Set structural tag scope ("auto" or "always"). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L899) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L903)

set_structural_tag_schema

@@ -1996,7 +1996,7 @@ set_structural_tag_schema(schema: str) -> None Set structural tag schema mode ("auto" or "strict"). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L903) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L907)

set_disaggregated_endpoint

@@ -2006,7 +2006,7 @@ set_disaggregated_endpoint(bootstrap_host: str | None = None, bootstrap_port: in Set the disaggregated endpoint for the model -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L907) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L911)
@@ -2019,7 +2019,7 @@ from dynamo.llm import ModelType Values are Chat, Completions, Embedding, Classify, Pooling, TensorBased, Images, Audios, Videos, Realtime, and Empty (no OpenAI surface). -[`lib/bindings/python/src/dynamo/_core.pyi#L1637`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1637) +[`lib/bindings/python/src/dynamo/_core.pyi#L1641`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1641) **Public methods** @@ -2031,7 +2031,7 @@ supports_chat() -> bool Return True if this model type supports chat. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1667) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1671)

supports_embedding

@@ -2041,7 +2041,7 @@ supports_embedding() -> bool Return True if this model type supports /v1/embeddings. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1671) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1675)

supports_classify

@@ -2051,7 +2051,7 @@ supports_classify() -> bool Return True if this model type supports /v1/classify. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1675) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1679)

supports_pooling

@@ -2061,7 +2061,7 @@ supports_pooling() -> bool Return True if this model type supports /v1/pooling. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1679) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1683)
@@ -2132,7 +2132,7 @@ A collection of prefix matching scores of workers for a given token ids. 'scores from dynamo.llm import OverlapScores ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L935`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L935) +[`lib/bindings/python/src/dynamo/_core.pyi#L939`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L939) @@ -2146,7 +2146,7 @@ from dynamo.llm import PythonAsyncEngine PythonAsyncEngine(generator: Any, event_loop: Any) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1472`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1472) +[`lib/bindings/python/src/dynamo/_core.pyi#L1476`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1476) **Public methods** @@ -2158,7 +2158,7 @@ __init__(generator: Any, event_loop: Any) -> None Wrap a Python generator and event loop for use with Dynamo services. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1477) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1481) @@ -2175,7 +2175,7 @@ RadixTree() -> None Thread-safe: operations route to a dedicated background thread and long calls release the Python GIL. -[`lib/bindings/python/src/dynamo/_core.pyi#L962`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L962) +[`lib/bindings/python/src/dynamo/_core.pyi#L966`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L966) **Public methods** @@ -2187,7 +2187,7 @@ __init__() -> None Create a new RadixTree instance. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L970) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L974)

find_matches

@@ -2210,7 +2210,7 @@ If True, stop searching after finding the first match - `OverlapScores` — OverlapScores containing worker matching scores and frequencies -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L976) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L980)

apply_event

@@ -2233,7 +2233,7 @@ Serialized KV cache event as bytes - `ValueError` — If the event bytes cannot be deserialized -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L991) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L995)

remove_worker

@@ -2249,7 +2249,7 @@ Remove all blocks associated with a specific worker. ID of the worker to remove -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1004) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1008)

clear_all_blocks

@@ -2265,7 +2265,7 @@ Clear all blocks for a specific worker. ID of the worker whose blocks should be cleared -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1013) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1017)

dump_tree_as_events

@@ -2279,7 +2279,7 @@ Dump the current RadixTree state as a list of JSON-serialized KV cache events. - `List[str]` — List of JSON-serialized KV cache events as strings -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1022) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1026)
@@ -2315,7 +2315,7 @@ from dynamo.llm import RouterConfig RouterConfig(mode: RouterMode, config: Optional[KvRouterConfig] = None, active_decode_blocks_threshold: Optional[float] = None, active_prefill_tokens_threshold: Optional[int] = None, active_prefill_tokens_threshold_frac: Optional[float] = None, enforce_disagg: bool = False, session_affinity_ttl_secs: Optional[int] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1694`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1694) +[`lib/bindings/python/src/dynamo/_core.pyi#L1698`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1698) **Public methods** @@ -2351,7 +2351,7 @@ Deprecated and ignored. Routing topology and readiness come from registered work Router-local session-affinity idle TTL in seconds. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1699) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1703) @@ -2361,7 +2361,7 @@ Router mode for load balancing requests across workers from dynamo.llm import RouterMode ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1683`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1683) +[`lib/bindings/python/src/dynamo/_core.pyi#L1687`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1687) @@ -2371,7 +2371,7 @@ A policy-class queue cap rejected the request. from dynamo.llm import RouterQueueLimitExceeded ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L3278`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3278) +[`lib/bindings/python/src/dynamo/_core.pyi#L3282`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3282) @@ -2392,7 +2392,7 @@ and ``0.0`` is neutral. Matching weights are summed and squashed with ``tanh``, so opposite preferences cancel before Dynamo converts the bounded bias into a strictly positive score multiplier. -[`lib/bindings/python/src/dynamo/_core.pyi#L915`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L915) +[`lib/bindings/python/src/dynamo/_core.pyi#L919`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L919) **Public methods** @@ -2404,7 +2404,7 @@ __init__(required_taints: Optional[Set[str]] = None, preferred_taints: Optional[ No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L929) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L933) @@ -2561,6 +2561,16 @@ Free a finished reservation, releasing its tracked load. [source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L806) +

abort_reservation

+ +```python +abort_reservation(selection_id: str) -> None +``` + +Abort a failed reservation, releasing its load and reporting failure. + +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L810) +

loads

```python @@ -2569,7 +2579,7 @@ loads(*, model_name: Optional[str] = None, routing_group: Optional[str] = None) Current per-model active load (pending counts + per-worker potential loads). -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L810) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L814)

potential_loads

@@ -2579,7 +2589,7 @@ potential_loads(request: JsonLike) -> JsonLike Per-worker potential loads for a prompt, without booking. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L816) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L820)
@@ -2662,7 +2672,7 @@ Each worker has exactly one role; values are not combinable. Use the needs (Prefill AND Decode) OR a single Aggregated peer is expressed as `[[WorkerType.Prefill, WorkerType.Decode], [WorkerType.Aggregated]]`. -[`lib/bindings/python/src/dynamo/_core.pyi#L2224`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2224) +[`lib/bindings/python/src/dynamo/_core.pyi#L2228`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2228) @@ -2735,7 +2745,7 @@ from dynamo.llm import fetch_model fetch_model(remote_name: str, ignore_weights: bool = False) -> str ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2363`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2363) +[`lib/bindings/python/src/dynamo/_core.pyi#L2367`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2367) @@ -2749,7 +2759,7 @@ from dynamo.llm import lora_name_to_id lora_name_to_id(lora_name: str) -> int ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2321`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2321) +[`lib/bindings/python/src/dynamo/_core.pyi#L2325`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2325) @@ -2763,7 +2773,7 @@ from dynamo.llm import make_engine make_engine(distributed_runtime: DistributedRuntime, args: EntrypointArgs) -> EngineConfig ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L2380`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2380) +[`lib/bindings/python/src/dynamo/_core.pyi#L2384`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2384) @@ -2799,7 +2809,7 @@ strategies — for example a prefill tier registered with When `ignore_weights` is true, remote HuggingFace model resolution skips weight files and downloads only the metadata needed for registration. -[`lib/bindings/python/src/dynamo/_core.pyi#L2243`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2243) +[`lib/bindings/python/src/dynamo/_core.pyi#L2247`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2247) @@ -2816,7 +2826,7 @@ run_input(distributed_runtime: DistributedRuntime, input: str, engine_config: En ``frontend_route_extensions`` supplies additional HTTP routes to the frontend (HTTP input only); see ``FrontendRoute``. -[`lib/bindings/python/src/dynamo/_core.pyi#L2446`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2446) +[`lib/bindings/python/src/dynamo/_core.pyi#L2450`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2450) @@ -2874,7 +2884,7 @@ unregister_model(endpoint: Endpoint, lora_name: Optional[str] = None) -> None If lora_name is provided, unregisters a LoRA adapter instead of a base model. -[`lib/bindings/python/src/dynamo/_core.pyi#L2299`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2299) +[`lib/bindings/python/src/dynamo/_core.pyi#L2303`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2303) @@ -2891,5 +2901,5 @@ update_model_taints(endpoint: Endpoint, taints: Set[str]) -> None Reserved 'dynamo.topology/' taints are derived from the model's topology metadata and cannot be supplied by callers. -[`lib/bindings/python/src/dynamo/_core.pyi#L2310`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2310) +[`lib/bindings/python/src/dynamo/_core.pyi#L2314`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2314) diff --git a/docs/fern/pages/reference/api/python/mocker.mdx b/docs/fern/pages/reference/api/python/mocker.mdx index aa8505ff91bb..80a12b33bc6d 100644 --- a/docs/fern/pages/reference/api/python/mocker.mdx +++ b/docs/fern/pages/reference/api/python/mocker.mdx @@ -20,7 +20,7 @@ from dynamo.mocker import MockEngineArgs MockEngineArgs(engine_type: str = 'vllm', num_gpu_blocks: Optional[int] = None, block_size: int = 0, max_num_seqs: Optional[int] = 256, max_num_batched_tokens: Optional[int] = 8192, enable_prefix_caching: bool = True, enable_chunked_prefill: bool = True, speedup_ratio: float = 1.0, decode_speedup_ratio: float = 1.0, dp_size: int = 1, startup_time: Optional[float] = None, worker_type: str = 'aggregated', planner_profile_data: Optional[str | os.PathLike[str]] = None, aic_backend: Optional[str] = None, aic_system: Optional[str] = None, aic_backend_version: Optional[str] = None, aic_tp_size: Optional[int] = None, aic_model_path: Optional[str] = None, aic_moe_tp_size: Optional[int] = None, aic_moe_ep_size: Optional[int] = None, aic_attention_dp_size: Optional[int] = None, aic_nextn: Optional[int] = None, aic_nextn_accept_rates: Optional[str] = None, aic_mtp_seed: int = 42, aic_gemm_dtype: Optional[str] = None, aic_moe_dtype: Optional[str] = None, aic_fmha_dtype: Optional[str] = None, aic_kv_cache_dtype: Optional[str] = None, aic_comm_dtype: Optional[str] = None, gpu_memory_utilization: Optional[float] = None, mem_fraction_static: Optional[float] = None, free_gpu_memory_fraction: Optional[float] = None, enable_local_indexer: bool = False, bootstrap_port: Optional[int] = None, handoff_session_timeout_ms: int = 300000, kv_bytes_per_token: Optional[int] = None, kv_transfer_bandwidth: Optional[float] = None, kv_transfer_timing_mode: str = 'full_prompt', reasoning: Optional[ReasoningConfig] = None, response_replay_trace_path: Optional[str | os.PathLike[str]] = None, zmq_kv_events_port: Optional[int] = None, zmq_replay_port: Optional[int] = None, preemption_mode: str = 'lifo', router_queue_policy: Optional[str] = None, sglang: Optional[SglangArgs] = None, trtllm: Optional[TrtllmArgs] = None, num_g2_blocks: Optional[int] = None, num_g3_blocks: Optional[int] = None, offload_batch_size: Optional[int] = None, bandwidth_g1_to_g2_gbps: Optional[float] = None, bandwidth_g2_to_g1_gbps: Optional[float] = None, bandwidth_g2_to_g3_gbps: Optional[float] = None, bandwidth_g3_to_g2_gbps: Optional[float] = None, enable_g4_storage: bool = False, bandwidth_g2_to_g4_gbps: Optional[float] = None, bandwidth_g4_to_g2_gbps: Optional[float] = None, max_model_len: Optional[int] = None, g1_backend: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1922`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1922) +[`lib/bindings/python/src/dynamo/_core.pyi#L1926`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1926) **Public methods** @@ -32,7 +32,7 @@ __init__(engine_type: str = 'vllm', num_gpu_blocks: Optional[int] = None, block_ No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1923) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1927)

from_json

@@ -42,7 +42,7 @@ from_json(config_json: str) -> MockEngineArgs No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1986) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1990)

copy

@@ -52,7 +52,7 @@ copy() -> MockEngineArgs No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1990) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1994)

is_prefill

@@ -62,7 +62,7 @@ is_prefill() -> bool No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2190) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2194)

is_decode

@@ -72,7 +72,7 @@ is_decode() -> bool No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2192) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2196)

with_overrides

@@ -82,7 +82,7 @@ with_overrides(bootstrap_port: Optional[int] = None, zmq_kv_events_port: Optiona No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2194) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2198) @@ -122,7 +122,7 @@ from dynamo.mocker import ReasoningConfig ReasoningConfig(start_thinking_token_id: int, end_thinking_token_id: int, thinking_ratio: float) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1894`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1894) +[`lib/bindings/python/src/dynamo/_core.pyi#L1898`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1898) **Public methods** @@ -134,7 +134,7 @@ __init__(start_thinking_token_id: int, end_thinking_token_id: int, thinking_rati No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1895) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1899) @@ -148,7 +148,7 @@ from dynamo.mocker import SglangArgs SglangArgs(schedule_policy: Optional[str] = None, page_size: Optional[int] = None, max_prefill_tokens: Optional[int] = None, chunked_prefill_size: Optional[int] = None, clip_max_new_tokens: Optional[int] = None, schedule_conservativeness: Optional[float] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1903`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1903) +[`lib/bindings/python/src/dynamo/_core.pyi#L1907`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1907) **Public methods** @@ -160,7 +160,7 @@ __init__(schedule_policy: Optional[str] = None, page_size: Optional[int] = None, No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1904) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1908) @@ -174,7 +174,7 @@ from dynamo.mocker import TrtllmArgs TrtllmArgs(capacity_scheduler_policy: Optional[str] = None) -> None ``` -[`lib/bindings/python/src/dynamo/_core.pyi#L1915`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1915) +[`lib/bindings/python/src/dynamo/_core.pyi#L1919`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1919) **Public methods** @@ -186,7 +186,7 @@ __init__(capacity_scheduler_policy: Optional[str] = None) -> None No summary available. -[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1916) +[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1920) diff --git a/lib/bindings/python/rust/llm/kv.rs b/lib/bindings/python/rust/llm/kv.rs index 6e1fcf31c655..910c33c4c731 100644 --- a/lib/bindings/python/rust/llm/kv.rs +++ b/lib/bindings/python/rust/llm/kv.rs @@ -796,6 +796,21 @@ impl SelectionService { }) } + /// Abort a failed reservation, releasing its load and reporting failure. + fn abort_reservation<'p>( + &self, + py: Python<'p>, + selection_id: String, + ) -> PyResult> { + let core = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + core.abort_reservation(&selection_id) + .await + .map_err(selection_to_pyerr)?; + Ok(()) + }) + } + /// Current per-model active load (pending counts + per-worker potential loads). #[pyo3(signature = (*, model_name = None, routing_group = None))] fn loads( diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi index 1401ac456b7e..22b9b128cbca 100644 --- a/lib/bindings/python/src/dynamo/_core.pyi +++ b/lib/bindings/python/src/dynamo/_core.pyi @@ -807,6 +807,10 @@ class SelectionService: """Free a finished reservation, releasing its tracked load.""" ... + async def abort_reservation(self, selection_id: str) -> None: + """Abort a failed reservation, releasing its load and reporting failure.""" + ... + def loads( self, *, model_name: Optional[str] = None, routing_group: Optional[str] = None ) -> JsonLike: diff --git a/lib/bindings/python/tests/test_kv_bindings.py b/lib/bindings/python/tests/test_kv_bindings.py index 66035f009813..27cf637d8ff5 100644 --- a/lib/bindings/python/tests/test_kv_bindings.py +++ b/lib/bindings/python/tests/test_kv_bindings.py @@ -238,6 +238,10 @@ async def test_selection_service_selects_registered_worker(monkeypatch): assert selected["worker_id"] == 1 assert selected["dp_rank"] == 0 assert selected["endpoint"] == "http://worker-1:8000" + + # Unknown releases are idempotent, but both terminal paths must be exposed. + await service.free_reservation("unknown-completed-selection") + await service.abort_reservation("unknown-aborted-selection") finally: await service.shutdown_async() diff --git a/lib/kv-router/src/services/selection/core/mod.rs b/lib/kv-router/src/services/selection/core/mod.rs index bd6c88593ac4..a52709a43564 100644 --- a/lib/kv-router/src/services/selection/core/mod.rs +++ b/lib/kv-router/src/services/selection/core/mod.rs @@ -101,6 +101,12 @@ struct ReservationBooking { lora_name: Option, } +#[derive(Clone, Copy)] +enum ReservationOutcome { + Completed, + Aborted, +} + #[derive(Debug, Clone)] pub struct SelectionServiceConfig { pub port: u16, @@ -752,7 +758,7 @@ impl SelectionCore { ) .await?; let mode = if book { - ScheduleMode::Tracked { + ScheduleMode::TrackedWithLifecycle { request_id: selection_id.clone().ok_or_else(|| { SelectionError::Internal( "booked selection did not include a selection ID".to_string(), @@ -810,15 +816,34 @@ impl SelectionCore { } result = entry.scheduler.schedule_request(schedule_request) => result?, }; - let endpoint = self + let endpoint = match self .catalog .schedulable_endpoint(response.best_worker.worker_id, &key) - .ok_or_else(|| { - SelectionError::Internal(format!( + { + Some(endpoint) => endpoint, + None => { + if book && let Some(selection_id) = selection_id.as_deref() { + match entry.scheduler.abort_if_present(selection_id).await { + Ok(true) => {} + Ok(false) => tracing::error!( + %selection_id, + %key, + "Booked selection disappeared before endpoint resolution cleanup" + ), + Err(error) => tracing::error!( + %selection_id, + %key, + %error, + "Failed to abort booked selection after endpoint resolution failed" + ), + } + } + return Err(SelectionError::Internal(format!( "selected worker {} is no longer schedulable", response.best_worker.worker_id - )) - })?; + ))); + } + }; let overlap = MooncakeOverlapSummary::from_selected_worker_tiers( &response.selected_worker_tiers, entry.block_size, @@ -1033,6 +1058,13 @@ impl SelectionCore { lora_name, } = booking; + if entry.scheduler.has_admission_policy() { + return Err(SelectionError::BadRequest( + "create_reservation is not supported with a queue admission policy; use select_and_reserve" + .to_string(), + )); + } + // Strict booking: never lazily recreate a worker/rank removed since the // reservation was resolved. entry @@ -1073,17 +1105,39 @@ impl SelectionCore { } pub async fn free_reservation(&self, selection_id: &str) -> Result<(), SelectionError> { - let entries = self.initialized_entries(); + self.release_reservation(selection_id, ReservationOutcome::Completed) + .await + } + + pub async fn abort_reservation(&self, selection_id: &str) -> Result<(), SelectionError> { + self.release_reservation(selection_id, ReservationOutcome::Aborted) + .await + } + + async fn release_reservation( + &self, + selection_id: &str, + outcome: ReservationOutcome, + ) -> Result<(), SelectionError> { + let mut entries = self.initialized_entries(); + entries.sort_unstable_by(|left, right| { + (&left.key.model_name, &left.key.routing_group) + .cmp(&(&right.key.model_name, &right.key.routing_group)) + }); for entry in entries { - match entry.scheduler.free(selection_id).await { - Ok(()) => return Ok(()), - Err(SequenceError::RequestNotFound { .. }) => continue, + let result = match outcome { + ReservationOutcome::Completed => { + entry.scheduler.free_if_present(selection_id).await + } + ReservationOutcome::Aborted => entry.scheduler.abort_if_present(selection_id).await, + }; + match result { + Ok(true) => return Ok(()), + Ok(false) | Err(SequenceError::RequestNotFound { .. }) => continue, Err(error) => return Err(error.into()), } } - Err(SelectionError::NotFound(format!( - "reservation {selection_id} not found" - ))) + Ok(()) } pub fn add_output_block( @@ -1266,7 +1320,16 @@ impl Drop for SelectionCore { mod tests { use super::*; use crate::protocols::StorageTier; + use crate::scheduling::selector::{ + WorkerInputView, WorkerPicker, WorkerSelectionContext, WorkerSelectionPolicy, + }; + use crate::scheduling::{ + QueueAdmissionDecision, QueueAdmissionEvent, QueueAdmissionId, QueueAdmissionPolicy, + QueueAdmissionRequest, WorkerSelectionPolicyError, + }; use crate::services::indexer::backend::test_util::store_event; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{OnceLock, Weak}; use std::time::Duration; fn test_config(use_kv_events: bool) -> crate::config::KvRouterConfig { @@ -1277,6 +1340,62 @@ mod tests { } } + struct CatalogRemovingPicker { + core: Weak, + worker_id: WorkerId, + } + + impl WorkerPicker for CatalogRemovingPicker { + fn pick( + &mut self, + _context: &WorkerSelectionContext<'_>, + input: WorkerInputView<'_>, + ) -> Result { + assert!(!input.candidates().is_empty()); + self.core + .upgrade() + .expect("selection core") + .catalog + .set_lifecycle(self.worker_id, WorkerLifecycle::Unschedulable, Vec::new()) + .expect("selected worker catalog record"); + Ok(0) + } + } + + struct AbortCountingAdmissionPolicy { + aborted: Arc, + } + + impl QueueAdmissionPolicy for AbortCountingAdmissionPolicy { + fn admit(&mut self, _request: QueueAdmissionRequest<'_>) -> QueueAdmissionDecision { + QueueAdmissionDecision::Ready + } + + fn on_event(&mut self, event: QueueAdmissionEvent<'_>, _ready: &mut Vec) { + if matches!(event, QueueAdmissionEvent::Aborted { .. }) { + self.aborted.fetch_add(1, Ordering::Relaxed); + } + } + } + + struct DeferredAbortAdmissionPolicy { + deferred: Arc, + aborted: Arc, + } + + impl QueueAdmissionPolicy for DeferredAbortAdmissionPolicy { + fn admit(&mut self, _request: QueueAdmissionRequest<'_>) -> QueueAdmissionDecision { + self.deferred.fetch_add(1, Ordering::Relaxed); + QueueAdmissionDecision::Defer + } + + fn on_event(&mut self, event: QueueAdmissionEvent<'_>, _ready: &mut Vec) { + if matches!(event, QueueAdmissionEvent::Aborted { .. }) { + self.aborted.fetch_add(1, Ordering::Relaxed); + } + } + } + fn worker(worker_id: WorkerId) -> WorkerRequest { WorkerRequest { worker_id, @@ -1483,6 +1602,154 @@ mod tests { ); } + #[tokio::test] + async fn endpoint_resolution_failure_aborts_the_booked_admission() { + let config = test_config(false); + let tracking_hash = Arc::new(TrackingHashContext::from_config(&config).unwrap()); + let aborted = Arc::new(AtomicUsize::new(0)); + let observed_aborted = Arc::clone(&aborted); + let core_ref = Arc::new(OnceLock::>::new()); + let factory_core_ref = Arc::clone(&core_ref); + let factory: WorkerSelectionPolicyFactory = Arc::new( + move |config: &crate::config::KvRouterConfig, worker_type, _partition| { + WorkerSelectionPolicy::new( + config.clone(), + worker_type.as_str(), + Vec::new(), + Box::new(CatalogRemovingPicker { + core: factory_core_ref + .get() + .expect("core reference installed") + .clone(), + worker_id: 1, + }), + ) + .with_admission_policy(Box::new(AbortCountingAdmissionPolicy { + aborted: Arc::clone(&observed_aborted), + })) + }, + ); + let core = Arc::new(SelectionCore::new_inner( + config, + 1, + CancellationToken::new(), + None, + Some(factory), + true, + SelectionCacheConfig::default(), + tracking_hash, + )); + core_ref + .set(Arc::downgrade(&core)) + .expect("install core reference"); + core.upsert_worker(worker(1)).await.expect("worker upsert"); + + let error = core + .select_and_reserve(reserve_request("endpoint-race")) + .await + .expect_err("endpoint lookup should fail after catalog removal"); + assert!(matches!( + error, + SelectionError::Internal(message) + if message.contains("selected worker 1 is no longer schedulable") + )); + assert_eq!(aborted.load(Ordering::Relaxed), 1); + let entry = core + .entry(&RoutingPartitionId::new("model", "default")) + .expect("selection entry"); + assert!( + !entry + .scheduler + .free_if_present("endpoint-race") + .await + .unwrap() + ); + } + + #[tokio::test] + async fn aborting_deferred_reservation_retracts_selection() { + let config = test_config(false); + let tracking_hash = Arc::new(TrackingHashContext::from_config(&config).unwrap()); + let deferred = Arc::new(AtomicUsize::new(0)); + let aborted = Arc::new(AtomicUsize::new(0)); + let observed_deferred = Arc::clone(&deferred); + let observed_aborted = Arc::clone(&aborted); + let factory: WorkerSelectionPolicyFactory = Arc::new( + move |config: &crate::config::KvRouterConfig, worker_type, _partition| { + WorkerSelectionPolicy::default(config.clone(), worker_type.as_str()) + .with_admission_policy(Box::new(DeferredAbortAdmissionPolicy { + deferred: Arc::clone(&observed_deferred), + aborted: Arc::clone(&observed_aborted), + })) + }, + ); + let core = Arc::new(SelectionCore::new_inner( + config, + 1, + CancellationToken::new(), + None, + Some(factory), + true, + SelectionCacheConfig::default(), + tracking_hash, + )); + core.upsert_worker(worker(1)).await.expect("worker upsert"); + + let selection_core = Arc::clone(&core); + let selection = tokio::spawn(async move { + selection_core + .select_and_reserve(reserve_request("deferred-reservation")) + .await + }); + tokio::time::timeout(Duration::from_secs(1), async { + while deferred.load(Ordering::Relaxed) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("reservation was not deferred"); + + core.abort_reservation("deferred-reservation") + .await + .expect("abort deferred reservation"); + let error = tokio::time::timeout(Duration::from_secs(1), selection) + .await + .expect("aborted selection did not finish") + .expect("selection task panicked") + .expect_err("aborted selection must not reserve a worker"); + assert!(matches!( + error, + SelectionError::Scheduler(KvSchedulerError::BookingFailed(message)) + if message.contains("aborted before dispatch") + )); + assert_eq!(aborted.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn reservation_release_remains_idempotent_after_worker_removal() { + let core = SelectionCore::new_local( + test_config(false), + 1, + CancellationToken::new(), + SelectionCacheConfig::default(), + ); + core.upsert_worker(worker(1)).await.expect("worker upsert"); + core.select_and_reserve(reserve_request("removed-worker-reservation")) + .await + .expect("reservation"); + + core.delete_worker(1).await.expect("worker removal"); + core.free_reservation("removed-worker-reservation") + .await + .expect("release after worker removal"); + core.free_reservation("removed-worker-reservation") + .await + .expect("repeated release"); + core.abort_reservation("unknown-reservation") + .await + .expect("unknown abort remains idempotent"); + } + #[tokio::test] async fn shutdown_reports_not_ready_and_rejects_new_work() { let core = SelectionCore::new_local( diff --git a/lib/kv-router/src/services/selection/server.rs b/lib/kv-router/src/services/selection/server.rs index 0f62fe92a2a8..4ce90082405d 100644 --- a/lib/kv-router/src/services/selection/server.rs +++ b/lib/kv-router/src/services/selection/server.rs @@ -160,6 +160,16 @@ async fn delete_reservation( } } +async fn abort_reservation( + State(state): State>, + Path(selection_id): Path, +) -> Response { + match state.service.abort_reservation(&selection_id).await { + Ok(()) => json_ok(StatusCode::OK), + Err(error) => error.into_response(), + } +} + async fn add_output_block( State(state): State>, Path(selection_id): Path, @@ -320,6 +330,10 @@ pub(crate) fn create_router(state: Arc) -> Router { "/reservations/{selection_id}/output_block", post(add_output_block), ) + .route( + "/reservations/{selection_id}/abort", + post(abort_reservation), + ) .route("/reservations/{selection_id}", delete(delete_reservation)) .route("/workers", post(create_worker).get(list_workers)) .route( diff --git a/lib/kv-router/src/services/selection/service.rs b/lib/kv-router/src/services/selection/service.rs index 37a5afcdd453..08fd0289b454 100644 --- a/lib/kv-router/src/services/selection/service.rs +++ b/lib/kv-router/src/services/selection/service.rs @@ -326,6 +326,10 @@ impl SelectionService { self.core.free_reservation(selection_id).await } + pub async fn abort_reservation(&self, selection_id: &str) -> Result<(), SelectionError> { + self.core.abort_reservation(selection_id).await + } + pub fn ready(&self) -> ReadyResponse { self.core.ready() } diff --git a/lib/kv-router/src/services/selection/tests.rs b/lib/kv-router/src/services/selection/tests.rs index 260f606f3180..20def1271218 100644 --- a/lib/kv-router/src/services/selection/tests.rs +++ b/lib/kv-router/src/services/selection/tests.rs @@ -22,13 +22,16 @@ use crate::protocols::{ BlockExtraInfo, BlockHashOptions, BlockMmObjectInfo, OverlapScores, StorageTier, WorkerId, WorkerWithDpRank, compute_block_hash_for_seq, compute_seq_hash_for_block, }; -use crate::scheduling::WorkerSelectionPolicyError; use crate::scheduling::config::RouterConfigOverride; use crate::scheduling::overlap::build_overlap_scores_response; use crate::scheduling::selector::{ WorkerCandidate, WorkerFilter, WorkerInputView, WorkerPicker, WorkerScorer, WorkerSelectionContext, WorkerSelectionPolicy, }; +use crate::scheduling::{ + QueueAdmissionDecision, QueueAdmissionEvent, QueueAdmissionPolicy, QueueAdmissionRequest, + WorkerSelectionPolicyError, +}; use crate::{TrackingHashContext, TrackingHashScope}; use tempfile::NamedTempFile; @@ -123,6 +126,51 @@ impl WorkerFilter for RejectWorker { } } +struct CountingAdmissionPolicy { + admissions: Arc, +} + +impl QueueAdmissionPolicy for CountingAdmissionPolicy { + fn admit(&mut self, _request: QueueAdmissionRequest<'_>) -> QueueAdmissionDecision { + self.admissions.fetch_add(1, Ordering::Relaxed); + QueueAdmissionDecision::Ready + } +} + +#[derive(Default)] +struct AdmissionLifecycleCounts { + admissions: AtomicUsize, + completed: AtomicUsize, + aborted: AtomicUsize, +} + +struct RecordingAdmissionPolicy { + counts: Arc, +} + +impl QueueAdmissionPolicy for RecordingAdmissionPolicy { + fn admit(&mut self, _request: QueueAdmissionRequest<'_>) -> QueueAdmissionDecision { + self.counts.admissions.fetch_add(1, Ordering::Relaxed); + QueueAdmissionDecision::Ready + } + + fn on_event( + &mut self, + event: QueueAdmissionEvent<'_>, + _ready: &mut Vec, + ) { + match event { + QueueAdmissionEvent::Completed { .. } => { + self.counts.completed.fetch_add(1, Ordering::Relaxed); + } + QueueAdmissionEvent::Aborted { .. } => { + self.counts.aborted.fetch_add(1, Ordering::Relaxed); + } + _ => {} + } + } +} + fn normalize_prompt(request: &PromptRequest) -> super::input::NormalizedPrompt { let config = test_config(); let context = TrackingHashContext::from_config(&config).unwrap(); @@ -198,6 +246,18 @@ async fn patch(app: Router, uri: &str, body: &str) -> Response { .unwrap() } +async fn delete(app: Router, uri: &str) -> Response { + app.oneshot( + Request::builder() + .method("DELETE") + .uri(uri) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap() +} + async fn register_worker(app: Router, max_tokens: Option) -> Response { register_worker_id(app, 1, max_tokens).await } @@ -371,6 +431,173 @@ async fn worker_selection_policy_factory_is_per_partition_composes_filter_scorer ); } +#[tokio::test] +async fn admission_policy_supports_atomic_reservations_and_rejects_two_step_booking() { + let admissions = Arc::new(AtomicUsize::new(0)); + let observed = Arc::clone(&admissions); + let app = native_policy_app(move |config, worker_type, _partition| { + WorkerSelectionPolicy::new( + config.clone(), + worker_type.as_str(), + vec![Box::new(WorkerIdScorer)], + Box::new(LowestCostPicker), + ) + .with_admission_policy(Box::new(CountingAdmissionPolicy { + admissions: Arc::clone(&observed), + })) + }) + .await; + assert_eq!( + register_worker_id(app.clone(), 1, None).await.status(), + StatusCode::CREATED + ); + + let reserved = post( + app.clone(), + "/select_and_reserve", + r#"{"model_name":"model","token_ids":[1,2,3,4],"selection_id":"atomic-policy","session_id":"session-a"}"#, + ) + .await; + assert_eq!(reserved.status(), StatusCode::OK); + assert_eq!(admissions.load(Ordering::Relaxed), 1); + + let selected = post( + app.clone(), + "/select", + r#"{"model_name":"model","token_ids":[1,2,3,4],"selection_id":"two-step-policy","session_id":"session-b"}"#, + ) + .await; + assert_eq!(selected.status(), StatusCode::OK); + let replay = post( + app, + "/reservations", + r#"{"model_name":"model","selection_id":"two-step-policy"}"#, + ) + .await; + assert_eq!(replay.status(), StatusCode::BAD_REQUEST); + assert_eq!(admissions.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn admission_completion_survives_worker_removal() { + let counts = Arc::new(AdmissionLifecycleCounts::default()); + let observed = Arc::clone(&counts); + let app = native_policy_app(move |config, worker_type, _partition| { + WorkerSelectionPolicy::new( + config.clone(), + worker_type.as_str(), + vec![Box::new(WorkerIdScorer)], + Box::new(LowestCostPicker), + ) + .with_admission_policy(Box::new(RecordingAdmissionPolicy { + counts: Arc::clone(&observed), + })) + }) + .await; + assert_eq!( + register_worker_id(app.clone(), 1, None).await.status(), + StatusCode::CREATED + ); + + let reserved = post( + app.clone(), + "/select_and_reserve", + r#"{"model_name":"model","token_ids":[1,2,3,4],"selection_id":"removed-worker","session_id":"session-a"}"#, + ) + .await; + assert_eq!(reserved.status(), StatusCode::OK); + assert_eq!( + delete(app.clone(), "/workers/1").await.status(), + StatusCode::OK + ); + + let released = delete(app, "/reservations/removed-worker").await; + assert_eq!(released.status(), StatusCode::OK); + assert_eq!(counts.admissions.load(Ordering::Relaxed), 1); + assert_eq!(counts.completed.load(Ordering::Relaxed), 1); + assert_eq!(counts.aborted.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn reservation_abort_reports_aborted_instead_of_completed() { + let counts = Arc::new(AdmissionLifecycleCounts::default()); + let observed = Arc::clone(&counts); + let app = native_policy_app(move |config, worker_type, _partition| { + WorkerSelectionPolicy::new( + config.clone(), + worker_type.as_str(), + vec![Box::new(WorkerIdScorer)], + Box::new(LowestCostPicker), + ) + .with_admission_policy(Box::new(RecordingAdmissionPolicy { + counts: Arc::clone(&observed), + })) + }) + .await; + assert_eq!( + register_worker_id(app.clone(), 1, None).await.status(), + StatusCode::CREATED + ); + + let reserved = post( + app.clone(), + "/select_and_reserve", + r#"{"model_name":"model","token_ids":[1,2,3,4],"selection_id":"aborted-reservation","session_id":"session-a"}"#, + ) + .await; + assert_eq!(reserved.status(), StatusCode::OK); + let aborted = post(app, "/reservations/aborted-reservation/abort", "{}").await; + assert_eq!(aborted.status(), StatusCode::OK); + assert_eq!(counts.completed.load(Ordering::Relaxed), 0); + assert_eq!(counts.aborted.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn reservation_release_skips_partitions_that_do_not_own_the_id() { + let counts = Arc::new(AdmissionLifecycleCounts::default()); + let observed = Arc::clone(&counts); + let app = native_policy_app(move |config, worker_type, _partition| { + WorkerSelectionPolicy::new( + config.clone(), + worker_type.as_str(), + vec![Box::new(WorkerIdScorer)], + Box::new(LowestCostPicker), + ) + .with_admission_policy(Box::new(RecordingAdmissionPolicy { + counts: Arc::clone(&observed), + })) + }) + .await; + for (worker_id, routing_group) in [(1, "a-first"), (2, "z-owner")] { + let worker = serde_json::json!({ + "worker_id": worker_id, + "model_name": "model", + "routing_group": routing_group, + "endpoint": format!("http://worker-{worker_id}:8000"), + "block_size": 4 + }); + assert_eq!( + post(app.clone(), "/workers", &worker.to_string()) + .await + .status(), + StatusCode::CREATED + ); + } + + let reserved = post( + app.clone(), + "/select_and_reserve", + r#"{"model_name":"model","routing_group":"z-owner","token_ids":[1,2,3,4],"selection_id":"owned-by-z","session_id":"session-a"}"#, + ) + .await; + assert_eq!(reserved.status(), StatusCode::OK); + assert_eq!( + delete(app, "/reservations/owned-by-z").await.status(), + StatusCode::OK + ); + assert_eq!(counts.completed.load(Ordering::Relaxed), 1); +} + #[tokio::test] async fn native_worker_selection_policy_rejects_non_finite_costs_before_booking() { let app = native_policy_app(|config, worker_type, _partition| { diff --git a/lib/llm/src/kv_router.rs b/lib/llm/src/kv_router.rs index e369c81a3e99..cc8ac26ab89a 100644 --- a/lib/llm/src/kv_router.rs +++ b/lib/llm/src/kv_router.rs @@ -1313,6 +1313,27 @@ where self.scheduler.free_if_worker(request_id, worker).await } + /// Release a completed booking with its final input-plus-output context. + pub async fn complete_if_worker( + &self, + request_id: &str, + worker: WorkerWithDpRank, + context_tokens: usize, + ) -> Result<(), SequenceError> { + self.scheduler + .complete_if_worker(request_id, worker, context_tokens) + .await + } + + /// Release an aborted booking only if it still belongs to `worker`. + pub async fn abort_if_worker( + &self, + request_id: &str, + worker: WorkerWithDpRank, + ) -> Result<(), SequenceError> { + self.scheduler.abort_if_worker(request_id, worker).await + } + /// Number of requests currently parked in the scheduler queue. pub fn pending_count(&self) -> usize { self.scheduler.pending_count() diff --git a/lib/llm/src/kv_router/push_router/request_guard.rs b/lib/llm/src/kv_router/push_router/request_guard.rs index 81accb56b198..ebfa98590125 100644 --- a/lib/llm/src/kv_router/push_router/request_guard.rs +++ b/lib/llm/src/kv_router/push_router/request_guard.rs @@ -55,25 +55,40 @@ where } } - async fn finish(&mut self) { + async fn release(&mut self, completed_context_tokens: Option) { if self.freed { return; } - if self.scheduler_tracked - && let Err(error) = self - .chooser - .free_if_worker(&self.context_id, self.worker) - .await - { - tracing::warn!( - request_id = %self.context_id, - worker = ?self.worker, - %error, - "Failed to free request" - ); + if self.scheduler_tracked { + let result = if let Some(context_tokens) = completed_context_tokens { + self.chooser + .complete_if_worker(&self.context_id, self.worker, context_tokens) + .await + } else { + self.chooser + .abort_if_worker(&self.context_id, self.worker) + .await + }; + if let Err(error) = result { + tracing::warn!( + request_id = %self.context_id, + worker = ?self.worker, + %error, + completed = completed_context_tokens.is_some(), + "Failed to release request" + ); + } } self.freed = true; } + + async fn complete(&mut self, context_tokens: usize) { + self.release(Some(context_tokens)).await; + } + + async fn abort(&mut self) { + self.release(None).await; + } } impl Drop for RequestCleanup @@ -97,7 +112,7 @@ where let context_id = self.context_id.clone(); let worker = self.worker; handle.spawn(async move { - let result = chooser.free_if_worker(&context_id, worker).await; + let result = chooser.abort_if_worker(&context_id, worker).await; if let Err(error) = result { tracing::warn!( request_id = %context_id, @@ -227,6 +242,10 @@ struct OutputBlockUpdate { decay_fraction: Option, } +fn routing_isl_tokens(request: &PreprocessedRequest) -> usize { + request.block_mm_routing_info().0.len() +} + /// Tracks when streamed output grows into a new scheduler accounting block. struct OutputBlockTracker { track_output_blocks: bool, @@ -269,6 +288,10 @@ impl OutputBlockTracker { .map(|expected| (1.0 - cumulative_osl as f64 / expected.max(1) as f64).max(0.0)); Some(OutputBlockUpdate { decay_fraction }) } + + fn context_tokens(&self, cumulative_osl: usize) -> usize { + self.isl_tokens.saturating_add(cumulative_osl) + } } /// Coordinates scheduler cleanup, observability, and streamed load tracking. @@ -301,7 +324,7 @@ where // Snapshot request-scoped inputs now so the guard can outlive the // PreprocessedRequest after it is moved into backend dispatch. let block_size = chooser.block_size() as usize; - let isl_tokens = request.token_ids.len(); + let isl_tokens = routing_isl_tokens(request); let expected_output_tokens = request .routing .as_ref() @@ -399,11 +422,14 @@ where pub(super) async fn finish(&mut self) { // Metrics must observe the completed request before cleanup releases its state. self.observability.record_metrics(); - self.cleanup.finish().await; + let context_tokens = self + .output_blocks + .context_tokens(self.observability.cumulative_osl()); + self.cleanup.complete(context_tokens).await; } pub(super) async fn abort(&mut self) { - self.cleanup.finish().await; + self.cleanup.abort().await; } } @@ -416,3 +442,28 @@ where self.observability.record_metrics(); } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocols::common::preprocessor::MmRoutingInfo; + + #[test] + fn terminal_context_uses_multimodal_routing_length() { + let request = PreprocessedRequest::builder() + .model("test".to_string()) + .token_ids(vec![1]) + .mm_routing_info(Some(MmRoutingInfo { + routing_token_ids: vec![1, 2, 3, 4], + block_mm_infos: Vec::new(), + expanded_prompt_len: 4, + })) + .stop_conditions(Default::default()) + .sampling_options(Default::default()) + .output_options(Default::default()) + .build() + .unwrap(); + + assert_eq!(routing_isl_tokens(&request), 4); + } +} diff --git a/lib/llm/src/kv_router/scheduler.rs b/lib/llm/src/kv_router/scheduler.rs index 91aa7f00153e..82036bc1eece 100644 --- a/lib/llm/src/kv_router/scheduler.rs +++ b/lib/llm/src/kv_router/scheduler.rs @@ -433,6 +433,31 @@ where Ok(()) } + /// Release a completed `request_id` and report its final context. + pub async fn complete_if_worker( + &self, + request_id: &str, + worker: WorkerWithDpRank, + context_tokens: usize, + ) -> Result<(), SequenceError> { + self.inner + .complete_if_worker(request_id, worker, context_tokens) + .await?; + self.update_queue_metrics(); + Ok(()) + } + + /// Release an aborted `request_id` only if it is still booked on `worker`. + pub async fn abort_if_worker( + &self, + request_id: &str, + worker: WorkerWithDpRank, + ) -> Result<(), SequenceError> { + self.inner.abort_if_worker(request_id, worker).await?; + self.update_queue_metrics(); + Ok(()) + } + pub fn pending_count(&self) -> usize { self.inner.pending_count() } From edd3df9dbde57eff131a839874aa0cbd32983892 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Mon, 17 Aug 2026 17:52:07 +0000 Subject: [PATCH 2/2] feat(router): restore live admission progress Signed-off-by: Ishan Dhanani --- lib/kv-router/src/lib.rs | 5 +- lib/kv-router/src/scheduling/queue.rs | 65 +++++++++++++++++-- .../src/scheduling/queue_admission/mod.rs | 14 ++++ .../queue_admission/policy_queue.rs | 5 +- lib/kv-router/src/scheduling/types.rs | 5 ++ lib/llm/src/kv_router.rs | 7 +- lib/llm/src/kv_router/push_router.rs | 34 +++++++++- .../kv_router/push_router/request_guard.rs | 27 +++++++- .../src/kv_router/push_router/selection.rs | 5 +- 9 files changed, 150 insertions(+), 17 deletions(-) diff --git a/lib/kv-router/src/lib.rs b/lib/kv-router/src/lib.rs index d8d2bd5a0d25..24c3210cc576 100644 --- a/lib/kv-router/src/lib.rs +++ b/lib/kv-router/src/lib.rs @@ -74,8 +74,9 @@ pub use scheduling::policy::{FcfsPolicy, RouterSchedulingPolicy, SchedulingPolic pub use scheduling::{ KvSchedulerError, PotentialLoad, QueueAdmissionDecision, QueueAdmissionEvent, QueueAdmissionId, QueueAdmissionPolicy, QueueAdmissionRequest, QueueAdmissionWorker, - QueueAdmissionWorkerSnapshot, SchedulingRequest, SchedulingResponse, SessionContext, - WorkerSelectionInputTrigger, WorkerSelectionKvHints, WorkerSelectionPolicyError, + QueueAdmissionWorkerSnapshot, RequestProgress, SchedulingRequest, SchedulingResponse, + SessionContext, WorkerSelectionInputTrigger, WorkerSelectionKvHints, + WorkerSelectionPolicyError, }; pub use selector::{ DefaultWorkerSelector, ScoredWorkerCandidate, WorkerCacheInput, WorkerCandidate, WorkerFilter, diff --git a/lib/kv-router/src/scheduling/queue.rs b/lib/kv-router/src/scheduling/queue.rs index 3bf164e5d891..9a964d9105c1 100644 --- a/lib/kv-router/src/scheduling/queue.rs +++ b/lib/kv-router/src/scheduling/queue.rs @@ -21,7 +21,8 @@ use super::policy_config::{PolicyClassConfig, PolicyProfile}; use super::prefill_load::{PrefillLoadEstimator, effective_prefill_tokens}; use super::queue_admission::{ PolicyQueue, QueueAdmissionDecision, QueueAdmissionEvent, QueueAdmissionId, - QueueAdmissionWorker, QueueAdmissionWorkerSnapshot, QueueSnapshot, WorkerPlacement, + QueueAdmissionWorker, QueueAdmissionWorkerSnapshot, QueueSnapshot, RequestProgress, + RequestProgressUpdater, WorkerPlacement, }; use super::selector::{DefaultWorkerSelector, WorkerSelector}; use super::types::{ @@ -59,6 +60,7 @@ struct QueuedRequest { enqueue_at: Instant, block_hashes: Option>, admission_id: Option, + request_progress: Option, } struct SelectedWorkerForRequest { @@ -979,7 +981,7 @@ impl< debug_assert!(inserted, "duplicate lifecycle request ID"); } - let admission_decision = if lifecycle_request_id.is_some() { + let (admission_decision, request_progress) = if lifecycle_request_id.is_some() { self.refresh_admission_worker_snapshot(); let workers = self.workers_with_configs.borrow(); let routing_eligibility = request.eligibility(); @@ -988,21 +990,25 @@ impl< .validate_worker_rank(&workers, worker) .is_ok() }; - self.pending.admit_with_admission_policy( + let (progress, updater) = RequestProgress::new(request.isl_tokens); + let decision = self.pending.admit_with_admission_policy( request .mode .tracked_request_id() .expect("lifecycle request is tracked"), request.isl_tokens, + progress, request.session_context.as_ref(), &self.admission_worker_snapshot, request.pinned_worker, request.allowed_worker_ids.as_ref(), request.routing_constraints.has_hard_constraints(), &is_worker_eligible, - ) + ); + let request_progress = decision.as_ref().map(|_| updater); + (decision, request_progress) } else { - None + (None, None) }; if let Some((admission_id, _)) = admission_decision { let replaced = self.managed_admission_request_ids.insert( @@ -1021,7 +1027,7 @@ impl< self.all_workers_prefill_busy(class, request.eligibility(), decay_now) }); if !should_queue { - return self.admit_one(request, decay_now, admission_id); + return self.admit_one(request, decay_now, admission_id, request_progress); } let snapshot = snapshot.unwrap_or_else(|| self.snapshot_for(&request)); @@ -1037,6 +1043,7 @@ impl< enqueue_at: decay_now, block_hashes, admission_id, + request_progress, }; let worker_count = self.workers_with_configs.borrow().len(); let enqueue = match admission_decision { @@ -1415,12 +1422,13 @@ impl< let class = self.profile.class(class_index); let queued = popped.into_payload(); let admission_id = queued.admission_id; + let request_progress = queued.request_progress; let request = queued.request; tracing::debug!( policy_class = class.name, "scheduling request from pending queue" ); - self.admit_one(request, admit_now, admission_id); + self.admit_one(request, admit_now, admission_id, request_progress); } } @@ -1506,6 +1514,7 @@ impl< target_cached_prefix_blocks, router_hint_candidates: request.router_hint_candidates.take(), potential_decode_blocks: selected.selection.potential_decode_blocks, + request_progress: None, }, }) } @@ -1517,6 +1526,7 @@ impl< mut request: SchedulingRequest, decay_now: Instant, admission_id: Option, + request_progress: Option, ) -> (bool, bool) { let selected = match self.select_worker_for_request(&mut request, decay_now) { Ok(s) => s, @@ -1541,6 +1551,7 @@ impl< target_cached_prefix_blocks, router_hint_candidates: request.router_hint_candidates.take(), potential_decode_blocks: selected.selection.potential_decode_blocks, + request_progress, }; let non_max_overlap_selection = selected.non_max_overlap_selection; @@ -1901,6 +1912,7 @@ mod tests { #[derive(Default)] struct AdmissionPolicyState { deferred: StdMutex>, + progress: StdMutex>, dispatched: AtomicUsize, reconciled: AtomicUsize, aborted: AtomicUsize, @@ -1916,6 +1928,7 @@ mod tests { fn admit(&mut self, request: QueueAdmissionRequest<'_>) -> QueueAdmissionDecision { assert_eq!(request.request_id(), "policy-request"); *self.state.deferred.lock().unwrap() = Some(request.id()); + *self.state.progress.lock().unwrap() = Some(request.progress().clone()); QueueAdmissionDecision::Defer } @@ -2548,6 +2561,30 @@ mod tests { .unwrap() .unwrap() .unwrap(); + let progress = response + .request_progress + .expect("policy-managed request receives a live-progress updater"); + assert_eq!( + state + .progress + .lock() + .unwrap() + .as_ref() + .expect("policy retained request progress") + .context_tokens(), + 32 + ); + progress.update_context_tokens(47); + assert_eq!( + state + .progress + .lock() + .unwrap() + .as_ref() + .expect("policy retained request progress") + .context_tokens(), + 47 + ); lease.as_mut().unwrap().disarm(); assert_eq!(state.dispatched.load(Ordering::Relaxed), 1); let reconciled_before_completion = state.reconciled.load(Ordering::Relaxed); @@ -2572,6 +2609,20 @@ mod tests { ); } + #[tokio::test] + async fn default_queue_does_not_create_live_progress() { + let (queue, slots) = make_queue(1, 16, 128, None); + let (request, response_rx) = make_request("default-request", 32); + + queue.enqueue(request).await; + let response = response_rx.await.unwrap().unwrap(); + + assert!(response.request_progress.is_none()); + slots + .free(&"default-request".to_owned(), Instant::now()) + .unwrap(); + } + #[tokio::test] async fn custom_admission_policy_rejects_duplicate_request_id_while_deferred() { let state = Arc::new(AdmissionPolicyState::default()); diff --git a/lib/kv-router/src/scheduling/queue_admission/mod.rs b/lib/kv-router/src/scheduling/queue_admission/mod.rs index 7fe3af8d6a36..ab80058af0c4 100644 --- a/lib/kv-router/src/scheduling/queue_admission/mod.rs +++ b/lib/kv-router/src/scheduling/queue_admission/mod.rs @@ -145,6 +145,7 @@ pub struct QueueAdmissionRequest<'a> { id: QueueAdmissionId, request_id: &'a str, context_tokens: usize, + progress: RequestProgress, session_context: Option<&'a SessionContext>, worker_snapshot: &'a QueueAdmissionWorkerSnapshot, eligibility: QueueAdmissionEligibility<'a>, @@ -163,6 +164,7 @@ impl<'a> QueueAdmissionRequest<'a> { id, request_id, context_tokens, + progress: RequestProgress::new(context_tokens).0, session_context, worker_snapshot, eligibility: QueueAdmissionEligibility::default(), @@ -174,6 +176,7 @@ impl<'a> QueueAdmissionRequest<'a> { id: QueueAdmissionId, request_id: &'a str, context_tokens: usize, + progress: RequestProgress, session_context: Option<&'a SessionContext>, worker_snapshot: &'a QueueAdmissionWorkerSnapshot, pinned_worker: Option, @@ -185,6 +188,7 @@ impl<'a> QueueAdmissionRequest<'a> { id, request_id, context_tokens, + progress, session_context, worker_snapshot, eligibility: QueueAdmissionEligibility { @@ -225,6 +229,16 @@ impl<'a> QueueAdmissionRequest<'a> { self.context_tokens } + /// Return the latest logical input-plus-output context for this request. + /// + /// The value starts at [`Self::context_tokens`] and advances monotonically + /// while the response stream crosses output-block boundaries. It can lag by + /// at most one such boundary; the terminal event carries the authoritative + /// final context. Policies may retain a clone for the request lifetime. + pub fn progress(&self) -> &RequestProgress { + &self.progress + } + pub fn session_context(&self) -> Option<&SessionContext> { self.session_context } diff --git a/lib/kv-router/src/scheduling/queue_admission/policy_queue.rs b/lib/kv-router/src/scheduling/queue_admission/policy_queue.rs index ff66b47b9a10..e8f0edd1d76f 100644 --- a/lib/kv-router/src/scheduling/queue_admission/policy_queue.rs +++ b/lib/kv-router/src/scheduling/queue_admission/policy_queue.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; use super::{ QueueAdmissionDecision, QueueAdmissionEvent, QueueAdmissionId, QueueAdmissionPolicy, - QueueAdmissionRequest, QueueAdmissionWorkerSnapshot, WorkerPlacement, + QueueAdmissionRequest, QueueAdmissionWorkerSnapshot, RequestProgress, WorkerPlacement, }; use crate::protocols::{WorkerId, WorkerWithDpRank}; use crate::scheduling::config::RouterQueuePolicy; @@ -479,6 +479,7 @@ impl PolicyQueue { &mut self, request_id: &str, context_tokens: usize, + progress: RequestProgress, session_context: Option<&SessionContext>, worker_snapshot: &QueueAdmissionWorkerSnapshot, pinned_worker: Option, @@ -493,6 +494,7 @@ impl PolicyQueue { id, request_id, context_tokens, + progress, session_context, worker_snapshot, pinned_worker, @@ -1046,6 +1048,7 @@ policy_classes: .admit_with_admission_policy( "request-1", 32, + RequestProgress::new(32).0, None, &snapshot, None, diff --git a/lib/kv-router/src/scheduling/types.rs b/lib/kv-router/src/scheduling/types.rs index 77808db9921b..56163a19a42e 100644 --- a/lib/kv-router/src/scheduling/types.rs +++ b/lib/kv-router/src/scheduling/types.rs @@ -12,6 +12,7 @@ use super::config::RouterConfigOverride; use super::filter::RoutingEligibility; use super::overlap::{OverlapSignals, SelectedWorkerTierSnapshot}; use super::prefill_load::effective_prefill_tokens; +use super::queue_admission::RequestProgressUpdater; pub use crate::protocols::PotentialLoad; use crate::protocols::{ LocalBlockHash, RoutingConstraints, SharedCacheHits, WorkerConfigLike, WorkerId, @@ -124,6 +125,10 @@ pub struct SchedulingResponse { pub target_cached_prefix_blocks: u32, pub router_hint_candidates: Option, pub potential_decode_blocks: usize, + /// Stream-owned updater for a custom queue-admission policy, when one manages this request. + /// + /// The ordinary scheduler path leaves this as `None` and performs no live-progress updates. + pub request_progress: Option, } /// A routing decision that selected less KV overlap than another eligible worker. diff --git a/lib/llm/src/kv_router.rs b/lib/llm/src/kv_router.rs index cc8ac26ab89a..060579ab260c 100644 --- a/lib/llm/src/kv_router.rs +++ b/lib/llm/src/kv_router.rs @@ -17,8 +17,8 @@ use dynamo_kv_router::{ }, router_hint::{RouterHint, RouterHintRootCandidates}, scheduling::{ - CacheHitEstimates, OverlapAnalysis, OverloadedWorkerProvider, ScheduleMode, - ScheduleRequest, TieredOverlapRefresher, WorkerAvailabilityProvider, + CacheHitEstimates, OverlapAnalysis, OverloadedWorkerProvider, RequestProgressUpdater, + ScheduleMode, ScheduleRequest, TieredOverlapRefresher, WorkerAvailabilityProvider, effective_prefill_tokens, overlap::cache_hit_estimates_from_tiered_matches, }, }; @@ -176,6 +176,7 @@ pub enum FindBestMatchOutcome { potential_decode_blocks: u64, routing_hashes: Option, router_hint: Option, + request_progress: Option, }, QueueRejected { rejection: scheduling::QueueRejection, @@ -1160,6 +1161,7 @@ where total_us = total_elapsed.as_micros() as u64, "find_best_match completed" ); + let request_progress = response.request_progress; match admission { FindBestMatchAdmission::WithAdmission { .. } => Ok( @@ -1171,6 +1173,7 @@ where potential_decode_blocks: response.potential_decode_blocks as u64, routing_hashes, router_hint, + request_progress, }), ), FindBestMatchAdmission::WithoutAdmission => Ok( diff --git a/lib/llm/src/kv_router/push_router.rs b/lib/llm/src/kv_router/push_router.rs index cbec4df49e15..ad68cff34e2d 100644 --- a/lib/llm/src/kv_router/push_router.rs +++ b/lib/llm/src/kv_router/push_router.rs @@ -289,6 +289,7 @@ where selected_worker, request, !is_query_only, + selection.request_progress.take(), ); let record_result: Result<(), Error> = async { @@ -717,7 +718,7 @@ mod tests { use dynamo_kv_router::{ DefaultWorkerSelector, WorkerSelectionPolicy, config::KvRouterConfig, - protocols::RoutingConstraints, + protocols::RoutingConstraints, scheduling::RequestProgress, }; use dynamo_runtime::{ DistributedRuntime, Runtime, @@ -798,6 +799,7 @@ mod tests { WorkerWithDpRank::from_worker_id(0), &request(), false, + None, ); let monitored = monitor_response_stream(source, context, guard); tokio::pin!(monitored); @@ -810,6 +812,36 @@ mod tests { runtime.shutdown(); } + #[tokio::test] + #[serial_test::serial] + async fn output_block_boundary_updates_live_admission_progress() { + let (router, runtime) = router(None).await; + let request = request(); + let initial_context_tokens = request.token_ids.len(); + let (progress, updater) = RequestProgress::new(initial_context_tokens); + let mut guard = RequestGuard::new( + Arc::clone(&router.chooser), + Arc::clone(&router.request_metrics), + "live-admission-progress".to_string(), + WorkerWithDpRank::from_worker_id(0), + &request, + false, + Some(updater), + ); + + guard + .on_item(&Annotated::from_data(LLMEngineOutput { + token_ids: vec![2; 16], + ..Default::default() + })) + .await; + + assert_eq!(progress.context_tokens(), initial_context_tokens + 16); + guard.abort().await; + drop(router); + runtime.shutdown(); + } + #[tokio::test] #[serial_test::serial] async fn stream_failure_releases_booking_before_error_is_observable() { diff --git a/lib/llm/src/kv_router/push_router/request_guard.rs b/lib/llm/src/kv_router/push_router/request_guard.rs index ebfa98590125..cdf2eb325ed9 100644 --- a/lib/llm/src/kv_router/push_router/request_guard.rs +++ b/lib/llm/src/kv_router/push_router/request_guard.rs @@ -3,7 +3,9 @@ use std::sync::Arc; -use dynamo_kv_router::{protocols::WorkerWithDpRank, selector::WorkerSelector}; +use dynamo_kv_router::{ + protocols::WorkerWithDpRank, scheduling::RequestProgressUpdater, selector::WorkerSelector, +}; use dynamo_runtime::{ error::DynamoError, metrics::frontend_perf::{STAGE_DISPATCH, StageGuard}, @@ -240,6 +242,7 @@ impl RequestObservability { struct OutputBlockUpdate { decay_fraction: Option, + update_scheduler: bool, } fn routing_isl_tokens(request: &PreprocessedRequest) -> usize { @@ -249,6 +252,7 @@ fn routing_isl_tokens(request: &PreprocessedRequest) -> usize { /// Tracks when streamed output grows into a new scheduler accounting block. struct OutputBlockTracker { track_output_blocks: bool, + track_request_progress: bool, current_total_blocks: usize, isl_tokens: usize, block_size: usize, @@ -258,12 +262,14 @@ struct OutputBlockTracker { impl OutputBlockTracker { fn new( track_output_blocks: bool, + track_request_progress: bool, isl_tokens: usize, block_size: usize, expected_output_tokens: Option, ) -> Self { Self { track_output_blocks, + track_request_progress, current_total_blocks: isl_tokens.div_ceil(block_size), isl_tokens, block_size, @@ -272,7 +278,7 @@ impl OutputBlockTracker { } fn observe(&mut self, cumulative_osl: usize) -> Option { - if !self.track_output_blocks { + if !self.track_output_blocks && !self.track_request_progress { return None; } @@ -286,7 +292,10 @@ impl OutputBlockTracker { let decay_fraction = self .expected_output_tokens .map(|expected| (1.0 - cumulative_osl as f64 / expected.max(1) as f64).max(0.0)); - Some(OutputBlockUpdate { decay_fraction }) + Some(OutputBlockUpdate { + decay_fraction, + update_scheduler: self.track_output_blocks, + }) } fn context_tokens(&self, cumulative_osl: usize) -> usize { @@ -305,6 +314,7 @@ where cleanup: RequestCleanup, observability: RequestObservability, output_blocks: OutputBlockTracker, + request_progress: Option, prefill_marked: bool, migration_state: Option, } @@ -320,6 +330,7 @@ where worker: WorkerWithDpRank, request: &PreprocessedRequest, scheduler_tracked: bool, + request_progress: Option, ) -> Self { // Snapshot request-scoped inputs now so the guard can outlive the // PreprocessedRequest after it is moved into backend dispatch. @@ -331,6 +342,7 @@ where .and_then(|routing| routing.expected_output_tokens); let track_output_blocks = scheduler_tracked && chooser.kv_router_config().router_track_output_blocks; + let track_request_progress = request_progress.is_some(); if scheduler_tracked { request_metrics.requests_started_total().inc(); } @@ -340,10 +352,12 @@ where observability: RequestObservability::new(request.tracker.clone(), request_metrics), output_blocks: OutputBlockTracker::new( track_output_blocks, + track_request_progress, isl_tokens, block_size, expected_output_tokens, ), + request_progress, prefill_marked: false, migration_state: request.migration_state.clone(), } @@ -404,6 +418,13 @@ where return; }; + if let Some(progress) = &self.request_progress { + progress.update_context_tokens(self.output_blocks.context_tokens(cumulative_osl)); + } + if !update.update_scheduler { + return; + } + if let Err(error) = self .cleanup .chooser diff --git a/lib/llm/src/kv_router/push_router/selection.rs b/lib/llm/src/kv_router/push_router/selection.rs index 9a4568c303d0..0a6c64aa84f2 100644 --- a/lib/llm/src/kv_router/push_router/selection.rs +++ b/lib/llm/src/kv_router/push_router/selection.rs @@ -8,7 +8,7 @@ use dynamo_kv_router::{ indexer::RoutingDecisionHashes, protocols::{BlockExtraInfo, RoutingConstraints, WorkerId, WorkerWithDpRank}, router_hint::RouterHint, - scheduling::RoutingEligibility, + scheduling::{RequestProgressUpdater, RoutingEligibility}, selector::WorkerSelector, }; use dynamo_runtime::{dynamo_nvtx_range, pipeline::Error}; @@ -33,6 +33,7 @@ pub(super) struct WorkerSelection { pub(super) cached_tokens: usize, pub(super) routing_hashes: Option, pub(super) router_hint: Option, + pub(super) request_progress: Option, } #[derive(Clone, Copy)] @@ -119,6 +120,7 @@ where potential_decode_blocks: _, routing_hashes, router_hint, + request_progress, } => Ok(WorkerSelection { worker, overlap_amount: overlap_blocks, @@ -126,6 +128,7 @@ where cached_tokens, routing_hashes, router_hint, + request_progress, }), FindBestMatchOutcome::QueueRejected { rejection } => Err(rejection.into()), }