Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
440 changes: 225 additions & 215 deletions docs/fern/pages/reference/api/python/_core.mdx

Large diffs are not rendered by default.

296 changes: 153 additions & 143 deletions docs/fern/pages/reference/api/python/llm.mdx

Large diffs are not rendered by default.

26 changes: 13 additions & 13 deletions docs/fern/pages/reference/api/python/mocker.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ from dynamo.mocker import MockEngineArgs
MockEngineArgs(engine_type: str = 'vllm', num_gpu_blocks: Optional[int] = None, block_size: int = 0, max_num_seqs: Optional[int] = 256, max_num_batched_tokens: Optional[int] = 8192, enable_prefix_caching: bool = True, enable_chunked_prefill: bool = True, speedup_ratio: float = 1.0, decode_speedup_ratio: float = 1.0, dp_size: int = 1, startup_time: Optional[float] = None, worker_type: str = 'aggregated', planner_profile_data: Optional[str | os.PathLike[str]] = None, aic_backend: Optional[str] = None, aic_system: Optional[str] = None, aic_backend_version: Optional[str] = None, aic_tp_size: Optional[int] = None, aic_model_path: Optional[str] = None, aic_moe_tp_size: Optional[int] = None, aic_moe_ep_size: Optional[int] = None, aic_attention_dp_size: Optional[int] = None, aic_nextn: Optional[int] = None, aic_nextn_accept_rates: Optional[str] = None, aic_mtp_seed: int = 42, aic_gemm_dtype: Optional[str] = None, aic_moe_dtype: Optional[str] = None, aic_fmha_dtype: Optional[str] = None, aic_kv_cache_dtype: Optional[str] = None, aic_comm_dtype: Optional[str] = None, gpu_memory_utilization: Optional[float] = None, mem_fraction_static: Optional[float] = None, free_gpu_memory_fraction: Optional[float] = None, enable_local_indexer: bool = False, bootstrap_port: Optional[int] = None, handoff_session_timeout_ms: int = 300000, kv_bytes_per_token: Optional[int] = None, kv_transfer_bandwidth: Optional[float] = None, kv_transfer_timing_mode: str = 'full_prompt', reasoning: Optional[ReasoningConfig] = None, response_replay_trace_path: Optional[str | os.PathLike[str]] = None, zmq_kv_events_port: Optional[int] = None, zmq_replay_port: Optional[int] = None, preemption_mode: str = 'lifo', router_queue_policy: Optional[str] = None, sglang: Optional[SglangArgs] = None, trtllm: Optional[TrtllmArgs] = None, num_g2_blocks: Optional[int] = None, num_g3_blocks: Optional[int] = None, offload_batch_size: Optional[int] = None, bandwidth_g1_to_g2_gbps: Optional[float] = None, bandwidth_g2_to_g1_gbps: Optional[float] = None, bandwidth_g2_to_g3_gbps: Optional[float] = None, bandwidth_g3_to_g2_gbps: Optional[float] = None, enable_g4_storage: bool = False, bandwidth_g2_to_g4_gbps: Optional[float] = None, bandwidth_g4_to_g2_gbps: Optional[float] = None, max_model_len: Optional[int] = None, g1_backend: Optional[str] = None) -> None
```

[`lib/bindings/python/src/dynamo/_core.pyi#L1922`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1922)
[`lib/bindings/python/src/dynamo/_core.pyi#L1926`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1926)

**Public methods**

Expand All @@ -32,7 +32,7 @@ __init__(engine_type: str = 'vllm', num_gpu_blocks: Optional[int] = None, block_

No summary available.

[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1923)
[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1927)

<h4 id="api-dynamo-core-mockengineargs-from-json">from_json</h4>

Expand All @@ -42,7 +42,7 @@ from_json(config_json: str) -> MockEngineArgs

No summary available.

[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1986)
[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1990)

<h4 id="api-dynamo-core-mockengineargs-copy">copy</h4>

Expand All @@ -52,7 +52,7 @@ copy() -> MockEngineArgs

No summary available.

[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1990)
[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1994)

<h4 id="api-dynamo-core-mockengineargs-is-prefill">is_prefill</h4>

Expand All @@ -62,7 +62,7 @@ is_prefill() -> bool

No summary available.

[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2190)
[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2194)

<h4 id="api-dynamo-core-mockengineargs-is-decode">is_decode</h4>

Expand All @@ -72,7 +72,7 @@ is_decode() -> bool

No summary available.

[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2192)
[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2196)

<h4 id="api-dynamo-core-mockengineargs-with-overrides">with_overrides</h4>

Expand All @@ -82,7 +82,7 @@ with_overrides(bootstrap_port: Optional[int] = None, zmq_kv_events_port: Optiona

No summary available.

[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2194)
[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L2198)
</Accordion>

<Accordion id="api-dynamo-mocker-args-profiledataresult" title="ProfileDataResult (class)">
Expand Down Expand Up @@ -122,7 +122,7 @@ from dynamo.mocker import ReasoningConfig
ReasoningConfig(start_thinking_token_id: int, end_thinking_token_id: int, thinking_ratio: float) -> None
```

[`lib/bindings/python/src/dynamo/_core.pyi#L1894`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1894)
[`lib/bindings/python/src/dynamo/_core.pyi#L1898`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1898)

**Public methods**

Expand All @@ -134,7 +134,7 @@ __init__(start_thinking_token_id: int, end_thinking_token_id: int, thinking_rati

No summary available.

[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1895)
[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1899)
</Accordion>

<Accordion id="api-dynamo-core-sglangargs" title="SglangArgs (class)">
Expand All @@ -148,7 +148,7 @@ from dynamo.mocker import SglangArgs
SglangArgs(schedule_policy: Optional[str] = None, page_size: Optional[int] = None, max_prefill_tokens: Optional[int] = None, chunked_prefill_size: Optional[int] = None, clip_max_new_tokens: Optional[int] = None, schedule_conservativeness: Optional[float] = None) -> None
```

[`lib/bindings/python/src/dynamo/_core.pyi#L1903`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1903)
[`lib/bindings/python/src/dynamo/_core.pyi#L1907`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1907)

**Public methods**

Expand All @@ -160,7 +160,7 @@ __init__(schedule_policy: Optional[str] = None, page_size: Optional[int] = None,

No summary available.

[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1904)
[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1908)
</Accordion>

<Accordion id="api-dynamo-core-trtllmargs" title="TrtllmArgs (class)">
Expand All @@ -174,7 +174,7 @@ from dynamo.mocker import TrtllmArgs
TrtllmArgs(capacity_scheduler_policy: Optional[str] = None) -> None
```

[`lib/bindings/python/src/dynamo/_core.pyi#L1915`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1915)
[`lib/bindings/python/src/dynamo/_core.pyi#L1919`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1919)

**Public methods**

Expand All @@ -186,7 +186,7 @@ __init__(capacity_scheduler_policy: Optional[str] = None) -> None

No summary available.

[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1916)
[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L1920)
</Accordion>

<Accordion id="api-dynamo-mocker-config-apply-worker-engine-args-overrides" title="apply_worker_engine_args_overrides (function)">
Expand Down
15 changes: 15 additions & 0 deletions lib/bindings/python/rust/llm/kv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -796,6 +796,21 @@ impl SelectionService {
})
}

/// Abort a failed reservation, releasing its load and reporting failure.
fn abort_reservation<'p>(
&self,
py: Python<'p>,
selection_id: String,
) -> PyResult<Bound<'p, PyAny>> {
let core = self.inner.clone();
pyo3_async_runtimes::tokio::future_into_py(py, async move {
core.abort_reservation(&selection_id)
.await
.map_err(selection_to_pyerr)?;
Ok(())
})
}

/// Current per-model active load (pending counts + per-worker potential loads).
#[pyo3(signature = (*, model_name = None, routing_group = None))]
fn loads(
Expand Down
4 changes: 4 additions & 0 deletions lib/bindings/python/src/dynamo/_core.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -807,6 +807,10 @@ class SelectionService:
"""Free a finished reservation, releasing its tracked load."""
...

async def abort_reservation(self, selection_id: str) -> None:
"""Abort a failed reservation, releasing its load and reporting failure."""
...

def loads(
self, *, model_name: Optional[str] = None, routing_group: Optional[str] = None
) -> JsonLike:
Expand Down
4 changes: 4 additions & 0 deletions lib/bindings/python/tests/test_kv_bindings.py
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,10 @@ async def test_selection_service_selects_registered_worker(monkeypatch):
assert selected["worker_id"] == 1
assert selected["dp_rank"] == 0
assert selected["endpoint"] == "http://worker-1:8000"

# Unknown releases are idempotent, but both terminal paths must be exposed.
await service.free_reservation("unknown-completed-selection")
await service.abort_reservation("unknown-aborted-selection")
finally:
await service.shutdown_async()

Expand Down
5 changes: 3 additions & 2 deletions lib/kv-router/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,9 @@ pub use scheduling::policy::{FcfsPolicy, RouterSchedulingPolicy, SchedulingPolic
pub use scheduling::{
KvSchedulerError, PotentialLoad, QueueAdmissionDecision, QueueAdmissionEvent, QueueAdmissionId,
QueueAdmissionPolicy, QueueAdmissionRequest, QueueAdmissionWorker,
QueueAdmissionWorkerSnapshot, SchedulingRequest, SchedulingResponse, SessionContext,
WorkerSelectionInputTrigger, WorkerSelectionKvHints, WorkerSelectionPolicyError,
QueueAdmissionWorkerSnapshot, RequestProgress, SchedulingRequest, SchedulingResponse,
SessionContext, WorkerSelectionInputTrigger, WorkerSelectionKvHints,
WorkerSelectionPolicyError,
};
pub use selector::{
DefaultWorkerSelector, ScoredWorkerCandidate, WorkerCacheInput, WorkerCandidate, WorkerFilter,
Expand Down
65 changes: 58 additions & 7 deletions lib/kv-router/src/scheduling/queue.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ use super::policy_config::{PolicyClassConfig, PolicyProfile};
use super::prefill_load::{PrefillLoadEstimator, effective_prefill_tokens};
use super::queue_admission::{
PolicyQueue, QueueAdmissionDecision, QueueAdmissionEvent, QueueAdmissionId,
QueueAdmissionWorker, QueueAdmissionWorkerSnapshot, QueueSnapshot, WorkerPlacement,
QueueAdmissionWorker, QueueAdmissionWorkerSnapshot, QueueSnapshot, RequestProgress,
RequestProgressUpdater, WorkerPlacement,
};
use super::selector::{DefaultWorkerSelector, WorkerSelector};
use super::types::{
Expand Down Expand Up @@ -59,6 +60,7 @@ struct QueuedRequest {
enqueue_at: Instant,
block_hashes: Option<Vec<LocalBlockHash>>,
admission_id: Option<QueueAdmissionId>,
request_progress: Option<RequestProgressUpdater>,
}

struct SelectedWorkerForRequest {
Expand Down Expand Up @@ -979,7 +981,7 @@ impl<
debug_assert!(inserted, "duplicate lifecycle request ID");
}

let admission_decision = if lifecycle_request_id.is_some() {
let (admission_decision, request_progress) = if lifecycle_request_id.is_some() {
self.refresh_admission_worker_snapshot();
let workers = self.workers_with_configs.borrow();
let routing_eligibility = request.eligibility();
Expand All @@ -988,21 +990,25 @@ impl<
.validate_worker_rank(&workers, worker)
.is_ok()
};
self.pending.admit_with_admission_policy(
let (progress, updater) = RequestProgress::new(request.isl_tokens);
let decision = self.pending.admit_with_admission_policy(
request
.mode
.tracked_request_id()
.expect("lifecycle request is tracked"),
request.isl_tokens,
progress,
request.session_context.as_ref(),
&self.admission_worker_snapshot,
request.pinned_worker,
request.allowed_worker_ids.as_ref(),
request.routing_constraints.has_hard_constraints(),
&is_worker_eligible,
)
);
let request_progress = decision.as_ref().map(|_| updater);
(decision, request_progress)
} else {
None
(None, None)
};
if let Some((admission_id, _)) = admission_decision {
let replaced = self.managed_admission_request_ids.insert(
Expand All @@ -1021,7 +1027,7 @@ impl<
self.all_workers_prefill_busy(class, request.eligibility(), decay_now)
});
if !should_queue {
return self.admit_one(request, decay_now, admission_id);
return self.admit_one(request, decay_now, admission_id, request_progress);
}

let snapshot = snapshot.unwrap_or_else(|| self.snapshot_for(&request));
Expand All @@ -1037,6 +1043,7 @@ impl<
enqueue_at: decay_now,
block_hashes,
admission_id,
request_progress,
};
let worker_count = self.workers_with_configs.borrow().len();
let enqueue = match admission_decision {
Expand Down Expand Up @@ -1415,12 +1422,13 @@ impl<
let class = self.profile.class(class_index);
let queued = popped.into_payload();
let admission_id = queued.admission_id;
let request_progress = queued.request_progress;
let request = queued.request;
tracing::debug!(
policy_class = class.name,
"scheduling request from pending queue"
);
self.admit_one(request, admit_now, admission_id);
self.admit_one(request, admit_now, admission_id, request_progress);
}
}

Expand Down Expand Up @@ -1506,6 +1514,7 @@ impl<
target_cached_prefix_blocks,
router_hint_candidates: request.router_hint_candidates.take(),
potential_decode_blocks: selected.selection.potential_decode_blocks,
request_progress: None,
},
})
}
Expand All @@ -1517,6 +1526,7 @@ impl<
mut request: SchedulingRequest,
decay_now: Instant,
admission_id: Option<QueueAdmissionId>,
request_progress: Option<RequestProgressUpdater>,
) -> (bool, bool) {
let selected = match self.select_worker_for_request(&mut request, decay_now) {
Ok(s) => s,
Expand All @@ -1541,6 +1551,7 @@ impl<
target_cached_prefix_blocks,
router_hint_candidates: request.router_hint_candidates.take(),
potential_decode_blocks: selected.selection.potential_decode_blocks,
request_progress,
};
let non_max_overlap_selection = selected.non_max_overlap_selection;

Expand Down Expand Up @@ -1901,6 +1912,7 @@ mod tests {
#[derive(Default)]
struct AdmissionPolicyState {
deferred: StdMutex<Option<QueueAdmissionId>>,
progress: StdMutex<Option<RequestProgress>>,
dispatched: AtomicUsize,
reconciled: AtomicUsize,
aborted: AtomicUsize,
Expand All @@ -1916,6 +1928,7 @@ mod tests {
fn admit(&mut self, request: QueueAdmissionRequest<'_>) -> QueueAdmissionDecision {
assert_eq!(request.request_id(), "policy-request");
*self.state.deferred.lock().unwrap() = Some(request.id());
*self.state.progress.lock().unwrap() = Some(request.progress().clone());
QueueAdmissionDecision::Defer
}

Expand Down Expand Up @@ -2548,6 +2561,30 @@ mod tests {
.unwrap()
.unwrap()
.unwrap();
let progress = response
.request_progress
.expect("policy-managed request receives a live-progress updater");
assert_eq!(
state
.progress
.lock()
.unwrap()
.as_ref()
.expect("policy retained request progress")
.context_tokens(),
32
);
progress.update_context_tokens(47);
assert_eq!(
state
.progress
.lock()
.unwrap()
.as_ref()
.expect("policy retained request progress")
.context_tokens(),
47
);
lease.as_mut().unwrap().disarm();
assert_eq!(state.dispatched.load(Ordering::Relaxed), 1);
let reconciled_before_completion = state.reconciled.load(Ordering::Relaxed);
Expand All @@ -2572,6 +2609,20 @@ mod tests {
);
}

#[tokio::test]
async fn default_queue_does_not_create_live_progress() {
let (queue, slots) = make_queue(1, 16, 128, None);
let (request, response_rx) = make_request("default-request", 32);

queue.enqueue(request).await;
let response = response_rx.await.unwrap().unwrap();

assert!(response.request_progress.is_none());
slots
.free(&"default-request".to_owned(), Instant::now())
.unwrap();
}

#[tokio::test]
async fn custom_admission_policy_rejects_duplicate_request_id_while_deferred() {
let state = Arc::new(AdmissionPolicyState::default());
Expand Down
Loading
Loading