Skip to content
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
147a63f
feat(router): add native scorer and picker policies
ishandhanani Aug 1, 2026
e14cdd8
fix(router): host additive scorer contributions
ishandhanani Aug 1, 2026
46ec9e0
refactor(router): share worker selection policy path
ishandhanani Aug 1, 2026
69eed39
perf(router): preserve default selection fast path
ishandhanani Aug 1, 2026
46063d7
perf(router): simplify native worker selection
ishandhanani Aug 1, 2026
0cb225b
feat(router): expose overlap to custom pickers
ishandhanani Aug 1, 2026
1a987dd
feat(router): group native worker policy inputs
ishandhanani Aug 1, 2026
d15c470
refactor(router): remove stale policy scaffolding
ishandhanani Aug 3, 2026
8e62e90
refactor(router): split worker selector modules
ishandhanani Aug 3, 2026
f30afb9
feat(router): support custom selectors in the frontend
ishandhanani Aug 2, 2026
e232362
refactor(router): unify frontend policy construction
ishandhanani Aug 2, 2026
5d654b7
refactor(frontend): remove redundant policy plumbing
ishandhanani Aug 3, 2026
3b5cdc9
feat(epp): accept prebuilt selection services
ishandhanani Aug 3, 2026
325fe10
fix(router): clarify native policy failures
ishandhanani Aug 4, 2026
20c7520
fix(epp): initialize selector before discovery
ishandhanani Aug 4, 2026
ec6c2b6
chore(router): refresh kvbm dependency lock
ishandhanani Aug 4, 2026
8e9c485
fix(epp): use prebuilt service replica port
ishandhanani Aug 5, 2026
c66968b
fix(router): tighten custom policy ownership
ishandhanani Aug 5, 2026
1d8ad4b
docs(router): document custom worker selection
ishandhanani Aug 5, 2026
38c7649
fix(router): address native policy review feedback
ishandhanani Aug 6, 2026
a46e9ea
Merge branch 'main' into idhanani/dyn-3712-native-worker-selection-api
ishandhanani Aug 6, 2026
344d95c
Merge branch 'main' into idhanani/dyn-3712-native-worker-selection-api
ishandhanani Aug 6, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 44 additions & 4 deletions deploy/inference-gateway/ext-proc/src/epp_router.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ use std::time::Duration;
use anyhow::Result;
use tokio::sync::Semaphore;

use dynamo_kv_router::services::selection::SelectionService;
use dynamo_llm::protocols::common::extensions::{
AgentHints, HEADER_REQUEST_PRIORITY, HEADER_REQUEST_STRICT_PRIORITY, resolve_request_priority,
};
Expand Down Expand Up @@ -68,16 +69,55 @@ pub struct EppRouter {
impl EppRouter {
/// Assemble the standalone runtime from the validated selector config.
pub async fn from_selector(cfg: EppStandaloneConfig) -> Result<Self> {
let selector = Arc::new(Selector::new(&cfg).await?);
let (renderer, reflector, reflector_ready) = Self::dependencies(&cfg).await?;
Ok(Self::from_selector_parts(
cfg,
renderer,
reflector,
reflector_ready,
selector,
))
}

/// Assemble a custom EPP image around a prebuilt selection service.
pub async fn from_selection_service(
cfg: EppStandaloneConfig,
service: Arc<SelectionService>,
Comment thread
ishandhanani marked this conversation as resolved.
Resolved
) -> Result<Self> {
let selector = Arc::new(Selector::from_service(&cfg, service).await?);
let (renderer, reflector, reflector_ready) = Self::dependencies(&cfg).await?;
Ok(Self::from_selector_parts(
cfg,
renderer,
reflector,
reflector_ready,
selector,
))
}

async fn dependencies(
cfg: &EppStandaloneConfig,
) -> Result<(VllmRenderClient, Arc<PodDiscovery>, Arc<AtomicBool>)> {
let renderer = VllmRenderClient::new(
&cfg.tokenizer_service_url,
Duration::from_millis(cfg.tokenization_timeout_ms),
cfg.tokenizer_max_response_bytes,
)?;

let (reflector, reflector_ready) = PodDiscovery::spawn(&cfg).await?;
let (reflector, reflector_ready) = PodDiscovery::spawn(cfg).await?;
let reflector = Arc::new(reflector);

let selector = Arc::new(Selector::new(&cfg).await?);
Ok((renderer, reflector, reflector_ready))
}

fn from_selector_parts(
cfg: EppStandaloneConfig,
renderer: VllmRenderClient,
reflector: Arc<PodDiscovery>,
reflector_ready: Arc<AtomicBool>,
selector: Arc<Selector>,
) -> Self {
let peer_ready = selector.peer_ready();

let defaults = RegistrationDefaults::from_config(&cfg);
Expand All @@ -87,7 +127,7 @@ impl EppRouter {
// Readiness is driven solely by the live pod+pool signal (see `is_ready`);
// we do not block startup on a schedulable worker. A valid, empty pool is
// ready immediately and returns 503 per-request until capacity appears.
Ok(Self {
Self {
renderer,
reflector,
selector,
Expand All @@ -96,7 +136,7 @@ impl EppRouter {
peer_ready,
model_name: cfg.model_name,
inflight: Arc::new(Semaphore::new(cfg.max_inflight_requests)),
})
}
}

/// Overall EPP readiness for the gRPC health signal: the pod reflector is
Expand Down
176 changes: 158 additions & 18 deletions deploy/inference-gateway/ext-proc/src/selector.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,21 @@ struct ReconcileState {
}

impl Selector {
fn validate_queueing_requirements(
cfg: &EppStandaloneConfig,
queueing_enabled: bool,
) -> Result<()> {
if queueing_enabled && cfg.max_num_batched_tokens.unwrap_or(0) == 0 {
anyhow::bail!(
"DYN_EPP_MAX_NUM_BATCHED_TOKENS is required (and must be > 0) because the router \
scheduling policy enables queueing for model {}; set it to the engine's \
--max-num-batched-tokens",
cfg.model_name
);
}
Ok(())
}

pub async fn new(cfg: &EppStandaloneConfig) -> Result<Self> {
Self::new_with_kv_router_config(cfg, kv_router_config_from_dynamo_env()).await
}
Expand All @@ -107,32 +122,16 @@ impl Selector {
cfg: &EppStandaloneConfig,
kv_router_config: KvRouterConfig,
) -> Result<Self> {
let cancel = CancellationToken::new();

// If queueing is enabled, we need to validate that the max_num_batched_tokens is set.
// Done once at startup to avoid validating on every reconcile.
let queueing_enabled = kv_router_config
.queueing_enabled(Some(&cfg.model_name))
.map_err(|e| anyhow!("resolving router policy for model {}: {e}", cfg.model_name))?;
if queueing_enabled && cfg.max_num_batched_tokens.unwrap_or(0) == 0 {
anyhow::bail!(
"DYN_EPP_MAX_NUM_BATCHED_TOKENS is required (and must be > 0) because the router \
scheduling policy enables queueing for model {}; set it to the engine's \
--max-num-batched-tokens",
cfg.model_name
);
}
Self::validate_queueing_requirements(cfg, queueing_enabled)?;

let mut builder =
SelectionServiceBuilder::new(kv_router_config).indexer_threads(cfg.selector_threads);

let replication: Option<(String, u16)> = match &cfg.peer_service {
Some(name) => Some((
name.clone(),
crate::peer_discovery::resolve_replica_sync_port(&cfg.namespace, name).await?,
)),
None => None,
};
let replication = Self::replication(cfg).await?;

if let Some((_, peer_sync_port)) = &replication {
builder = builder.replica_sync(*peer_sync_port, Vec::new());
Expand All @@ -145,6 +144,44 @@ impl Selector {
.map_err(|e| anyhow!("building embedded selection service: {e}"))?,
);

Self::from_service_with_replication(cfg, service, replication).await
}

/// Wrap a prebuilt selection service for use by a custom EPP image.
pub(crate) async fn from_service(
cfg: &EppStandaloneConfig,
service: Arc<SelectionService>,
) -> Result<Self> {
let queueing_enabled = service
.queueing_enabled(&cfg.model_name)
.map_err(|e| anyhow!("resolving router policy for model {}: {e}", cfg.model_name))?;
Self::validate_queueing_requirements(cfg, queueing_enabled)?;
if cfg.peer_service.is_some() && !service.replica_sync_enabled() {
anyhow::bail!(
"DYN_EPP_PEER_SERVICE requires a prebuilt SelectionService with replica sync enabled"
);
}
let replication = Self::replication(cfg).await?;
Self::from_service_with_replication(cfg, service, replication).await
}
Comment thread
ishandhanani marked this conversation as resolved.
Resolved

async fn replication(cfg: &EppStandaloneConfig) -> Result<Option<(String, u16)>> {
match &cfg.peer_service {
Some(name) => Ok(Some((
name.clone(),
crate::peer_discovery::resolve_replica_sync_port(&cfg.namespace, name).await?,
))),
None => Ok(None),
}
}

async fn from_service_with_replication(
cfg: &EppStandaloneConfig,
service: Arc<SelectionService>,
replication: Option<(String, u16)>,
) -> Result<Self> {
let cancel = CancellationToken::new();

let peer_ready = if let Some((service_name, peer_sync_port)) = replication {
// In replicated mode, we need to exclude ourselves from the peer set which requires the POD_IP
let self_ip = std::env::var("POD_IP")
Expand Down Expand Up @@ -356,8 +393,26 @@ impl Drop for Selector {

#[cfg(test)]
mod tests {
use dynamo_kv_router::{
WorkerInputView, WorkerPicker, WorkerSelectionContext, WorkerSelectionPolicy,
WorkerSelectionPolicyError,
};

use super::*;

struct FirstEligiblePicker;

impl WorkerPicker for FirstEligiblePicker {
fn pick(
&mut self,
_context: &WorkerSelectionContext<'_>,
input: WorkerInputView<'_>,
) -> Result<usize, WorkerSelectionPolicyError> {
assert!(!input.candidates().is_empty());
Ok(0)
}
}

fn model_policy_file() -> tempfile::NamedTempFile {
let policy_file = tempfile::NamedTempFile::new().expect("create policy file");
std::fs::write(
Expand Down Expand Up @@ -491,6 +546,40 @@ models:
selector
}

#[tokio::test]
async fn prebuilt_service_runs_custom_policy_through_reservation() {
let service = SelectionServiceBuilder::new(KvRouterConfig::default())
.indexer_threads(1)
.worker_selection_policy_factory(|config, worker_type| {
WorkerSelectionPolicy::new(
config.clone(),
worker_type,
Vec::new(),
Box::new(FirstEligiblePicker),
)
})
.build()
.await
.expect("custom selection service should build");
let selector = Selector::from_service(&test_config(), Arc::new(service))
.await
.expect("selector should accept a prebuilt service");
selector
.reconcile(&[schedulable_registration(1)])
.await
.expect("worker should register");

let response = selector
.select_and_reserve(select_request("custom-policy"))
.await
.expect("custom policy should select and reserve");
assert_eq!(response.worker_id, 1);
selector
.free_reservation("custom-policy")
.await
.expect("custom-policy reservation should be releasable");
}

/// Item 1: a successful reserve books load, and the final free releases it.
/// `free_reservation` is idempotent: a second free of the same id is a no-op
/// success (matching a duplicate completion signal from the gateway).
Expand Down Expand Up @@ -746,6 +835,57 @@ models:
.expect("threshold-free model should allow missing capacity");
}

#[tokio::test]
async fn prebuilt_service_rejects_missing_queue_capacity() {
let policy_file = model_policy_file();
let router_config = router_config_with_policy(&policy_file);
let service = Arc::new(
SelectionServiceBuilder::new(router_config)
.indexer_threads(1)
.build()
.await
.expect("selection service should build"),
);
let mut cfg = test_config();
cfg.model_name = "queueing-model".to_string();
cfg.max_num_batched_tokens = None;

let error = Selector::from_service(&cfg, service)
.await
.err()
.expect("queueing model must reject missing capacity");
assert!(
error
.to_string()
.contains("DYN_EPP_MAX_NUM_BATCHED_TOKENS is required"),
"{error}"
);
}

#[tokio::test]
async fn prebuilt_service_rejects_peer_discovery_without_replica_sync() {
let service = Arc::new(
SelectionServiceBuilder::new(KvRouterConfig::default())
.indexer_threads(1)
.build()
.await
.expect("selection service should build"),
);
let mut cfg = test_config();
cfg.peer_service = Some("does-not-exist".to_string());

let error = Selector::from_service(&cfg, service)
.await
.err()
.expect("peer discovery must require replica sync");
assert!(
error
.to_string()
.contains("SelectionService with replica sync enabled"),
"{error}"
);
}

#[tokio::test]
async fn duplicate_worker_ids_are_rejected_before_reconciliation() {
let selector = Selector::new(&test_config())
Expand Down
4 changes: 2 additions & 2 deletions lib/kv-router/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ bench = []
profile = []
standalone-indexer = ["dep:axum", "dep:serde_json", "dep:reqwest", "dep:zmq", "dep:chrono", "dep:tracing-appender"]
standalone-slot-tracker = ["dep:axum", "dep:serde_json", "dep:zmq"]
standalone-selection = ["standalone-indexer"]
standalone-selection = ["standalone-indexer", "dep:once_cell"]

[dependencies]
# repo
Expand All @@ -37,7 +37,7 @@ blake3 = { workspace = true }
dashmap = { workspace = true }
indexmap = { workspace = true }
ordered-float = { workspace = true }
once_cell = "1"
once_cell = { version = "1", optional = true }
derive_builder = { workspace = true }
fastrand = "2"
prometheus = { workspace = true, optional = true }
Expand Down
12 changes: 10 additions & 2 deletions lib/kv-router/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,14 @@ pub use radix_tree::RadixTree;
pub use scheduling::LocalScheduler;
pub use scheduling::PrefillLoadEstimator;
pub use scheduling::policy::{FcfsPolicy, RouterSchedulingPolicy, SchedulingPolicy, WsptPolicy};
pub use scheduling::{KvSchedulerError, PotentialLoad, SchedulingRequest, SchedulingResponse};
pub use selector::{DefaultWorkerSelector, WorkerSelector};
pub use scheduling::{
KvSchedulerError, PotentialLoad, SchedulingRequest, SchedulingResponse,
WorkerSelectionPolicyError,
};
pub use selector::{
DefaultWorkerPicker, DefaultWorkerScorer, DefaultWorkerSelector, ScoredWorkerCandidate,
WorkerCacheInput, WorkerCandidate, WorkerInputView, WorkerInputs, WorkerLoadInput,
WorkerPicker, WorkerRoutingInput, WorkerScorer, WorkerSelectionContext, WorkerSelectionPolicy,
WorkerSelector,
};
pub use tracking_hash::{TrackingHashAlgorithm, TrackingHashContext, TrackingHashScope};
Loading
Loading