Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
0624d41
feat(router): add custom policy catalog
ishandhanani Aug 7, 2026
00ccf79
fix(bindings): remove redundant python module feature
ishandhanani Aug 8, 2026
a2282a1
refactor(epp): move custom catalog ownership to deployers
ishandhanani Aug 8, 2026
7762e27
docs: add custom worker selection walkthrough
ishandhanani Aug 8, 2026
9ff6cc3
docs: tighten custom worker selection walkthrough
ishandhanani Aug 8, 2026
07cfd91
fix(router): honor custom policy entrypoints
ishandhanani Aug 8, 2026
0a700ba
fix(epp): box linked policy configuration
ishandhanani Aug 8, 2026
1cf5b4f
style(bindings): format custom policy selection
ishandhanani Aug 8, 2026
3adeec0
refactor(router): centralize resolved policy wiring
ishandhanani Aug 8, 2026
ef1980c
Merge remote-tracking branch 'origin/main' into codex/pr-12536-rebase-2
ishandhanani Aug 10, 2026
f79d4fe
ci(bindings): remove redundant policy check
ishandhanani Aug 10, 2026
db2f22f
build(router): restore catalog workspace integration
ishandhanani Aug 10, 2026
53fa82a
docs(router): add custom policy example
ishandhanani Aug 10, 2026
1747553
fix(router): reject custom policy on in-process engines
ishandhanani Aug 10, 2026
0d7da98
chore(router): assign custom policy example ownership
ishandhanani Aug 10, 2026
5574fc3
docs: ignore same-PR custom policy links
ishandhanani Aug 10, 2026
55f5bfc
Merge branch 'main' into idhanani/dyn-3713-sglang-cache-load-policy
ishandhanani Aug 10, 2026
a5edbd4
fix(build): copy policy examples into EPP builds
ishandhanani Aug 10, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/codeowners/areas.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ areas:
- lib/llm/src/kv_router/publisher/
- tests/router/
- tests/router/configs/
- lib/router-plugins/
Comment thread
ishandhanani marked this conversation as resolved.
Resolved
- examples/router/policy-class-queues.yaml
- label: backend-vllm
github_team: '@ai-dynamo/dynamo-backend-vllm-codeowners'
Expand Down
8 changes: 8 additions & 0 deletions .github/workflows/pre-merge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,10 @@ jobs:
if: matrix.dir == 'lib/bindings/python'
working-directory: ${{ matrix.dir }}
run: cargo test --locked --features aic-forward-pass -- --nocapture
- name: Compile Python bindings custom worker-selection policy support
Comment thread
ishandhanani marked this conversation as resolved.
Resolved
if: matrix.dir == 'lib/bindings/python'
working-directory: ${{ matrix.dir }}
run: cargo check --locked --features custom-policy,select-service
- name: Run KVBM offload Mooncake trace test
if: matrix.dir == '.'
working-directory: ${{ matrix.dir }}
Expand Down Expand Up @@ -322,6 +326,10 @@ jobs:
if: matrix.dir == 'lib/bindings/python'
working-directory: ${{ matrix.dir }}
run: cargo clippy --locked --no-deps --features aic-forward-pass --all-targets -- -D warnings
- name: Lint Python bindings custom worker-selection policy support
if: matrix.dir == 'lib/bindings/python'
working-directory: ${{ matrix.dir }}
run: cargo clippy --locked --no-deps --features custom-policy,select-service --all-targets -- -D warnings
- name: Run KVBM offload Mooncake trace clippy
if: matrix.dir == '.'
working-directory: ${{ matrix.dir }}
Expand Down
1 change: 1 addition & 0 deletions CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@

# === router (@ai-dynamo/dynamo-router-codeowners) ===
/lib/kv-router/ @ai-dynamo/dynamo-router-codeowners
/lib/router-plugins/ @ai-dynamo/dynamo-router-codeowners
/lib/kv-router/tests/ @ai-dynamo/dynamo-router-codeowners
/lib/kv-router/src/indexer/ @ai-dynamo/dynamo-router-codeowners
/lib/kv-router/src/recovery/ @ai-dynamo/dynamo-router-codeowners
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -416,8 +416,8 @@ def add_arguments(self, parser) -> None:
env_var="DYN_ROUTER_POLICY_CONFIG",
default=None,
help=(
"KV Router: Startup-only YAML policy-family and cache-bucket "
"queue configuration. "
"KV Router: Startup-only YAML configuration for policy-class queues "
"and custom worker-selection instances. "
"When omitted, router_queue_threshold and router_queue_policy define "
"one synthetic policy class; queueing remains disabled unless "
"router_queue_threshold is set."
Expand Down
2 changes: 1 addition & 1 deletion deploy/inference-gateway/ext-proc/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ pub use epp_standalone_config::{EppMode, EppStandaloneConfig, TokenizerProtocol}
pub use inference_pool::PoolState;
pub use picker::{Endpoint, EndpointPicker, PickResult, RequestInfo, ResponseUsage};
pub use pod_discovery::{PodDiscovery, RawWorker};
pub use runner::{run, run_with_selection_service};
pub use runner::{run, run_with_selection_service, run_with_worker_selection_policy_registry};
pub use selector::{OverlapSummary, SelectRequest, SelectResponse, Selector, WorkerRegistration};
pub use server::ExtProcServer;
pub use topology_adapter::{RegistrationDefaults, TopologyAdapter};
Expand Down
131 changes: 123 additions & 8 deletions deploy/inference-gateway/ext-proc/src/runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,13 @@
use std::sync::Arc;

use anyhow::Result;
use dynamo_kv_router::services::selection::SelectionService;
use dynamo_kv_router::WorkerSelectionPolicyFactory;
use dynamo_kv_router::config::{KvRouterConfig, try_kv_router_config_from_dynamo_env};
use dynamo_kv_router::services::selection::{SelectionService, WorkerSelectionPolicyRegistry};
use tokio::net::TcpListener;
use tokio_rustls::TlsAcceptor;

use crate::{EppMode, EppStandaloneConfig, ExtProcServer, Router, metrics};
use crate::{EppMode, EppStandaloneConfig, ExtProcServer, Router, Selector, metrics};

const GRPC_PORT: u16 = 9002;
const HEALTH_PORT: u16 = 9003;
Expand All @@ -39,6 +41,15 @@ struct Config {
component: String,
}

enum StandaloneSelectionService {
Default,
Existing(SelectionService),
LinkedWorkerSelectionPolicy {
kv_router_config: KvRouterConfig,
factory: WorkerSelectionPolicyFactory,
},
}

impl Config {
fn from_env() -> Self {
let namespace = env_or("DYN_NAMESPACE_PREFIX", "")
Expand Down Expand Up @@ -119,13 +130,60 @@ fn create_tls_acceptor() -> Result<TlsAcceptor> {
/// Run the stock EPP until it exits.
pub async fn run() -> Result<()> {
init_tracing();
run_inner(EppMode::from_env()?, None).await
let mode = EppMode::from_env()?;
if matches!(mode, EppMode::Standalone) {
let kv_router_config =
try_kv_router_config_from_dynamo_env().map_err(anyhow::Error::msg)?;
reject_unlinked_worker_selection_policy(&kv_router_config)?;
}
run_inner(mode, StandaloneSelectionService::Default).await
}

fn reject_unlinked_worker_selection_policy(config: &KvRouterConfig) -> Result<()> {
if let Some(instance) = config.selected_worker_selection_policy_instance()? {
anyhow::bail!(
"worker-selection instance {instance:?} is configured, but this stock EPP has no linked worker-selection policy catalog; run a custom EPP binary that links the catalog"
);
}
Ok(())
Comment thread
ishandhanani marked this conversation as resolved.
Resolved
}
Comment thread
ishandhanani marked this conversation as resolved.
Resolved

/// Run the standalone EPP around a caller-built selection service.
pub async fn run_with_selection_service(service: SelectionService) -> Result<()> {
init_tracing();
run_inner(EppMode::Standalone, Some(service)).await
run_inner(
EppMode::Standalone,
StandaloneSelectionService::Existing(service),
)
.await
}

/// Run EPP with policy types statically linked into the image and instances selected from YAML.
pub async fn run_with_worker_selection_policy_registry(
registry: WorkerSelectionPolicyRegistry,
) -> Result<()> {
init_tracing();
let mode = EppMode::from_env()?;
let kv_router_config = try_kv_router_config_from_dynamo_env().map_err(anyhow::Error::msg)?;
let Some(factory) = registry.resolve(&kv_router_config)? else {
return run_inner(mode, StandaloneSelectionService::Default).await;
};
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
Resolved
require_standalone_mode_for_linked_worker_selection_policy(mode)?;
run_inner(
mode,
StandaloneSelectionService::LinkedWorkerSelectionPolicy {
kv_router_config,
factory,
},
)
.await
}

fn require_standalone_mode_for_linked_worker_selection_policy(mode: EppMode) -> Result<()> {
if matches!(mode, EppMode::Standalone) {
return Ok(());
}
anyhow::bail!("linked worker-selection policies require DYN_EPP_MODE=standalone")
}

fn init_tracing() {
Expand All @@ -137,7 +195,10 @@ fn init_tracing() {
.try_init();
}

async fn run_inner(mode: EppMode, selection_service: Option<SelectionService>) -> Result<()> {
async fn run_inner(
mode: EppMode,
standalone_selection_service: StandaloneSelectionService,
) -> Result<()> {
let standalone = matches!(mode, EppMode::Standalone);

let config = Config::from_env();
Expand Down Expand Up @@ -187,11 +248,26 @@ async fn run_inner(mode: EppMode, selection_service: Option<SelectionService>) -
"Initializing standalone selector mode (no Dynamo runtime)..."
);
metrics::set_served_model(&selector_cfg.model_name);
let router = Arc::new(match selection_service {
Some(service) => {
let router = Arc::new(match standalone_selection_service {
StandaloneSelectionService::Default => {
crate::EppRouter::from_selector(selector_cfg).await?
}
StandaloneSelectionService::Existing(service) => {
crate::EppRouter::from_selection_service(selector_cfg, service).await?
}
StandaloneSelectionService::LinkedWorkerSelectionPolicy {
kv_router_config,
factory,
} => {
let service =
Selector::build_selection_service_with_worker_selection_policy_factory(
&selector_cfg,
kv_router_config,
factory,
)
.await?;
crate::EppRouter::from_selection_service(selector_cfg, service).await?
}
None => crate::EppRouter::from_selector(selector_cfg).await?,
});
let ready_router = router.clone();
serve(router, move || ready_router.is_ready(), health_reporter).await
Expand Down Expand Up @@ -318,3 +394,42 @@ async fn serve<P: crate::EndpointPicker>(
Ok(())
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn linked_policy_requires_standalone_epp() {
assert!(
require_standalone_mode_for_linked_worker_selection_policy(EppMode::Standalone).is_ok()
);
assert!(
require_standalone_mode_for_linked_worker_selection_policy(EppMode::DynamoRuntime)
.is_err()
);
}

#[test]
fn stock_epp_rejects_custom_worker_selection_policy() {
let policy_file = tempfile::NamedTempFile::new().unwrap();
std::fs::write(
policy_file.path(),
r#"
worker_selection:
default: custom
instances:
- name: custom
type: acme
parameters: {}
"#,
)
.unwrap();
let config = KvRouterConfig {
router_policy_config: Some(policy_file.path().display().to_string()),
..Default::default()
};

assert!(reject_unlinked_worker_selection_policy(&config).is_err());
}
}
69 changes: 48 additions & 21 deletions deploy/inference-gateway/ext-proc/src/selector.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ use dynamo_kv_router::protocols::RoutingConstraints;
use dynamo_kv_router::services::selection::{
PromptRequest, SelectAndReserveRequest as CoreSelectAndReserveRequest, SelectionError,
SelectionService, SelectionServiceBuilder, WorkerLifecycle, WorkerRequest as CoreWorkerRequest,
WorkerSelectionPolicyFactory,
};
use tokio::sync::Mutex;
use tokio_util::sync::CancellationToken;
Expand Down Expand Up @@ -118,10 +119,37 @@ impl Selector {
Self::new_with_kv_router_config(cfg, kv_router_config_from_dynamo_env()).await
}

/// Build a selection service using the custom policy compiled into this EPP image.
pub(crate) async fn build_selection_service_with_worker_selection_policy_factory(
cfg: &EppStandaloneConfig,
kv_router_config: KvRouterConfig,
factory: WorkerSelectionPolicyFactory,
) -> Result<SelectionService> {
Self::build_selection_service(cfg, kv_router_config, |builder| {
builder.worker_selection_policy_factory(move |config, worker_type, partition| {
factory(config, worker_type, partition)
})
})
.await
}

async fn new_with_kv_router_config(
cfg: &EppStandaloneConfig,
kv_router_config: KvRouterConfig,
) -> Result<Self> {
let service =
Self::build_selection_service(cfg, kv_router_config, |builder| builder).await?;
Self::from_service(cfg, service).await
}

async fn build_selection_service<F>(
cfg: &EppStandaloneConfig,
kv_router_config: KvRouterConfig,
configure: F,
) -> Result<SelectionService>
where
F: FnOnce(SelectionServiceBuilder) -> SelectionServiceBuilder,
{
// If queueing is enabled, we need to validate that the max_num_batched_tokens is set.
// Done once at startup to avoid validating on every reconcile.
let queueing_enabled = kv_router_config
Expand All @@ -137,14 +165,10 @@ impl Selector {
builder = builder.replica_sync(*peer_sync_port, Vec::new());
}

let service = Arc::new(
builder
.build()
.await
.map_err(|e| anyhow!("building embedded selection service: {e}"))?,
);

Self::from_service_with_replication(cfg, service, replication).await
configure(builder)
.build()
.await
.map_err(|e| anyhow!("building embedded selection service: {e}"))
}

/// Wrap a prebuilt selection service for use by a custom EPP image.
Expand Down Expand Up @@ -553,19 +577,22 @@ models:

#[tokio::test]
async fn prebuilt_service_runs_custom_policy_through_reservation() {
let service = SelectionServiceBuilder::new(KvRouterConfig::default())
.indexer_threads(1)
.worker_selection_policy_factory(|config, worker_type, _partition| {
WorkerSelectionPolicy::new(
config.clone(),
worker_type,
Vec::new(),
Box::new(FirstEligiblePicker),
)
})
.build()
.await
.expect("custom selection service should build");
let service = Selector::build_selection_service(
&test_config(),
KvRouterConfig::default(),
|builder| {
builder.worker_selection_policy_factory(|config, worker_type, _partition| {
WorkerSelectionPolicy::new(
config.clone(),
worker_type,
Vec::new(),
Box::new(FirstEligiblePicker),
)
})
},
)
.await
.expect("custom selection service should build");
let selector = Selector::from_service(&test_config(), service)
.await
.expect("selector should accept a prebuilt service");
Expand Down
2 changes: 1 addition & 1 deletion docs/fern/index.yml
Original file line number Diff line number Diff line change
Expand Up @@ -878,7 +878,7 @@ navigation:
- page: Building from Source
path: pages/developer-guide/advanced-customizations/building-from-source.md
- page: Custom Worker Selection
path: pages/developer-guide/advanced-customizations/custom-worker-selection.md
path: pages/developer-guide/advanced-customizations/custom-worker-selection.mdx
- page: Custom Vision Encoders
slug: custom-vision-encoders
path: pages/developer-guide/advanced-customizations/custom-vision-encoders.md
Expand Down
Loading
Loading