diff --git a/deploy/inference-gateway/ext-proc/src/epp.rs b/deploy/inference-gateway/ext-proc/src/epp.rs index b0c8288eaac8..c606914fe9a8 100644 --- a/deploy/inference-gateway/ext-proc/src/epp.rs +++ b/deploy/inference-gateway/ext-proc/src/epp.rs @@ -180,7 +180,6 @@ impl Router { None, model_name.clone(), actual_namespace.to_string(), - enable_eagle, // ext-proc constructs no KvWorkerMonitor; overload publishing is // unused on this path (matches the prior namespace-lookup miss). None, diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/router/router-guide.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/router/router-guide.md index 6fe4641ad4f2..2b5fc729d3cc 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/router/router-guide.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/router/router-guide.md @@ -189,9 +189,15 @@ This is not specific to router flags — any card difference splits a set the sa - **Applying the flag to only some replicas of a set splits it.** Whether by flag or by an exported `DYN_ROUTER_MODE` that reaches some processes and not others, every member of a set must end up with the same value. > [!IMPORTANT] -> An advertised configuration **replaces** the frontend's for that worker set rather than merging with it. If you set `--router-mode kv` on a worker and the frontend was tuned with KV flags such as `--router-temperature`, restate those flags on the worker too, or that set falls back to KV defaults. +> An advertised configuration **replaces** the frontend's for that worker set rather than merging with it. On a worker, a flag you do not pass means **the default**, not "inherit the frontend's value". If the frontend was tuned with `--router-kv-overlap-score-credit 2.5` and a worker advertises only `--router-mode kv`, that worker set routes with the default `1.0` — the tuning is not inherited, it is replaced. Restate on the worker every flag that matters for it. -Because the flags are shared with the frontend, they read the same environment variables — `--router-mode` reads `DYN_ROUTER_MODE`. Kubernetes scopes environment per service, so a frontend's value does not reach workers, and the launch scripts pass the flag rather than exporting it. The one case to watch is a shell that exports `DYN_ROUTER_MODE` and then starts both the frontend and workers: there the workers advertise it instead of inheriting. +Merging is not offered because it cannot be done correctly: the KV tuning is flat concrete values with no record of which ones were set explicitly, so a deliberate `1.0` is indistinguishable from a default `1.0`. + +Workers and the frontend share defaults exactly — both build their configuration from the same argument groups — so the values only diverge when the frontend is tuned and the worker is not. + +The flags also read the same environment variables, and this changes the answer depending on how you deploy. If the frontend's tuning comes from the environment and the worker shares that environment — a single shell starting both — the worker picks the tuning up and nothing is lost. Kubernetes scopes environment per service, so there the worker sees only its own flags. The same intent can therefore produce different routing in the two setups. `--router-mode` itself is the exception in the other direction: a shared `DYN_ROUTER_MODE` makes a worker advertise when you meant it to inherit. + +The `Activating prefill router` log line reports the configuration each hop actually resolved — mode, block size, session TTL, and KV tuning — which is the quickest way to confirm what a worker set ended up with. The frontend-only options (`--router-min-initial-workers`, `--enforce-disagg`, `--admission-control`) are not accepted by workers, since a model card does not carry them. diff --git a/lib/bindings/c/src/lib.rs b/lib/bindings/c/src/lib.rs index 65d11f08ae88..6f5257643c3a 100644 --- a/lib/bindings/c/src/lib.rs +++ b/lib/bindings/c/src/lib.rs @@ -861,7 +861,6 @@ pub unsafe extern "C" fn create_routers( None, model_name.clone(), actual_namespace.clone(), - enable_eagle, // C bindings construct no KvWorkerMonitor; overload publishing is // unused on this path (matches the prior namespace-lookup miss). None, diff --git a/lib/llm/src/discovery/model_manager.rs b/lib/llm/src/discovery/model_manager.rs index d04370d806df..01ddd469f96d 100644 --- a/lib/llm/src/discovery/model_manager.rs +++ b/lib/llm/src/discovery/model_manager.rs @@ -3287,7 +3287,6 @@ mod tests { None, "topology-model".to_string(), worker_set.namespace().to_string(), - false, None, ); let encoder = crate::kv_router::EncoderRouter::new( diff --git a/lib/llm/src/discovery/watcher.rs b/lib/llm/src/discovery/watcher.rs index c6b28ed5cf68..58bbd6e75ca4 100644 --- a/lib/llm/src/discovery/watcher.rs +++ b/lib/llm/src/discovery/watcher.rs @@ -651,8 +651,9 @@ where { let mut prefill_config = router_config.kv_router_config.clone(); prefill_config.router_track_active_blocks = false; - let prefill_enable_eagle = false; + // Fallback only: a prefill worker that declares its own + // `router_config` overrides this at activation time. Some(PrefillRouter::new_with_selector_factory( None, self.manager.clone(), @@ -665,7 +666,6 @@ where router_config.session_affinity_ttl_secs, model_name.clone(), namespace.clone(), - prefill_enable_eagle, worker_monitor.clone(), Some(allocator_trim.clone()), )) diff --git a/lib/llm/src/kv_router/prefill_router/activation.rs b/lib/llm/src/kv_router/prefill_router/activation.rs index fe1adae2b482..330700d43c4f 100644 --- a/lib/llm/src/kv_router/prefill_router/activation.rs +++ b/lib/llm/src/kv_router/prefill_router/activation.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use std::sync::atomic::Ordering; -use anyhow::Result; +use anyhow::{Context as _, Result}; use tokio::sync::{oneshot, watch}; use dynamo_kv_router::{ @@ -36,21 +36,92 @@ use crate::{ session_affinity::create_affinity_coordinator, }; +/// How the prefill worker set wants to be routed to, resolved from its cards. +#[derive(Debug)] +struct PrefillAdvertisement { + router_mode: RouterMode, + /// `None` when the card declared nothing, so the decode set's tuning applies. + kv_router_config: Option, + /// Taken from an advertised `router_config` whole, `None` included -- an + /// advertisement replaces the decode set's configuration rather than + /// merging with it. `None` here when nothing was advertised at all. + session_affinity_ttl: Option>, + is_eagle: bool, + /// The prefill workers' own block size, which is what their KV events are + /// keyed on. The decode set's value would index this pool at the wrong + /// granularity if the two ever differ. + kv_cache_block_size: u32, +} + +/// A prefill worker that declares its own `router_config` governs this hop; +/// one that declares nothing inherits `decode_router_mode`. That is what lets a +/// deployment run KV-routed prefill in front of round-robin decode. +fn resolve_advertisement_from_cards( + cards: &[ModelDeploymentCard], + decode_router_mode: RouterMode, +) -> Result { + let mode_of = |card: &ModelDeploymentCard| { + card.router_config + .as_ref() + .map_or(decode_router_mode, |config| config.router_mode) + }; + + let Some(first) = cards.first() else { + anyhow::bail!("no readable prefill model card; cannot resolve prefill routing"); + }; + + let advertisement = PrefillAdvertisement { + router_mode: mode_of(first), + kv_router_config: first + .router_config + .as_ref() + .map(|config| config.kv_router_config.clone()), + session_affinity_ttl: first.router_config.as_ref().map(|config| { + config + .session_affinity_ttl_secs + .map(std::time::Duration::from_secs) + }), + is_eagle: first.runtime_config.enable_eagle, + kv_cache_block_size: first.kv_cache_block_size, + }; + + // A fleet mid-rolling-update can disagree. First card wins, but a silent + // split means half the fleet is routed on the other half's terms. + let disagreeing = cards + .iter() + .skip(1) + .filter(|card| mode_of(card) != advertisement.router_mode) + .count(); + if disagreeing > 0 { + tracing::warn!( + resolved_mode = ?advertisement.router_mode, + disagreeing, + total = cards.len(), + "Prefill workers advertise conflicting router modes; using the first" + ); + } + + Ok(advertisement) +} + impl PrefillRouter { /// Create a disabled prefill router that will never activate (passthrough only) pub fn disabled( model_manager: Arc, - router_mode: RouterMode, + decode_router_mode: RouterMode, session_affinity_ttl_secs: Option, ) -> Arc { - Self::disabled_with_selector(model_manager, router_mode, session_affinity_ttl_secs) + Self::disabled_with_selector(model_manager, decode_router_mode, session_affinity_ttl_secs) } + /// `decode_router_mode` is the owning decode worker set's mode. It governs + /// decode-side decisions and is the fallback for the prefill hop; a prefill + /// worker that advertises its own `router_config` overrides the latter. #[expect(clippy::too_many_arguments)] pub fn new( activation_rx: oneshot::Receiver, model_manager: Arc, - router_mode: RouterMode, + decode_router_mode: RouterMode, kv_cache_block_size: u32, kv_router_config: Option, decode_router: Option>, @@ -58,13 +129,12 @@ impl PrefillRouter { session_affinity_ttl_secs: Option, model_name: String, namespace: String, - is_eagle: bool, worker_monitor: Option, ) -> Arc { Self::new_with_selector_factory( Some(activation_rx), model_manager, - router_mode, + decode_router_mode, kv_cache_block_size, kv_router_config, decode_router, @@ -78,7 +148,6 @@ impl PrefillRouter { session_affinity_ttl_secs, model_name, namespace, - is_eagle, worker_monitor, None, ) @@ -91,7 +160,7 @@ where { pub(crate) fn disabled_with_selector( model_manager: Arc, - router_mode: RouterMode, + decode_router_mode: RouterMode, session_affinity_ttl_secs: Option, ) -> Arc { Arc::new(Self { @@ -103,7 +172,7 @@ where decode_session_affinity: std::sync::OnceLock::new(), model_manager, cancel_token: tokio_util::sync::CancellationToken::new(), - router_mode, + decode_router_mode, session_affinity_ttl: session_affinity_ttl_secs.map(std::time::Duration::from_secs), conditional_disagg_policy: make_conditional_disagg_policy(None), conditional_disagg_prefill_busy_threshold: None, @@ -111,7 +180,6 @@ where prefill_load_estimator: None, model_name: String::new(), // Not used for disabled router namespace: String::new(), // Not used for disabled router - is_eagle: false, task_guard: None, lifecycle: std::sync::atomic::AtomicU8::new(PrefillLifecycleState::Pending as u8), #[cfg(test)] @@ -123,7 +191,7 @@ where pub(crate) fn new_with_selector_factory( activation_rx: Option>, model_manager: Arc, - router_mode: RouterMode, + decode_router_mode: RouterMode, kv_cache_block_size: u32, kv_router_config: Option, decode_router: Option>>, @@ -132,7 +200,6 @@ where session_affinity_ttl_secs: Option, model_name: String, namespace: String, - is_eagle: bool, worker_monitor: Option, task_guard: Option, ) -> Arc { @@ -156,7 +223,7 @@ where decode_session_affinity: std::sync::OnceLock::new(), model_manager: model_manager.clone(), cancel_token: cancel_token.clone(), - router_mode, + decode_router_mode, session_affinity_ttl: session_affinity_ttl_secs.map(std::time::Duration::from_secs), conditional_disagg_policy, conditional_disagg_prefill_busy_threshold, @@ -164,7 +231,6 @@ where prefill_load_estimator, model_name, namespace, - is_eagle, task_guard: task_guard.clone(), lifecycle: std::sync::atomic::AtomicU8::new(PrefillLifecycleState::Pending as u8), #[cfg(test)] @@ -223,11 +289,6 @@ where kv_cache_block_size: u32, kv_router_config: Option, ) -> Result<(PrefillBinding, Client)> { - tracing::info!( - router_mode = ?context.router_mode, - "Activating prefill router" - ); - let endpoint_id = endpoint.id(); // Start runtime config watcher for this endpoint (needed for get_disaggregated_endpoint) @@ -237,30 +298,52 @@ where .get_or_create_runtime_config_watcher(&endpoint) .await?; - let inner_router = if context.router_mode.is_kv_routing() { - let discovered_cards = endpoint - .component() - .drt() - .discovery() - .list(DiscoveryQuery::EndpointModels { - namespace: endpoint_id.namespace.clone(), - component: endpoint_id.component.clone(), - endpoint: endpoint_id.name.clone(), - }) - .await; - let is_eagle = match discovered_cards { - Ok(instances) => instances - .into_iter() - .find_map(|instance| instance.deserialize_model::().ok()) - .map_or(context.is_eagle, |card| card.runtime_config.enable_eagle), - Err(error) => { - tracing::warn!(%error, "Failed to read prefill model card; using configured EAGLE mode"); - context.is_eagle - } - }; + let advertisement = Self::resolve_prefill_advertisement(context, &endpoint).await?; + let prefill_router_mode = advertisement.router_mode; + + // Everything the hop uses comes from the prefill card when it says so, + // falling back to the decode set. A block size of 0 means the card never + // declared one, so it cannot be trusted over the decode set's. + let prefill_block_size = match advertisement.kv_cache_block_size { + 0 => kv_cache_block_size, + advertised => advertised, + }; + let prefill_session_affinity_ttl = advertisement + .session_affinity_ttl + .unwrap_or(context.session_affinity_ttl); + + // A prefill card that declares a mode may declare KV tuning alongside it; + // honoring only half of its `RouterConfig` would be a trap. Whichever + // config wins, `router_track_active_blocks` stays off: prefill routing is + // prompt-side, and crediting decode blocks here would double-count load. + let advertised_kv_tuning = advertisement.kv_router_config.is_some(); + let prefill_kv_config = match advertisement.kv_router_config { + Some(mut advertised) => { + advertised.router_track_active_blocks = false; + Some(advertised) + } + None => kv_router_config, + }; + // Logged once per activation, and deliberately the *resolved* values + // rather than what the card asked for. An advertisement replaces the + // decode set's configuration rather than merging with it, so a worker + // that names a mode without restating tuning silently gets defaults -- + // this line is what makes that answerable without a rebuild. + tracing::info!( + ?prefill_router_mode, + decode_router_mode = ?context.decode_router_mode, + advertised = advertised_kv_tuning, + is_eagle = advertisement.is_eagle, + block_size = prefill_block_size, + session_affinity_ttl = ?prefill_session_affinity_ttl, + kv_tuning = ?prefill_kv_config, + "Activating prefill router" + ); + + let inner_router = if prefill_router_mode.is_kv_routing() { // Create KV chooser using the endpoint (this is a prefill router) - let effective_kv_router_config = kv_router_config.clone().unwrap_or_default(); + let effective_kv_router_config = prefill_kv_config.clone().unwrap_or_default(); let selector = (context.worker_selector_factory)( &effective_kv_router_config, crate::worker_type::WorkerType::Prefill, @@ -270,21 +353,21 @@ where .model_manager .kv_chooser_for_with_selector( &endpoint, - kv_cache_block_size, + prefill_block_size, selector, - kv_router_config, + prefill_kv_config, context.prefill_load_estimator.clone(), Some(crate::worker_type::WorkerType::Prefill), WORKER_TYPE_PREFILL, Some(context.model_name.clone()), - is_eagle, + advertisement.is_eagle, ) .await?; // Extract client from kv_chooser to ensure shared state let client = kv_chooser.client().clone(); let affinity = - create_affinity_coordinator(context.session_affinity_ttl, client.clone()).await?; + create_affinity_coordinator(prefill_session_affinity_ttl, client.clone()).await?; let prefill_client = client.clone(); // Build the PushRouter for prefill with KV mode using the shared client @@ -308,15 +391,15 @@ where // Create client for simple router let client = endpoint.client().await?; let affinity = - create_affinity_coordinator(context.session_affinity_ttl, client.clone()).await?; + create_affinity_coordinator(prefill_session_affinity_ttl, client.clone()).await?; let prefill_client = client.clone(); - // Create simple push router with the frontend's router mode + // Create simple push router with the resolved prefill router mode // Note: Per-worker metrics (active_prefill_tokens, active_decode_blocks) are only // available in KV routing mode where the router has actual bookkeeping. let push_router = PushRouter::>::from_client_with_monitor( client, - context.router_mode, + prefill_router_mode, None, // worker_monitor ) .await?; @@ -326,7 +409,7 @@ where crate::session_affinity::SessionAffinityPushRouter::new_with_coordinator( push_router, affinity, - context.router_mode.is_direct_routing(), + prefill_router_mode.is_direct_routing(), ), )), prefill_client, @@ -337,11 +420,55 @@ where PrefillBinding { endpoint_id, router: inner_router.0, + prefill_router_mode, }, inner_router.1, )) } + /// Fetch the prefill worker set's own cards and resolve how the prefill hop + /// should be routed. + /// + /// Returns `Err` rather than silently inheriting when the cards cannot be + /// read. `drive_target` retries activation with backoff, and a delayed + /// activation is far cheaper than routing an entire deployment with the + /// wrong strategy because of one transient discovery miss — a downgrade that + /// would persist until the binding was rebuilt. + async fn resolve_prefill_advertisement( + context: &PrefillBuildContext, + endpoint: &Endpoint, + ) -> Result { + let endpoint_id = endpoint.id(); + let instances = endpoint + .component() + .drt() + .discovery() + .list(DiscoveryQuery::EndpointModels { + namespace: endpoint_id.namespace.clone(), + component: endpoint_id.component.clone(), + endpoint: endpoint_id.name.clone(), + }) + .await + .with_context(|| format!("listing prefill model cards for {endpoint_id}"))?; + + // An unparseable card is not just a missing EAGLE hint any more: it + // drops a worker out of the vote that decides how this hop is routed. + let cards: Vec = instances + .into_iter() + .filter_map(|instance| { + instance + .deserialize_model::() + .inspect_err(|error| { + tracing::debug!(%error, %endpoint_id, "Skipping unreadable prefill card") + }) + .ok() + }) + .collect(); + + resolve_advertisement_from_cards(&cards, context.decode_router_mode) + .with_context(|| format!("prefill endpoint {endpoint_id}")) + } + /// Attach the freshly-created prefill `Client` to this WorkerSet's monitor (handed in /// at construction). The monitor then publishes the overloaded set to the prefill pool /// and watches the prefill endpoint for metric cleanup. No-op for a disabled router. @@ -400,7 +527,7 @@ where } let build_context = PrefillBuildContext { model_manager: router_ref.model_manager.clone(), - router_mode: router_ref.router_mode, + decode_router_mode: router_ref.decode_router_mode, worker_selector_factory: router_ref .worker_selector_factory .clone() @@ -408,7 +535,6 @@ where prefill_load_estimator: router_ref.prefill_load_estimator.clone(), session_affinity_ttl: router_ref.session_affinity_ttl, model_name: router_ref.model_name.clone(), - is_eagle: router_ref.is_eagle, }; drop(router_ref); let build = Self::build_binding( @@ -521,3 +647,110 @@ where self.target.lock().clone() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::entrypoint::RouterConfig; + use dynamo_kv_router::config::KvRouterConfig; + + fn card(router_config: Option) -> ModelDeploymentCard { + let mut card = ModelDeploymentCard::with_name_only("test-model"); + card.router_config = router_config; + card + } + + fn card_with_block_size(block_size: u32) -> ModelDeploymentCard { + let mut card = card(None); + card.kv_cache_block_size = block_size; + card + } + + #[test] + fn inherits_decode_mode_when_card_advertises_nothing() { + // The pre-override behavior: a prefill worker that says nothing is + // routed exactly as the decode set is. Every deployment predating this + // feature lands here, so it must not shift. + let cards = vec![card(None)]; + let resolved = + resolve_advertisement_from_cards(&cards, RouterMode::RoundRobin).expect("resolves"); + assert_eq!(resolved.router_mode, RouterMode::RoundRobin); + assert!(resolved.kv_router_config.is_none()); + } + + #[test] + fn card_router_config_overrides_decode_mode() { + // The headline case: KV-routed prefill in front of round-robin decode. + let cards = vec![card(Some(RouterConfig::new( + RouterMode::KV, + KvRouterConfig::default(), + )))]; + let resolved = + resolve_advertisement_from_cards(&cards, RouterMode::RoundRobin).expect("resolves"); + assert_eq!(resolved.router_mode, RouterMode::KV); + assert!(resolved.kv_router_config.is_some()); + } + + #[test] + fn first_card_wins_when_the_fleet_disagrees() { + // A rolling update can leave old (inheriting) and new (advertising) + // prefill workers side by side. Resolution must stay deterministic + // rather than depending on which card happened to be read. + let cards = vec![ + card(Some(RouterConfig::new( + RouterMode::KV, + KvRouterConfig::default(), + ))), + card(None), + ]; + let resolved = + resolve_advertisement_from_cards(&cards, RouterMode::RoundRobin).expect("resolves"); + assert_eq!(resolved.router_mode, RouterMode::KV); + } + + #[test] + fn block_size_comes_from_the_prefill_card() { + // The prefill pool's KV events are keyed on its own block size. Taking + // the decode set's would index this pool at the wrong granularity. + let cards = vec![card_with_block_size(64)]; + let resolved = resolve_advertisement_from_cards(&cards, RouterMode::KV).expect("resolves"); + assert_eq!(resolved.kv_cache_block_size, 64); + } + + #[test] + fn session_affinity_ttl_is_taken_whole_or_not_at_all() { + // An advertisement replaces the decode set's configuration rather than + // merging, so a card that advertises without a TTL means "no affinity", + // not "inherit the frontend's". A card advertising nothing means inherit. + let advertised = vec![card(Some(RouterConfig::new( + RouterMode::KV, + KvRouterConfig::default(), + )))]; + assert_eq!( + resolve_advertisement_from_cards(&advertised, RouterMode::KV) + .expect("resolves") + .session_affinity_ttl, + Some(None), + ); + + let silent = vec![card(None)]; + assert_eq!( + resolve_advertisement_from_cards(&silent, RouterMode::KV) + .expect("resolves") + .session_affinity_ttl, + None, + ); + } + + #[test] + fn no_readable_card_is_an_error_not_a_silent_inherit() { + // Activation must fail and be retried rather than quietly routing the + // whole deployment with the decode set's mode. + let error = resolve_advertisement_from_cards(&[], RouterMode::KV) + .expect_err("empty card list must not resolve"); + assert!( + error.to_string().contains("no readable prefill model card"), + "unexpected error: {error}" + ); + } +} diff --git a/lib/llm/src/kv_router/prefill_router/conditional_bypass.rs b/lib/llm/src/kv_router/prefill_router/conditional_bypass.rs index 7557f6a1cd28..9ec3eb3f4408 100644 --- a/lib/llm/src/kv_router/prefill_router/conditional_bypass.rs +++ b/lib/llm/src/kv_router/prefill_router/conditional_bypass.rs @@ -74,7 +74,10 @@ where session_affinity: Option<&SessionAffinityId>, decode_affinity_target: Option, ) -> Result> { - if !self.router_mode.is_kv_routing() { + // Conditional disagg peeks the decode router to find the cache-hot + // decode worker, so it needs the *decode* set in KV mode. A KV prefill + // hop in front of a non-KV decode set cannot make this decision. + if !self.decode_router_mode.is_kv_routing() { return Ok(None); } diff --git a/lib/llm/src/kv_router/prefill_router/mod.rs b/lib/llm/src/kv_router/prefill_router/mod.rs index 8f3405027e04..2e37d8c58d46 100644 --- a/lib/llm/src/kv_router/prefill_router/mod.rs +++ b/lib/llm/src/kv_router/prefill_router/mod.rs @@ -193,7 +193,10 @@ where decode_session_affinity: OnceLock, model_manager: Arc, cancel_token: CancellationToken, - router_mode: RouterMode, + /// Mode of the decode set that owns this router. Governs decode-side + /// decisions, and is the fallback for the prefill hop -- not its mode. That + /// lives on [`PrefillBinding::prefill_router_mode`]. + decode_router_mode: RouterMode, session_affinity_ttl: Option, conditional_disagg_policy: Box, /// Resolved once at construction: dedicated threshold if set, otherwise @@ -206,7 +209,6 @@ where model_name: String, /// Namespace (used for logging / lifecycle messages). namespace: String, - is_eagle: bool, task_guard: Option, /// Initialization and worker availability state. lifecycle: AtomicU8, @@ -220,6 +222,10 @@ where { endpoint_id: EndpointId, router: InnerPrefillRouter, + /// Resolved at activation from the prefill card. Lives here rather than on + /// `PrefillRouter` because it is unknowable until a target is discovered, + /// and changes when the binding is rebuilt. + prefill_router_mode: RouterMode, } struct PrefillBuildContext @@ -227,12 +233,12 @@ where Sel: WorkerSelector + Send + 'static, { model_manager: Arc, - router_mode: RouterMode, + /// Fallback mode for the prefill hop when the prefill card advertises none. + decode_router_mode: RouterMode, worker_selector_factory: WorkerSelectorFactory, prefill_load_estimator: Option>, session_affinity_ttl: Option, model_name: String, - is_eagle: bool, } pub(crate) trait PrefillRouterLifecycle: Send + Sync { @@ -381,13 +387,6 @@ where .as_ref() .and_then(|r| r.prefill_worker_id); - if self.router_mode.is_direct_routing() && preselected_worker.is_none() { - return Err(anyhow::anyhow!( - "Prefill worker ID required in Direct routing mode but none found in request. \ - Expected prefill_worker_id to be set via x-dynamo-prefill-instance-id header by external router (e.g., EPP)." - )); - } - let tracker = prefill_req.tracker.clone(); let mut prefill_context = Context::with_id_and_metadata(prefill_req, request_id.clone(), metadata.clone()); @@ -400,6 +399,16 @@ where let Some(binding) = self.binding.load_full() else { return next.generate(context.map(|_| req)).await; }; + + // Keyed on the prefill mode, not the decode set's, and checked after + // the binding loads because that is where the prefill mode lives. + if binding.prefill_router_mode.is_direct_routing() && preselected_worker.is_none() { + return Err(anyhow::anyhow!( + "Prefill worker ID required in Direct routing mode but none found in request. \ + Expected prefill_worker_id to be set via x-dynamo-prefill-instance-id header by external router (e.g., EPP)." + )); + } + let router = &binding.router; let endpoint_id = &binding.endpoint_id; let prefill_result: Result<(PrefillOutcome, Option)> = async { @@ -868,7 +877,6 @@ mod tests { None, "test-model".to_string(), "test-namespace".to_string(), - false, None, ); let task_state = Arc::downgrade(&router.activation_task_state); diff --git a/lib/llm/src/kv_router/prefill_router/query.rs b/lib/llm/src/kv_router/prefill_router/query.rs index c7449bf5b208..171f07f6bb03 100644 --- a/lib/llm/src/kv_router/prefill_router/query.rs +++ b/lib/llm/src/kv_router/prefill_router/query.rs @@ -309,6 +309,7 @@ mod tests { name: endpoint_name.to_string(), }, router: InnerPrefillRouter::SimpleRouter(shared.clone()), + prefill_router_mode: mode, }, ))); prefill.lifecycle.store( diff --git a/tests/router/common.py b/tests/router/common.py index 31d5e144e415..8cd89b4842ad 100644 --- a/tests/router/common.py +++ b/tests/router/common.py @@ -3667,3 +3667,275 @@ async def run_direct_mode_tests(): asyncio.run(run_direct_mode_tests()) logger.info("Direct-mode disagg E2E test passed") + + +def _test_disagg_per_role_router_modes( + prefill_workers, + decode_workers, + block_size: int, + request, + frontend_port: int, + test_payload: dict, + store_backend: str = "etcd", + request_plane: str = "nats", +): + """Validate that prefill and decode tiers can run different router modes. + + The prefill mockers advertise ``RouterConfig(RouterMode.KV)`` in their model + deployment cards while the decode mockers advertise nothing, so decode + inherits the frontend's ``--router-mode round-robin``. This asserts the two + hops are genuinely routed on different terms: + + 1. Progressive prefix-extending requests converge on ONE prefill worker, + which only KV routing produces — round-robin would rotate. + 2. Those same requests spread across MORE THAN ONE decode worker, which only + round-robin produces — KV routing would converge there too. + + Assertion 2 is the one that fails if the per-role override is ignored and + the whole pipeline silently runs in a single mode. + """ + num_requests = 4 + + with FrontendRouterProcess( + request, + block_size, + frontend_port, + decode_workers.namespace, + store_backend, + request_plane=request_plane, + router_mode="round-robin", + min_initial_workers=decode_workers.num_workers, + ): + frontend_url = f"http://localhost:{frontend_port}" + chat_url = f"{frontend_url}/v1/chat/completions" + + logger.info( + "Waiting for prefill and decode workers to register with the " + "round-robin frontend..." + ) + asyncio.run( + wait_for_frontend_ready( + frontend_url=frontend_url, + expected_num_workers=( + prefill_workers.num_workers + decode_workers.num_workers + ), + timeout=120, + engine_workers=[prefill_workers, decode_workers], + store_backend=store_backend, + request_plane=request_plane, + ) + ) + + async def send_progressive_requests(): + prefill_worker_ids: list[int] = [] + decode_worker_ids: list[int] = [] + base_content = test_payload["messages"][0]["content"] + + async with aiohttp.ClientSession() as session: + for i in range(num_requests): + payload = { + **test_payload, + "messages": [ + { + "role": "user", + "content": " ".join([base_content] * (i + 1)), + } + ], + "nvext": {"extra_fields": ["worker_id"]}, + "stream": True, + } + + async with session.post(chat_url, json=payload) as response: + assert ( + response.status == 200 + ), f"Request {i + 1} failed with status {response.status}" + + prefill_wid = None + decode_wid = None + body = await response.text() + for data in parse_sse_json_chunks(body): + worker_id_info = data.get("nvext", {}).get("worker_id", {}) + if "prefill_worker_id" in worker_id_info: + prefill_wid = worker_id_info["prefill_worker_id"] + if "decode_worker_id" in worker_id_info: + decode_wid = worker_id_info["decode_worker_id"] + + logger.info( + f"Request {i + 1}: prefill_worker_id={prefill_wid}, " + f"decode_worker_id={decode_wid}" + ) + if prefill_wid is not None: + prefill_worker_ids.append(prefill_wid) + if decode_wid is not None: + decode_worker_ids.append(decode_wid) + + await asyncio.sleep(1) + + return prefill_worker_ids, decode_worker_ids + + prefill_ids, decode_ids = asyncio.run(send_progressive_requests()) + + logger.info(f"Collected prefill_worker_ids: {prefill_ids}") + logger.info(f"Collected decode_worker_ids: {decode_ids}") + + assert len(prefill_ids) == num_requests, ( + f"Expected {num_requests} prefill_worker_ids, got {len(prefill_ids)}. " + f"A prefill hop must have run for every request." + ) + assert ( + len(decode_ids) == num_requests + ), f"Expected {num_requests} decode_worker_ids, got {len(decode_ids)}." + + # The prefill tier advertised KV, so prefix reuse must concentrate it. + # As in the all-KV disagg test, the TCP request plane can show a + # transient on the first request before the initial "stored" KV events + # are ingested, so only requests 2..N are required to converge there. + converged = prefill_ids[1:] if request_plane == "tcp" else prefill_ids + assert len(set(converged)) == 1, ( + f"Prefill advertised RouterMode.KV, so prefix-extending requests must " + f"converge on one prefill worker; got {set(converged)}. " + f"Full list: {prefill_ids}" + ) + + # The decode tier inherited round-robin, so it must NOT converge. If the + # prefill card's override had leaked into the decode router (or the + # decode set had been dragged into KV mode), these would collapse to one. + assert len(set(decode_ids)) > 1, ( + f"Decode inherited round-robin, so requests must spread across " + f"workers; all {num_requests} landed on {set(decode_ids)}. " + f"This means the per-role router config did not take effect." + ) + + assert prefill_ids[0] not in set(decode_ids), ( + f"Prefill worker {prefill_ids[0]} should not appear in the decode " + f"worker set {set(decode_ids)}." + ) + + logger.info( + "Verified per-role router modes: prefill converged on " + f"{set(converged)} (KV) while decode spread across " + f"{set(decode_ids)} (round-robin)" + ) + + +def _test_disagg_per_role_session_affinity( + prefill_workers, + decode_workers, + block_size: int, + request, + frontend_port: int, + test_payload: dict, + store_backend: str = "etcd", + request_plane: str = "nats", +): + """Validate that session affinity is configured per hop, not per deployment. + + The prefill workers advertise a session-affinity TTL; the decode workers + advertise the same router mode without one. Both hops then run round-robin, + so the only thing that can pin a session is its own affinity setting: + + 1. Repeated requests carrying one session id stay on ONE prefill worker, + because that hop has affinity. + 2. The same requests spread across MULTIPLE decode workers, because that hop + does not -- round-robin rotates. + + Assertion 2 is the one that fails if the two hops share a TTL: decode would + pin alongside prefill. + + This covers configurability, not expiry. Proving a TTL elapses would mean + sleeping past it and asserting a rebind, which is timing-dependent and can + re-select the same worker. + """ + num_requests = 4 + + with FrontendRouterProcess( + request, + block_size, + frontend_port, + decode_workers.namespace, + store_backend, + request_plane=request_plane, + router_mode="round-robin", + min_initial_workers=decode_workers.num_workers, + ): + frontend_url = f"http://localhost:{frontend_port}" + chat_url = f"{frontend_url}/v1/chat/completions" + + asyncio.run( + wait_for_frontend_ready( + frontend_url=frontend_url, + expected_num_workers=( + prefill_workers.num_workers + decode_workers.num_workers + ), + timeout=120, + engine_workers=[prefill_workers, decode_workers], + store_backend=store_backend, + request_plane=request_plane, + ) + ) + + session_headers = {"x-dynamo-session-id": f"per-role-ttl-{uuid.uuid4()}"} + content = test_payload["messages"][0]["content"] + + async def send_session_requests(): + prefill_ids: list[int] = [] + decode_ids: list[int] = [] + async with aiohttp.ClientSession() as session: + for i in range(num_requests): + payload = { + **test_payload, + "messages": [{"role": "user", "content": content}], + "nvext": {"extra_fields": ["worker_id"]}, + "stream": True, + "max_tokens": 1, + } + async with session.post( + chat_url, json=payload, headers=session_headers + ) as response: + assert ( + response.status == 200 + ), f"Request {i + 1} failed with status {response.status}" + # worker_id repeats across chunks of one stream; record + # it once per request so the counts match the requests. + prefill_wid = None + decode_wid = None + body = await response.text() + for data in parse_sse_json_chunks(body): + worker_id_info = data.get("nvext", {}).get("worker_id", {}) + if "prefill_worker_id" in worker_id_info: + prefill_wid = worker_id_info["prefill_worker_id"] + if "decode_worker_id" in worker_id_info: + decode_wid = worker_id_info["decode_worker_id"] + if prefill_wid is not None: + prefill_ids.append(prefill_wid) + if decode_wid is not None: + decode_ids.append(decode_wid) + await asyncio.sleep(0.5) + return prefill_ids, decode_ids + + prefill_ids, decode_ids = asyncio.run(send_session_requests()) + logger.info(f"Session-pinned prefill_worker_ids: {prefill_ids}") + logger.info(f"Session-pinned decode_worker_ids: {decode_ids}") + + assert ( + len(prefill_ids) == num_requests + ), f"Expected {num_requests} prefill_worker_ids, got {len(prefill_ids)}." + assert ( + len(decode_ids) == num_requests + ), f"Expected {num_requests} decode_worker_ids, got {len(decode_ids)}." + + assert len(set(prefill_ids)) == 1, ( + f"Prefill advertised a session-affinity TTL, so one session must stay " + f"pinned to one prefill worker; got {set(prefill_ids)}. Full: {prefill_ids}" + ) + assert len(set(decode_ids)) > 1, ( + f"Decode advertised no session-affinity TTL, so the same session must " + f"not be pinned there; all {num_requests} requests landed on " + f"{set(decode_ids)}. That means the hops are sharing one TTL." + ) + + logger.info( + "Verified per-hop session affinity: prefill pinned to " + f"{set(prefill_ids)} (TTL advertised) while decode spread across " + f"{set(decode_ids)} (no TTL)" + ) diff --git a/tests/router/mocker_process.py b/tests/router/mocker_process.py index bcb883fa0540..fadf0d17e575 100644 --- a/tests/router/mocker_process.py +++ b/tests/router/mocker_process.py @@ -119,6 +119,15 @@ def _build_mocker_command( str(mocker_args["response_replay_trace_path"]), ] ) + if "router_mode" in mocker_args: + command.extend(["--router-mode", str(mocker_args["router_mode"])]) + if "router_session_affinity_ttl_secs" in mocker_args: + command.extend( + [ + "--router-session-affinity-ttl-secs", + str(mocker_args["router_session_affinity_ttl_secs"]), + ] + ) return command diff --git a/tests/router/test_router_e2e_with_mockers.py b/tests/router/test_router_e2e_with_mockers.py index bf50acfca50e..e9bb2eb7d226 100644 --- a/tests/router/test_router_e2e_with_mockers.py +++ b/tests/router/test_router_e2e_with_mockers.py @@ -23,6 +23,8 @@ from tests.router.common import ( _test_busy_threshold_endpoint, _test_disagg_direct_mode, + _test_disagg_per_role_router_modes, + _test_disagg_per_role_session_affinity, _test_disagg_router_overload_529, _test_disagg_topology_required_prefill_pin_match_and_mismatch, _test_python_router_bindings, @@ -1425,6 +1427,93 @@ def run_case(prefill_workers, decode_workers): run_case(prefill_workers, decode_workers) +@pytest.mark.timeout(180) +def test_disagg_per_role_router_modes( + request, + runtime_services_dynamic_ports, + predownload_tokenizers, +): + """KV-routed prefill in front of round-robin decode, via per-role MDC config. + + The prefill mockers advertise RouterConfig(RouterMode.KV) in their cards; the + decode mockers advertise nothing and inherit the frontend's round-robin. Both + hops must then route on their own terms rather than sharing one mode. + """ + logger.info("Starting per-role router mode disagg test (KV prefill, RR decode)") + + shared_namespace = f"test-namespace-{generate_random_suffix()}" + base_mocker_args = { + "speedup_ratio": SPEEDUP_RATIO, + "block_size": BLOCK_SIZE, + } + + with launch_disagg_workers( + request, + shared_namespace, + "prefill_first", + # Only the prefill tier advertises a mode; decode inherits the frontend's. + prefill_mocker_args={**base_mocker_args, "router_mode": "kv"}, + decode_mocker_args=base_mocker_args, + num_prefill_mockers=2, + num_decode_mockers=2, + enable_disagg_bootstrap=False, + ) as (prefill_workers, decode_workers): + _test_disagg_per_role_router_modes( + prefill_workers=prefill_workers, + decode_workers=decode_workers, + block_size=BLOCK_SIZE, + request=request, + frontend_port=allocate_frontend_ports(request, 1)[0], + test_payload=TEST_PAYLOAD, + request_plane="nats", + ) + + +@pytest.mark.timeout(180) +def test_disagg_per_role_session_affinity( + request, + runtime_services_dynamic_ports, + predownload_tokenizers, +): + """Session affinity is configured per hop: prefill pins, decode does not. + + Both tiers advertise round-robin; only prefill advertises a TTL. If the two + hops shared a session-affinity setting, decode would pin alongside prefill. + """ + logger.info("Starting per-hop session affinity disagg test") + + shared_namespace = f"test-namespace-{generate_random_suffix()}" + base_mocker_args = { + "speedup_ratio": SPEEDUP_RATIO, + "block_size": BLOCK_SIZE, + } + + with launch_disagg_workers( + request, + shared_namespace, + "prefill_first", + # Same mode on both tiers, so affinity is the only difference between them. + prefill_mocker_args={ + **base_mocker_args, + "router_mode": "round-robin", + "router_session_affinity_ttl_secs": 300, + }, + decode_mocker_args={**base_mocker_args, "router_mode": "round-robin"}, + num_prefill_mockers=2, + num_decode_mockers=2, + enable_disagg_bootstrap=False, + ) as (prefill_workers, decode_workers): + _test_disagg_per_role_session_affinity( + prefill_workers=prefill_workers, + decode_workers=decode_workers, + block_size=BLOCK_SIZE, + request=request, + frontend_port=allocate_frontend_ports(request, 1)[0], + test_payload=TEST_PAYLOAD, + request_plane="nats", + ) + + @pytest.mark.timeout(180) def test_router_decisions_disagg_router_aic( request,