From 6b3585b9f1884bd0f8b3d80c170ef74f8fea2fa8 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 11 Sep 2026 11:26:17 -0700 Subject: [PATCH 1/3] feat(mocker): publish native KV events from the vLLM gRPC server Signed-off-by: jthomson04 --- Cargo.lock | 3 + lib/mocker/servers/vllm/Cargo.toml | 5 +- lib/mocker/servers/vllm/README.md | 23 +++ lib/mocker/servers/vllm/src/main.rs | 3 +- lib/mocker/servers/vllm/src/server.rs | 64 ++++++- lib/mocker/servers/vllm/src/server_tests.rs | 86 ++++++++- lib/mocker/servers/vllm/tests/common/mod.rs | 190 ++++++++++++++++++++ lib/mocker/servers/vllm/tests/sidecar.rs | 14 ++ lib/mocker/src/services/zmq_events.rs | 143 +++++++++++++-- 9 files changed, 505 insertions(+), 26 deletions(-) create mode 100644 lib/mocker/servers/vllm/tests/common/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 5bbc2229579d..ed0d2155fa9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3149,7 +3149,10 @@ dependencies = [ "blake3", "clap", "dynamo-backend-common", + "dynamo-kv-router", + "dynamo-llm", "dynamo-mocker", + "dynamo-runtime", "dynamo-vllm-sidecar", "futures", "prost-types 0.13.5", diff --git a/lib/mocker/servers/vllm/Cargo.toml b/lib/mocker/servers/vllm/Cargo.toml index 17bcd78bf655..99a06f6b7066 100644 --- a/lib/mocker/servers/vllm/Cargo.toml +++ b/lib/mocker/servers/vllm/Cargo.toml @@ -17,7 +17,7 @@ name = "dynamo-vllm-mocker-server" path = "src/main.rs" [dependencies] -dynamo-mocker = { workspace = true } +dynamo-mocker = { workspace = true, features = ["zmq-events"] } dynamo-vllm-sidecar = { path = "../../../sidecar/vllm" } anyhow = { workspace = true } @@ -34,5 +34,8 @@ tracing-subscriber = { workspace = true } uuid = { workspace = true } [dev-dependencies] +dynamo-llm = { path = "../../../llm", default-features = false } +dynamo-kv-router = { workspace = true } +dynamo-runtime = { workspace = true } dynamo-backend-common = { workspace = true, features = ["testing"] } tokio-stream = { workspace = true, features = ["net"] } diff --git a/lib/mocker/servers/vllm/README.md b/lib/mocker/servers/vllm/README.md index 68ee7efc0506..dba030d62815 100644 --- a/lib/mocker/servers/vllm/README.md +++ b/lib/mocker/servers/vllm/README.md @@ -38,6 +38,29 @@ Synthetic output plans are limited to 32,768 tokens. LiveEngine uses a small, fixed response buffer for each request and cancels slow consumers rather than turning declared output length into a second admission-control policy. +## KV events + +Like regular mock workers, the server publishes KV cache events when prefix +caching is enabled, except in decode mode. It uses the existing Mocker ZMQ +publisher and reports the endpoint through native engine discovery. The +sidecar forwards these events to Dynamo's router. + +The event publisher binds a free port by default. For a fixed port and an +optional replay port, add these settings to the server command above: + +```bash +--extra-engine-args '{"block_size":4,"zmq_kv_events_port":5557,"zmq_replay_port":5558}' +``` + +Run the sidecar as shown above and use a frontend with `--router-mode kv`. +Sidecar discovers the event endpoint without an additional setting. When +running across containers or hosts, expose the event port; a fixed port can +make this easier. + +Set `"enable_prefix_caching":false` to disable both prefix caching and KV +events. Decode servers do not publish events. As with regular mock workers, +publisher setup failures are logged and serving continues without KV events. + ## Disaggregated wire-flow Run separate endpoints for the two emulated vLLM roles: diff --git a/lib/mocker/servers/vllm/src/main.rs b/lib/mocker/servers/vllm/src/main.rs index 2e512c120d40..f28a7f44c8e8 100644 --- a/lib/mocker/servers/vllm/src/main.rs +++ b/lib/mocker/servers/vllm/src/main.rs @@ -73,7 +73,8 @@ async fn main() -> anyhow::Result<()> { max_concurrent_requests: args.max_concurrent_requests, }, engine_args, - )?; + ) + .await?; tracing::info!( listen = %args.listen, diff --git a/lib/mocker/servers/vllm/src/server.rs b/lib/mocker/servers/vllm/src/server.rs index 7230a92a6e86..c296f01c0ea3 100644 --- a/lib/mocker/servers/vllm/src/server.rs +++ b/lib/mocker/servers/vllm/src/server.rs @@ -6,9 +6,12 @@ use std::pin::Pin; use std::sync::Arc; use clap::ValueEnum; -use dynamo_mocker::common::protocols::{EngineType, MockEngineArgs, OutputSignal, WorkerType}; -use dynamo_mocker::live::{LiveEngine, LiveRequest, stable_request_uuid}; +use dynamo_mocker::common::protocols::{ + EngineType, KvEventPublishers, MockEngineArgs, OutputSignal, WorkerType, +}; +use dynamo_mocker::live::{LiveEngine, LiveEngineConfig, LiveRequest, stable_request_uuid}; use dynamo_mocker::scheduler::MockerMetrics; +use dynamo_mocker::services::zmq_events::ZmqKvEventSink; use dynamo_vllm_sidecar::proto as pb; use futures::Stream; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; @@ -67,11 +70,16 @@ pub struct VllmMockerService { model_info: Arc, server_info: Arc, engine: LiveEngine, + kv_event_sources: Arc>, request_permits: Arc, } impl VllmMockerService { - pub fn new(config: MockerServerConfig, engine_args: MockEngineArgs) -> anyhow::Result { + pub async fn new( + config: MockerServerConfig, + engine_args: MockEngineArgs, + ) -> anyhow::Result { + let engine_args = engine_args.normalized()?; anyhow::ensure!( engine_args.engine_type == EngineType::Vllm, "Mocker engine_type must be vllm" @@ -134,11 +142,57 @@ impl VllmMockerService { .unwrap_or_default(), rl_capabilities: None, }; + // The wire role is separate from the aggregated scheduler used to + // emulate disaggregated requests. + let sink = if engine_args.needs_kv_publisher() && config.mode != ServerMode::Decode { + match ZmqKvEventSink::bind( + engine_args.zmq_kv_events_port, + engine_args.zmq_replay_port, + DP_RANK, + u32::try_from(engine_args.block_size)?, + ) + .await + { + Ok(sink) => Some(sink), + Err(error) => { + tracing::error!(dp_rank = DP_RANK, %error, "Failed to create ZMQ KV event sink"); + None + } + } + } else { + None + }; + let kv_event_sources = sink + .as_ref() + .map(|sink| pb::KvEventSource { + transport: "zmq".to_string(), + endpoint: sink.endpoint().to_string(), + topic: String::new(), + data_parallel_rank: Some(DP_RANK), + replay_endpoint: sink.replay_endpoint().unwrap_or_default().to_string(), + encoding: "msgpack".to_string(), + schema_version: 1, + ..Default::default() + }) + .into_iter() + .collect(); + let engine = LiveEngine::start_with_config( + engine_args, + DP_RANK, + LiveEngineConfig { + kv_event_publishers: KvEventPublishers::new( + None, + sink.map(|sink| Arc::new(sink) as _), + ), + ..Default::default() + }, + )?; Ok(Self { config: Arc::new(config), model_info: Arc::new(model_info), server_info: Arc::new(server_info), - engine: LiveEngine::start(engine_args, DP_RANK)?, + engine, + kv_event_sources: Arc::new(kv_event_sources), request_permits: Arc::new(Semaphore::new(max_concurrent_requests)), }) } @@ -319,7 +373,7 @@ impl pb::control_server::Control for VllmMockerService { _request: Request, ) -> Result, Status> { Ok(Response::new(pb::GetKvEventSourcesResponse { - sources: Vec::new(), + sources: (*self.kv_event_sources).clone(), })) } diff --git a/lib/mocker/servers/vllm/src/server_tests.rs b/lib/mocker/servers/vllm/src/server_tests.rs index a09e87bb6034..8801df829477 100644 --- a/lib/mocker/servers/vllm/src/server_tests.rs +++ b/lib/mocker/servers/vllm/src/server_tests.rs @@ -155,14 +155,15 @@ fn text_prompts_fail_with_an_actionable_status() { assert!(error.message().contains("token_ids")); } -#[test] -fn service_rejects_non_vllm_or_multi_rank_engines() { +#[tokio::test] +async fn service_rejects_non_vllm_or_multi_rank_engines() { let sglang = MockEngineArgs::builder() .engine_type(EngineType::Sglang) .build() .unwrap(); assert!( VllmMockerService::new(MockerServerConfig::default(), sglang) + .await .err() .unwrap() .to_string() @@ -172,6 +173,7 @@ fn service_rejects_non_vllm_or_multi_rank_engines() { let multi_rank = MockEngineArgs::builder().dp_size(2).build().unwrap(); assert!( VllmMockerService::new(MockerServerConfig::default(), multi_rank) + .await .err() .unwrap() .to_string() @@ -184,6 +186,7 @@ fn service_rejects_non_vllm_or_multi_rank_engines() { .unwrap(); assert!( VllmMockerService::new(MockerServerConfig::default(), disaggregated) + .await .err() .unwrap() .to_string() @@ -196,6 +199,7 @@ fn service_rejects_non_vllm_or_multi_rank_engines() { }; assert!( VllmMockerService::new(disabled, MockEngineArgs::default()) + .await .err() .unwrap() .to_string() @@ -208,8 +212,9 @@ fn service_rejects_non_vllm_or_multi_rank_engines() { /// capability absence; this test catches it at the Control RPC boundary. #[tokio::test] async fn unsupported_rl_control_reports_unimplemented() { - let service = - VllmMockerService::new(MockerServerConfig::default(), MockEngineArgs::default()).unwrap(); + let service = VllmMockerService::new(MockerServerConfig::default(), MockEngineArgs::default()) + .await + .unwrap(); let server_info = pb::control_server::Control::get_server_info( &service, Request::new(pb::GetServerInfoRequest {}), @@ -241,7 +246,9 @@ async fn unary_generate_maps_capacity_rejection_to_resource_exhausted() { .speedup_ratio(0.0) .build() .unwrap(); - let service = VllmMockerService::new(MockerServerConfig::default(), args).unwrap(); + let service = VllmMockerService::new(MockerServerConfig::default(), args) + .await + .unwrap(); let mut oversized = request("oversized"); oversized.prompt = Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds { ids: vec![1, 2, 3, 4, 5], @@ -269,6 +276,7 @@ async fn concurrent_request_limit_rejects_a_stalled_stream() { }, args, ) + .await .unwrap(); let mut first_request = request("stalled"); first_request.stopping.as_mut().unwrap().max_new_tokens = 100; @@ -337,7 +345,9 @@ fn decode_rejects_a_handoff_missing_the_opacity_sentinel() { #[tokio::test] async fn unary_generate_accumulates_output_and_terminal_metadata() { - let service = VllmMockerService::new(MockerServerConfig::default(), admitting_args()).unwrap(); + let service = VllmMockerService::new(MockerServerConfig::default(), admitting_args()) + .await + .unwrap(); let mut routed_request = Request::new(request("unary")); routed_request.metadata_mut().insert( @@ -390,7 +400,9 @@ async fn streaming_generate_maps_capacity_rejection_to_resource_exhausted() { .speedup_ratio(0.0) .build() .unwrap(); - let service = VllmMockerService::new(MockerServerConfig::default(), args).unwrap(); + let service = VllmMockerService::new(MockerServerConfig::default(), args) + .await + .unwrap(); let mut oversized = request("oversized-stream"); oversized.prompt = Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds { ids: vec![1, 2, 3, 4, 5], @@ -422,7 +434,9 @@ async fn streaming_survives_a_producer_that_outruns_a_stalled_consumer() { // The producer runs instantly while the consumer stalls, so the request // races far past LiveEngine's fixed per-request buffer. The server's pump // must absorb the burst instead of shedding the stream into an INTERNAL. - let service = VllmMockerService::new(MockerServerConfig::default(), admitting_args()).unwrap(); + let service = VllmMockerService::new(MockerServerConfig::default(), admitting_args()) + .await + .unwrap(); let mut bursty = request("bursty"); bursty.stopping.as_mut().unwrap().max_new_tokens = 50; @@ -459,3 +473,59 @@ async fn streaming_survives_a_producer_that_outruns_a_stalled_consumer() { assert_eq!(finish.num_output_tokens, 50); assert_eq!(service.active_request_count(), 0); } + +#[tokio::test] +async fn kv_event_discovery_follows_regular_mocker_rules() { + for (mode, prefix_caching, expected_sources) in [ + (ServerMode::Aggregated, true, 1), + (ServerMode::Prefill, true, 1), + (ServerMode::Decode, true, 0), + (ServerMode::Aggregated, false, 0), + ] { + let mut args = admitting_args(); + args.enable_prefix_caching = prefix_caching; + let service = VllmMockerService::new( + MockerServerConfig { + mode, + ..Default::default() + }, + args, + ) + .await + .unwrap(); + assert_eq!( + pb::control_server::Control::get_kv_event_sources( + &service, + Request::new(pb::GetKvEventSourcesRequest {}) + ) + .await + .unwrap() + .into_inner() + .sources + .len(), + expected_sources + ); + } +} + +#[tokio::test] +async fn failed_kv_publisher_is_not_advertised() { + let occupied = tokio::net::TcpListener::bind("0.0.0.0:0").await.unwrap(); + let mut args = admitting_args(); + args.zmq_kv_events_port = Some(occupied.local_addr().unwrap().port()); + let service = VllmMockerService::new(MockerServerConfig::default(), args) + .await + .unwrap(); + assert_eq!( + pb::control_server::Control::get_kv_event_sources( + &service, + Request::new(pb::GetKvEventSourcesRequest {}) + ) + .await + .unwrap() + .into_inner() + .sources + .len(), + 0 + ); +} diff --git a/lib/mocker/servers/vllm/tests/common/mod.rs b/lib/mocker/servers/vllm/tests/common/mod.rs new file mode 100644 index 000000000000..495736e44eec --- /dev/null +++ b/lib/mocker/servers/vllm/tests/common/mod.rs @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! End-to-end assertions for native vLLM KV events through sidecar. + +use std::sync::Arc; +use std::time::Duration; + +use dynamo_backend_common::{ + GenerateContext, KvEventSource, LLMEngine, PreprocessedRequest, StopConditions, +}; +use dynamo_kv_router::indexer::{KvIndexerInterface, KvIndexerMetrics, LocalKvIndexer}; +use dynamo_kv_router::protocols::{KV_EVENT_SUBJECT, KvCacheEventData, RouterEvent}; +use dynamo_llm::kv_router::publisher::{KvEventPublisher, KvEventSourceConfig}; +use dynamo_runtime::distributed::DistributedConfig; +use dynamo_runtime::transports::event_plane::EventSubscriber; +use dynamo_runtime::{DistributedRuntime, Runtime}; +use futures::StreamExt; + +async fn generate(engine: &impl LLMEngine, tokens: Vec) { + let request = PreprocessedRequest::builder() + .model("mocker-model".to_string()) + .token_ids(tokens) + .sampling_options(Default::default()) + .output_options(Default::default()) + .stop_conditions(StopConditions { + max_tokens: Some(1), + ignore_eos: Some(true), + ..Default::default() + }) + .build() + .unwrap(); + let context = dynamo_backend_common::testing::mock_context(); + let mut output = engine + .generate(request, GenerateContext::new(context, None)) + .await + .unwrap(); + while let Some(output) = output.next().await { + output.unwrap(); + } +} + +pub async fn check_kv_events(engine: &impl LLMEngine) { + let sources = engine.kv_event_sources().await.unwrap(); + assert_eq!(sources.len(), 1); + let KvEventSource::Zmq { + endpoint: source_endpoint, + topic, + dp_rank, + } = &sources[0] + else { + panic!("expected a native ZMQ source"); + }; + assert_eq!(*dp_rank, 0); + assert!(topic.is_empty()); + assert!(source_endpoint.starts_with("tcp://127.0.0.1:")); + assert_ne!(source_endpoint.rsplit(':').next(), Some("0")); + + let drt = DistributedRuntime::new( + Runtime::from_current().unwrap(), + DistributedConfig::process_local(), + ) + .await + .unwrap(); + let endpoint = drt + .namespace("mocker-kv-test") + .unwrap() + .component("worker") + .unwrap() + .endpoint("generate"); + let mut subscriber = EventSubscriber::for_endpoint(&endpoint, KV_EVENT_SUBJECT) + .await + .unwrap() + .typed::>(); + let mut relay = KvEventPublisher::new_with_local_indexer( + endpoint, + 4, + Some(KvEventSourceConfig::Zmq { + endpoint: source_endpoint.clone(), + topic: topic.clone(), + image_token_id: None, + video_token_id: None, + }), + false, + *dp_rank, + None, + ) + .unwrap(); + + // Exercise both real ZMQ subscription handshakes. Distinct warmup prompts + // produce fresh stores even if an earlier event was dropped before connect. + tokio::time::timeout(Duration::from_secs(10), async { + for n in 100.. { + generate(engine, vec![n; 4]).await; + if let Ok(Some(batch)) = + tokio::time::timeout(Duration::from_millis(50), subscriber.next()).await + { + batch.unwrap(); + break; + } + } + }) + .await + .expect("KV relay did not become ready"); + + let cancel = dynamo_runtime::CancellationToken::new(); + let indexer = LocalKvIndexer::new( + cancel.clone(), + 4, + Arc::new(KvIndexerMetrics::new_unregistered()), + 100, + ); + let prompt = vec![11, 22, 33, 44, 55, 66, 77, 88]; + generate(engine, prompt.clone()).await; + let mut last_event_id = None; + let mut stored_hashes = Vec::new(); + + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let (_, events) = subscriber.next().await.unwrap().unwrap(); + for event in events { + assert_eq!(event.event.dp_rank, 0); + if let Some(last) = last_event_id { + assert!(event.event.event_id > last); + } + last_event_id = Some(event.event.event_id); + if let KvCacheEventData::Stored(data) = &event.event.data { + stored_hashes.extend(data.blocks.iter().map(|block| block.block_hash)); + } + indexer.apply_event_with_buffer(event).await.unwrap(); + } + let scores = indexer + .find_matches_for_request(&prompt, None, None, None) + .await + .unwrap(); + if scores.scores.values().any(|&blocks| blocks == 2) { + break; + } + } + }) + .await + .expect("stored prompt did not reach the router index"); + + // A cache hit must keep the original indexed blocks usable. + generate(engine, prompt.clone()).await; + assert!( + indexer + .find_matches_for_request(&prompt, None, None, None) + .await + .unwrap() + .scores + .values() + .any(|&blocks| blocks == 2) + ); + + // The test engine has eight blocks. Force eviction with unrelated prompts. + for n in 200..212 { + generate(engine, vec![n; 8]).await; + } + tokio::time::timeout(Duration::from_secs(5), async { + let mut removed_stored_block = false; + loop { + let (_, events) = subscriber.next().await.unwrap().unwrap(); + for event in events { + assert_eq!(event.event.dp_rank, 0); + assert!(event.event.event_id > last_event_id.unwrap()); + last_event_id = Some(event.event.event_id); + if let KvCacheEventData::Removed(data) = &event.event.data { + removed_stored_block |= data + .block_hashes + .iter() + .any(|hash| stored_hashes.contains(hash)); + } + indexer.apply_event_with_buffer(event).await.unwrap(); + } + let scores = indexer + .find_matches_for_request(&prompt, None, None, None) + .await + .unwrap(); + if removed_stored_block && scores.scores.values().all(|&blocks| blocks == 0) { + break; + } + } + }) + .await + .expect("eviction did not remove the prompt from the router index"); + relay.shutdown(); + cancel.cancel(); + engine.cleanup().await.unwrap(); +} diff --git a/lib/mocker/servers/vllm/tests/sidecar.rs b/lib/mocker/servers/vllm/tests/sidecar.rs index 830fde1e96e6..2d51937ff047 100644 --- a/lib/mocker/servers/vllm/tests/sidecar.rs +++ b/lib/mocker/servers/vllm/tests/sidecar.rs @@ -32,6 +32,7 @@ impl RunningServer { }, engine_args, ) + .await .unwrap(); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); @@ -246,3 +247,16 @@ async fn dropping_sidecar_stream_cancels_mocker_work() { .await .expect("dropping the gRPC stream should cancel scheduler work promptly"); } + +mod common; + +#[tokio::test] +async fn sidecar_relays_stored_and_evicted_blocks() { + let mut args = fast_engine_args(); + args.num_gpu_blocks = 8; + args.max_num_seqs = Some(1); + let server = RunningServer::start(ServerMode::Aggregated, args).await; + let engine = sidecar(&server.endpoint, DisaggregationMode::Aggregated).await; + engine.start(0).await.unwrap(); + common::check_kv_events(&engine).await; +} diff --git a/lib/mocker/src/services/zmq_events.rs b/lib/mocker/src/services/zmq_events.rs index 015a69f6a256..80f208a09e31 100644 --- a/lib/mocker/src/services/zmq_events.rs +++ b/lib/mocker/src/services/zmq_events.rs @@ -13,7 +13,7 @@ use dynamo_kv_router::protocols::{KvCacheEvent, KvCacheEventData, StorageTier}; use futures::{Sink, SinkExt, StreamExt}; use serde::Serialize; use tmq::{ - Context, Multipart, SocketBuilder, + AsZmqSocket, Context, Multipart, SocketBuilder, publish::{Publish, publish}, router::{Router, router}, }; @@ -56,6 +56,8 @@ enum ZmqRawKvEvent { pub struct ZmqKvEventSink { tx: mpsc::UnboundedSender>, + endpoint: String, + replay_endpoint: Option, } impl ZmqKvEventSink { @@ -65,27 +67,51 @@ impl ZmqKvEventSink { dp_rank: u32, block_size: u32, ) -> Result { - let (tx, mut rx) = mpsc::unbounded_channel::>(); + Self::bind(Some(port), replay_port, dp_rank, block_size).await + } - let endpoint = format!("tcp://0.0.0.0:{port}"); - let pub_socket = bind_pub_socket(&endpoint) - .await - .map_err(|e| anyhow::anyhow!("ZMQ PUB bind to {endpoint} failed: {e}"))?; - tracing::info!("ZmqKvEventSink bound to {endpoint} for dp_rank {dp_rank}"); + /// Bind the publisher, using an OS-assigned port when `port` is absent. + pub async fn bind( + port: Option, + replay_port: Option, + dp_rank: u32, + block_size: u32, + ) -> Result { + let (tx, mut rx) = mpsc::unbounded_channel::>(); - let mut router_socket = if let Some(rp) = replay_port { + // Reserve the explicit replay port before choosing an automatic PUB port. + let (mut router_socket, replay_endpoint) = if let Some(rp) = replay_port { let replay_ep = format!("tcp://0.0.0.0:{rp}"); let sock = bind_router_socket(&replay_ep) .await .map_err(|e| anyhow::anyhow!("ZMQ ROUTER bind to {replay_ep} failed: {e}"))?; + let replay_ep = sock + .get_socket() + .get_last_endpoint()? + .map_err(|_| anyhow::anyhow!("ZMQ replay endpoint is not UTF-8"))?; tracing::info!( "ZmqKvEventSink replay ROUTER bound to {replay_ep} for dp_rank {dp_rank}" ); - Some(sock) + (Some(sock), Some(replay_ep)) } else { - None + (None, None) }; + let endpoint = format!( + "tcp://0.0.0.0:{}", + port.map_or("*".to_string(), |p| p.to_string()) + ); + let pub_socket = bind_pub_socket(&endpoint) + .await + .map_err(|e| anyhow::anyhow!("ZMQ PUB bind to {endpoint} failed: {e}"))?; + let endpoint = pub_socket + .lock() + .await + .get_socket() + .get_last_endpoint()? + .map_err(|_| anyhow::anyhow!("ZMQ PUB endpoint is not UTF-8"))?; + tracing::info!("ZmqKvEventSink bound to {endpoint} for dp_rank {dp_rank}"); + tokio::spawn(async move { let mut seq_num: u64 = 0; let mut ring_buffer: VecDeque<(u64, Bytes)> = VecDeque::new(); @@ -219,7 +245,20 @@ impl ZmqKvEventSink { } }); - Ok(Self { tx }) + Ok(Self { + tx, + endpoint, + replay_endpoint, + }) + } + + /// Bound address for engine discovery; sidecar resolves the wildcard host. + pub fn endpoint(&self) -> &str { + &self.endpoint + } + + pub fn replay_endpoint(&self) -> Option<&str> { + self.replay_endpoint.as_deref() } fn enqueue(&self, events: Vec) -> anyhow::Result<()> { @@ -404,6 +443,88 @@ mod tests { } } + #[tokio::test] + async fn automatic_ports_are_distinct_and_released_on_drop() { + use std::time::Duration; + + let first = ZmqKvEventSink::bind(None, None, 0, 4).await.unwrap(); + let second = ZmqKvEventSink::bind(None, None, 0, 4).await.unwrap(); + let port: u16 = first + .endpoint() + .rsplit_once(':') + .unwrap() + .1 + .parse() + .unwrap(); + assert_ne!(port, 0); + assert_ne!(first.endpoint(), second.endpoint()); + assert_eq!(first.endpoint(), format!("tcp://0.0.0.0:{port}")); + assert!(ZmqKvEventSink::new(port, None, 0, 4).await.is_err()); + drop(first); + let rebound = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Ok(sink) = ZmqKvEventSink::new(port, None, 0, 4).await { + break sink; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("dropping the sink must release its PUB socket"); + assert_eq!(rebound.endpoint(), format!("tcp://0.0.0.0:{port}")); + } + + #[tokio::test] + async fn automatic_publisher_honors_explicit_replay_port() { + use std::time::Duration; + use tmq::dealer::dealer; + + let reserved = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let replay_port = reserved.local_addr().unwrap().port(); + drop(reserved); + let sink = ZmqKvEventSink::bind(None, Some(replay_port), 0, 4) + .await + .unwrap(); + assert_eq!( + sink.replay_endpoint(), + Some(format!("tcp://0.0.0.0:{replay_port}").as_str()) + ); + sink.publish(RawKvEvent { + event: stored_event(), + block_token_ids: Some(vec![vec![1, 2, 3, 4]]), + storage_tier: StorageTier::Device, + }) + .unwrap(); + let ctx = Context::new(); + let mut replay = dealer(&ctx) + .set_linger(0) + .connect(&format!("tcp://127.0.0.1:{replay_port}")) + .unwrap(); + let frames = tokio::time::timeout(Duration::from_secs(5), async { + loop { + replay + .send(Multipart::from(vec![ + Vec::new(), + 0u64.to_be_bytes().to_vec(), + ])) + .await + .unwrap(); + let frames = multipart_message(replay.next().await.unwrap().unwrap()); + assert_eq!(frames.len(), 4); + if frames[2] != (-1i64).to_be_bytes() { + break frames; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("replay must return the published event"); + assert_eq!(frames[2], 0u64.to_be_bytes()); + let batch = dynamo_kv_router::zmq_wire::decode_event_batch(&frames[3]).unwrap(); + assert_eq!(batch.data_parallel_rank, Some(0)); + assert_eq!(batch.events.len(), 1); + } + #[test] fn host_pinned_zmq_stored_event_carries_medium() { let events = convert_to_zmq_events( From 4b1b8c7c213494d36b1811346cdb99c0b491d73a Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 11 Sep 2026 13:24:33 -0700 Subject: [PATCH 2/3] fix(mocker): simplify vLLM KV event setup and validation Signed-off-by: jthomson04 --- lib/mocker/servers/vllm/README.md | 33 ++++++++++--- lib/mocker/servers/vllm/src/main.rs | 2 +- lib/mocker/servers/vllm/src/server.rs | 4 +- lib/mocker/servers/vllm/src/server_tests.rs | 28 +++++++++-- lib/mocker/servers/vllm/tests/common/mod.rs | 51 ++++++++++----------- lib/mocker/servers/vllm/tests/sidecar.rs | 3 +- lib/mocker/src/services/zmq_events.rs | 21 ++++----- 7 files changed, 90 insertions(+), 52 deletions(-) diff --git a/lib/mocker/servers/vllm/README.md b/lib/mocker/servers/vllm/README.md index dba030d62815..6e294eb83c72 100644 --- a/lib/mocker/servers/vllm/README.md +++ b/lib/mocker/servers/vllm/README.md @@ -45,17 +45,36 @@ caching is enabled, except in decode mode. It uses the existing Mocker ZMQ publisher and reports the endpoint through native engine discovery. The sidecar forwards these events to Dynamo's router. -The event publisher binds a free port by default. For a fixed port and an -optional replay port, add these settings to the server command above: +The event publisher binds a free port by default. For this gRPC server, an +unset `zmq_kv_events_port` selects an automatic ZMQ port. The sidecar discovers +the endpoint and replaces its wildcard address with the host from +`--grpc-endpoint`. Use a frontend with `--router-mode kv`. + +Automatic ports work for local processes and containers that share a network +namespace, including containers in one Kubernetes pod. Use a fixed port when +port mappings, a Service, or firewall rules need a known event port. For example: ```bash ---extra-engine-args '{"block_size":4,"zmq_kv_events_port":5557,"zmq_replay_port":5558}' +cargo run -p dynamo-vllm-mocker --bin dynamo-vllm-mocker-server -- \ + --listen 0.0.0.0:50051 \ + --model mocker-model \ + --extra-engine-args '{"speedup_ratio":1000,"block_size":64,"zmq_kv_events_port":5557}' + +cargo run -p dynamo-vllm-sidecar --bin dynamo-vllm-sidecar -- \ + --grpc-endpoint mock-host:50051 ``` -Run the sidecar as shown above and use a frontend with `--router-mode kv`. -Sidecar discovers the event endpoint without an additional setting. When -running across containers or hosts, expose the event port; a fixed port can -make this easier. +Replace `mock-host` with a host reachable from the sidecar. Expose both TCP +ports on that host, preserving the event port number. For Docker port mapping, +use `-p 50051:50051 -p 5557:5557` on the mock-server container. The sidecar will +connect to `mock-host:5557` for events. + +An explicit replay client can use the existing optional replay socket by adding +`"zmq_replay_port":5558` to the engine arguments. The current sidecar receiver +does not consume the advertised replay endpoint. The shared native PUB/SUB path +can lose events before the subscription is ready, and restarting only the +sidecar does not rebuild the index for blocks already in the mock server's +cache. This server uses that existing path without additional recovery. Set `"enable_prefix_caching":false` to disable both prefix caching and KV events. Decode servers do not publish events. As with regular mock workers, diff --git a/lib/mocker/servers/vllm/src/main.rs b/lib/mocker/servers/vllm/src/main.rs index f28a7f44c8e8..bd8f3021b194 100644 --- a/lib/mocker/servers/vllm/src/main.rs +++ b/lib/mocker/servers/vllm/src/main.rs @@ -51,7 +51,7 @@ fn load_engine_args(value: Option<&str>) -> anyhow::Result { .map_err(anyhow::Error::msg) .with_context(|| format!("failed to load --extra-engine-args from {path}"))?, }; - args.normalized().context("invalid Mocker engine arguments") + Ok(args) } #[tokio::main] diff --git a/lib/mocker/servers/vllm/src/server.rs b/lib/mocker/servers/vllm/src/server.rs index c296f01c0ea3..c300c0591349 100644 --- a/lib/mocker/servers/vllm/src/server.rs +++ b/lib/mocker/servers/vllm/src/server.rs @@ -79,7 +79,6 @@ impl VllmMockerService { config: MockerServerConfig, engine_args: MockEngineArgs, ) -> anyhow::Result { - let engine_args = engine_args.normalized()?; anyhow::ensure!( engine_args.engine_type == EngineType::Vllm, "Mocker engine_type must be vllm" @@ -93,6 +92,7 @@ impl VllmMockerService { engine_args.worker_type == WorkerType::Aggregated, "Mocker worker_type must be aggregated; use the server mode for the emulated wire role" ); + let engine_args = engine_args.normalized()?; let max_concurrent_requests = config.max_concurrent_requests; let model_info = pb::ModelInfo { model_id: config.model.clone(), @@ -149,7 +149,7 @@ impl VllmMockerService { engine_args.zmq_kv_events_port, engine_args.zmq_replay_port, DP_RANK, - u32::try_from(engine_args.block_size)?, + server_info.kv_block_size, ) .await { diff --git a/lib/mocker/servers/vllm/src/server_tests.rs b/lib/mocker/servers/vllm/src/server_tests.rs index 8801df829477..e84b298b2ec5 100644 --- a/lib/mocker/servers/vllm/src/server_tests.rs +++ b/lib/mocker/servers/vllm/src/server_tests.rs @@ -12,6 +12,7 @@ use std::collections::BTreeMap; fn admitting_args() -> MockEngineArgs { MockEngineArgs::builder() .block_size(4) + .enable_prefix_caching(false) .num_gpu_blocks(4096) .max_num_seqs(Some(64)) .max_num_batched_tokens(Some(1024)) @@ -157,10 +158,12 @@ fn text_prompts_fail_with_an_actionable_status() { #[tokio::test] async fn service_rejects_non_vllm_or_multi_rank_engines() { - let sglang = MockEngineArgs::builder() + let mut sglang = MockEngineArgs::builder() .engine_type(EngineType::Sglang) .build() .unwrap(); + // Service-specific errors must take priority over general validation. + sglang.num_gpu_blocks = 0; assert!( VllmMockerService::new(MockerServerConfig::default(), sglang) .await @@ -170,7 +173,8 @@ async fn service_rejects_non_vllm_or_multi_rank_engines() { .contains("engine_type") ); - let multi_rank = MockEngineArgs::builder().dp_size(2).build().unwrap(); + let mut multi_rank = MockEngineArgs::builder().dp_size(2).build().unwrap(); + multi_rank.num_gpu_blocks = 0; assert!( VllmMockerService::new(MockerServerConfig::default(), multi_rank) .await @@ -180,10 +184,11 @@ async fn service_rejects_non_vllm_or_multi_rank_engines() { .contains("dp_size") ); - let disaggregated = MockEngineArgs::builder() + let mut disaggregated = MockEngineArgs::builder() .worker_type(WorkerType::Prefill) .build() .unwrap(); + disaggregated.num_gpu_blocks = 0; assert!( VllmMockerService::new(MockerServerConfig::default(), disaggregated) .await @@ -205,6 +210,17 @@ async fn service_rejects_non_vllm_or_multi_rank_engines() { .to_string() .contains("max_concurrent_requests") ); + + let mut invalid = admitting_args(); + invalid.num_gpu_blocks = 0; + assert!( + VllmMockerService::new(MockerServerConfig::default(), invalid) + .await + .err() + .unwrap() + .to_string() + .contains("num_gpu_blocks") + ); } /// Regression: a mocker without RL capabilities could classify an unsupported @@ -212,7 +228,7 @@ async fn service_rejects_non_vllm_or_multi_rank_engines() { /// capability absence; this test catches it at the Control RPC boundary. #[tokio::test] async fn unsupported_rl_control_reports_unimplemented() { - let service = VllmMockerService::new(MockerServerConfig::default(), MockEngineArgs::default()) + let service = VllmMockerService::new(MockerServerConfig::default(), admitting_args()) .await .unwrap(); let server_info = pb::control_server::Control::get_server_info( @@ -240,6 +256,7 @@ async fn unsupported_rl_control_reports_unimplemented() { async fn unary_generate_maps_capacity_rejection_to_resource_exhausted() { let args = MockEngineArgs::builder() .block_size(4) + .enable_prefix_caching(false) .num_gpu_blocks(1) .max_num_seqs(Some(8)) .max_num_batched_tokens(Some(64)) @@ -264,6 +281,7 @@ async fn unary_generate_maps_capacity_rejection_to_resource_exhausted() { async fn concurrent_request_limit_rejects_a_stalled_stream() { let args = MockEngineArgs::builder() .block_size(4) + .enable_prefix_caching(false) .num_gpu_blocks(128) .max_num_seqs(Some(1)) .speedup_ratio(0.01) @@ -394,6 +412,7 @@ async fn streaming_generate_maps_capacity_rejection_to_resource_exhausted() { // the rejection arrives as a later stream item after prompt info. let args = MockEngineArgs::builder() .block_size(4) + .enable_prefix_caching(false) .num_gpu_blocks(1) .max_num_seqs(Some(8)) .max_num_batched_tokens(Some(64)) @@ -512,6 +531,7 @@ async fn kv_event_discovery_follows_regular_mocker_rules() { async fn failed_kv_publisher_is_not_advertised() { let occupied = tokio::net::TcpListener::bind("0.0.0.0:0").await.unwrap(); let mut args = admitting_args(); + args.enable_prefix_caching = true; args.zmq_kv_events_port = Some(occupied.local_addr().unwrap().port()); let service = VllmMockerService::new(MockerServerConfig::default(), args) .await diff --git a/lib/mocker/servers/vllm/tests/common/mod.rs b/lib/mocker/servers/vllm/tests/common/mod.rs index 495736e44eec..358f86e06d01 100644 --- a/lib/mocker/servers/vllm/tests/common/mod.rs +++ b/lib/mocker/servers/vllm/tests/common/mod.rs @@ -10,7 +10,9 @@ use dynamo_backend_common::{ GenerateContext, KvEventSource, LLMEngine, PreprocessedRequest, StopConditions, }; use dynamo_kv_router::indexer::{KvIndexerInterface, KvIndexerMetrics, LocalKvIndexer}; -use dynamo_kv_router::protocols::{KV_EVENT_SUBJECT, KvCacheEventData, RouterEvent}; +use dynamo_kv_router::protocols::{ + KV_EVENT_SUBJECT, KvCacheEventData, RouterEvent, compute_block_hash_for_seq, +}; use dynamo_llm::kv_router::publisher::{KvEventPublisher, KvEventSourceConfig}; use dynamo_runtime::distributed::DistributedConfig; use dynamo_runtime::transports::event_plane::EventSubscriber; @@ -40,7 +42,7 @@ async fn generate(engine: &impl LLMEngine, tokens: Vec) { } } -pub async fn check_kv_events(engine: &impl LLMEngine) { +pub async fn check_kv_events(engine: &impl LLMEngine, block_size: u32) { let sources = engine.kv_event_sources().await.unwrap(); assert_eq!(sources.len(), 1); let KvEventSource::Zmq { @@ -74,7 +76,7 @@ pub async fn check_kv_events(engine: &impl LLMEngine) { .typed::>(); let mut relay = KvEventPublisher::new_with_local_indexer( endpoint, - 4, + block_size, Some(KvEventSourceConfig::Zmq { endpoint: source_endpoint.clone(), topic: topic.clone(), @@ -91,7 +93,7 @@ pub async fn check_kv_events(engine: &impl LLMEngine) { // produce fresh stores even if an earlier event was dropped before connect. tokio::time::timeout(Duration::from_secs(10), async { for n in 100.. { - generate(engine, vec![n; 4]).await; + generate(engine, vec![n; block_size as usize]).await; if let Ok(Some(batch)) = tokio::time::timeout(Duration::from_millis(50), subscriber.next()).await { @@ -106,11 +108,14 @@ pub async fn check_kv_events(engine: &impl LLMEngine) { let cancel = dynamo_runtime::CancellationToken::new(); let indexer = LocalKvIndexer::new( cancel.clone(), - 4, + block_size, Arc::new(KvIndexerMetrics::new_unregistered()), 100, ); let prompt = vec![11, 22, 33, 44, 55, 66, 77, 88]; + let prompt_hashes = compute_block_hash_for_seq(&prompt, block_size, Default::default()); + let expected_blocks = u32::try_from(prompt_hashes.len()).unwrap(); + assert!(expected_blocks > 0); generate(engine, prompt.clone()).await; let mut last_event_id = None; let mut stored_hashes = Vec::new(); @@ -125,7 +130,12 @@ pub async fn check_kv_events(engine: &impl LLMEngine) { } last_event_id = Some(event.event.event_id); if let KvCacheEventData::Stored(data) = &event.event.data { - stored_hashes.extend(data.blocks.iter().map(|block| block.block_hash)); + stored_hashes.extend( + data.blocks + .iter() + .filter(|block| prompt_hashes.contains(&block.tokens_hash)) + .map(|block| block.block_hash), + ); } indexer.apply_event_with_buffer(event).await.unwrap(); } @@ -133,7 +143,12 @@ pub async fn check_kv_events(engine: &impl LLMEngine) { .find_matches_for_request(&prompt, None, None, None) .await .unwrap(); - if scores.scores.values().any(|&blocks| blocks == 2) { + if scores + .scores + .values() + .any(|&blocks| blocks == expected_blocks) + { + assert_eq!(stored_hashes.len(), prompt_hashes.len()); break; } } @@ -141,24 +156,11 @@ pub async fn check_kv_events(engine: &impl LLMEngine) { .await .expect("stored prompt did not reach the router index"); - // A cache hit must keep the original indexed blocks usable. - generate(engine, prompt.clone()).await; - assert!( - indexer - .find_matches_for_request(&prompt, None, None, None) - .await - .unwrap() - .scores - .values() - .any(|&blocks| blocks == 2) - ); - // The test engine has eight blocks. Force eviction with unrelated prompts. for n in 200..212 { - generate(engine, vec![n; 8]).await; + generate(engine, vec![n; 2 * block_size as usize]).await; } tokio::time::timeout(Duration::from_secs(5), async { - let mut removed_stored_block = false; loop { let (_, events) = subscriber.next().await.unwrap().unwrap(); for event in events { @@ -166,10 +168,7 @@ pub async fn check_kv_events(engine: &impl LLMEngine) { assert!(event.event.event_id > last_event_id.unwrap()); last_event_id = Some(event.event.event_id); if let KvCacheEventData::Removed(data) = &event.event.data { - removed_stored_block |= data - .block_hashes - .iter() - .any(|hash| stored_hashes.contains(hash)); + stored_hashes.retain(|hash| !data.block_hashes.contains(hash)); } indexer.apply_event_with_buffer(event).await.unwrap(); } @@ -177,7 +176,7 @@ pub async fn check_kv_events(engine: &impl LLMEngine) { .find_matches_for_request(&prompt, None, None, None) .await .unwrap(); - if removed_stored_block && scores.scores.values().all(|&blocks| blocks == 0) { + if stored_hashes.is_empty() && scores.scores.values().all(|&blocks| blocks == 0) { break; } } diff --git a/lib/mocker/servers/vllm/tests/sidecar.rs b/lib/mocker/servers/vllm/tests/sidecar.rs index 2d51937ff047..0a9e7639486a 100644 --- a/lib/mocker/servers/vllm/tests/sidecar.rs +++ b/lib/mocker/servers/vllm/tests/sidecar.rs @@ -255,8 +255,9 @@ async fn sidecar_relays_stored_and_evicted_blocks() { let mut args = fast_engine_args(); args.num_gpu_blocks = 8; args.max_num_seqs = Some(1); + let block_size = u32::try_from(args.block_size).unwrap(); let server = RunningServer::start(ServerMode::Aggregated, args).await; let engine = sidecar(&server.endpoint, DisaggregationMode::Aggregated).await; engine.start(0).await.unwrap(); - common::check_kv_events(&engine).await; + common::check_kv_events(&engine, block_size).await; } diff --git a/lib/mocker/src/services/zmq_events.rs b/lib/mocker/src/services/zmq_events.rs index 80f208a09e31..e34391fa7cad 100644 --- a/lib/mocker/src/services/zmq_events.rs +++ b/lib/mocker/src/services/zmq_events.rs @@ -475,19 +475,18 @@ mod tests { } #[tokio::test] - async fn automatic_publisher_honors_explicit_replay_port() { + async fn automatic_replay_port_serves_events_and_rejects_a_second_bind() { use std::time::Duration; use tmq::dealer::dealer; - let reserved = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let replay_port = reserved.local_addr().unwrap().port(); - drop(reserved); - let sink = ZmqKvEventSink::bind(None, Some(replay_port), 0, 4) - .await - .unwrap(); - assert_eq!( - sink.replay_endpoint(), - Some(format!("tcp://0.0.0.0:{replay_port}").as_str()) + let sink = ZmqKvEventSink::bind(None, Some(0), 0, 4).await.unwrap(); + let replay_endpoint = sink.replay_endpoint().unwrap(); + let replay_port: u16 = replay_endpoint.rsplit_once(':').unwrap().1.parse().unwrap(); + assert_ne!(replay_port, 0); + assert!( + ZmqKvEventSink::bind(None, Some(replay_port), 0, 4) + .await + .is_err() ); sink.publish(RawKvEvent { event: stored_event(), @@ -498,7 +497,7 @@ mod tests { let ctx = Context::new(); let mut replay = dealer(&ctx) .set_linger(0) - .connect(&format!("tcp://127.0.0.1:{replay_port}")) + .connect(&replay_endpoint.replace("0.0.0.0", "127.0.0.1")) .unwrap(); let frames = tokio::time::timeout(Duration::from_secs(5), async { loop { From 27b9f9cefafbe84ef9f71f3de13a6c3329332a04 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 11 Sep 2026 16:14:14 -0700 Subject: [PATCH 3/3] fix(mocker): preserve synchronous service construction Signed-off-by: jthomson04 --- lib/mocker/servers/vllm/src/main.rs | 3 +- lib/mocker/servers/vllm/src/server.rs | 9 +--- lib/mocker/servers/vllm/src/server_tests.rs | 31 +++----------- lib/mocker/servers/vllm/tests/sidecar.rs | 1 - lib/mocker/src/services/zmq_events.rs | 46 ++++++++++++--------- 5 files changed, 36 insertions(+), 54 deletions(-) diff --git a/lib/mocker/servers/vllm/src/main.rs b/lib/mocker/servers/vllm/src/main.rs index bd8f3021b194..f0388652d5dd 100644 --- a/lib/mocker/servers/vllm/src/main.rs +++ b/lib/mocker/servers/vllm/src/main.rs @@ -73,8 +73,7 @@ async fn main() -> anyhow::Result<()> { max_concurrent_requests: args.max_concurrent_requests, }, engine_args, - ) - .await?; + )?; tracing::info!( listen = %args.listen, diff --git a/lib/mocker/servers/vllm/src/server.rs b/lib/mocker/servers/vllm/src/server.rs index c300c0591349..e0b30b4f5c79 100644 --- a/lib/mocker/servers/vllm/src/server.rs +++ b/lib/mocker/servers/vllm/src/server.rs @@ -75,10 +75,7 @@ pub struct VllmMockerService { } impl VllmMockerService { - pub async fn new( - config: MockerServerConfig, - engine_args: MockEngineArgs, - ) -> anyhow::Result { + pub fn new(config: MockerServerConfig, engine_args: MockEngineArgs) -> anyhow::Result { anyhow::ensure!( engine_args.engine_type == EngineType::Vllm, "Mocker engine_type must be vllm" @@ -150,9 +147,7 @@ impl VllmMockerService { engine_args.zmq_replay_port, DP_RANK, server_info.kv_block_size, - ) - .await - { + ) { Ok(sink) => Some(sink), Err(error) => { tracing::error!(dp_rank = DP_RANK, %error, "Failed to create ZMQ KV event sink"); diff --git a/lib/mocker/servers/vllm/src/server_tests.rs b/lib/mocker/servers/vllm/src/server_tests.rs index e84b298b2ec5..9e13134a5aba 100644 --- a/lib/mocker/servers/vllm/src/server_tests.rs +++ b/lib/mocker/servers/vllm/src/server_tests.rs @@ -166,7 +166,6 @@ async fn service_rejects_non_vllm_or_multi_rank_engines() { sglang.num_gpu_blocks = 0; assert!( VllmMockerService::new(MockerServerConfig::default(), sglang) - .await .err() .unwrap() .to_string() @@ -177,7 +176,6 @@ async fn service_rejects_non_vllm_or_multi_rank_engines() { multi_rank.num_gpu_blocks = 0; assert!( VllmMockerService::new(MockerServerConfig::default(), multi_rank) - .await .err() .unwrap() .to_string() @@ -191,7 +189,6 @@ async fn service_rejects_non_vllm_or_multi_rank_engines() { disaggregated.num_gpu_blocks = 0; assert!( VllmMockerService::new(MockerServerConfig::default(), disaggregated) - .await .err() .unwrap() .to_string() @@ -204,7 +201,6 @@ async fn service_rejects_non_vllm_or_multi_rank_engines() { }; assert!( VllmMockerService::new(disabled, MockEngineArgs::default()) - .await .err() .unwrap() .to_string() @@ -215,7 +211,6 @@ async fn service_rejects_non_vllm_or_multi_rank_engines() { invalid.num_gpu_blocks = 0; assert!( VllmMockerService::new(MockerServerConfig::default(), invalid) - .await .err() .unwrap() .to_string() @@ -228,9 +223,7 @@ async fn service_rejects_non_vllm_or_multi_rank_engines() { /// capability absence; this test catches it at the Control RPC boundary. #[tokio::test] async fn unsupported_rl_control_reports_unimplemented() { - let service = VllmMockerService::new(MockerServerConfig::default(), admitting_args()) - .await - .unwrap(); + let service = VllmMockerService::new(MockerServerConfig::default(), admitting_args()).unwrap(); let server_info = pb::control_server::Control::get_server_info( &service, Request::new(pb::GetServerInfoRequest {}), @@ -263,9 +256,7 @@ async fn unary_generate_maps_capacity_rejection_to_resource_exhausted() { .speedup_ratio(0.0) .build() .unwrap(); - let service = VllmMockerService::new(MockerServerConfig::default(), args) - .await - .unwrap(); + let service = VllmMockerService::new(MockerServerConfig::default(), args).unwrap(); let mut oversized = request("oversized"); oversized.prompt = Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds { ids: vec![1, 2, 3, 4, 5], @@ -294,7 +285,6 @@ async fn concurrent_request_limit_rejects_a_stalled_stream() { }, args, ) - .await .unwrap(); let mut first_request = request("stalled"); first_request.stopping.as_mut().unwrap().max_new_tokens = 100; @@ -363,9 +353,7 @@ fn decode_rejects_a_handoff_missing_the_opacity_sentinel() { #[tokio::test] async fn unary_generate_accumulates_output_and_terminal_metadata() { - let service = VllmMockerService::new(MockerServerConfig::default(), admitting_args()) - .await - .unwrap(); + let service = VllmMockerService::new(MockerServerConfig::default(), admitting_args()).unwrap(); let mut routed_request = Request::new(request("unary")); routed_request.metadata_mut().insert( @@ -419,9 +407,7 @@ async fn streaming_generate_maps_capacity_rejection_to_resource_exhausted() { .speedup_ratio(0.0) .build() .unwrap(); - let service = VllmMockerService::new(MockerServerConfig::default(), args) - .await - .unwrap(); + let service = VllmMockerService::new(MockerServerConfig::default(), args).unwrap(); let mut oversized = request("oversized-stream"); oversized.prompt = Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds { ids: vec![1, 2, 3, 4, 5], @@ -453,9 +439,7 @@ async fn streaming_survives_a_producer_that_outruns_a_stalled_consumer() { // The producer runs instantly while the consumer stalls, so the request // races far past LiveEngine's fixed per-request buffer. The server's pump // must absorb the burst instead of shedding the stream into an INTERNAL. - let service = VllmMockerService::new(MockerServerConfig::default(), admitting_args()) - .await - .unwrap(); + let service = VllmMockerService::new(MockerServerConfig::default(), admitting_args()).unwrap(); let mut bursty = request("bursty"); bursty.stopping.as_mut().unwrap().max_new_tokens = 50; @@ -510,7 +494,6 @@ async fn kv_event_discovery_follows_regular_mocker_rules() { }, args, ) - .await .unwrap(); assert_eq!( pb::control_server::Control::get_kv_event_sources( @@ -533,9 +516,7 @@ async fn failed_kv_publisher_is_not_advertised() { let mut args = admitting_args(); args.enable_prefix_caching = true; args.zmq_kv_events_port = Some(occupied.local_addr().unwrap().port()); - let service = VllmMockerService::new(MockerServerConfig::default(), args) - .await - .unwrap(); + let service = VllmMockerService::new(MockerServerConfig::default(), args).unwrap(); assert_eq!( pb::control_server::Control::get_kv_event_sources( &service, diff --git a/lib/mocker/servers/vllm/tests/sidecar.rs b/lib/mocker/servers/vllm/tests/sidecar.rs index 0a9e7639486a..5da00090f142 100644 --- a/lib/mocker/servers/vllm/tests/sidecar.rs +++ b/lib/mocker/servers/vllm/tests/sidecar.rs @@ -32,7 +32,6 @@ impl RunningServer { }, engine_args, ) - .await .unwrap(); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); diff --git a/lib/mocker/src/services/zmq_events.rs b/lib/mocker/src/services/zmq_events.rs index e34391fa7cad..fb6bfadfa17a 100644 --- a/lib/mocker/src/services/zmq_events.rs +++ b/lib/mocker/src/services/zmq_events.rs @@ -7,7 +7,7 @@ use std::collections::VecDeque; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; -use anyhow::Result; +use anyhow::{Context as _, Result}; use bytes::Bytes; use dynamo_kv_router::protocols::{KvCacheEvent, KvCacheEventData, StorageTier}; use futures::{Sink, SinkExt, StreamExt}; @@ -67,23 +67,25 @@ impl ZmqKvEventSink { dp_rank: u32, block_size: u32, ) -> Result { - Self::bind(Some(port), replay_port, dp_rank, block_size).await + Self::bind(Some(port), replay_port, dp_rank, block_size) } /// Bind the publisher, using an OS-assigned port when `port` is absent. - pub async fn bind( + /// Requires an active Tokio runtime for the background publisher task. + pub fn bind( port: Option, replay_port: Option, dp_rank: u32, block_size: u32, ) -> Result { + let runtime = tokio::runtime::Handle::try_current() + .context("ZmqKvEventSink::bind requires an active Tokio runtime")?; let (tx, mut rx) = mpsc::unbounded_channel::>(); // Reserve the explicit replay port before choosing an automatic PUB port. let (mut router_socket, replay_endpoint) = if let Some(rp) = replay_port { let replay_ep = format!("tcp://0.0.0.0:{rp}"); let sock = bind_router_socket(&replay_ep) - .await .map_err(|e| anyhow::anyhow!("ZMQ ROUTER bind to {replay_ep} failed: {e}"))?; let replay_ep = sock .get_socket() @@ -102,17 +104,15 @@ impl ZmqKvEventSink { port.map_or("*".to_string(), |p| p.to_string()) ); let pub_socket = bind_pub_socket(&endpoint) - .await .map_err(|e| anyhow::anyhow!("ZMQ PUB bind to {endpoint} failed: {e}"))?; let endpoint = pub_socket - .lock() - .await .get_socket() .get_last_endpoint()? .map_err(|_| anyhow::anyhow!("ZMQ PUB endpoint is not UTF-8"))?; tracing::info!("ZmqKvEventSink bound to {endpoint} for dp_rank {dp_rank}"); - tokio::spawn(async move { + let pub_socket: SharedPubSocket = Arc::new(Mutex::new(pub_socket)); + runtime.spawn(async move { let mut seq_num: u64 = 0; let mut ring_buffer: VecDeque<(u64, Bytes)> = VecDeque::new(); @@ -386,13 +386,13 @@ where configure_receive_builder(builder).set_sndtimeo(ZMQ_SNDTIMEOUT_MS) } -async fn bind_pub_socket(endpoint: &str) -> Result { +fn bind_pub_socket(endpoint: &str) -> Result { let ctx = Context::new(); let socket = configure_send_builder(publish(&ctx)).bind(endpoint)?; - Ok(Arc::new(Mutex::new(socket))) + Ok(socket) } -async fn bind_router_socket(endpoint: &str) -> Result { +fn bind_router_socket(endpoint: &str) -> Result { let ctx = Context::new(); let socket = configure_bidirectional_builder(router(&ctx)).bind(endpoint)?; Ok(socket) @@ -443,12 +443,24 @@ mod tests { } } + #[test] + fn bind_requires_an_active_runtime() { + let error = ZmqKvEventSink::bind(None, None, 0, 4) + .err() + .expect("binding without a runtime must fail"); + assert!( + error + .to_string() + .contains("requires an active Tokio runtime") + ); + } + #[tokio::test] async fn automatic_ports_are_distinct_and_released_on_drop() { use std::time::Duration; - let first = ZmqKvEventSink::bind(None, None, 0, 4).await.unwrap(); - let second = ZmqKvEventSink::bind(None, None, 0, 4).await.unwrap(); + let first = ZmqKvEventSink::bind(None, None, 0, 4).unwrap(); + let second = ZmqKvEventSink::bind(None, None, 0, 4).unwrap(); let port: u16 = first .endpoint() .rsplit_once(':') @@ -479,15 +491,11 @@ mod tests { use std::time::Duration; use tmq::dealer::dealer; - let sink = ZmqKvEventSink::bind(None, Some(0), 0, 4).await.unwrap(); + let sink = ZmqKvEventSink::bind(None, Some(0), 0, 4).unwrap(); let replay_endpoint = sink.replay_endpoint().unwrap(); let replay_port: u16 = replay_endpoint.rsplit_once(':').unwrap().1.parse().unwrap(); assert_ne!(replay_port, 0); - assert!( - ZmqKvEventSink::bind(None, Some(replay_port), 0, 4) - .await - .is_err() - ); + assert!(ZmqKvEventSink::bind(None, Some(replay_port), 0, 4).is_err()); sink.publish(RawKvEvent { event: stored_event(), block_token_ids: Some(vec![vec![1, 2, 3, 4]]),