Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion lib/mocker/servers/vllm/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ name = "dynamo-vllm-mocker-server"
path = "src/main.rs"

[dependencies]
dynamo-mocker = { workspace = true }
dynamo-mocker = { workspace = true, features = ["zmq-events"] }
dynamo-vllm-sidecar = { path = "../../../sidecar/vllm" }

anyhow = { workspace = true }
Expand All @@ -34,5 +34,8 @@ tracing-subscriber = { workspace = true }
uuid = { workspace = true }

[dev-dependencies]
dynamo-llm = { path = "../../../llm", default-features = false }
dynamo-kv-router = { workspace = true }
dynamo-runtime = { workspace = true }
dynamo-backend-common = { workspace = true, features = ["testing"] }
tokio-stream = { workspace = true, features = ["net"] }
42 changes: 42 additions & 0 deletions lib/mocker/servers/vllm/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,48 @@ Synthetic output plans are limited to 32,768 tokens. LiveEngine uses a small,
fixed response buffer for each request and cancels slow consumers rather than
turning declared output length into a second admission-control policy.

## KV events

Like regular mock workers, the server publishes KV cache events when prefix
caching is enabled, except in decode mode. It uses the existing Mocker ZMQ
publisher and reports the endpoint through native engine discovery. The
sidecar forwards these events to Dynamo's router.

The event publisher binds a free port by default. For this gRPC server, an
unset `zmq_kv_events_port` selects an automatic ZMQ port. The sidecar discovers
the endpoint and replaces its wildcard address with the host from
`--grpc-endpoint`. Use a frontend with `--router-mode kv`.

Automatic ports work for local processes and containers that share a network
namespace, including containers in one Kubernetes pod. Use a fixed port when
port mappings, a Service, or firewall rules need a known event port. For example:

```bash
cargo run -p dynamo-vllm-mocker --bin dynamo-vllm-mocker-server -- \
--listen 0.0.0.0:50051 \
--model mocker-model \
--extra-engine-args '{"speedup_ratio":1000,"block_size":64,"zmq_kv_events_port":5557}'

cargo run -p dynamo-vllm-sidecar --bin dynamo-vllm-sidecar -- \
--grpc-endpoint mock-host:50051
```

Replace `mock-host` with a host reachable from the sidecar. Expose both TCP
ports on that host, preserving the event port number. For Docker port mapping,
use `-p 50051:50051 -p 5557:5557` on the mock-server container. The sidecar will
connect to `mock-host:5557` for events.

An explicit replay client can use the existing optional replay socket by adding
`"zmq_replay_port":5558` to the engine arguments. The current sidecar receiver
does not consume the advertised replay endpoint. The shared native PUB/SUB path
can lose events before the subscription is ready, and restarting only the
sidecar does not rebuild the index for blocks already in the mock server's
cache. This server uses that existing path without additional recovery.

Set `"enable_prefix_caching":false` to disable both prefix caching and KV
events. Decode servers do not publish events. As with regular mock workers,
publisher setup failures are logged and serving continues without KV events.

## Disaggregated wire-flow

Run separate endpoints for the two emulated vLLM roles:
Expand Down
5 changes: 3 additions & 2 deletions lib/mocker/servers/vllm/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ fn load_engine_args(value: Option<&str>) -> anyhow::Result<MockEngineArgs> {
.map_err(anyhow::Error::msg)
.with_context(|| format!("failed to load --extra-engine-args from {path}"))?,
};
args.normalized().context("invalid Mocker engine arguments")
Ok(args)
}

#[tokio::main]
Expand All @@ -73,7 +73,8 @@ async fn main() -> anyhow::Result<()> {
max_concurrent_requests: args.max_concurrent_requests,
},
engine_args,
)?;
)
.await?;

tracing::info!(
listen = %args.listen,
Expand Down
64 changes: 59 additions & 5 deletions lib/mocker/servers/vllm/src/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@ use std::pin::Pin;
use std::sync::Arc;

use clap::ValueEnum;
use dynamo_mocker::common::protocols::{EngineType, MockEngineArgs, OutputSignal, WorkerType};
use dynamo_mocker::live::{LiveEngine, LiveRequest, stable_request_uuid};
use dynamo_mocker::common::protocols::{
EngineType, KvEventPublishers, MockEngineArgs, OutputSignal, WorkerType,
};
use dynamo_mocker::live::{LiveEngine, LiveEngineConfig, LiveRequest, stable_request_uuid};
use dynamo_mocker::scheduler::MockerMetrics;
use dynamo_mocker::services::zmq_events::ZmqKvEventSink;
use dynamo_vllm_sidecar::proto as pb;
use futures::Stream;
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
Expand Down Expand Up @@ -67,11 +70,15 @@ pub struct VllmMockerService {
model_info: Arc<pb::ModelInfo>,
server_info: Arc<pb::ServerInfo>,
engine: LiveEngine,
kv_event_sources: Arc<Vec<pb::KvEventSource>>,
request_permits: Arc<Semaphore>,
}

impl VllmMockerService {
pub fn new(config: MockerServerConfig, engine_args: MockEngineArgs) -> anyhow::Result<Self> {
pub async fn new(
Comment thread
jthomson04 marked this conversation as resolved.
Resolved
config: MockerServerConfig,
engine_args: MockEngineArgs,
) -> anyhow::Result<Self> {
anyhow::ensure!(
engine_args.engine_type == EngineType::Vllm,
"Mocker engine_type must be vllm"
Expand All @@ -85,6 +92,7 @@ impl VllmMockerService {
engine_args.worker_type == WorkerType::Aggregated,
"Mocker worker_type must be aggregated; use the server mode for the emulated wire role"
);
let engine_args = engine_args.normalized()?;
let max_concurrent_requests = config.max_concurrent_requests;
let model_info = pb::ModelInfo {
model_id: config.model.clone(),
Expand Down Expand Up @@ -134,11 +142,57 @@ impl VllmMockerService {
.unwrap_or_default(),
rl_capabilities: None,
};
// The wire role is separate from the aggregated scheduler used to
// emulate disaggregated requests.
let sink = if engine_args.needs_kv_publisher() && config.mode != ServerMode::Decode {
match ZmqKvEventSink::bind(
engine_args.zmq_kv_events_port,
engine_args.zmq_replay_port,
DP_RANK,
server_info.kv_block_size,
)
.await
{
Ok(sink) => Some(sink),
Err(error) => {
tracing::error!(dp_rank = DP_RANK, %error, "Failed to create ZMQ KV event sink");
None
}
}
} else {
None
};
let kv_event_sources = sink
.as_ref()
.map(|sink| pb::KvEventSource {
transport: "zmq".to_string(),
endpoint: sink.endpoint().to_string(),
topic: String::new(),
data_parallel_rank: Some(DP_RANK),
replay_endpoint: sink.replay_endpoint().unwrap_or_default().to_string(),
encoding: "msgpack".to_string(),
schema_version: 1,
..Default::default()
})
.into_iter()
.collect();
let engine = LiveEngine::start_with_config(
engine_args,
DP_RANK,
LiveEngineConfig {
kv_event_publishers: KvEventPublishers::new(
None,
sink.map(|sink| Arc::new(sink) as _),
),
..Default::default()
},
)?;
Ok(Self {
config: Arc::new(config),
model_info: Arc::new(model_info),
server_info: Arc::new(server_info),
engine: LiveEngine::start(engine_args, DP_RANK)?,
engine,
kv_event_sources: Arc::new(kv_event_sources),
request_permits: Arc::new(Semaphore::new(max_concurrent_requests)),
})
}
Expand Down Expand Up @@ -319,7 +373,7 @@ impl pb::control_server::Control for VllmMockerService {
_request: Request<pb::GetKvEventSourcesRequest>,
) -> Result<Response<pb::GetKvEventSourcesResponse>, Status> {
Ok(Response::new(pb::GetKvEventSourcesResponse {
sources: Vec::new(),
sources: (*self.kv_event_sources).clone(),
}))
}

Expand Down
Loading
Loading