Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ sysinfo = { version = "0.39.6", default-features = false, features = ["system"]
tempfile = "3"
thiserror = { version = "2.0.17" }
tmq = { version = "0.5.0" }
zmq = { version = "0.10" }
tokio = { version = "=1.54.0", features = ["full"] }
tokio-stream = { version = "0.1" }
tokio-util = { version = "0.7.19", features = ["codec", "net", "rt", "io-util"] }
Expand Down
1 change: 1 addition & 0 deletions lib/backend-common/src/publisher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ fn setup_kv_publishers(
video_token_id: None,
expected_rank: Some(dp_rank),
liveness: heartbeat_timeout.map(|timeout| (dp_rank, timeout)),
recovery: None,
}),
None,
),
Expand Down
1 change: 1 addition & 0 deletions lib/bindings/kvbm/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions lib/bindings/python/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions lib/bindings/python/rust/llm/kv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1333,6 +1333,7 @@ impl KvEventPublisher {
topic: zmq_topic.unwrap_or_default(),
image_token_id,
video_token_id,
recovery: None,
});

if kv_block_size == 0 {
Expand Down
2 changes: 1 addition & 1 deletion lib/kv-router/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ thiserror = { workspace = true }
tokio = { workspace = true }
tokio-util = { workspace = true }
tracing = { workspace = true }
zmq = { version = "0.10", optional = true }
zmq = { workspace = true, optional = true }
uuid = { workspace = true }
xxhash-rust = { workspace = true }
zeroize = { workspace = true }
Expand Down
9 changes: 9 additions & 0 deletions lib/kv-router/src/indexer/local.rs
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,15 @@ impl LocalKvIndexer {
drop(unusable_snapshot);
}

/// Wait until every admitted event has been applied on every tier; event errors stay logged.
pub async fn flush_and_wait(&self) -> Result<(), KvRouterError> {
self.indexer.flush_and_wait().await?;
for indexer in self.all_lower_tier_indexers() {
indexer.flush_and_wait().await?;
}
Ok(())
}

/// Apply event with buffering.
///
/// Stored and Removed events are recorded after successful queue admission; this does not
Expand Down
1 change: 1 addition & 0 deletions lib/llm/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ validator = { workspace = true }
url = { workspace = true }
uuid = { workspace = true }
xxhash-rust = { workspace = true }
zmq = { workspace = true }
modelexpress-client = { workspace = true }
modelexpress-common = { workspace = true }

Expand Down
6 changes: 6 additions & 0 deletions lib/llm/src/kv_router/publisher/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@ KV routing events are advisory:
- Outbound IDs remain monotonic and are never reused.
- Do not add per-event or per-chunk acknowledgements or publication retries.

Sources that opt into replay recovery (`zmq_recovery.rs`) are the exception:
DEP #14897 requires a verified local index before registration, so recovery
sends one barrier before `Success` and waits until the index has finished every
earlier batch. There are no per-batch acknowledgements, and bad events are
still handled as on the live path.

`Cleared` is different: it is an ordering barrier and must complete across every
affected physical tier before publication.

Expand Down
4 changes: 4 additions & 0 deletions lib/llm/src/kv_router/publisher/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ processor publishes normalized events through the configured event plane.

- `zmq_listener.rs`: receives engine ZMQ batches and converts wire events into
placement-aware events.
- `zmq_recovery.rs`: opt-in replay recovery that catches a ZMQ source up from
the engine's replay buffer and reports whether the local index is complete.
A completed handshake does not prove the live subscription is active; a batch
lost in that window is repaired at the next live gap (see `ZmqRecoveryConfig`).
- `event_processor.rs`: owns the async receive loop and preserves event ordering.
- `batching.rs`: coalesces adjacent store/remove events before publish.
- `dedup.rs`: gates duplicate removes with per-rank, per-tier refcounts.
Expand Down
39 changes: 37 additions & 2 deletions lib/llm/src/kv_router/publisher/event_processor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ pub(super) async fn run_event_processor_loop<P: RouterEventBatchSink + 'static>(
publisher: P,
worker_id: u64,
cancellation_token: CancellationToken,
mut rx: mpsc::UnboundedReceiver<Vec<PlacementEvent>>,
mut rx: mpsc::UnboundedReceiver<super::PublisherInput>,
local_indexer: Option<Arc<LocalKvIndexer>>,
timeout_ms: Option<u64>,
max_batch_blocks: usize,
Expand All @@ -47,6 +47,34 @@ pub(super) async fn run_event_processor_loop<P: RouterEventBatchSink + 'static>(
publish_output(&publisher, worker_id, &output).await;
break;
};
let event_batch = match event_batch {
super::PublisherInput::Events(events) => events,
super::PublisherInput::Barrier(done) => {
let mut output = Vec::new();
batching_state.flush(&local_indexer, worker_id, &mut dedup, &mut output).await;
// Recovery applies events like live ingestion, then waits for the index to finish them.
let result = tokio::select! {
biased;
_ = cancellation_token.cancelled() => Err(anyhow::anyhow!("KV event publisher cancelled during recovery")),
result = flush_local_indexer(&local_indexer) => result,
};
if let Err(error) = &result {
if cancellation_token.is_cancelled() {
tracing::debug!(%error, "KV recovery barrier cancelled");
} else {
tracing::error!(%error, "KV recovery barrier failed; stopping KV event publisher");
}
}
let has_failed = result.is_err();
let _ = done.send(result);
if has_failed {
// Keep the shared token alive so the listener can report the failure.
break;
}
publish_output(&publisher, worker_id, &output).await;
continue;
}
};
let mut output = Vec::new();

// Process the complete source list before returning to `select!` so
Expand Down Expand Up @@ -166,6 +194,13 @@ pub(super) async fn run_event_processor_loop<P: RouterEventBatchSink + 'static>(
}
}

async fn flush_local_indexer(local_indexer: &Option<Arc<LocalKvIndexer>>) -> anyhow::Result<()> {
Comment thread
jthomson04 marked this conversation as resolved.
Resolved
let indexer = local_indexer
.as_ref()
.ok_or_else(|| anyhow::anyhow!("KV recovery requires a local indexer"))?;
Ok(indexer.flush_and_wait().await?)
}

async fn publish_output<P: RouterEventBatchSink>(
publisher: &P,
worker_id: u64,
Expand All @@ -188,7 +223,7 @@ pub(super) async fn start_event_processor<P: RouterEventBatchSink + 'static>(
publisher: P,
worker_id: u64,
cancellation_token: CancellationToken,
rx: mpsc::UnboundedReceiver<Vec<PlacementEvent>>,
rx: mpsc::UnboundedReceiver<super::PublisherInput>,
local_indexer: Option<Arc<LocalKvIndexer>>,
batching_timeout_ms: Option<u64>,
) {
Expand Down
Loading
Loading