Skip to content

Commit 7096a57

Browse files
jthomson04claude
andcommitted
feat(mocker): opt-in KV state-agent (KVCR) mode
Add `--kv-state-agent` so a GPU-free deployment exercises the full KVCR router path: frontend KV router + `python -m dynamo.kv_state_agent` + mocker workers. - Each worker DP rank attaches its raw ZMQ stream to the namespace's state agent through a KvStateAttachmentOwner, with a unique CacheOwner per worker x rank, and advertises `kv_event_source_mode = state_agent_v2`. The in-process legacy relay is skipped so events are published once. - Host-pinned (G2) events are tagged `ownership: "kvcr"`; G1 events stay framework-owned. - The ZMQ sink now recovers G2 token IDs from the blocks' earlier G1 stores. AISimulate's G2 residency carries none, so ZMQ mode previously dropped every G2 event and no lower tier reached the router. - `--kv-state-agent-router-hints` advertises router-hint capability with placeholder source endpoints so the router runs its hinted lower-tier lookup; the mocker logs and ignores the hints. - The ZMQ PUB socket uses vLLM's send high-water mark (100k messages) instead of libzmq's default of 1000, so a lagging subscriber sees vLLM-like buffering. - examples/backends/mocker/launch/agg_kvcr_state_agent.sh launches the state agent, mockers and frontend. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: jthomson04 <jwillthomson19@gmail.com>
1 parent 2db5333 commit 7096a57

10 files changed

Lines changed: 518 additions & 32 deletions

File tree

‎components/src/dynamo/mocker/args.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -560,6 +560,23 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
560560
"Each worker's DP ranks bind on base_port + dp_rank. "
561561
"Used alongside --zmq-kv-events-ports for gap recovery. (default: None, disabled)",
562562
)
563+
parser.add_argument(
564+
"--kv-state-agent",
565+
action="store_true",
566+
default=False,
567+
help="Publish KV events through an external KV state agent "
568+
"(python -m dynamo.kv_state_agent) instead of an in-process relay. Host-pinned "
569+
"events are tagged KVCR-owned, so the router indexes them under a CacheOwner per "
570+
"worker DP rank. Requires --zmq-kv-events-ports and --num-host-blocks.",
571+
)
572+
parser.add_argument(
573+
"--kv-state-agent-router-hints",
574+
action="store_true",
575+
default=False,
576+
help="With --kv-state-agent, advertise router-hint capability and a placeholder "
577+
"source control endpoint per CacheOwner so the router runs its hinted lower-tier "
578+
"lookup. The mocker ignores the hints it receives.",
579+
)
563580
parser.add_argument(
564581
"--bootstrap-ports",
565582
type=str,
@@ -733,6 +750,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
733750
f"got {len(args.zmq_replay_ports_list)}: {args.zmq_replay_ports_list}"
734751
)
735752

753+
if args.kv_state_agent:
754+
if not args.zmq_kv_events_ports_list:
755+
raise ValueError("--kv-state-agent requires --zmq-kv-events-ports")
756+
# An --extra-engine-args file carries its own native_host_offload.
757+
if args.num_host_blocks is None and args.extra_engine_args is None:
758+
raise ValueError("--kv-state-agent requires --num-host-blocks")
759+
elif args.kv_state_agent_router_hints:
760+
raise ValueError("--kv-state-agent-router-hints requires --kv-state-agent")
761+
736762
# Set endpoint default based on worker type if not explicitly provided
737763
if args.endpoint is None:
738764
if args.is_prefill_worker:

‎components/src/dynamo/mocker/config.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,13 @@
1515
from dynamo.common.utils.topology import apply_topology_config
1616
from dynamo.llm import ModelRuntimeConfig
1717

18+
# Fixed indexer domain and pool for mocker CacheOwners, in the canonical
19+
# `domain/pool/owner` format (see examples/backends/vllm/deploy/kvcr/agg.yaml).
20+
_KV_STATE_AGENT_INDEXER_DOMAIN = (
21+
"01010101010101010101010101010101:02020202020202020202020202020202"
22+
)
23+
_KV_STATE_AGENT_POOL = "0303030303030303"
24+
1825

1926
def normalize_mocker_config(config: Mapping[str, Any] | str | None = None) -> dict:
2027
from dynamo._core import _normalize_mocker_config
@@ -195,13 +202,15 @@ def apply_worker_engine_args_overrides(
195202
bootstrap_port: int | None = None,
196203
zmq_kv_events_port: int | None = None,
197204
zmq_replay_port: int | None = None,
205+
kv_state_agent: bool | None = None,
198206
ais_mtp_seed: int | None = None,
199207
) -> dict:
200208
config = copy.deepcopy(dict(engine_args))
201209
for key, value in (
202210
("bootstrap_port", bootstrap_port),
203211
("zmq_kv_events_port", zmq_kv_events_port),
204212
("zmq_replay_port", zmq_replay_port),
213+
("kv_state_agent", kv_state_agent),
205214
):
206215
if value is not None:
207216
config.setdefault("dynamo", {})[key] = value
@@ -237,3 +246,46 @@ def build_runtime_config(
237246
rc.set_disaggregated_endpoint(bootstrap_host=host, bootstrap_port=port)
238247
apply_topology_config(rc)
239248
return rank["block_size"], rc
249+
250+
251+
def build_kv_state_agent_descriptors(
252+
engine_args: Mapping[str, Any],
253+
*,
254+
worker_id: int,
255+
kv_state_endpoint: str,
256+
zmq_kv_events_port: int,
257+
router_hint_worker_type: str | None = None,
258+
) -> list[dict]:
259+
"""KV state-agent attachment descriptors for one mocker worker, one per DP rank.
260+
261+
The worker ID makes each CacheOwner unique across mocker processes. With
262+
``router_hint_worker_type``, each owner advertises a placeholder source
263+
control endpoint: the router builds hints for it, and the mocker ignores them.
264+
"""
265+
descriptors = []
266+
for rank in range(engine_args["dp_size"]):
267+
descriptors.append(
268+
{
269+
"cache_owner_id": (
270+
f"{_KV_STATE_AGENT_INDEXER_DOMAIN}/{_KV_STATE_AGENT_POOL}/"
271+
f"{worker_id:016x}{rank:016x}"
272+
),
273+
"global_dp_rank": rank,
274+
"kv_state_endpoint": kv_state_endpoint,
275+
"indexer_domain_id": _KV_STATE_AGENT_INDEXER_DOMAIN,
276+
"kv_block_size": engine_args["engine"]["block_size"],
277+
"raw_zmq_endpoint": f"tcp://127.0.0.1:{zmq_kv_events_port + rank}",
278+
"raw_topic": "",
279+
"router_hint_source": (
280+
{
281+
"source_control_endpoint": (
282+
f"tcp://mocker-{worker_id:x}-{rank}.invalid:1"
283+
),
284+
"worker_type": router_hint_worker_type,
285+
}
286+
if router_hint_worker_type is not None
287+
else None
288+
),
289+
}
290+
)
291+
return descriptors

‎components/src/dynamo/mocker/main.py‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import argparse
77
import asyncio
88
import functools
9+
import json
910
import logging
1011
import os
1112
import signal
@@ -16,13 +17,25 @@
1617
os.environ.setdefault("DYN_COMPUTE_THREADS", "0")
1718

1819
from dynamo.common.configuration.groups.router_args import build_router_config
20+
from dynamo.common.constants import (
21+
KV_HINT_TRANSFER_CAPABILITY_KEY,
22+
KV_HINT_TRANSFER_WORKER_TYPE_RUNTIME_KEY,
23+
)
1924
from dynamo.common.utils.runtime import create_runtime
20-
from dynamo.llm import EngineType, EntrypointArgs, fetch_model, make_engine, run_input
25+
from dynamo.llm import (
26+
EngineType,
27+
EntrypointArgs,
28+
KvStateAttachmentOwner,
29+
fetch_model,
30+
make_engine,
31+
run_input,
32+
)
2133
from dynamo.runtime.logging import configure_dynamo_logging
2234

2335
from .args import parse_args, resolve_planner_profile_data
2436
from .config import (
2537
apply_worker_engine_args_overrides,
38+
build_kv_state_agent_descriptors,
2639
build_runtime_config,
2740
load_mocker_engine_args,
2841
)
@@ -121,6 +134,47 @@ async def worker():
121134
del profile_data_result # Triggers tmpdir cleanup via __del__
122135

123136

137+
async def start_kv_state_agent_owner(
138+
args: argparse.Namespace,
139+
runtime,
140+
engine_args: dict,
141+
runtime_config,
142+
) -> KvStateAttachmentOwner:
143+
"""Attach one worker's raw ZMQ streams to the namespace's KV state agent."""
144+
runtime_config.kv_event_source_mode = "state_agent_v2"
145+
hint_worker_type = None
146+
if args.kv_state_agent_router_hints:
147+
hint_worker_type = (
148+
"prefill"
149+
if args.is_prefill_worker
150+
else "decode"
151+
if args.is_decode_worker
152+
else "aggregated"
153+
)
154+
# Source endpoints ride in the descriptors, as for vLLM state-agent workers.
155+
runtime_config.set_engine_specific(
156+
KV_HINT_TRANSFER_WORKER_TYPE_RUNTIME_KEY, json.dumps(hint_worker_type)
157+
)
158+
runtime_config.set_engine_specific(
159+
KV_HINT_TRANSFER_CAPABILITY_KEY, json.dumps(True)
160+
)
161+
endpoint = runtime.endpoint(args.endpoint)
162+
worker_id = endpoint.connection_id()
163+
owner = KvStateAttachmentOwner(
164+
endpoint,
165+
worker_id,
166+
build_kv_state_agent_descriptors(
167+
engine_args,
168+
worker_id=worker_id,
169+
kv_state_endpoint=args.endpoint,
170+
zmq_kv_events_port=engine_args["dynamo"]["zmq_kv_events_port"],
171+
router_hint_worker_type=hint_worker_type,
172+
),
173+
)
174+
await owner.start()
175+
return owner
176+
177+
124178
def compute_stagger_delay(num_workers: int, stagger_delay: float) -> float:
125179
"""Compute the stagger delay based on worker count to give the frontend time to process registrations.
126180
Returns the delay in seconds between worker launches.
@@ -154,6 +208,9 @@ async def launch_workers(args: argparse.Namespace, base_engine_args):
154208
"""
155209
futures = []
156210
runtimes = []
211+
# Discovery leases own the attachment intents; holding the owners keeps
212+
# their handles alive until the runtimes shut down.
213+
attachment_owners = []
157214
shutdown_requested = False
158215

159216
def on_worker_done(worker_num: int, future: asyncio.Future) -> None:
@@ -231,6 +288,7 @@ def on_worker_done(worker_num: int, future: asyncio.Future) -> None:
231288
if args.zmq_replay_ports_list
232289
else None
233290
),
291+
kv_state_agent=args.kv_state_agent or None,
234292
ais_mtp_seed=(
235293
(base_engine_args["engine"]["aic_mtp_seed"] + worker_id) % (1 << 64)
236294
if base_engine_args["engine"]["aic_nextn"] is not None
@@ -243,6 +301,12 @@ def on_worker_done(worker_num: int, future: asyncio.Future) -> None:
243301
kv_cache_block_size, runtime_config = build_runtime_config(worker_engine_args)
244302
if args.sglang_generate:
245303
runtime_config.set_engine_specific("sglang_generate", "true")
304+
if args.kv_state_agent:
305+
attachment_owners.append(
306+
await start_kv_state_agent_owner(
307+
args, runtime, worker_engine_args, runtime_config
308+
)
309+
)
246310

247311
# Create EntrypointArgs for this worker
248312
entrypoint_args = EntrypointArgs(

‎components/src/dynamo/mocker/tests/unit/test_config.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -423,3 +423,62 @@ async def launch_workers(_args, config):
423423
assert engine["kv_cache_bytes_per_token"] == 2048
424424
assert engine["kv_transfer_bytes_per_token"] == 2048
425425
assert engine["native_host_offload"]["num_host_blocks"] == 16
426+
427+
428+
@pytest.mark.parametrize(
429+
"flags, error",
430+
[
431+
(["--num-host-blocks", "8"], "requires --zmq-kv-events-ports"),
432+
(["--zmq-kv-events-ports", "5557"], "requires --num-host-blocks"),
433+
],
434+
)
435+
def test_kv_state_agent_requires_zmq_ports_and_host_blocks(flags, error):
436+
with pytest.raises(ValueError, match=error):
437+
parse_args(["--kv-state-agent", *flags])
438+
439+
440+
def test_kv_state_agent_router_hints_require_kv_state_agent():
441+
with pytest.raises(ValueError, match="requires --kv-state-agent"):
442+
parse_args(["--kv-state-agent-router-hints"])
443+
444+
445+
def test_kv_state_agent_descriptors_cover_each_dp_rank():
446+
args = parse_args(
447+
[
448+
"--kv-state-agent",
449+
"--kv-state-agent-router-hints",
450+
"--zmq-kv-events-ports",
451+
"5557",
452+
"--num-host-blocks",
453+
"8",
454+
"--kv-bytes-per-token",
455+
"1024",
456+
"--data-parallel-size",
457+
"2",
458+
]
459+
)
460+
config = CONFIG.apply_worker_engine_args_overrides(
461+
CONFIG.build_mocker_engine_args(args),
462+
zmq_kv_events_port=5557,
463+
kv_state_agent=True,
464+
)
465+
assert config["dynamo"]["kv_state_agent"] is True
466+
467+
descriptors = CONFIG.build_kv_state_agent_descriptors(
468+
config,
469+
worker_id=0xABC,
470+
kv_state_endpoint=args.endpoint,
471+
zmq_kv_events_port=5557,
472+
router_hint_worker_type="aggregated",
473+
)
474+
assert [d["global_dp_rank"] for d in descriptors] == [0, 1]
475+
assert [d["raw_zmq_endpoint"] for d in descriptors] == [
476+
"tcp://127.0.0.1:5557",
477+
"tcp://127.0.0.1:5558",
478+
]
479+
assert len({d["cache_owner_id"] for d in descriptors}) == 2
480+
for descriptor in descriptors:
481+
assert descriptor["cache_owner_id"].startswith(
482+
descriptor["indexer_domain_id"] + "/"
483+
)
484+
assert descriptor["router_hint_source"]["worker_type"] == "aggregated"
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/bin/bash
2+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# GPU-free KVCR router path: KV router frontend + external KV state agent +
6+
# mocker workers. Each mocker DP rank attaches its raw ZMQ stream to the state
7+
# agent; host-pinned (G2) events are KVCR-owned, so the router indexes them in
8+
# its HostPinned lower tier under one CacheOwner per worker DP rank. With
9+
# ROUTER_HINTS=1 workers advertise router hints, so the router also runs its
10+
# hinted lower-tier lookup. Mockers ignore the hints: this measures router cost,
11+
# not KV transfer benefit.
12+
set -e
13+
trap 'echo Cleaning up...; kill 0' EXIT
14+
15+
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
16+
source "$SCRIPT_DIR/../../../common/launch_utils.sh"
17+
18+
MODEL="${MODEL:-Qwen/Qwen3-0.6B}"
19+
BLOCK_SIZE="${BLOCK_SIZE:-64}"
20+
NUM_WORKERS="${NUM_WORKERS:-2}"
21+
DP_SIZE="${DP_SIZE:-1}"
22+
NUM_HOST_BLOCKS="${NUM_HOST_BLOCKS:-4096}"
23+
ZMQ_BASE_PORT="${ZMQ_BASE_PORT:-20080}"
24+
ROUTER_HINTS="${ROUTER_HINTS:-1}"
25+
STATE_AGENT_SYSTEM_PORT="${STATE_AGENT_SYSTEM_PORT:-9091}"
26+
27+
HTTP_PORT="${DYN_HTTP_PORT:-8000}"
28+
print_launch_banner "Launching KVCR state agent + KV router + mocker" "$MODEL" "$HTTP_PORT"
29+
30+
# 1. One state agent per namespace, sized for every worker DP rank. Workers
31+
# wait up to DYN_KV_STATE_AGENT_HOST_DISCOVERY_TIMEOUT_SECS (30 s) for it.
32+
DYN_COMPONENT=kv_state_agent \
33+
DYN_HEALTH_CHECK_ENABLED=false \
34+
DYN_SYSTEM_PORT="$STATE_AGENT_SYSTEM_PORT" \
35+
python3 -m dynamo.kv_state_agent --max-slots $((NUM_WORKERS * DP_SIZE)) &
36+
37+
for _ in $(seq 1 60); do
38+
curl -sf "http://127.0.0.1:${STATE_AGENT_SYSTEM_PORT}/live" >/dev/null && break
39+
sleep 1
40+
done
41+
42+
# 2. Mocker workers. Each worker's DP ranks publish on base_port + dp_rank.
43+
ZMQ_PORTS=$(seq -s, "$ZMQ_BASE_PORT" "$DP_SIZE" $((ZMQ_BASE_PORT + (NUM_WORKERS - 1) * DP_SIZE)))
44+
HINT_ARGS=()
45+
[ "$ROUTER_HINTS" = "1" ] && HINT_ARGS=(--kv-state-agent-router-hints)
46+
python3 -m dynamo.mocker \
47+
--model-path "$MODEL" \
48+
--block-size "$BLOCK_SIZE" \
49+
--num-workers "$NUM_WORKERS" \
50+
--data-parallel-size "$DP_SIZE" \
51+
--num-host-blocks "$NUM_HOST_BLOCKS" \
52+
--zmq-kv-events-ports "$ZMQ_PORTS" \
53+
--kv-state-agent \
54+
"${HINT_ARGS[@]}" &
55+
56+
# 3. Frontend with KV routing.
57+
python3 -m dynamo.frontend \
58+
--router-mode kv \
59+
--kv-cache-block-size "$BLOCK_SIZE" &
60+
61+
wait_any_exit

‎lib/llm/src/mocker.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -642,9 +642,19 @@ impl MockerExecutionContext {
642642
replay_port,
643643
dp_rank,
644644
args.block_size as u32,
645+
args.runtime.kv_state_agent,
645646
)
646647
.await
647648
{
649+
// The external KV state agent consumes the raw stream;
650+
// a local relay would publish every event twice.
651+
Ok(sink) if args.runtime.kv_state_agent => (
652+
KvEventPublishers::new(
653+
None,
654+
Some(Arc::new(sink) as Arc<dyn RawKvEventSink>),
655+
),
656+
None,
657+
),
648658
Ok(sink) => {
649659
let source_config = Some(KvEventSourceConfig::Zmq {
650660
endpoint: format!("tcp://127.0.0.1:{zmq_port}"),
@@ -824,6 +834,10 @@ impl AsyncEngine<SingleIn<PreprocessedRequest>, ManyOut<Annotated<LLMEngineOutpu
824834
let native_sglang_terminal = native_sglang.is_some();
825835

826836
let dp_rank = self.resolve_dp_rank(&request);
837+
if let Some(kv_hint) = &request.kv_hint {
838+
// Mocker workers have no KV transfer path; hints are observed only.
839+
tracing::debug!(request_id = ctx.id(), ?kv_hint, "Ignoring router KV hint");
840+
}
827841

828842
// A client can pick dp_rank with `x-dynamo-dp-rank` plus an explicit worker.
829843
if dp_rank >= self.engine_args.dp_size {

0 commit comments

Comments
 (0)