Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions components/src/dynamo/mocker/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -560,6 +560,23 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
"Each worker's DP ranks bind on base_port + dp_rank. "
"Used alongside --zmq-kv-events-ports for gap recovery. (default: None, disabled)",
)
parser.add_argument(
"--kv-state-agent",
action="store_true",
default=False,
help="Publish KV events through an external KV state agent "
"(python -m dynamo.kv_state_agent) instead of an in-process relay. Host-pinned "
"events are tagged KVCR-owned, so the router indexes them under a CacheOwner per "
"worker DP rank. Requires --zmq-kv-events-ports and --num-host-blocks.",
)
parser.add_argument(
"--kv-state-agent-router-hints",
action="store_true",
default=False,
help="With --kv-state-agent, advertise router-hint capability and a placeholder "
"source control endpoint per CacheOwner so the router runs its hinted lower-tier "
"lookup. The mocker ignores the hints it receives.",
)
parser.add_argument(
"--bootstrap-ports",
type=str,
Expand Down Expand Up @@ -733,6 +750,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
f"got {len(args.zmq_replay_ports_list)}: {args.zmq_replay_ports_list}"
)

if args.kv_state_agent:
if not args.zmq_kv_events_ports_list:
raise ValueError("--kv-state-agent requires --zmq-kv-events-ports")
# An --extra-engine-args file carries its own native_host_offload.
if args.num_host_blocks is None and args.extra_engine_args is None:
raise ValueError("--kv-state-agent requires --num-host-blocks")
elif args.kv_state_agent_router_hints:
raise ValueError("--kv-state-agent-router-hints requires --kv-state-agent")

# Set endpoint default based on worker type if not explicitly provided
if args.endpoint is None:
if args.is_prefill_worker:
Expand Down
52 changes: 52 additions & 0 deletions components/src/dynamo/mocker/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@
from dynamo.common.utils.topology import apply_topology_config
from dynamo.llm import ModelRuntimeConfig

# Fixed indexer domain and pool for mocker CacheOwners, in the canonical
# `domain/pool/owner` format (see examples/backends/vllm/deploy/kvcr/agg.yaml).
_KV_STATE_AGENT_INDEXER_DOMAIN = (
"01010101010101010101010101010101:02020202020202020202020202020202"
)
_KV_STATE_AGENT_POOL = "0303030303030303"


def normalize_mocker_config(config: Mapping[str, Any] | str | None = None) -> dict:
from dynamo._core import _normalize_mocker_config
Expand Down Expand Up @@ -195,13 +202,15 @@ def apply_worker_engine_args_overrides(
bootstrap_port: int | None = None,
zmq_kv_events_port: int | None = None,
zmq_replay_port: int | None = None,
kv_state_agent: bool | None = None,
ais_mtp_seed: int | None = None,
) -> dict:
config = copy.deepcopy(dict(engine_args))
for key, value in (
("bootstrap_port", bootstrap_port),
("zmq_kv_events_port", zmq_kv_events_port),
("zmq_replay_port", zmq_replay_port),
("kv_state_agent", kv_state_agent),
):
if value is not None:
config.setdefault("dynamo", {})[key] = value
Expand Down Expand Up @@ -237,3 +246,46 @@ def build_runtime_config(
rc.set_disaggregated_endpoint(bootstrap_host=host, bootstrap_port=port)
apply_topology_config(rc)
return rank["block_size"], rc


def build_kv_state_agent_descriptors(
engine_args: Mapping[str, Any],
*,
kv_state_endpoint: str,
zmq_kv_events_port: int,
router_hint_worker_type: str | None = None,
) -> list[dict]:
"""KV state-agent attachment descriptors for one mocker worker, one per DP rank.

Each CacheOwner slot is its rank's ZMQ port: unique on the state agent's
host (raw endpoints are loopback) and stable across mocker restarts, so a
restarted worker reattaches to its durable slot. With
``router_hint_worker_type``, each owner advertises a placeholder source
control endpoint: the router builds hints for it, and the mocker ignores them.
"""
descriptors = []
for rank in range(engine_args["dp_size"]):
port = zmq_kv_events_port + rank
descriptors.append(
{
"cache_owner_id": (
f"{_KV_STATE_AGENT_INDEXER_DOMAIN}/{_KV_STATE_AGENT_POOL}/"
f"{port:032x}"
),
"global_dp_rank": rank,
"kv_state_endpoint": kv_state_endpoint,
"indexer_domain_id": _KV_STATE_AGENT_INDEXER_DOMAIN,
"kv_block_size": engine_args["engine"]["block_size"],
"raw_zmq_endpoint": f"tcp://127.0.0.1:{port}",
"raw_topic": "",
"router_hint_source": (
{
"source_control_endpoint": f"tcp://mocker-{port}.invalid:1",
"worker_type": router_hint_worker_type,
}
if router_hint_worker_type is not None
else None
),
}
)
return descriptors
64 changes: 63 additions & 1 deletion components/src/dynamo/mocker/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import argparse
import asyncio
import functools
import json
import logging
import os
import signal
Expand All @@ -16,13 +17,25 @@
os.environ.setdefault("DYN_COMPUTE_THREADS", "0")

from dynamo.common.configuration.groups.router_args import build_router_config
from dynamo.common.constants import (
KV_HINT_TRANSFER_CAPABILITY_KEY,
KV_HINT_TRANSFER_WORKER_TYPE_RUNTIME_KEY,
)
from dynamo.common.utils.runtime import create_runtime
from dynamo.llm import EngineType, EntrypointArgs, fetch_model, make_engine, run_input
from dynamo.llm import (
EngineType,
EntrypointArgs,
KvStateAttachmentOwner,
fetch_model,
make_engine,
run_input,
)
from dynamo.runtime.logging import configure_dynamo_logging

from .args import parse_args, resolve_planner_profile_data
from .config import (
apply_worker_engine_args_overrides,
build_kv_state_agent_descriptors,
build_runtime_config,
load_mocker_engine_args,
)
Expand Down Expand Up @@ -121,6 +134,45 @@ async def worker():
del profile_data_result # Triggers tmpdir cleanup via __del__


async def start_kv_state_agent_owner(
args: argparse.Namespace,
runtime,
engine_args: dict,
runtime_config,
) -> KvStateAttachmentOwner:
"""Attach one worker's raw ZMQ streams to the namespace's KV state agent."""
runtime_config.kv_event_source_mode = "state_agent_v2"
hint_worker_type = None
if args.kv_state_agent_router_hints:
hint_worker_type = (
"prefill"
if args.is_prefill_worker
else "decode"
if args.is_decode_worker
else "aggregated"
)
# Source endpoints ride in the descriptors, as for vLLM state-agent workers.
runtime_config.set_engine_specific(
KV_HINT_TRANSFER_WORKER_TYPE_RUNTIME_KEY, json.dumps(hint_worker_type)
)
runtime_config.set_engine_specific(
KV_HINT_TRANSFER_CAPABILITY_KEY, json.dumps(True)
)
endpoint = runtime.endpoint(args.endpoint)
owner = KvStateAttachmentOwner(
endpoint,
endpoint.connection_id(),
build_kv_state_agent_descriptors(
engine_args,
kv_state_endpoint=args.endpoint,
zmq_kv_events_port=engine_args["dynamo"]["zmq_kv_events_port"],
router_hint_worker_type=hint_worker_type,
),
)
await owner.start()
return owner


def compute_stagger_delay(num_workers: int, stagger_delay: float) -> float:
"""Compute the stagger delay based on worker count to give the frontend time to process registrations.
Returns the delay in seconds between worker launches.
Expand Down Expand Up @@ -154,6 +206,9 @@ async def launch_workers(args: argparse.Namespace, base_engine_args):
"""
futures = []
runtimes = []
# Discovery leases own the attachment intents; holding the owners keeps
# their handles alive until the runtimes shut down.
attachment_owners = []
shutdown_requested = False

def on_worker_done(worker_num: int, future: asyncio.Future) -> None:
Expand Down Expand Up @@ -231,6 +286,7 @@ def on_worker_done(worker_num: int, future: asyncio.Future) -> None:
if args.zmq_replay_ports_list
else None
),
kv_state_agent=args.kv_state_agent or None,
ais_mtp_seed=(
(base_engine_args["engine"]["aic_mtp_seed"] + worker_id) % (1 << 64)
if base_engine_args["engine"]["aic_nextn"] is not None
Expand All @@ -243,6 +299,12 @@ def on_worker_done(worker_num: int, future: asyncio.Future) -> None:
kv_cache_block_size, runtime_config = build_runtime_config(worker_engine_args)
if args.sglang_generate:
runtime_config.set_engine_specific("sglang_generate", "true")
if worker_engine_args["dynamo"]["kv_state_agent"]:
attachment_owners.append(
await start_kv_state_agent_owner(
args, runtime, worker_engine_args, runtime_config
)
)

# Create EntrypointArgs for this worker
entrypoint_args = EntrypointArgs(
Expand Down
63 changes: 63 additions & 0 deletions components/src/dynamo/mocker/tests/unit/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ def test_profile_is_an_explicit_timing_provider(tmp_path):
{"engine": {"max_model_len": 0}},
{"gpu_memory_utilization": 1.1},
{"gpu_memory_utilization": float("nan")},
{"dynamo": {"kv_state_agent": True}},
],
)
def test_canonical_validation_rejects_bad_input(bad):
Expand Down Expand Up @@ -423,3 +424,65 @@ async def launch_workers(_args, config):
assert engine["kv_cache_bytes_per_token"] == 2048
assert engine["kv_transfer_bytes_per_token"] == 2048
assert engine["native_host_offload"]["num_host_blocks"] == 16


@pytest.mark.parametrize(
"flags, error",
[
(["--num-host-blocks", "8"], "requires --zmq-kv-events-ports"),
(["--zmq-kv-events-ports", "5557"], "requires --num-host-blocks"),
],
)
def test_kv_state_agent_requires_zmq_ports_and_host_blocks(flags, error):
with pytest.raises(ValueError, match=error):
parse_args(["--kv-state-agent", *flags])


def test_kv_state_agent_router_hints_require_kv_state_agent():
with pytest.raises(ValueError, match="requires --kv-state-agent"):
parse_args(["--kv-state-agent-router-hints"])


def test_kv_state_agent_descriptors_cover_each_dp_rank():
args = parse_args(
[
"--kv-state-agent",
"--kv-state-agent-router-hints",
"--zmq-kv-events-ports",
"5557",
"--num-host-blocks",
"8",
"--kv-bytes-per-token",
"1024",
"--data-parallel-size",
"2",
]
)
config = CONFIG.apply_worker_engine_args_overrides(
CONFIG.build_mocker_engine_args(args),
zmq_kv_events_port=5557,
kv_state_agent=True,
)
assert config["dynamo"]["kv_state_agent"] is True

descriptors = CONFIG.build_kv_state_agent_descriptors(
config,
kv_state_endpoint=args.endpoint,
zmq_kv_events_port=5557,
router_hint_worker_type="aggregated",
)
assert [d["global_dp_rank"] for d in descriptors] == [0, 1]
assert [d["raw_zmq_endpoint"] for d in descriptors] == [
"tcp://127.0.0.1:5557",
"tcp://127.0.0.1:5558",
]
# Slots come from the rank's ZMQ port, so they survive worker restarts.
assert [d["cache_owner_id"].rsplit("/", 1)[1] for d in descriptors] == [
f"{5557:032x}",
f"{5558:032x}",
]
for descriptor in descriptors:
assert descriptor["cache_owner_id"].startswith(
descriptor["indexer_domain_id"] + "/"
)
assert descriptor["router_hint_source"]["worker_type"] == "aggregated"
58 changes: 58 additions & 0 deletions examples/backends/mocker/launch/agg_kvcr_state_agent.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# GPU-free KVCR router path: KV router frontend + external KV state agent +
# mocker workers. Each mocker DP rank attaches its raw ZMQ stream to the state
# agent; host-pinned (G2) events are KVCR-owned, so the router indexes them in
# its HostPinned lower tier under one CacheOwner per worker DP rank. With
# ROUTER_HINTS=1 workers advertise router hints, so the router also runs its
# hinted lower-tier lookup. Mockers ignore the hints: this measures router cost,
# not KV transfer benefit.
set -e
trap 'echo Cleaning up...; kill 0' EXIT

SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
source "$SCRIPT_DIR/../../../common/launch_utils.sh"

MODEL="${MODEL:-Qwen/Qwen3-0.6B}"
BLOCK_SIZE="${BLOCK_SIZE:-64}"
NUM_WORKERS="${NUM_WORKERS:-2}"
DP_SIZE="${DP_SIZE:-1}"
NUM_HOST_BLOCKS="${NUM_HOST_BLOCKS:-4096}"
ZMQ_BASE_PORT="${ZMQ_BASE_PORT:-20080}"
ROUTER_HINTS="${ROUTER_HINTS:-1}"
STATE_AGENT_SYSTEM_PORT="${STATE_AGENT_SYSTEM_PORT:-9091}"

HTTP_PORT="${DYN_HTTP_PORT:-8000}"
print_launch_banner "Launching KVCR state agent + KV router + mocker" "$MODEL" "$HTTP_PORT"

# 1. One state agent per namespace, sized for every worker DP rank. Workers
# wait up to DYN_KV_STATE_AGENT_HOST_DISCOVERY_TIMEOUT_SECS (30 s) for it.
DYN_COMPONENT=kv_state_agent \
DYN_HEALTH_CHECK_ENABLED=false \
DYN_SYSTEM_PORT="$STATE_AGENT_SYSTEM_PORT" \
python3 -m dynamo.kv_state_agent --max-slots $((NUM_WORKERS * DP_SIZE)) &

wait_for_ready "http://127.0.0.1:${STATE_AGENT_SYSTEM_PORT}/live" 60

# 2. Mocker workers. Each worker's DP ranks publish on base_port + dp_rank.
ZMQ_PORTS=$(seq -s, "$ZMQ_BASE_PORT" "$DP_SIZE" $((ZMQ_BASE_PORT + (NUM_WORKERS - 1) * DP_SIZE)))
HINT_ARGS=()
[ "$ROUTER_HINTS" = "1" ] && HINT_ARGS=(--kv-state-agent-router-hints)
python3 -m dynamo.mocker \
--model-path "$MODEL" \
--block-size "$BLOCK_SIZE" \
--num-workers "$NUM_WORKERS" \
--data-parallel-size "$DP_SIZE" \
--num-host-blocks "$NUM_HOST_BLOCKS" \
--zmq-kv-events-ports "$ZMQ_PORTS" \
--kv-state-agent \
"${HINT_ARGS[@]}" &

# 3. Frontend with KV routing.
python3 -m dynamo.frontend \
--router-mode kv \
--kv-cache-block-size "$BLOCK_SIZE" &

wait_any_exit
14 changes: 14 additions & 0 deletions lib/llm/src/mocker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -642,9 +642,19 @@ impl MockerExecutionContext {
replay_port,
dp_rank,
args.block_size as u32,
args.runtime.kv_state_agent,
)
.await
{
// The external KV state agent consumes the raw stream;
// a local relay would publish every event twice.
Ok(sink) if args.runtime.kv_state_agent => (
KvEventPublishers::new(
None,
Some(Arc::new(sink) as Arc<dyn RawKvEventSink>),
),
None,
),
Ok(sink) => {
let source_config = Some(KvEventSourceConfig::Zmq {
endpoint: format!("tcp://127.0.0.1:{zmq_port}"),
Expand Down Expand Up @@ -824,6 +834,10 @@ impl AsyncEngine<SingleIn<PreprocessedRequest>, ManyOut<Annotated<LLMEngineOutpu
let native_sglang_terminal = native_sglang.is_some();

let dp_rank = self.resolve_dp_rank(&request);
if let Some(kv_hint) = &request.kv_hint {
// Mocker workers have no KV transfer path; hints are observed only.
tracing::debug!(request_id = ctx.id(), ?kv_hint, "Ignoring router KV hint");
}

// A client can pick dp_rank with `x-dynamo-dp-rank` plus an explicit worker.
if dp_rank >= self.engine_args.dp_size {
Expand Down
Loading
Loading