Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions components/src/dynamo/mocker/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -257,9 +257,10 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser.add_argument(
"--g1-backend",
choices=["kvbm", "native"],
default="kvbm",
help="G1 manager for the shared vLLM/TRT-LLM scheduler (default: "
"kvbm). Use 'native' for the self-contained physical block-pool model.",
default=None,
help="G1 manager for the shared vLLM/TRT-LLM scheduler. When unset, "
"native is used unless KVBM G2/G3/G4 offloading selects KVBM. Explicit "
"native cannot be combined with KVBM offloading. Ignored for SGLang.",
)
parser.add_argument(
"--enable-chunked-prefill",
Expand Down Expand Up @@ -605,7 +606,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
type=non_negative_int,
default=None,
help="Enable KVBM mock offload with this many per-worker G2 host blocks. "
"Set to 0 to disable.",
"This automatically selects the KVBM G1 backend. Set to 0 to disable.",
)
parser.add_argument(
"--num-g3-blocks",
Expand Down Expand Up @@ -699,6 +700,23 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
)

args = parser.parse_args(argv)

kvbm_offload_enabled = (
(args.num_g2_blocks or 0) > 0
or (args.num_g3_blocks or 0) > 0
or args.enable_g4_storage
)
if (
args.engine_type in ("vllm", "trtllm")
and args.g1_backend == "native"
and kvbm_offload_enabled
):
parser.error(
"--g1-backend native cannot be combined with KVBM G2/G3/G4 "
"offload; omit --g1-backend to select KVBM automatically or set "
"--g1-backend kvbm explicitly"
)

validate_worker_type_args(args)

# Validate num_workers
Expand Down
2 changes: 1 addition & 1 deletion components/src/dynamo/mocker/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ def build_mocker_engine_args(args: argparse.Namespace) -> MockEngineArgs:
args, "max_num_batched_tokens", _DEFAULT_MAX_NUM_BATCHED_TOKENS
),
enable_prefix_caching=getattr(args, "enable_prefix_caching", True),
g1_backend=getattr(args, "g1_backend", "kvbm"),
g1_backend=getattr(args, "g1_backend", None),
enable_chunked_prefill=getattr(args, "enable_chunked_prefill", True),
speedup_ratio=getattr(args, "speedup_ratio", 1.0),
decode_speedup_ratio=getattr(args, "decode_speedup_ratio", 1.0),
Expand Down
83 changes: 82 additions & 1 deletion components/src/dynamo/mocker/tests/unit/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ def make_args(**overrides):
"max_num_seqs": 256,
"max_num_batched_tokens": 8192,
"enable_prefix_caching": True,
"g1_backend": "kvbm",
"g1_backend": None,
"enable_chunked_prefill": True,
"preemption_mode": "lifo",
"speedup_ratio": 1.0,
Expand Down Expand Up @@ -144,6 +144,57 @@ def test_build_mocker_engine_args_trtllm_accepts_native_g1():
assert engine_args.block_size == 32


def test_mocker_defaults_to_native_g1_across_python_entrypoints():
cli_args = parse_args([])
assert cli_args.g1_backend is None
assert CONFIG.build_mocker_engine_args(cli_args).g1_backend == "native"
assert MockEngineArgs().g1_backend == "native"
assert MockEngineArgs.from_json("{}").g1_backend == "native"


@pytest.mark.parametrize(
"offload",
[
{"num_g2_blocks": 8},
{"num_g2_blocks": 8, "num_g3_blocks": 16},
{"num_g2_blocks": 8, "enable_g4_storage": True},
],
)
def test_mocker_offload_automatically_selects_kvbm_g1(offload):
engine_args = MockEngineArgs(**offload)

assert engine_args.g1_backend == "kvbm"


@pytest.mark.parametrize("engine_type", ["vllm", "trtllm"])
def test_mocker_explicit_native_with_offload_is_rejected(engine_type):
with pytest.raises(Exception, match="omit g1_backend"):
MockEngineArgs(
engine_type=engine_type,
g1_backend="native",
num_g2_blocks=8,
)


def test_mocker_explicit_native_accepts_disabled_offload():
engine_args = MockEngineArgs(
g1_backend="native",
num_g2_blocks=0,
num_g3_blocks=0,
)

assert engine_args.g1_backend == "native"


@pytest.mark.parametrize("engine_type", ["VLLM", "TRTLLM"])
def test_mocker_uppercase_json_offload_selects_kvbm_g1(engine_type):
engine_args = MockEngineArgs.from_json(
json.dumps({"engine_type": engine_type, "num_g2_blocks": 8})
)

assert engine_args.g1_backend == "kvbm"


@pytest.mark.parametrize("engine_type", ["vllm", "trtllm"])
def test_build_mocker_engine_args_accepts_native_g1_with_mtp(engine_type):
engine_args = CONFIG.build_mocker_engine_args(
Expand Down Expand Up @@ -425,6 +476,36 @@ def test_mocker_cli_accepts_native_g1_for_trtllm():
assert args.g1_backend == "native"


@pytest.mark.parametrize("engine_type", ["vllm", "trtllm"])
def test_mocker_cli_rejects_explicit_native_g1_with_offload(engine_type, capsys):
with pytest.raises(SystemExit):
parse_args(
[
"--engine-type",
engine_type,
"--g1-backend",
"native",
"--num-g2-blocks",
"8",
]
)

assert "omit --g1-backend" in capsys.readouterr().err


def test_mocker_cli_ignores_explicit_g1_backend_for_sglang():
args = parse_args(
[
"--engine-type",
"sglang",
"--g1-backend",
"native",
]
)

assert args.g1_backend == "native"


def test_mocker_cli_accepts_max_model_len():
args = parse_args(["--max-model-len", "32768"])

Expand Down
13 changes: 12 additions & 1 deletion docs/fern/dynosim/modeling.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,19 @@ For the shared vLLM/TensorRT-LLM core, `g1_backend` selects the GPU-tier block m
- `kvbm` uses the KVBM logical block manager and its lineage-aware inactive pool.
- `native` uses Mocker's self-contained physical block-pool model.

When `g1_backend` is unset, Mocker selects `native` unless `num_g2_blocks`,
`num_g3_blocks`, or `enable_g4_storage` enables a lower tier. Lower-tier configuration
automatically selects `kvbm`. An explicit `g1_backend: native` conflicts with lower-tier
configuration and fails validation; omit `g1_backend` to select KVBM automatically or set it to
`kvbm`.

Both managers model G1 capacity, prefix reuse, request ownership, eviction, and router-visible KV
events. The SGLang core uses its own token-pool and radix-cache implementation.
events. Their event visibility and allocation timing can differ, which can change KV-router
decisions and numeric replay results. Set `g1_backend` explicitly when comparing a new run with a
baseline created under a specific manager.

The SGLang core uses its own token-pool and radix-cache implementation. It ignores `g1_backend` and
does not support KVBM lower-tier offload.

## Timing Sources

Expand Down
1 change: 1 addition & 0 deletions lib/bench/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ router-test-support = [
satf = ["dep:dynamo-data-gen"]
test-support = [
"router-test-support",
"dynamo-mocker/test-support",
"dep:flate2",
"dep:tempfile",
"dep:tracing",
Expand Down
49 changes: 49 additions & 0 deletions lib/bench/tests/mooncake_trace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ use dynamo_mocker::common::protocols::{EngineType, MockEngineArgs, SglangArgs};
use dynamo_mocker::loadgen::{ReplayRequestHashes, SessionTrace, Trace, TurnTrace};
use dynamo_mocker::replay::{
ReplayKvEventVisibility, ReplayTimedKvEvent, ReplayTimedRequest, ReplayWorkerArtifacts,
native_g1_parent_chain_artifact,
};
use mooncake_open_loop::{
MooncakeOperationPayload, OpenLoopConfig, PreparedMooncakeCorpus, prepare_mooncake_corpus,
Expand Down Expand Up @@ -1203,6 +1204,54 @@ async fn mooncake_open_loop_smoke_completes_exact_ids_and_drains() -> anyhow::Re
Ok(())
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn native_g1_parent_chain_replays_across_indexer_variants() -> anyhow::Result<()> {
let artifacts = vec![native_g1_parent_chain_artifact(BLOCK_SIZE as usize)];
let stored = artifacts[0]
.kv_events
.iter()
.filter_map(|event| match &event.event.data {
KvCacheEventData::Stored(stored) => Some(stored),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(stored.len(), 2);
assert_eq!(
stored.iter().map(|event| event.blocks.len()).sum::<usize>(),
3,
"fixture must publish two prefix blocks plus the promoted tail"
);

let variants = [
MooncakeIndexerConfig::radix_tree(),
MooncakeIndexerConfig::nested_map(8, NUM_EVENT_WORKERS),
MooncakeIndexerConfig::concurrent_radix_tree(NUM_EVENT_WORKERS),
MooncakeIndexerConfig::concurrent_radix_tree_compressed(NUM_EVENT_WORKERS),
MooncakeIndexerConfig::branch_sharded_crtc(2, NUM_EVENT_WORKERS, 2),
];

for config in &variants {
let label = config.short_name();
let (scores, metrics) =
collect_prepared_corpus_overlap_scores_with_metrics(config, &artifacts).await?;

assert_eq!(
support::stored_parent_not_found_count(metrics.as_ref()),
0,
"{label} rejected a Native G1 Stored event before its parent was visible"
);
assert_eq!(scores.len(), 1, "{label} should resolve the fixture query");
assert_eq!(scores[0].query_len, 3);
assert_eq!(
scores[0].scores.values().copied().max(),
Some(3),
"{label} should index the complete three-block parent chain"
);
}

Ok(())
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn mooncake_approx_ttl_drain_leaves_indexer_dumps_empty() -> anyhow::Result<()> {
let warning_count = support::warning_counter(&["dynamo_kv_router::indexer", "dynamo_mocker"]);
Expand Down
14 changes: 13 additions & 1 deletion lib/bench/tests/support/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, OnceLock};

use anyhow::Context;
use dynamo_kv_router::indexer::{KvIndexerMetrics, METRIC_WARNING_DUPLICATE_STORE};
use dynamo_kv_router::indexer::{
KvIndexerMetrics, METRIC_EVENT_STORED, METRIC_STATUS_PARENT_NOT_FOUND,
METRIC_WARNING_DUPLICATE_STORE,
};
use tracing::{Event, Level, Subscriber};
use tracing_subscriber::Registry;
use tracing_subscriber::layer::{Context as LayerContext, Layer};
Expand Down Expand Up @@ -76,3 +79,12 @@ pub fn duplicate_store_warning_count(metrics: &KvIndexerMetrics) -> u64 {
.expect("duplicate_store warning metric should exist")
.get()
}

#[allow(dead_code)]
pub fn stored_parent_not_found_count(metrics: &KvIndexerMetrics) -> u64 {
metrics
.kv_cache_events_applied
.get_metric_with_label_values(&[METRIC_EVENT_STORED, METRIC_STATUS_PARENT_NOT_FOUND])
.expect("stored parent-not-found metric should exist")
.get()
}
12 changes: 7 additions & 5 deletions lib/bindings/python/rust/llm/replay.rs
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ impl MockEngineArgs {
#[pymethods]
impl MockEngineArgs {
#[new]
#[pyo3(signature = (engine_type="vllm", num_gpu_blocks=None, block_size=0, max_num_seqs=Some(256), max_num_batched_tokens=Some(8192), enable_prefix_caching=true, enable_chunked_prefill=true, speedup_ratio=1.0, decode_speedup_ratio=1.0, dp_size=1, startup_time=None, worker_type="aggregated", planner_profile_data=None, aic_backend=None, aic_system=None, aic_backend_version=None, aic_tp_size=None, aic_model_path=None, aic_moe_tp_size=None, aic_moe_ep_size=None, aic_attention_dp_size=None, aic_nextn=None, aic_nextn_accept_rates=None, aic_mtp_seed=42, aic_gemm_dtype=None, aic_moe_dtype=None, aic_fmha_dtype=None, aic_kv_cache_dtype=None, aic_comm_dtype=None, gpu_memory_utilization=None, mem_fraction_static=None, free_gpu_memory_fraction=None, enable_local_indexer=false, bootstrap_port=None, handoff_session_timeout_ms=300000, kv_bytes_per_token=None, kv_transfer_bandwidth=None, kv_transfer_timing_mode="full_prompt", reasoning=None, response_replay_trace_path=None, zmq_kv_events_port=None, zmq_replay_port=None, preemption_mode="lifo", router_queue_policy=None, sglang=None, trtllm=None, num_g2_blocks=None, num_g3_blocks=None, offload_batch_size=None, bandwidth_g1_to_g2_gbps=None, bandwidth_g2_to_g1_gbps=None, bandwidth_g2_to_g3_gbps=None, bandwidth_g3_to_g2_gbps=None, enable_g4_storage=false, bandwidth_g2_to_g4_gbps=None, bandwidth_g4_to_g2_gbps=None, max_model_len=None, g1_backend="kvbm"))]
#[pyo3(signature = (engine_type="vllm", num_gpu_blocks=None, block_size=0, max_num_seqs=Some(256), max_num_batched_tokens=Some(8192), enable_prefix_caching=true, enable_chunked_prefill=true, speedup_ratio=1.0, decode_speedup_ratio=1.0, dp_size=1, startup_time=None, worker_type="aggregated", planner_profile_data=None, aic_backend=None, aic_system=None, aic_backend_version=None, aic_tp_size=None, aic_model_path=None, aic_moe_tp_size=None, aic_moe_ep_size=None, aic_attention_dp_size=None, aic_nextn=None, aic_nextn_accept_rates=None, aic_mtp_seed=42, aic_gemm_dtype=None, aic_moe_dtype=None, aic_fmha_dtype=None, aic_kv_cache_dtype=None, aic_comm_dtype=None, gpu_memory_utilization=None, mem_fraction_static=None, free_gpu_memory_fraction=None, enable_local_indexer=false, bootstrap_port=None, handoff_session_timeout_ms=300000, kv_bytes_per_token=None, kv_transfer_bandwidth=None, kv_transfer_timing_mode="full_prompt", reasoning=None, response_replay_trace_path=None, zmq_kv_events_port=None, zmq_replay_port=None, preemption_mode="lifo", router_queue_policy=None, sglang=None, trtllm=None, num_g2_blocks=None, num_g3_blocks=None, offload_batch_size=None, bandwidth_g1_to_g2_gbps=None, bandwidth_g2_to_g1_gbps=None, bandwidth_g2_to_g3_gbps=None, bandwidth_g3_to_g2_gbps=None, enable_g4_storage=false, bandwidth_g2_to_g4_gbps=None, bandwidth_g4_to_g2_gbps=None, max_model_len=None, g1_backend=None))]
Comment thread
dreamtalen marked this conversation as resolved.
Resolved
#[allow(clippy::too_many_arguments)]
fn new(
engine_type: &str,
Expand Down Expand Up @@ -242,12 +242,12 @@ impl MockEngineArgs {
bandwidth_g2_to_g4_gbps: Option<f64>,
bandwidth_g4_to_g2_gbps: Option<f64>,
max_model_len: Option<usize>,
g1_backend: &str,
g1_backend: Option<&str>,
) -> PyResult<Self> {
let engine_type = parse_mocker_engine_type(engine_type)?;
let worker_type = parse_worker_type(worker_type)?;
let preemption_mode = parse_preemption_mode(preemption_mode)?;
let g1_backend = parse_g1_backend(g1_backend)?;
let g1_backend = g1_backend.map(parse_g1_backend).transpose()?;
let kv_transfer_timing_mode = kv_transfer_timing_mode
.parse()
.map_err(|error: String| PyException::new_err(error))?;
Expand All @@ -266,7 +266,6 @@ impl MockEngineArgs {
.max_num_seqs(max_num_seqs)
.max_num_batched_tokens(max_num_batched_tokens)
.enable_prefix_caching(enable_prefix_caching)
.g1_backend(g1_backend)
.enable_chunked_prefill(enable_chunked_prefill)
.speedup_ratio(speedup_ratio)
.decode_speedup_ratio(decode_speedup_ratio)
Expand Down Expand Up @@ -321,6 +320,9 @@ impl MockEngineArgs {
if let Some(num_gpu_blocks) = num_gpu_blocks {
builder = builder.num_gpu_blocks(num_gpu_blocks);
}
if let Some(g1_backend) = g1_backend {
builder = builder.g1_backend(g1_backend);
}

if let Some(npz_path) = planner_profile_data {
let perf_model = PerfModel::from_npz(&npz_path).map_err(|e| {
Expand Down Expand Up @@ -403,7 +405,7 @@ impl MockEngineArgs {

#[getter]
fn g1_backend(&self) -> &'static str {
match self.inner.g1_backend {
match self.inner.resolved_g1_backend() {
RsG1Backend::Kvbm => "kvbm",
RsG1Backend::Native => "native",
}
Expand Down
2 changes: 1 addition & 1 deletion lib/bindings/python/src/dynamo/_core.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -2095,7 +2095,7 @@ class MockEngineArgs:
bandwidth_g2_to_g4_gbps: Optional[float] = None,
bandwidth_g4_to_g2_gbps: Optional[float] = None,
max_model_len: Optional[int] = None,
g1_backend: str = "kvbm",
g1_backend: Optional[str] = None,
) -> None:
...

Expand Down
1 change: 1 addition & 0 deletions lib/mocker/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ profile = []
# Test/benchmark-only deterministic replay plumbing. Production replay never
# enables this feature and keeps the existing random selector behavior.
replay-bench = ["dynamo-kv-router/bench"]
test-support = []
aic-forward-pass = ["dep:aiconfigurator-core"]
zmq-events = ["dep:tmq"]
# Enables G1↔G2 offload simulation via kvbm-engine.
Expand Down
Loading
Loading