From 93fb7b278f87efe3a876f5431234ff2b5ab40931 Mon Sep 17 00:00:00 2001 From: Yongming Ding Date: Mon, 27 Jul 2026 14:55:55 -0700 Subject: [PATCH 1/2] feat(mocker): default G1 to native and preserve KV event order Signed-off-by: Yongming Ding --- components/src/dynamo/mocker/args.py | 10 +- components/src/dynamo/mocker/config.py | 2 +- .../dynamo/mocker/tests/unit/test_config.py | 39 ++++- lib/bench/Cargo.toml | 1 + lib/bench/tests/mooncake_trace.rs | 49 ++++++ lib/bench/tests/support/mod.rs | 14 +- lib/bindings/python/rust/llm/replay.rs | 12 +- lib/bindings/python/src/dynamo/_core.pyi | 2 +- lib/mocker/Cargo.toml | 1 + lib/mocker/src/common/protocols.rs | 146 ++++++++++++------ lib/mocker/src/kv_manager/g1_manager.rs | 29 +++- lib/mocker/src/replay/artifacts.rs | 70 +++++++++ lib/mocker/src/replay/mod.rs | 3 + lib/mocker/src/replay/offline/agg.rs | 22 ++- lib/mocker/src/scheduler/vllm/core.rs | 2 +- lib/mocker/src/scheduler/vllm/policy/tests.rs | 2 +- lib/mocker/src/scheduler/vllm/tests.rs | 4 + 17 files changed, 336 insertions(+), 72 deletions(-) diff --git a/components/src/dynamo/mocker/args.py b/components/src/dynamo/mocker/args.py index 7fa805ded50a..b1d7e2293f67 100644 --- a/components/src/dynamo/mocker/args.py +++ b/components/src/dynamo/mocker/args.py @@ -257,9 +257,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument( "--g1-backend", choices=["kvbm", "native"], - default="kvbm", - help="G1 manager for the shared vLLM/TRT-LLM scheduler (default: " - "kvbm). Use 'native' for the self-contained physical block-pool model.", + default=None, + help="G1 manager for the shared vLLM/TRT-LLM scheduler. When unset, " + "native is used unless KVBM G2/G3/G4 offloading selects KVBM. An " + "explicit native selection is also overridden when offloading requires " + "KVBM. Ignored for SGLang.", ) parser.add_argument( "--enable-chunked-prefill", @@ -605,7 +607,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=non_negative_int, default=None, help="Enable KVBM mock offload with this many per-worker G2 host blocks. " - "Set to 0 to disable.", + "This automatically selects the KVBM G1 backend. Set to 0 to disable.", ) parser.add_argument( "--num-g3-blocks", diff --git a/components/src/dynamo/mocker/config.py b/components/src/dynamo/mocker/config.py index 53e805ad90b9..111f50c320d8 100644 --- a/components/src/dynamo/mocker/config.py +++ b/components/src/dynamo/mocker/config.py @@ -288,7 +288,7 @@ def build_mocker_engine_args(args: argparse.Namespace) -> MockEngineArgs: args, "max_num_batched_tokens", _DEFAULT_MAX_NUM_BATCHED_TOKENS ), enable_prefix_caching=getattr(args, "enable_prefix_caching", True), - g1_backend=getattr(args, "g1_backend", "kvbm"), + g1_backend=getattr(args, "g1_backend", None), enable_chunked_prefill=getattr(args, "enable_chunked_prefill", True), speedup_ratio=getattr(args, "speedup_ratio", 1.0), decode_speedup_ratio=getattr(args, "decode_speedup_ratio", 1.0), diff --git a/components/src/dynamo/mocker/tests/unit/test_config.py b/components/src/dynamo/mocker/tests/unit/test_config.py index 4efb7eae4c15..582355f4b96e 100644 --- a/components/src/dynamo/mocker/tests/unit/test_config.py +++ b/components/src/dynamo/mocker/tests/unit/test_config.py @@ -39,7 +39,7 @@ def make_args(**overrides): "max_num_seqs": 256, "max_num_batched_tokens": 8192, "enable_prefix_caching": True, - "g1_backend": "kvbm", + "g1_backend": None, "enable_chunked_prefill": True, "preemption_mode": "lifo", "speedup_ratio": 1.0, @@ -144,6 +144,43 @@ def test_build_mocker_engine_args_trtllm_accepts_native_g1(): assert engine_args.block_size == 32 +def test_mocker_defaults_to_native_g1_across_python_entrypoints(): + cli_args = parse_args([]) + assert cli_args.g1_backend is None + assert CONFIG.build_mocker_engine_args(cli_args).g1_backend == "native" + assert MockEngineArgs().g1_backend == "native" + assert MockEngineArgs.from_json("{}").g1_backend == "native" + + +@pytest.mark.parametrize( + "offload", + [ + {"num_g2_blocks": 8}, + {"num_g2_blocks": 8, "num_g3_blocks": 16}, + {"num_g2_blocks": 8, "enable_g4_storage": True}, + ], +) +def test_mocker_offload_automatically_selects_kvbm_g1(offload): + engine_args = MockEngineArgs(**offload) + + assert engine_args.g1_backend == "kvbm" + + +def test_mocker_explicit_native_with_offload_selects_kvbm_g1(): + engine_args = MockEngineArgs(g1_backend="native", num_g2_blocks=8) + + assert engine_args.g1_backend == "kvbm" + + +@pytest.mark.parametrize("engine_type", ["VLLM", "TRTLLM"]) +def test_mocker_uppercase_json_offload_selects_kvbm_g1(engine_type): + engine_args = MockEngineArgs.from_json( + json.dumps({"engine_type": engine_type, "num_g2_blocks": 8}) + ) + + assert engine_args.g1_backend == "kvbm" + + @pytest.mark.parametrize("engine_type", ["vllm", "trtllm"]) def test_build_mocker_engine_args_accepts_native_g1_with_mtp(engine_type): engine_args = CONFIG.build_mocker_engine_args( diff --git a/lib/bench/Cargo.toml b/lib/bench/Cargo.toml index 215149035f69..d1d6cf69c62e 100644 --- a/lib/bench/Cargo.toml +++ b/lib/bench/Cargo.toml @@ -74,6 +74,7 @@ router-test-support = [ satf = ["dep:dynamo-data-gen"] test-support = [ "router-test-support", + "dynamo-mocker/test-support", "dep:flate2", "dep:tempfile", "dep:tracing", diff --git a/lib/bench/tests/mooncake_trace.rs b/lib/bench/tests/mooncake_trace.rs index 08a666fb933e..f422555f0726 100644 --- a/lib/bench/tests/mooncake_trace.rs +++ b/lib/bench/tests/mooncake_trace.rs @@ -40,6 +40,7 @@ use dynamo_mocker::common::protocols::{EngineType, MockEngineArgs, SglangArgs}; use dynamo_mocker::loadgen::{ReplayRequestHashes, SessionTrace, Trace, TurnTrace}; use dynamo_mocker::replay::{ ReplayKvEventVisibility, ReplayTimedKvEvent, ReplayTimedRequest, ReplayWorkerArtifacts, + native_g1_parent_chain_artifact, }; use mooncake_open_loop::{ MooncakeOperationPayload, OpenLoopConfig, PreparedMooncakeCorpus, prepare_mooncake_corpus, @@ -1203,6 +1204,54 @@ async fn mooncake_open_loop_smoke_completes_exact_ids_and_drains() -> anyhow::Re Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn native_g1_parent_chain_replays_across_indexer_variants() -> anyhow::Result<()> { + let artifacts = vec![native_g1_parent_chain_artifact(BLOCK_SIZE as usize)]; + let stored = artifacts[0] + .kv_events + .iter() + .filter_map(|event| match &event.event.data { + KvCacheEventData::Stored(stored) => Some(stored), + _ => None, + }) + .collect::>(); + assert_eq!(stored.len(), 2); + assert_eq!( + stored.iter().map(|event| event.blocks.len()).sum::(), + 3, + "fixture must publish two prefix blocks plus the promoted tail" + ); + + let variants = [ + MooncakeIndexerConfig::radix_tree(), + MooncakeIndexerConfig::nested_map(8, NUM_EVENT_WORKERS), + MooncakeIndexerConfig::concurrent_radix_tree(NUM_EVENT_WORKERS), + MooncakeIndexerConfig::concurrent_radix_tree_compressed(NUM_EVENT_WORKERS), + MooncakeIndexerConfig::branch_sharded_crtc(2, NUM_EVENT_WORKERS, 2), + ]; + + for config in &variants { + let label = config.short_name(); + let (scores, metrics) = + collect_prepared_corpus_overlap_scores_with_metrics(config, &artifacts).await?; + + assert_eq!( + support::stored_parent_not_found_count(metrics.as_ref()), + 0, + "{label} rejected a Native G1 Stored event before its parent was visible" + ); + assert_eq!(scores.len(), 1, "{label} should resolve the fixture query"); + assert_eq!(scores[0].query_len, 3); + assert_eq!( + scores[0].scores.values().copied().max(), + Some(3), + "{label} should index the complete three-block parent chain" + ); + } + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn mooncake_approx_ttl_drain_leaves_indexer_dumps_empty() -> anyhow::Result<()> { let warning_count = support::warning_counter(&["dynamo_kv_router::indexer", "dynamo_mocker"]); diff --git a/lib/bench/tests/support/mod.rs b/lib/bench/tests/support/mod.rs index 883e59f8682f..93725c28a545 100644 --- a/lib/bench/tests/support/mod.rs +++ b/lib/bench/tests/support/mod.rs @@ -6,7 +6,10 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, OnceLock}; use anyhow::Context; -use dynamo_kv_router::indexer::{KvIndexerMetrics, METRIC_WARNING_DUPLICATE_STORE}; +use dynamo_kv_router::indexer::{ + KvIndexerMetrics, METRIC_EVENT_STORED, METRIC_STATUS_PARENT_NOT_FOUND, + METRIC_WARNING_DUPLICATE_STORE, +}; use tracing::{Event, Level, Subscriber}; use tracing_subscriber::Registry; use tracing_subscriber::layer::{Context as LayerContext, Layer}; @@ -76,3 +79,12 @@ pub fn duplicate_store_warning_count(metrics: &KvIndexerMetrics) -> u64 { .expect("duplicate_store warning metric should exist") .get() } + +#[allow(dead_code)] +pub fn stored_parent_not_found_count(metrics: &KvIndexerMetrics) -> u64 { + metrics + .kv_cache_events_applied + .get_metric_with_label_values(&[METRIC_EVENT_STORED, METRIC_STATUS_PARENT_NOT_FOUND]) + .expect("stored parent-not-found metric should exist") + .get() +} diff --git a/lib/bindings/python/rust/llm/replay.rs b/lib/bindings/python/rust/llm/replay.rs index 2c38b8dd9409..3c4f4fe6f6c6 100644 --- a/lib/bindings/python/rust/llm/replay.rs +++ b/lib/bindings/python/rust/llm/replay.rs @@ -182,7 +182,7 @@ impl MockEngineArgs { #[pymethods] impl MockEngineArgs { #[new] - #[pyo3(signature = (engine_type="vllm", num_gpu_blocks=None, block_size=0, max_num_seqs=Some(256), max_num_batched_tokens=Some(8192), enable_prefix_caching=true, enable_chunked_prefill=true, speedup_ratio=1.0, decode_speedup_ratio=1.0, dp_size=1, startup_time=None, worker_type="aggregated", planner_profile_data=None, aic_backend=None, aic_system=None, aic_backend_version=None, aic_tp_size=None, aic_model_path=None, aic_moe_tp_size=None, aic_moe_ep_size=None, aic_attention_dp_size=None, aic_nextn=None, aic_nextn_accept_rates=None, aic_mtp_seed=42, aic_gemm_dtype=None, aic_moe_dtype=None, aic_fmha_dtype=None, aic_kv_cache_dtype=None, aic_comm_dtype=None, gpu_memory_utilization=None, mem_fraction_static=None, free_gpu_memory_fraction=None, enable_local_indexer=false, bootstrap_port=None, handoff_session_timeout_ms=300000, kv_bytes_per_token=None, kv_transfer_bandwidth=None, kv_transfer_timing_mode="full_prompt", reasoning=None, response_replay_trace_path=None, zmq_kv_events_port=None, zmq_replay_port=None, preemption_mode="lifo", router_queue_policy=None, sglang=None, trtllm=None, num_g2_blocks=None, num_g3_blocks=None, offload_batch_size=None, bandwidth_g1_to_g2_gbps=None, bandwidth_g2_to_g1_gbps=None, bandwidth_g2_to_g3_gbps=None, bandwidth_g3_to_g2_gbps=None, enable_g4_storage=false, bandwidth_g2_to_g4_gbps=None, bandwidth_g4_to_g2_gbps=None, max_model_len=None, g1_backend="kvbm"))] + #[pyo3(signature = (engine_type="vllm", num_gpu_blocks=None, block_size=0, max_num_seqs=Some(256), max_num_batched_tokens=Some(8192), enable_prefix_caching=true, enable_chunked_prefill=true, speedup_ratio=1.0, decode_speedup_ratio=1.0, dp_size=1, startup_time=None, worker_type="aggregated", planner_profile_data=None, aic_backend=None, aic_system=None, aic_backend_version=None, aic_tp_size=None, aic_model_path=None, aic_moe_tp_size=None, aic_moe_ep_size=None, aic_attention_dp_size=None, aic_nextn=None, aic_nextn_accept_rates=None, aic_mtp_seed=42, aic_gemm_dtype=None, aic_moe_dtype=None, aic_fmha_dtype=None, aic_kv_cache_dtype=None, aic_comm_dtype=None, gpu_memory_utilization=None, mem_fraction_static=None, free_gpu_memory_fraction=None, enable_local_indexer=false, bootstrap_port=None, handoff_session_timeout_ms=300000, kv_bytes_per_token=None, kv_transfer_bandwidth=None, kv_transfer_timing_mode="full_prompt", reasoning=None, response_replay_trace_path=None, zmq_kv_events_port=None, zmq_replay_port=None, preemption_mode="lifo", router_queue_policy=None, sglang=None, trtllm=None, num_g2_blocks=None, num_g3_blocks=None, offload_batch_size=None, bandwidth_g1_to_g2_gbps=None, bandwidth_g2_to_g1_gbps=None, bandwidth_g2_to_g3_gbps=None, bandwidth_g3_to_g2_gbps=None, enable_g4_storage=false, bandwidth_g2_to_g4_gbps=None, bandwidth_g4_to_g2_gbps=None, max_model_len=None, g1_backend=None))] #[allow(clippy::too_many_arguments)] fn new( engine_type: &str, @@ -242,12 +242,12 @@ impl MockEngineArgs { bandwidth_g2_to_g4_gbps: Option, bandwidth_g4_to_g2_gbps: Option, max_model_len: Option, - g1_backend: &str, + g1_backend: Option<&str>, ) -> PyResult { let engine_type = parse_mocker_engine_type(engine_type)?; let worker_type = parse_worker_type(worker_type)?; let preemption_mode = parse_preemption_mode(preemption_mode)?; - let g1_backend = parse_g1_backend(g1_backend)?; + let g1_backend = g1_backend.map(parse_g1_backend).transpose()?; let kv_transfer_timing_mode = kv_transfer_timing_mode .parse() .map_err(|error: String| PyException::new_err(error))?; @@ -266,7 +266,6 @@ impl MockEngineArgs { .max_num_seqs(max_num_seqs) .max_num_batched_tokens(max_num_batched_tokens) .enable_prefix_caching(enable_prefix_caching) - .g1_backend(g1_backend) .enable_chunked_prefill(enable_chunked_prefill) .speedup_ratio(speedup_ratio) .decode_speedup_ratio(decode_speedup_ratio) @@ -321,6 +320,9 @@ impl MockEngineArgs { if let Some(num_gpu_blocks) = num_gpu_blocks { builder = builder.num_gpu_blocks(num_gpu_blocks); } + if let Some(g1_backend) = g1_backend { + builder = builder.g1_backend(g1_backend); + } if let Some(npz_path) = planner_profile_data { let perf_model = PerfModel::from_npz(&npz_path).map_err(|e| { @@ -403,7 +405,7 @@ impl MockEngineArgs { #[getter] fn g1_backend(&self) -> &'static str { - match self.inner.g1_backend { + match self.inner.resolved_g1_backend() { RsG1Backend::Kvbm => "kvbm", RsG1Backend::Native => "native", } diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi index 1ed43a4f202f..61851f0ca401 100644 --- a/lib/bindings/python/src/dynamo/_core.pyi +++ b/lib/bindings/python/src/dynamo/_core.pyi @@ -2095,7 +2095,7 @@ class MockEngineArgs: bandwidth_g2_to_g4_gbps: Optional[float] = None, bandwidth_g4_to_g2_gbps: Optional[float] = None, max_model_len: Optional[int] = None, - g1_backend: str = "kvbm", + g1_backend: Optional[str] = None, ) -> None: ... diff --git a/lib/mocker/Cargo.toml b/lib/mocker/Cargo.toml index 89ba2eaa2c86..dbafcc435b1e 100644 --- a/lib/mocker/Cargo.toml +++ b/lib/mocker/Cargo.toml @@ -18,6 +18,7 @@ profile = [] # Test/benchmark-only deterministic replay plumbing. Production replay never # enables this feature and keeps the existing random selector behavior. replay-bench = ["dynamo-kv-router/bench"] +test-support = [] aic-forward-pass = ["dep:aiconfigurator-core"] zmq-events = ["dep:tmq"] # Enables G1↔G2 offload simulation via kvbm-engine. diff --git a/lib/mocker/src/common/protocols.rs b/lib/mocker/src/common/protocols.rs index 9293b0133373..3a65ecf3770e 100644 --- a/lib/mocker/src/common/protocols.rs +++ b/lib/mocker/src/common/protocols.rs @@ -34,14 +34,15 @@ pub enum MockerEvictionBackend { /// G1 implementation used by the shared vLLM/TRT-LLM mock scheduler. /// -/// `Kvbm` preserves the existing kvbm-logical implementation while `Native` -/// selects the self-contained physical-copy pool. Both remain available until -/// the KVBM G1 implementation is removed. +/// `Native` is the default self-contained physical-copy pool. `Kvbm` preserves +/// the existing kvbm-logical implementation and is selected automatically when +/// the legacy G2/G3/G4 offload path is enabled. SGLang ignores this setting and +/// uses its own KV manager. #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum G1Backend { - #[default] Kvbm, + #[default] Native, } @@ -703,9 +704,11 @@ pub struct MockEngineArgs { #[builder(default = true)] pub enable_prefix_caching: bool, - /// G1 block-manager implementation for the shared vLLM/TRT-LLM scheduler. - #[builder(default)] - pub g1_backend: G1Backend, + /// Requested G1 block-manager implementation for the shared vLLM/TRT-LLM + /// scheduler. `None` selects native unless legacy offload requires KVBM. + /// Ignored by the SGLang scheduler, which uses `SglangKvManager`. + #[builder(default = "None", setter(strip_option))] + pub g1_backend: Option, #[builder(default = true)] pub enable_chunked_prefill: bool, @@ -1003,26 +1006,6 @@ fn validate_mock_engine_args(args: &MockEngineArgs) -> Result<(), ValidationErro )); } - if args.g1_backend == G1Backend::Native && args.engine_type == EngineType::Sglang { - return Err(mock_engine_args_validation_error( - "native_g1_requires_shared_scheduler", - format!( - "g1_backend=native is supported only for engine_type=vllm or trtllm, got engine_type={:?}", - args.engine_type - ), - )); - } - - if args.g1_backend == G1Backend::Native - && (args.num_g2_blocks.is_some() || args.num_g3_blocks.is_some() || args.enable_g4_storage) - { - return Err(mock_engine_args_validation_error( - "native_g1_legacy_offload_conflict", - "g1_backend=native cannot be combined with the legacy kvbm-offload G2/G3/G4 path" - .to_string(), - )); - } - if args.num_g3_blocks.is_some() && args.num_g2_blocks.is_none() { return Err(mock_engine_args_validation_error( "g3_requires_g2", @@ -1394,6 +1377,7 @@ impl MockEngineArgs { pub fn normalized(mut self) -> anyhow::Result { self.materialize_defaults(); + self.resolve_g1_backend(); self.validate_config()?; Ok(self) } @@ -1436,6 +1420,33 @@ impl MockEngineArgs { } } + fn resolve_g1_backend(&mut self) { + let requires_kvbm = matches!(self.engine_type, EngineType::Vllm | EngineType::Trtllm) + && (self.num_g2_blocks.is_some() + || self.num_g3_blocks.is_some() + || self.enable_g4_storage); + + self.g1_backend = Some(match (self.g1_backend, requires_kvbm) { + (Some(G1Backend::Native), true) => { + tracing::info!( + requested_g1_backend = "native", + selected_g1_backend = "kvbm", + "KVBM offload requires the KVBM G1 backend; overriding explicit selection" + ); + G1Backend::Kvbm + } + (Some(backend), _) => backend, + (None, true) => G1Backend::Kvbm, + (None, false) => G1Backend::Native, + }); + } + + /// Return the resolved backend, using the native default for raw, + /// unnormalized arguments. + pub fn resolved_g1_backend(&self) -> G1Backend { + self.g1_backend.unwrap_or_default() + } + fn validate_config(&mut self) -> anyhow::Result<()> { self.validate() .map_err(|error| anyhow::anyhow!("Failed to validate MockEngineArgs: {error}"))?; @@ -1648,7 +1659,7 @@ mod tests { assert_eq!(restored.max_model_len, Some(32768)); assert_eq!(restored.max_num_seqs, None); assert_eq!(restored.max_num_batched_tokens, None); - assert_eq!(restored.g1_backend, G1Backend::Native); + assert_eq!(restored.g1_backend, Some(G1Backend::Native)); assert_eq!( restored.kv_transfer_timing_mode, KvTransferTimingMode::FullPrompt @@ -1659,14 +1670,16 @@ mod tests { fn test_mock_engine_args_accepts_legacy_enum_case_and_writes_lowercase() { let args = MockEngineArgs::from_json_str( &json!({ - "engine_type": "Vllm", + "engine_type": "VLLM", "worker_type": "Aggregated", "preemption_mode": "Lifo", + "num_g2_blocks": 8, }) .to_string(), ) .unwrap(); + assert_eq!(args.g1_backend, Some(G1Backend::Kvbm)); let serialized = serde_json::to_value(args).unwrap(); assert_eq!(serialized["engine_type"], "vllm"); assert_eq!(serialized["worker_type"], "aggregated"); @@ -1842,35 +1855,70 @@ mod tests { .unwrap_or_else(|error| { panic!("native G1 should support {engine_type:?}: {error}") }); - assert_eq!(args.g1_backend, G1Backend::Native); + assert_eq!(args.g1_backend, Some(G1Backend::Native)); } } #[test] - fn test_native_g1_rejects_sglang_and_legacy_kvbm_offload() { - let sglang = MockEngineArgs::builder() - .engine_type(EngineType::Sglang) - .g1_backend(G1Backend::Native) - .build() - .unwrap() - .normalized() - .unwrap_err(); - assert!( - sglang.to_string().contains("engine_type=vllm or trtllm"), - "unexpected error: {sglang}" - ); + fn test_g1_backend_defaults_to_native() { + let default_args = MockEngineArgs::default(); + assert_eq!(default_args.g1_backend, Some(G1Backend::Native)); + + let json_args = MockEngineArgs::from_json_str("{}").unwrap(); + assert_eq!(json_args.g1_backend, Some(G1Backend::Native)); + } + + #[test] + fn test_legacy_kvbm_offload_selects_kvbm_g1() { + let configs = [ + MockEngineArgs::builder() + .num_g2_blocks(Some(8)) + .build() + .unwrap(), + MockEngineArgs::builder() + .num_g2_blocks(Some(8)) + .num_g3_blocks(Some(16)) + .build() + .unwrap(), + MockEngineArgs::builder() + .num_g2_blocks(Some(8)) + .enable_g4_storage(true) + .build() + .unwrap(), + ]; - let offload = MockEngineArgs::builder() + for config in configs { + let args = config.normalized().unwrap(); + assert_eq!(args.g1_backend, Some(G1Backend::Kvbm)); + } + } + + #[test] + fn test_explicit_native_g1_with_offload_selects_kvbm() { + let args = MockEngineArgs::builder() .g1_backend(G1Backend::Native) .num_g2_blocks(Some(8)) .build() .unwrap() .normalized() - .unwrap_err(); - assert!( - offload.to_string().contains("legacy kvbm-offload"), - "unexpected error: {offload}" - ); + .unwrap(); + + assert_eq!(args.g1_backend, Some(G1Backend::Kvbm)); + } + + #[test] + fn test_g1_backend_is_ignored_for_sglang() { + for g1_backend in [G1Backend::Kvbm, G1Backend::Native] { + let args = MockEngineArgs::builder() + .engine_type(EngineType::Sglang) + .g1_backend(g1_backend) + .build() + .unwrap() + .normalized() + .unwrap(); + assert_eq!(args.engine_type, EngineType::Sglang); + assert_eq!(args.g1_backend, Some(g1_backend)); + } } #[test] @@ -1886,7 +1934,7 @@ mod tests { .unwrap_or_else(|error| { panic!("native G1 MTP should support {engine_type:?}: {error}") }); - assert_eq!(args.g1_backend, G1Backend::Native); + assert_eq!(args.g1_backend, Some(G1Backend::Native)); assert_eq!(args.aic_nextn, Some(1)); } } diff --git a/lib/mocker/src/kv_manager/g1_manager.rs b/lib/mocker/src/kv_manager/g1_manager.rs index 746a0c8527ed..b4d2612f7866 100644 --- a/lib/mocker/src/kv_manager/g1_manager.rs +++ b/lib/mocker/src/kv_manager/g1_manager.rs @@ -223,6 +223,11 @@ impl G1Manager { sequence: &mut ActiveSequence, ) { if let G1ManagerBackend::Native(manager) = &mut self.backend { + // A single scheduling decision can complete several prompt blocks + // and the mutable tail together. Publish/cache the earlier full + // blocks first so the promoted tail never references a parent that + // is not cache-visible yet. + manager.finalize_computed_prefix(owner, computed_before, computed_after); if let Some(promote) = sequence.promote_computed_tail(computed_after) { assert!( matches!( @@ -232,7 +237,6 @@ impl G1Manager { "computed-tail promotion must be infallible" ); } - manager.finalize_computed_prefix(owner, computed_before, computed_after); } } @@ -577,6 +581,8 @@ impl G1Manager { #[cfg(test)] mod tests { + use dynamo_kv_router::protocols::KvCacheEventData; + use super::*; #[test] @@ -594,4 +600,25 @@ mod tests { let _ = manager.process_for_request(Uuid::from_u128(1), &event, 0); } + + #[test] + fn native_finalization_publishes_parent_before_promoted_tail() { + let stored = crate::replay::native_g1_parent_chain_artifact(4) + .kv_events + .into_iter() + .map(|event| match event.event.data { + KvCacheEventData::Stored(stored) => stored, + other => panic!("expected Stored event, got {other:?}"), + }) + .collect::>(); + assert_eq!(stored.len(), 2); + assert_eq!(stored[0].blocks.len(), 2); + assert_eq!(stored[0].parent_hash, None); + assert_eq!(stored[1].blocks.len(), 1); + assert_eq!( + stored[1].parent_hash, + Some(stored[0].blocks[1].block_hash), + "the promoted tail must be published after its parent block" + ); + } } diff --git a/lib/mocker/src/replay/artifacts.rs b/lib/mocker/src/replay/artifacts.rs index a013486c4d3c..d0a0243c5c89 100644 --- a/lib/mocker/src/replay/artifacts.rs +++ b/lib/mocker/src/replay/artifacts.rs @@ -7,6 +7,76 @@ use uuid::Uuid; use crate::common::protocols::OutputSignal; use crate::loadgen::ReplayRequestHashes; +/// Build the minimal Native G1 artifact that exercises the producer→indexer +/// parent-ordering contract. +/// +/// This is test-only plumbing shared by the mocker unit regression and +/// downstream indexer parity coverage. +#[cfg(any(test, feature = "test-support"))] +#[doc(hidden)] +pub fn native_g1_parent_chain_artifact(block_size: usize) -> ReplayWorkerArtifacts { + use crate::common::protocols::{G1Backend, KvEventPublishers}; + use crate::common::sequence::ActiveSequence; + use crate::kv_manager::{G1Acquire, G1Manager}; + use crate::scheduler::capture_router_event_sink; + + assert!(block_size >= 2, "block size must be at least 2"); + let computed_after = block_size + .checked_mul(3) + .expect("ordering regression token count overflow"); + let prompt_len = computed_after - 2; + let prompt_len_u32 = + u32::try_from(prompt_len).expect("ordering regression prompt length must fit in u32"); + let mut tokens = (0..prompt_len_u32).collect::>(); + let mut sequence = ActiveSequence::new(tokens.clone(), 2, Some(block_size), true, false); + let owner = Uuid::from_u128(1); + let (events, sink) = capture_router_event_sink(1); + let publishers = KvEventPublishers::new(Some(sink), None); + let mut manager = G1Manager::new_with_backend(3, block_size, publishers, 0, G1Backend::Native); + + let creation = sequence + .take_creation_signal() + .expect("three-block sequence must allocate G1 blocks"); + assert!(matches!( + manager.process_for_request(owner, &creation, 0), + G1Acquire::Ready(3) + )); + + for token in [prompt_len_u32, prompt_len_u32 + 1] { + assert!(sequence.push(token).is_none()); + tokens.push(token); + } + manager.finalize_computed_prefix(owner, 0, computed_after, &mut sequence); + + let kv_events = events + .drain() + .into_iter() + .enumerate() + .map(|(ordinal, event)| ReplayTimedKvEvent { + event: event.event, + storage_tier: event.storage_tier, + timestamp_us: ordinal as u64, + }) + .collect::>(); + let request_timestamp = kv_events.len() as u64; + + ReplayWorkerArtifacts { + requests: vec![ReplayTimedRequest { + uuid: owner, + timestamp_us: request_timestamp, + scheduled_ready_at_ms: request_timestamp as f64 / 1000.0, + input_length: tokens.len(), + output_length: 0, + replay_hashes: ReplayRequestHashes::from_tokens( + &tokens, + u32::try_from(block_size).expect("block size must fit in u32"), + ), + }], + output_signals: Vec::new(), + kv_events, + } +} + #[derive(Debug, Clone)] pub struct ReplayTimedRequest { pub uuid: Uuid, diff --git a/lib/mocker/src/replay/mod.rs b/lib/mocker/src/replay/mod.rs index b3ab7a7557d9..b86b5aad7d74 100644 --- a/lib/mocker/src/replay/mod.rs +++ b/lib/mocker/src/replay/mod.rs @@ -16,6 +16,9 @@ use std::sync::Arc; use crate::common::protocols::{DirectRequest, MockEngineArgs}; use dynamo_kv_router::PrefillLoadEstimator; +#[cfg(any(test, feature = "test-support"))] +#[doc(hidden)] +pub use artifacts::native_g1_parent_chain_artifact; pub use artifacts::{ ReplayTimedKvEvent, ReplayTimedOutputSignal, ReplayTimedRequest, ReplayWorkerArtifacts, }; diff --git a/lib/mocker/src/replay/offline/agg.rs b/lib/mocker/src/replay/offline/agg.rs index 0197c4d03034..e06ed7d36003 100644 --- a/lib/mocker/src/replay/offline/agg.rs +++ b/lib/mocker/src/replay/offline/agg.rs @@ -2059,14 +2059,16 @@ mod tests { fn test_multi_worker_trace_kv_router_debug_snapshot_tracks_queue_and_cached_dispatch() { let policy = RouterQueuePolicy::Fcfs; let args = queueing_router_args(policy); + let mut router_config = queueing_router_config(policy); + router_config.router_queue_threshold = Some(0.0); let mut runtime = AggRuntime::new( &args, - Some(queueing_router_config(policy)), + Some(router_config), None, normalize_trace_requests( vec![ DirectRequest { - tokens: vec![11; 64], + tokens: vec![11; 128], max_output_tokens: 8, output_token_ids: None, uuid: Some(Uuid::from_u128(11)), @@ -2075,7 +2077,7 @@ mod tests { ..Default::default() }, DirectRequest { - tokens: vec![22; 64], + tokens: vec![22; 192], max_output_tokens: 8, output_token_ids: None, uuid: Some(Uuid::from_u128(22)), @@ -2089,7 +2091,7 @@ mod tests { output_token_ids: None, uuid: Some(Uuid::from_u128(33)), dp_rank: 0, - arrival_timestamp_ms: Some(0.1), + arrival_timestamp_ms: Some(15.0), ..Default::default() }, ], @@ -2117,13 +2119,19 @@ mod tests { .collect::>(), vec![1, 1] ); - assert!(initial_router.indexer.total_cached_blocks > 0); + assert_eq!( + initial_router.indexer.total_cached_blocks, 0, + "native G1 publishes blocks only after their tokens are computed" + ); - assert!(runtime.advance_one_timestamp().unwrap()); + while runtime.now_ms < 15.0 { + assert!(runtime.advance_one_timestamp().unwrap()); + } let queued = runtime.debug_snapshot(); let queued_router = queued.router.as_ref().unwrap(); - assert_eq!(queued.now_ms, 0.1); + assert_eq!(queued.now_ms, 15.0); + assert!(queued_router.indexer.total_cached_blocks > 0); assert_eq!(queued.router_pending_request_ids, vec![Uuid::from_u128(33)]); assert_eq!(queued_router.pending.len(), 1); assert_eq!(queued_router.pending[0].uuid, Uuid::from_u128(33)); diff --git a/lib/mocker/src/scheduler/vllm/core.rs b/lib/mocker/src/scheduler/vllm/core.rs index 92c14d4663bc..cdc95b53bffd 100644 --- a/lib/mocker/src/scheduler/vllm/core.rs +++ b/lib/mocker/src/scheduler/vllm/core.rs @@ -499,7 +499,7 @@ impl VllmCore { args.block_size, kv_event_publishers, dp_rank, - args.g1_backend, + args.resolved_g1_backend(), args.enable_prefix_caching, ), args, diff --git a/lib/mocker/src/scheduler/vllm/policy/tests.rs b/lib/mocker/src/scheduler/vllm/policy/tests.rs index fc59304c27d9..4e89c9ecfee8 100644 --- a/lib/mocker/src/scheduler/vllm/policy/tests.rs +++ b/lib/mocker/src/scheduler/vllm/policy/tests.rs @@ -988,7 +988,7 @@ mod trtllm { #[test] fn native_g1_runs_under_trtllm_no_evict_policy() { let mut args = capacity_args(); - args.g1_backend = G1Backend::Native; + args.g1_backend = Some(G1Backend::Native); let mut core = VllmCore::new(args); receive(&mut core, Uuid::from_u128(201), 0..4, 4); receive(&mut core, Uuid::from_u128(202), 100..104, 4); diff --git a/lib/mocker/src/scheduler/vllm/tests.rs b/lib/mocker/src/scheduler/vllm/tests.rs index 66a02184cd2a..aa0f04b6b756 100644 --- a/lib/mocker/src/scheduler/vllm/tests.rs +++ b/lib/mocker/src/scheduler/vllm/tests.rs @@ -1373,6 +1373,10 @@ mod core_behavior { #[test] fn test_running_request_catches_up_decode_tail_before_promote() { let args = MockEngineArgs::builder() + // This test exercises the compatibility `G1Manager::process` + // entrypoint, which is intentionally KVBM-only. Production + // scheduler paths use the owner-aware entrypoint for native G1. + .g1_backend(G1Backend::Kvbm) .block_size(4) .num_gpu_blocks(8) .max_num_batched_tokens(Some(8)) From fce5fb9a64ce77276f8b4d2214606e4b2986c537 Mon Sep 17 00:00:00 2001 From: Yongming Ding Date: Mon, 27 Jul 2026 23:16:50 -0700 Subject: [PATCH 2/2] fix(mocker): address review comments Signed-off-by: Yongming Ding --- components/src/dynamo/mocker/args.py | 22 ++++- .../dynamo/mocker/tests/unit/test_config.py | 50 ++++++++++- docs/fern/dynosim/modeling.md | 13 ++- lib/mocker/src/common/protocols.rs | 88 ++++++++++++++----- lib/mocker/src/replay/offline/agg.rs | 28 +++--- lib/mocker/src/scheduler/vllm/tests.rs | 28 +++++- 6 files changed, 178 insertions(+), 51 deletions(-) diff --git a/components/src/dynamo/mocker/args.py b/components/src/dynamo/mocker/args.py index b1d7e2293f67..89d37a993026 100644 --- a/components/src/dynamo/mocker/args.py +++ b/components/src/dynamo/mocker/args.py @@ -259,9 +259,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: choices=["kvbm", "native"], default=None, help="G1 manager for the shared vLLM/TRT-LLM scheduler. When unset, " - "native is used unless KVBM G2/G3/G4 offloading selects KVBM. An " - "explicit native selection is also overridden when offloading requires " - "KVBM. Ignored for SGLang.", + "native is used unless KVBM G2/G3/G4 offloading selects KVBM. Explicit " + "native cannot be combined with KVBM offloading. Ignored for SGLang.", ) parser.add_argument( "--enable-chunked-prefill", @@ -701,6 +700,23 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) args = parser.parse_args(argv) + + kvbm_offload_enabled = ( + (args.num_g2_blocks or 0) > 0 + or (args.num_g3_blocks or 0) > 0 + or args.enable_g4_storage + ) + if ( + args.engine_type in ("vllm", "trtllm") + and args.g1_backend == "native" + and kvbm_offload_enabled + ): + parser.error( + "--g1-backend native cannot be combined with KVBM G2/G3/G4 " + "offload; omit --g1-backend to select KVBM automatically or set " + "--g1-backend kvbm explicitly" + ) + validate_worker_type_args(args) # Validate num_workers diff --git a/components/src/dynamo/mocker/tests/unit/test_config.py b/components/src/dynamo/mocker/tests/unit/test_config.py index 582355f4b96e..cb656c58f09b 100644 --- a/components/src/dynamo/mocker/tests/unit/test_config.py +++ b/components/src/dynamo/mocker/tests/unit/test_config.py @@ -166,10 +166,24 @@ def test_mocker_offload_automatically_selects_kvbm_g1(offload): assert engine_args.g1_backend == "kvbm" -def test_mocker_explicit_native_with_offload_selects_kvbm_g1(): - engine_args = MockEngineArgs(g1_backend="native", num_g2_blocks=8) +@pytest.mark.parametrize("engine_type", ["vllm", "trtllm"]) +def test_mocker_explicit_native_with_offload_is_rejected(engine_type): + with pytest.raises(Exception, match="omit g1_backend"): + MockEngineArgs( + engine_type=engine_type, + g1_backend="native", + num_g2_blocks=8, + ) - assert engine_args.g1_backend == "kvbm" + +def test_mocker_explicit_native_accepts_disabled_offload(): + engine_args = MockEngineArgs( + g1_backend="native", + num_g2_blocks=0, + num_g3_blocks=0, + ) + + assert engine_args.g1_backend == "native" @pytest.mark.parametrize("engine_type", ["VLLM", "TRTLLM"]) @@ -462,6 +476,36 @@ def test_mocker_cli_accepts_native_g1_for_trtllm(): assert args.g1_backend == "native" +@pytest.mark.parametrize("engine_type", ["vllm", "trtllm"]) +def test_mocker_cli_rejects_explicit_native_g1_with_offload(engine_type, capsys): + with pytest.raises(SystemExit): + parse_args( + [ + "--engine-type", + engine_type, + "--g1-backend", + "native", + "--num-g2-blocks", + "8", + ] + ) + + assert "omit --g1-backend" in capsys.readouterr().err + + +def test_mocker_cli_ignores_explicit_g1_backend_for_sglang(): + args = parse_args( + [ + "--engine-type", + "sglang", + "--g1-backend", + "native", + ] + ) + + assert args.g1_backend == "native" + + def test_mocker_cli_accepts_max_model_len(): args = parse_args(["--max-model-len", "32768"]) diff --git a/docs/fern/dynosim/modeling.md b/docs/fern/dynosim/modeling.md index b16afa3e5207..6521811e5a2e 100644 --- a/docs/fern/dynosim/modeling.md +++ b/docs/fern/dynosim/modeling.md @@ -36,8 +36,19 @@ For the shared vLLM/TensorRT-LLM core, `g1_backend` selects the GPU-tier block m - `kvbm` uses the KVBM logical block manager and its lineage-aware inactive pool. - `native` uses Mocker's self-contained physical block-pool model. +When `g1_backend` is unset, Mocker selects `native` unless `num_g2_blocks`, +`num_g3_blocks`, or `enable_g4_storage` enables a lower tier. Lower-tier configuration +automatically selects `kvbm`. An explicit `g1_backend: native` conflicts with lower-tier +configuration and fails validation; omit `g1_backend` to select KVBM automatically or set it to +`kvbm`. + Both managers model G1 capacity, prefix reuse, request ownership, eviction, and router-visible KV -events. The SGLang core uses its own token-pool and radix-cache implementation. +events. Their event visibility and allocation timing can differ, which can change KV-router +decisions and numeric replay results. Set `g1_backend` explicitly when comparing a new run with a +baseline created under a specific manager. + +The SGLang core uses its own token-pool and radix-cache implementation. It ignores `g1_backend` and +does not support KVBM lower-tier offload. ## Timing Sources diff --git a/lib/mocker/src/common/protocols.rs b/lib/mocker/src/common/protocols.rs index 3a65ecf3770e..e8ec19e82bc5 100644 --- a/lib/mocker/src/common/protocols.rs +++ b/lib/mocker/src/common/protocols.rs @@ -708,6 +708,7 @@ pub struct MockEngineArgs { /// scheduler. `None` selects native unless legacy offload requires KVBM. /// Ignored by the SGLang scheduler, which uses `SglangKvManager`. #[builder(default = "None", setter(strip_option))] + #[serde(skip_serializing_if = "Option::is_none")] pub g1_backend: Option, #[builder(default = true)] @@ -1006,6 +1007,14 @@ fn validate_mock_engine_args(args: &MockEngineArgs) -> Result<(), ValidationErro )); } + if args.g1_backend == Some(G1Backend::Native) && args.requires_kvbm_g1() { + return Err(mock_engine_args_validation_error( + "native_g1_legacy_offload_conflict", + "g1_backend=native cannot be combined with KVBM G2/G3/G4 offload; omit g1_backend to select KVBM automatically or set g1_backend=kvbm explicitly" + .to_string(), + )); + } + if args.num_g3_blocks.is_some() && args.num_g2_blocks.is_none() { return Err(mock_engine_args_validation_error( "g3_requires_g2", @@ -1420,31 +1429,33 @@ impl MockEngineArgs { } } + fn requires_kvbm_g1(&self) -> bool { + matches!(self.engine_type, EngineType::Vllm | EngineType::Trtllm) + && (self.num_g2_blocks.is_some_and(|blocks| blocks > 0) + || self.num_g3_blocks.is_some_and(|blocks| blocks > 0) + || self.enable_g4_storage) + } + fn resolve_g1_backend(&mut self) { - let requires_kvbm = matches!(self.engine_type, EngineType::Vllm | EngineType::Trtllm) - && (self.num_g2_blocks.is_some() - || self.num_g3_blocks.is_some() - || self.enable_g4_storage); - - self.g1_backend = Some(match (self.g1_backend, requires_kvbm) { - (Some(G1Backend::Native), true) => { - tracing::info!( - requested_g1_backend = "native", - selected_g1_backend = "kvbm", - "KVBM offload requires the KVBM G1 backend; overriding explicit selection" - ); + if self.g1_backend.is_none() { + self.g1_backend = Some(if self.requires_kvbm_g1() { G1Backend::Kvbm - } - (Some(backend), _) => backend, - (None, true) => G1Backend::Kvbm, - (None, false) => G1Backend::Native, - }); + } else { + G1Backend::Native + }); + } } - /// Return the resolved backend, using the native default for raw, - /// unnormalized arguments. + /// Return the selected backend, resolving an unset raw configuration from + /// its engine and lower-tier offload settings. pub fn resolved_g1_backend(&self) -> G1Backend { - self.g1_backend.unwrap_or_default() + self.g1_backend.unwrap_or_else(|| { + if self.requires_kvbm_g1() { + G1Backend::Kvbm + } else { + G1Backend::Native + } + }) } fn validate_config(&mut self) -> anyhow::Result<()> { @@ -1666,6 +1677,14 @@ mod tests { ); } + #[test] + fn test_mock_engine_args_json_omits_unset_g1_backend() { + let args = MockEngineArgs::builder().build().unwrap(); + let serialized = serde_json::to_value(args).unwrap(); + + assert!(serialized.get("g1_backend").is_none()); + } + #[test] fn test_mock_engine_args_accepts_legacy_enum_case_and_writes_lowercase() { let args = MockEngineArgs::from_json_str( @@ -1888,22 +1907,45 @@ mod tests { ]; for config in configs { + assert_eq!(config.resolved_g1_backend(), G1Backend::Kvbm); let args = config.normalized().unwrap(); assert_eq!(args.g1_backend, Some(G1Backend::Kvbm)); } } #[test] - fn test_explicit_native_g1_with_offload_selects_kvbm() { + fn test_explicit_native_g1_with_offload_is_rejected() { + for engine_type in [EngineType::Vllm, EngineType::Trtllm] { + let error = MockEngineArgs::builder() + .engine_type(engine_type) + .g1_backend(G1Backend::Native) + .num_g2_blocks(Some(8)) + .build() + .unwrap() + .normalized() + .unwrap_err(); + + assert!( + error.to_string().contains("omit g1_backend"), + "unexpected error for {engine_type:?}: {error}" + ); + } + } + + #[test] + fn test_explicit_native_g1_accepts_disabled_offload() { let args = MockEngineArgs::builder() .g1_backend(G1Backend::Native) - .num_g2_blocks(Some(8)) + .num_g2_blocks(Some(0)) + .num_g3_blocks(Some(0)) .build() .unwrap() .normalized() .unwrap(); - assert_eq!(args.g1_backend, Some(G1Backend::Kvbm)); + assert_eq!(args.g1_backend, Some(G1Backend::Native)); + assert_eq!(args.num_g2_blocks, None); + assert_eq!(args.num_g3_blocks, None); } #[test] diff --git a/lib/mocker/src/replay/offline/agg.rs b/lib/mocker/src/replay/offline/agg.rs index e06ed7d36003..99719a381833 100644 --- a/lib/mocker/src/replay/offline/agg.rs +++ b/lib/mocker/src/replay/offline/agg.rs @@ -1120,7 +1120,7 @@ mod tests { run_trace_workload_multi_collect_with_stats, run_trace_workload_single_collect, }; use super::*; - use crate::common::protocols::{EngineType, SglangArgs}; + use crate::common::protocols::{EngineType, G1Backend, SglangArgs}; use crate::loadgen::{AgenticTrace, AgenticTurnTrace, SessionTrace, Trace, TurnTrace}; use crate::replay::offline::extensions::kv_router::{ReplayKvRouterConfig, RouterQueuePolicy}; use crate::replay::{TraceRequestStatsSnapshot, normalize_trace_requests}; @@ -2058,17 +2058,17 @@ mod tests { #[test] fn test_multi_worker_trace_kv_router_debug_snapshot_tracks_queue_and_cached_dispatch() { let policy = RouterQueuePolicy::Fcfs; - let args = queueing_router_args(policy); - let mut router_config = queueing_router_config(policy); - router_config.router_queue_threshold = Some(0.0); + let mut args = queueing_router_args(policy); + // Preserve this snapshot's historical KVBM event-visibility semantics. + args.g1_backend = Some(G1Backend::Kvbm); let mut runtime = AggRuntime::new( &args, - Some(router_config), + Some(queueing_router_config(policy)), None, normalize_trace_requests( vec![ DirectRequest { - tokens: vec![11; 128], + tokens: vec![11; 64], max_output_tokens: 8, output_token_ids: None, uuid: Some(Uuid::from_u128(11)), @@ -2077,7 +2077,7 @@ mod tests { ..Default::default() }, DirectRequest { - tokens: vec![22; 192], + tokens: vec![22; 64], max_output_tokens: 8, output_token_ids: None, uuid: Some(Uuid::from_u128(22)), @@ -2091,7 +2091,7 @@ mod tests { output_token_ids: None, uuid: Some(Uuid::from_u128(33)), dp_rank: 0, - arrival_timestamp_ms: Some(15.0), + arrival_timestamp_ms: Some(0.1), ..Default::default() }, ], @@ -2119,19 +2119,13 @@ mod tests { .collect::>(), vec![1, 1] ); - assert_eq!( - initial_router.indexer.total_cached_blocks, 0, - "native G1 publishes blocks only after their tokens are computed" - ); + assert!(initial_router.indexer.total_cached_blocks > 0); - while runtime.now_ms < 15.0 { - assert!(runtime.advance_one_timestamp().unwrap()); - } + assert!(runtime.advance_one_timestamp().unwrap()); let queued = runtime.debug_snapshot(); let queued_router = queued.router.as_ref().unwrap(); - assert_eq!(queued.now_ms, 15.0); - assert!(queued_router.indexer.total_cached_blocks > 0); + assert_eq!(queued.now_ms, 0.1); assert_eq!(queued.router_pending_request_ids, vec![Uuid::from_u128(33)]); assert_eq!(queued_router.pending.len(), 1); assert_eq!(queued_router.pending[0].uuid, Uuid::from_u128(33)); diff --git a/lib/mocker/src/scheduler/vllm/tests.rs b/lib/mocker/src/scheduler/vllm/tests.rs index aa0f04b6b756..8c05a1eb453c 100644 --- a/lib/mocker/src/scheduler/vllm/tests.rs +++ b/lib/mocker/src/scheduler/vllm/tests.rs @@ -1657,10 +1657,15 @@ mod router_events { assert!(pass.kv_events.iter().all(|event| event.event.dp_rank == 0)); } + #[rstest] + #[case::kvbm(G1Backend::Kvbm)] + #[case::native(G1Backend::Native)] #[tokio::test] - async fn test_completion_events_apply_cleanly() { + async fn test_completion_events_apply_cleanly(#[case] backend: G1Backend) { let harness = RouterIndexerHarness::new(4, ROUTER_TEST_WORKER_ID); - let mut core = VllmCore::new_with_kv_capture(router_args(), ROUTER_TEST_WORKER_ID); + let mut args = router_args(); + args.g1_backend = Some(backend); + let mut core = VllmCore::new_with_kv_capture(args, ROUTER_TEST_WORKER_ID); core.receive(DirectRequest { tokens: (0..8).collect(), max_output_tokens: 4, @@ -1681,7 +1686,7 @@ mod router_events { harness.apply_events(pass.kv_events).await; } - assert!(saw_store); + assert!(saw_store, "backend={backend:?}"); assert!(harness.ok_count(METRIC_EVENT_STORED) > 0); assert_eq!(core.kv_manager.num_active_blocks(), 0); harness.assert_no_event_warnings(); @@ -2869,7 +2874,9 @@ mod offload { use uuid::Uuid; use crate::common::handoff::HandoffId; - use crate::common::protocols::{DirectRequest, MockEngineArgs, MoveBlock, WorkerType}; + use crate::common::protocols::{ + DirectRequest, G1Backend, MockEngineArgs, MoveBlock, WorkerType, + }; use crate::common::sequence::ActiveSequence; use crate::kv_manager::G1Acquire; use crate::kvbm_offload::shared_g3::shared_g3_test_guard_blocking; @@ -2984,6 +2991,7 @@ mod offload { #[test] fn decode_growth_waits_without_preempting() { let args = MockEngineArgs::builder() + .g1_backend(G1Backend::Kvbm) .num_gpu_blocks(3) .block_size(4) .max_num_batched_tokens(Some(64)) @@ -3069,6 +3077,7 @@ mod offload { #[test] fn speculative_decode_reservation_waits_without_preempting() { let args = MockEngineArgs::builder() + .g1_backend(G1Backend::Kvbm) .num_gpu_blocks(5) .block_size(4) .max_num_batched_tokens(Some(64)) @@ -3150,6 +3159,7 @@ mod offload { #[tokio::test] async fn execute_pass_ticks_offload_engine_when_attached() { let args = MockEngineArgs::builder() + .g1_backend(G1Backend::Kvbm) .num_gpu_blocks(8) .block_size(4) .max_num_batched_tokens(Some(64)) @@ -3176,6 +3186,7 @@ mod offload { async fn live_destination_eviction_uses_command_application_time() { let block_size = 4; let args = MockEngineArgs::builder() + .g1_backend(G1Backend::Kvbm) .num_gpu_blocks(2) .block_size(block_size) .max_num_batched_tokens(Some(64)) @@ -3293,6 +3304,7 @@ mod offload { #[tokio::test] async fn ready_swap_ins_drain_on_pass_entry() { let args = MockEngineArgs::builder() + .g1_backend(G1Backend::Kvbm) .num_gpu_blocks(8) .block_size(4) .max_num_batched_tokens(Some(64)) @@ -3369,6 +3381,7 @@ mod offload { #[tokio::test] async fn g2_swap_in_reserves_destination_slot_before_transfer() { let args = MockEngineArgs::builder() + .g1_backend(G1Backend::Kvbm) .num_gpu_blocks(1) .block_size(4) .max_num_batched_tokens(Some(64)) @@ -3450,6 +3463,7 @@ mod offload { fn tick_offload_only_completes_g3_staging_before_same_timestamp_admission() { let _guard = shared_g3_test_guard_blocking(); let args = MockEngineArgs::builder() + .g1_backend(G1Backend::Kvbm) // Use two blocks so the staged hit still has a reusable prefix // after vLLM's required final-block recompute. .num_gpu_blocks(2) @@ -3534,6 +3548,7 @@ mod offload { async fn partial_g2_swap_in_pins_cached_g1_prefix() { let block_size = 4; let args = MockEngineArgs::builder() + .g1_backend(G1Backend::Kvbm) .num_gpu_blocks(3) .block_size(block_size) .max_num_batched_tokens(Some(64)) @@ -3608,6 +3623,7 @@ mod offload { async fn cancel_parked_swap_in_releases_ownership_and_isolates_uuid_reuse() { let block_size = 4; let args = MockEngineArgs::builder() + .g1_backend(G1Backend::Kvbm) .num_gpu_blocks(3) .block_size(block_size) .max_num_batched_tokens(Some(64)) @@ -3759,6 +3775,7 @@ mod offload { #[tokio::test] async fn completed_swap_ins_reenter_front_preserving_order() { let args = MockEngineArgs::builder() + .g1_backend(G1Backend::Kvbm) .num_gpu_blocks(2) .block_size(4) .max_num_batched_tokens(Some(64)) @@ -3866,6 +3883,7 @@ mod offload { // 4 tokens per block; sequence has 16 tokens → 4 full blocks. let block_size = 4; let args = MockEngineArgs::builder() + .g1_backend(G1Backend::Kvbm) .num_gpu_blocks(16) .block_size(block_size) .max_num_batched_tokens(Some(64)) @@ -4001,6 +4019,7 @@ mod offload { async fn run_mode() -> ModeReport { let args = MockEngineArgs::builder() + .g1_backend(G1Backend::Kvbm) .num_gpu_blocks(32) .block_size(BLOCK_SIZE) .max_num_batched_tokens(Some(256)) @@ -4128,6 +4147,7 @@ mod offload { fn destination_reservation_retries_after_presence_filtered_offload() { let block_size = 4; let args = MockEngineArgs::builder() + .g1_backend(G1Backend::Kvbm) .num_gpu_blocks(2) .block_size(block_size) .max_num_batched_tokens(Some(64))