Skip to content
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,11 @@ base:
engine:
type: vllm
connector: null
# ~1.56 TB of MXFP4 shards may take the full VLLM_ENGINE_READY_TIMEOUT_S.
# Loading ~1.56 TB of MXFP4 shards plus the latency_specialized decode autotune can take
# most of VLLM_ENGINE_READY_TIMEOUT_S.
health_check:
interval_seconds: 10
max_attempts: 360
max_attempts: 720
roles:
agg:
nodes: 1
Expand All @@ -26,8 +27,6 @@ base:
args:
served-model-name: moonshotai/Kimi-K3.1-Pro
tensor-parallel-size: 8
# Shards the MLA latent and indexer-key caches across the TP group: ~7x the KV tokens of plain TP8.
decode-context-parallel-size: 8
trust-remote-code: true
language-model-only: true
load-format: fastsafetensors
Expand All @@ -46,7 +45,7 @@ base:
# TOTAL_CPU_DRAM_GB split across the 8 TP ranks.
kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":281125000000,"lazy_offload":false}}'
env:
VLLM_ENGINE_READY_TIMEOUT_S: '3600'
VLLM_ENGINE_READY_TIMEOUT_S: '7200'
PYTHONUNBUFFERED: '1'
# Identical prefixes must hash to identical block keys run-to-run.
PYTHONHASHSEED: '42'
Expand All @@ -58,125 +57,143 @@ base:
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '2249'

# Admission is 2x CONC. The decode graphs cover max-num-seqs x (1 + draft tokens);
# throughput runs replace block rejection with the golden acceptance length.
override_c1:
# Admission is 2x CONC. DSpark drafts match the SGLang recipe's DFLASH drafts: 7 through c8,
# 6 at c16, 5 at c24-c40 and 4 from c48. The largest decode graph covers max-num-seqs x (1 + drafts),
# rounded up to a default capture size; throughput runs replace block rejection with the
# golden acceptance length. Plain TP8 at c1-c2, where the KV fits on GPU and DCP's decode
# collectives cost most; DCP8 from c4 for ~7x the KV tokens.
# Low-latency decode (the fused CuTe DSL path) runs only where every decode graph has at
# most 64 rows (c1-c4): this image launches its 65-320-row MoE experts on 152 CTAs that
# must all be resident, which a 148-SM B300 cannot hold.
override_tp8_c1:
roles:
agg:
args:
additional-config: '{"decode_backend":"latency_specialized"}'
max-num-seqs: 2
max-cudagraph-capture-size: 12
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
max-cudagraph-capture-size: 16
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: '1'

override_c2:
override_tp8_c2:
roles:
agg:
args:
additional-config: '{"decode_backend":"latency_specialized"}'
max-num-seqs: 4
max-cudagraph-capture-size: 24
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
max-cudagraph-capture-size: 32
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: '2'

override_c4:
override_dcp8_c4:
roles:
agg:
args:
additional-config: '{"decode_backend":"latency_specialized"}'
decode-context-parallel-size: 8
max-num-seqs: 8
max-cudagraph-capture-size: 48
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
max-cudagraph-capture-size: 64
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: '4'

override_c8:
override_dcp8_c8:
roles:
agg:
args:
decode-context-parallel-size: 8
max-num-seqs: 16
max-cudagraph-capture-size: 96
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
max-cudagraph-capture-size: 128
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: '8'

override_c16:
override_dcp8_c16:
roles:
agg:
args:
decode-context-parallel-size: 8
max-num-seqs: 32
max-cudagraph-capture-size: 160
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
max-cudagraph-capture-size: 224
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":6,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: '16'

override_c24:
override_dcp8_c24:
roles:
agg:
args:
decode-context-parallel-size: 8
max-num-seqs: 48
max-cudagraph-capture-size: 240
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
max-cudagraph-capture-size: 288
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: '24'

override_c32:
override_dcp8_c32:
roles:
agg:
args:
decode-context-parallel-size: 8
max-num-seqs: 64
max-cudagraph-capture-size: 256
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
max-cudagraph-capture-size: 384
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: '32'

override_c40:
override_dcp8_c40:
roles:
agg:
args:
decode-context-parallel-size: 8
max-num-seqs: 80
max-cudagraph-capture-size: 320
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
max-cudagraph-capture-size: 480
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: '40'

override_c48:
override_dcp8_c48:
roles:
agg:
args:
decode-context-parallel-size: 8
max-num-seqs: 96
max-cudagraph-capture-size: 384
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
max-cudagraph-capture-size: 480
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: '48'

override_c56:
override_dcp8_c56:
roles:
agg:
args:
decode-context-parallel-size: 8
max-num-seqs: 112
max-cudagraph-capture-size: 336
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
max-cudagraph-capture-size: 560
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: '56'

override_c70:
override_dcp8_c70:
roles:
agg:
args:
decode-context-parallel-size: 8
max-num-seqs: 140
max-cudagraph-capture-size: 420
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
max-cudagraph-capture-size: 704
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: '70'
5 changes: 3 additions & 2 deletions inferencex-e2e/configs/nvidia-master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5957,8 +5957,9 @@ kimik3.1-fp4-b300-vllm-agentic-dspark:
# 0.75 resolves to 2249 GB of host DRAM, which the recipe's Simple CPU offload splits across the 8 TP ranks.
- dram-utilization: 0.75
search-space:
# The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. DSpark drafts 5 tokens up to conc 8, then 4, 3 and 2.
- { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: agentic.yaml }
# The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. DSpark drafts 7 tokens up to conc 8, then 6, 5 and 4.
- { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2], srt-recipe: agentic.yaml }
- { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: agentic.yaml }

dsv41flash-fp4-b200-vllm-agentic-dspark:
image: vllm/vllm-openai:cu134-nightly-40941f2c4ba96fba0a7e282a5355e89cd74d9ac5@sha256:13e92fcd97268013d13d07fb97d80803227cf0beae0161f8dce95a8200eaafb0
Expand Down
7 changes: 7 additions & 0 deletions inferencex-e2e/perf-changelog.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12255,3 +12255,10 @@
- "Prefill CUDA graphs are disabled because the image's default prefill graph faults with an illegal address during long-context replay on H100; decode CUDA graphs are unchanged."
- "Serve the pinned checkpoint revision 46a050bec13002ad29ee795c16e61cf4fa3c4a9c offline from the shared H100 Hugging Face cache."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4717

- config-keys:
- kimik3.1-fp4-b300-vllm-agentic-dspark
description:
- "Kimi-K3.1-Pro B300 vLLM: enable the recipe's low-latency decode (decode_backend=latency_specialized) at c1-c4, whose decode graphs stay within 64 rows (the image's wider MoE needs 152 resident CTAs, more than a B300's 148 SMs), and run c1-c2 at plain TP8 instead of TP8 x DCP8."
- "Match the SGLang recipe's draft depths (DSpark 7 tokens through c8, 6 at c16, 5 at c24-c40, 4 from c48), size each decode graph cap to max-num-seqs x (1 + drafts), and raise the engine-ready timeout and health check to 7200 s."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4833
Loading