diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml index a6ad375bbd..21413562a5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml @@ -14,10 +14,11 @@ base: engine: type: vllm connector: null - # ~1.56 TB of MXFP4 shards may take the full VLLM_ENGINE_READY_TIMEOUT_S. + # Loading ~1.56 TB of MXFP4 shards plus the latency_specialized decode autotune can take + # most of VLLM_ENGINE_READY_TIMEOUT_S. health_check: interval_seconds: 10 - max_attempts: 360 + max_attempts: 720 roles: agg: nodes: 1 @@ -26,8 +27,6 @@ base: args: served-model-name: moonshotai/Kimi-K3.1-Pro tensor-parallel-size: 8 - # Shards the MLA latent and indexer-key caches across the TP group: ~7x the KV tokens of plain TP8. - decode-context-parallel-size: 8 trust-remote-code: true language-model-only: true load-format: fastsafetensors @@ -46,7 +45,7 @@ base: # TOTAL_CPU_DRAM_GB split across the 8 TP ranks. kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":281125000000,"lazy_offload":false}}' env: - VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_ENGINE_READY_TIMEOUT_S: '7200' PYTHONUNBUFFERED: '1' # Identical prefixes must hash to identical block keys run-to-run. PYTHONHASHSEED: '42' @@ -58,125 +57,143 @@ base: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2249' -# Admission is 2x CONC. The decode graphs cover max-num-seqs x (1 + draft tokens); -# throughput runs replace block rejection with the golden acceptance length. -override_c1: +# Admission is 2x CONC. DSpark drafts match the SGLang recipe's DFLASH drafts: 7 through c8, +# 6 at c16, 5 at c24-c40 and 4 from c48. The largest decode graph covers max-num-seqs x (1 + drafts), +# rounded up to a default capture size; throughput runs replace block rejection with the +# golden acceptance length. Plain TP8 at c1-c2, where the KV fits on GPU and DCP's decode +# collectives cost most; DCP8 from c4 for ~7x the KV tokens. +# Low-latency decode (the fused CuTe DSL path) runs only where every decode graph has at +# most 64 rows (c1-c4): this image launches its 65-320-row MoE experts on 152 CTAs that +# must all be resident, which a 148-SM B300 cannot hold. +override_tp8_c1: roles: agg: args: + additional-config: '{"decode_backend":"latency_specialized"}' max-num-seqs: 2 - max-cudagraph-capture-size: 12 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 16 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '1' -override_c2: +override_tp8_c2: roles: agg: args: + additional-config: '{"decode_backend":"latency_specialized"}' max-num-seqs: 4 - max-cudagraph-capture-size: 24 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 32 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '2' -override_c4: +override_dcp8_c4: roles: agg: args: + additional-config: '{"decode_backend":"latency_specialized"}' + decode-context-parallel-size: 8 max-num-seqs: 8 - max-cudagraph-capture-size: 48 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 64 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '4' -override_c8: +override_dcp8_c8: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 16 - max-cudagraph-capture-size: 96 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 128 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '8' -override_c16: +override_dcp8_c16: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 32 - max-cudagraph-capture-size: 160 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 224 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":6,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '16' -override_c24: +override_dcp8_c24: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 48 - max-cudagraph-capture-size: 240 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 288 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '24' -override_c32: +override_dcp8_c32: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 64 - max-cudagraph-capture-size: 256 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 384 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '32' -override_c40: +override_dcp8_c40: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 80 - max-cudagraph-capture-size: 320 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 480 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '40' -override_c48: +override_dcp8_c48: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 96 - max-cudagraph-capture-size: 384 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 480 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '48' -override_c56: +override_dcp8_c56: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 112 - max-cudagraph-capture-size: 336 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 560 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '56' -override_c70: +override_dcp8_c70: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 140 - max-cudagraph-capture-size: 420 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 704 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '70' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index d486c9f442..521117a5f3 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5957,8 +5957,9 @@ kimik3.1-fp4-b300-vllm-agentic-dspark: # 0.75 resolves to 2249 GB of host DRAM, which the recipe's Simple CPU offload splits across the 8 TP ranks. - dram-utilization: 0.75 search-space: - # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. DSpark drafts 5 tokens up to conc 8, then 4, 3 and 2. - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: agentic.yaml } + # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. DSpark drafts 7 tokens up to conc 8, then 6, 5 and 4. + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: agentic.yaml } dsv41flash-fp4-b200-vllm-agentic-dspark: image: vllm/vllm-openai:cu134-nightly-40941f2c4ba96fba0a7e282a5355e89cd74d9ac5@sha256:13e92fcd97268013d13d07fb97d80803227cf0beae0161f8dce95a8200eaafb0 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 18b0f5d5a6..1db54f77c1 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12255,3 +12255,10 @@ - "Prefill CUDA graphs are disabled because the image's default prefill graph faults with an illegal address during long-context replay on H100; decode CUDA graphs are unchanged." - "Serve the pinned checkpoint revision 46a050bec13002ad29ee795c16e61cf4fa3c4a9c offline from the shared H100 Hugging Face cache." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4717 + +- config-keys: + - kimik3.1-fp4-b300-vllm-agentic-dspark + description: + - "Kimi-K3.1-Pro B300 vLLM: enable the recipe's low-latency decode (decode_backend=latency_specialized) at c1-c4, whose decode graphs stay within 64 rows (the image's wider MoE needs 152 resident CTAs, more than a B300's 148 SMs), and run c1-c2 at plain TP8 instead of TP8 x DCP8." + - "Match the SGLang recipe's draft depths (DSpark 7 tokens through c8, 6 at c16, 5 at c24-c40, 4 from c48), size each decode graph cap to max-num-seqs x (1 + drafts), and raise the engine-ready timeout and health check to 7200 s." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4833