From e0e54a03a1ec34bbdd4bf8b5707e4fd7719486f7 Mon Sep 17 00:00:00 2001 From: zjy0516 Date: Thu, 5 Nov 2026 10:04:36 +0000 Subject: [PATCH 1/5] [K3.1-Pro] vLLM B300: enable low-latency decode, TP8 at c1-c2 Turn on decode_backend=latency_specialized at every point, run c1-c2 at plain TP8 instead of TP8 x DCP8, and raise the c1 and c70 graph caps to 16 and 432 so the full decode step is captured. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0188HSzNcNLyDbocjggjUSvq --- .../kimik3.1/vllm/b300-fp4-mtp/agentic.yaml | 45 ++++++++++++------- inferencex-e2e/configs/nvidia-master.yaml | 3 +- inferencex-e2e/perf-changelog.yaml | 6 +++ 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml index a6ad375bbd..cb95c87f64 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml @@ -26,8 +26,6 @@ base: args: served-model-name: moonshotai/Kimi-K3.1-Pro tensor-parallel-size: 8 - # Shards the MLA latent and indexer-key caches across the TP group: ~7x the KV tokens of plain TP8. - decode-context-parallel-size: 8 trust-remote-code: true language-model-only: true load-format: fastsafetensors @@ -36,6 +34,8 @@ base: reasoning-parser: kimi_k3 kv-cache-dtype: fp8 attention-config: '{"indexer_kv_dtype":"mxfp4"}' + # The recipe's Low-latency decode: decode steps run the fused CuTe DSL path (TP8 or TP8 x DCP8, no EP). + additional-config: '{"decode_backend":"latency_specialized"}' # The K3.1 week-1 precision policy requires an FP32 KDA recurrent state. mamba-ssm-cache-dtype: float32 enable-prefix-caching: true @@ -58,20 +58,22 @@ base: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2249' -# Admission is 2x CONC. The decode graphs cover max-num-seqs x (1 + draft tokens); -# throughput runs replace block rejection with the golden acceptance length. -override_c1: +# Admission is 2x CONC. The largest decode graph covers max-num-seqs x (1 + draft tokens), +# rounded up to a default capture size; throughput runs replace block rejection with the +# golden acceptance length. Plain TP8 at c1-c2, where the KV fits on GPU and DCP's decode +# collectives cost most; DCP8 from c4 for ~7x the KV tokens. +override_tp8_c1: roles: agg: args: max-num-seqs: 2 - max-cudagraph-capture-size: 12 + max-cudagraph-capture-size: 16 speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '1' -override_c2: +override_tp8_c2: roles: agg: args: @@ -82,10 +84,11 @@ override_c2: env: CONC: '2' -override_c4: +override_dcp8_c4: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 8 max-cudagraph-capture-size: 48 speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' @@ -93,10 +96,11 @@ override_c4: env: CONC: '4' -override_c8: +override_dcp8_c8: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 16 max-cudagraph-capture-size: 96 speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' @@ -104,10 +108,11 @@ override_c8: env: CONC: '8' -override_c16: +override_dcp8_c16: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 32 max-cudagraph-capture-size: 160 speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' @@ -115,10 +120,11 @@ override_c16: env: CONC: '16' -override_c24: +override_dcp8_c24: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 48 max-cudagraph-capture-size: 240 speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' @@ -126,10 +132,11 @@ override_c24: env: CONC: '24' -override_c32: +override_dcp8_c32: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 64 max-cudagraph-capture-size: 256 speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' @@ -137,10 +144,11 @@ override_c32: env: CONC: '32' -override_c40: +override_dcp8_c40: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 80 max-cudagraph-capture-size: 320 speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' @@ -148,10 +156,11 @@ override_c40: env: CONC: '40' -override_c48: +override_dcp8_c48: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 96 max-cudagraph-capture-size: 384 speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' @@ -159,10 +168,11 @@ override_c48: env: CONC: '48' -override_c56: +override_dcp8_c56: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 112 max-cudagraph-capture-size: 336 speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' @@ -170,12 +180,13 @@ override_c56: env: CONC: '56' -override_c70: +override_dcp8_c70: roles: agg: args: + decode-context-parallel-size: 8 max-num-seqs: 140 - max-cudagraph-capture-size: 420 + max-cudagraph-capture-size: 432 speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 4a781453e7..29af1c328c 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5447,7 +5447,8 @@ kimik3.1-fp4-b300-vllm-agentic-dspark: - dram-utilization: 0.75 search-space: # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. DSpark drafts 5 tokens up to conc 8, then 4, 3 and 2. - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: agentic.yaml } dsv41flash-fp4-b200-vllm-agentic-dspark: image: vllm/vllm-openai:v0.32.0@sha256:befccf0e7e4089aa23926bb6f887a85c0d46c2a99e6de5c5ffdf5cb7957c5b34 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 025645e525..87c9eaea7d 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12043,3 +12043,9 @@ - "Add Kimi-K3.1-Pro MXFP4 AgentX on one 8x MI355X node with SGLang (lmsysorg/sglang:dev-kimi-k31-pro-mi35x) and the DFlash2 draft at 5 draft tokens with block verification." - "TP8 monokernel at concurrency 1 and 4 with GPU-resident KV; TP8/DCP8 with the HiCache host tier at concurrency 8, 16, 32 and 48." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4808 + +- config-keys: + - kimik3.1-fp4-b300-vllm-agentic-dspark + description: + - "Kimi-K3.1-Pro B300 vLLM: enable the recipe's low-latency decode (decode_backend=latency_specialized) at every point, run c1-c2 at plain TP8 instead of TP8 x DCP8, and raise the c1 and c70 graph caps to 16 and 432 so the full decode step is captured." + pr-link: TBD From 15015e4ed444e18f93813763a250d254413bbb89 Mon Sep 17 00:00:00 2001 From: zjy0516 Date: Thu, 5 Nov 2026 11:33:58 +0000 Subject: [PATCH 2/5] [K3.1-Pro] vLLM B300: raise engine-ready timeout and health check to 7200 s The latency_specialized decode path adds a second FlashInfer autotune pass at startup; on a local GB300 run it took ~40 min to become ready. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0188HSzNcNLyDbocjggjUSvq --- .../kimik3.1/vllm/b300-fp4-mtp/agentic.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml index cb95c87f64..526289d99f 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml @@ -14,10 +14,11 @@ base: engine: type: vllm connector: null - # ~1.56 TB of MXFP4 shards may take the full VLLM_ENGINE_READY_TIMEOUT_S. + # Loading ~1.56 TB of MXFP4 shards plus the latency_specialized decode autotune can take + # most of VLLM_ENGINE_READY_TIMEOUT_S. health_check: interval_seconds: 10 - max_attempts: 360 + max_attempts: 720 roles: agg: nodes: 1 @@ -46,7 +47,7 @@ base: # TOTAL_CPU_DRAM_GB split across the 8 TP ranks. kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":281125000000,"lazy_offload":false}}' env: - VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_ENGINE_READY_TIMEOUT_S: '7200' PYTHONUNBUFFERED: '1' # Identical prefixes must hash to identical block keys run-to-run. PYTHONHASHSEED: '42' From 55582cd5ac2375e9da76b504c0e3ab3d42d7ad08 Mon Sep 17 00:00:00 2001 From: zjy0516 Date: Thu, 5 Nov 2026 14:55:27 +0000 Subject: [PATCH 3/5] [K3.1-Pro] vLLM B300: match SGLang's draft depths DSpark now drafts 7 tokens through c8, 6 at c16, 5 at c24-c40 and 4 from c48, the same draft counts as the SGLang DFLASH recipe, and each decode graph cap covers max-num-seqs x (1 + drafts). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0188HSzNcNLyDbocjggjUSvq --- .../kimik3.1/vllm/b300-fp4-mtp/agentic.yaml | 45 ++++++++++--------- inferencex-e2e/configs/nvidia-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 3 +- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml index 526289d99f..f08baa8b1a 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml @@ -59,7 +59,8 @@ base: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2249' -# Admission is 2x CONC. The largest decode graph covers max-num-seqs x (1 + draft tokens), +# Admission is 2x CONC. DSpark drafts match the SGLang recipe's DFLASH drafts: 7 through c8, +# 6 at c16, 5 at c24-c40 and 4 from c48. The largest decode graph covers max-num-seqs x (1 + drafts), # rounded up to a default capture size; throughput runs replace block rejection with the # golden acceptance length. Plain TP8 at c1-c2, where the KV fits on GPU and DCP's decode # collectives cost most; DCP8 from c4 for ~7x the KV tokens. @@ -69,7 +70,7 @@ override_tp8_c1: args: max-num-seqs: 2 max-cudagraph-capture-size: 16 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '1' @@ -79,8 +80,8 @@ override_tp8_c2: agg: args: max-num-seqs: 4 - max-cudagraph-capture-size: 24 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 32 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '2' @@ -91,8 +92,8 @@ override_dcp8_c4: args: decode-context-parallel-size: 8 max-num-seqs: 8 - max-cudagraph-capture-size: 48 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 64 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '4' @@ -103,8 +104,8 @@ override_dcp8_c8: args: decode-context-parallel-size: 8 max-num-seqs: 16 - max-cudagraph-capture-size: 96 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 128 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '8' @@ -115,8 +116,8 @@ override_dcp8_c16: args: decode-context-parallel-size: 8 max-num-seqs: 32 - max-cudagraph-capture-size: 160 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 224 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":6,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '16' @@ -127,8 +128,8 @@ override_dcp8_c24: args: decode-context-parallel-size: 8 max-num-seqs: 48 - max-cudagraph-capture-size: 240 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 288 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '24' @@ -139,8 +140,8 @@ override_dcp8_c32: args: decode-context-parallel-size: 8 max-num-seqs: 64 - max-cudagraph-capture-size: 256 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 384 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '32' @@ -151,8 +152,8 @@ override_dcp8_c40: args: decode-context-parallel-size: 8 max-num-seqs: 80 - max-cudagraph-capture-size: 320 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 480 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '40' @@ -163,8 +164,8 @@ override_dcp8_c48: args: decode-context-parallel-size: 8 max-num-seqs: 96 - max-cudagraph-capture-size: 384 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 480 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '48' @@ -175,8 +176,8 @@ override_dcp8_c56: args: decode-context-parallel-size: 8 max-num-seqs: 112 - max-cudagraph-capture-size: 336 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 560 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '56' @@ -187,8 +188,8 @@ override_dcp8_c70: args: decode-context-parallel-size: 8 max-num-seqs: 140 - max-cudagraph-capture-size: 432 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 704 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '70' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 29af1c328c..673f5f435c 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5446,7 +5446,7 @@ kimik3.1-fp4-b300-vllm-agentic-dspark: # 0.75 resolves to 2249 GB of host DRAM, which the recipe's Simple CPU offload splits across the 8 TP ranks. - dram-utilization: 0.75 search-space: - # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. DSpark drafts 5 tokens up to conc 8, then 4, 3 and 2. + # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. DSpark drafts 7 tokens up to conc 8, then 6, 5 and 4. - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2], srt-recipe: agentic.yaml } - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: agentic.yaml } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 87c9eaea7d..ebfdb77a1a 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12047,5 +12047,6 @@ - config-keys: - kimik3.1-fp4-b300-vllm-agentic-dspark description: - - "Kimi-K3.1-Pro B300 vLLM: enable the recipe's low-latency decode (decode_backend=latency_specialized) at every point, run c1-c2 at plain TP8 instead of TP8 x DCP8, and raise the c1 and c70 graph caps to 16 and 432 so the full decode step is captured." + - "Kimi-K3.1-Pro B300 vLLM: enable the recipe's low-latency decode (decode_backend=latency_specialized) at every point and run c1-c2 at plain TP8 instead of TP8 x DCP8." + - "Match the SGLang recipe's draft depths (DSpark 7 tokens through c8, 6 at c16, 5 at c24-c40, 4 from c48), size each decode graph cap to max-num-seqs x (1 + drafts), and raise the engine-ready timeout and health check to 7200 s." pr-link: TBD From 05a295fc96c221ca244d36f82dec0ce3a5c5a73a Mon Sep 17 00:00:00 2001 From: zjy0516 Date: Thu, 5 Nov 2026 16:07:29 +0000 Subject: [PATCH 4/5] chore(kimik3.1): link the B300 vLLM changelog entry to its PR Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0188HSzNcNLyDbocjggjUSvq --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index ebfdb77a1a..9ef6ca463e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12049,4 +12049,4 @@ description: - "Kimi-K3.1-Pro B300 vLLM: enable the recipe's low-latency decode (decode_backend=latency_specialized) at every point and run c1-c2 at plain TP8 instead of TP8 x DCP8." - "Match the SGLang recipe's draft depths (DSpark 7 tokens through c8, 6 at c16, 5 at c24-c40, 4 from c48), size each decode graph cap to max-num-seqs x (1 + drafts), and raise the engine-ready timeout and health check to 7200 s." - pr-link: TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4833 From 947a5e905f231580d40cd602dae7c8645745ee0e Mon Sep 17 00:00:00 2001 From: zjy0516 Date: Fri, 6 Nov 2026 02:23:41 +0000 Subject: [PATCH 5/5] [K3.1-Pro] vLLM B300: run low-latency decode only at c1-c4 The kimi-k31 image launches the wide (65-320 row) MoE experts on 152 CTAs that must all be co-resident; a B300 has 148 SMs, so every point whose decode graphs exceed 64 rows (c8-c70) failed engine init. Keep latency_specialized where graphs stay within 64 rows and run the stock decode above. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0188HSzNcNLyDbocjggjUSvq --- .../kimik3.1/vllm/b300-fp4-mtp/agentic.yaml | 8 ++++++-- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml index f08baa8b1a..21413562a5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml @@ -35,8 +35,6 @@ base: reasoning-parser: kimi_k3 kv-cache-dtype: fp8 attention-config: '{"indexer_kv_dtype":"mxfp4"}' - # The recipe's Low-latency decode: decode steps run the fused CuTe DSL path (TP8 or TP8 x DCP8, no EP). - additional-config: '{"decode_backend":"latency_specialized"}' # The K3.1 week-1 precision policy requires an FP32 KDA recurrent state. mamba-ssm-cache-dtype: float32 enable-prefix-caching: true @@ -64,10 +62,14 @@ base: # rounded up to a default capture size; throughput runs replace block rejection with the # golden acceptance length. Plain TP8 at c1-c2, where the KV fits on GPU and DCP's decode # collectives cost most; DCP8 from c4 for ~7x the KV tokens. +# Low-latency decode (the fused CuTe DSL path) runs only where every decode graph has at +# most 64 rows (c1-c4): this image launches its 65-320-row MoE experts on 152 CTAs that +# must all be resident, which a 148-SM B300 cannot hold. override_tp8_c1: roles: agg: args: + additional-config: '{"decode_backend":"latency_specialized"}' max-num-seqs: 2 max-cudagraph-capture-size: 16 speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' @@ -79,6 +81,7 @@ override_tp8_c2: roles: agg: args: + additional-config: '{"decode_backend":"latency_specialized"}' max-num-seqs: 4 max-cudagraph-capture-size: 32 speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' @@ -90,6 +93,7 @@ override_dcp8_c4: roles: agg: args: + additional-config: '{"decode_backend":"latency_specialized"}' decode-context-parallel-size: 8 max-num-seqs: 8 max-cudagraph-capture-size: 64 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index edf2097b5e..e3349ee5c4 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12228,6 +12228,6 @@ - config-keys: - kimik3.1-fp4-b300-vllm-agentic-dspark description: - - "Kimi-K3.1-Pro B300 vLLM: enable the recipe's low-latency decode (decode_backend=latency_specialized) at every point and run c1-c2 at plain TP8 instead of TP8 x DCP8." + - "Kimi-K3.1-Pro B300 vLLM: enable the recipe's low-latency decode (decode_backend=latency_specialized) at c1-c4, whose decode graphs stay within 64 rows (the image's wider MoE needs 152 resident CTAs, more than a B300's 148 SMs), and run c1-c2 at plain TP8 instead of TP8 x DCP8." - "Match the SGLang recipe's draft depths (DSpark 7 tokens through c8, 6 at c16, 5 at c24-c40, 4 from c48), size each decode graph cap to max-num-seqs x (1 + drafts), and raise the engine-ready timeout and health check to 7200 s." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4833