From e9a7477a1541bea1a8c031eebc01127f32291367 Mon Sep 17 00:00:00 2001 From: hyukjlee Date: Sat, 7 Nov 2026 20:56:21 +0900 Subject: [PATCH 1/4] perf(amd): Kimi-K3.1 MI355X vLLM agentic: per-conc max-num-seqs, conc 1 DCP1, conc 4 seven draft tokens Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml | 18 +++++++++++------- inferencex-e2e/configs/amd-master.yaml | 4 ++-- inferencex-e2e/perf-changelog.yaml | 10 ++++++++++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml index 045d0cfb49..40185500fc 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -102,6 +102,10 @@ override_gpu_kv_c1: agg: args: max-num-seqs: 1 + # DCP1 at conc 1: no DCP collectives per decode step. + decode-context-parallel-size: null + dcp-comm-backend: null + cp-kv-cache-interleave-size: null compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16]}' additional-config: '{"decode_backend":"latency_specialized","kda_prefill_backend":"fused"}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' @@ -119,8 +123,8 @@ override_dram_c4: agg: args: max-num-seqs: 2 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[6,12,16,24,32,48,64,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[8,16,24,32,48,64,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '4' @@ -131,8 +135,8 @@ override_dram_c10: roles: agg: args: - max-num-seqs: 6 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[6,12,18,24,30,36,48,64,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' + max-num-seqs: 7 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[6,12,18,24,30,36,42,48,64,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: @@ -144,7 +148,7 @@ override_dram_c16: roles: agg: args: - max-num-seqs: 10 + max-num-seqs: 8 compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,64,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: @@ -157,7 +161,7 @@ override_dram_c24: roles: agg: args: - max-num-seqs: 16 + max-num-seqs: 13 compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: @@ -170,7 +174,7 @@ override_dram_c32: roles: agg: args: - max-num-seqs: 24 + max-num-seqs: 20 max-num-batched-tokens: 10240 gpu-memory-utilization: 0.91 compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 02c05727c3..7eee4e0470 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -940,8 +940,8 @@ kimik3.1-fp4-mi355x-vllm-agentic-dspark: # 0.6 resolves to 1799 GB of host DRAM, which the recipe's Simple CPU offload splits across the 8 TP ranks. - dram-utilization: 0.6 search-space: - # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. c1 keeps the KV cache on the GPUs. - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml } + # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. c1 keeps the KV cache on the GPUs, without DCP. + - { tp: 8, ep: 1, dcp-size: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml } - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [4, 10, 16, 24, 32, 48], srt-recipe: agentic.yaml } # MiniMax-M3.1-Flash AgentX on MI355X with SGLang: day-zero image from the SGLang diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3473c978b2..89e8d4df98 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12419,3 +12419,13 @@ - "Add concurrency 56." - "Concurrency 8 to 56: SGLANG_HICACHE_MAMBA_HOST_RATIO 12 -> 10; the image's larger device pools grew the pinned host pools past the node's GTT (hipHostMalloc failed at concurrency 32)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4846 + +- config-keys: + - kimik3.1-fp4-mi355x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Concurrency 1: DCP8 -> DCP1 (decode-context-parallel-size, dcp-comm-backend and cp-kv-cache-interleave-size removed; dcp-size 1)." + - "Concurrency 4: DSpark draft tokens 5 -> 7 (golden AL 3.41 -> 3.69); decode capture sizes 8 and 16." + - "max-num-seqs at concurrency 10/16/24/32: 6/10/16/24 -> 7/8/13/20; concurrency 10 adds capture size 42. Concurrency 48 unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX From 28b46131b49fc36bb18ac85879c0eab9b7f971b4 Mon Sep 17 00:00:00 2001 From: hyukjlee Date: Sat, 7 Nov 2026 20:56:47 +0900 Subject: [PATCH 2/4] perf-changelog: link #4887 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 89e8d4df98..55c387e435 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12428,4 +12428,4 @@ - "Concurrency 1: DCP8 -> DCP1 (decode-context-parallel-size, dcp-comm-backend and cp-kv-cache-interleave-size removed; dcp-size 1)." - "Concurrency 4: DSpark draft tokens 5 -> 7 (golden AL 3.41 -> 3.69); decode capture sizes 8 and 16." - "max-num-seqs at concurrency 10/16/24/32: 6/10/16/24 -> 7/8/13/20; concurrency 10 adds capture size 42. Concurrency 48 unchanged." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4887 From e9a33875c5d7803c3874120e4f1c7d06ec4d9be6 Mon Sep 17 00:00:00 2001 From: hyukjlee Date: Sat, 7 Nov 2026 21:10:48 +0900 Subject: [PATCH 3/4] Drop added comments Co-Authored-By: Claude Opus 5.5 --- .../srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml | 1 - inferencex-e2e/configs/amd-master.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml index 40185500fc..c7c3ea7e5f 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -102,7 +102,6 @@ override_gpu_kv_c1: agg: args: max-num-seqs: 1 - # DCP1 at conc 1: no DCP collectives per decode step. decode-context-parallel-size: null dcp-comm-backend: null cp-kv-cache-interleave-size: null diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 7eee4e0470..2feb17bcb2 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -940,7 +940,7 @@ kimik3.1-fp4-mi355x-vllm-agentic-dspark: # 0.6 resolves to 1799 GB of host DRAM, which the recipe's Simple CPU offload splits across the 8 TP ranks. - dram-utilization: 0.6 search-space: - # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. c1 keeps the KV cache on the GPUs, without DCP. + # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. c1 keeps the KV cache on the GPUs. - { tp: 8, ep: 1, dcp-size: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml } - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [4, 10, 16, 24, 32, 48], srt-recipe: agentic.yaml } From 336f12129e47f5ca5bf6c2888c75227eb96fbb6a Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 8 Nov 2026 11:17:39 +0900 Subject: [PATCH 4/4] docs(changelog): sync Kimi-K3.1 MI355X vLLM AgentX block with the diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The image stays vllm/vllm-openai-rocm:kimi-k31@sha256:5f47cc1004d9a7ac65f8a913725edb20c62c99a07e31cbf4ba21741bd0b4f9b1 (vLLM 0.0.0+mum.14d1ec2336). - Concurrency 1 moves from TP8 x DCP8 to TP8 x DCP1 (master config dcp-size 8 -> 1; the recipe's c1 override drops decode-context-parallel-size, dcp-comm-backend and cp-kv-cache-interleave-size). It keeps GPU-resident KV, 7 draft tokens and the latency_specialized decode backend. Concurrency 4-48 stay TP8 x DCP8 with DRAM offload. - Concurrency 4: DSpark draft tokens 5 -> 7, so throughput runs use golden acceptance length 3.69 instead of 3.41. The decode capture sizes change from 6, 12 to 8, 16 to match max-num-seqs 2 x (1 + 7). - max-num-seqs at concurrency 10/16/24/32 changes from 6/10/16/24 to 7/8/13/20. Concurrency 10 adds capture size 42 (7 x (1 + 5)). Otherwise the draft lengths, capture lists, max-num-batched-tokens and gpu-memory-utilization at these points, and every setting at concurrency 48, are unchanged. - No change to the DSpark draft's precision: Inferact/Kimi-K3.1-Pro-DSpark-preview (revision 6a2ef20a) ships all-BF16 weights with no quantization_config, the speculative config names the draft and sets no quantization, so get_draft_quant_config() returns None and K3DSparkModel loads its layers in BF16 with the target's BF16 embedding and LM head. Its KV cache stays FP8, the same as the target's. DCP1 at concurrency 1 and 7 draft tokens at concurrency 4 do not change these types. - 镜像保持为 vllm/vllm-openai-rocm:kimi-k31@sha256:5f47cc1004d9a7ac65f8a913725edb20c62c99a07e31cbf4ba21741bd0b4f9b1(vLLM 0.0.0+mum.14d1ec2336)。 - 并发 1 从 TP8 x DCP8 改为 TP8 x DCP1(master config 的 dcp-size 8 -> 1;recipe 的 c1 override 去掉 decode-context-parallel-size、dcp-comm-backend 和 cp-kv-cache-interleave-size)。它仍使用 GPU 常驻 KV、7 个草稿 token 和 latency_specialized decode 后端。并发 4-48 仍为 TP8 x DCP8 并使用 DRAM offload。 - 并发 4:DSpark 草稿 token 数 5 -> 7,因此吞吐测试使用 golden acceptance length 3.69 而不是 3.41。decode capture sizes 从 6、12 改为 8、16,以匹配 max-num-seqs 2 x (1 + 7)。 - 并发 10/16/24/32 的 max-num-seqs 从 6/10/16/24 改为 7/8/13/20。并发 10 新增 capture size 42(7 x (1 + 5))。除此之外,这些点的草稿长度、capture 列表、max-num-batched-tokens 和 gpu-memory-utilization,以及并发 48 的所有设置均不变。 - DSpark 草稿精度无变化:Inferact/Kimi-K3.1-Pro-DSpark-preview(revision 6a2ef20a)发布的权重全部为 BF16 且没有 quantization_config,speculative config 指定了草稿且未设置量化,因此 get_draft_quant_config() 返回 None,K3DSparkModel 以 BF16 加载各层,并使用目标模型的 BF16 embedding 和 LM head。其 KV cache 保持 FP8,与目标模型相同。并发 1 的 DCP1 和并发 4 的 7 个草稿 token 都不改变这些类型。 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 119f820e9c..3ec4ff362a 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12446,7 +12446,9 @@ scenario-type: - agentic-coding description: - - "Concurrency 1: DCP8 -> DCP1 (decode-context-parallel-size, dcp-comm-backend and cp-kv-cache-interleave-size removed; dcp-size 1)." - - "Concurrency 4: DSpark draft tokens 5 -> 7 (golden AL 3.41 -> 3.69); decode capture sizes 8 and 16." - - "max-num-seqs at concurrency 10/16/24/32: 6/10/16/24 -> 7/8/13/20; concurrency 10 adds capture size 42. Concurrency 48 unchanged." + - "The image stays vllm/vllm-openai-rocm:kimi-k31@sha256:5f47cc1004d9a7ac65f8a913725edb20c62c99a07e31cbf4ba21741bd0b4f9b1 (vLLM 0.0.0+mum.14d1ec2336)." + - "Concurrency 1 moves from TP8 x DCP8 to TP8 x DCP1 (master config dcp-size 8 -> 1; the recipe's c1 override drops decode-context-parallel-size, dcp-comm-backend and cp-kv-cache-interleave-size). It keeps GPU-resident KV, 7 draft tokens and the latency_specialized decode backend. Concurrency 4-48 stay TP8 x DCP8 with DRAM offload." + - "Concurrency 4: DSpark draft tokens 5 -> 7, so throughput runs use golden acceptance length 3.69 instead of 3.41. The decode capture sizes change from 6, 12 to 8, 16 to match max-num-seqs 2 x (1 + 7)." + - "max-num-seqs at concurrency 10/16/24/32 changes from 6/10/16/24 to 7/8/13/20. Concurrency 10 adds capture size 42 (7 x (1 + 5)). Otherwise the draft lengths, capture lists, max-num-batched-tokens and gpu-memory-utilization at these points, and every setting at concurrency 48, are unchanged." + - "No change to the DSpark draft's precision: Inferact/Kimi-K3.1-Pro-DSpark-preview (revision 6a2ef20a) ships all-BF16 weights with no quantization_config, the speculative config names the draft and sets no quantization, so get_draft_quant_config() returns None and K3DSparkModel loads its layers in BF16 with the target's BF16 embedding and LM head. Its KV cache stays FP8, the same as the target's. DCP1 at concurrency 1 and 7 draft tokens at concurrency 4 do not change these types." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4887