From 3bec727312286b19953119dc5fa091124d71f908 Mon Sep 17 00:00:00 2001 From: hyukjlee Date: Fri, 13 Nov 2026 01:14:22 +0900 Subject: [PATCH 01/10] perf(amd): Kimi-K3.1 MI355X vLLM AgentX: image kimi-k31-2ded1ca, env flags, c1 nst 7/5, c4 GPU KV DCP8 latency_specialized, c48 nst 3 Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml | 52 ++++++++++++------- inferencex-e2e/configs/amd-master.yaml | 11 ++-- inferencex-e2e/perf-changelog.yaml | 14 +++++ 3 files changed, 55 insertions(+), 22 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml index 33b08cc274..5f87ccb21d 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -61,12 +61,17 @@ base: VLLM_USE_V2_MODEL_RUNNER: '1' VLLM_FASTSAFETENSORS_QUEUE_SIZE: '-1' # K3.1 has no torch.compile submodule, so only breakable graphs capture the prefill - # sizes the throughput points list. The conc-1 override turns this off. + # sizes the throughput points list. The conc-1 and conc-4 overrides turn this off. VLLM_USE_BREAKABLE_CUDAGRAPH: '1' VLLM_ROCM_AITER_MLA_ASM_PADDING: asm # Verify the draft tokens with the AITER assembly DCP kernel. VLLM_ROCM_AITER_MLA_DCP_VERIFY: asm VLLM_ROCM_DCP_AITER_ALLGATHER: '1' + # DCP V-up reduce-scatter; needs VLLM_ROCM_DCP_AITER_ALLGATHER. + VLLM_ROCM_DCP_VUP_RS: '1' + VLLM_ROCM_DSA_MULTISTREAM: '1' + # VRAM reserve for unprofiled DCP prefill transients. + VLLM_ROCM_MEM_HEADROOM_GIB: '8' VLLM_ROCM_USE_AITER_FLYDSL_SPARSE_MLA: '1' # AITER reads its own variable for the same kernels. AITER_FLYDSL_SPARSE_MLA: '1' @@ -94,8 +99,9 @@ base: AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' -# max-num-seqs stays below CONC from c4 to c32, and equals it at c1 and c48. The capture list is -# the decode sizes, multiples of one plus the draft token count, then the prefill ladder. +# max-num-seqs stays below CONC from c4 to c32, and equals it at c1 and c48. c1 and c4 capture +# decode sizes only; the others list decode sizes, multiples of one plus the draft token count, +# then the prefill ladder. # Throughput runs replace block rejection with the golden acceptance length. override_gpu_kv_c1: roles: @@ -117,31 +123,41 @@ override_gpu_kv_c1: CONC: '1' KV_OFFLOADING: 'none' -override_dram_c4: +override_gpu_kv_c1_nst5: roles: agg: args: - max-num-seqs: 2 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[8,16,24,32,48,64,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-num-seqs: 1 + decode-context-parallel-size: null + dcp-comm-backend: null + cp-kv-cache-interleave-size: null + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":6,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[6]}' + additional-config: '{"decode_backend":"latency_specialized","kda_prefill_backend":"fused"}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: null + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' benchmark: env: - CONC: '4' - KV_OFFLOADING: 'dram' - TOTAL_CPU_DRAM_GB: '1799' + CONC: '1' + KV_OFFLOADING: 'none' -override_dram_c10: +override_gpu_kv_c4: roles: agg: args: - max-num-seqs: 7 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[6,12,18,24,30,36,42,48,64,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' + max-num-seqs: 2 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":12,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[6,12]}' + additional-config: '{"decode_backend":"latency_specialized","kda_prefill_backend":"fused"}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: null + env: + # Decode-only capture sizes, as at conc 1. + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' benchmark: env: - CONC: '10' - KV_OFFLOADING: 'dram' - TOTAL_CPU_DRAM_GB: '1799' + CONC: '4' + KV_OFFLOADING: 'none' override_dram_c16: roles: @@ -199,8 +215,8 @@ override_dram_c48: prefill-schedule-interval: 8 long-prefill-token-threshold: 10240 long-prefill-token-threshold-adaptive: true - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224,231,238,245,252,259,266,273,280,287,294,301,308,315,322,329,336,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":6,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '48' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 5de9147e7f..e36571447d 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -974,7 +974,7 @@ minimaxm31flash-fp4-mi355x-vllm-agentic-dspark: - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [64], srt-recipe: agentic.yaml:override_tp4_dspark3_c64_offload } kimik3.1-fp4-mi355x-vllm-agentic-dspark: - image: vllm/vllm-openai-rocm:kimi-k31-d73dd529@sha256:5b54f223a150a9baad41a5d511e159e7cbf43e308c6cd44ad3f6a2463c245b3c + image: vllm/vllm-openai-rocm:kimi-k31-2ded1ca@sha256:5661b15c5fcfddaf48a7cdc7d03aa31f4146013b0ea7cb38c5538ec1a8e7a46f model: moonshotai/Kimi-K3.1-Pro model-prefix: kimik3.1 runner: cluster:mi355x-amds @@ -987,9 +987,12 @@ kimik3.1-fp4-mi355x-vllm-agentic-dspark: # 0.6 resolves to 1799 GB of host DRAM, which the recipe's Simple CPU offload splits across the 8 TP ranks. - dram-utilization: 0.6 search-space: - # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. c1 keeps the KV cache on the GPUs. - - { tp: 8, ep: 1, dcp-size: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [4, 16, 24, 32, 48], srt-recipe: agentic.yaml } + # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. c1 and c4 keep the KV cache on the GPUs. + # c1 runs twice, with 7 and with 5 draft tokens; each row names its override. + - { tp: 8, ep: 1, dcp-size: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml:override_gpu_kv_c1 } + - { tp: 8, ep: 1, dcp-size: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml:override_gpu_kv_c1_nst5 } + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [16, 24, 32, 48], srt-recipe: agentic.yaml } # MiniMax-M3.1-Flash AgentX on MI355X with SGLang: day-zero image from the SGLang # cookbook (https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1), diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 671d2b377a..d25220f434 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12785,3 +12785,17 @@ - "Concurrency 12 and 24: CUDA graphs also for prefill and mixed steps up to 4096 tokens (#4878 recipe without the TEP shared-expert variable); DSpark drafts 3 tokens; prefill-schedule-interval 8." - "Concurrency 48 and 64: EP8 MegaMoE with FULL_DECODE_ONLY graphs, max-num-batched-tokens 16384, prefill-schedule-interval 8 and gpu-memory-utilization 0.97." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4871 + +- config-keys: + - kimik3.1-fp4-mi355x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from vllm/vllm-openai-rocm:kimi-k31-d73dd529 (sha256:5b54f223) to vllm/vllm-openai-rocm:kimi-k31-2ded1ca (sha256:5661b15c), vLLM mum-tracking 2ded1caaf9, which includes a KV null-block fix." + - "Every point sets VLLM_ROCM_DCP_VUP_RS=1 (DCP V-up reduce-scatter, on top of the existing VLLM_ROCM_DCP_AITER_ALLGATHER=1), VLLM_ROCM_DSA_MULTISTREAM=1 and VLLM_ROCM_MEM_HEADROOM_GIB=8." + - "Concurrency 1 runs two variants, both TP8 x DCP1 with GPU-resident KV, the latency_specialized decode backend and breakable CUDA graphs off: override_gpu_kv_c1 keeps 7 draft tokens (golden AL 3.69) and capture sizes 2-16; the new override_gpu_kv_c1_nst5 uses 5 draft tokens (golden AL 3.41) with capture size 6. Each master-config row selects its override." + - "Concurrency 4 drops DRAM offload (kv-offloading none, override_gpu_kv_c4) and stays TP8 x DCP8 with max-num-seqs 2; it switches to the latency_specialized decode backend, 5 draft tokens (golden AL 3.41, from 7), decode-only capture sizes 6 and 12 and breakable CUDA graphs off." + - "Concurrency 48 uses 3 draft tokens (golden AL 2.88, from 6) with decode capture sizes 4-192." + - "Remove override_dram_c10, unused since concurrency 10 left the sweep." + - "No change to the DSpark draft's precision: Inferact/Kimi-K3.1-Pro-DSpark-preview (revision 6a2ef20a) still loads in BF16 with no quantization and an FP8 KV cache; the new draft lengths do not change these types." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/5043 From 63c89f414c193a0c7f2874465175eece528778e3 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 13 Nov 2026 09:34:06 +0900 Subject: [PATCH 02/10] Kimi-K3.1 MI355X vLLM AgentX c1: VLLM_KIMI_SD_DSA_PROLOGUE=1 Both concurrency-1 overrides (override_gpu_kv_c1, override_gpu_kv_c1_nst5) run each DSA layer's specialized-decode prologue as one launch (mum PR 3, D1a). It needs DCP1, so the DCP8 c4 override is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml | 3 +++ inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml index 5f87ccb21d..16d4764205 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -118,6 +118,8 @@ override_gpu_kv_c1: env: # This point captures only the decode sizes, so breakable graphs buy nothing. VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + # Runs each DSA layer's decode prologue as one launch; needs DCP1. + VLLM_KIMI_SD_DSA_PROLOGUE: '1' benchmark: env: CONC: '1' @@ -137,6 +139,7 @@ override_gpu_kv_c1_nst5: kv-transfer-config: null env: VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + VLLM_KIMI_SD_DSA_PROLOGUE: '1' benchmark: env: CONC: '1' diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 2234553409..52f669b590 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12804,7 +12804,7 @@ description: - "Update the image from vllm/vllm-openai-rocm:kimi-k31-d73dd529 (sha256:5b54f223) to vllm/vllm-openai-rocm:kimi-k31-2ded1ca (sha256:5661b15c), vLLM mum-tracking 2ded1caaf9, which includes a KV null-block fix." - "Every point sets VLLM_ROCM_DCP_VUP_RS=1 (DCP V-up reduce-scatter, on top of the existing VLLM_ROCM_DCP_AITER_ALLGATHER=1), VLLM_ROCM_DSA_MULTISTREAM=1 and VLLM_ROCM_MEM_HEADROOM_GIB=8." - - "Concurrency 1 runs two variants, both TP8 x DCP1 with GPU-resident KV, the latency_specialized decode backend and breakable CUDA graphs off: override_gpu_kv_c1 keeps 7 draft tokens (golden AL 3.69) and capture sizes 2-16; the new override_gpu_kv_c1_nst5 uses 5 draft tokens (golden AL 3.41) with capture size 6. Each master-config row selects its override." + - "Concurrency 1 runs two variants, both TP8 x DCP1 with GPU-resident KV, the latency_specialized decode backend, VLLM_KIMI_SD_DSA_PROLOGUE=1 (each DSA layer's decode prologue in one launch) and breakable CUDA graphs off: override_gpu_kv_c1 keeps 7 draft tokens (golden AL 3.69) and capture sizes 2-16; the new override_gpu_kv_c1_nst5 uses 5 draft tokens (golden AL 3.41) with capture size 6. Each master-config row selects its override." - "Concurrency 4 drops DRAM offload (kv-offloading none, override_gpu_kv_c4) and stays TP8 x DCP8 with max-num-seqs 2; it switches to the latency_specialized decode backend, 5 draft tokens (golden AL 3.41, from 7), decode-only capture sizes 6 and 12 and breakable CUDA graphs off." - "Concurrency 48 uses 3 draft tokens (golden AL 2.88, from 6) with decode capture sizes 4-192." - "Remove override_dram_c10, unused since concurrency 10 left the sweep." From 393d5f6b1b7fbcc25f95834266bf6ad34210732f Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 13 Nov 2026 09:37:26 +0900 Subject: [PATCH 03/10] Kimi-K3.1 MI355X vLLM AgentX: drop the c1 five-draft-token variant Remove override_gpu_kv_c1_nst5 and its master-config row; c1 runs once with override_gpu_kv_c1 (7 draft tokens), selected by CONC as on main. Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml | 20 ------------------- inferencex-e2e/configs/amd-master.yaml | 4 +--- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 2 insertions(+), 24 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml index 16d4764205..b381b6861d 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -125,26 +125,6 @@ override_gpu_kv_c1: CONC: '1' KV_OFFLOADING: 'none' -override_gpu_kv_c1_nst5: - roles: - agg: - args: - max-num-seqs: 1 - decode-context-parallel-size: null - dcp-comm-backend: null - cp-kv-cache-interleave-size: null - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":6,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[6]}' - additional-config: '{"decode_backend":"latency_specialized","kda_prefill_backend":"fused"}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - kv-transfer-config: null - env: - VLLM_USE_BREAKABLE_CUDAGRAPH: '0' - VLLM_KIMI_SD_DSA_PROLOGUE: '1' - benchmark: - env: - CONC: '1' - KV_OFFLOADING: 'none' - override_gpu_kv_c4: roles: agg: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 93d1c89a98..f10972e63b 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -988,9 +988,7 @@ kimik3.1-fp4-mi355x-vllm-agentic-dspark: - dram-utilization: 0.6 search-space: # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. c1 and c4 keep the KV cache on the GPUs. - # c1 runs twice, with 7 and with 5 draft tokens; each row names its override. - - { tp: 8, ep: 1, dcp-size: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml:override_gpu_kv_c1 } - - { tp: 8, ep: 1, dcp-size: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml:override_gpu_kv_c1_nst5 } + - { tp: 8, ep: 1, dcp-size: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml } - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4], srt-recipe: agentic.yaml } - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [16, 24, 32, 48], srt-recipe: agentic.yaml } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 52f669b590..55ddfa888a 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12804,7 +12804,7 @@ description: - "Update the image from vllm/vllm-openai-rocm:kimi-k31-d73dd529 (sha256:5b54f223) to vllm/vllm-openai-rocm:kimi-k31-2ded1ca (sha256:5661b15c), vLLM mum-tracking 2ded1caaf9, which includes a KV null-block fix." - "Every point sets VLLM_ROCM_DCP_VUP_RS=1 (DCP V-up reduce-scatter, on top of the existing VLLM_ROCM_DCP_AITER_ALLGATHER=1), VLLM_ROCM_DSA_MULTISTREAM=1 and VLLM_ROCM_MEM_HEADROOM_GIB=8." - - "Concurrency 1 runs two variants, both TP8 x DCP1 with GPU-resident KV, the latency_specialized decode backend, VLLM_KIMI_SD_DSA_PROLOGUE=1 (each DSA layer's decode prologue in one launch) and breakable CUDA graphs off: override_gpu_kv_c1 keeps 7 draft tokens (golden AL 3.69) and capture sizes 2-16; the new override_gpu_kv_c1_nst5 uses 5 draft tokens (golden AL 3.41) with capture size 6. Each master-config row selects its override." + - "Concurrency 1 (TP8 x DCP1, latency_specialized decode backend) sets VLLM_KIMI_SD_DSA_PROLOGUE=1, which runs each DSA layer's decode prologue in one launch." - "Concurrency 4 drops DRAM offload (kv-offloading none, override_gpu_kv_c4) and stays TP8 x DCP8 with max-num-seqs 2; it switches to the latency_specialized decode backend, 5 draft tokens (golden AL 3.41, from 7), decode-only capture sizes 6 and 12 and breakable CUDA graphs off." - "Concurrency 48 uses 3 draft tokens (golden AL 2.88, from 6) with decode capture sizes 4-192." - "Remove override_dram_c10, unused since concurrency 10 left the sweep." From 326296985072c2d7f20fc2f639a11e7757757b5b Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 13 Nov 2026 16:50:55 +0900 Subject: [PATCH 04/10] Kimi-K3.1 MI355X vLLM AgentX: image kimi-k31-6d74a15, c16-c32 prefill cadence Image kimi-k31-6d74a15 (mum-tracking 6d74a152f) adds VLLM_PREFILL_CADENCE_NO_ESCAPE (mum PR 21). c16, c24 and c32 set prefill-schedule-interval 32 with the flag on, since their max-num-seqs is below the concurrency and the capacity escape would drop the hold. Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml | 14 ++++++++++++-- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 3 ++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml index b381b6861d..7173a180b1 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -147,8 +147,14 @@ override_dram_c16: agg: args: max-num-seqs: 8 + # Admit prefill only every 32nd engine step so decode runs in longer uninterrupted stretches. + prefill-schedule-interval: 32 compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,64,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + env: + # max-num-seqs is below CONC, so requests always wait and vLLM would otherwise + # drop the prefill hold (capacity escape); keep it. + VLLM_PREFILL_CADENCE_NO_ESCAPE: '1' benchmark: env: CONC: '16' @@ -160,8 +166,11 @@ override_dram_c24: agg: args: max-num-seqs: 13 + prefill-schedule-interval: 32 compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + env: + VLLM_PREFILL_CADENCE_NO_ESCAPE: '1' benchmark: env: CONC: '24' @@ -175,12 +184,13 @@ override_dram_c32: max-num-seqs: 20 max-num-batched-tokens: 10240 gpu-memory-utilization: 0.91 - # Admit prefill only every 8th engine step so decode runs in longer uninterrupted stretches. - prefill-schedule-interval: 8 + prefill-schedule-interval: 32 long-prefill-token-threshold: 10240 long-prefill-token-threshold-adaptive: true compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + env: + VLLM_PREFILL_CADENCE_NO_ESCAPE: '1' benchmark: env: CONC: '32' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index f10972e63b..943a3748cf 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -974,7 +974,7 @@ minimaxm31flash-fp4-mi355x-vllm-agentic-dspark: - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [32, 64], srt-recipe: agentic.yaml } kimik3.1-fp4-mi355x-vllm-agentic-dspark: - image: vllm/vllm-openai-rocm:kimi-k31-2ded1ca@sha256:5661b15c5fcfddaf48a7cdc7d03aa31f4146013b0ea7cb38c5538ec1a8e7a46f + image: vllm/vllm-openai-rocm:kimi-k31-6d74a15@sha256:c470d0de88e8bb664a07006d44e6603a413c9fffd36cc4add4d4b420d4bde5e0 model: moonshotai/Kimi-K3.1-Pro model-prefix: kimik3.1 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 55ddfa888a..9bf6d07f97 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12802,11 +12802,12 @@ scenario-type: - agentic-coding description: - - "Update the image from vllm/vllm-openai-rocm:kimi-k31-d73dd529 (sha256:5b54f223) to vllm/vllm-openai-rocm:kimi-k31-2ded1ca (sha256:5661b15c), vLLM mum-tracking 2ded1caaf9, which includes a KV null-block fix." + - "Update the image from vllm/vllm-openai-rocm:kimi-k31-d73dd529 (sha256:5b54f223) to vllm/vllm-openai-rocm:kimi-k31-6d74a15 (sha256:c470d0de), vLLM mum-tracking 6d74a152f, which includes a KV null-block fix and VLLM_PREFILL_CADENCE_NO_ESCAPE." - "Every point sets VLLM_ROCM_DCP_VUP_RS=1 (DCP V-up reduce-scatter, on top of the existing VLLM_ROCM_DCP_AITER_ALLGATHER=1), VLLM_ROCM_DSA_MULTISTREAM=1 and VLLM_ROCM_MEM_HEADROOM_GIB=8." - "Concurrency 1 (TP8 x DCP1, latency_specialized decode backend) sets VLLM_KIMI_SD_DSA_PROLOGUE=1, which runs each DSA layer's decode prologue in one launch." - "Concurrency 4 drops DRAM offload (kv-offloading none, override_gpu_kv_c4) and stays TP8 x DCP8 with max-num-seqs 2; it switches to the latency_specialized decode backend, 5 draft tokens (golden AL 3.41, from 7), decode-only capture sizes 6 and 12 and breakable CUDA graphs off." - "Concurrency 48 uses 3 draft tokens (golden AL 2.88, from 6) with decode capture sizes 4-192." + - "Concurrency 16, 24 and 32 set prefill-schedule-interval 32 (from 1, 1 and 8) and VLLM_PREFILL_CADENCE_NO_ESCAPE=1. Their max-num-seqs is below the concurrency, so requests always wait; without the flag vLLM drops the prefill hold (capacity escape)." - "Remove override_dram_c10, unused since concurrency 10 left the sweep." - "No change to the DSpark draft's precision: Inferact/Kimi-K3.1-Pro-DSpark-preview (revision 6a2ef20a) still loads in BF16 with no quantization and an FP8 KV cache; the new draft lengths do not change these types." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/5043 From 579bd80c782700962ba49792b1a812525a375f5a Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 13 Nov 2026 17:04:45 +0900 Subject: [PATCH 05/10] Kimi-K3.1 MI355X vLLM AgentX c48: drop the prefill cadence and long-prefill threshold c48 no longer sets prefill-schedule-interval 8, long-prefill-token-threshold 10240 or long-prefill-token-threshold-adaptive; it uses vLLM's defaults. Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml | 4 ---- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml index 7173a180b1..10af14a32f 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -204,10 +204,6 @@ override_dram_c48: max-num-seqs: 48 max-num-batched-tokens: 10240 gpu-memory-utilization: 0.91 - # Admit prefill only every 8th engine step so decode runs in longer uninterrupted stretches. - prefill-schedule-interval: 8 - long-prefill-token-threshold: 10240 - long-prefill-token-threshold-adaptive: true compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 9bf6d07f97..03318528d7 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12806,7 +12806,7 @@ - "Every point sets VLLM_ROCM_DCP_VUP_RS=1 (DCP V-up reduce-scatter, on top of the existing VLLM_ROCM_DCP_AITER_ALLGATHER=1), VLLM_ROCM_DSA_MULTISTREAM=1 and VLLM_ROCM_MEM_HEADROOM_GIB=8." - "Concurrency 1 (TP8 x DCP1, latency_specialized decode backend) sets VLLM_KIMI_SD_DSA_PROLOGUE=1, which runs each DSA layer's decode prologue in one launch." - "Concurrency 4 drops DRAM offload (kv-offloading none, override_gpu_kv_c4) and stays TP8 x DCP8 with max-num-seqs 2; it switches to the latency_specialized decode backend, 5 draft tokens (golden AL 3.41, from 7), decode-only capture sizes 6 and 12 and breakable CUDA graphs off." - - "Concurrency 48 uses 3 draft tokens (golden AL 2.88, from 6) with decode capture sizes 4-192." + - "Concurrency 48 uses 3 draft tokens (golden AL 2.88, from 6) with decode capture sizes 4-192, and drops prefill-schedule-interval 8, long-prefill-token-threshold 10240 and long-prefill-token-threshold-adaptive (back to vLLM defaults)." - "Concurrency 16, 24 and 32 set prefill-schedule-interval 32 (from 1, 1 and 8) and VLLM_PREFILL_CADENCE_NO_ESCAPE=1. Their max-num-seqs is below the concurrency, so requests always wait; without the flag vLLM drops the prefill hold (capacity escape)." - "Remove override_dram_c10, unused since concurrency 10 left the sweep." - "No change to the DSpark draft's precision: Inferact/Kimi-K3.1-Pro-DSpark-preview (revision 6a2ef20a) still loads in BF16 with no quantization and an FP8 KV cache; the new draft lengths do not change these types." From a65a6e7fa9158dafc22a7c6f4aa514f148159d01 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 13 Nov 2026 17:24:33 +0900 Subject: [PATCH 06/10] Kimi-K3.1 MI355X vLLM AgentX: add c52 and c56 override_dram_c52 and override_dram_c56 copy c48 (TP8 x DCP8, DRAM offload, 3 draft tokens, max-num-batched-tokens 10240, gpu-memory-utilization 0.91) with max-num-seqs equal to the concurrency and decode capture sizes up to max-num-seqs x 4 (208, 224); the DRAM row lists 52 and 56. Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml | 32 ++++++++++++++++++- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 1 + 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml index 10af14a32f..578d5eb5e5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -99,7 +99,7 @@ base: AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' -# max-num-seqs stays below CONC from c4 to c32, and equals it at c1 and c48. c1 and c4 capture +# max-num-seqs stays below CONC from c4 to c32, and equals it at c1 and c48 to c56. c1 and c4 capture # decode sizes only; the others list decode sizes, multiples of one plus the draft token count, # then the prefill ladder. # Throughput runs replace block rejection with the golden acceptance length. @@ -212,3 +212,33 @@ override_dram_c48: KV_OFFLOADING: 'dram' TOTAL_CPU_DRAM_GB: '1799' +override_dram_c52: + roles: + agg: + args: + max-num-seqs: 52 + max-num-batched-tokens: 10240 + gpu-memory-utilization: 0.91 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '52' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +override_dram_c56: + roles: + agg: + args: + max-num-seqs: 56 + max-num-batched-tokens: 10240 + gpu-memory-utilization: 0.91 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '56' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 943a3748cf..11348ce9ce 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -990,7 +990,7 @@ kimik3.1-fp4-mi355x-vllm-agentic-dspark: # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. c1 and c4 keep the KV cache on the GPUs. - { tp: 8, ep: 1, dcp-size: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml } - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4], srt-recipe: agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [16, 24, 32, 48], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [16, 24, 32, 48, 52, 56], srt-recipe: agentic.yaml } # MiniMax-M3.1-Flash AgentX on MI355X with SGLang: day-zero image from the SGLang # cookbook (https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1), diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 03318528d7..4ac9d88814 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12808,6 +12808,7 @@ - "Concurrency 4 drops DRAM offload (kv-offloading none, override_gpu_kv_c4) and stays TP8 x DCP8 with max-num-seqs 2; it switches to the latency_specialized decode backend, 5 draft tokens (golden AL 3.41, from 7), decode-only capture sizes 6 and 12 and breakable CUDA graphs off." - "Concurrency 48 uses 3 draft tokens (golden AL 2.88, from 6) with decode capture sizes 4-192, and drops prefill-schedule-interval 8, long-prefill-token-threshold 10240 and long-prefill-token-threshold-adaptive (back to vLLM defaults)." - "Concurrency 16, 24 and 32 set prefill-schedule-interval 32 (from 1, 1 and 8) and VLLM_PREFILL_CADENCE_NO_ESCAPE=1. Their max-num-seqs is below the concurrency, so requests always wait; without the flag vLLM drops the prefill hold (capacity escape)." + - "Add concurrency 52 and 56 (override_dram_c52, override_dram_c56) on the c48 settings: TP8 x DCP8, DRAM offload, 3 draft tokens, max-num-batched-tokens 10240, gpu-memory-utilization 0.91, max-num-seqs equal to the concurrency, decode capture sizes 4-208 and 4-224." - "Remove override_dram_c10, unused since concurrency 10 left the sweep." - "No change to the DSpark draft's precision: Inferact/Kimi-K3.1-Pro-DSpark-preview (revision 6a2ef20a) still loads in BF16 with no quantization and an FP8 KV cache; the new draft lengths do not change these types." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/5043 From c3003a587d6be70a44523410fb449c121013b038 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 13 Nov 2026 17:25:28 +0900 Subject: [PATCH 07/10] Kimi-K3.1 MI355X vLLM AgentX: add c60 override_dram_c60 copies c48 with max-num-seqs 60 and decode capture sizes up to 240 (max-num-seqs x 4); the DRAM row lists 60. Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml | 17 ++++++++++++++++- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml index 578d5eb5e5..cfc21b9fc7 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -99,7 +99,7 @@ base: AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' -# max-num-seqs stays below CONC from c4 to c32, and equals it at c1 and c48 to c56. c1 and c4 capture +# max-num-seqs stays below CONC from c4 to c32, and equals it at c1 and c48 to c60. c1 and c4 capture # decode sizes only; the others list decode sizes, multiples of one plus the draft token count, # then the prefill ladder. # Throughput runs replace block rejection with the golden acceptance length. @@ -242,3 +242,18 @@ override_dram_c56: KV_OFFLOADING: 'dram' TOTAL_CPU_DRAM_GB: '1799' +override_dram_c60: + roles: + agg: + args: + max-num-seqs: 60 + max-num-batched-tokens: 10240 + gpu-memory-utilization: 0.91 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '60' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 11348ce9ce..e888453842 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -990,7 +990,7 @@ kimik3.1-fp4-mi355x-vllm-agentic-dspark: # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. c1 and c4 keep the KV cache on the GPUs. - { tp: 8, ep: 1, dcp-size: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml } - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4], srt-recipe: agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [16, 24, 32, 48, 52, 56], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [16, 24, 32, 48, 52, 56, 60], srt-recipe: agentic.yaml } # MiniMax-M3.1-Flash AgentX on MI355X with SGLang: day-zero image from the SGLang # cookbook (https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1), diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 4ac9d88814..49784583a9 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12808,7 +12808,7 @@ - "Concurrency 4 drops DRAM offload (kv-offloading none, override_gpu_kv_c4) and stays TP8 x DCP8 with max-num-seqs 2; it switches to the latency_specialized decode backend, 5 draft tokens (golden AL 3.41, from 7), decode-only capture sizes 6 and 12 and breakable CUDA graphs off." - "Concurrency 48 uses 3 draft tokens (golden AL 2.88, from 6) with decode capture sizes 4-192, and drops prefill-schedule-interval 8, long-prefill-token-threshold 10240 and long-prefill-token-threshold-adaptive (back to vLLM defaults)." - "Concurrency 16, 24 and 32 set prefill-schedule-interval 32 (from 1, 1 and 8) and VLLM_PREFILL_CADENCE_NO_ESCAPE=1. Their max-num-seqs is below the concurrency, so requests always wait; without the flag vLLM drops the prefill hold (capacity escape)." - - "Add concurrency 52 and 56 (override_dram_c52, override_dram_c56) on the c48 settings: TP8 x DCP8, DRAM offload, 3 draft tokens, max-num-batched-tokens 10240, gpu-memory-utilization 0.91, max-num-seqs equal to the concurrency, decode capture sizes 4-208 and 4-224." + - "Add concurrency 52, 56 and 60 (override_dram_c52, override_dram_c56, override_dram_c60) on the c48 settings: TP8 x DCP8, DRAM offload, 3 draft tokens, max-num-batched-tokens 10240, gpu-memory-utilization 0.91, max-num-seqs equal to the concurrency, decode capture sizes 4-208, 4-224 and 4-240." - "Remove override_dram_c10, unused since concurrency 10 left the sweep." - "No change to the DSpark draft's precision: Inferact/Kimi-K3.1-Pro-DSpark-preview (revision 6a2ef20a) still loads in BF16 with no quantization and an FP8 KV cache; the new draft lengths do not change these types." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/5043 From b383c2874989396ef68949106b8073f73d67b89a Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 13 Nov 2026 17:29:09 +0900 Subject: [PATCH 08/10] Kimi-K3.1 MI355X vLLM AgentX c32: drop the long-prefill threshold c32 no longer sets long-prefill-token-threshold 10240 or long-prefill-token-threshold-adaptive; prefill-schedule-interval 32 and VLLM_PREFILL_CADENCE_NO_ESCAPE=1 stay. Co-Authored-By: Claude Opus 5.5 --- .../srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml | 2 -- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml index cfc21b9fc7..f057f1445a 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -185,8 +185,6 @@ override_dram_c32: max-num-batched-tokens: 10240 gpu-memory-utilization: 0.91 prefill-schedule-interval: 32 - long-prefill-token-threshold: 10240 - long-prefill-token-threshold-adaptive: true compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' env: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 49784583a9..edac6a6734 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12807,7 +12807,7 @@ - "Concurrency 1 (TP8 x DCP1, latency_specialized decode backend) sets VLLM_KIMI_SD_DSA_PROLOGUE=1, which runs each DSA layer's decode prologue in one launch." - "Concurrency 4 drops DRAM offload (kv-offloading none, override_gpu_kv_c4) and stays TP8 x DCP8 with max-num-seqs 2; it switches to the latency_specialized decode backend, 5 draft tokens (golden AL 3.41, from 7), decode-only capture sizes 6 and 12 and breakable CUDA graphs off." - "Concurrency 48 uses 3 draft tokens (golden AL 2.88, from 6) with decode capture sizes 4-192, and drops prefill-schedule-interval 8, long-prefill-token-threshold 10240 and long-prefill-token-threshold-adaptive (back to vLLM defaults)." - - "Concurrency 16, 24 and 32 set prefill-schedule-interval 32 (from 1, 1 and 8) and VLLM_PREFILL_CADENCE_NO_ESCAPE=1. Their max-num-seqs is below the concurrency, so requests always wait; without the flag vLLM drops the prefill hold (capacity escape)." + - "Concurrency 16, 24 and 32 set prefill-schedule-interval 32 (from 1, 1 and 8) and VLLM_PREFILL_CADENCE_NO_ESCAPE=1. Their max-num-seqs is below the concurrency, so requests always wait; without the flag vLLM drops the prefill hold (capacity escape). Concurrency 32 drops long-prefill-token-threshold 10240 and long-prefill-token-threshold-adaptive." - "Add concurrency 52, 56 and 60 (override_dram_c52, override_dram_c56, override_dram_c60) on the c48 settings: TP8 x DCP8, DRAM offload, 3 draft tokens, max-num-batched-tokens 10240, gpu-memory-utilization 0.91, max-num-seqs equal to the concurrency, decode capture sizes 4-208, 4-224 and 4-240." - "Remove override_dram_c10, unused since concurrency 10 left the sweep." - "No change to the DSpark draft's precision: Inferact/Kimi-K3.1-Pro-DSpark-preview (revision 6a2ef20a) still loads in BF16 with no quantization and an FP8 KV cache; the new draft lengths do not change these types." From 34099327ddbff18dbba256b580a50efcd7f6671d Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 13 Nov 2026 18:10:01 +0900 Subject: [PATCH 09/10] Kimi-K3.1 MI355X vLLM AgentX: drop c52, c56 and c60 Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml | 48 +------------------ inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 1 - 3 files changed, 2 insertions(+), 49 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml index f057f1445a..09839520ae 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -99,7 +99,7 @@ base: AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' -# max-num-seqs stays below CONC from c4 to c32, and equals it at c1 and c48 to c60. c1 and c4 capture +# max-num-seqs stays below CONC from c4 to c32, and equals it at c1 and c48. c1 and c4 capture # decode sizes only; the others list decode sizes, multiples of one plus the draft token count, # then the prefill ladder. # Throughput runs replace block rejection with the golden acceptance length. @@ -209,49 +209,3 @@ override_dram_c48: CONC: '48' KV_OFFLOADING: 'dram' TOTAL_CPU_DRAM_GB: '1799' - -override_dram_c52: - roles: - agg: - args: - max-num-seqs: 52 - max-num-batched-tokens: 10240 - gpu-memory-utilization: 0.91 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '52' - KV_OFFLOADING: 'dram' - TOTAL_CPU_DRAM_GB: '1799' - -override_dram_c56: - roles: - agg: - args: - max-num-seqs: 56 - max-num-batched-tokens: 10240 - gpu-memory-utilization: 0.91 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '56' - KV_OFFLOADING: 'dram' - TOTAL_CPU_DRAM_GB: '1799' - -override_dram_c60: - roles: - agg: - args: - max-num-seqs: 60 - max-num-batched-tokens: 10240 - gpu-memory-utilization: 0.91 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '60' - KV_OFFLOADING: 'dram' - TOTAL_CPU_DRAM_GB: '1799' - diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 2af89229d8..4aa163afcf 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -990,7 +990,7 @@ kimik3.1-fp4-mi355x-vllm-agentic-dspark: # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. c1 and c4 keep the KV cache on the GPUs. - { tp: 8, ep: 1, dcp-size: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml } - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4], srt-recipe: agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [16, 24, 32, 48, 52, 56, 60], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [16, 24, 32, 48], srt-recipe: agentic.yaml } # MiniMax-M3.1-Flash AgentX on MI355X with SGLang: day-zero image from the SGLang # cookbook (https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1), diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index d968bced98..c5fec22b05 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12855,7 +12855,6 @@ - "Concurrency 4 drops DRAM offload (kv-offloading none, override_gpu_kv_c4) and stays TP8 x DCP8 with max-num-seqs 2; it switches to the latency_specialized decode backend, 5 draft tokens (golden AL 3.41, from 7), decode-only capture sizes 6 and 12 and breakable CUDA graphs off." - "Concurrency 48 uses 3 draft tokens (golden AL 2.88, from 6) with decode capture sizes 4-192, and drops prefill-schedule-interval 8, long-prefill-token-threshold 10240 and long-prefill-token-threshold-adaptive (back to vLLM defaults)." - "Concurrency 16, 24 and 32 set prefill-schedule-interval 32 (from 1, 1 and 8) and VLLM_PREFILL_CADENCE_NO_ESCAPE=1. Their max-num-seqs is below the concurrency, so requests always wait; without the flag vLLM drops the prefill hold (capacity escape). Concurrency 32 drops long-prefill-token-threshold 10240 and long-prefill-token-threshold-adaptive." - - "Add concurrency 52, 56 and 60 (override_dram_c52, override_dram_c56, override_dram_c60) on the c48 settings: TP8 x DCP8, DRAM offload, 3 draft tokens, max-num-batched-tokens 10240, gpu-memory-utilization 0.91, max-num-seqs equal to the concurrency, decode capture sizes 4-208, 4-224 and 4-240." - "Remove override_dram_c10, unused since concurrency 10 left the sweep." - "No change to the DSpark draft's precision: Inferact/Kimi-K3.1-Pro-DSpark-preview (revision 6a2ef20a) still loads in BF16 with no quantization and an FP8 KV cache; the new draft lengths do not change these types." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/5043 From 6cbe3c5e64f2928a47f0f57c494465cdbfe4e6c4 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 13 Nov 2026 22:51:46 +0900 Subject: [PATCH 10/10] perf(amd): sync Kimi-K3.1 MI355X vLLM DSpark AgentX block with the diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update the image from vllm/vllm-openai-rocm:kimi-k31-d73dd529 (sha256:5b54f223) to vllm/vllm-openai-rocm:kimi-k31-6d74a15 (sha256:c470d0de), vLLM mum-tracking 6d74a152f. - Split concurrency 4 out of the DRAM-offload row into its own TP8 x DCP8 row with kv-offloading none (recipe override override_dram_c4 renamed override_gpu_kv_c4). Concurrency 1 (TP8 x DCP1, GPU KV) and 16, 24, 32 and 48 (TP8 x DCP8, Simple CPU DRAM offload) keep their layouts. Remove override_dram_c10, unused since concurrency 10 left the sweep. - Every point sets VLLM_ROCM_DCP_VUP_RS=1 (DCP V-up reduce-scatter, on top of the existing VLLM_ROCM_DCP_AITER_ALLGATHER=1), VLLM_ROCM_DSA_MULTISTREAM=1 and VLLM_ROCM_MEM_HEADROOM_GIB=8. - Concurrency 1 (latency_specialized decode backend) sets VLLM_KIMI_SD_DSA_PROLOGUE=1, which runs each DSA layer's decode prologue in one launch. - Concurrency 4 keeps max-num-seqs 2 and switches to the latency_specialized decode backend, 5 draft tokens (golden AL 3.41, from 7), decode-only capture sizes 6 and 12 and breakable CUDA graphs off. - Concurrency 16, 24 and 32 set prefill-schedule-interval 32 (from 1, 1 and 8) and VLLM_PREFILL_CADENCE_NO_ESCAPE=1. Their max-num-seqs is below the concurrency, so requests always wait; without the flag vLLM drops the prefill hold (capacity escape). Concurrency 32 drops long-prefill-token-threshold 10240 and long-prefill-token-threshold-adaptive. - Concurrency 48 uses 3 draft tokens (golden AL 2.88, from 6) with decode capture sizes 4-192, and drops prefill-schedule-interval 8, long-prefill-token-threshold 10240 and long-prefill-token-threshold-adaptive (back to vLLM defaults). - The new vLLM (mum-tracking d73dd529 to 6d74a152f) zeroes the KV null block after CUDA graph capture in the V2 runner (seungrokj/mum#17) and adds the opt-in VLLM_PREFILL_CADENCE_NO_ESCAPE hold (seungrokj/mum#21), the DCP V-up-first combine and multi-stream DSA decode, VLLM_ROCM_MEM_HEADROOM_GIB and the one-launch DSA decode prologue. It also fuses the MXFP4 indexer decode prep, the q_a/kv_a RMSNorm with the sigmoid gate, the KDA f_b_proj GEMM, the AttnRes register tile and the latent-MoE norm, compiles the FlyDSL sparse MLA at load and sizes the packed DSA KV blocks tightly. - No change to the DSpark draft's data types: speculative-config does not pin a revision, so Inferact/Kimi-K3.1-Pro-DSpark-preview resolves to Hugging Face main 6a2ef20a (current since 2026-11-05), whose weights are all BF16 with no quantization_config. vLLM 6d74a152f loads it unquantized with an FP8 KV cache, as d73dd529 did; VLLM_ROCM_DCP_VUP_RS also combines the draft's DCP attention output, in BF16 with an FP32 LSE. 中文: - 将镜像从 vllm/vllm-openai-rocm:kimi-k31-d73dd529(sha256:5b54f223)更新为 vllm/vllm-openai-rocm:kimi-k31-6d74a15(sha256:c470d0de),即 vLLM mum-tracking 6d74a152f。 - 将并发 4 从 DRAM offload 行拆出,单独成为 kv-offloading none 的 TP8 x DCP8 行(recipe override override_dram_c4 改名为 override_gpu_kv_c4)。并发 1(TP8 x DCP1,KV 在 GPU 上)以及 16、24、32、48(TP8 x DCP8,Simple CPU DRAM offload)的布局不变。删除 override_dram_c10,并发 10 退出扫描后已不再使用。 - 所有点设置 VLLM_ROCM_DCP_VUP_RS=1(DCP V-up reduce-scatter,叠加在已有的 VLLM_ROCM_DCP_AITER_ALLGATHER=1 之上)、VLLM_ROCM_DSA_MULTISTREAM=1 和 VLLM_ROCM_MEM_HEADROOM_GIB=8。 - 并发 1(latency_specialized decode 后端)设置 VLLM_KIMI_SD_DSA_PROLOGUE=1,将每个 DSA 层的 decode prologue 合并为一次 launch。 - 并发 4 保持 max-num-seqs 2,改用 latency_specialized decode 后端、5 个草稿 token(golden AL 3.41,原为 7)、仅 decode 的 capture sizes 6 和 12,并关闭 breakable CUDA graph。 - 并发 16、24、32 设置 prefill-schedule-interval 32(原为 1、1、8)和 VLLM_PREFILL_CADENCE_NO_ESCAPE=1。它们的 max-num-seqs 低于并发数,请求始终在排队;不设此标志时 vLLM 会放弃 prefill 保持(capacity escape)。并发 32 去掉 long-prefill-token-threshold 10240 和 long-prefill-token-threshold-adaptive。 - 并发 48 使用 3 个草稿 token(golden AL 2.88,原为 6),decode capture sizes 为 4-192,并去掉 prefill-schedule-interval 8、long-prefill-token-threshold 10240 和 long-prefill-token-threshold-adaptive(恢复 vLLM 默认值)。 - 新 vLLM(mum-tracking d73dd529 到 6d74a152f)在 V2 runner 中于 CUDA graph capture 之后将 KV null block 清零(seungrokj/mum#17),并新增可选的 VLLM_PREFILL_CADENCE_NO_ESCAPE 保持(seungrokj/mum#21)、DCP V-up-first 合并与多流 DSA decode、VLLM_ROCM_MEM_HEADROOM_GIB 以及单次 launch 的 DSA decode prologue。它还融合了 MXFP4 indexer decode 准备、q_a/kv_a RMSNorm 与 sigmoid gate、KDA f_b_proj GEMM、AttnRes register tile 和 latent-MoE norm,在加载时编译 FlyDSL sparse MLA,并收紧 packed DSA KV block 的大小。 - DSpark 草稿的数据类型无变化:speculative-config 未固定 revision,因此 Inferact/Kimi-K3.1-Pro-DSpark-preview 解析为 Hugging Face main 6a2ef20a(自 2026-11-05 起为当前版本),其权重全部为 BF16,没有 quantization_config。vLLM 6d74a152f 不做量化地加载它,KV cache 为 FP8,与 d73dd529 相同;VLLM_ROCM_DCP_VUP_RS 也用于合并草稿的 DCP attention 输出,使用 BF16 和 FP32 LSE。 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 74bf25c909..1389af7979 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12861,12 +12861,13 @@ scenario-type: - agentic-coding description: - - "Update the image from vllm/vllm-openai-rocm:kimi-k31-d73dd529 (sha256:5b54f223) to vllm/vllm-openai-rocm:kimi-k31-6d74a15 (sha256:c470d0de), vLLM mum-tracking 6d74a152f, which includes a KV null-block fix and VLLM_PREFILL_CADENCE_NO_ESCAPE." + - "Update the image from vllm/vllm-openai-rocm:kimi-k31-d73dd529 (sha256:5b54f223) to vllm/vllm-openai-rocm:kimi-k31-6d74a15 (sha256:c470d0de), vLLM mum-tracking 6d74a152f." + - "Split concurrency 4 out of the DRAM-offload row into its own TP8 x DCP8 row with kv-offloading none (recipe override override_dram_c4 renamed override_gpu_kv_c4). Concurrency 1 (TP8 x DCP1, GPU KV) and 16, 24, 32 and 48 (TP8 x DCP8, Simple CPU DRAM offload) keep their layouts. Remove override_dram_c10, unused since concurrency 10 left the sweep." - "Every point sets VLLM_ROCM_DCP_VUP_RS=1 (DCP V-up reduce-scatter, on top of the existing VLLM_ROCM_DCP_AITER_ALLGATHER=1), VLLM_ROCM_DSA_MULTISTREAM=1 and VLLM_ROCM_MEM_HEADROOM_GIB=8." - - "Concurrency 1 (TP8 x DCP1, latency_specialized decode backend) sets VLLM_KIMI_SD_DSA_PROLOGUE=1, which runs each DSA layer's decode prologue in one launch." - - "Concurrency 4 drops DRAM offload (kv-offloading none, override_gpu_kv_c4) and stays TP8 x DCP8 with max-num-seqs 2; it switches to the latency_specialized decode backend, 5 draft tokens (golden AL 3.41, from 7), decode-only capture sizes 6 and 12 and breakable CUDA graphs off." - - "Concurrency 48 uses 3 draft tokens (golden AL 2.88, from 6) with decode capture sizes 4-192, and drops prefill-schedule-interval 8, long-prefill-token-threshold 10240 and long-prefill-token-threshold-adaptive (back to vLLM defaults)." + - "Concurrency 1 (latency_specialized decode backend) sets VLLM_KIMI_SD_DSA_PROLOGUE=1, which runs each DSA layer's decode prologue in one launch." + - "Concurrency 4 keeps max-num-seqs 2 and switches to the latency_specialized decode backend, 5 draft tokens (golden AL 3.41, from 7), decode-only capture sizes 6 and 12 and breakable CUDA graphs off." - "Concurrency 16, 24 and 32 set prefill-schedule-interval 32 (from 1, 1 and 8) and VLLM_PREFILL_CADENCE_NO_ESCAPE=1. Their max-num-seqs is below the concurrency, so requests always wait; without the flag vLLM drops the prefill hold (capacity escape). Concurrency 32 drops long-prefill-token-threshold 10240 and long-prefill-token-threshold-adaptive." - - "Remove override_dram_c10, unused since concurrency 10 left the sweep." - - "No change to the DSpark draft's precision: Inferact/Kimi-K3.1-Pro-DSpark-preview (revision 6a2ef20a) still loads in BF16 with no quantization and an FP8 KV cache; the new draft lengths do not change these types." + - "Concurrency 48 uses 3 draft tokens (golden AL 2.88, from 6) with decode capture sizes 4-192, and drops prefill-schedule-interval 8, long-prefill-token-threshold 10240 and long-prefill-token-threshold-adaptive (back to vLLM defaults)." + - "The new vLLM (mum-tracking d73dd529 to 6d74a152f) zeroes the KV null block after CUDA graph capture in the V2 runner (seungrokj/mum#17) and adds the opt-in VLLM_PREFILL_CADENCE_NO_ESCAPE hold (seungrokj/mum#21), the DCP V-up-first combine and multi-stream DSA decode, VLLM_ROCM_MEM_HEADROOM_GIB and the one-launch DSA decode prologue. It also fuses the MXFP4 indexer decode prep, the q_a/kv_a RMSNorm with the sigmoid gate, the KDA f_b_proj GEMM, the AttnRes register tile and the latent-MoE norm, compiles the FlyDSL sparse MLA at load and sizes the packed DSA KV blocks tightly." + - "No change to the DSpark draft's data types: speculative-config does not pin a revision, so Inferact/Kimi-K3.1-Pro-DSpark-preview resolves to Hugging Face main 6a2ef20a (current since 2026-11-05), whose weights are all BF16 with no quantization_config. vLLM 6d74a152f loads it unquantized with an FP8 KV cache, as d73dd529 did; VLLM_ROCM_DCP_VUP_RS also combines the draft's DCP attention output, in BF16 with an FP32 LSE." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/5043