From e85978ac46c1761c0ae524d6969f99c19fb814b8 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 15 Nov 2026 22:03:56 +0900 Subject: [PATCH 1/3] perf(amd): Kimi-K3.1 MI355X vLLM AgentX: image kimi-k31-5c14fe8, SP prefill, c1/c4 decode levers Bump the image to vllm/vllm-openai-rocm:kimi-k31-5c14fe8 (mum-tracking 5c14fe8de: seungrokj/mum#26 and seungrokj/mum#28 on top of 6d74a152f). Enable VLLM_K3_SP_PREFILL on every point, and set the latency_specialized decode levers explicitly at concurrency 1 and 4. Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml | 18 ++++++++++++++++++ inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 11 +++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml index 09839520ae..68219d9bf4 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -72,6 +72,8 @@ base: VLLM_ROCM_DSA_MULTISTREAM: '1' # VRAM reserve for unprofiled DCP prefill transients. VLLM_ROCM_MEM_HEADROOM_GIB: '8' + # Token-shard the residual stream across TP for prefill batches of >= 4096 tokens. + VLLM_K3_SP_PREFILL: '1' VLLM_ROCM_USE_AITER_FLYDSL_SPARSE_MLA: '1' # AITER reads its own variable for the same kernels. AITER_FLYDSL_SPARSE_MLA: '1' @@ -120,6 +122,15 @@ override_gpu_kv_c1: VLLM_USE_BREAKABLE_CUDAGRAPH: '0' # Runs each DSA layer's decode prologue as one launch; needs DCP1. VLLM_KIMI_SD_DSA_PROLOGUE: '1' + # D1a also runs the MXFP4 indexer prep; needs VLLM_KIMI_SD_DSA_PROLOGUE. + VLLM_KIMI_SD_DSA_PROLOGUE_INDEXER: '1' + # latency_specialized decode levers; read only under that backend. + VLLM_K31_BEAT_MARKOV: '1' + VLLM_K31_BEAT_DRAFTMETA: '1' + VLLM_K31_BEAT_K2SYNC: '1' + VLLM_K31_BEAT_EAGERPREP: '1' + VLLM_K31_BEAT_KDAK1: '1' + VLLM_K3_INDEXER_SAMPLED_TOPK: '1' benchmark: env: CONC: '1' @@ -137,6 +148,13 @@ override_gpu_kv_c4: env: # Decode-only capture sizes, as at conc 1. VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + # latency_specialized decode levers; read only under that backend. + VLLM_K31_BEAT_MARKOV: '1' + VLLM_K31_BEAT_DRAFTMETA: '1' + VLLM_K31_BEAT_K2SYNC: '1' + VLLM_K31_BEAT_EAGERPREP: '1' + VLLM_K31_BEAT_KDAK1: '1' + VLLM_K3_INDEXER_SAMPLED_TOPK: '1' benchmark: env: CONC: '4' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 4aa163afcf..ee6fbf74c1 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -974,7 +974,7 @@ minimaxm31flash-fp4-mi355x-vllm-agentic-dspark: - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [32, 64], srt-recipe: agentic.yaml } kimik3.1-fp4-mi355x-vllm-agentic-dspark: - image: vllm/vllm-openai-rocm:kimi-k31-6d74a15@sha256:c470d0de88e8bb664a07006d44e6603a413c9fffd36cc4add4d4b420d4bde5e0 + image: vllm/vllm-openai-rocm:kimi-k31-5c14fe8@sha256:a61996b4c270ec09111802f62b44f5c20ad4cb73068be022b2804a287b924b9a model: moonshotai/Kimi-K3.1-Pro model-prefix: kimik3.1 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index ef2a146d5d..24f2851f86 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12898,3 +12898,14 @@ - "Sweep the B200 Wan2.2-T2V-A14B vLLM-Omni baseline on 1, 2, 4 and 8 GPUs at C1 on the same wan22.yaml release-v1 workload, so the dashboard shows a GPU-count curve." - "Deployment per count: 1 GPU runs both CFG branches with Ulysses 1 and no VAE patch parallelism; 2 GPUs CFG parallel 2 x Ulysses 1; 4 GPUs CFG parallel 2 x Ulysses 2; 8 GPUs keeps CFG parallel 2 x Ulysses 4. Tiled VAE patch parallelism spans every GPU from 2 GPUs up." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/5084 + +- config-keys: + - kimik3.1-fp4-mi355x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from vllm/vllm-openai-rocm:kimi-k31-6d74a15 (sha256:c470d0de) to vllm/vllm-openai-rocm:kimi-k31-5c14fe8 (sha256:a61996b4), vLLM mum-tracking 5c14fe8de, which adds seungrokj/mum#26 and seungrokj/mum#28 on top of 6d74a152f." + - "Every point sets VLLM_K3_SP_PREFILL=1 (seungrokj/mum#26): prefill batches of at least 4096 tokens keep the attn-res residual stream token-sharded across TP, replacing each layer's three all-reduces with reduce-scatters and all-gathers over symmetric memory that sum the TP partials in FP32 in a fixed order." + - "Concurrency 1 and 4 (latency_specialized decode backend) set the seungrokj/mum#28 levers explicitly: VLLM_K31_BEAT_MARKOV, VLLM_K31_BEAT_DRAFTMETA, VLLM_K31_BEAT_K2SYNC, VLLM_K31_BEAT_EAGERPREP, VLLM_K31_BEAT_KDAK1 and VLLM_K3_INDEXER_SAMPLED_TOPK, plus VLLM_KIMI_SD_DSA_PROLOGUE_INDEXER at concurrency 1, where VLLM_KIMI_SD_DSA_PROLOGUE is on. Concurrency 16-48 use the stock decode backend, which ignores these flags." + - "No change to layouts, draft token counts, capture sizes or the DSpark draft's data types." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 41861637f57bdcdab790e1867322b5e0ac83999b Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 15 Nov 2026 22:04:32 +0900 Subject: [PATCH 2/3] docs(changelog): link the Kimi-K3.1 MI355X entry to #5089 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 24f2851f86..5cd3ebd18b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12908,4 +12908,4 @@ - "Every point sets VLLM_K3_SP_PREFILL=1 (seungrokj/mum#26): prefill batches of at least 4096 tokens keep the attn-res residual stream token-sharded across TP, replacing each layer's three all-reduces with reduce-scatters and all-gathers over symmetric memory that sum the TP partials in FP32 in a fixed order." - "Concurrency 1 and 4 (latency_specialized decode backend) set the seungrokj/mum#28 levers explicitly: VLLM_K31_BEAT_MARKOV, VLLM_K31_BEAT_DRAFTMETA, VLLM_K31_BEAT_K2SYNC, VLLM_K31_BEAT_EAGERPREP, VLLM_K31_BEAT_KDAK1 and VLLM_K3_INDEXER_SAMPLED_TOPK, plus VLLM_KIMI_SD_DSA_PROLOGUE_INDEXER at concurrency 1, where VLLM_KIMI_SD_DSA_PROLOGUE is on. Concurrency 16-48 use the stock decode backend, which ignores these flags." - "No change to layouts, draft token counts, capture sizes or the DSpark draft's data types." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/5089 From 75d74552457a3f44438c6b9bb9906062fc8f82c6 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 15 Nov 2026 22:08:51 +0900 Subject: [PATCH 3/3] perf(amd): drop Kimi-K3.1 c1/c4 lever flags that match their defaults Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml | 16 ---------------- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml index 68219d9bf4..3909562ea0 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -122,15 +122,6 @@ override_gpu_kv_c1: VLLM_USE_BREAKABLE_CUDAGRAPH: '0' # Runs each DSA layer's decode prologue as one launch; needs DCP1. VLLM_KIMI_SD_DSA_PROLOGUE: '1' - # D1a also runs the MXFP4 indexer prep; needs VLLM_KIMI_SD_DSA_PROLOGUE. - VLLM_KIMI_SD_DSA_PROLOGUE_INDEXER: '1' - # latency_specialized decode levers; read only under that backend. - VLLM_K31_BEAT_MARKOV: '1' - VLLM_K31_BEAT_DRAFTMETA: '1' - VLLM_K31_BEAT_K2SYNC: '1' - VLLM_K31_BEAT_EAGERPREP: '1' - VLLM_K31_BEAT_KDAK1: '1' - VLLM_K3_INDEXER_SAMPLED_TOPK: '1' benchmark: env: CONC: '1' @@ -148,13 +139,6 @@ override_gpu_kv_c4: env: # Decode-only capture sizes, as at conc 1. VLLM_USE_BREAKABLE_CUDAGRAPH: '0' - # latency_specialized decode levers; read only under that backend. - VLLM_K31_BEAT_MARKOV: '1' - VLLM_K31_BEAT_DRAFTMETA: '1' - VLLM_K31_BEAT_K2SYNC: '1' - VLLM_K31_BEAT_EAGERPREP: '1' - VLLM_K31_BEAT_KDAK1: '1' - VLLM_K3_INDEXER_SAMPLED_TOPK: '1' benchmark: env: CONC: '4' diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 5cd3ebd18b..22a33ec210 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12906,6 +12906,6 @@ description: - "Update the image from vllm/vllm-openai-rocm:kimi-k31-6d74a15 (sha256:c470d0de) to vllm/vllm-openai-rocm:kimi-k31-5c14fe8 (sha256:a61996b4), vLLM mum-tracking 5c14fe8de, which adds seungrokj/mum#26 and seungrokj/mum#28 on top of 6d74a152f." - "Every point sets VLLM_K3_SP_PREFILL=1 (seungrokj/mum#26): prefill batches of at least 4096 tokens keep the attn-res residual stream token-sharded across TP, replacing each layer's three all-reduces with reduce-scatters and all-gathers over symmetric memory that sum the TP partials in FP32 in a fixed order." - - "Concurrency 1 and 4 (latency_specialized decode backend) set the seungrokj/mum#28 levers explicitly: VLLM_K31_BEAT_MARKOV, VLLM_K31_BEAT_DRAFTMETA, VLLM_K31_BEAT_K2SYNC, VLLM_K31_BEAT_EAGERPREP, VLLM_K31_BEAT_KDAK1 and VLLM_K3_INDEXER_SAMPLED_TOPK, plus VLLM_KIMI_SD_DSA_PROLOGUE_INDEXER at concurrency 1, where VLLM_KIMI_SD_DSA_PROLOGUE is on. Concurrency 16-48 use the stock decode backend, which ignores these flags." + - "Concurrency 1 and 4 (latency_specialized decode backend) pick up the seungrokj/mum#28 levers through their defaults (all 1): VLLM_K31_BEAT_MARKOV, VLLM_K31_BEAT_DRAFTMETA, VLLM_K31_BEAT_K2SYNC, VLLM_K31_BEAT_EAGERPREP, VLLM_K31_BEAT_KDAK1 and VLLM_K3_INDEXER_SAMPLED_TOPK, plus VLLM_KIMI_SD_DSA_PROLOGUE_INDEXER at concurrency 1, where VLLM_KIMI_SD_DSA_PROLOGUE is on. Concurrency 16-48 use the stock decode backend, which ignores these flags." - "No change to layouts, draft token counts, capture sizes or the DSpark draft's data types." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/5089