diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml index 03ed0f8ce2..1b0c1db5af 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml @@ -59,64 +59,12 @@ override_tp8_c1: args: tensor-parallel-size: 8 max-num-seqs: 2 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,256,512,1024,2048]}' + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[4,8,256,512,1024,2048]}' benchmark: env: CONC: '1' KV_OFFLOADING: none -override_tp4_c1: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - max-num-seqs: 2 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,256,512,1024,2048]}' - benchmark: - env: - CONC: '1' - KV_OFFLOADING: none - -override_tp4_c5: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - max-num-seqs: 8 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,256,512,1024,2048]}' - benchmark: - env: - CONC: '5' - KV_OFFLOADING: none - -override_tp4_c10: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - max-num-seqs: 15 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,256,512,1024,2048]}' - benchmark: - env: - CONC: '10' - KV_OFFLOADING: none - -override_tp4_c15: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - max-num-seqs: 23 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,256,512,1024,2048]}' - benchmark: - env: - CONC: '15' - KV_OFFLOADING: none - override_tp4_c20: roles: agg: @@ -124,28 +72,27 @@ override_tp4_c20: args: tensor-parallel-size: 4 max-num-seqs: 30 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,256,512,1024,2048]}' + max-num-active-seqs: 6 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,256,512,1024,2048,3072,4096,5120,6144,7168,8192,10240,12288,14336,16384]}' benchmark: env: CONC: '20' KV_OFFLOADING: none -override_tp4_c30_dram: +override_tp4_c24: roles: agg: gpus: 4 args: tensor-parallel-size: 4 - max-num-seqs: 45 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,256,512,1024,2048]}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1499000000000,"lazy_offload":true}}' + max-num-seqs: 7 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,256,512,1024,2048,2560,3072,3584,4096,5120,6144,7168,8192]}' env: - VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: mnnvl benchmark: env: - CONC: '30' - KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1499' + CONC: '24' + KV_OFFLOADING: none override_tp2_c24_dram: roles: @@ -153,10 +100,11 @@ override_tp2_c24_dram: gpus: 2 args: tensor-parallel-size: 2 - gpu-memory-utilization: 0.92 max-num-seqs: 36 + gpu-memory-utilization: 0.92 + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,256,512,1024,2048]}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":804232626176,"lazy_offload":true}}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":749000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' benchmark: @@ -165,88 +113,23 @@ override_tp2_c24_dram: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '749' -override_tp4_c40_dram: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - max-num-seqs: 60 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,256,512,1024,2048]}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1499000000000,"lazy_offload":true}}' - env: - VLLM_USE_SIMPLE_KV_OFFLOAD: '1' - benchmark: - env: - CONC: '40' - KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1499' - -override_tp4_c48_dram: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - max-num-seqs: 72 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,512,1024,2048]}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1499000000000,"lazy_offload":true}}' - env: - VLLM_USE_SIMPLE_KV_OFFLOAD: '1' - benchmark: - env: - CONC: '48' - KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1499' - -override_tp4_c64_dram: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - max-num-seqs: 96 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384,512,1024,2048]}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1499000000000,"lazy_offload":true}}' - env: - VLLM_USE_SIMPLE_KV_OFFLOAD: '1' - benchmark: - env: - CONC: '64' - KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1499' - -override_tp2_c32_dram: - roles: - agg: - gpus: 2 - args: - tensor-parallel-size: 2 - gpu-memory-utilization: 0.92 - max-num-seqs: 48 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,256,512,1024,2048]}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":804232626176,"lazy_offload":true}}' - env: - VLLM_USE_SIMPLE_KV_OFFLOAD: '1' - benchmark: - env: - CONC: '32' - KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '749' -override_tp2_c40_dram: +override_tp2_c28_dram: + slurm: + time_limit: "02:20:00" roles: agg: gpus: 2 args: tensor-parallel-size: 2 + max-num-seqs: 36 gpu-memory-utilization: 0.92 - max-num-seqs: 60 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,256,512,1024,2048]}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":804232626176,"lazy_offload":true}}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,256,512,1024,2048]}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":749000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' benchmark: env: - CONC: '40' + CONC: '28' KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '749' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 9e524ab5f1..1eabff5a14 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5436,7 +5436,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 2 dp-attn: false minimaxm3-fp4-b300-vllm-agentic-mtp: - image: vllm/vllm-openai:nightly-5555a2d8f7116b2b4877f66516f4128176cf722a@sha256:e5c9e6bb82f645e8fca04e4d395743ee5ed473a62b98942d65f83c35d0c69301 + image: vllm/vllm-openai:nightly-49cb2bbf71cfd36ee5aa33d1e184b1cddff55294@sha256:8c1434bfee7d7626eef78ef70600434ffba4a1b8813cb1d1c8707ad25f47035e model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: cluster:b300-dsxe @@ -5449,11 +5449,8 @@ minimaxm3-fp4-b300-vllm-agentic-mtp: - dram-utilization: 1.0 search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml } - - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20], srt-recipe: agentic.yaml } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [30, 40, 48, 64], srt-recipe: agentic.yaml } - - dram-utilization: 1.0 - search-space: - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24, 32, 40], srt-recipe: agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [20, 24], srt-recipe: agentic.yaml } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24, 28], srt-recipe: agentic.yaml } minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 59fccba611..717bcbbe29 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11281,3 +11281,12 @@ - "The new image includes vllm-project/vllm#60683 (Kimi-K3 specialized decode on MI355X), #59552 (AITER Gluon chunk_kda for KDA prefill), #59634 (fused GLA gate GEMM and MLA FP8 prefill q/k/v prep) and #59824 (DSpark adaptive verification on MI355X, not enabled here)." - "Draft model data type is unchanged: the Inferact/Kimi-K3-DSpark draft is BF16 with no quantization_config, it gets no quantization and does not inherit the target quantization-config at either image, and its KV cache stays FP8 via speculative-config kv_cache_dtype." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4393 + +- config-keys: + - minimaxm3-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Pin vLLM to digest-pinned nightly-49cb2bbf71cfd36ee5aa33d1e184b1cddff55294; configure TP8 c1 and TP4 c20/c24 without offload, plus TP2 c24/c28 with 749 GB lazy DRAM offload and CUTLASS MSA decode." + - "Set TP4 c20 max-num-active-seqs to 6 and TP2 c28 Slurm time limit to 02:20:00." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4357