Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -59,104 +59,52 @@ override_tp8_c1:
args:
tensor-parallel-size: 8
max-num-seqs: 2
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,256,512,1024,2048]}'
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[4,8,256,512,1024,2048]}'
benchmark:
env:
CONC: '1'
KV_OFFLOADING: none

override_tp4_c1:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
max-num-seqs: 2
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,256,512,1024,2048]}'
benchmark:
env:
CONC: '1'
KV_OFFLOADING: none

override_tp4_c5:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
max-num-seqs: 8
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,256,512,1024,2048]}'
benchmark:
env:
CONC: '5'
KV_OFFLOADING: none

override_tp4_c10:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
max-num-seqs: 15
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,256,512,1024,2048]}'
benchmark:
env:
CONC: '10'
KV_OFFLOADING: none

override_tp4_c15:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
max-num-seqs: 23
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,256,512,1024,2048]}'
benchmark:
env:
CONC: '15'
KV_OFFLOADING: none

override_tp4_c20:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
max-num-seqs: 30
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,256,512,1024,2048]}'
max-num-active-seqs: 6
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,256,512,1024,2048,3072,4096,5120,6144,7168,8192,10240,12288,14336,16384]}'
benchmark:
env:
CONC: '20'
KV_OFFLOADING: none

override_tp4_c30_dram:
override_tp4_c24:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
max-num-seqs: 45
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,256,512,1024,2048]}'
kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1499000000000,"lazy_offload":true}}'
max-num-seqs: 7
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,256,512,1024,2048,2560,3072,3584,4096,5120,6144,7168,8192]}'
env:
VLLM_USE_SIMPLE_KV_OFFLOAD: '1'
VLLM_FLASHINFER_ALLREDUCE_BACKEND: mnnvl
benchmark:
env:
CONC: '30'
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '1499'
CONC: '24'
KV_OFFLOADING: none

override_tp2_c24_dram:
roles:
agg:
gpus: 2
args:
tensor-parallel-size: 2
gpu-memory-utilization: 0.92
max-num-seqs: 36
gpu-memory-utilization: 0.92
attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}'
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,256,512,1024,2048]}'
kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":804232626176,"lazy_offload":true}}'
kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":749000000000,"lazy_offload":true}}'
env:
VLLM_USE_SIMPLE_KV_OFFLOAD: '1'
benchmark:
Expand All @@ -165,88 +113,23 @@ override_tp2_c24_dram:
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '749'

override_tp4_c40_dram:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
max-num-seqs: 60
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,256,512,1024,2048]}'
kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1499000000000,"lazy_offload":true}}'
env:
VLLM_USE_SIMPLE_KV_OFFLOAD: '1'
benchmark:
env:
CONC: '40'
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '1499'

override_tp4_c48_dram:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
max-num-seqs: 72
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,512,1024,2048]}'
kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1499000000000,"lazy_offload":true}}'
env:
VLLM_USE_SIMPLE_KV_OFFLOAD: '1'
benchmark:
env:
CONC: '48'
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '1499'

override_tp4_c64_dram:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
max-num-seqs: 96
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384,512,1024,2048]}'
kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1499000000000,"lazy_offload":true}}'
env:
VLLM_USE_SIMPLE_KV_OFFLOAD: '1'
benchmark:
env:
CONC: '64'
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '1499'

override_tp2_c32_dram:
roles:
agg:
gpus: 2
args:
tensor-parallel-size: 2
gpu-memory-utilization: 0.92
max-num-seqs: 48
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,256,512,1024,2048]}'
kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":804232626176,"lazy_offload":true}}'
env:
VLLM_USE_SIMPLE_KV_OFFLOAD: '1'
benchmark:
env:
CONC: '32'
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '749'
override_tp2_c40_dram:
override_tp2_c28_dram:
slurm:
time_limit: "02:20:00"
roles:
agg:
gpus: 2
args:
tensor-parallel-size: 2
max-num-seqs: 36
gpu-memory-utilization: 0.92
max-num-seqs: 60
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,256,512,1024,2048]}'
kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":804232626176,"lazy_offload":true}}'
attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}'
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,256,512,1024,2048]}'
kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":749000000000,"lazy_offload":true}}'
env:
VLLM_USE_SIMPLE_KV_OFFLOAD: '1'
benchmark:
env:
CONC: '40'
CONC: '28'
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '749'
9 changes: 3 additions & 6 deletions inferencex-e2e/configs/nvidia-master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5436,7 +5436,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp:
ep: 2
dp-attn: false
minimaxm3-fp4-b300-vllm-agentic-mtp:
image: vllm/vllm-openai:nightly-5555a2d8f7116b2b4877f66516f4128176cf722a@sha256:e5c9e6bb82f645e8fca04e4d395743ee5ed473a62b98942d65f83c35d0c69301
image: vllm/vllm-openai:nightly-49cb2bbf71cfd36ee5aa33d1e184b1cddff55294@sha256:8c1434bfee7d7626eef78ef70600434ffba4a1b8813cb1d1c8707ad25f47035e
model: nvidia/MiniMax-M3-NVFP4
model-prefix: minimaxm3
runner: cluster:b300-dsxe
Expand All @@ -5449,11 +5449,8 @@ minimaxm3-fp4-b300-vllm-agentic-mtp:
- dram-utilization: 1.0
search-space:
- { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml }
- { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20], srt-recipe: agentic.yaml }
- { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [30, 40, 48, 64], srt-recipe: agentic.yaml }
- dram-utilization: 1.0
search-space:
- { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24, 32, 40], srt-recipe: agentic.yaml }
- { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [20, 24], srt-recipe: agentic.yaml }
- { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24, 28], srt-recipe: agentic.yaml }

minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg:
image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7
Expand Down
9 changes: 9 additions & 0 deletions inferencex-e2e/perf-changelog.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11281,3 +11281,12 @@
- "The new image includes vllm-project/vllm#60683 (Kimi-K3 specialized decode on MI355X), #59552 (AITER Gluon chunk_kda for KDA prefill), #59634 (fused GLA gate GEMM and MLA FP8 prefill q/k/v prep) and #59824 (DSpark adaptive verification on MI355X, not enabled here)."
- "Draft model data type is unchanged: the Inferact/Kimi-K3-DSpark draft is BF16 with no quantization_config, it gets no quantization and does not inherit the target quantization-config at either image, and its KV cache stays FP8 via speculative-config kv_cache_dtype."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4393

- config-keys:
- minimaxm3-fp4-b300-vllm-agentic-mtp
scenario-type:
- agentic-coding
description:
- "Pin vLLM to digest-pinned nightly-49cb2bbf71cfd36ee5aa33d1e184b1cddff55294; configure TP8 c1 and TP4 c20/c24 without offload, plus TP2 c24/c28 with 749 GB lazy DRAM offload and CUTLASS MSA decode."
- "Set TP4 c20 max-num-active-seqs to 6 and TP2 c28 Slurm time limit to 02:20:00."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4357
Loading