diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml index fbdd71b397..b958364afc 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml @@ -189,7 +189,9 @@ override_tp4_c30_dram: args: tensor-parallel-size: 4 max-num-seqs: 60 - max-cudagraph-capture-size: 240 + max-cudagraph-capture-size: 4096 + cudagraph-capture-sizes: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 256, 320, 384, 448, 512, 640, 768, 896, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096] + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1073000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' @@ -257,7 +259,9 @@ override_tp4_c38_dram: args: tensor-parallel-size: 4 max-num-seqs: 76 - max-cudagraph-capture-size: 304 + max-cudagraph-capture-size: 4096 + cudagraph-capture-sizes: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 384, 448, 512, 640, 768, 896, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096] + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1073000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' @@ -283,3 +287,133 @@ override_tp4_c40_dram: CONC: '40' KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '1073' + +# Low-concurrency arms that cap the number of sequences the scheduler keeps active +# (max-num-active-seqs) below CONC, with a three- or five-token draft. Selected by +# name from the master config. + +override_cap1_k5_tp4_c10: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-active-seqs: 1 + gpu-memory-utilization: 0.925 + max-cudagraph-capture-size: 4096 + cudagraph-capture-sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 640, 768, 1024, 1536, 2048, 3072, 4096] + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":5,"attention_backend":"FLASHINFER"}' + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: mnnvl + benchmark: + env: + CONC: '10' + KV_OFFLOADING: none + +override_cap1_k5_tp8_c10: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-active-seqs: 1 + gpu-memory-utilization: 0.9 + max-cudagraph-capture-size: 4096 + cudagraph-capture-sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 640, 768, 1024, 1536, 2048, 3072, 4096] + no-enable-flashinfer-autotune: false + enable-flashinfer-autotune: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":5,"attention_backend":"FLASHINFER"}' + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: mnnvl + benchmark: + env: + CONC: '10' + KV_OFFLOADING: none + +# TP4 DRAM concurrency 20 with 4 and 3 active sequences on the c10 key's image; the CPU KV +# pool is the 1,073 GB node allowance. +override_cap4_tp4_c20_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-active-seqs: 4 + gpu-memory-utilization: 0.9 + max-cudagraph-capture-size: 8192 + cudagraph-capture-sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 640, 768, 1024, 1536, 2048, 3072, 4096, 6144, 8192] + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1073000000000,"lazy_offload":true}}' + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: mnnvl + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '20' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1073' + +override_cap3_tp4_c20_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-active-seqs: 3 + gpu-memory-utilization: 0.9 + max-cudagraph-capture-size: 8192 + cudagraph-capture-sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 640, 768, 1024, 1536, 2048, 3072, 4096, 6144, 8192] + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1073000000000,"lazy_offload":true}}' + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: mnnvl + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '20' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1073' + +# CONC 32 with active sequences capped below CONC on the c10 key's image; the CPU KV pool is cut to the 1,073 GB node allowance. + +override_cap6_tp4_c32_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-active-seqs: 6 + gpu-memory-utilization: 0.925 + max-cudagraph-capture-size: 4096 + cudagraph-capture-sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 640, 768, 1024, 1536, 2048, 3072, 4096] + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1073000000000,"lazy_offload":true}}' + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: mnnvl + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1073' + +override_cap16_tp4_c32_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-active-seqs: 16 + gpu-memory-utilization: 0.925 + max-cudagraph-capture-size: 4096 + cudagraph-capture-sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 640, 768, 1024, 1536, 2048, 3072, 4096] + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1073000000000,"lazy_offload":true}}' + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: mnnvl + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1073' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index daecee68c0..e8e6986cb0 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5558,7 +5558,7 @@ minimaxm3-fp4-b300-trtllm-agentic-mtp: # enabled by framework=dynamo-vllm. minimaxm3-fp4-b200-vllm-agentic-mtp: - image: vllm/vllm-openai:nightly-af7f9488c2210d67e1033ecdc845b087ee7fe92b + image: vllm/vllm-openai:nightly-af7f9488c2210d67e1033ecdc845b087ee7fe92b@sha256:8e2ffbb9f866de678b22e7446903b1ee5a5766c51a948013277477c462a68a77 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: cluster:b200-nscale @@ -5570,11 +5570,33 @@ minimaxm3-fp4-b200-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.683 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml } - - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20], srt-recipe: agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [15], srt-recipe: agentic.yaml:override_tp4_c15 } + - dram-utilization: 1.0 + search-space: + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [30], srt-recipe: agentic.yaml:override_tp4_c30_dram } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32], srt-recipe: agentic.yaml:override_tp4_c32_dram } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [38], srt-recipe: agentic.yaml:override_tp4_c38_dram } +minimaxm3-fp4-b200-vllm-agentic-mtp-c10: + image: vllm/vllm-openai:nightly-ac68c3087215e0a4f3cdfa218508c6aada57235d@sha256:98adb2311118263a453ec0acd5d4b72bda3db26cecc0fe8ffc94b9ab03c951f9 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: cluster:b200-nscale + precision: fp4 + framework: vllm + multinode: false + srt-recipe-dir: minimaxm3/vllm/b200-fp4-mtp + scenarios: + agentic-coding: + - dram-utilization: 0.683 + search-space: + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [10], srt-recipe: agentic.yaml:override_cap1_k5_tp4_c10 } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [10], srt-recipe: agentic.yaml:override_cap1_k5_tp8_c10 } - dram-utilization: 1.0 search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [15, 20, 25, 30, 32, 34, 36, 38, 40], srt-recipe: agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [20], srt-recipe: agentic.yaml:override_cap4_tp4_c20_dram } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [20], srt-recipe: agentic.yaml:override_cap3_tp4_c20_dram } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32], srt-recipe: agentic.yaml:override_cap6_tp4_c32_dram } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32], srt-recipe: agentic.yaml:override_cap16_tp4_c32_dram } minimaxm31flash-fp4-b200-vllm-agentic-dspark: image: vllm/vllm-openai:minimax-m31-flash@sha256:86f582dae25ea92802353727164ae4eb03b37b18baf9119b805ffaec2a7ef07c model: MiniMaxAI/MiniMax-M3.1-Flash @@ -5612,7 +5634,6 @@ minimaxm31flash-fp4-gb200-vllm-agentic-dspark: - search-space: - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } - { tp: 4, ep: 4, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 128], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } - minimaxm3-fp4-b200-trtllm-agentic-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 model: nvidia/MiniMax-M3-NVFP4 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 1dea6c7678..917413d64e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11043,3 +11043,13 @@ description: - 'Refine GB300 Kimi-K3 AgentX points to be good on all four p90 charts: replace the decode-capped 1P2D-2P5D points with uncapped 1P2D c48-64 and 3P4D c240, add specialized-decode 1P3D c16 and 1P2D c24/32, keep 2P3D c240 and aggregated TP16 c1, and drop aggregated TP8 c2/c4. Pin cp-kv-cache-interleave-size to 1 for the specialized-decode disaggregated points.' pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4373 + +- config-keys: + - minimaxm3-fp4-b200-vllm-agentic-mtp + - minimaxm3-fp4-b200-vllm-agentic-mtp-c10 + scenario-type: + - agentic-coding + description: + - "Pin the MiniMax-M3 B200 vLLM images by digest and retune the TP4 concurrency 15-38 entries, with DRAM KV offload sized to the 1,073 GB node allowance." + - "Add minimaxm3-fp4-b200-vllm-agentic-mtp-c10 with TP4 and TP8 concurrency 10 and TP4 DRAM concurrency 20 and 32 entries that cap active sequences." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4382