Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
base:
schema: 2
name: minimaxm31flash-fp4-b200-vllm-agentic
resources:
gpu_type: b200
gpus_per_node: 8
frontend:
type: vllm
enable_multiple_frontends: false
observability:
enabled: false
tachometer:
enabled: false
engine:
type: vllm
connector: null
# Cold loads of the 250 GB checkpoint from the hub cache run past half an hour.
health_check:
interval_seconds: 5
max_attempts: 1440
roles:
agg:
nodes: 1
workers: 1
args:
served-model-name: MiniMaxAI/MiniMax-M3.1-Flash
language-model-only: true
trust-remote-code: true
reasoning-parser: minimax_m3
tool-call-parser: minimax_m3
enable-auto-tool-choice: true
# MSA sparse/index cache needs 128-token blocks.
block-size: 128
# The checkpoint is QAT-trained for an NVFP4 KV cache.
kv-cache-dtype: nvfp4
attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}'
disable-hybrid-kv-cache-manager: true
enable-prefix-caching: true
max-model-len: 1048576
max-num-batched-tokens: 8192
compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}'
disable-uvicorn-access-log: true
env:
VLLM_FLOAT32_MATMUL_PRECISION: high
PYTHONUNBUFFERED: '1'
VLLM_ENGINE_READY_TIMEOUT_S: '7200'
benchmark:
env:
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:'

zip_override_tp4:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
max-num-seqs: [2, 8, 16, 32, 64]
# Golden AL 4.04.
speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: ['1', '4', '8', '16', '32']

zip_override_dep4:
frontend:
type: vllm-router
args:
policy: consistent_hash
prometheus-host: 127.0.0.1
prometheus-port: 18000
request-timeout-secs: 14400
disable-retries: true
setup_script: vllm-router.sh
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 1
Comment thread
Oseltamivir marked this conversation as resolved.
Resolved
data-parallel-size: 4
enable-expert-parallel: true
max-num-seqs: [32, 64]
# Golden AL 2.88.
speculative-config: '{"method":"dspark","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: ['64', '128']
15 changes: 15 additions & 0 deletions inferencex-e2e/configs/nvidia-master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5529,6 +5529,21 @@ minimaxm3-fp4-b200-vllm-agentic-mtp:
- dram-utilization: 1.0
search-space:
- { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [15, 20, 25, 30, 32, 34, 36, 38, 40], srt-recipe: agentic.yaml }
minimaxm31flash-fp4-b200-vllm-agentic-dspark:
image: vllm/vllm-openai:minimax-m31-flash@sha256:86f582dae25ea92802353727164ae4eb03b37b18baf9119b805ffaec2a7ef07c
model: MiniMaxAI/MiniMax-M3.1-Flash
model-prefix: minimaxm31flash
runner: cluster:b200-nscale
precision: fp4
framework: vllm
multinode: false
srt-recipe-dir: minimaxm31flash/vllm/b200-fp4-mtp
scenarios:
agentic-coding:
- search-space:
- { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml }
- { tp: 4, ep: 4, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 128], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml }

minimaxm3-fp4-b200-trtllm-agentic-mtp:
image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1
model: nvidia/MiniMax-M3-NVFP4
Expand Down
10 changes: 10 additions & 0 deletions inferencex-e2e/perf-changelog.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10872,6 +10872,16 @@
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4395
scenario-type: [agentic-coding]

- config-keys:
- minimaxm31flash-fp4-b200-vllm-agentic-dspark
scenario-type: [agentic-coding]
description:
- "Add day-0 MiniMax-M3.1-Flash (MiniMaxAI/MiniMax-M3.1-Flash, NVFP4 experts + NVFP4 KV cache) AgentX recipe on B200 with the vLLM day-0 image vllm/vllm-openai:minimax-m31-flash."
- "Serve flags follow recipes.vllm.ai Blackwell TP4: block-size 128, nvfp4 KV and indexer cache, prefill query quantization, FULL_AND_PIECEWISE graphs, minimax_m3 parsers, text-only."
- "DSpark with the built-in drafter (probabilistic drafting, FLASH_ATTN drafter, fp8 drafter KV); synthetic acceptance from the golden curve minimaxm31flash_dspark.yaml (#4425)."
- "TP4 c1-c32 drafting 7 tokens (golden AL 4.04); DEP4 (TP1 x DP4 + EP4 behind vllm-router) c64-c128 drafting 3 tokens (golden AL 2.88)."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4426

- config-keys:
- dsv41flash-fp4-h100-vllm-agentic-dspark
scenario-type:
Expand Down
Loading