Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
# MiniMax-M3.1-Flash AgentX on MI355X with vLLM, following the MI355X TP4
# recipe at https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3.1-Flash and
# https://vllm.ai/blog/2026-10-23-minimax-m31-flash, with native DSpark.
# Checkpoint: AMD's Quark export amd/MiniMax-M3.1-Flash-Quark-MXFP4 (routed
# experts stored in OCP MXFP4, run as W4A4 on the AITER MoE; every other layer
# and the dspark/ draft as in MiniMaxAI/MiniMax-M3.1-Flash). Long prefill
# steps run with dual-batch overlap (DBO). Sequence-parallel prefill
# (VLLM_ROCM_MINIMAX_M3_SP_PREFILL) stays off: the image does not
# enable it on this checkpoint.
base:
schema: 2
name: minimaxm31flash-fp4-mi355x-vllm-agentic
resources:
gpu_type: mi355x
gpus_per_node: 8
frontend:
type: vllm
enable_multiple_frontends: false
observability:
enabled: false
tachometer:
enabled: false
engine:
type: vllm
connector: null
# Cold loads of the 240 GB checkpoint can take the full
# VLLM_ENGINE_READY_TIMEOUT_S.
health_check:
interval_seconds: 10
max_attempts: 720
roles:
agg:
nodes: 1
workers: 1
gpus: 4
args:
served-model-name: amd/MiniMax-M3.1-Flash-Quark-MXFP4
tensor-parallel-size: 4
gpu-memory-utilization: 0.9
language-model-only: true
trust-remote-code: true
reasoning-parser: minimax_m3
tool-call-parser: minimax_m3
enable-auto-tool-choice: true
# MSA sparse/index cache needs 128-token blocks.
block-size: 128
# The checkpoint is QAT-trained for an NVFP4 KV cache; decode indexer
# scoring splits across TP ranks by context.
kv-cache-dtype: nvfp4
attention-config: '{"indexer_kv_dtype":"nvfp4","indexer_decode_cp":true}'
# The export's MXFP4 experts load as stored (W4A4, AITER_MXFP4_MXFP4).
moe-backend: aiter
disable-hybrid-kv-cache-manager: true
enable-prefix-caching: true
max-model-len: 1048576
# 32K-token prefill steps, so DBO (from 16384 tokens) can split them.
max-num-batched-tokens: 32768
# DBO: a prefill step of 16384+ tokens runs as two microbatches, one's
# all-reduce under the other's compute; decode steps stay whole.
enable-dbo: true
dbo-prefill-token-threshold: 16384
dbo-decode-token-threshold: 32768
disable-uvicorn-access-log: true
env:
# The export's recommended MoE setup: AITER MoE, and the MXFP8 shared
# expert stays a separate MLP (no shared-expert fusion).
VLLM_ROCM_USE_AITER: '1'
VLLM_ROCM_USE_AITER_MOE: '1'
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: '0'
# DBO needs the V2 model runner (the V1 runner refuses DSpark).
VLLM_USE_V2_MODEL_RUNNER: '1'
# Bind each HIP stream to one hardware queue. The image's HIP moves an
# active stream to the least loaded queue, which can put DBO's
# all-reduce stream on the compute stream's queue and serialize them.
DEBUG_HIP_DYNAMIC_QUEUES: '0'
PYTHONNOUSERSITE: '1'
PYTHONUNBUFFERED: '1'
VLLM_ENGINE_READY_TIMEOUT_S: '7200'
# Outlast AIPerf's pooled-connection idle gaps.
VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900'
benchmark:
env:
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:'

# Native zip expansion creates one variant per concurrency. TP4 is the
# upstream MI355X layout; admission is 2x CONC. Throughput runs replace block
# rejection with the golden acceptance length.

# c1-c8: DSpark 7 with the recipe's mono decode. A pure decode step of up to
# 16 tokens (2 requests x 8) runs each layer as one fused kernel; larger steps
# and steps with prefill run on the normal path, eager. AgentX tool-call gaps
# keep fewer requests decoding than CONC, so c4 and c8 still reach it. Mono
# needs FP8 KV and indexer caches in the shuffled KV layout and no piecewise
# graphs. No DBO here: 8192-token prefill steps on the V1 runner.
zip_override_tp4_dspark7_mono:
roles:
agg:
args:
gpu-memory-utilization: 0.85
kv-cache-dtype: fp8
attention-config: '{"indexer_kv_dtype":"fp8"}'
compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}'
max-num-batched-tokens: 8192
enable-dbo: null
dbo-prefill-token-threshold: null
dbo-decode-token-threshold: null
speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"TRITON_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
max-num-seqs: [2, 4, 8, 16]
env:
VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT: '1'
VLLM_ROCM_MINIMAX_M3_MONO: '1'
VLLM_USE_V2_MODEL_RUNNER: null
DEBUG_HIP_DYNAMIC_QUEUES: null
benchmark:
env:
CONC: ['1', '2', '4', '8']

# c16-c64: DSpark 3 (the blog's MI355X draft length from c8) on the NVFP4 KV
# cache with DBO.
zip_override_tp4_dspark3:
roles:
agg:
args:
speculative-config: '{"method":"dspark","num_speculative_tokens":3,"attention_backend":"TRITON_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
max-num-seqs: [32, 64, 128]
benchmark:
env:
CONC: ['16', '32', '64']
21 changes: 21 additions & 0 deletions inferencex-e2e/configs/amd-master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1532,6 +1532,27 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark:
- { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: agentic.yaml }
- { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: agentic.yaml }

# MiniMax-M3.1-Flash AgentX on MI355X: day-zero vLLM ROCm image from
# https://vllm.ai/blog/2026-10-23-minimax-m31-flash, verified 2026-10-23.
# AMD's Quark MXFP4 export (routed experts in MXFP4, W4A4 on the AITER MoE)
# at TP4 with native DSpark: 7 drafts, mono decode and an FP8 KV cache at
# c1-c8; 3 drafts, an NVFP4 KV cache and DBO from c16. Throughput runs use the
# minimaxm31flash_dspark golden AL.
minimaxm31flash-fp4-mi355x-vllm-agentic-dspark:
image: vllm/vllm-openai-rocm:minimax-m31-flash@sha256:ae8681212aa5291d8c27f06e90016435e3fca144a68fd5c439231f6475f74569
model: amd/MiniMax-M3.1-Flash-Quark-MXFP4
model-prefix: minimaxm31flash
runner: cluster:mi355x-amds
precision: fp4
framework: vllm
multinode: false
srt-recipe-dir: minimaxm31flash/vllm/mi355x-fp4
scenarios:
agentic-coding:
- dram-utilization: 0.80
search-space:
- { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: agentic.yaml }

# DeepSeek-V4.1-Flash on ATOM, following the upstream TP2 / TP4 AgentX recipe.
# Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval.
dsv41flash-fp4-mi355x-atom-agentic-dspark:
Expand Down
10 changes: 10 additions & 0 deletions inferencex-e2e/perf-changelog.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10929,6 +10929,16 @@
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4408
scenario-type: [agentic-coding]

- config-keys:
- minimaxm31flash-fp4-mi355x-vllm-agentic-dspark
description:
- "Add day-zero MiniMax-M3.1-Flash AgentX on MI355X with vLLM image vllm/vllm-openai-rocm:minimax-m31-flash (https://vllm.ai/blog/2026-10-23-minimax-m31-flash), on AMD's Quark export amd/MiniMax-M3.1-Flash-Quark-MXFP4 (MXFP4 routed experts + NVFP4 KV)."
- "Serve flags follow the recipes.vllm.ai MI355X TP4 recipe: block size 128, nvfp4 KV and indexer cache with indexer decode CP, hybrid KV cache manager disabled, minimax_m3 reasoning and tool parsers, 1M context, native DSpark (TRITON_ATTN drafter, fp8 draft KV, probabilistic draft sampling). MoE as the export recommends: MXFP4 experts loaded as stored on the AITER W4A4 MoE (moe-backend aiter), no shared-expert fusion."
- "Search space: TP4 at conc 1-64, no KV offloading; throughput runs use the minimaxm31flash_dspark golden AL curve."
- "Conc 1-8: DSpark 7 with the recipe's mono decode (VLLM_ROCM_MINIMAX_M3_MONO=1): a pure decode step of up to 16 tokens runs each layer as one fused kernel, on FP8 KV and indexer caches with FULL_DECODE_ONLY graphs; AgentX tool-call gaps keep c4 and c8 steps small enough to reach it."
- "Conc 16-64: DSpark 3 (the blog's MI355X draft length from c8) on the NVFP4 KV cache, with 32K-token prefill steps under DBO."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4420

- config-keys:
- minimaxm31flash-fp4-h200-vllm-agentic-dspark
description:
Expand Down
Loading