diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..eefabc3c6c --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,191 @@ +base: + schema: 2 + name: minimaxm31flash-fp4-b300-sglang-agentic + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # Cold loads of the 250 GB checkpoint plus JIT MSA kernel builds and graph capture outlast the default. + health_check: + max_attempts: 1440 + interval_seconds: 5 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + # Serve flags follow the cookbook's B300 cells at + # https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1. + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + trust-remote-code: true + reasoning-parser: minimax-m3 + tool-call-parser: minimax-m3 + # Seven-token DSpark, the bundled draft's dspark_block_size. Throughput runs + # switch to synthetic acceptance at the golden acceptance length. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 7 + speculative-use-block-verification: true + cuda-graph-backend-prefill: breakable + tokenizer-worker-num: 8 + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + FMHA_SM100_ALLOW_JIT: '1' + benchmark: + env: + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# TP4 Balanced cell (TRT-LLM routed MoE). The scheduler batch is 2x CONC (AgentX +# subagent fan-out), capped at the cookbook's 128; the decode graph batch matches it. + +override_tp4_c1: + roles: + agg: + args: + tensor-parallel-size: 4 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 32768 + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + mem-fraction-static: 0.8 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c4: + roles: + agg: + args: + tensor-parallel-size: 4 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 32768 + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + mem-fraction-static: 0.8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp4_c8: + roles: + agg: + args: + tensor-parallel-size: 4 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 32768 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + mem-fraction-static: 0.8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp4_c16: + roles: + agg: + args: + tensor-parallel-size: 4 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 32768 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + mem-fraction-static: 0.8 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp4_c32: + roles: + agg: + args: + tensor-parallel-size: 4 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 32768 + max-running-requests: 64 + cuda-graph-max-bs-decode: 64 + mem-fraction-static: 0.8 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: none + +# High-Throughput cell: TP8 with attention DP2 and EP8 (MegaMoE). The scheduler +# batch is 2x CONC up to the cookbook's 256, and each of the two attention-DP +# ranks captures decode graphs for half of it. + +override_tp8_ep8_c64: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + attn-dp-size: 2 + enable-dp-lm-head: true + expert-parallel-size: 8 + chunked-prefill-size: 65536 + max-running-requests: 128 + cuda-graph-max-bs-decode: 64 + mem-fraction-static: 0.8 + env: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '32768' + benchmark: + env: + CONC: '64' + KV_OFFLOADING: none + +override_tp8_ep8_c128: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + attn-dp-size: 2 + enable-dp-lm-head: true + expert-parallel-size: 8 + chunked-prefill-size: 65536 + max-running-requests: 256 + cuda-graph-max-bs-decode: 128 + mem-fraction-static: 0.8 + env: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '32768' + benchmark: + env: + CONC: '128' + KV_OFFLOADING: none + +override_tp8_ep8_c256: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + attn-dp-size: 2 + enable-dp-lm-head: true + expert-parallel-size: 8 + chunked-prefill-size: 65536 + max-running-requests: 256 + cuda-graph-max-bs-decode: 128 + mem-fraction-static: 0.8 + env: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '32768' + benchmark: + env: + CONC: '256' + KV_OFFLOADING: none diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 1a9bfc58c7..1f7ec0b5a1 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9444,6 +9444,30 @@ dsv41flash-fp4-b300-sglang-agentic-dspark: - { tp: 2, ep: 2, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [64, 256], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [256], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } +# MiniMax-M3.1-Flash AgentX on B300 with SGLang: day-zero image from the SGLang +# cookbook (https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1), +# manifest digest verified 2026-10-24. Serve flags follow the cookbook's +# B300 Balanced (TP4) and High-Throughput (TP8, attention DP2, EP8). +# Seven-token DSpark (bundled draft); synthetic acceptance follows +# infx/golden_al_distribution/minimaxm31flash_dspark.yaml. +minimaxm31flash-fp4-b300-sglang-agentic-dspark: + image: lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87 + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:b300-dsxe + precision: fp4 + framework: sglang + multinode: false + srt-recipe-dir: minimaxm31flash/sglang/b300-fp4-mtp + scenarios: + agentic-coding: + - search-space: + # Balanced: TP4. + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } + # High-Throughput: TP8 + EP8 with attention DP2 inside the server (--attn-dp-size 2, + # not full DP attention, so dp-attn stays false). + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 128, 256], srt-recipe: agentic.yaml } + # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: image: lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4@sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 8b10b8473e..d546a08e59 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11082,6 +11082,15 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4444 scenario-type: [agentic-coding] +- config-keys: + - minimaxm31flash-fp4-b300-sglang-agentic-dspark + description: + - "Add day-zero MiniMax-M3.1-Flash (MiniMaxAI/MiniMax-M3.1-Flash, NVFP4) AgentX on B300 with SGLang image lmsysorg/sglang:dev-minimax-m31-flash (https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1)." + - "Serve flags follow the cookbook's B300 Balanced (TP4) and High-Throughput (TP8, attention DP2, EP8): DSpark with block verification, breakable prefill graphs, minimax-m3 reasoning and tool parsers, 8 tokenizer workers." + - "Seven-token DSpark (the bundled draft's dspark_block_size); throughput runs use synthetic acceptance from the minimaxm31flash_dspark golden curve (AL 4.04 at 7 draft tokens, #4425)." + - "Search space: Balanced TP4 at conc 1-32; TP8 + EP8 with attention DP2 at conc 64-256." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4441 + - config-keys: - minimaxm31flash-fp4-mi355x-sglang-agentic-dspark description: