diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..486fee3239 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml @@ -0,0 +1,118 @@ +schema: 2 + +# MiniMax-M3.1-Flash AgentX on two GB300 trays (8 GPUs), aggregated through +# Dynamo vLLM. DEP8 only: the NVFP4 KV-cache attention kernels need 16 query +# heads per KV head on each rank (64 query / 4 KV heads), so attention TP8 does +# not start; low and mid concurrency stay on the single-tray TP4 recipe. Serve flags follow the single-node GB300 recipe +# (single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp); the +# Dynamo setup follows minimaxm3/vllm/gb200-fp4/agentx. +base: + identity: + model: {} + container: {} + frameworks: {dynamo: "1.6.0.dev20261016"} + # 1.6.0.dev20261014+ imports the vLLM engine-launch helpers lazily + # (ai-dynamo/dynamo#16378); earlier builds fail at import on this vLLM. + dynamo: {install: true, source: {pypi: "1.6.0.dev20261016"}} + environment: {ETCD_LEASE_TTL: "7200"} + slurm: {time_limit: "12:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: "gb300" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + trust-remote-code: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 + engine: + type: vllm + connector: + roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + # Rendering the chat template for long AgentX conversations exceeds the + # 30 s default under concurrent load; vLLM then aborts the request. + VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: "600" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "@fabric.rdma-device-ports" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "@fabric.rdma-devices" + args: + served-model-name: "MiniMaxAI/MiniMax-M3.1-Flash" + pipeline-parallel-size: 1 + trust-remote-code: true + language-model-only: true + enable-prefix-caching: true + kv-cache-metrics: true + # MSA sparse/index cache needs 128-token blocks. + block-size: 128 + # The checkpoint is QAT-trained for an NVFP4 KV cache. + kv-cache-dtype: "nvfp4" + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + max-model-len: 1048576 + max-num-batched-tokens: 8192 + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + kv_events: true + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + +# DEP8: TP1 x DP8 + EP8, one DP launch per tray, with vLLM Simple CPU KV +# offload. Each DP rank gets a quarter of its tray's DRAM allowance (179 GiB of +# the 771 GB that dram-utilization 0.80 leaves per tray); three-token DSpark +# (golden AL 2.88). +override_dep8_vllm_simple: + name: "minimaxm31flash-vllm-agg-gb300-dep8-vllm-simple-agentic" + engine: + dp_launch_mode: per_node + roles: + agg: + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + max-num-seqs: 32 + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":192199557120,"cpu_bytes_to_use_per_rank":192199557120,"lazy_offload":true}}' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 1b6ec2946f..d46fe0a069 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5608,6 +5608,38 @@ minimaxm31flash-fp4-gb200-vllm-agentic-dspark: - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } - { tp: 4, ep: 4, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 128], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } +# MiniMax-M3.1-Flash AgentX on two GB300 trays (8 GPUs) through Dynamo vLLM: +# DEP8 (TP1 x DP8 + EP8) with vLLM Simple CPU KV offload for high concurrency, +# 3-token DSpark on the FlashInfer PrimiTS drafter. Attention TP8 is not +# possible with the NVFP4 KV cache (16 query heads per KV head per rank needed), +# so low and mid concurrency stay on minimaxm31flash-fp4-gb300-vllm-agentic-dspark. +minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg: + image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.6.0.dev20261016" } + multinode: true + disagg: false + srt-recipe-dir: minimaxm31flash/vllm/gb300-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [72, 96, 128] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_dep8_vllm_simple + worker: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + minimaxm3-fp4-b200-trtllm-agentic-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 model: nvidia/MiniMax-M3-NVFP4 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 06d2fb2edc..d71eb6c21a 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10997,3 +10997,12 @@ - "Add day-0 MiniMax-M3.1-Flash (MiniMaxAI/MiniMax-M3.1-Flash, NVFP4 experts + NVFP4 KV cache) AgentX on GB200, matching the B200 recipe (#4426) with the arm64 build of the same day-0 vLLM image (minimax-m31-flash-arm64-cu134)." - "DSpark with the built-in drafter (probabilistic drafting, FLASH_ATTN drafter, fp8 drafter KV); TP4 c1-c32 drafting 7 tokens (golden AL 4.04); DEP4 (TP1 x DP4 + EP4 behind vllm-router) c64-c128 drafting 3 tokens (golden AL 2.88)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4433 + +- config-keys: + - minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg + scenario-type: [agentic-coding] + description: + - "Add MiniMax-M3.1-Flash AgentX on two GB300 trays (8 GPUs) aggregated through Dynamo vLLM, with the single-node GB300 serve flags (NVFP4 KV and indexer cache, 600 s chat-template render timeout) and the day-0 arm64 vLLM image." + - "DEP8 (TP1 x DP8 + EP8, per-node DP launch) c72/c96/c128 with 3-token DSpark on the FlashInfer PrimiTS drafter (golden AL 2.88) and vLLM Simple CPU KV offload (179 GiB per DP rank, lazy), on Dynamo 1.6.0.dev20261016." + - "No TP8/TEP8 points: vLLM's NVFP4 KV-cache attention needs 16 query heads per KV head on each rank (64 query / 4 KV heads), which attention TP8 halves to 8." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4436