From 56d4dcd70050eaaf6d45b4ad9e142ae223974f64 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 8 Oct 2026 10:03:50 -0700 Subject: [PATCH 1/5] config: update B300 MiniMax-M3 TRT-LLM AgentX recipe --- .../trtllm/b300-fp4-mtp/agentic.yaml | 333 +++++++++++++----- inferencex-e2e/configs/nvidia-master.yaml | 8 +- inferencex-e2e/perf-changelog.yaml | 9 + 3 files changed, 254 insertions(+), 96 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml index 8f2a74bbdd..7eb965e31d 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml @@ -5,7 +5,7 @@ base: name: minimaxm3-fp4-b300-trtllm-agentic model: path: hf:nvidia/MiniMax-M3-NVFP4 - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 + container: nvcr.io#nvidia/tensorrt-llm/release:1.4.0rc0.dev202610070001 precision: fp4 resources: gpu_type: b300 @@ -51,11 +51,13 @@ base: kv_cache_config: free_gpu_memory_fraction: 0.94 enable_block_reuse: true - block_reuse_policy: per_conversation + block_reuse_config: + policy: per_conversation tokens_per_block: 128 use_kv_cache_manager_v2: true - dtype: fp8 + dtype: nvfp4 event_buffer_max_size: 0 + iteration_stats_interval: 10 speculative_config: decoding_type: Eagle3 max_draft_len: 3 @@ -69,14 +71,16 @@ base: stream_interval: 20 print_iter_log: true enable_iter_perf_stats: true + iter_perf_stats_interval: 10 return_perf_metrics: true num_postprocess_workers: 8 + enable_tokenization_cache: true enable_attention_dp: false env: TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - TLLM_PROFILE_LOG_RANKS: all + TLLM_PROFILE_LOG_RANKS: '0' PYTHONNOUSERSITE: '1' TRTLLM_ENABLE_PDL: '1' ENROOT_ALLOW_DEV: 'yes' @@ -85,10 +89,11 @@ base: TQDM_DISABLE: '1' HF_HUB_DISABLE_PROGRESS_BARS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + MAXSMS: '32' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - # Piecewise graph capture and autotuning took 22 minutes at TP4 c5; larger - # batch ladders and TP8 outlast srt-slurm's 1800 s default. + # TP2 uses the short capture ladder; TP4/TP8 variants use the measured + # extended ladder. Synthetic acceptance is selected from the golden curve. health_check: max_attempts: 1440 interval_seconds: 5 @@ -102,9 +107,66 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization KV_OFFLOADING: dram -# One variant per point. The batch limit is CONC with graphs captured at every -# size to 20 and a thinned ladder at 25 and 30. The host pool is 288 GiB at TP8 -# and about 362 GiB below. +# One variant per measured point. Draft length and capture ladder follow the +# matching local ser_tp*_mtp*_c*.yaml profile. + +override_tp2_c10: + roles: + agg: + gpus: 2 + args: + tensor_parallel_size: 2 + max_batch_size: 10 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '10' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c15: + roles: + agg: + gpus: 2 + args: + tensor_parallel_size: 2 + max_batch_size: 15 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '15' + TOTAL_CPU_DRAM_GB: '599' + override_tp2_c20: roles: agg: @@ -213,13 +275,13 @@ override_tp2_c30: CONC: '30' TOTAL_CPU_DRAM_GB: '599' -override_tp4_c5: +override_tp2_c35: roles: agg: - gpus: 4 + gpus: 2 args: - tensor_parallel_size: 4 - max_batch_size: 5 + tensor_parallel_size: 2 + max_batch_size: 35 cuda_graph_config: batch_sizes: - 1 @@ -227,20 +289,37 @@ override_tp4_c5: - 3 - 4 - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 26 + - 28 + - 30 + - 32 + - 34 + - 35 kv_cache_config: host_cache_size: 388554555392 benchmark: env: - CONC: '5' - TOTAL_CPU_DRAM_GB: '1199' + CONC: '35' + TOTAL_CPU_DRAM_GB: '599' -override_tp4_c10: +override_tp2_c40: roles: agg: - gpus: 4 + gpus: 2 args: - tensor_parallel_size: 4 - max_batch_size: 10 + tensor_parallel_size: 2 + max_batch_size: 40 cuda_graph_config: batch_sizes: - 1 @@ -253,20 +332,34 @@ override_tp4_c10: - 8 - 9 - 10 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 26 + - 28 + - 30 + - 32 + - 34 + - 36 + - 38 + - 40 kv_cache_config: host_cache_size: 388554555392 benchmark: env: - CONC: '10' - TOTAL_CPU_DRAM_GB: '1199' + CONC: '40' + TOTAL_CPU_DRAM_GB: '599' -override_tp4_c15: +override_tp2_c45: roles: agg: - gpus: 4 + gpus: 2 args: - tensor_parallel_size: 4 - max_batch_size: 15 + tensor_parallel_size: 2 + max_batch_size: 45 cuda_graph_config: batch_sizes: - 1 @@ -279,25 +372,70 @@ override_tp4_c15: - 8 - 9 - 10 - - 11 - - 12 - - 13 - 14 - - 15 + - 16 + - 18 + - 20 + - 22 + - 24 + - 26 + - 28 + - 30 + - 32 + - 34 + - 36 + - 38 + - 40 + - 42 + - 44 + - 45 kv_cache_config: host_cache_size: 388554555392 benchmark: env: - CONC: '15' + CONC: '45' + TOTAL_CPU_DRAM_GB: '599' + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 1 + cuda_graph_config: + batch_sizes: + - 1 + kv_cache_config: + host_cache_size: 388554555392 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 + speculative_config: + max_draft_len: 7 + benchmark: + env: + CONC: '1' TOTAL_CPU_DRAM_GB: '1199' -override_tp4_c20: +override_tp4_c5: roles: agg: gpus: 4 args: tensor_parallel_size: 4 - max_batch_size: 20 + max_batch_size: 5 cuda_graph_config: batch_sizes: - 1 @@ -305,35 +443,36 @@ override_tp4_c20: - 3 - 4 - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 13 - - 14 - - 15 - - 16 - - 17 - - 18 - - 19 - - 20 kv_cache_config: host_cache_size: 388554555392 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 + speculative_config: + max_draft_len: 5 benchmark: env: - CONC: '20' + CONC: '5' TOTAL_CPU_DRAM_GB: '1199' -override_tp4_c25: +override_tp4_c10: roles: agg: gpus: 4 args: tensor_parallel_size: 4 - max_batch_size: 25 + max_batch_size: 10 cuda_graph_config: batch_sizes: - 1 @@ -346,30 +485,36 @@ override_tp4_c25: - 8 - 9 - 10 - - 11 - - 12 - - 13 - - 14 - - 15 - - 17 - - 19 - - 21 - - 23 - - 25 kv_cache_config: host_cache_size: 388554555392 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 + speculative_config: + max_draft_len: 5 benchmark: env: - CONC: '25' + CONC: '10' TOTAL_CPU_DRAM_GB: '1199' -override_tp4_c30: +override_tp4_c15: roles: agg: gpus: 4 args: tensor_parallel_size: 4 - max_batch_size: 30 + max_batch_size: 15 cuda_graph_config: batch_sizes: - 1 @@ -384,19 +529,28 @@ override_tp4_c30: - 10 - 11 - 12 + - 13 - 14 - - 16 - - 18 - - 20 - - 22 - - 24 - - 27 - - 30 + - 15 kv_cache_config: host_cache_size: 388554555392 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 benchmark: env: - CONC: '30' + CONC: '15' TOTAL_CPU_DRAM_GB: '1199' override_tp8_c1: @@ -411,28 +565,23 @@ override_tp8_c1: - 1 kv_cache_config: host_cache_size: 309237645312 - benchmark: - env: - CONC: '1' - TOTAL_CPU_DRAM_GB: '2399' - -override_tp8_c5: - roles: - agg: - gpus: 8 - args: - tensor_parallel_size: 8 - max_batch_size: 5 - cuda_graph_config: - batch_sizes: + torch_compile_config: + capture_num_tokens: - 1 - - 2 - - 3 - - 4 - - 5 - kv_cache_config: - host_cache_size: 309237645312 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 + speculative_config: + max_draft_len: 7 benchmark: env: - CONC: '5' + CONC: '1' TOTAL_CPU_DRAM_GB: '2399' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index ac296d2809..af0f729130 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5783,7 +5783,7 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p5d_tp2_tp2_c120_eval" decode: { num-worker: 5, tp: 2, ep: 1, dp-attn: false } minimaxm3-fp4-b300-trtllm-agentic-mtp: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 + image: nvcr.io#nvidia/tensorrt-llm/release:1.4.0rc0.dev202610070001 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: cluster:b300-dsxe @@ -5794,9 +5794,9 @@ minimaxm3-fp4-b300-trtllm-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [20, 25, 30], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [10, 15, 20, 25, 30, 35, 40, 45], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5, 10, 15], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve # covers the middle/high-interactivity range omitted by the one-decode DEP diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index a93c093d41..097f81b43d 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9410,3 +9410,12 @@ - "TP4 concurrency 32 sets ATOM_PREFILL_DECODE_INTERVAL=16: after each prefill, ATOM runs at least 16 decode passes and batches the prefills that arrive in the meantime (ATOM's PrefillDelayer, off by default). Prefill no longer interrupts decode as often, so P90 interactivity rises while TTFT rises. Local 3600 s replay: 108.2 -> 150.9 tok/s/user, ITL p90 9.24 -> 6.63 ms, TTFT p50 0.51 -> 1.14 s, throughput unchanged." - "TP2 concurrency 64 sets ATOM_PREFILL_DECODE_INTERVAL=4. At this load each batched prefill forward serves several requests, so throughput rises as well as interactivity. Local 3600 s replay: 30.3 -> 38.8 tok/s/user, 105.4k -> 115.2k tok/s/GPU, 6487 -> 6941 requests, TTFT p90 2.3 -> 6.4 s. All other points are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3603 + +- config-keys: + - minimaxm3-fp4-b300-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the B300 TRT-LLM MiniMax-M3 AgentX recipe to the 13 scheduled TP2/TP4/TP8 profiles with NVFP4 KV cache, EAGLE3 speculative decoding, and native DRAM KV offload." + - "Pin the recipe and master config image to nvcr.io#nvidia/tensorrt-llm/release:1.4.0rc0.dev202610070001." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD From 0a1d0659cdbacefab280b9e4fc3dd4db69d0b972 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 8 Oct 2026 10:04:46 -0700 Subject: [PATCH 2/5] config: add PR link to perf changelog --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 097f81b43d..646a97c128 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9418,4 +9418,4 @@ description: - "Update the B300 TRT-LLM MiniMax-M3 AgentX recipe to the 13 scheduled TP2/TP4/TP8 profiles with NVFP4 KV cache, EAGLE3 speculative decoding, and native DRAM KV offload." - "Pin the recipe and master config image to nvcr.io#nvidia/tensorrt-llm/release:1.4.0rc0.dev202610070001." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3835 From 329af637d9f8e6eba3427fc6126ef3554d5f3eea Mon Sep 17 00:00:00 2001 From: Ankur Singh Date: Fri, 23 Oct 2026 12:23:32 -0700 Subject: [PATCH 3/5] Drop the TRT-LLM chat store patch and pin MiniMax-M3 thinking mode for B300 AgentX --- .../minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml | 4 +++- inferencex-e2e/perf-changelog.yaml | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml index 9e0928f5fd..2fcf6a6e65 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml @@ -14,7 +14,6 @@ base: engine: type: trtllm served_model_name: nvidia/MiniMax-M3-NVFP4 - setup_script: minimaxm3-trtllm-agentx.sh roles: agg: nodes: 1 @@ -97,6 +96,9 @@ base: AIPERF_SERVER_METRICS_URLS: http://localhost:8000/prometheus/metrics AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization KV_OFFLOADING: dram + # Pin MiniMax-M3 thinking on for the replay, matching the thinking_on + # golden acceptance curve. The client splits this value on whitespace. + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"thinking_mode":"enabled"}}' # One variant per measured point. Draft length and capture ladder follow the # matching local ser_tp*_mtp*_c*.yaml profile. The host pool is 288 GiB at TP8 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 29ef713108..5d1ac0a657 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11060,5 +11060,8 @@ - agentic-coding description: - "Update the B300 TRT-LLM MiniMax-M3 AgentX recipe to the 13 scheduled TP2/TP4/TP8 profiles with NVFP4 KV cache, EAGLE3 speculative decoding, and native DRAM KV offload." - - "Pin the recipe and master config image to nvcr.io#nvidia/tensorrt-llm/release:1.4.0rc0.dev202610070001." + - "Pin the master config image to nvcr.io#nvidia/tensorrt-llm/release:1.4.0rc0.dev202610070001." + - "Drop the TensorRT-LLM chat store patch (setup_script minimaxm3-trtllm-agentx.sh) from the B300 recipe." + - "Pin thinking_mode enabled for the AgentX replay through chat_template_kwargs (AIPERF_EXTRA_INPUTS), matching the thinking_on golden acceptance curve." + - "Rebase the host pools onto the #3804 DRAM policy: dram-utilization 1.0, host_cache_size 374500000000 bytes per rank at TP2, 374750000000 at TP4 and 309237645312 at TP8, TOTAL_CPU_DRAM_GB 749/1499/2999." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3835 From 6055328a380b0838863cac199ff19a9e79767a3f Mon Sep 17 00:00:00 2001 From: Ankur Singh Date: Mon, 26 Oct 2026 12:03:14 -0700 Subject: [PATCH 4/5] Add the MiniMax-M3 tool parser and keep TP4 c20/c25 and TP8 c5 for B300 TRT-LLM AgentX --- .../trtllm/b300-fp4-mtp/agentic.yaml | 138 +++++++++++++++++- inferencex-e2e/configs/nvidia-master.yaml | 4 +- inferencex-e2e/perf-changelog.yaml | 3 +- 3 files changed, 141 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml index 2fcf6a6e65..f9a80dd78b 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml @@ -18,7 +18,8 @@ base: agg: nodes: 1 workers: 1 - extra_args: [--chat_template, /model/chat_template.jinja] + # --tool_parser is a trtllm-serve CLI flag; no YAML args key reaches it. + extra_args: [--chat_template, /model/chat_template.jinja, --tool_parser, minimax_m3] args: moe_expert_parallel_size: 1 max_seq_len: 1048576 @@ -547,6 +548,106 @@ override_tp4_c15: CONC: '15' TOTAL_CPU_DRAM_GB: '1499' +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 20 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + kv_cache_config: + host_cache_size: 374750000000 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 + benchmark: + env: + CONC: '20' + TOTAL_CPU_DRAM_GB: '1499' + +override_tp4_c25: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 25 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 17 + - 19 + - 21 + - 23 + - 25 + kv_cache_config: + host_cache_size: 374750000000 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 + benchmark: + env: + CONC: '25' + TOTAL_CPU_DRAM_GB: '1499' + override_tp8_c1: roles: agg: @@ -579,3 +680,38 @@ override_tp8_c1: env: CONC: '1' TOTAL_CPU_DRAM_GB: '2999' + +override_tp8_c5: + roles: + agg: + gpus: 8 + args: + tensor_parallel_size: 8 + max_batch_size: 5 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + kv_cache_config: + host_cache_size: 309237645312 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 + benchmark: + env: + CONC: '5' + TOTAL_CPU_DRAM_GB: '2999' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 4d2f5ba7cb..ebf759a4c5 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5546,8 +5546,8 @@ minimaxm3-fp4-b300-trtllm-agentic-mtp: - dram-utilization: 1.0 search-space: - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [10, 15, 20, 25, 30, 35, 40, 45], srt-recipe: agentic.yaml } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5, 10, 15], srt-recipe: agentic.yaml } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1], srt-recipe: agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5, 10, 15, 20, 25], srt-recipe: agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5], srt-recipe: agentic.yaml } # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve # covers the middle/high-interactivity range omitted by the one-decode DEP diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 5d1ac0a657..2eddc34789 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11059,9 +11059,10 @@ scenario-type: - agentic-coding description: - - "Update the B300 TRT-LLM MiniMax-M3 AgentX recipe to the 13 scheduled TP2/TP4/TP8 profiles with NVFP4 KV cache, EAGLE3 speculative decoding, and native DRAM KV offload." + - "Update the B300 TRT-LLM MiniMax-M3 AgentX recipe to 16 scheduled TP2/TP4/TP8 profiles with NVFP4 KV cache, EAGLE3 speculative decoding, and native DRAM KV offload; TP4 c20/c25 and TP8 c5 are kept from the published curve and TP4 c30 is dropped." - "Pin the master config image to nvcr.io#nvidia/tensorrt-llm/release:1.4.0rc0.dev202610070001." - "Drop the TensorRT-LLM chat store patch (setup_script minimaxm3-trtllm-agentx.sh) from the B300 recipe." - "Pin thinking_mode enabled for the AgentX replay through chat_template_kwargs (AIPERF_EXTRA_INPUTS), matching the thinking_on golden acceptance curve." - "Rebase the host pools onto the #3804 DRAM policy: dram-utilization 1.0, host_cache_size 374500000000 bytes per rank at TP2, 374750000000 at TP4 and 309237645312 at TP8, TOTAL_CPU_DRAM_GB 749/1499/2999." + - "Parse MiniMax-M3 tool calls with trtllm-serve --tool_parser minimax_m3." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3835 From 3a57440f325a9d66b70bcced7b7aa7891aba3b77 Mon Sep 17 00:00:00 2001 From: Ankur Singh Date: Mon, 26 Oct 2026 12:07:41 -0700 Subject: [PATCH 5/5] Drop the TP4 c15 point from the B300 TRT-LLM MiniMax-M3 AgentX curve --- .../trtllm/b300-fp4-mtp/agentic.yaml | 45 ------------------- inferencex-e2e/configs/nvidia-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 2 insertions(+), 47 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml index f9a80dd78b..f6267bba3e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml @@ -503,51 +503,6 @@ override_tp4_c10: CONC: '10' TOTAL_CPU_DRAM_GB: '1499' -override_tp4_c15: - roles: - agg: - gpus: 4 - args: - tensor_parallel_size: 4 - max_batch_size: 15 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 13 - - 14 - - 15 - kv_cache_config: - host_cache_size: 374750000000 - torch_compile_config: - capture_num_tokens: - - 1 - - 256 - - 512 - - 768 - - 1024 - - 1536 - - 2048 - - 3072 - - 4096 - - 6144 - - 8192 - - 12288 - benchmark: - env: - CONC: '15' - TOTAL_CPU_DRAM_GB: '1499' - override_tp4_c20: roles: agg: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index b840eb609e..375651d859 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5546,7 +5546,7 @@ minimaxm3-fp4-b300-trtllm-agentic-mtp: - dram-utilization: 1.0 search-space: - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [10, 15, 20, 25, 30, 35, 40, 45], srt-recipe: agentic.yaml } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5, 10, 15, 20, 25], srt-recipe: agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5, 10, 20, 25], srt-recipe: agentic.yaml } - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5], srt-recipe: agentic.yaml } # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3f8d2eb118..8fa04bafea 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11272,7 +11272,7 @@ scenario-type: - agentic-coding description: - - "Update the B300 TRT-LLM MiniMax-M3 AgentX recipe to 16 scheduled TP2/TP4/TP8 profiles with NVFP4 KV cache, EAGLE3 speculative decoding, and native DRAM KV offload; TP4 c20/c25 and TP8 c5 are kept from the published curve and TP4 c30 is dropped." + - "Update the B300 TRT-LLM MiniMax-M3 AgentX recipe to 15 scheduled TP2/TP4/TP8 profiles with NVFP4 KV cache, EAGLE3 speculative decoding, and native DRAM KV offload; TP4 c20/c25 and TP8 c5 are kept from the published curve, and TP4 c15 and c30 are dropped." - "Pin the master config image to nvcr.io#nvidia/tensorrt-llm/release:1.4.0rc0.dev202610070001." - "Drop the TensorRT-LLM chat store patch (setup_script minimaxm3-trtllm-agentx.sh) from the B300 recipe." - "Pin thinking_mode enabled for the AgentX replay through chat_template_kwargs (AIPERF_EXTRA_INPUTS), matching the thinking_on golden acceptance curve."