diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml index a5d7319918..f6267bba3e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml @@ -14,12 +14,12 @@ base: engine: type: trtllm served_model_name: nvidia/MiniMax-M3-NVFP4 - setup_script: minimaxm3-trtllm-agentx.sh roles: agg: nodes: 1 workers: 1 - extra_args: [--chat_template, /model/chat_template.jinja] + # --tool_parser is a trtllm-serve CLI flag; no YAML args key reaches it. + extra_args: [--chat_template, /model/chat_template.jinja, --tool_parser, minimax_m3] args: moe_expert_parallel_size: 1 max_seq_len: 1048576 @@ -45,11 +45,13 @@ base: kv_cache_config: free_gpu_memory_fraction: 0.94 enable_block_reuse: true - block_reuse_policy: per_conversation + block_reuse_config: + policy: per_conversation tokens_per_block: 128 use_kv_cache_manager_v2: true - dtype: fp8 + dtype: nvfp4 event_buffer_max_size: 0 + iteration_stats_interval: 10 speculative_config: decoding_type: Eagle3 max_draft_len: 3 @@ -63,14 +65,16 @@ base: stream_interval: 20 print_iter_log: true enable_iter_perf_stats: true + iter_perf_stats_interval: 10 return_perf_metrics: true num_postprocess_workers: 8 + enable_tokenization_cache: true enable_attention_dp: false env: TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - TLLM_PROFILE_LOG_RANKS: all + TLLM_PROFILE_LOG_RANKS: '0' PYTHONNOUSERSITE: '1' TRTLLM_ENABLE_PDL: '1' ENROOT_ALLOW_DEV: 'yes' @@ -79,10 +83,11 @@ base: TQDM_DISABLE: '1' HF_HUB_DISABLE_PROGRESS_BARS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + MAXSMS: '32' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - # Piecewise graph capture and autotuning took 22 minutes at TP4 c5; larger - # batch ladders and TP8 outlast srt-slurm's 1800 s default. + # TP2 uses the short capture ladder; TP4/TP8 variants use the measured + # extended ladder. Synthetic acceptance is selected from the golden curve. health_check: max_attempts: 1440 interval_seconds: 5 @@ -92,10 +97,71 @@ base: AIPERF_SERVER_METRICS_URLS: http://localhost:8000/prometheus/metrics AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization KV_OFFLOADING: dram + # Pin MiniMax-M3 thinking on for the replay, matching the thinking_on + # golden acceptance curve. The client splits this value on whitespace. + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"thinking_mode":"enabled"}}' -# One variant per point. The batch limit is CONC with graphs captured at every -# size to 20 and a thinned ladder at 25 and 30. The host pool is 288 GiB at TP8 +# One variant per measured point. Draft length and capture ladder follow the +# matching local ser_tp*_mtp*_c*.yaml profile. The host pool is 288 GiB at TP8 # and about 349 GiB below. + +override_tp2_c10: + roles: + agg: + gpus: 2 + args: + tensor_parallel_size: 2 + max_batch_size: 10 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + kv_cache_config: + host_cache_size: 374500000000 + benchmark: + env: + CONC: '10' + TOTAL_CPU_DRAM_GB: '749' + +override_tp2_c15: + roles: + agg: + gpus: 2 + args: + tensor_parallel_size: 2 + max_batch_size: 15 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + kv_cache_config: + host_cache_size: 374500000000 + benchmark: + env: + CONC: '15' + TOTAL_CPU_DRAM_GB: '749' + override_tp2_c20: roles: agg: @@ -204,13 +270,13 @@ override_tp2_c30: CONC: '30' TOTAL_CPU_DRAM_GB: '749' -override_tp4_c5: +override_tp2_c35: roles: agg: - gpus: 4 + gpus: 2 args: - tensor_parallel_size: 4 - max_batch_size: 5 + tensor_parallel_size: 2 + max_batch_size: 35 cuda_graph_config: batch_sizes: - 1 @@ -218,20 +284,37 @@ override_tp4_c5: - 3 - 4 - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 26 + - 28 + - 30 + - 32 + - 34 + - 35 kv_cache_config: - host_cache_size: 374750000000 + host_cache_size: 374500000000 benchmark: env: - CONC: '5' - TOTAL_CPU_DRAM_GB: '1499' + CONC: '35' + TOTAL_CPU_DRAM_GB: '749' -override_tp4_c10: +override_tp2_c40: roles: agg: - gpus: 4 + gpus: 2 args: - tensor_parallel_size: 4 - max_batch_size: 10 + tensor_parallel_size: 2 + max_batch_size: 40 cuda_graph_config: batch_sizes: - 1 @@ -244,20 +327,34 @@ override_tp4_c10: - 8 - 9 - 10 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 26 + - 28 + - 30 + - 32 + - 34 + - 36 + - 38 + - 40 kv_cache_config: - host_cache_size: 374750000000 + host_cache_size: 374500000000 benchmark: env: - CONC: '10' - TOTAL_CPU_DRAM_GB: '1499' + CONC: '40' + TOTAL_CPU_DRAM_GB: '749' -override_tp4_c15: +override_tp2_c45: roles: agg: - gpus: 4 + gpus: 2 args: - tensor_parallel_size: 4 - max_batch_size: 15 + tensor_parallel_size: 2 + max_batch_size: 45 cuda_graph_config: batch_sizes: - 1 @@ -270,25 +367,107 @@ override_tp4_c15: - 8 - 9 - 10 - - 11 - - 12 - - 13 - 14 - - 15 + - 16 + - 18 + - 20 + - 22 + - 24 + - 26 + - 28 + - 30 + - 32 + - 34 + - 36 + - 38 + - 40 + - 42 + - 44 + - 45 + kv_cache_config: + host_cache_size: 374500000000 + benchmark: + env: + CONC: '45' + TOTAL_CPU_DRAM_GB: '749' + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 1 + cuda_graph_config: + batch_sizes: + - 1 kv_cache_config: host_cache_size: 374750000000 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 + speculative_config: + max_draft_len: 7 benchmark: env: - CONC: '15' + CONC: '1' TOTAL_CPU_DRAM_GB: '1499' -override_tp4_c20: +override_tp4_c5: roles: agg: gpus: 4 args: tensor_parallel_size: 4 - max_batch_size: 20 + max_batch_size: 5 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + kv_cache_config: + host_cache_size: 374750000000 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 + speculative_config: + max_draft_len: 5 + benchmark: + env: + CONC: '5' + TOTAL_CPU_DRAM_GB: '1499' + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 10 cuda_graph_config: batch_sizes: - 1 @@ -301,30 +480,36 @@ override_tp4_c20: - 8 - 9 - 10 - - 11 - - 12 - - 13 - - 14 - - 15 - - 16 - - 17 - - 18 - - 19 - - 20 kv_cache_config: host_cache_size: 374750000000 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 + speculative_config: + max_draft_len: 5 benchmark: env: - CONC: '20' + CONC: '10' TOTAL_CPU_DRAM_GB: '1499' -override_tp4_c25: +override_tp4_c20: roles: agg: gpus: 4 args: tensor_parallel_size: 4 - max_batch_size: 25 + max_batch_size: 20 cuda_graph_config: batch_sizes: - 1 @@ -342,25 +527,39 @@ override_tp4_c25: - 13 - 14 - 15 + - 16 - 17 + - 18 - 19 - - 21 - - 23 - - 25 + - 20 kv_cache_config: host_cache_size: 374750000000 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 benchmark: env: - CONC: '25' + CONC: '20' TOTAL_CPU_DRAM_GB: '1499' -override_tp4_c30: +override_tp4_c25: roles: agg: gpus: 4 args: tensor_parallel_size: 4 - max_batch_size: 30 + max_batch_size: 25 cuda_graph_config: batch_sizes: - 1 @@ -375,19 +574,33 @@ override_tp4_c30: - 10 - 11 - 12 + - 13 - 14 - - 16 - - 18 - - 20 - - 22 - - 24 - - 27 - - 30 + - 15 + - 17 + - 19 + - 21 + - 23 + - 25 kv_cache_config: host_cache_size: 374750000000 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 benchmark: env: - CONC: '30' + CONC: '25' TOTAL_CPU_DRAM_GB: '1499' override_tp8_c1: @@ -402,6 +615,22 @@ override_tp8_c1: - 1 kv_cache_config: host_cache_size: 309237645312 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 + speculative_config: + max_draft_len: 7 benchmark: env: CONC: '1' @@ -423,6 +652,20 @@ override_tp8_c5: - 5 kv_cache_config: host_cache_size: 309237645312 + torch_compile_config: + capture_num_tokens: + - 1 + - 256 + - 512 + - 768 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 12288 benchmark: env: CONC: '5' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 9e524ab5f1..375651d859 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5532,7 +5532,7 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: dp-attn: false decode: { num-worker: 5, tp: 2, ep: 1, dp-attn: false } minimaxm3-fp4-b300-trtllm-agentic-mtp: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 + image: nvcr.io#nvidia/tensorrt-llm/release:1.4.0rc0.dev202610070001 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: cluster:b300-dsxe @@ -5545,8 +5545,8 @@ minimaxm3-fp4-b300-trtllm-agentic-mtp: # TP2/TP4 host caches take the 3 TB policy share per GPU. - dram-utilization: 1.0 search-space: - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [20, 25, 30], srt-recipe: agentic.yaml } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30], srt-recipe: agentic.yaml } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [10, 15, 20, 25, 30, 35, 40, 45], srt-recipe: agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5, 10, 20, 25], srt-recipe: agentic.yaml } - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5], srt-recipe: agentic.yaml } # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 8471f866d2..8fa04bafea 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11266,3 +11266,16 @@ - "Add TP4 DRAM KV offload points at conc 40, 48 and 56." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4491 scenario-type: [agentic-coding] + +- config-keys: + - minimaxm3-fp4-b300-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the B300 TRT-LLM MiniMax-M3 AgentX recipe to 15 scheduled TP2/TP4/TP8 profiles with NVFP4 KV cache, EAGLE3 speculative decoding, and native DRAM KV offload; TP4 c20/c25 and TP8 c5 are kept from the published curve, and TP4 c15 and c30 are dropped." + - "Pin the master config image to nvcr.io#nvidia/tensorrt-llm/release:1.4.0rc0.dev202610070001." + - "Drop the TensorRT-LLM chat store patch (setup_script minimaxm3-trtllm-agentx.sh) from the B300 recipe." + - "Pin thinking_mode enabled for the AgentX replay through chat_template_kwargs (AIPERF_EXTRA_INPUTS), matching the thinking_on golden acceptance curve." + - "Rebase the host pools onto the #3804 DRAM policy: dram-utilization 1.0, host_cache_size 374500000000 bytes per rank at TP2, 374750000000 at TP4 and 309237645312 at TP8, TOTAL_CPU_DRAM_GB 749/1499/2999." + - "Parse MiniMax-M3 tool calls with trtllm-serve --tool_parser minimax_m3." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3835