diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..045d0cfb49 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,198 @@ +base: + schema: 2 + name: kimik3.1-fp4-mi355x-vllm-agentic + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # A node's first run downloads ~1.56 TB of MXFP4 shards into its local HF cache before loading. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + cp-kv-cache-interleave-size: 1 + trust-remote-code: true + moe-backend: auto + seed: 7 + language-model-only: true + load-format: fastsafetensors + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + max-model-len: 1048576 + enable-prefix-caching: true + prefix-match-unit: 64 + kv-cache-dtype: fp8 + # The K3.1 week-1 precision policy requires an FP32 KDA recurrent state. + mamba-ssm-cache-dtype: float32 + # The policy's FP4 indexer cache; the image's AITER pads the FP4 indexer pages it needs. + attention-config: '{"indexer_kv_dtype":"mxfp4"}' + additional-config: '{"kda_prefill_backend":"fused"}' + # Prefix-cached prefill chunks end on K3.1's 1152-token packed DSA block, so the budget floors to a multiple of 1152. + max-num-batched-tokens: 16384 + # Kernel scratch, up to HSA_SCRATCH_SINGLE_LIMIT_ASYNC per dispatch, is allocated outside this budget, so leave headroom. + gpu-memory-utilization: 0.90 + stream-interval: 10 + disable-uvicorn-access-log: true + # 224.875 GB per rank; TOTAL_CPU_DRAM_GB (1799) split across the 8 TP ranks. + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' + env: + VLLM_ROCM_USE_AITER: '1' + VLLM_ROCM_USE_AITER_MOE_SITUV2: '1' + SAFETENSORS_FAST_GPU: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_FASTSAFETENSORS_QUEUE_SIZE: '-1' + # K3.1 has no torch.compile submodule, so only breakable graphs capture the prefill + # sizes the throughput points list. The conc-1 override turns this off. + VLLM_USE_BREAKABLE_CUDAGRAPH: '1' + VLLM_ROCM_AITER_MLA_ASM_PADDING: asm + # Verify the draft tokens with the AITER assembly DCP kernel. + VLLM_ROCM_AITER_MLA_DCP_VERIFY: asm + VLLM_ROCM_DCP_AITER_ALLGATHER: '1' + VLLM_ROCM_USE_AITER_FLYDSL_SPARSE_MLA: '1' + # AITER reads its own variable for the same kernels. + AITER_FLYDSL_SPARSE_MLA: '1' + VLLM_ROCM_DSA_DCP_GLOBAL_KEY_PREFILL: '1' + VLLM_ROCM_DSA_DCP_ROWSHARD_PREFILL: '1' + VLLM_USE_DIRECT_DCP_A2A: '0' + VLLM_USE_DIRECT_DCP_Q_GATHER: '0' + VLLM_USE_DIRECT_DCP_KV_GATHER: '0' + # The K3.1 policy and MODELS.md keep all-reduce in BF16, so quick reduce must not quantize. + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: NONE + AITER_FLYDSL_STAGE2_FP8: '0' + AITER_BF16_FP8_MOE_BOUND: '0' + HSA_NO_SCRATCH_RECLAIM: '1' + # Back the offload pool with transparent huge pages. Torch's CPU allocator + # only calls madvise(MADV_HUGEPAGE) when this is set, and the host is in + # madvise mode, so without it the pool is all 4 KiB pages. + THP_MEM_ALLOC_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1200' + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + # Identical prefixes must hash to identical block keys run-to-run. + PYTHONHASHSEED: '42' + benchmark: + env: + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# max-num-seqs stays below CONC from c4 to c32, and equals it at c1 and c48. The capture list is +# the decode sizes, multiples of one plus the draft token count, then the prefill ladder. +# Throughput runs replace block rejection with the golden acceptance length. +override_gpu_kv_c1: + roles: + agg: + args: + max-num-seqs: 1 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16]}' + additional-config: '{"decode_backend":"latency_specialized","kda_prefill_backend":"fused"}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: null + env: + # This point captures only the decode sizes, so breakable graphs buy nothing. + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + benchmark: + env: + CONC: '1' + KV_OFFLOADING: 'none' + +override_dram_c4: + roles: + agg: + args: + max-num-seqs: 2 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[6,12,16,24,32,48,64,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '4' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +override_dram_c10: + roles: + agg: + args: + max-num-seqs: 6 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[6,12,18,24,30,36,48,64,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '10' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +override_dram_c16: + roles: + agg: + args: + max-num-seqs: 10 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,64,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '16' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +override_dram_c24: + roles: + agg: + args: + max-num-seqs: 16 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16384,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240,12288,14336,16384]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '24' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +override_dram_c32: + roles: + agg: + args: + max-num-seqs: 24 + max-num-batched-tokens: 10240 + gpu-memory-utilization: 0.91 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,128,160,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '32' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +override_dram_c48: + roles: + agg: + args: + max-num-seqs: 48 + max-num-batched-tokens: 10240 + gpu-memory-utilization: 0.91 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":10240,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,224,256,288,320,352,384,416,448,480,512,640,768,896,1024,1280,1536,1792,2048,2560,3072,3584,4096,4608,5120,6144,6400,7168,8192,8704,10240]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":3,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '48' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 9e9e23dc1a..8b7a89722a 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -926,6 +926,24 @@ minimaxm31flash-fp4-mi355x-vllm-agentic-dspark: - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: agentic.yaml } - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [64], srt-recipe: agentic.yaml:override_tp4_dspark3_c64_offload } +kimik3.1-fp4-mi355x-vllm-agentic-dspark: + image: vllm/vllm-openai-rocm:kimi-k31@sha256:5f47cc1004d9a7ac65f8a913725edb20c62c99a07e31cbf4ba21741bd0b4f9b1 + model: moonshotai/Kimi-K3.1-Pro + model-prefix: kimik3.1 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + srt-recipe-dir: kimik3.1/vllm/mi355x-fp4-mtp + scenarios: + agentic-coding: + # 0.6 resolves to 1799 GB of host DRAM, which the recipe's Simple CPU offload splits across the 8 TP ranks. + - dram-utilization: 0.6 + search-space: + # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. c1 keeps the KV cache on the GPUs. + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [4, 10, 16, 24, 32, 48], srt-recipe: agentic.yaml } + # MiniMax-M3.1-Flash AgentX on MI355X with SGLang: day-zero image from the SGLang # cookbook (https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1), # manifest digest verified 2026-10-24. Serve flags follow the cookbook's diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 42401e2a75..97190a884e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12035,6 +12035,21 @@ - "Add Kimi-K3.1-Pro day-0 vLLM AgentX on B300: TP8 x DCP8 with DSpark (Inferact/Kimi-K3.1-Pro-DSpark-preview) and Simple CPU KV offload, conc 1-70." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4796 +- config-keys: + - kimik3.1-fp4-mi355x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add Kimi-K3.1-Pro vLLM AgentX on one 8x MI355X node with vLLM's day-0 image vllm/vllm-openai-rocm:kimi-k31@sha256:5f47cc1004d9a7ac65f8a913725edb20c62c99a07e31cbf4ba21741bd0b4f9b1 (vLLM 0.0.0+mum.14d1ec2336, AITER 52846bc2)." + - "TP8 x DCP8 (a2a DCP backend) with the Inferact/Kimi-K3.1-Pro-DSpark-preview draft: GPU-resident KV at concurrency 1, and SimpleCPUOffloadConnector DRAM offload at concurrency 4, 10, 16, 24, 32 and 48 (224.875 GB per rank, 1799 GB per node, which is what dram-utilization 0.6 resolves to)." + - "Per point at concurrency 1/4/10/16/24/32/48: max-num-seqs 1/2/6/10/16/24/48 and DSpark draft tokens 7/5/5/4/4/3/3. max-num-batched-tokens is 16384 and gpu-memory-utilization 0.90 through concurrency 24; both change to 10240 and 0.91 at concurrency 32 and 48." + - "Every point compiles in mode 3 with FULL_AND_PIECEWISE CUDA graphs. Concurrency 1 runs the latency_specialized decode backend, captures sizes 2-16 and turns breakable CUDA graphs off; the offload points keep VLLM_USE_BREAKABLE_CUDAGRAPH=1 and capture max-num-seqs multiples of (1 + draft tokens) plus a prefill ladder up to max-num-batched-tokens." + - "Common settings: FP8 KV cache, FP32 KDA state (mamba-ssm-cache-dtype float32), MXFP4 indexer cache (indexer_kv_dtype mxfp4), fused KDA prefill, prefix caching with prefix-match-unit 64, max-model-len 1048576, fastsafetensors loading, stream-interval 10, AITER ASM DCP verify, AITER DCP allgather, FlyDSL sparse MLA, the image's HSA_NO_SCRATCH_RECLAIM=1, transparent huge pages for the offload pool (THP_MEM_ALLOC_ENABLE=1), and 1200 s execute-model and 7200 s engine-ready timeouts." + - "K3.1 week-1 precision policy: linear layers and shared experts stay BF16 (no quantization-config); routed experts run A4W4, MXFP4 weights with FP4 activations, through the SiTUv2 FlyDSL MoE (VLLM_ROCM_USE_AITER_MOE_SITUV2=1, the same as the image's auto default); MoE stage 2 stays BF16 (AITER_FLYDSL_STAGE2_FP8=0); all-reduce stays BF16 (VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=NONE)." + - "Verification is block rejection with probabilistic draft sampling. Throughput runs replace it with the golden acceptance length for the draft length: 3.69 at 7 tokens, 3.41 at 5, 3.19 at 4 and 2.88 at 3. Evals and AL gates verify for real." + - "No change to the DSpark draft's precision: Inferact/Kimi-K3.1-Pro-DSpark-preview (revision 6a2ef20a) ships all-BF16 weights with no quantization_config, the speculative config sets no quantization, so get_draft_quant_config() returns None and K3DSparkModel loads its layers in BF16 with the target's BF16 embedding and LM head. Its KV cache is FP8, the same as the target's." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4806 + - config-keys: - kimik3.1-fp4-mi355x-sglang-agentic-dflash scenario-type: