diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/b200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/b200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..8a3221a3ca --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/b200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,168 @@ +schema: 2 + +base: + identity: + model: {} + container: {} + resources: + gpu_type: b200 + gpus_per_node: 8 + slurm: + time_limit: '8:00:00' + dynamo: + install: false + # One aggregate worker serves the public port itself; no router is needed. + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + # One 8xB200 node cannot hold the ~1.56 TB MXFP4 checkpoint. + nodes: 2 + workers: 1 + gpus: 16 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + PYTHONNOUSERSITE: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + # The SGLang Kimi K3.1 Pro cookbook's two-node B200 layout. + tp-size: 16 + dcp-size: 16 + mem-fraction-static: 0.85 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + kv-cache-dtype: fp8_e4m3 + # The K3.1 week-1 precision policy requires FP32 KDA recurrent state. + mamba-ssm-dtype: float32 + reasoning-parser: kimi_k3 + tool-call-parser: kimi_k3 + # DFLASH verifies the current token plus 7 drafts per step. + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-num-draft-tokens: 8 + speculative-use-block-verification: true + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-draft-attention-backend: cutedsl_mla + # Without ReplaySSM DFLASH keeps one KDA state per draft token and the KV pool collapses. + enable-linear-replayssm-spec: true + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + # SGLang serves the port itself; conversation-aware routing needs a Dynamo frontend. + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + sbatch_directives: + mem: '0' + srun_options: + mem: '0' + container-remap-root: '' + +# Admission is 2x CONC. Seven drafts through CONC 14; four (the draft's native block of 5) +# from CONC 24, where the verify batch grows with the running requests. +override_tp16_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + +override_tp16_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + +override_tp16_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + +override_tp16_c14: + roles: + agg: + args: + max-running-requests: 28 + cuda-graph-max-bs-decode: 28 + +override_tp16_c24: + roles: + agg: + args: + max-running-requests: 48 + cuda-graph-max-bs-decode: 48 + speculative-num-draft-tokens: 5 + +override_tp16_c32: + roles: + agg: + args: + max-running-requests: 64 + cuda-graph-max-bs-decode: 64 + speculative-num-draft-tokens: 5 + +override_tp16_c48: + roles: + agg: + args: + max-running-requests: 96 + cuda-graph-max-bs-decode: 96 + speculative-num-draft-tokens: 5 + +override_tp16_c56: + roles: + agg: + args: + max-running-requests: 112 + cuda-graph-max-bs-decode: 112 + speculative-num-draft-tokens: 5 + +override_tp16_c72: + roles: + agg: + args: + max-running-requests: 144 + cuda-graph-max-bs-decode: 144 + speculative-num-draft-tokens: 5 + +override_tp16_c80: + roles: + agg: + args: + max-running-requests: 160 + cuda-graph-max-bs-decode: 160 + speculative-num-draft-tokens: 5 + +override_tp16_c96_nospec: + roles: + agg: + args: + max-running-requests: 192 + cuda-graph-max-bs-decode: 192 + speculative-algorithm: null + speculative-draft-model-path: null + speculative-num-draft-tokens: null + speculative-use-block-verification: null + speculative-draft-kv-cache-dtype: null + speculative-draft-attention-backend: null + enable-linear-replayssm-spec: null diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 4a781453e7..e14fc00e4e 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -4809,6 +4809,151 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true +kimik3.1-fp4-b200-dynamo-sglang-agentic-dflash: + image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391 + model: moonshotai/Kimi-K3.1-Pro + model-prefix: kimik3.1 + runner: cluster:b200-nscale + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: false + srt-recipe-dir: kimik3.1/sglang/b200-fp4/agentx + scenarios: + agentic-coding: + - search-space: + - spec-decoding: draft_model + kv-offloading: none + conc-list: [1] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp16_c1 + power: true + worker: + num-worker: 1 + tp: 16 + dcp-size: 16 + ep: 1 + dp-attn: false + - spec-decoding: draft_model + kv-offloading: none + conc-list: [4] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp16_c4 + power: true + worker: + num-worker: 1 + tp: 16 + dcp-size: 16 + ep: 1 + dp-attn: false + - spec-decoding: draft_model + kv-offloading: none + conc-list: [8] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp16_c8 + power: true + worker: + num-worker: 1 + tp: 16 + dcp-size: 16 + ep: 1 + dp-attn: false + - spec-decoding: draft_model + kv-offloading: none + conc-list: [14] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp16_c14 + power: true + worker: + num-worker: 1 + tp: 16 + dcp-size: 16 + ep: 1 + dp-attn: false + - spec-decoding: draft_model + kv-offloading: none + conc-list: [24] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp16_c24 + power: true + worker: + num-worker: 1 + tp: 16 + dcp-size: 16 + ep: 1 + dp-attn: false + - spec-decoding: draft_model + kv-offloading: none + conc-list: [32] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp16_c32 + power: true + worker: + num-worker: 1 + tp: 16 + dcp-size: 16 + ep: 1 + dp-attn: false + - spec-decoding: draft_model + kv-offloading: none + conc-list: [48] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp16_c48 + power: true + worker: + num-worker: 1 + tp: 16 + dcp-size: 16 + ep: 1 + dp-attn: false + - spec-decoding: draft_model + kv-offloading: none + conc-list: [56] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp16_c56 + power: true + worker: + num-worker: 1 + tp: 16 + dcp-size: 16 + ep: 1 + dp-attn: false + - spec-decoding: draft_model + kv-offloading: none + conc-list: [72] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp16_c72 + power: true + worker: + num-worker: 1 + tp: 16 + dcp-size: 16 + ep: 1 + dp-attn: false + - spec-decoding: draft_model + kv-offloading: none + conc-list: [80] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp16_c80 + power: true + worker: + num-worker: 1 + tp: 16 + dcp-size: 16 + ep: 1 + dp-attn: false + - kv-offloading: none + conc-list: [96] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp16_c96_nospec + power: true + worker: + num-worker: 1 + tp: 16 + dcp-size: 16 + ep: 1 + dp-attn: false + glm5.2-fp4-gb200-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-20261020-ab580b2e@sha256:0af411bb320a95bb5e1c58b4c20c9852920a19aa9f12f148f69731b0dca3b576 model: nvidia/GLM-5.2-NVFP4 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 025645e525..96e8ca9439 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12043,3 +12043,12 @@ - "Add Kimi-K3.1-Pro MXFP4 AgentX on one 8x MI355X node with SGLang (lmsysorg/sglang:dev-kimi-k31-pro-mi35x) and the DFlash2 draft at 5 draft tokens with block verification." - "TP8 monokernel at concurrency 1 and 4 with GPU-resident KV; TP8/DCP8 with the HiCache host tier at concurrency 8, 16, 32 and 48." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4808 + +- config-keys: + - kimik3.1-fp4-b200-dynamo-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Add Kimi K3.1 Pro B200 SGLang AgentX on lmsysorg/sglang:dev-kimi-k31-pro: one aggregated worker over two nodes, TP16 with DCP16, no KV offload, concurrency 1, 4, 8, 14, 24, 32, 48, 56, 72, 80 with DFLASH and 96 without speculation." + - "DFLASH uses RadixArk/Kimi-K3.1-Dflash2-Preview with block verification, 7 draft tokens through concurrency 14 and 4 above." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4801