Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
schema: 2

base:
identity:
model: {}
container: {}
resources:
gpu_type: b200
gpus_per_node: 8
slurm:
time_limit: '8:00:00'
dynamo:
install: false
# One aggregate worker serves the public port itself; no router is needed.
frontend:
type: sglang
enable_multiple_frontends: false
observability:
enabled: false
tachometer:
enabled: false
engine: sglang
roles:
agg:
# One 8xB200 node cannot hold the ~1.56 TB MXFP4 checkpoint.
nodes: 2
workers: 1
gpus: 16
env:
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
PYTHONUNBUFFERED: '1'
PYTHONNOUSERSITE: '1'
# Outlast AIPerf's pooled connections past Uvicorn's keep-alive.
SGLANG_TIMEOUT_KEEP_ALIVE: '1800'
args:
served-model-name: moonshotai/Kimi-K3.1-Pro
trust-remote-code: true
# The SGLang Kimi K3.1 Pro cookbook's two-node B200 layout.
tp-size: 16
dcp-size: 16
mem-fraction-static: 0.85
weight-loader-prefetch-checkpoints: true
model-loader-extra-config: '{"enable_multithread_load": true}'
kv-cache-dtype: fp8_e4m3
# The K3.1 week-1 precision policy requires FP32 KDA recurrent state.
mamba-ssm-dtype: float32
reasoning-parser: kimi_k3
tool-call-parser: kimi_k3
# DFLASH verifies the current token plus 7 drafts per step.
speculative-algorithm: DFLASH
speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview
speculative-num-draft-tokens: 8
speculative-use-block-verification: true
speculative-draft-kv-cache-dtype: fp8_e4m3
speculative-draft-attention-backend: cutedsl_mla
# Without ReplaySSM DFLASH keeps one KDA state per draft token and the KV pool collapses.
enable-linear-replayssm-spec: true
watchdog-timeout: 3600
enable-metrics: true
enable-cache-report: true
health_check:
max_attempts: 1440
interval_seconds: 10
benchmark:
env:
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
# Warmup can leave bytes unacknowledged past AIPerf's 30 s default.
AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
AGENTIC_WARMUP_GRACE_PERIOD: '3600'
# SGLang serves the port itself; conversation-aware routing needs a Dynamo frontend.
AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
sbatch_directives:
mem: '0'
srun_options:
mem: '0'
container-remap-root: ''

# Admission is 2x CONC. Seven drafts through CONC 14; four (the draft's native block of 5)
# from CONC 24, where the verify batch grows with the running requests.
override_tp16_c1:
roles:
agg:
args:
max-running-requests: 2
cuda-graph-max-bs-decode: 2

override_tp16_c4:
roles:
agg:
args:
max-running-requests: 8
cuda-graph-max-bs-decode: 8

override_tp16_c8:
roles:
agg:
args:
max-running-requests: 16
cuda-graph-max-bs-decode: 16

override_tp16_c14:
roles:
agg:
args:
max-running-requests: 28
cuda-graph-max-bs-decode: 28

override_tp16_c24:
roles:
agg:
args:
max-running-requests: 48
cuda-graph-max-bs-decode: 48
speculative-num-draft-tokens: 5

override_tp16_c32:
roles:
agg:
args:
max-running-requests: 64
cuda-graph-max-bs-decode: 64
speculative-num-draft-tokens: 5

override_tp16_c48:
roles:
agg:
args:
max-running-requests: 96
cuda-graph-max-bs-decode: 96
speculative-num-draft-tokens: 5

override_tp16_c56:
roles:
agg:
args:
max-running-requests: 112
cuda-graph-max-bs-decode: 112
speculative-num-draft-tokens: 5

override_tp16_c72:
roles:
agg:
args:
max-running-requests: 144
cuda-graph-max-bs-decode: 144
speculative-num-draft-tokens: 5

override_tp16_c80:
roles:
agg:
args:
max-running-requests: 160
cuda-graph-max-bs-decode: 160
speculative-num-draft-tokens: 5

override_tp16_c96_nospec:
roles:
agg:
args:
max-running-requests: 192
cuda-graph-max-bs-decode: 192
speculative-algorithm: null
speculative-draft-model-path: null
speculative-num-draft-tokens: null
speculative-use-block-verification: null
speculative-draft-kv-cache-dtype: null
speculative-draft-attention-backend: null
enable-linear-replayssm-spec: null
145 changes: 145 additions & 0 deletions inferencex-e2e/configs/nvidia-master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4809,6 +4809,151 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-disagg:
ep: 8
dp-attn: true

kimik3.1-fp4-b200-dynamo-sglang-agentic-dflash:
image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391
model: moonshotai/Kimi-K3.1-Pro
model-prefix: kimik3.1
runner: cluster:b200-nscale
precision: fp4
framework: dynamo-sglang
multinode: true
disagg: false
srt-recipe-dir: kimik3.1/sglang/b200-fp4/agentx
scenarios:
agentic-coding:
- search-space:
- spec-decoding: draft_model
kv-offloading: none
conc-list: [1]
num-nodes: 2
srt-recipe: agg-variants.yaml:override_tp16_c1
power: true
worker:
num-worker: 1
tp: 16
dcp-size: 16
ep: 1
dp-attn: false
- spec-decoding: draft_model
kv-offloading: none
conc-list: [4]
num-nodes: 2
srt-recipe: agg-variants.yaml:override_tp16_c4
power: true
worker:
num-worker: 1
tp: 16
dcp-size: 16
ep: 1
dp-attn: false
- spec-decoding: draft_model
kv-offloading: none
conc-list: [8]
num-nodes: 2
srt-recipe: agg-variants.yaml:override_tp16_c8
power: true
worker:
num-worker: 1
tp: 16
dcp-size: 16
ep: 1
dp-attn: false
- spec-decoding: draft_model
kv-offloading: none
conc-list: [14]
num-nodes: 2
srt-recipe: agg-variants.yaml:override_tp16_c14
power: true
worker:
num-worker: 1
tp: 16
dcp-size: 16
ep: 1
dp-attn: false
- spec-decoding: draft_model
kv-offloading: none
conc-list: [24]
num-nodes: 2
srt-recipe: agg-variants.yaml:override_tp16_c24
power: true
worker:
num-worker: 1
tp: 16
dcp-size: 16
ep: 1
dp-attn: false
- spec-decoding: draft_model
kv-offloading: none
conc-list: [32]
num-nodes: 2
srt-recipe: agg-variants.yaml:override_tp16_c32
power: true
worker:
num-worker: 1
tp: 16
dcp-size: 16
ep: 1
dp-attn: false
- spec-decoding: draft_model
kv-offloading: none
conc-list: [48]
num-nodes: 2
srt-recipe: agg-variants.yaml:override_tp16_c48
power: true
worker:
num-worker: 1
Comment on lines +4817 to +4904

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔴 If this merges before #4798, all ten DFLASH rows fail at binding with No golden curve for model prefix 'kimik3.1' and method 'dflash'. Only the c96 no-spec row would launch. identities.yaml has no kimik3.1 dflash identity, and this diff does not add one. Fix: add the kimi-k3.1-pro__dflash__kimi-k3.1-dflash2-preview__default-block identity and its curve in this change. Otherwise, gate the merge on #4798 or leave the DFLASH rows out until it lands.

Why this was flagged

The condition is merging this PR before the stacked #4798. Every search-space row with spec-decoding: draft_model (nvidia-master.yaml:4817-4944) resolves a golden AL curve by model prefix kimik3.1 and method dflash. infx/golden_al_distribution/identities.yaml has kimik3.1 only under method dspark (line 105-106). The only dflash identity there is for musespark (line 170-171). identity.py:260 therefore raises ValueError("No golden curve for model prefix 'kimik3.1' and method 'dflash'") for 10 of the 11 rows, so the sweep produces only the c96 no-spec point. The PR description acknowledges the dependency, but nothing in this diff enforces the merge order.

Verification: The diff does not touch infx/golden_al_distribution/identities.yaml. agg-variants.yaml sets speculative-algorithm: DFLASH (line 50). identities.yaml selects kimik3.1 only with method: dspark (line 106), so identity.py:260 raises ValueError. Result: all ten spec-decoding: draft_model rows fail to launch.

tp: 16
dcp-size: 16
ep: 1
dp-attn: false
- spec-decoding: draft_model
kv-offloading: none
conc-list: [56]
num-nodes: 2
srt-recipe: agg-variants.yaml:override_tp16_c56
power: true
worker:
num-worker: 1
tp: 16
dcp-size: 16
ep: 1
dp-attn: false
- spec-decoding: draft_model
kv-offloading: none
conc-list: [72]
num-nodes: 2
srt-recipe: agg-variants.yaml:override_tp16_c72
power: true
worker:
num-worker: 1
tp: 16
dcp-size: 16
ep: 1
dp-attn: false
- spec-decoding: draft_model
kv-offloading: none
conc-list: [80]
num-nodes: 2
srt-recipe: agg-variants.yaml:override_tp16_c80
power: true
worker:
num-worker: 1
tp: 16
dcp-size: 16
ep: 1
dp-attn: false
- kv-offloading: none
conc-list: [96]
num-nodes: 2
srt-recipe: agg-variants.yaml:override_tp16_c96_nospec
power: true
worker:
num-worker: 1
tp: 16
dcp-size: 16
ep: 1
dp-attn: false

glm5.2-fp4-gb200-dynamo-sglang-agentic-agg:
image: lmsysorg/sglang:nightly-dev-20261020-ab580b2e@sha256:0af411bb320a95bb5e1c58b4c20c9852920a19aa9f12f148f69731b0dca3b576
model: nvidia/GLM-5.2-NVFP4
Expand Down
9 changes: 9 additions & 0 deletions inferencex-e2e/perf-changelog.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12043,3 +12043,12 @@
- "Add Kimi-K3.1-Pro MXFP4 AgentX on one 8x MI355X node with SGLang (lmsysorg/sglang:dev-kimi-k31-pro-mi35x) and the DFlash2 draft at 5 draft tokens with block verification."
- "TP8 monokernel at concurrency 1 and 4 with GPU-resident KV; TP8/DCP8 with the HiCache host tier at concurrency 8, 16, 32 and 48."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4808

- config-keys:
- kimik3.1-fp4-b200-dynamo-sglang-agentic-dflash
scenario-type:
- agentic-coding
description:
- "Add Kimi K3.1 Pro B200 SGLang AgentX on lmsysorg/sglang:dev-kimi-k31-pro: one aggregated worker over two nodes, TP16 with DCP16, no KV offload, concurrency 1, 4, 8, 14, 24, 32, 48, 56, 72, 80 with DFLASH and 96 without speculation."
- "DFLASH uses RadixArk/Kimi-K3.1-Dflash2-Preview with block verification, 7 draft tokens through concurrency 14 and 4 above."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4801
Loading