Skip to content
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ base:
SGLANG_TIMEOUT_KEEP_ALIVE: '1800'
# Explicitly disable the prohibited draft precision conversion path.
SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '0'
# NVIDIA runtime optimizations: compact verify sampling (verify tokens sampled on the top-k support before
# temperature and top-p; same distribution), post-verify CUDA graphs and their fast paths.
SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION: '1'
args:
served-model-name: nvidia/Qwen3.8-Flash-Next-NVFP4
revision: fc694b54fb0174e0913e6adf86691ef85a4ead47
Expand Down Expand Up @@ -64,7 +67,7 @@ base:
# The native connector injects model-specific golden AL for throughput;
# quality evaluations use real verification. No manual AL override.
enable-linear-replayssm-spec: true
cuda-graph-backend-prefill: disabled
cuda-graph-backend-prefill: breakable
cuda-graph-backend-decode: full
cuda-graph-max-bs-decode: 96
cuda-graph-bs-decode: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96]
Expand Down Expand Up @@ -101,43 +104,171 @@ base:
TOTAL_CPU_DRAM_GB: '241'
KV_OFFLOADING: dram

# Only client concurrency changes; serving/admission/graph/cache settings stay fixed.
override_c1:
benchmark:
env:
CONC: '1'

override_c2:
benchmark:
env:
CONC: '2'

override_c4:
override_tp4_c3:
roles:
agg:
gpus: 4
env:
# Captures the decode step's host-side metadata preparation in the decode CUDA graph.
SGLANG_ENABLE_METADATA_GLUE_GRAPH: '1'
args:
<<: &gpu_only_args
# null drops the base setting, so SGLang's default applies.
enable-hierarchical-cache: null
hicache-size: null
hicache-io-backend: null
hicache-mem-layout: null
hicache-write-policy: null
mamba-radix-cache-strategy: null
mamba-track-interval: null
max-mamba-cache-size: null
# Dense small sizes: decode batches up to 16 pad by at most one request.
cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96]
tokenizer-worker-num: 6
tensor-parallel-size: 4
expert-parallel-size: 4
# SGLang's custom all-reduce instead of the FlashInfer fused all-reduce.
enforce-disable-flashinfer-allreduce-fusion: true
# Keep the per-layer embedding table in GPU memory instead of pinned host memory.
ple-offload-embedding: null
ple-offload-backend: null
no-ple-offload-embedding: true
# Running cap: one request decodes at a time; the other sessions wait for their first token.
max-running-requests: 1
benchmark:
env:
CONC: '4'
<<: &gpu_only_client
KV_OFFLOADING: none
TOTAL_CPU_DRAM_GB: '0'
CONC: '3'

override_c8:
roles:
agg:
args:
<<: *gpu_only_args
# Running cap with shortest-prefill-first admission: two requests decode; short new requests start first.
max-running-requests: 2
schedule-policy: shortest-prefill-first
benchmark:
env:
<<: *gpu_only_client
CONC: '8'

override_c16:
override_c20:
roles:
agg:
env:
# Cache-aware admission skips its in-batch prefix check, which holds short new requests behind long ones.
IN_BATCH_PREFIX_CACHING_CHECK_THRESHOLD: '-1'
# One fused MoE routing kernel for 17+ token batches instead of the split path; outputs are bitwise equal.
FLASHINFER_ROUTING_FORCE_BLOCK_PER_TOKEN: '0'
args:
<<: *gpu_only_args
kv-cache-dtype: fp8_e4m3
speculative-draft-kv-cache-dtype: fp8_e4m3
# Context length: no decode-time Mamba checkpoints; the AgentX replay never re-sends model output.
mamba-track-interval: 262144
cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96]
# Running cap with shortest-prefill-first admission: short new requests start first; long ones wait.
max-running-requests: 6
schedule-policy: shortest-prefill-first
# Hold new prefills until two running slots are free, so they share one prefill pass.
min-free-slots-delay: 2
# FP8 QSA indexer: the sparse-attention indexer keeps its keys in FP8 and scores with FP8 GEMMs.
qsa-indexer-dtype: fp8_e4m3
benchmark:
env:
CONC: '16'
<<: *gpu_only_client
CONC: '20'

override_c32:
override_c36:
roles:
agg:
Comment on lines +185 to +187

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 nit (optional): The new recipe comment includes past benchmark results, which the AGENTS.md comments rule does not allow in code. The phrase "they gave no gain at 36 and 56" records past results, and that history belongs in git and perf-changelog.yaml. The out-of-memory failure at concurrency 80 counts as a measured failure, so that part can stay. Fix: shorten the comment to the technical reason only, for example "Post-verify CUDA graphs captured lazily while serving exhaust GPU memory at high concurrency; compact verify sampling stays on."

Why this was flagged

The comment on the &hicache_env anchor (agentic.yaml:185-187) explains why the post-verify CUDA graph flags are turned off for override_c36, override_c56 and override_c80. Its clause "they gave no gain at 36 and 56" reports earlier benchmark outcomes. AGENTS.md (Comments, mandatory, every file) bans comments about past results and points to git and perf-changelog.yaml as the record. It allows only technical explanations, such as a measured failure. Nothing breaks at runtime. The cost is that the recipe goes against the repo's mandatory comment convention, and the comment will go stale once the image or settings change.

Verification: The diff adds this text at agentic.yaml:190-191, inside the &hicache_env anchor under override_c36. AGENTS.md:54-58 says: "No history: how code evolved, what it replaced, where it came from, or past results." The OOM at concurrency 80 is a measured failure, so that part is allowed. "they gave no gain at 36 and 56" reports an earlier benchmark outcome. Nothing breaks at runtime.

env:
<<: &hicache_env
# The switch's post-verify CUDA graphs and their fast paths stay off here: capturing them lazily while
# serving ran out of GPU memory at concurrency 80, and they gave no gain at 36 and 56.
# Compact verify sampling stays on.
SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH: '0'
SGLANG_OPT_SPEC_VERIFY_REUSE_STAGED_INPUTS: '0'
SGLANG_OPT_SPEC_POST_VERIFY_ONE_LAUNCH_LOAD: '0'
SGLANG_OPT_SMALL_FIELD_COPY_VECTORIZED: '0'
SGLANG_OPT_SPEC_PACKED_VERIFY_BCAST: '0'
SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH: '0'
IN_BATCH_PREFIX_CACHING_CHECK_THRESHOLD: '-1'
FLASHINFER_ROUTING_FORCE_BLOCK_PER_TOKEN: '0'
args:
<<: &hicache_fp8_kv_args
kv-cache-dtype: fp8_e4m3
speculative-draft-kv-cache-dtype: fp8_e4m3
# Context length: no decode-time Mamba checkpoints; the AgentX replay never re-sends model output.
mamba-track-interval: 262144
min-free-slots-delay: 2
# Two draft steps (three draft tokens): a smaller verify batch per decode step at these running sizes.
speculative-num-steps: 2
speculative-num-draft-tokens: 3
max-mamba-cache-size: 720
cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 56, 64, 72, 80, 88, 96]
max-running-requests: 12
schedule-policy: shortest-prefill-first
benchmark:
env:
CONC: '32'
CONC: '36'

override_c48:
override_c56:
roles:
agg:
env: *hicache_env
args:
<<: *hicache_fp8_kv_args
max-mamba-cache-size: 720
cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 60, 64, 72, 80, 88, 96]
# Running cap: at most 35 requests decode; the rest wait for their first token.
max-running-requests: 35
# Hold prefill until the queue reaches min(20% of running, 4) requests or 300 ms pass; at most four per pass.
enable-prefill-delayer: true
prefill-delayer-queue-min-ratio: 0.2
prefill-delayer-max-delay-ms: 300
prefill-max-requests: 4
# After a prefill pass, run ten decode rounds before the next while 12 or more requests are running.
prefill-decode-interval: 10
prefill-decode-interval-min-bs: 12
# FP8 QSA indexer: the sparse-attention indexer keeps its keys in FP8 and scores with FP8 GEMMs.
qsa-indexer-dtype: fp8_e4m3
benchmark:
env:
CONC: '48'
CONC: '56'

override_c64:
override_c80:
roles:
agg:
env: *hicache_env
args:
<<: *hicache_fp8_kv_args
speculative-num-steps: 2
speculative-num-draft-tokens: 3
mem-fraction-static: 0.90
# The whole per-GPU host allowance (TOTAL_CPU_DRAM_GB 241).
hicache-size: 241
max-mamba-cache-size: 448
max-running-requests: 72
cuda-graph-max-bs-decode: 128
# Every size to 80, so decode batches at this concurrency never pad.
cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128]
# Cache only the prompt on finish; the AgentX replay never re-sends model output.
strip-thinking-cache: true
# At most two Mamba states per conversation path; stale interior states skip host write-back.
mamba-max-states-per-path: 2
# The concurrency-56 admission merging, triggered at 10% of running, with eight decode rounds between prefills.
enable-prefill-delayer: true
prefill-delayer-queue-min-ratio: 0.1
prefill-delayer-max-delay-ms: 300
prefill-max-requests: 4
prefill-decode-interval: 8
prefill-decode-interval-min-bs: 12
# FP8 QSA indexer: the sparse-attention indexer keeps its keys in FP8 and scores with FP8 GEMMs.
qsa-indexer-dtype: fp8_e4m3
benchmark:
env:
CONC: '64'
CONC: '80'
18 changes: 5 additions & 13 deletions inferencex-e2e/configs/nvidia-master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6904,9 +6904,8 @@ dsv41flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg:
ep: 32
dp-attn: true

# One serving GPU, stock ARM64 release, native AgentX client and proportional host cache.
qwen3.8next-fp4-gb300-sglang-agentic-mtp:
image: lmsysorg/sglang:v0.5.22-cu130@sha256:06441be3de19a816ddbac917535e920cbfc7fc43171abc08417394f43634ffb6
image: lmsysorg/sglang:nightly-dev-cu13-20261103-797af717@sha256:c7c348bc52c410f0e367a46a2a733146017e3de9709694cd42f004fe5d6c763f
model: nvidia/Qwen3.8-Flash-Next-NVFP4
model-prefix: qwen3.8next
runner: cluster:gb300-nv
Expand All @@ -6916,19 +6915,12 @@ qwen3.8next-fp4-gb300-sglang-agentic-mtp:
srt-recipe-dir: qwen3.8next/sglang/gb300-fp4-mtp
scenarios:
agentic-coding:
# Use the GB300 profile's proportional per-worker host DRAM allowance.
# Generated allowance: 225 GB. Recipe H224 reserves alignment headroom.
# HiCache rows take the GB300 profile's proportional per-GPU host DRAM allowance (241 GB).
- dram-utilization: 1.0
search-space:
- tp: 1
require-power: true
ep: 1
dp-attn: false
spec-decoding: mtp
kv-offloading: dram
kv-offload-backend: { name: hicache }
conc-list: [1, 2, 4, 8, 16, 32, 48, 64]
srt-recipe: agentic.yaml
- { tp: 4, ep: 4, dp-attn: false, require-power: true, spec-decoding: mtp, kv-offloading: none, conc-list: [3], srt-recipe: agentic.yaml }
- { tp: 1, ep: 1, dp-attn: false, require-power: true, spec-decoding: mtp, kv-offloading: none, conc-list: [8, 20], srt-recipe: agentic.yaml }
- { tp: 1, ep: 1, dp-attn: false, require-power: true, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [36, 56, 80], srt-recipe: agentic.yaml }

musespark-fp4-gb300-sglang-agentic-dflash:
image: lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc
Expand Down
9 changes: 9 additions & 0 deletions inferencex-e2e/perf-changelog.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12352,3 +12352,12 @@
description:
- "Sweep H3 C1 latency on 1, 2, 4 and 8 MI355X GPUs with SGLang Ulysses across all GPUs; the MI355X recipe now sets the Ulysses degree from the GPU count, which keeps the 4-GPU point unchanged."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4818

- config-keys:
- qwen3.8next-fp4-gb300-sglang-agentic-mtp
scenario-type:
- agentic-coding
description:
- "Update to lmsysorg/sglang:nightly-dev-cu13-20261103-797af717 and enable SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION and breakable prefill CUDA graphs."
- "Replace the search space with TP4/EP4 at concurrency 3, TP1/EP1 at concurrency 8 and 20, and TP1/EP1 with HiCache at concurrency 36, 56 and 80; concurrency 20, 36, 56 and 80 use fp8_e4m3 kv-cache-dtype and speculative-draft-kv-cache-dtype (was bfloat16), concurrency 20, 56 and 80 set qsa-indexer-dtype fp8_e4m3, and concurrency 36 and 80 use 2 speculative steps instead of 3."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4787
Loading