Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
base:
schema: 2
name: glm5.3-fp4-b300-sglang-agentic
identity:
model:
revision: e3b8f9f2fad29727dfa95e766c5cac18ea1414bf
resources:
gpu_type: b300
gpus_per_node: 8
frontend:
type: sglang
enable_multiple_frontends: false
observability:
enabled: false
tachometer:
enabled: false
engine: sglang
roles:
agg:
nodes: 1
workers: 1
gpus: 8
args:
served-model-name: nvidia/GLM-5.3-NVFP4
trust-remote-code: true
tensor-parallel-size: 8
expert-parallel-size: 1
quantization: modelopt_fp4
# GLM-5.3 emits the GLM-4.7 tool-call format.
tool-call-parser: glm47
reasoning-parser: glm45
context-length: 1048576
chunked-prefill-size: 8192
max-prefill-tokens: 8192
mem-fraction-static: 0.86
max-running-requests: 256
cuda-graph-max-bs-decode: 64
# The connector selects golden AL 2.99 for K=3 from glm5.3_mtp.yaml.
speculative-algorithm: NEXTN
speculative-num-steps: 3
speculative-eagle-topk: 1
speculative-num-draft-tokens: 4
# Overlap checkpoint reads with weight loading.
model-loader-extra-config: '{"enable_multithread_load":true}'
weight-loader-prefetch-checkpoints: true
weight-loader-drop-cache-after-load: false
watchdog-timeout: 1800
enable-cache-report: true
enable-metrics: true
env:
PYTHONNOUSERSITE: '1'
# Outlast AIPerf's pooled connections past Uvicorn's keep-alive.
SGLANG_TIMEOUT_KEEP_ALIVE: '900'
benchmark:
env:
# Warmup can leave bytes unacknowledged past AIPerf's 30 s default.
AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'

override_c1:
roles:
agg:
args:
max-running-requests: 2
cuda-graph-max-bs-decode: 2
# With expandable segments, lifts tokens/s/GPU at the same interactivity.
enable-torch-symm-mem: true
env:
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
benchmark:
env:
CONC: '1'
KV_OFFLOADING: none

override_c4:
roles:
agg:
args:
max-running-requests: 8
cuda-graph-max-bs-decode: 8
enable-torch-symm-mem: true
env:
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
benchmark:
env:
CONC: '4'
KV_OFFLOADING: none

override_c8:
roles:
agg:
args:
enable-torch-symm-mem: true
env:
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
benchmark:
env:
CONC: '8'
KV_OFFLOADING: none

override_c10:
roles:
agg:
args:
enable-torch-symm-mem: true
env:
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
benchmark:
env:
CONC: '10'
KV_OFFLOADING: none

override_c13:
roles:
agg:
args:
enable-torch-symm-mem: true
env:
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
benchmark:
env:
CONC: '13'
KV_OFFLOADING: none

override_c14:
roles:
agg:
args:
enable-torch-symm-mem: true
env:
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
benchmark:
env:
CONC: '14'
KV_OFFLOADING: none

override_c22:
roles:
agg:
args:
enable-hierarchical-cache: true
# GB per rank; each TP rank holds full MLA KV. The DSA indexer pool adds
# about 69 GB/rank, about 2.95 TB of host DRAM per node in total.
hicache-size: 300
hicache-write-policy: write_back
hicache-io-backend: direct
hicache-mem-layout: page_first_direct
benchmark:
env:
CONC: '22'
KV_OFFLOADING: dram

override_c24:
roles:
agg:
args:
enable-hierarchical-cache: true
hicache-size: 300
hicache-write-policy: write_back
hicache-io-backend: direct
hicache-mem-layout: page_first_direct
benchmark:
env:
CONC: '24'
KV_OFFLOADING: dram

override_c25:
roles:
agg:
args:
enable-hierarchical-cache: true
hicache-size: 300
hicache-write-policy: write_back
hicache-io-backend: direct
hicache-mem-layout: page_first_direct
benchmark:
env:
CONC: '25'
KV_OFFLOADING: dram

override_c28:
roles:
agg:
args:
enable-hierarchical-cache: true
hicache-size: 300
hicache-write-policy: write_back
hicache-io-backend: direct
hicache-mem-layout: page_first_direct
benchmark:
env:
CONC: '28'
KV_OFFLOADING: dram
17 changes: 17 additions & 0 deletions inferencex-e2e/configs/nvidia-master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10185,3 +10185,20 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-disagg:
ep: 1
dp-attn: false
decode: { num-worker: 1, tp: 16, dcp-size: 16, ep: 1, dp-attn: false }

glm5.3-fp4-b300-sglang-agentic-mtp:
image: lmsysorg/sglang:v0.5.20-cu130@sha256:b27fce60bc5494c118c4910702812bcfa8cee67abcdd1ff8b0902f21647552f4
model: nvidia/GLM-5.3-NVFP4
model-prefix: glm5.3
runner: cluster:b300-dsxe
precision: fp4
framework: sglang
multinode: false
srt-recipe-dir: glm5.3/sglang/b300-fp4-mtp
scenarios:
agentic-coding:
- search-space:
- { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 10, 13, 14], srt-recipe: agentic.yaml }
- dram-utilization: 1.0
search-space:
- { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [22, 24, 25, 28], srt-recipe: agentic.yaml }
1 change: 1 addition & 0 deletions inferencex-e2e/configs/runners.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,7 @@ clusters:
DeepSeek-V4-Pro-NVFP4: {root: scratch, dir: DeepSeek-V4-Pro-NVFP4}
GLM-5.2-FP8: {root: scratch, dir: GLM-5.2-FP8}
GLM-5.2-NVFP4: {root: scratch, dir: GLM-5.2-NVFP4}
GLM-5.3-NVFP4: {root: scratch, dir: GLM-5.3-NVFP4}
Kimi-K3: {root: scratch, dir: Kimi-K3}
MiniMax-M3: {root: scratch, dir: MiniMax-M3}
MiniMax-M3-MXFP8: {root: scratch, dir: MiniMax-M3-MXFP8}
Expand Down
9 changes: 9 additions & 0 deletions inferencex-e2e/perf-changelog.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10823,3 +10823,12 @@
- "Use fastokens in both worker roles and BF16 decode index logits."
- "Disable decode KV-event publishing for round-robin routing; retain prefill KV events and metrics on both roles."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4303

- config-keys:
- glm5.3-fp4-b300-sglang-agentic-mtp
scenario-type:
- agentic-coding
description:
- "Add GLM-5.3 NVFP4 B300 AgentX on SGLang v0.5.20 (lmsysorg/sglang:v0.5.20-cu130), TP8 with NEXTN MTP K3."
- "Concurrency 1, 4, 8, 10, 13 and 14 run without host offload; concurrency 22, 24, 25 and 28 use a 300 GB/rank HiCache host tier."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3930
Loading