diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3/sglang/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..2bc1e71481 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3/sglang/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,193 @@ +base: + schema: 2 + name: glm5.3-fp4-b300-sglang-agentic + identity: + model: + revision: e3b8f9f2fad29727dfa95e766c5cac18ea1414bf + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: nvidia/GLM-5.3-NVFP4 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 1 + quantization: modelopt_fp4 + # GLM-5.3 emits the GLM-4.7 tool-call format. + tool-call-parser: glm47 + reasoning-parser: glm45 + context-length: 1048576 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + mem-fraction-static: 0.86 + max-running-requests: 256 + cuda-graph-max-bs-decode: 64 + # The connector selects golden AL 2.99 for K=3 from glm5.3_mtp.yaml. + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + # Overlap checkpoint reads with weight loading. + model-loader-extra-config: '{"enable_multithread_load":true}' + weight-loader-prefetch-checkpoints: true + weight-loader-drop-cache-after-load: false + watchdog-timeout: 1800 + enable-cache-report: true + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + benchmark: + env: + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + # With expandable segments, lifts tokens/s/GPU at the same interactivity. + enable-torch-symm-mem: true + env: + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + enable-torch-symm-mem: true + env: + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_c8: + roles: + agg: + args: + enable-torch-symm-mem: true + env: + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_c10: + roles: + agg: + args: + enable-torch-symm-mem: true + env: + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + benchmark: + env: + CONC: '10' + KV_OFFLOADING: none + +override_c13: + roles: + agg: + args: + enable-torch-symm-mem: true + env: + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + benchmark: + env: + CONC: '13' + KV_OFFLOADING: none + +override_c14: + roles: + agg: + args: + enable-torch-symm-mem: true + env: + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + benchmark: + env: + CONC: '14' + KV_OFFLOADING: none + +override_c22: + roles: + agg: + args: + enable-hierarchical-cache: true + # GB per rank; each TP rank holds full MLA KV. The DSA indexer pool adds + # about 69 GB/rank, about 2.95 TB of host DRAM per node in total. + hicache-size: 300 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '22' + KV_OFFLOADING: dram + +override_c24: + roles: + agg: + args: + enable-hierarchical-cache: true + hicache-size: 300 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + +override_c25: + roles: + agg: + args: + enable-hierarchical-cache: true + hicache-size: 300 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '25' + KV_OFFLOADING: dram + +override_c28: + roles: + agg: + args: + enable-hierarchical-cache: true + hicache-size: 300 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '28' + KV_OFFLOADING: dram diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 9e87e0fb01..a8111d731f 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -10185,3 +10185,20 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-disagg: ep: 1 dp-attn: false decode: { num-worker: 1, tp: 16, dcp-size: 16, ep: 1, dp-attn: false } + +glm5.3-fp4-b300-sglang-agentic-mtp: + image: lmsysorg/sglang:v0.5.20-cu130@sha256:b27fce60bc5494c118c4910702812bcfa8cee67abcdd1ff8b0902f21647552f4 + model: nvidia/GLM-5.3-NVFP4 + model-prefix: glm5.3 + runner: cluster:b300-dsxe + precision: fp4 + framework: sglang + multinode: false + srt-recipe-dir: glm5.3/sglang/b300-fp4-mtp + scenarios: + agentic-coding: + - search-space: + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 10, 13, 14], srt-recipe: agentic.yaml } + - dram-utilization: 1.0 + search-space: + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [22, 24, 25, 28], srt-recipe: agentic.yaml } diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index c6f759c960..54b337e5f6 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -524,6 +524,7 @@ clusters: DeepSeek-V4-Pro-NVFP4: {root: scratch, dir: DeepSeek-V4-Pro-NVFP4} GLM-5.2-FP8: {root: scratch, dir: GLM-5.2-FP8} GLM-5.2-NVFP4: {root: scratch, dir: GLM-5.2-NVFP4} + GLM-5.3-NVFP4: {root: scratch, dir: GLM-5.3-NVFP4} Kimi-K3: {root: scratch, dir: Kimi-K3} MiniMax-M3: {root: scratch, dir: MiniMax-M3} MiniMax-M3-MXFP8: {root: scratch, dir: MiniMax-M3-MXFP8} diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 38564eb03d..03119255d5 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10823,3 +10823,12 @@ - "Use fastokens in both worker roles and BF16 decode index logits." - "Disable decode KV-event publishing for round-robin routing; retain prefill KV events and metrics on both roles." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4303 + +- config-keys: + - glm5.3-fp4-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add GLM-5.3 NVFP4 B300 AgentX on SGLang v0.5.20 (lmsysorg/sglang:v0.5.20-cu130), TP8 with NEXTN MTP K3." + - "Concurrency 1, 4, 8, 10, 13 and 14 run without host offload; concurrency 22, 24, 25 and 28 use a 300 GB/rank HiCache host tier." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3930