From 638f9533b14541cd0df208f7c3347e2deb38b79b Mon Sep 17 00:00:00 2001 From: thomawan Date: Thu, 14 May 2026 15:36:42 +0800 Subject: [PATCH 1/2] Update glm-5-fp8 mtp image/setting for mi355 sglang --- .github/configs/amd-master.yaml | 6 +++--- benchmarks/single_node/glm5_fp8_mi355x.sh | 7 ++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index d3dad32555..d18309c2fd 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -387,7 +387,7 @@ glm5-fp8-mi355x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } glm5-fp8-mi355x-sglang-mtp: - image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260415 + image: lmsysorg/sglang-rocm:v0.5.11-rocm720-mi35x-20260513 model: zai-org/GLM-5-FP8 model-prefix: glm5 runner: mi355x @@ -399,12 +399,12 @@ glm5-fp8-mi355x-sglang-mtp: - isl: 1024 osl: 1024 search-space: - - { tp: 4, conc-start: 4, conc-end: 128, spec-decoding: mtp } + - { tp: 4, conc-start: 4, conc-end: 256, spec-decoding: mtp } - { tp: 8, conc-start: 4, conc-end: 8, spec-decoding: mtp } - isl: 8192 osl: 1024 search-space: - - { tp: 4, conc-start: 4, conc-end: 128, spec-decoding: mtp } + - { tp: 4, conc-start: 4, conc-end: 256, spec-decoding: mtp } - { tp: 8, conc-start: 4, conc-end: 8, spec-decoding: mtp } glm5-fp8-mi355x-atom: diff --git a/benchmarks/single_node/glm5_fp8_mi355x.sh b/benchmarks/single_node/glm5_fp8_mi355x.sh index 5c517cdcb9..cd99536b9c 100755 --- a/benchmarks/single_node/glm5_fp8_mi355x.sh +++ b/benchmarks/single_node/glm5_fp8_mi355x.sh @@ -15,11 +15,6 @@ if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi -# GLM-5 requires transformers with glm_moe_dsa model type support. -# However, the Image rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260219 doesn't provide this support. -python3 -m pip install -U --no-cache-dir \ - "git+https://github.com/huggingface/transformers.git@6ed9ee36f608fd145168377345bfc4a5de12e1e2" - if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi # ROCm / SGLang performance tuning for MI355X @@ -51,6 +46,8 @@ python3 -m sglang.launch_server \ --nsa-prefill-backend tilelang \ --nsa-decode-backend tilelang $EVAL_CONTEXT_ARGS \ --kv-cache-dtype fp8_e4m3 \ + --max-running-requests ${CONC} \ + --cuda-graph-max-bs ${CONC} \ --disable-radix-cache> $SERVER_LOG 2>&1 & SERVER_PID=$! From d821908de10e9ff2f3e0aa5cff7ef632ecf05aef Mon Sep 17 00:00:00 2001 From: thomawan Date: Thu, 14 May 2026 15:46:02 +0800 Subject: [PATCH 2/2] Update change log --- perf-changelog.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6d7faff5fa..683ce101a8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -2460,3 +2460,11 @@ description: - "Add DeepSeek-V4-Pro FP4 MI355X ATOM benchmark config; bump image to rocm/atom-dev:nightly_202605101539, expand concurrency range (conc 4–1024), and simplify runtime script" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1311 + +- config-keys: + - glm5-fp8-mi355x-sglang-mtp + description: + - "Image: lmsysorg/sglang-rocm:v0.5.11-rocm720-mi35x-20260513" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1377 + +