Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
7c4ddcd
Add structured GSM8K artifacts and opt-in completeness checks
sammysun0711 Oct 23, 2026
58045a8
Keep completed token-limited answers in GSM8K accuracy reporting
sammysun0711 Oct 23, 2026
aaf458d
Add unified Qwen4 MI308X CI using the existing evaluation scripts
sammysun0711 Oct 23, 2026
71a68fa
Simplify Qwen4 CI to one workflow and driver
sammysun0711 Oct 25, 2026
4f7f33e
Keep GSM8K CI extensions to request completion accounting
sammysun0711 Oct 25, 2026
5cd922b
Use minimal GSM8K results and add full local accuracy checks
sammysun0711 Oct 25, 2026
2111007
Explain CI authentication failures and report execution status
sammysun0711 Oct 25, 2026
b8baa9f
Show CI progress and keep build state outside home
sammysun0711 Oct 25, 2026
c413cd4
Reuse the base runtime when installing candidate SGLang
sammysun0711 Oct 25, 2026
4cc7b62
Fix install issue & split 6 case from workflow
sammysun0711 Oct 25, 2026
447c780
simplify ci test
sammysun0711 Oct 26, 2026
6a9b6d6
Simplify Qwen CI and enable per-case retries
sammysun0711 Oct 30, 2026
33cf62a
Address review comments
sammysun0711 Oct 30, 2026
bce046f
minior cleanup
sammysun0711 Oct 30, 2026
b2ad2b0
change model test order
sammysun0711 Oct 30, 2026
e347014
Doc: add run-qwen4-ci label info
sammysun0711 Oct 30, 2026
14f008a
ci: adjust ci summary report
sammysun0711 Oct 31, 2026
4a5acb3
CI: adjust test case timeout to 60 minutes
sammysun0711 Oct 31, 2026
23031f7
update git actions for node.js 24
sammysun0711 Oct 31, 2026
46281ce
Add daily pressure test
sammysun0711 Nov 1, 2026
6ccd086
ci: adjust pressure test for concurrency case
sammysun0711 Nov 1, 2026
c9b8240
ci: adjust accuracy gate to 3%
sammysun0711 Nov 2, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
268 changes: 268 additions & 0 deletions .github/workflows/qwen4-ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,268 @@
name: Qwen4 MI308X CI

on:
pull_request:
branches: [qwen4_v0.5.21]
types: [opened, reopened, synchronize, ready_for_review, labeled]
schedule:
- cron: '0 20 * * *' # 04:00 China time (UTC+8)
workflow_dispatch:
inputs:
sglang_ref:
description: 'SGLang ref; empty uses this workflow ref'
type: string
aiter_ref:
description: 'AITER ref; empty uses qwen38-flash-next-dev'
type: string
smoke:
description: 'Diagnostic two-request/two-example run; no accuracy gate'
type: boolean
default: false
case:
description: 'Comma-separated case IDs; empty runs all six'
type: string
workflow_call:
inputs:
harness_sha:
description: 'Optional controller revision for an AITER caller'
type: string
default: ''
sglang_ref:
description: 'SGLang ref to test from an AITER caller'
type: string
default: qwen4_v0.5.21
aiter_ref:
description: 'AITER ref to test'
type: string
default: ''
smoke:
type: boolean
default: false
case:
type: string
default: ''
secrets:
DOCKER_PASSWORD:
required: false

permissions:
contents: read

defaults:
run:
shell: bash

concurrency:
group: qwen4-${{ github.repository }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
validate:
name: Qwen4 CPU validation
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.sources.outputs.matrix }}
harness_sha: ${{ steps.sources.outputs.harness_sha }}
sglang_repository: ${{ steps.sources.outputs.sglang_repository }}
sglang_sha: ${{ steps.sources.outputs.sglang_sha }}
aiter_sha: ${{ steps.sources.outputs.aiter_sha }}
env:
SGLANG_REPOSITORY: ${{ !inputs.sglang_ref && github.repository == 'apinge/sglang' && github.event.pull_request.head.repo.full_name || 'apinge/sglang' }}
SGLANG_REF: ${{ inputs.sglang_ref || (github.repository == 'apinge/sglang' && (github.event.pull_request.head.sha || github.sha)) || 'qwen4_v0.5.21' }}
AITER_REF: ${{ inputs.aiter_ref || (github.repository == 'apinge/aiter' && (github.event.pull_request.head.sha || github.sha)) || 'qwen38-flash-next-dev' }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: apinge/sglang
ref: ${{ inputs.harness_sha || (github.repository == 'apinge/sglang' && github.sha) || 'qwen4_v0.5.21' }}
persist-credentials: false
- name: Validate Qwen4 driver and evaluation shell scripts
run: |
python3 -m venv "$RUNNER_TEMP/qwen4-tests"
"$RUNNER_TEMP/qwen4-tests/bin/pip" install pytest==8.3.5
"$RUNNER_TEMP/qwen4-tests/bin/python" -m pytest scripts/ci/test_qwen4_ci.py -q
for script in evaluation/*.sh; do bash -n "$script"; done
echo "Driver unit tests and evaluation shell syntax checks passed." >> "$GITHUB_STEP_SUMMARY"
- name: Checkout candidate to freeze its revision
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ env.SGLANG_REPOSITORY }}
ref: ${{ env.SGLANG_REF }}
path: candidate
persist-credentials: false
- name: Checkout AITER to freeze its revision
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: apinge/aiter
ref: ${{ env.AITER_REF }}
path: aiter-source
persist-credentials: false
- name: Select cases and preserve source revisions for retries
id: sources
env:
SELECTED_CASES: ${{ inputs.case }}
run: |
python3 - <<'PY'
import json
import os
import subprocess
import sys
from pathlib import Path

sys.path.insert(0, "scripts/ci")
from qwen4_ci import CASES

ids = [case[0] for case in CASES]
selected = {name.strip() for name in os.environ.get("SELECTED_CASES", "").split(",") if name.strip()}
unknown = selected - set(ids)
if unknown:
raise SystemExit(f"Unknown cases: {', '.join(sorted(unknown))}")
revisions = {
name: subprocess.check_output(["git", "-C", path, "rev-parse", "HEAD"], text=True).strip()
for name, path in (("harness_sha", "."), ("sglang_sha", "candidate"), ("aiter_sha", "aiter-source"))
}
outputs = {
**revisions,
"sglang_repository": os.environ["SGLANG_REPOSITORY"],
"matrix": json.dumps({"case": [name for name in ids if not selected or name in selected]}),
}
with Path(os.environ["GITHUB_OUTPUT"]).open("a") as stream:
for name, value in outputs.items():
stream.write(f"{name}={value}\n")
print(f"Selected cases: {outputs['matrix']}")
print(f"Source revisions: {revisions}")
PY

gpu:
name: Qwen4 (${{ matrix.case }})
needs: validate
if: >-
github.event_name != 'pull_request' ||
(!github.event.pull_request.draft &&
(github.event.pull_request.head.repo.full_name == github.repository ||
contains(github.event.pull_request.labels.*.name, 'run-qwen4-ci')))
runs-on: ${{ fromJSON(vars.QWEN4_RUNNER_LABELS || '["self-hosted"]') }}
timeout-minutes: 60
strategy:
fail-fast: false
max-parallel: 1
matrix: ${{ fromJSON(needs.validate.outputs.matrix) }}
env:
QWEN4_WORK_ROOT: ${{ vars.QWEN4_WORK_ROOT || '/tmp' }}
MODELS_ROOT: ${{ vars.QWEN4_MODELS_ROOT || '/raid/models' }}
IMAGE: qwen4-ci:${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.case }}
CONTAINER: qwen4-ci-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.case }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: apinge/sglang
ref: ${{ needs.validate.outputs.harness_sha }}
persist-credentials: false
- name: Checkout candidate source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ needs.validate.outputs.sglang_repository }}
ref: ${{ needs.validate.outputs.sglang_sha }}
path: candidate
persist-credentials: false
- name: Set temporary paths
run: |
mkdir -p "$QWEN4_WORK_ROOT"
work_dir=$(mktemp -d "$QWEN4_WORK_ROOT/qwen4-ci.XXXXXX")
mkdir -p "$work_dir/results" "$work_dir/docker" "$work_dir/buildx"
echo "RESULTS=$work_dir/results" >> "$GITHUB_ENV"
echo "DOCKER_CONFIG=$work_dir/docker" >> "$GITHUB_ENV"
echo "BUILDX_CONFIG=$work_dir/buildx" >> "$GITHUB_ENV"
- name: Authenticate image pull
env:
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
DOCKER_USERNAME: ${{ vars.QWEN4_DOCKER_USERNAME || 'alishared' }}
run: |
if [ -n "$DOCKER_PASSWORD" ]; then
printf '%s' "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
fi
- name: Build case image using the pinned sources and Docker cache
env:
AITER_REF: ${{ needs.validate.outputs.aiter_sha }}
BASE_IMAGE: ${{ vars.QWEN4_BASE_IMAGE || 'rocm/ali-private:ubuntu22.04_rocm7.14.0_cp310_torch2.12.0_sglang_d0a5b96_aiter_b7aed05_20261020' }}
run: |
docker build --network=host --progress=plain --build-arg "AITER_REF=$AITER_REF" \
--build-arg "BASE_IMAGE=$BASE_IMAGE" --build-arg SGLANG_SOURCE=candidate \
-f evaluation/Dockerfile.ci -t "$IMAGE" . \
2>&1 | tee "$RESULTS/build.log"
- name: Run evaluation scripts and accuracy gate
env:
CASE_ID: ${{ matrix.case }}
SMOKE: ${{ inputs.smoke }}
DAILY_PRESSURE: ${{ github.event_name == 'schedule' && github.event.schedule == '0 20 * * *' }}
run: |
trap 'docker stop -t 5 "$CONTAINER" >/dev/null 2>&1 || true' INT TERM
touch /tmp/qwen4-mi308x.lock
args=(--output /results/evaluation --case "$CASE_ID")
if [ "$SMOKE" = true ]; then args+=(--smoke); fi
if [ "$DAILY_PRESSURE" = true ]; then args+=(--pressure); fi
docker run -d --name "$CONTAINER" --device /dev/kfd --device /dev/dri \
--ipc=host --network=host --group-add video --ulimit nofile=65536:65536 --security-opt seccomp=unconfined \
-v "$MODELS_ROOT:/models:ro" -v "$RESULTS:/results" \
-v /tmp/qwen4-mi308x.lock:/tmp/qwen4-mi308x.lock \
"$IMAGE" sleep 3600
docker exec "$CONTAINER" python3 scripts/ci/qwen4_ci.py "${args[@]}" &
wait "$!"
- name: Preserve provenance and clean up
if: always()
run: |
if docker inspect "$CONTAINER" >/dev/null 2>&1; then
docker cp "$CONTAINER:/ci/build.json" "$RESULTS/build.json" || true
docker exec "$CONTAINER" chown -R "$(id -u):$(id -g)" /results || true
docker rm -f "$CONTAINER"
fi
docker image rm "$IMAGE" 2>/dev/null || true
if [ -n "${DOCKER_CONFIG:-}" ]; then rm -rf -- "$DOCKER_CONFIG"; fi
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: always()
with:
name: qwen4-results-${{ matrix.case }}-${{ github.run_attempt }}
path: |
${{ env.RESULTS }}/**
!${{ env.RESULTS }}/**/cache/**
!${{ env.RESULTS }}/**/core
!${{ env.RESULTS }}/**/core.*
if-no-files-found: warn
retention-days: 30

gate:
name: Qwen4 CI
needs: [validate, gpu]
if: >-
always() &&
(github.event_name != 'pull_request' || !github.event.pull_request.draft)
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
env:
VALIDATION_RESULT: ${{ needs.validate.result }}
CASES_RESULT: ${{ needs.gpu.result }}
steps:
- name: Checkout the summary formatter
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: apinge/sglang
ref: ${{ needs.validate.outputs.harness_sha || inputs.harness_sha || (github.repository == 'apinge/sglang' && github.sha) || 'qwen4_v0.5.21' }}
persist-credentials: false
- name: Collect case results from this run, including earlier retry attempts
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
github-token: ${{ github.token }}
run-id: ${{ github.run_id }}
pattern: qwen4-results-*
path: ${{ runner.temp }}/qwen4-case-results
- name: Summarize results and require every selected case to pass
if: always()
env:
CASE_MATRIX: ${{ needs.validate.outputs.matrix }}
CASE_FILTER: ${{ inputs.case }}
SMOKE: ${{ inputs.smoke }}
DAILY_PRESSURE: ${{ github.event_name == 'schedule' && github.event.schedule == '0 20 * * *' }}
GH_TOKEN: ${{ github.token }}
run: python3 scripts/ci/qwen4_ci.py summarize
37 changes: 37 additions & 0 deletions evaluation/Dockerfile.ci
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
ARG BASE_IMAGE=rocm/ali-private:ubuntu22.04_rocm7.14.0_cp310_torch2.12.0_sglang_d0a5b96_aiter_b7aed05_20261020
FROM ${BASE_IMAGE}

# Standalone concurrency client; add its missing packages without resolving or
# upgrading the base image's serving runtime dependencies.
RUN python3 -m pip install --no-deps --no-cache-dir \
aliyun-python-sdk-core==2.16.1 aliyun-python-sdk-kms==2.16.5 \
crcmod==1.7 jmespath==1.1.0 oss2==2.19.1 pycryptodome==4.2.0

ARG AITER_REF=qwen38-flash-next-dev
ENV AITER_USE_SYSTEM_TRITON=1 GPU_ARCHS=gfx942 PYTORCH_ROCM_ARCH=gfx942 SGLANG_BUILD_RUST_EXTS=none PYTHONPATH=""

# Freeze the companion ref once. Evaluator and pyhip pins match the recorded gate.
RUN git init /ci/aiter && \
git -C /ci/aiter remote add origin https://github.com/apinge/aiter.git && \
git -C /ci/aiter fetch --depth=1 origin "$AITER_REF" && \
git -C /ci/aiter checkout --detach FETCH_HEAD && \
git -C /ci/aiter submodule update --init --recursive
RUN git init /ci/pyhip && \
git -C /ci/pyhip remote add origin https://github.com/tingqli/pyhip.git && \
git -C /ci/pyhip fetch --depth=1 origin edde17b08ffe0dab0360fe3b1aa042b56ac72e1c && \
git -C /ci/pyhip checkout --detach FETCH_HEAD && \
git -C /ci/pyhip submodule update --init --recursive
RUN git init /ci/client && \
git -C /ci/client remote add origin https://github.com/apinge/sglang.git && \
git -C /ci/client fetch --depth=1 origin fc6ddbbf7aac80167ea04cc00bf200af8adf2b85 && \
git -C /ci/client checkout --detach FETCH_HEAD

ARG SGLANG_SOURCE=.
COPY ${SGLANG_SOURCE} /ci/sglang
# Keep the current CI recipes when testing an older candidate revision.
COPY evaluation /ci/sglang/evaluation
COPY scripts/ci/qwen4_ci.py /ci/sglang/scripts/ci/qwen4_ci.py
RUN python3 /ci/sglang/scripts/ci/qwen4_ci.py install
WORKDIR /ci/sglang
ENTRYPOINT []
CMD ["python3", "scripts/ci/qwen4_ci.py", "--output", "/results"]
Loading
Loading