From a3c6646c6f4cb56ce14747b5c09426033106e372 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 08:05:06 +0000 Subject: [PATCH 01/17] feat(speedbench): collect GLM-5.3-Flash natural MTP acceptance --- .github/workflows/speedbench-al.yml | 17 +- .../speedbench/glm53flash_fp8_b300_sglang.sh | 29 ++ inferencex-e2e/configs/runners.yaml | 1 + inferencex-e2e/infx/bench/__main__.py | 1 + inferencex-e2e/infx/bench/speedbench_al.py | 303 ++++++++++++++++++ .../infx/tests/bench/test_speedbench_al.py | 95 ++++++ 6 files changed, 441 insertions(+), 5 deletions(-) create mode 100644 inferencex-e2e/benchmarks/single_node/speedbench/glm53flash_fp8_b300_sglang.sh create mode 100644 inferencex-e2e/infx/bench/speedbench_al.py create mode 100644 inferencex-e2e/infx/tests/bench/test_speedbench_al.py diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index e036c0c98c..16936d196d 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -19,13 +19,19 @@ on: # zizmor: ignore[concurrency-limits] type: string default: 'dsv4' framework: - description: "Serving framework; selects the _fp4_b300_[_].sh collector" + description: "Serving framework; selects the __b300_[_].sh collector" required: false type: choice options: [vllm, sglang] default: 'vllm' + precision: + description: "Checkpoint precision; selects the collector and queue metadata" + required: false + type: choice + options: [fp4, fp8, bf16] + default: 'fp4' collector-variant: - description: "Collector variant suffix; non-empty selects _fp4_b300__.sh (e.g. probabilistic_sample_method_block_rejection_sample_method)" + description: "Collector variant suffix; non-empty selects __b300__.sh" required: false type: string default: '' @@ -97,7 +103,7 @@ env: HF_HUB_CACHE: '/mnt/hf_hub_cache/' MODEL: ${{ inputs.model }} MODEL_PREFIX: ${{ inputs.model-prefix }} - PRECISION: fp4 + PRECISION: ${{ inputs.precision }} FRAMEWORK: ${{ inputs.framework }} EXP_NAME: ${{ inputs.model-prefix }}_speedbench IMAGE: ${{ inputs.image }} @@ -155,11 +161,12 @@ jobs: --arg runner "$RUNNER" \ --arg model_prefix "$MODEL_PREFIX" \ --arg framework "$FRAMEWORK" \ + --arg precision "$PRECISION" \ '[{ runner: $runner, framework: $framework, "model-prefix": $model_prefix, - precision: "fp4", + precision: $precision, "spec-decoding": "mtp" }]') scored=$(printf '%s' "$entry" | @@ -250,7 +257,7 @@ jobs: run: | if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi collector_dir=benchmarks/single_node/speedbench - collector="${MODEL_PREFIX}_fp4_b300_${FRAMEWORK}" + collector="${MODEL_PREFIX}_${PRECISION}_b300_${FRAMEWORK}" if [[ -n "$COLLECTOR_VARIANT" ]]; then collector+="_${COLLECTOR_VARIANT}" fi diff --git a/inferencex-e2e/benchmarks/single_node/speedbench/glm53flash_fp8_b300_sglang.sh b/inferencex-e2e/benchmarks/single_node/speedbench/glm53flash_fp8_b300_sglang.sh new file mode 100644 index 0000000000..7439d60583 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/speedbench/glm53flash_fp8_b300_sglang.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -eo pipefail +repo_root="$(cd "$(dirname "$0")/../../.." && pwd)" +source "$repo_root/benchmarks/check_env.sh" +check_env_vars \ + CATEGORY CHAT_TEMPLATE_KWARGS_ON EP_SIZE IMAGE MODEL MODEL_PATH MTP_LIST \ + OUT_YAML PORT SPEEDBENCH_OUTPUT_LEN THINKING_MODES TP + +export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" +exec python3 -m infx.bench speedbench-al \ + --concurrency 32 \ + --temperature 1.0 \ + --top-p 0.95 \ + -- \ + --tp-size "$TP" --ep-size "$EP_SIZE" \ + --trust-remote-code \ + --mem-fraction-static 0.80 \ + --context-length 32768 \ + --chunked-prefill-size 8192 \ + --max-prefill-tokens 8192 \ + --max-running-requests 32 \ + --cuda-graph-max-bs 32 \ + --kv-cache-dtype fp8_e4m3 \ + --dsa-prefill-backend trtllm \ + --dsa-decode-backend trtllm \ + --moe-runner-backend flashinfer_trtllm \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ + --disable-radix-cache diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index c54e232edb..ff16164590 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -521,6 +521,7 @@ clusters: GLM-5.2-FP8: {root: scratch, dir: GLM-5.2-FP8} GLM-5.2-NVFP4: {root: scratch, dir: GLM-5.2-NVFP4} GLM-5.3-NVFP4: {root: scratch, dir: GLM-5.3-NVFP4} + GLM-5.3-Flash: {root: data-models, dir: GLM-5.3-Flash} Kimi-K3: {root: scratch, dir: Kimi-K3} MiniMax-M3: {root: scratch, dir: MiniMax-M3} MiniMax-M3-MXFP8: {root: scratch, dir: MiniMax-M3-MXFP8} diff --git a/inferencex-e2e/infx/bench/__main__.py b/inferencex-e2e/infx/bench/__main__.py index cb33a7ea8e..6ef5583aa1 100644 --- a/inferencex-e2e/infx/bench/__main__.py +++ b/inferencex-e2e/infx/bench/__main__.py @@ -13,6 +13,7 @@ "fixed-seq": "infx.bench.fixed_seq", "agentic": "infx.bench.agentic.run", "eval": "infx.bench.eval", + "speedbench-al": "infx.bench.speedbench_al", } diff --git a/inferencex-e2e/infx/bench/speedbench_al.py b/inferencex-e2e/infx/bench/speedbench_al.py new file mode 100644 index 0000000000..7dd1c0acdd --- /dev/null +++ b/inferencex-e2e/infx/bench/speedbench_al.py @@ -0,0 +1,303 @@ +from __future__ import annotations + +import argparse +import contextlib +import hashlib +import importlib.metadata +import json +import math +import os +import signal +import socket +import subprocess +import sys +import tempfile +import urllib.request +from collections.abc import Mapping, Sequence +from pathlib import Path + +from infx.bench import env, proc, server + +PREPARE_URL = ( + "https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/" + "bcf059af55c20a89f797724598f9908d126153e6/" + "nemo_skills/dataset/speed-bench/prepare.py" +) + + +def levels(value: str) -> list[int]: + parsed = [env.parse_positive_int("MTP_LIST", item) for item in value.split()] + if not parsed or len(parsed) != len(set(parsed)): + raise env.InputError("MTP_LIST must contain distinct positive integers") + return parsed + + +def select_category(source: Path, destination: Path, category: str) -> int: + count = 0 + with source.open(encoding="utf-8") as rows, destination.open("w", encoding="utf-8") as out: + for line in rows: + row = json.loads(line) + if row.get("category") != category: + continue + messages = row["messages"] + # SGLang's loader consumes only the first user turn; reject lossy conversion. + if ( + len(messages) != 1 + or messages[0]["role"] != "user" + or not isinstance(messages[0]["content"], str) + or not messages[0]["content"] + ): + raise env.BenchError("SPEED-Bench row must contain exactly one nonempty user turn") + out.write(json.dumps({"turns": [messages[0]["content"]]}) + "\n") + count += 1 + if count == 0: + raise env.BenchError(f"No SPEED-Bench rows for category {category!r}") + return count + + +def read_acceptance(path: Path, *, expected: int, mtp: int) -> float: + rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] + if len(rows) != 1: + raise env.BenchError(f"Expected one benchmark result in {path}, got {len(rows)}") + result = rows[0] + if result.get("completed") != expected or result.get("total_output_tokens", 0) <= 0: + raise env.BenchError(f"Incomplete SPEED-Bench result: {result.get('completed')}/{expected}") + value = result.get("accept_length") + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or not 1 <= value <= mtp + 1 + ): + raise env.BenchError(f"Invalid acceptance length for MTP={mtp}: {value!r}") + return float(value) + + +def prepare_dataset(directory: Path, category: str) -> tuple[Path, int]: + constraints = directory / "constraints.txt" + constraints.write_text( + "".join( + f"{name}=={importlib.metadata.version(name)}\n" + for name in ("huggingface-hub", "transformers", "tokenizers") + ), + encoding="utf-8", + ) + run( + [ + sys.executable, + "-m", + "pip", + "install", + "-q", + "-c", + str(constraints), + "datasets", + "tiktoken", + ] + ) + prepare = directory / "prepare.py" + with urllib.request.urlopen(PREPARE_URL, timeout=60) as response: + prepare.write_bytes(response.read()) + run([sys.executable, str(prepare), "--config", "qualitative", "--output_dir", str(directory)]) + selected = directory / "selected.jsonl" + return selected, select_category(directory / "qualitative.jsonl", selected, category) + + +def run(command: Sequence[str]) -> None: + code = proc.call(command) + if code: + raise env.BenchError(f"Command exited with status {code}: {command[0]}") + + +def free_port(requested: int) -> int: + with socket.socket() as sock: + try: + sock.bind(("127.0.0.1", requested)) + except OSError: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def stop(child: subprocess.Popen[bytes]) -> None: + with contextlib.suppress(ProcessLookupError): + os.killpg(child.pid, signal.SIGTERM) + try: + child.wait(timeout=30) + except subprocess.TimeoutExpired: + pass + finally: + # The launcher can exit before its GPU workers. + with contextlib.suppress(ProcessLookupError): + os.killpg(child.pid, signal.SIGKILL) + child.wait() + + +def collect_cell( + *, + model_path: str, + mtp: int, + port: int, + dataset: Path, + count: int, + output_len: int, + results: Path, + args: argparse.Namespace, +) -> float: + command = [ + sys.executable, + "-m", + "sglang.launch_server", + "--model-path", + model_path, + "--host", + "127.0.0.1", + "--port", + str(port), + *args.server_args, + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + str(mtp), + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + str(mtp + 1), + ] + log = results / f"server_on_mtp{mtp}.log" + result = results / f"speedbench_on_mtp{mtp}.jsonl" + result.unlink(missing_ok=True) + proc.echo(command) + with log.open("wb") as sink: + child = subprocess.Popen( + command, stdout=sink, stderr=subprocess.STDOUT, start_new_session=True + ) + try: + server.wait_ready(f"http://127.0.0.1:{port}/health", pid=child.pid, log=log) + run( + [ + sys.executable, + "-m", + "sglang.bench_serving", + "--backend", + "sglang", + "--host", + "127.0.0.1", + "--port", + str(port), + "--model", + model_path, + "--dataset-name", + "speed-bench", + "--dataset-path", + str(dataset), + "--speed-bench-output-len", + str(output_len), + "--num-prompts", + str(count), + "--max-concurrency", + str(args.concurrency), + "--temperature", + str(args.temperature), + "--top-p", + str(args.top_p), + # No warmup generations may contaminate server-wide acceptance counters. + "--warmup-requests", + "0", + "--seed", + "42", + "--disable-ignore-eos", + "--output-file", + str(result), + "--output-details", + ] + ) + return read_acceptance(result, expected=count, mtp=mtp) + finally: + stop(child) + + +def write_matrix(path: Path, model: str, values: Mapping[int, float], metadata: Path) -> None: + text = f"# Collection provenance: {metadata.name}\n{json.dumps(model.rsplit('/', 1)[-1].lower())}:\n" + text += " thinking_on:\n" + text += "".join(f" {level}: {value:.6f}\n" for level, value in values.items()) + path.write_text(text, encoding="utf-8") + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--concurrency", type=int, required=True) + parser.add_argument("--temperature", type=float, required=True) + parser.add_argument("--top-p", type=float, required=True) + parser.add_argument("server_args", nargs=argparse.REMAINDER) + args = parser.parse_args(argv) + if args.server_args[:1] == ["--"]: + args.server_args = args.server_args[1:] + if args.concurrency < 1 or not args.temperature > 0 or not 0 < args.top_p <= 1: + raise env.InputError("Invalid concurrency or sampling parameters") + inputs = env.require( + "CATEGORY", + "CHAT_TEMPLATE_KWARGS_ON", + "IMAGE", + "MODEL", + "MODEL_PATH", + "MTP_LIST", + "OUT_YAML", + "PORT", + "SPEEDBENCH_OUTPUT_LEN", + "THINKING_MODES", + ) + # The upstream SPEED-Bench loader cannot forward chat-template kwargs. + if inputs["THINKING_MODES"] != "on" or json.loads(inputs["CHAT_TEMPLATE_KWARGS_ON"]) != {}: + raise env.InputError( + "SGLang SPEED-Bench requires THINKING_MODES=on and CHAT_TEMPLATE_KWARGS_ON={}" + ) + mtp_levels = levels(inputs["MTP_LIST"]) + output_len = env.positive_int("SPEEDBENCH_OUTPUT_LEN") + requested_port = env.positive_int("PORT") + model = Path(inputs["MODEL_PATH"]) + if not (model / "config.json").is_file(): + raise env.BenchError( + f"Staged checkpoint missing: {model / 'config.json'}; no download attempted" + ) + out = Path(inputs["OUT_YAML"]) + out.unlink(missing_ok=True) + results = out.parent / "speedbench_results" + results.mkdir(exist_ok=True) + metadata = results / "speedbench_provenance.json" + with tempfile.TemporaryDirectory(prefix="infx-speedbench-") as temp: + dataset, count = prepare_dataset(Path(temp), inputs["CATEGORY"]) + provenance = { + "inputs": inputs, + "server_args": args.server_args, + "concurrency": args.concurrency, + "temperature": args.temperature, + "top_p": args.top_p, + "thinking": "chat template default; reasoning_effort=max for GLM-5.3-Flash", + "draft": "embedded MTP, upstream checkpoint-default precision", + "verification": "standard (no block or synthetic acceptance flags)", + "prepare_url": PREPARE_URL, + "num_prompts": count, + "dataset_sha256": hashlib.sha256(dataset.read_bytes()).hexdigest(), + "config_sha256": hashlib.sha256((model / "config.json").read_bytes()).hexdigest(), + "sglang_version": importlib.metadata.version("sglang"), + "complete": False, + } + metadata.write_text(json.dumps(provenance, indent=2) + "\n", encoding="utf-8") + values: dict[int, float] = {} + for mtp in mtp_levels: + print(f"Collecting thinking=on MTP={mtp}, {count} prompts", flush=True) + values[mtp] = collect_cell( + model_path=str(model), + mtp=mtp, + port=free_port(requested_port), + dataset=dataset, + count=count, + output_len=output_len, + results=results, + args=args, + ) + write_matrix(out, inputs["MODEL"], values, metadata) + print(f"MTP={mtp}: AL={values[mtp]:.6f}", flush=True) + provenance["complete"] = True + metadata.write_text(json.dumps(provenance, indent=2) + "\n", encoding="utf-8") + return 0 diff --git a/inferencex-e2e/infx/tests/bench/test_speedbench_al.py b/inferencex-e2e/infx/tests/bench/test_speedbench_al.py new file mode 100644 index 0000000000..a1afcd8a75 --- /dev/null +++ b/inferencex-e2e/infx/tests/bench/test_speedbench_al.py @@ -0,0 +1,95 @@ +import json + +import pytest +import yaml + +from infx.bench.env import BenchError, InputError +from infx.bench.speedbench_al import levels, read_acceptance, select_category, write_matrix + + +def test_select_category_preserves_all_matching_prompts(tmp_path): + source = tmp_path / "source.jsonl" + source.write_text( + "\n".join( + json.dumps( + { + "category": category, + "messages": [{"role": "user", "content": content}], + } + ) + for category, content in [("coding", "one"), ("math", "two"), ("coding", "three")] + ) + ) + destination = tmp_path / "selected.jsonl" + assert select_category(source, destination, "coding") == 2 + assert [json.loads(row) for row in destination.read_text().splitlines()] == [ + {"turns": ["one"]}, + {"turns": ["three"]}, + ] + with pytest.raises(BenchError, match="No SPEED-Bench"): + select_category(source, destination, "missing") + + +@pytest.mark.parametrize( + "messages", + [ + [], + [{"role": "system", "content": "instruction"}, {"role": "user", "content": "question"}], + [{"role": "user", "content": ""}], + [{"role": "user", "content": ["multimodal"]}], + ], +) +def test_select_category_rejects_lossy_conversion(tmp_path, messages): + source = tmp_path / "source.jsonl" + source.write_text(json.dumps({"category": "coding", "messages": messages})) + with pytest.raises(BenchError, match="exactly one"): + select_category(source, tmp_path / "selected.jsonl", "coding") + + +def test_acceptance_and_matrix(tmp_path): + result = tmp_path / "result.jsonl" + result.write_text( + json.dumps({"completed": 20, "total_output_tokens": 100, "accept_length": 2.3456789}) + ) + value = read_acceptance(result, expected=20, mtp=3) + matrix = tmp_path / "matrix.yaml" + write_matrix(matrix, "zai-org/GLM-5.3-Flash", {3: value}, tmp_path / "provenance.json") + assert yaml.safe_load(matrix.read_text()) == {"glm-5.3-flash": {"thinking_on": {3: 2.345679}}} + + +@pytest.mark.parametrize( + "updates", + [ + {"completed": 19}, + {"total_output_tokens": 0}, + {"accept_length": None}, + {"accept_length": "N/A"}, + {"accept_length": float("nan")}, + {"accept_length": float("inf")}, + {"accept_length": True}, + {"accept_length": 0.99}, + {"accept_length": 4.01}, + ], +) +def test_reject_incomplete_or_invalid_measurements(tmp_path, updates): + result = tmp_path / "result.jsonl" + result.write_text( + json.dumps({"completed": 20, "total_output_tokens": 100, "accept_length": 2.5} | updates) + ) + with pytest.raises(BenchError): + read_acceptance(result, expected=20, mtp=3) + + +@pytest.mark.parametrize("content", ["", "{}\n{}\n"]) +def test_reject_missing_or_stale_results(tmp_path, content): + result = tmp_path / "result.jsonl" + result.write_text(content) + with pytest.raises(BenchError, match="Expected one"): + read_acceptance(result, expected=20, mtp=3) + + +def test_levels(): + assert levels("1 3 8") == [1, 3, 8] + for invalid in ("", "1 1", "0", "-1", "a"): + with pytest.raises(InputError): + levels(invalid) From 8ea3013a89b3f5fdd4de7c58c31d484ec948798c Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 08:09:31 +0000 Subject: [PATCH 02/17] fix(speedbench): use vLLM B300 for GLM-5.3-Flash collection --- .../speedbench/glm53flash_fp8_b300_sglang.sh | 29 ----- .../speedbench/glm53flash_fp8_b300_vllm.sh | 27 +++++ inferencex-e2e/infx/bench/speedbench_al.py | 100 +++++++++++------- .../infx/tests/bench/test_speedbench_al.py | 48 ++++++--- 4 files changed, 122 insertions(+), 82 deletions(-) delete mode 100644 inferencex-e2e/benchmarks/single_node/speedbench/glm53flash_fp8_b300_sglang.sh create mode 100644 inferencex-e2e/benchmarks/single_node/speedbench/glm53flash_fp8_b300_vllm.sh diff --git a/inferencex-e2e/benchmarks/single_node/speedbench/glm53flash_fp8_b300_sglang.sh b/inferencex-e2e/benchmarks/single_node/speedbench/glm53flash_fp8_b300_sglang.sh deleted file mode 100644 index 7439d60583..0000000000 --- a/inferencex-e2e/benchmarks/single_node/speedbench/glm53flash_fp8_b300_sglang.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -repo_root="$(cd "$(dirname "$0")/../../.." && pwd)" -source "$repo_root/benchmarks/check_env.sh" -check_env_vars \ - CATEGORY CHAT_TEMPLATE_KWARGS_ON EP_SIZE IMAGE MODEL MODEL_PATH MTP_LIST \ - OUT_YAML PORT SPEEDBENCH_OUTPUT_LEN THINKING_MODES TP - -export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" -exec python3 -m infx.bench speedbench-al \ - --concurrency 32 \ - --temperature 1.0 \ - --top-p 0.95 \ - -- \ - --tp-size "$TP" --ep-size "$EP_SIZE" \ - --trust-remote-code \ - --mem-fraction-static 0.80 \ - --context-length 32768 \ - --chunked-prefill-size 8192 \ - --max-prefill-tokens 8192 \ - --max-running-requests 32 \ - --cuda-graph-max-bs 32 \ - --kv-cache-dtype fp8_e4m3 \ - --dsa-prefill-backend trtllm \ - --dsa-decode-backend trtllm \ - --moe-runner-backend flashinfer_trtllm \ - --reasoning-parser glm45 \ - --tool-call-parser glm47 \ - --disable-radix-cache diff --git a/inferencex-e2e/benchmarks/single_node/speedbench/glm53flash_fp8_b300_vllm.sh b/inferencex-e2e/benchmarks/single_node/speedbench/glm53flash_fp8_b300_vllm.sh new file mode 100644 index 0000000000..088c65a168 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/speedbench/glm53flash_fp8_b300_vllm.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -eo pipefail +repo_root="$(cd "$(dirname "$0")/../../.." && pwd)" +source "$repo_root/benchmarks/check_env.sh" +check_env_vars \ + CATEGORY CHAT_TEMPLATE_KWARGS_ON IMAGE MODEL MODEL_PATH MTP_LIST \ + OUT_YAML PORT SPEEDBENCH_OUTPUT_LEN THINKING_MODES TP + +export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_USE_RUST_FRONTEND=0 +exec python3 -m infx.bench speedbench-al \ + --concurrency 1 \ + --temperature 1.0 \ + --top-p 0.95 \ + -- \ + --tensor-parallel-size "$TP" --data-parallel-size 1 \ + --trust-remote-code \ + --gpu-memory-utilization 0.80 \ + --max-model-len 32768 \ + --max-num-batched-tokens 8192 \ + --max-num-seqs 32 \ + --kv-cache-dtype fp8 \ + --tokenizer-mode auto \ + --reasoning-parser glm47 \ + --chat-template-content-format string \ + --no-enable-prefix-caching diff --git a/inferencex-e2e/infx/bench/speedbench_al.py b/inferencex-e2e/infx/bench/speedbench_al.py index 7dd1c0acdd..e1dd7dff41 100644 --- a/inferencex-e2e/infx/bench/speedbench_al.py +++ b/inferencex-e2e/infx/bench/speedbench_al.py @@ -40,7 +40,7 @@ def select_category(source: Path, destination: Path, category: str) -> int: if row.get("category") != category: continue messages = row["messages"] - # SGLang's loader consumes only the first user turn; reject lossy conversion. + # Coding collection is single-turn; do not silently flatten conversations. if ( len(messages) != 1 or messages[0]["role"] != "user" @@ -48,7 +48,7 @@ def select_category(source: Path, destination: Path, category: str) -> int: or not messages[0]["content"] ): raise env.BenchError("SPEED-Bench row must contain exactly one nonempty user turn") - out.write(json.dumps({"turns": [messages[0]["content"]]}) + "\n") + out.write(json.dumps(row) + "\n") count += 1 if count == 0: raise env.BenchError(f"No SPEED-Bench rows for category {category!r}") @@ -56,13 +56,17 @@ def select_category(source: Path, destination: Path, category: str) -> int: def read_acceptance(path: Path, *, expected: int, mtp: int) -> float: - rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] - if len(rows) != 1: - raise env.BenchError(f"Expected one benchmark result in {path}, got {len(rows)}") - result = rows[0] - if result.get("completed") != expected or result.get("total_output_tokens", 0) <= 0: + try: + result = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise env.BenchError(f"Expected one benchmark result in {path}") from error + if ( + result.get("completed") != expected + or result.get("failed", 0) != 0 + or result.get("total_output_tokens", 0) <= 0 + ): raise env.BenchError(f"Incomplete SPEED-Bench result: {result.get('completed')}/{expected}") - value = result.get("accept_length") + value = result.get("spec_decode_acceptance_length") if ( isinstance(value, bool) or not isinstance(value, (int, float)) @@ -70,6 +74,16 @@ def read_acceptance(path: Path, *, expected: int, mtp: int) -> float: or not 1 <= value <= mtp + 1 ): raise env.BenchError(f"Invalid acceptance length for MTP={mtp}: {value!r}") + drafts = result.get("spec_decode_num_drafts") + accepted = result.get("spec_decode_accepted_tokens") + if ( + type(drafts) is not int + or drafts <= 0 + or type(accepted) is not int + or not 0 <= accepted <= mtp * drafts + or not math.isclose(value, 1 + accepted / drafts, rel_tol=1e-9) + ): + raise env.BenchError("Missing or inconsistent speculative decoding counter deltas") return float(value) @@ -100,7 +114,10 @@ def prepare_dataset(directory: Path, category: str) -> tuple[Path, int]: prepare.write_bytes(response.read()) run([sys.executable, str(prepare), "--config", "qualitative", "--output_dir", str(directory)]) selected = directory / "selected.jsonl" - return selected, select_category(directory / "qualitative.jsonl", selected, category) + dataset = directory / "qualitative.jsonl" + count = select_category(dataset, selected, category) + selected.replace(dataset) + return dataset, count def run(command: Sequence[str]) -> None: @@ -144,27 +161,19 @@ def collect_cell( args: argparse.Namespace, ) -> float: command = [ - sys.executable, - "-m", - "sglang.launch_server", - "--model-path", + "vllm", + "serve", model_path, "--host", "127.0.0.1", "--port", str(port), *args.server_args, - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - str(mtp), - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - str(mtp + 1), + "--speculative-config", + json.dumps({"method": "mtp", "num_speculative_tokens": mtp}), ] log = results / f"server_on_mtp{mtp}.log" - result = results / f"speedbench_on_mtp{mtp}.jsonl" + result = results / f"speedbench_on_mtp{mtp}.json" result.unlink(missing_ok=True) proc.echo(command) with log.open("wb") as sink: @@ -175,11 +184,11 @@ def collect_cell( server.wait_ready(f"http://127.0.0.1:{port}/health", pid=child.pid, log=log) run( [ - sys.executable, - "-m", - "sglang.bench_serving", + "vllm", + "bench", + "serve", "--backend", - "sglang", + "vllm", "--host", "127.0.0.1", "--port", @@ -187,28 +196,35 @@ def collect_cell( "--model", model_path, "--dataset-name", - "speed-bench", + "speed_bench", "--dataset-path", - str(dataset), + str(dataset.parent), "--speed-bench-output-len", str(output_len), "--num-prompts", - str(count), + "-1", "--max-concurrency", str(args.concurrency), "--temperature", str(args.temperature), "--top-p", str(args.top_p), - # No warmup generations may contaminate server-wide acceptance counters. - "--warmup-requests", + "--chat-template-kwargs", + os.environ["CHAT_TEMPLATE_KWARGS_ON"], + "--trust-remote-code", + # vLLM records counter deltas around the formal request window. + "--num-warmups", + "0", + "--ready-check-timeout-sec", "0", "--seed", "42", - "--disable-ignore-eos", - "--output-file", - str(result), - "--output-details", + "--save-result", + "--save-detailed", + "--result-dir", + str(results), + "--result-filename", + result.name, ] ) return read_acceptance(result, expected=count, mtp=mtp) @@ -246,10 +262,13 @@ def main(argv: list[str]) -> int: "SPEEDBENCH_OUTPUT_LEN", "THINKING_MODES", ) - # The upstream SPEED-Bench loader cannot forward chat-template kwargs. - if inputs["THINKING_MODES"] != "on" or json.loads(inputs["CHAT_TEMPLATE_KWARGS_ON"]) != {}: + # GLM-5.3-Flash always thinks; enable_thinking=false does not disable it. + if inputs["THINKING_MODES"] != "on" or json.loads(inputs["CHAT_TEMPLATE_KWARGS_ON"]) not in ( + {}, + {"reasoning_effort": "max"}, + ): raise env.InputError( - "SGLang SPEED-Bench requires THINKING_MODES=on and CHAT_TEMPLATE_KWARGS_ON={}" + "GLM-5.3-Flash collection requires THINKING_MODES=on and max reasoning effort" ) mtp_levels = levels(inputs["MTP_LIST"]) output_len = env.positive_int("SPEEDBENCH_OUTPUT_LEN") @@ -272,14 +291,15 @@ def main(argv: list[str]) -> int: "concurrency": args.concurrency, "temperature": args.temperature, "top_p": args.top_p, - "thinking": "chat template default; reasoning_effort=max for GLM-5.3-Flash", + "thinking": "reasoning_effort=max for GLM-5.3-Flash", "draft": "embedded MTP, upstream checkpoint-default precision", "verification": "standard (no block or synthetic acceptance flags)", "prepare_url": PREPARE_URL, "num_prompts": count, "dataset_sha256": hashlib.sha256(dataset.read_bytes()).hexdigest(), "config_sha256": hashlib.sha256((model / "config.json").read_bytes()).hexdigest(), - "sglang_version": importlib.metadata.version("sglang"), + "vllm_version": importlib.metadata.version("vllm"), + "frontend": os.environ.get("VLLM_USE_RUST_FRONTEND"), "complete": False, } metadata.write_text(json.dumps(provenance, indent=2) + "\n", encoding="utf-8") diff --git a/inferencex-e2e/infx/tests/bench/test_speedbench_al.py b/inferencex-e2e/infx/tests/bench/test_speedbench_al.py index a1afcd8a75..1d868ddeea 100644 --- a/inferencex-e2e/infx/tests/bench/test_speedbench_al.py +++ b/inferencex-e2e/infx/tests/bench/test_speedbench_al.py @@ -23,8 +23,8 @@ def test_select_category_preserves_all_matching_prompts(tmp_path): destination = tmp_path / "selected.jsonl" assert select_category(source, destination, "coding") == 2 assert [json.loads(row) for row in destination.read_text().splitlines()] == [ - {"turns": ["one"]}, - {"turns": ["three"]}, + {"category": "coding", "messages": [{"role": "user", "content": "one"}]}, + {"category": "coding", "messages": [{"role": "user", "content": "three"}]}, ] with pytest.raises(BenchError, match="No SPEED-Bench"): select_category(source, destination, "missing") @@ -47,9 +47,17 @@ def test_select_category_rejects_lossy_conversion(tmp_path, messages): def test_acceptance_and_matrix(tmp_path): - result = tmp_path / "result.jsonl" + result = tmp_path / "result.json" result.write_text( - json.dumps({"completed": 20, "total_output_tokens": 100, "accept_length": 2.3456789}) + json.dumps( + { + "completed": 20, + "total_output_tokens": 100, + "spec_decode_acceptance_length": 2.3456789, + "spec_decode_num_drafts": 10000000, + "spec_decode_accepted_tokens": 13456789, + } + ) ) value = read_acceptance(result, expected=20, mtp=3) matrix = tmp_path / "matrix.yaml" @@ -62,19 +70,33 @@ def test_acceptance_and_matrix(tmp_path): [ {"completed": 19}, {"total_output_tokens": 0}, - {"accept_length": None}, - {"accept_length": "N/A"}, - {"accept_length": float("nan")}, - {"accept_length": float("inf")}, - {"accept_length": True}, - {"accept_length": 0.99}, - {"accept_length": 4.01}, + {"failed": 1}, + {"spec_decode_acceptance_length": None}, + {"spec_decode_acceptance_length": "N/A"}, + {"spec_decode_acceptance_length": float("nan")}, + {"spec_decode_acceptance_length": float("inf")}, + {"spec_decode_acceptance_length": True}, + {"spec_decode_acceptance_length": 0.99}, + {"spec_decode_acceptance_length": 4.01}, + {"spec_decode_num_drafts": 0}, + {"spec_decode_num_drafts": None}, + {"spec_decode_accepted_tokens": -1}, + {"spec_decode_accepted_tokens": 301}, ], ) def test_reject_incomplete_or_invalid_measurements(tmp_path, updates): - result = tmp_path / "result.jsonl" + result = tmp_path / "result.json" result.write_text( - json.dumps({"completed": 20, "total_output_tokens": 100, "accept_length": 2.5} | updates) + json.dumps( + { + "completed": 20, + "total_output_tokens": 100, + "spec_decode_acceptance_length": 2.5, + "spec_decode_num_drafts": 200, + "spec_decode_accepted_tokens": 300, + } + | updates + ) ) with pytest.raises(BenchError): read_acceptance(result, expected=20, mtp=3) From 7e8ca71fa27bc2a70324b9fe978f8deefed8c332 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 08:20:48 +0000 Subject: [PATCH 03/17] fix(speedbench): preserve multi-turn rows and drain GPU workers --- inferencex-e2e/infx/bench/speedbench_al.py | 29 ++++++++++++-- .../infx/tests/bench/test_speedbench_al.py | 38 ++++++++++++++++++- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/inferencex-e2e/infx/bench/speedbench_al.py b/inferencex-e2e/infx/bench/speedbench_al.py index e1dd7dff41..291fe9eed6 100644 --- a/inferencex-e2e/infx/bench/speedbench_al.py +++ b/inferencex-e2e/infx/bench/speedbench_al.py @@ -12,6 +12,7 @@ import subprocess import sys import tempfile +import time import urllib.request from collections.abc import Mapping, Sequence from pathlib import Path @@ -40,14 +41,14 @@ def select_category(source: Path, destination: Path, category: str) -> int: if row.get("category") != category: continue messages = row["messages"] - # Coding collection is single-turn; do not silently flatten conversations. + # vLLM samples the first turn of each row, including multi-turn rows. if ( - len(messages) != 1 + not messages or messages[0]["role"] != "user" or not isinstance(messages[0]["content"], str) or not messages[0]["content"] ): - raise env.BenchError("SPEED-Bench row must contain exactly one nonempty user turn") + raise env.BenchError("SPEED-Bench row must start with a nonempty text user turn") out.write(json.dumps(row) + "\n") count += 1 if count == 0: @@ -147,6 +148,24 @@ def stop(child: subprocess.Popen[bytes]) -> None: with contextlib.suppress(ProcessLookupError): os.killpg(child.pid, signal.SIGKILL) child.wait() + wait_gpu_idle() + + +def wait_gpu_idle() -> None: + # Reaping the launcher does not wait for TP workers to release their CUDA contexts. + deadline = time.monotonic() + 120 + while True: + output = subprocess.check_output( + ["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"], + text=True, + timeout=10, + ) + used = [int(line.strip()) for line in output.splitlines() if line.strip()] + if used and max(used) < 2000: + return + if time.monotonic() >= deadline: + raise env.BenchError(f"GPU memory did not drain after server shutdown: {used} MiB") + time.sleep(3) def collect_cell( @@ -296,6 +315,10 @@ def main(argv: list[str]) -> int: "verification": "standard (no block or synthetic acceptance flags)", "prepare_url": PREPARE_URL, "num_prompts": count, + "prompt_selection": "upstream vLLM SPEED-Bench: first user turn of each category row", + "rows_with_followup_turns": sum( + len(json.loads(line)["messages"]) > 1 for line in dataset.read_text().splitlines() + ), "dataset_sha256": hashlib.sha256(dataset.read_bytes()).hexdigest(), "config_sha256": hashlib.sha256((model / "config.json").read_bytes()).hexdigest(), "vllm_version": importlib.metadata.version("vllm"), diff --git a/inferencex-e2e/infx/tests/bench/test_speedbench_al.py b/inferencex-e2e/infx/tests/bench/test_speedbench_al.py index 1d868ddeea..e1a8279550 100644 --- a/inferencex-e2e/infx/tests/bench/test_speedbench_al.py +++ b/inferencex-e2e/infx/tests/bench/test_speedbench_al.py @@ -3,6 +3,7 @@ import pytest import yaml +from infx.bench import speedbench_al from infx.bench.env import BenchError, InputError from infx.bench.speedbench_al import levels, read_acceptance, select_category, write_matrix @@ -39,13 +40,46 @@ def test_select_category_preserves_all_matching_prompts(tmp_path): [{"role": "user", "content": ["multimodal"]}], ], ) -def test_select_category_rejects_lossy_conversion(tmp_path, messages): +def test_select_category_rejects_invalid_first_turn(tmp_path, messages): source = tmp_path / "source.jsonl" source.write_text(json.dumps({"category": "coding", "messages": messages})) - with pytest.raises(BenchError, match="exactly one"): + with pytest.raises(BenchError, match="start with"): select_category(source, tmp_path / "selected.jsonl", "coding") +def test_select_category_preserves_multiturn_rows_for_native_vllm_loader(tmp_path): + source = tmp_path / "source.jsonl" + row = { + "category": "coding", + "messages": [ + {"role": "user", "content": "Write a parser"}, + {"role": "user", "content": "Add error handling"}, + ], + } + source.write_text(json.dumps(row)) + destination = tmp_path / "selected.jsonl" + assert select_category(source, destination, "coding") == 1 + assert json.loads(destination.read_text()) == row + + +def test_wait_gpu_idle_waits_for_workers(monkeypatch): + readings = iter(["15000\n16000\n", "1000\n1900\n"]) + monkeypatch.setattr(speedbench_al.subprocess, "check_output", lambda *a, **kw: next(readings)) + waits = [] + monkeypatch.setattr(speedbench_al.time, "sleep", waits.append) + speedbench_al.wait_gpu_idle() + assert waits == [3] + + +@pytest.mark.parametrize("output", ["", "3000\n4000\n"]) +def test_wait_gpu_idle_fails_closed(monkeypatch, output): + monkeypatch.setattr(speedbench_al.subprocess, "check_output", lambda *a, **kw: output) + clock = iter([0, 121]) + monkeypatch.setattr(speedbench_al.time, "monotonic", lambda: next(clock)) + with pytest.raises(BenchError, match="did not drain"): + speedbench_al.wait_gpu_idle() + + def test_acceptance_and_matrix(tmp_path): result = tmp_path / "result.json" result.write_text( From 48627c4a3b8157789edff95972247e39716aacc2 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:05:02 +0000 Subject: [PATCH 04/17] feat(golden-al): commit measured GLM-5.3-Flash MTP curve --- .../golden_al_distribution/glm5.3flash_mtp.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 inferencex-e2e/infx/golden_al_distribution/glm5.3flash_mtp.yaml diff --git a/inferencex-e2e/infx/golden_al_distribution/glm5.3flash_mtp.yaml b/inferencex-e2e/infx/golden_al_distribution/glm5.3flash_mtp.yaml new file mode 100644 index 0000000000..8216550753 --- /dev/null +++ b/inferencex-e2e/infx/golden_al_distribution/glm5.3flash_mtp.yaml @@ -0,0 +1,13 @@ +# Source GitHub Actions run: https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41288481245 +# SPEED-Bench coding, 80 first-user-turn prompts, concurrency 1, max reasoning effort. +# vLLM B300 TP8, temperature 1.0, top_p 0.95, natural EOS, output cap 4096. +glm-5.3-flash: + thinking_on: + 1: 1.858382 + 2: 2.518506 + 3: 3.004589 + 4: 3.349344 + 5: 3.603536 + 6: 3.790803 + 7: 3.925358 + 8: 3.992497 From 12eea95ef965295bd2dc0baf0cbb06f018a486fd Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:09:10 +0000 Subject: [PATCH 05/17] fix(ci): allow authorized sweeps on stacked pull requests --- .github/workflows/README.md | 2 +- .github/workflows/run-sweep.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 1822672c27..a3b9d45166 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -162,7 +162,7 @@ test-config --config-keys dsr1-fp8-h200-sglang --evals-only --all-evals --config ## PR Sweep Labels -`run-sweep.yml` sweeps only same-repository PRs that change +`run-sweep.yml` sweeps only same-repository PRs, including stacked PRs, that change `inferencex-e2e/perf-changelog.yaml`, using the appended entries as the matrix. Apply exactly one primary label, normally `full-sweep-fail-fast`. Fork PRs use the [trusted dispatch](#trusted-external-fork-sweep-dispatch-poc). diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 19f6d4d2c6..03842620bf 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -18,8 +18,6 @@ concurrency: on: pull_request: - branches: - - main types: - synchronize - labeled From d49bdf1d416b420b4a3b45e029b75955c3fce53f Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:10:32 +0000 Subject: [PATCH 06/17] fix(bench): preserve golden acceptance precision in native overrides --- .../infx/srt_slurm/synthetic_acceptance.py | 8 ++--- .../srt_slurm/test_synthetic_acceptance.py | 36 +++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index e7da9b72bb..7ae0f9f9ae 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -248,16 +248,16 @@ def build_overrides( elif worker_engine == "atom": key = "spec-decode-acceptance-length" if worker_al is not None and worker_spec: - overrides += ["--set", f"{prefix}.args.{key}={worker_al:g}"] + overrides += ["--set", f"{prefix}.args.{key}={worker_al:.15g}"] elif key in (role.get("args") or {}): overrides += ["--unset", f"{prefix}.args.{key}"] elif worker_al is not None and worker_spec: values = ( - (f"{worker_al:g}", "match-expected", "real-draft-token") + (f"{worker_al:.15g}", "match-expected", "real-draft-token") if engine == "sglang" - else (f"{worker_al:g}", "match-expected") + else (f"{worker_al:.15g}", "match-expected") if engine == "tilert" - else (f"{worker_al - 1:g}",) + else (f"{worker_al - 1:.15g}",) ) for key, value in zip(variables, values, strict=True): overrides += ["--set", f"{prefix}.env.{key}={json.dumps(value)}"] diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index e687fc922c..0a309f4047 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -762,6 +762,42 @@ def test_sglang_abbreviated_algorithm_selects_golden( } +@pytest.mark.parametrize( + ("engine", "args", "key", "expected"), + [ + ( + "sglang", + {"speculative-algorithm": "EAGLE", "speculative-num-steps": 3}, + "SGLANG_SIMULATE_ACC_LEN", + "3.603536", + ), + ( + "tilert", + {"with-mtp": True, "num-mtp": 3}, + "TILERT_SIMULATE_ACC_LEN", + "3.603536", + ), + ( + "trt", + {"speculative_config": {"decoding_type": "mtp", "max_draft_len": 3}}, + "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS", + "2.603536", + ), + ], +) +def test_precise_golden_length_survives_native_overrides( + golden_dir: Path, engine: str, args: dict[str, Any], key: str, expected: str +) -> None: + (golden_dir / "dsv4_mtp.yaml").write_text( + yaml.safe_dump({"deepseek-v4-pro": {"thinking_on": {3: 3.603536}}}) + ) + recipe = {"roles": {"decode": {"args": args}}} + result = apply_native( + recipe, build_overrides(recipe, engine, ENV, golden_dir=golden_dir) + ) + assert result["roles"]["decode"]["env"][key] == expected + + def test_sglang_conflicting_algorithm_aliases_fail(golden_dir: Path) -> None: recipe = { "roles": { From 86f215a33bfc8b4326d6c3900ec18bd8d5c9d6c5 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:22:19 +0000 Subject: [PATCH 07/17] fix(ci): preserve canary gates after skipped reuse checks --- .github/workflows/run-sweep.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 03842620bf..acf9c1995c 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -316,6 +316,7 @@ jobs: name: canary-select needs: setup if: >- + !cancelled() && needs.setup.result == 'success' && ( contains(github.event.pull_request.labels.*.name, 'full-sweep-enabled') || contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') @@ -366,7 +367,7 @@ jobs: canary-sweep: needs: canary-select - if: ${{ needs.canary-select.outputs.canary-config != '' && needs.canary-select.outputs.canary-config != '[]' }} + if: ${{ !cancelled() && needs.canary-select.result == 'success' && needs.canary-select.outputs.canary-config != '' && needs.canary-select.outputs.canary-config != '[]' }} uses: $/.github/workflows/benchmark-tmpl.yml name: canary / strategy: @@ -390,7 +391,7 @@ jobs: canary-multi-node-sweep: needs: canary-select - if: ${{ needs.canary-select.outputs.multi-node-canary-config != '' && needs.canary-select.outputs.multi-node-canary-config != '[]' }} + if: ${{ !cancelled() && needs.canary-select.result == 'success' && needs.canary-select.outputs.multi-node-canary-config != '' && needs.canary-select.outputs.multi-node-canary-config != '[]' }} uses: $/.github/workflows/benchmark-multinode-tmpl.yml name: multi-node canary / strategy: From 60795af86bb8fc7bceeea55def078451515f605d Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:36:23 +0000 Subject: [PATCH 08/17] feat(bench): add native GLM-5.3-Flash FP8 MI325X AgentX lane --- .../configs/stage-hf-checkpoint.sh | 32 +++++ .../mi325x-fp8/agentx/agg-variants.yaml | 125 ++++++++++++++++++ inferencex-e2e/configs/amd-master.yaml | 18 +++ .../srt_slurm/test_checkpoint_staging.py | 70 ++++++++++ 4 files changed, 245 insertions(+) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/stage-hf-checkpoint.sh create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml create mode 100644 inferencex-e2e/infx/tests/srt_slurm/test_checkpoint_staging.py diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/stage-hf-checkpoint.sh b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/stage-hf-checkpoint.sh new file mode 100644 index 0000000000..2da787d3d7 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/stage-hf-checkpoint.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -eo pipefail + +source /infmax-workspace/benchmarks/check_env.sh +check_env_vars HF_HUB_CACHE HF_CHECKPOINT_REPO HF_CHECKPOINT_REVISION HF_CHECKPOINT_BUDGET_BYTES + +if [[ ! "$HF_CHECKPOINT_REVISION" =~ ^[0-9a-f]{40}$ ]] || + [[ ! "$HF_CHECKPOINT_BUDGET_BYTES" =~ ^[1-9][0-9]*$ ]]; then + echo "Checkpoint staging requires a commit SHA and a positive byte budget" >&2 + exit 1 +fi +if [[ "$HF_HUB_OFFLINE" == "1" || "$TRANSFORMERS_OFFLINE" == "1" ]]; then + echo "Checkpoint staging requires online Hugging Face access" >&2 + exit 1 +fi + +mkdir -p "$HF_HUB_CACHE" +cache_name="models--${HF_CHECKPOINT_REPO//\//--}" +exec 9>"$HF_HUB_CACHE/.${cache_name}.download.lock" +flock -x 9 +available=$(df -PB1 "$HF_HUB_CACHE" | awk 'NR == 2 {print $4}') +cached=0 +if [[ -d "$HF_HUB_CACHE/$cache_name" ]]; then + cached=$(du -sB1 "$HF_HUB_CACHE/$cache_name" | awk '{print $1}') +fi +if (( available + cached < HF_CHECKPOINT_BUDGET_BYTES )); then + echo "Insufficient shared-cache disk: available=$available cached=$cached required=$HF_CHECKPOINT_BUDGET_BYTES" >&2 + exit 1 +fi +hf download "$HF_CHECKPOINT_REPO" \ + --revision "$HF_CHECKPOINT_REVISION" \ + --cache-dir "$HF_HUB_CACHE" diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml new file mode 100644 index 0000000000..4755ed3ce8 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml @@ -0,0 +1,125 @@ +schema: 2 + +base: + identity: + model: + revision: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c + container: {} + frameworks: + sglang: v0.5.21 + resources: + gpu_type: mi325x + gpus_per_node: 8 + setup_script: stage-hf-checkpoint.sh + environment: + HF_CHECKPOINT_REPO: zai-org/GLM-5.3-Flash + HF_CHECKPOINT_REVISION: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c + # The 328 GB checkpoint needs download and metadata headroom. + HF_CHECKPOINT_BUDGET_BYTES: '400000000000' + HF_HUB_OFFLINE: '0' + TRANSFORMERS_OFFLINE: '0' + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONNOUSERSITE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + SGLANG_USE_AITER: '1' + AITER_ONLINE_TUNE: '0' + ROCM_QUICK_REDUCE_QUANTIZATION: 'NONE' + SGLANG_DSA_FUSE_TOPK: 'false' + SGLANG_OPT_USE_TOPK_V2: 'false' + SGLANG_GLM_NEXTN_MOE_PTPC: '0' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '0' + args: + served-model-name: zai-org/GLM-5.3-Flash + revision: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 4 + quantization: fp8 + context-length: 1048576 + page-size: 64 + mem-fraction-static: 0.82 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + attention-backend: dsa + dsa-prefill-backend: tilelang + dsa-decode-backend: tilelang + dsa-topk-backend: torch + linear-attn-backend: triton + linear-attn-verify-backend: triton + kv-cache-dtype: bfloat16 + moe-runner-backend: aiter + cuda-graph-backend-prefill: disabled + speculative-algorithm: EAGLE + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + speculative-attention-mode: prefill + speculative-moe-a2a-backend: none + reasoning-parser: glm45 + tool-call-parser: glm47 + watchdog-timeout: 1800 + dist-timeout: 600 + enable-metrics: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + env: + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +override_tp4_ep4_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + +override_tp4_ep4_c2: + roles: + agg: + args: + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + +override_tp4_ep4_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + +override_tp4_ep4_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + +override_tp4_ep4_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs-decode: 24 + +override_tp4_ep4_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index fbaaa07287..be18521df2 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1451,3 +1451,21 @@ kimik3-fp4-mi355x-vllm-disagg-agentic: # Inherits the same P-only scratch-reclaim setting as 1P1D c48. - TOTAL_CPU_DRAM_GB=1799 decode: { num-worker: 2, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } +glm5.3flash-fp8-mi325x-sglang-agentic-mtp: + image: lmsysorg/sglang:v0.5.21-rocm10-mi30x@sha256:4420a7e492fce623285c635b9a6a7e09ccf77a3c812006494e4414655d46662d + model: zai-org/GLM-5.3-Flash + model-prefix: glm5.3flash + runner: cluster:mi325x-amd + precision: fp8 + framework: sglang + multinode: false + srt-recipe-dir: glm5.3flash/sglang/mi325x-fp8 + scenarios: + agentic-coding: + - search-space: + - { tp: 4, ep: 4, conc-list: [1], spec-decoding: mtp, kv-offloading: none, srt-recipe: agentx/agg-variants.yaml:override_tp4_ep4_c1 } + - { tp: 4, ep: 4, conc-list: [2], spec-decoding: mtp, kv-offloading: none, srt-recipe: agentx/agg-variants.yaml:override_tp4_ep4_c2 } + - { tp: 4, ep: 4, conc-list: [4], spec-decoding: mtp, kv-offloading: none, srt-recipe: agentx/agg-variants.yaml:override_tp4_ep4_c4 } + - { tp: 4, ep: 4, conc-list: [8], spec-decoding: mtp, kv-offloading: none, srt-recipe: agentx/agg-variants.yaml:override_tp4_ep4_c8 } + - { tp: 4, ep: 4, conc-list: [12], spec-decoding: mtp, kv-offloading: none, srt-recipe: agentx/agg-variants.yaml:override_tp4_ep4_c12 } + - { tp: 4, ep: 4, conc-list: [16], spec-decoding: mtp, kv-offloading: none, srt-recipe: agentx/agg-variants.yaml:override_tp4_ep4_c16 } diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_checkpoint_staging.py b/inferencex-e2e/infx/tests/srt_slurm/test_checkpoint_staging.py new file mode 100644 index 0000000000..45b397ce83 --- /dev/null +++ b/inferencex-e2e/infx/tests/srt_slurm/test_checkpoint_staging.py @@ -0,0 +1,70 @@ +"""Exercise checkpoint staging without downloading weights or using a real cache.""" + +import os +import subprocess + +import pytest + +from infx.tests.bench.stubs import executable +from infx.tests.srt_slurm.test_setup_scripts import CONFIGS, MOUNT, ROOT + + +@pytest.mark.parametrize( + ("changes", "cached", "success"), + [ + ({}, False, True), + ({"HF_CHECKPOINT_REVISION": "main"}, False, False), + ({"HF_CHECKPOINT_BUDGET_BYTES": "0"}, False, False), + ({"HF_HUB_OFFLINE": "1"}, False, False), + ({"TRANSFORMERS_OFFLINE": "1"}, False, False), + ({"AVAILABLE": "10"}, False, False), + ({"AVAILABLE": "10"}, True, True), + ], +) +def test_staging_requires_pinned_online_download_and_sufficient_cache_space( + tmp_path, changes, cached, success +): + tools = tmp_path / "bin" + tools.mkdir() + executable(tools / "df", '#!/bin/bash\nprintf "header\\nfs 1000 0 %s\\n" "$AVAILABLE"\n') + executable(tools / "du", '#!/bin/bash\nprintf "100 cached\\n"\n') + executable(tools / "hf", '#!/bin/bash\nprintf "%s\\n" "$@" > "$DOWNLOAD_TRACE"\n') + mount = tmp_path / "mount.sh" + mount.write_text(MOUNT) + cache = tmp_path / "cache" + if cached: + (cache / "models--org--model").mkdir(parents=True) + trace = tmp_path / "download" + revision = "a" * 40 + result = subprocess.run( + ["bash", str(CONFIGS / "stage-hf-checkpoint.sh")], + env={ + "PATH": f"{tools}{os.pathsep}{os.environ['PATH']}", + "BASH_ENV": str(mount), + "STUB_WORKSPACE": str(ROOT), + "HF_HUB_CACHE": str(cache), + "HF_CHECKPOINT_REPO": "org/model", + "HF_CHECKPOINT_REVISION": revision, + "HF_CHECKPOINT_BUDGET_BYTES": "100", + "HF_HUB_OFFLINE": "0", + "TRANSFORMERS_OFFLINE": "0", + "AVAILABLE": "200", + "DOWNLOAD_TRACE": str(trace), + **changes, + }, + capture_output=True, + text=True, + check=False, + ) + assert (result.returncode == 0) is success, result.stdout + result.stderr + assert trace.exists() is success + if success: + assert trace.read_text().splitlines() == [ + "download", + "org/model", + "--revision", + revision, + "--cache-dir", + str(cache), + ] + assert (cache / ".models--org--model.download.lock").exists() From f4fb79f1de5252ef6d6e0cda408c2f27effdab65 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:36:59 +0000 Subject: [PATCH 09/17] docs(perf): record native GLM-5.3-Flash MI325X sweep --- inferencex-e2e/perf-changelog.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 88ea87392c..3280cfaeb0 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11769,3 +11769,11 @@ - "6 DEP2 prefill workers with a DEP8 decode at conc 280-400 or a DEP16 decode at conc 300; MegaMoE, DSpark block 7 with fp8 draft KV, Dynamo KV router with fastokens for prefill and least-loaded routing across decode DP ranks." - "Prefill: shortest-prefill-first, chunk 8192, mem-fraction-static 0.85, HiCache 200 GB per rank write_back." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4686 + +- config-keys: + - glm5.3flash-fp8-mi325x-sglang-agentic-mtp + scenario-type: [agentic-coding] + description: + - "Add native GLM-5.3-Flash FP8 AgentX on MI325X with immutable official SGLang v0.5.21 ROCm10 gfx942, TP4/EP4, native MTP5, full 1M context, and concurrency 1, 2, 4, 8, 12, and 16." + - "Stage checkpoint revision 46a050bec13002ad29ee795c16e61cf4fa3c4a9c in the canonical shared Hugging Face cache with locking and a 400 GB disk budget; use shared golden acceptance for throughput and canonical real-verification accuracy evaluation." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4722 From 9bc577e9a62bb5db75e268d7845149fe0d08d801 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:49:28 +0000 Subject: [PATCH 10/17] fix(bench): guard pinned checkpoint space and offline serving --- .../configs/stage-hf-checkpoint.sh | 56 +++++--- .../mi325x-fp8/agentx/agg-variants.yaml | 7 +- .../srt_slurm/test_checkpoint_staging.py | 126 ++++++++++++------ 3 files changed, 132 insertions(+), 57 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/stage-hf-checkpoint.sh b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/stage-hf-checkpoint.sh index 2da787d3d7..d489fc180a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/stage-hf-checkpoint.sh +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/stage-hf-checkpoint.sh @@ -9,24 +9,46 @@ if [[ ! "$HF_CHECKPOINT_REVISION" =~ ^[0-9a-f]{40}$ ]] || echo "Checkpoint staging requires a commit SHA and a positive byte budget" >&2 exit 1 fi -if [[ "$HF_HUB_OFFLINE" == "1" || "$TRANSFORMERS_OFFLINE" == "1" ]]; then - echo "Checkpoint staging requires online Hugging Face access" >&2 - exit 1 -fi - mkdir -p "$HF_HUB_CACHE" cache_name="models--${HF_CHECKPOINT_REPO//\//--}" exec 9>"$HF_HUB_CACHE/.${cache_name}.download.lock" flock -x 9 -available=$(df -PB1 "$HF_HUB_CACHE" | awk 'NR == 2 {print $4}') -cached=0 -if [[ -d "$HF_HUB_CACHE/$cache_name" ]]; then - cached=$(du -sB1 "$HF_HUB_CACHE/$cache_name" | awk '{print $1}') -fi -if (( available + cached < HF_CHECKPOINT_BUDGET_BYTES )); then - echo "Insufficient shared-cache disk: available=$available cached=$cached required=$HF_CHECKPOINT_BUDGET_BYTES" >&2 - exit 1 -fi -hf download "$HF_CHECKPOINT_REPO" \ - --revision "$HF_CHECKPOINT_REVISION" \ - --cache-dir "$HF_HUB_CACHE" +# Only staging goes online; the parent worker's serving environment is unchanged. +HF_HUB_OFFLINE=0 TRANSFORMERS_OFFLINE=0 python3 - <<'PY' +import os +import shutil +from pathlib import Path + +from huggingface_hub import HfApi, snapshot_download, try_to_load_from_cache + +repo = os.environ["HF_CHECKPOINT_REPO"] +revision = os.environ["HF_CHECKPOINT_REVISION"] +cache = os.environ["HF_HUB_CACHE"] +budget = int(os.environ["HF_CHECKPOINT_BUDGET_BYTES"]) +info = HfApi().model_info(repo, revision=revision, files_metadata=True) +files = info.siblings +if info.sha != revision or not files or any( + type(file.size) is not int or file.size < 0 for file in files +): + raise SystemExit("Checkpoint metadata must identify the pinned revision and every file size") + + +def cached(file): + path = try_to_load_from_cache(repo, file.rfilename, revision=revision, cache_dir=cache) + return isinstance(path, str) and Path(path).is_file() and Path(path).stat().st_size == file.size + + +total = sum(file.size for file in files) +missing = [file for file in files if not cached(file)] +remaining = sum(file.size for file in missing) +# Do not credit old revisions or incomplete blobs. A complete snapshot needs no reserve. +required = remaining + max(0, budget - total) if missing else 0 +available = shutil.disk_usage(cache).free +print(f"Checkpoint disk: available={available} remaining={remaining} required={required}", flush=True) +if available < required: + raise SystemExit("Insufficient shared-cache disk for the pinned checkpoint") +if missing: + snapshot_download(repo, revision=revision, cache_dir=cache) + if any(not cached(file) for file in files): + raise SystemExit("Pinned checkpoint is incomplete after download") +PY diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml index 4755ed3ce8..996a8dc1a8 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml @@ -16,8 +16,6 @@ base: HF_CHECKPOINT_REVISION: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c # The 328 GB checkpoint needs download and metadata headroom. HF_CHECKPOINT_BUDGET_BYTES: '400000000000' - HF_HUB_OFFLINE: '0' - TRANSFORMERS_OFFLINE: '0' frontend: type: sglang enable_multiple_frontends: false @@ -32,6 +30,8 @@ base: workers: 1 gpus: 4 env: + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' PYTHONNOUSERSITE: '1' SGLANG_TIMEOUT_KEEP_ALIVE: '900' SGLANG_USE_AITER: '1' @@ -78,9 +78,12 @@ base: interval_seconds: 10 benchmark: env: + HF_HUB_OFFLINE: '0' + TRANSFORMERS_OFFLINE: '0' AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' + TOKENIZER: /mnt/hf_hub_cache/models--zai-org--GLM-5.3-Flash/snapshots/46a050bec13002ad29ee795c16e61cf4fa3c4a9c override_tp4_ep4_c1: roles: diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_checkpoint_staging.py b/inferencex-e2e/infx/tests/srt_slurm/test_checkpoint_staging.py index 45b397ce83..7462edaee9 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_checkpoint_staging.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_checkpoint_staging.py @@ -1,55 +1,111 @@ -"""Exercise checkpoint staging without downloading weights or using a real cache.""" +"""Exercise checkpoint staging against a fake Hub and real temporary cache files.""" import os import subprocess +import sys +from pathlib import Path import pytest -from infx.tests.bench.stubs import executable from infx.tests.srt_slurm.test_setup_scripts import CONFIGS, MOUNT, ROOT +REVISION = "a" * 40 +HUB = ''' +import json, os, shutil +from pathlib import Path +from types import SimpleNamespace + +root = Path(os.environ["HF_HUB_CACHE"]) / "models--org--model" +revision = os.environ["HF_CHECKPOINT_REVISION"] +manifest = json.loads(os.environ["MANIFEST"]) +shutil.disk_usage = lambda path: SimpleNamespace(free=int(os.environ["AVAILABLE"])) + +class HfApi: + def model_info(self, repo, *, revision, files_metadata): + assert repo == "org/model" and files_metadata + assert os.environ["HF_HUB_OFFLINE"] == os.environ["TRANSFORMERS_OFFLINE"] == "0" + return SimpleNamespace(sha=os.environ.get("RETURNED_SHA", revision), + siblings=[SimpleNamespace(rfilename=n, size=s) for n, s in manifest]) + +def try_to_load_from_cache(repo, filename, *, revision, cache_dir): + assert repo == "org/model" and cache_dir == os.environ["HF_HUB_CACHE"] + return str(root / "snapshots" / revision / filename) + +def snapshot_download(repo, *, revision, cache_dir): + assert repo == "org/model" and cache_dir == os.environ["HF_HUB_CACHE"] + Path(os.environ["DOWNLOAD_TRACE"]).write_text(revision) + for filename, size in manifest: + path = root / "snapshots" / revision / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"x" * size) + if os.environ.get("INCOMPLETE_DOWNLOAD"): + path.unlink() +''' + @pytest.mark.parametrize( - ("changes", "cached", "success"), + ("changes", "cached", "old_revision", "success", "downloaded"), [ - ({}, False, True), - ({"HF_CHECKPOINT_REVISION": "main"}, False, False), - ({"HF_CHECKPOINT_BUDGET_BYTES": "0"}, False, False), - ({"HF_HUB_OFFLINE": "1"}, False, False), - ({"TRANSFORMERS_OFFLINE": "1"}, False, False), - ({"AVAILABLE": "10"}, False, False), - ({"AVAILABLE": "10"}, True, True), + ({}, {}, False, True, True), + ({"HF_CHECKPOINT_REVISION": "main"}, {}, False, False, False), + ({"HF_CHECKPOINT_BUDGET_BYTES": "0"}, {}, False, False, False), + ({"AVAILABLE": "0"}, {"config.json": 2, "weights": 8}, False, True, False), + ({"AVAILABLE": "0"}, {}, True, False, False), + ({"AVAILABLE": "0"}, {"weights": 7}, False, False, False), + ({"AVAILABLE": "3"}, {"weights": 8}, False, False, False), + ({"AVAILABLE": "4"}, {"weights": 8}, True, True, True), + ({"MANIFEST": "[]"}, {}, False, False, False), + ({"MANIFEST": '[["weights", null]]'}, {}, False, False, False), + ({"MANIFEST": '[["weights", -1]]'}, {}, False, False, False), + ({"MANIFEST": '[["weights", true]]'}, {}, False, False, False), + ({"MANIFEST": '[["weights", 1.5]]'}, {}, False, False, False), + ({"RETURNED_SHA": "b" * 40}, {}, False, False, False), + ({"INCOMPLETE_DOWNLOAD": "1"}, {}, False, False, True), + ({"AVAILABLE": "0", "BROKEN_LINK": "1"}, {}, False, False, False), ], ) -def test_staging_requires_pinned_online_download_and_sufficient_cache_space( - tmp_path, changes, cached, success -): - tools = tmp_path / "bin" - tools.mkdir() - executable(tools / "df", '#!/bin/bash\nprintf "header\\nfs 1000 0 %s\\n" "$AVAILABLE"\n') - executable(tools / "du", '#!/bin/bash\nprintf "100 cached\\n"\n') - executable(tools / "hf", '#!/bin/bash\nprintf "%s\\n" "$@" > "$DOWNLOAD_TRACE"\n') +def test_pinned_checkpoint_staging(tmp_path, changes, cached, old_revision, success, downloaded): + (tmp_path / "huggingface_hub.py").write_text(HUB) mount = tmp_path / "mount.sh" mount.write_text(MOUNT) cache = tmp_path / "cache" - if cached: - (cache / "models--org--model").mkdir(parents=True) + storage = cache / "models--org--model" + snapshot = storage / "snapshots" / REVISION + snapshot.mkdir(parents=True) + for name, size in cached.items(): + blob = storage / "blobs" / name + blob.parent.mkdir(exist_ok=True) + blob.write_bytes(b"x" * size) + (snapshot / name).symlink_to(blob) + if old_revision: + old = storage / "snapshots" / ("b" * 40) + old.mkdir() + (old / "weights").write_bytes(b"x" * 1000) + (storage / "blobs").mkdir(exist_ok=True) + (storage / "blobs" / "weights.incomplete").write_bytes(b"x" * 1000) + if changes.get("BROKEN_LINK"): + (snapshot / "weights").symlink_to(storage / "blobs" / "missing") trace = tmp_path / "download" - revision = "a" * 40 + parent_env = tmp_path / "parent-env" result = subprocess.run( - ["bash", str(CONFIGS / "stage-hf-checkpoint.sh")], + ["bash", "-c", 'bash "$STAGING_SCRIPT"; status=$?; ' + 'printf "%s %s" "$HF_HUB_OFFLINE" "$TRANSFORMERS_OFFLINE" > "$PARENT_ENV"; exit "$status"'], env={ - "PATH": f"{tools}{os.pathsep}{os.environ['PATH']}", + "PATH": f"{Path(sys.executable).parent}{os.pathsep}{os.environ['PATH']}", + "PYTHONPATH": str(tmp_path), "BASH_ENV": str(mount), "STUB_WORKSPACE": str(ROOT), + "STAGING_SCRIPT": str(CONFIGS / "stage-hf-checkpoint.sh"), "HF_HUB_CACHE": str(cache), "HF_CHECKPOINT_REPO": "org/model", - "HF_CHECKPOINT_REVISION": revision, - "HF_CHECKPOINT_BUDGET_BYTES": "100", - "HF_HUB_OFFLINE": "0", - "TRANSFORMERS_OFFLINE": "0", - "AVAILABLE": "200", + "HF_CHECKPOINT_REVISION": REVISION, + "HF_CHECKPOINT_BUDGET_BYTES": "12", + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "AVAILABLE": "20", + "MANIFEST": '[["config.json", 2], ["weights", 8]]', "DOWNLOAD_TRACE": str(trace), + "PARENT_ENV": str(parent_env), **changes, }, capture_output=True, @@ -57,14 +113,8 @@ def test_staging_requires_pinned_online_download_and_sufficient_cache_space( check=False, ) assert (result.returncode == 0) is success, result.stdout + result.stderr - assert trace.exists() is success - if success: - assert trace.read_text().splitlines() == [ - "download", - "org/model", - "--revision", - revision, - "--cache-dir", - str(cache), - ] + assert trace.exists() is downloaded + assert parent_env.read_text() == "1 1" + if downloaded: + assert trace.read_text() == REVISION assert (cache / ".models--org--model.download.lock").exists() From f9620895bbfa3cf5777e3448e212271f181ed564 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:09:55 +0000 Subject: [PATCH 11/17] fix(bench): use staged tokenizer for offline GLM processor --- .../glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml index 996a8dc1a8..0f6ef35713 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml @@ -44,6 +44,7 @@ base: args: served-model-name: zai-org/GLM-5.3-Flash revision: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c + tokenizer-path: /mnt/hf_hub_cache/models--zai-org--GLM-5.3-Flash/snapshots/46a050bec13002ad29ee795c16e61cf4fa3c4a9c trust-remote-code: true tensor-parallel-size: 4 expert-parallel-size: 4 From 817767f23ffb2716f91ac27c2f2c508c57228062 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:22:43 +0000 Subject: [PATCH 12/17] fix(bench): use upstream unfused mHC on MI325X --- .../glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml index 0f6ef35713..2d3242f942 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml @@ -39,6 +39,7 @@ base: ROCM_QUICK_REDUCE_QUANTIZATION: 'NONE' SGLANG_DSA_FUSE_TOPK: 'false' SGLANG_OPT_USE_TOPK_V2: 'false' + SGLANG_OPT_FUSE_MHC_POST_PRE: '0' SGLANG_GLM_NEXTN_MOE_PTPC: '0' SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '0' args: From 99527d1eac1298c439d2f58fbb3e35a86421b6af Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:29:48 +0000 Subject: [PATCH 13/17] fix(bench): restore canonical AgentX tokenizer lookup --- .../glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml index 2d3242f942..6d181f2da9 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml @@ -85,7 +85,6 @@ base: AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' - TOKENIZER: /mnt/hf_hub_cache/models--zai-org--GLM-5.3-Flash/snapshots/46a050bec13002ad29ee795c16e61cf4fa3c4a9c override_tp4_ep4_c1: roles: From b6c466048bda2deceb0ce81fd847053c8f9b7599 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:46:17 +0000 Subject: [PATCH 14/17] fix(bench): clear truthy AgentX client offline flags --- .../glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml index 6d181f2da9..abc6ff049a 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml @@ -80,8 +80,8 @@ base: interval_seconds: 10 benchmark: env: - HF_HUB_OFFLINE: '0' - TRANSFORMERS_OFFLINE: '0' + HF_HUB_OFFLINE: '' + TRANSFORMERS_OFFLINE: '' AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' From 5b39492d614501647218785e4c05bc677880994d Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 19:00:47 +0000 Subject: [PATCH 15/17] feat(mi325x): add paired GLM-5.3-Flash HiCache controls --- .../sglang/mi325x-fp8/agentx/hicache.yaml | 111 ++++++++++++++++++ inferencex-e2e/configs/amd-master.yaml | 52 ++++++++ 2 files changed, 163 insertions(+) create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/hicache.yaml diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/hicache.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/hicache.yaml new file mode 100644 index 0000000000..98fd1ea3cc --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/hicache.yaml @@ -0,0 +1,111 @@ +schema: 2 + +base: + identity: + model: + revision: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c + container: {} + frameworks: + sglang: v0.5.22 + resources: + gpu_type: mi325x + gpus_per_node: 8 + setup_script: stage-hf-checkpoint.sh + environment: + HF_CHECKPOINT_REPO: zai-org/GLM-5.3-Flash + HF_CHECKPOINT_REVISION: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c + HF_CHECKPOINT_BUDGET_BYTES: '400000000000' + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + SGLANG_USE_AITER: '1' + AITER_ONLINE_TUNE: '0' + ROCM_QUICK_REDUCE_QUANTIZATION: 'NONE' + SGLANG_DSA_FUSE_TOPK: 'false' + SGLANG_OPT_USE_TOPK_V2: 'false' + SGLANG_OPT_FUSE_MHC_POST_PRE: '0' + SGLANG_GLM_NEXTN_MOE_PTPC: '0' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '0' + args: + served-model-name: zai-org/GLM-5.3-Flash + revision: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c + tokenizer-path: /mnt/hf_hub_cache/models--zai-org--GLM-5.3-Flash/snapshots/46a050bec13002ad29ee795c16e61cf4fa3c4a9c + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 4 + quantization: fp8 + context-length: 1048576 + page-size: 64 + mem-fraction-static: 0.82 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + attention-backend: dsa + dsa-prefill-backend: tilelang + dsa-decode-backend: tilelang + dsa-topk-backend: torch + linear-attn-backend: triton + linear-attn-verify-backend: triton + kv-cache-dtype: bfloat16 + moe-runner-backend: aiter + cuda-graph-backend-prefill: disabled + speculative-algorithm: EAGLE + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + speculative-attention-mode: prefill + speculative-moe-a2a-backend: none + reasoning-parser: glm45 + tool-call-parser: glm47 + watchdog-timeout: 1800 + dist-timeout: 600 + enable-metrics: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + env: + HF_HUB_OFFLINE: '' + TRANSFORMERS_OFFLINE: '' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +zip_override_off: + roles: + agg: + args: + max-running-requests: [24, 32, 48, 64] + cuda-graph-max-bs-decode: [24, 32, 48, 64] + benchmark: + env: + CONC: ['12', '16', '24', '32'] + +zip_override_on: + roles: + agg: + args: + max-running-requests: [24, 32, 48, 64] + cuda-graph-max-bs-decode: [24, 32, 48, 64] + enable-hierarchical-cache: true + # Keep size and ratio unset so the combined cgroup-aware guard stays active. + hicache-host-memory-fraction: 0.60 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: ['12', '16', '24', '32'] diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index be18521df2..dbae37e2f7 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1469,3 +1469,55 @@ glm5.3flash-fp8-mi325x-sglang-agentic-mtp: - { tp: 4, ep: 4, conc-list: [8], spec-decoding: mtp, kv-offloading: none, srt-recipe: agentx/agg-variants.yaml:override_tp4_ep4_c8 } - { tp: 4, ep: 4, conc-list: [12], spec-decoding: mtp, kv-offloading: none, srt-recipe: agentx/agg-variants.yaml:override_tp4_ep4_c12 } - { tp: 4, ep: 4, conc-list: [16], spec-decoding: mtp, kv-offloading: none, srt-recipe: agentx/agg-variants.yaml:override_tp4_ep4_c16 } + +glm5.3flash-fp8-mi325x-sglang-agentic-mtp-hicache: + image: lmsysorg/sglang:v0.5.22-rocm10-mi30x@sha256:5658cd0d8de0c68de10d2e349296f55ceff98053182fa87b2a7e942e5e7f3c2a + model: zai-org/GLM-5.3-Flash + model-prefix: glm5.3flash + runner: cluster:mi325x-amd + precision: fp8 + framework: sglang + multinode: false + srt-recipe-dir: glm5.3flash/sglang/mi325x-fp8 + scenarios: + agentic-coding: + # Inventory-based GPU-share bookkeeping, not the live HiCache host fraction. + - dram-utilization: 1.0 + search-space: + - { tp: 4, ep: 4, conc-list: [12], spec-decoding: mtp, kv-offloading: none, srt-recipe: "agentx/hicache.yaml:zip_override_off[0]" } + - { tp: 4, ep: 4, conc-list: [16], spec-decoding: mtp, kv-offloading: none, srt-recipe: "agentx/hicache.yaml:zip_override_off[1]" } + - { tp: 4, ep: 4, conc-list: [24], spec-decoding: mtp, kv-offloading: none, srt-recipe: "agentx/hicache.yaml:zip_override_off[2]" } + - { tp: 4, ep: 4, conc-list: [32], spec-decoding: mtp, kv-offloading: none, srt-recipe: "agentx/hicache.yaml:zip_override_off[3]" } + - { tp: 4, ep: 4, conc-list: [12], spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, srt-recipe: "agentx/hicache.yaml:zip_override_on[0]" } + - { tp: 4, ep: 4, conc-list: [16], spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, srt-recipe: "agentx/hicache.yaml:zip_override_on[1]" } + - { tp: 4, ep: 4, conc-list: [24], spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, srt-recipe: "agentx/hicache.yaml:zip_override_on[2]" } + - { tp: 4, ep: 4, conc-list: [32], spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, srt-recipe: "agentx/hicache.yaml:zip_override_on[3]" } + +glm5.3flash-fp8-mi325x-sglang-agentic-mtp-hicache-eval-off: + image: lmsysorg/sglang:v0.5.22-rocm10-mi30x@sha256:5658cd0d8de0c68de10d2e349296f55ceff98053182fa87b2a7e942e5e7f3c2a + model: zai-org/GLM-5.3-Flash + model-prefix: glm5.3flash + runner: cluster:mi325x-amd + precision: fp8 + framework: sglang + multinode: false + srt-recipe-dir: glm5.3flash/sglang/mi325x-fp8 + scenarios: + agentic-coding: + - search-space: + - { tp: 4, ep: 4, conc-list: [16], spec-decoding: mtp, kv-offloading: none, srt-recipe: "agentx/hicache.yaml:zip_override_off[1]" } + +glm5.3flash-fp8-mi325x-sglang-agentic-mtp-hicache-eval-on: + image: lmsysorg/sglang:v0.5.22-rocm10-mi30x@sha256:5658cd0d8de0c68de10d2e349296f55ceff98053182fa87b2a7e942e5e7f3c2a + model: zai-org/GLM-5.3-Flash + model-prefix: glm5.3flash + runner: cluster:mi325x-amd + precision: fp8 + framework: sglang + multinode: false + srt-recipe-dir: glm5.3flash/sglang/mi325x-fp8 + scenarios: + agentic-coding: + - dram-utilization: 1.0 + search-space: + - { tp: 4, ep: 4, conc-list: [32], spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, srt-recipe: "agentx/hicache.yaml:zip_override_on[3]" } From 85b55fb6a24bd0e3bff9c5ce269981e6b4b6368e Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 19:01:18 +0000 Subject: [PATCH 16/17] test(mi325x): select paired HiCache sweep and accuracy controls --- inferencex-e2e/perf-changelog.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3280cfaeb0..23fe6571a6 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11777,3 +11777,22 @@ - "Add native GLM-5.3-Flash FP8 AgentX on MI325X with immutable official SGLang v0.5.21 ROCm10 gfx942, TP4/EP4, native MTP5, full 1M context, and concurrency 1, 2, 4, 8, 12, and 16." - "Stage checkpoint revision 46a050bec13002ad29ee795c16e61cf4fa3c4a9c in the canonical shared Hugging Face cache with locking and a 400 GB disk budget; use shared golden acceptance for throughput and canonical real-verification accuracy evaluation." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4722 + +- config-keys: + - glm5.3flash-fp8-mi325x-sglang-agentic-mtp-hicache + scenario-type: [agentic-coding] + description: + - "Add paired GLM-5.3-Flash FP8 MI325X cache OFF/ON throughput at concurrency 12, 16, 24, and 32 on immutable official SGLang v0.5.22 ROCm10 mi30x, preserving TP4/EP4, BF16 KV, native MTP5, full 1M context, and the canonical AgentX workload." + - "Use native HiCache automatic host-memory fraction 0.60 without an explicit size or ratio; preserve whole-node exclusive resources and distinguish nominal DRAM metadata from actual host-pool allocation." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4735 + no-evals: true + +- config-keys: + - glm5.3flash-fp8-mi325x-sglang-agentic-mtp-hicache-eval-off + - glm5.3flash-fp8-mi325x-sglang-agentic-mtp-hicache-eval-on + scenario-type: [agentic-coding] + description: + - "Run fresh canonical GLM-5.3-Flash FP8 MI325X accuracy controls on the child image with cache OFF at concurrency 16 and ON at concurrency 32, without synthetic acceptance." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4735 + evals-only: true + all-evals: true From 5499d9b676c24ca5acbf7abb14c4aeaebee6ade2 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:31:31 +0000 Subject: [PATCH 17/17] fix(ci): gate MI325X sweep on successful canary completion --- .github/workflows/run-sweep.yml | 254 +++++++++---- .../infx/tests/workflows/test_canary_gate.py | 345 ++++++++++++++++++ 2 files changed, 536 insertions(+), 63 deletions(-) create mode 100644 inferencex-e2e/infx/tests/workflows/test_canary_gate.py diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index acf9c1995c..7ff3ae1ac1 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -316,7 +316,8 @@ jobs: name: canary-select needs: setup if: >- - !cancelled() && needs.setup.result == 'success' && + !cancelled() && + needs.setup.result == 'success' && ( contains(github.event.pull_request.labels.*.name, 'full-sweep-enabled') || contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') @@ -366,8 +367,15 @@ jobs: } >> "$GITHUB_OUTPUT" canary-sweep: - needs: canary-select - if: ${{ !cancelled() && needs.canary-select.result == 'success' && needs.canary-select.outputs.canary-config != '' && needs.canary-select.outputs.canary-config != '[]' }} + needs: [setup, canary-select] + if: >- + ${{ + !cancelled() && + needs.setup.result == 'success' && + needs.canary-select.result == 'success' && + needs.canary-select.outputs.canary-config != '' && + needs.canary-select.outputs.canary-config != '[]' + }} uses: $/.github/workflows/benchmark-tmpl.yml name: canary / strategy: @@ -390,8 +398,15 @@ jobs: agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }} canary-multi-node-sweep: - needs: canary-select - if: ${{ !cancelled() && needs.canary-select.result == 'success' && needs.canary-select.outputs.multi-node-canary-config != '' && needs.canary-select.outputs.multi-node-canary-config != '[]' }} + needs: [setup, canary-select] + if: >- + ${{ + !cancelled() && + needs.setup.result == 'success' && + needs.canary-select.result == 'success' && + needs.canary-select.outputs.multi-node-canary-config != '' && + needs.canary-select.outputs.multi-node-canary-config != '[]' + }} uses: $/.github/workflows/benchmark-multinode-tmpl.yml name: multi-node canary / strategy: @@ -415,22 +430,144 @@ jobs: scenario-type: agentic-coding agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }} - sweep-multi-node-1k1k: + canary-gate: + name: canary-gate needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] + if: ${{ !cancelled() && needs.setup.result == 'success' }} + runs-on: ubuntu-latest + outputs: + search-space-config: ${{ steps.validate.outputs.search-space-config }} + steps: + - name: Validate canary completion + id: validate + shell: python + env: + SWEEP_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} + SETUP_RESULT: ${{ needs.setup.result }} + SETUP_PLAN: ${{ needs.setup.outputs.search-space-config }} + SELECTOR_RESULT: ${{ needs.canary-select.result }} + SINGLE_CANARY_RESULT: ${{ needs.canary-sweep.result }} + MULTI_CANARY_RESULT: ${{ needs.canary-multi-node-sweep.result }} + SINGLE_CANARY_CONFIG: ${{ needs.canary-select.outputs.canary-config }} + MULTI_CANARY_CONFIG: ${{ needs.canary-select.outputs.multi-node-canary-config }} + REMAINING_PLAN: ${{ needs.canary-select.outputs.remaining-search-space-config }} + run: | + import copy + import json + import os + from pathlib import Path + + def require(condition, message): + if not condition: + raise SystemExit(message) + + def rows(plan, family, group): + groups = plan.get(family) + if groups is None: + groups = {} + require(isinstance(groups, dict), f"Invalid {family} plan") + values = groups.get(group) + if values is None: + values = [] + require( + isinstance(values, list) and all(isinstance(row, dict) for row in values), + f"Invalid {family}.{group} rows", + ) + return values + + def concurrency(row, multi=False): + value = row["conc"] + if multi: + require(isinstance(value, list) and len(value) > 0, "Invalid multi-node concurrency") + value = value[0] + require(type(value) is int and value > 0, "Invalid canary concurrency") + return value + + labels = json.loads(os.environ["SWEEP_LABELS"]) + require(isinstance(labels, list), "Invalid sweep labels") + primary = set(labels) & { + "full-sweep-enabled", "full-sweep-fail-fast", "non-canary-full-sweep-enabled", + } + require(len(primary) == 1, "Exactly one primary sweep label is required") + require(os.environ["SETUP_RESULT"] == "success", "Setup did not succeed") + plan = json.loads(os.environ["SETUP_PLAN"]) + require(isinstance(plan, dict), "Invalid setup plan") + selector_result = os.environ["SELECTOR_RESULT"] + single_result = os.environ["SINGLE_CANARY_RESULT"] + multi_result = os.environ["MULTI_CANARY_RESULT"] + + if primary == {"non-canary-full-sweep-enabled"}: + require( + selector_result == single_result == multi_result == "skipped", + "Non-canary mode has unexpected canary jobs", + ) + remaining = plan + else: + require(selector_result == "success", "Required canary selection did not succeed") + single_candidates = [ + row for group in ("1k1k", "8k1k", "agentic") + for row in rows(plan, "single_node", group) + if row.get("run-eval") is not True + ] + multi_candidates = [ + row for row in rows(plan, "multi_node", "agentic") + if row.get("run-eval") is not True + ] + single = min(single_candidates, key=concurrency) if single_candidates else None + multi = ( + min(multi_candidates, key=lambda row: concurrency(row, multi=True)) + if single is None and multi_candidates else None + ) + require( + json.loads(os.environ["SINGLE_CANARY_CONFIG"]) == ([] if single is None else [single]), + "Single-node canary does not match the eligible setup plan", + ) + require( + json.loads(os.environ["MULTI_CANARY_CONFIG"]) == ([] if multi is None else [multi]), + "Multi-node canary does not match the eligible setup plan", + ) + require( + single_result == ("skipped" if single is None else "success"), + "Required single-node canary did not succeed", + ) + require( + multi_result == ("skipped" if multi is None else "success"), + "Required multi-node canary did not succeed", + ) + remaining = copy.deepcopy(plan) + for family, groups, selected in ( + ("single_node", ("1k1k", "8k1k", "agentic"), single), + ("multi_node", ("agentic",), multi), + ): + remaining[family] = remaining.get(family) or {} + for group in groups: + values = copy.deepcopy(rows(plan, family, group)) + if selected is not None and selected in values: + values.remove(selected) + remaining[family][group] = values + require( + json.loads(os.environ["REMAINING_PLAN"]) == remaining, + "Remaining matrix must remove only the selected canary", + ) + + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + output.write("search-space-config=" + json.dumps(remaining, separators=(",", ":")) + "\n") + + sweep-multi-node-1k1k: + needs: [setup, canary-gate] if: >- ${{ !cancelled() && needs.setup.result == 'success' && - (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && - (needs.canary-multi-node-sweep.result == 'success' || needs.canary-multi-node-sweep.result == 'skipped') && - toJson(fromJson(needs.setup.outputs.search-space-config).multi_node['1k1k']) != 'null' + needs.canary-gate.result == 'success' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).multi_node['1k1k']) != 'null' }} uses: $/.github/workflows/benchmark-multinode-tmpl.yml name: multi-node 1k1k / strategy: fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') }} matrix: - config: ${{ fromJson(needs.setup.outputs.search-space-config).multi_node['1k1k'] }} + config: ${{ fromJson(needs.canary-gate.outputs.search-space-config).multi_node['1k1k'] }} secrets: INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} with: &multi-node-inputs @@ -446,42 +583,40 @@ jobs: run-eval: false sweep-multi-node-8k1k: - needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] + needs: [setup, canary-gate] if: >- ${{ !cancelled() && needs.setup.result == 'success' && - (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && - (needs.canary-multi-node-sweep.result == 'success' || needs.canary-multi-node-sweep.result == 'skipped') && - toJson(fromJson(needs.setup.outputs.search-space-config).multi_node['8k1k']) != 'null' + needs.canary-gate.result == 'success' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).multi_node['8k1k']) != 'null' }} uses: $/.github/workflows/benchmark-multinode-tmpl.yml name: multi-node 8k1k / strategy: fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') }} matrix: - config: ${{ fromJson(needs.setup.outputs.search-space-config).multi_node['8k1k'] }} + config: ${{ fromJson(needs.canary-gate.outputs.search-space-config).multi_node['8k1k'] }} secrets: INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} with: *multi-node-inputs sweep-single-node-1k1k: - needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] + needs: [setup, canary-gate] if: >- ${{ !cancelled() && needs.setup.result == 'success' && - (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && - (needs.canary-multi-node-sweep.result == 'success' || needs.canary-multi-node-sweep.result == 'skipped') && - toJson(fromJson((needs.canary-sweep.result == 'success' && needs.canary-select.outputs.remaining-search-space-config) || needs.setup.outputs.search-space-config).single_node['1k1k']) != 'null' && - toJson(fromJson((needs.canary-sweep.result == 'success' && needs.canary-select.outputs.remaining-search-space-config) || needs.setup.outputs.search-space-config).single_node['1k1k']) != '[]' + needs.canary-gate.result == 'success' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).single_node['1k1k']) != 'null' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).single_node['1k1k']) != '[]' }} uses: $/.github/workflows/benchmark-tmpl.yml name: single-node 1k1k / strategy: fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') }} matrix: - config: ${{ fromJson((needs.canary-sweep.result == 'success' && needs.canary-select.outputs.remaining-search-space-config) || needs.setup.outputs.search-space-config).single_node['1k1k'] }} + config: ${{ fromJson(needs.canary-gate.outputs.search-space-config).single_node['1k1k'] }} secrets: INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} with: &single-node-inputs @@ -495,43 +630,41 @@ jobs: run-eval: ${{ matrix.config.run-eval }} sweep-single-node-8k1k: - needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] + needs: [setup, canary-gate] if: >- ${{ !cancelled() && needs.setup.result == 'success' && - (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && - (needs.canary-multi-node-sweep.result == 'success' || needs.canary-multi-node-sweep.result == 'skipped') && - toJson(fromJson((needs.canary-sweep.result == 'success' && needs.canary-select.outputs.remaining-search-space-config) || needs.setup.outputs.search-space-config).single_node['8k1k']) != 'null' && - toJson(fromJson((needs.canary-sweep.result == 'success' && needs.canary-select.outputs.remaining-search-space-config) || needs.setup.outputs.search-space-config).single_node['8k1k']) != '[]' + needs.canary-gate.result == 'success' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).single_node['8k1k']) != 'null' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).single_node['8k1k']) != '[]' }} uses: $/.github/workflows/benchmark-tmpl.yml name: single-node 8k1k / strategy: fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') }} matrix: - config: ${{ fromJson((needs.canary-sweep.result == 'success' && needs.canary-select.outputs.remaining-search-space-config) || needs.setup.outputs.search-space-config).single_node['8k1k'] }} + config: ${{ fromJson(needs.canary-gate.outputs.search-space-config).single_node['8k1k'] }} secrets: INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} with: *single-node-inputs sweep-agentic: - needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] + needs: [setup, canary-gate] if: >- ${{ !cancelled() && needs.setup.result == 'success' && - (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && - (needs.canary-multi-node-sweep.result == 'success' || needs.canary-multi-node-sweep.result == 'skipped') && - toJson(fromJson((needs.canary-sweep.result == 'success' && needs.canary-select.outputs.remaining-search-space-config) || needs.setup.outputs.search-space-config).single_node['agentic']) != 'null' && - toJson(fromJson((needs.canary-sweep.result == 'success' && needs.canary-select.outputs.remaining-search-space-config) || needs.setup.outputs.search-space-config).single_node['agentic']) != '[]' + needs.canary-gate.result == 'success' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).single_node['agentic']) != 'null' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).single_node['agentic']) != '[]' }} uses: $/.github/workflows/benchmark-tmpl.yml name: agentic / strategy: fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') }} matrix: - config: ${{ fromJson((needs.canary-sweep.result == 'success' && needs.canary-select.outputs.remaining-search-space-config) || needs.setup.outputs.search-space-config).single_node['agentic'] }} + config: ${{ fromJson(needs.canary-gate.outputs.search-space-config).single_node['agentic'] }} secrets: INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} with: @@ -548,22 +681,21 @@ jobs: agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }} sweep-multi-node-agentic: - needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] + needs: [setup, canary-gate] if: >- ${{ !cancelled() && needs.setup.result == 'success' && - (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && - (needs.canary-multi-node-sweep.result == 'success' || needs.canary-multi-node-sweep.result == 'skipped') && - toJson(fromJson((needs.canary-multi-node-sweep.result == 'success' && needs.canary-select.outputs.remaining-search-space-config) || needs.setup.outputs.search-space-config).multi_node['agentic']) != 'null' && - toJson(fromJson((needs.canary-multi-node-sweep.result == 'success' && needs.canary-select.outputs.remaining-search-space-config) || needs.setup.outputs.search-space-config).multi_node['agentic']) != '[]' + needs.canary-gate.result == 'success' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).multi_node['agentic']) != 'null' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).multi_node['agentic']) != '[]' }} uses: $/.github/workflows/benchmark-multinode-tmpl.yml name: multi-node agentic / strategy: fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') }} matrix: - config: ${{ fromJson((needs.canary-multi-node-sweep.result == 'success' && needs.canary-select.outputs.remaining-search-space-config) || needs.setup.outputs.search-space-config).multi_node['agentic'] }} + config: ${{ fromJson(needs.canary-gate.outputs.search-space-config).multi_node['agentic'] }} secrets: INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} with: @@ -582,22 +714,21 @@ jobs: agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }} sweep-evals: - needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] + needs: [setup, canary-gate] if: >- ${{ !cancelled() && needs.setup.result == 'success' && - (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && - (needs.canary-multi-node-sweep.result == 'success' || needs.canary-multi-node-sweep.result == 'skipped') && - toJson(fromJson(needs.setup.outputs.search-space-config).evals) != '[]' && - toJson(fromJson(needs.setup.outputs.search-space-config).evals) != 'null' + needs.canary-gate.result == 'success' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).evals) != '[]' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).evals) != 'null' }} uses: $/.github/workflows/benchmark-tmpl.yml name: eval / strategy: fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') }} matrix: - config: ${{ fromJson(needs.setup.outputs.search-space-config).evals }} + config: ${{ fromJson(needs.canary-gate.outputs.search-space-config).evals }} secrets: INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} with: @@ -614,22 +745,21 @@ jobs: eval-only: true sweep-agentic-evals: - needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] + needs: [setup, canary-gate] if: >- ${{ !cancelled() && needs.setup.result == 'success' && - (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && - (needs.canary-multi-node-sweep.result == 'success' || needs.canary-multi-node-sweep.result == 'skipped') && - toJson(fromJson(needs.setup.outputs.search-space-config).agentic_evals) != '[]' && - toJson(fromJson(needs.setup.outputs.search-space-config).agentic_evals) != 'null' + needs.canary-gate.result == 'success' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).agentic_evals) != '[]' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).agentic_evals) != 'null' }} uses: $/.github/workflows/benchmark-tmpl.yml name: agentic eval / strategy: fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') }} matrix: - config: ${{ fromJson(needs.setup.outputs.search-space-config).agentic_evals }} + config: ${{ fromJson(needs.canary-gate.outputs.search-space-config).agentic_evals }} secrets: INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} with: @@ -648,22 +778,21 @@ jobs: scenario-type: agentic-coding sweep-multi-node-evals: - needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] + needs: [setup, canary-gate] if: >- ${{ !cancelled() && needs.setup.result == 'success' && - (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && - (needs.canary-multi-node-sweep.result == 'success' || needs.canary-multi-node-sweep.result == 'skipped') && - toJson(fromJson(needs.setup.outputs.search-space-config).multinode_evals) != '[]' && - toJson(fromJson(needs.setup.outputs.search-space-config).multinode_evals) != 'null' + needs.canary-gate.result == 'success' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).multinode_evals) != '[]' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).multinode_evals) != 'null' }} uses: $/.github/workflows/benchmark-multinode-tmpl.yml name: multi-node eval / strategy: fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') }} matrix: - config: ${{ fromJson(needs.setup.outputs.search-space-config).multinode_evals }} + config: ${{ fromJson(needs.canary-gate.outputs.search-space-config).multinode_evals }} secrets: INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} with: @@ -682,22 +811,21 @@ jobs: eval-conc: ${{ (matrix.config['eval-framework'] || 'lm-eval') == 'lm-eval' && matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} sweep-multi-node-agentic-evals: - needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] + needs: [setup, canary-gate] if: >- ${{ !cancelled() && needs.setup.result == 'success' && - (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && - (needs.canary-multi-node-sweep.result == 'success' || needs.canary-multi-node-sweep.result == 'skipped') && - toJson(fromJson(needs.setup.outputs.search-space-config).multinode_agentic_evals) != '[]' && - toJson(fromJson(needs.setup.outputs.search-space-config).multinode_agentic_evals) != 'null' + needs.canary-gate.result == 'success' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).multinode_agentic_evals) != '[]' && + toJson(fromJson(needs.canary-gate.outputs.search-space-config).multinode_agentic_evals) != 'null' }} uses: $/.github/workflows/benchmark-multinode-tmpl.yml name: multi-node agentic eval / strategy: fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') }} matrix: - config: ${{ fromJson(needs.setup.outputs.search-space-config).multinode_agentic_evals }} + config: ${{ fromJson(needs.canary-gate.outputs.search-space-config).multinode_agentic_evals }} secrets: INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} with: diff --git a/inferencex-e2e/infx/tests/workflows/test_canary_gate.py b/inferencex-e2e/infx/tests/workflows/test_canary_gate.py new file mode 100644 index 0000000000..06a071bdaa --- /dev/null +++ b/inferencex-e2e/infx/tests/workflows/test_canary_gate.py @@ -0,0 +1,345 @@ +import copy +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[4] + + +@pytest.fixture +def jobs(): + return yaml.safe_load((ROOT / ".github/workflows/run-sweep.yml").read_text())["jobs"] + + +def read_outputs(path): + return dict(line.split("=", 1) for line in path.read_text().splitlines()) + + +def select(jobs, tmp_path, plan): + step = jobs["canary-select"]["steps"][0] + output = tmp_path / "selection" + subprocess.run( + ["bash", "-e", "-o", "pipefail", "-c", step["run"]], + env={**os.environ, "SEARCH_SPACE": json.dumps(plan), "GITHUB_OUTPUT": str(output)}, + check=True, + capture_output=True, + text=True, + ) + return read_outputs(output) + + +def context(plan, selection): + return { + "setup": {"result": "success", "outputs": {"search-space-config": json.dumps(plan)}}, + "canary-select": {"result": "success", "outputs": selection}, + "canary-sweep": { + "result": "success" if json.loads(selection["canary-config"]) else "skipped", + }, + "canary-multi-node-sweep": { + "result": "success" if json.loads(selection["multi-node-canary-config"]) else "skipped", + }, + } + + +def run_gate(jobs, tmp_path, needs, labels=None): + step = jobs["canary-gate"]["steps"][0] + output = tmp_path / "gate-output" + env = {} + for name, expression in step["env"].items(): + if name == "SWEEP_LABELS": + env[name] = json.dumps(["full-sweep-fail-fast"] if labels is None else labels) + continue + value = {"needs": needs} + for key in expression.removeprefix("${{ ").removesuffix(" }}").split("."): + value = value.get(key, {}) if isinstance(value, dict) else "" + env[name] = value if isinstance(value, str) else "" + result = subprocess.run( + [sys.executable, "-c", step["run"]], + env={ + **os.environ, + **env, + "GITHUB_OUTPUT": str(output), + }, + check=False, + capture_output=True, + text=True, + ) + if result.returncode: + assert not output.exists() + return result, None + return result, json.loads(read_outputs(output)["search-space-config"]) + + +def point(conc, mode="none", **extra): + return {"conc": conc, "kv-offloading": mode, "model": "glm5.3flash", **extra} + + +@pytest.fixture +def agentic_plan(): + return { + "single_node": { + "agentic": [ + point(conc, mode) for mode in ["none", "dram"] for conc in [24, 32, 48, 64] + ], + }, + "multi_node": {}, + "agentic_evals": [point(64, "dram", **{"run-eval": True, "eval-only": True})], + } + + +@pytest.mark.parametrize("label", ["full-sweep-fail-fast", "full-sweep-enabled"]) +def test_actual_selector_and_gate_release_only_remaining_points( + jobs, + tmp_path, + agentic_plan, + label, +): + original = copy.deepcopy(agentic_plan) + selected = select(jobs, tmp_path, agentic_plan) + assert json.loads(selected["canary-config"]) == [point(24)] + assert json.loads(selected["multi-node-canary-config"]) == [] + needs = context(agentic_plan, selected) + needs["reuse-sweep-gate"] = {"result": "skipped"} + result, remaining = run_gate(jobs, tmp_path, needs, [label]) + assert result.returncode == 0, result.stderr + assert remaining["single_node"]["agentic"] == original["single_node"]["agentic"][1:] + assert remaining["agentic_evals"] == original["agentic_evals"] + assert agentic_plan == original + + +@pytest.mark.parametrize("status", ["skipped", "failure", "cancelled"]) +def test_required_selector_must_succeed(jobs, tmp_path, agentic_plan, status): + needs = context(agentic_plan, select(jobs, tmp_path, agentic_plan)) + needs["canary-select"]["result"] = status + result, _ = run_gate(jobs, tmp_path, needs) + assert result.returncode != 0 + assert "Required canary selection did not succeed" in result.stderr + + +@pytest.mark.parametrize("status", ["skipped", "failure", "cancelled"]) +@pytest.mark.parametrize("multi", [False, True]) +def test_selected_canary_must_succeed(jobs, tmp_path, status, multi): + plan = ( + {"multi_node": {"agentic": [point([8]), point([16])]}} + if multi + else {"single_node": {"agentic": [point(8), point(16)]}} + ) + needs = context(plan, select(jobs, tmp_path, plan)) + needs["canary-multi-node-sweep" if multi else "canary-sweep"]["result"] = status + result, _ = run_gate(jobs, tmp_path, needs) + assert result.returncode != 0 + assert "canary did not succeed" in result.stderr + + +@pytest.mark.parametrize( + "output", + [ + "canary-config", + "multi-node-canary-config", + "remaining-search-space-config", + ], +) +@pytest.mark.parametrize("bad", [None, "", "{invalid", "null", "{}"]) +def test_missing_or_malformed_selector_outputs_fail_closed( + jobs, + tmp_path, + agentic_plan, + output, + bad, +): + needs = context(agentic_plan, select(jobs, tmp_path, agentic_plan)) + if bad is None: + del needs["canary-select"]["outputs"][output] + else: + needs["canary-select"]["outputs"][output] = bad + result, _ = run_gate(jobs, tmp_path, needs) + assert result.returncode != 0 + + +def test_forged_empty_selection_cannot_bypass_eligible_work(jobs, tmp_path, agentic_plan): + needs = context(agentic_plan, select(jobs, tmp_path, agentic_plan)) + needs["canary-select"]["outputs"]["canary-config"] = "[]" + needs["canary-sweep"]["result"] = "skipped" + result, _ = run_gate(jobs, tmp_path, needs) + assert result.returncode != 0 + assert "does not match the eligible setup plan" in result.stderr + + +@pytest.mark.parametrize("replacement", [[point(32)], [point(24), point(32)], [point(1)]]) +def test_selector_must_choose_exactly_the_expected_minimum( + jobs, + tmp_path, + agentic_plan, + replacement, +): + needs = context(agentic_plan, select(jobs, tmp_path, agentic_plan)) + needs["canary-select"]["outputs"]["canary-config"] = json.dumps(replacement) + result, _ = run_gate(jobs, tmp_path, needs) + assert result.returncode != 0 + + +@pytest.mark.parametrize("mutation", ["retain-canary", "drop-another-point", "drop-eval"]) +def test_remaining_matrix_cannot_duplicate_or_discard_work( + jobs, + tmp_path, + agentic_plan, + mutation, +): + selected = select(jobs, tmp_path, agentic_plan) + remaining = json.loads(selected["remaining-search-space-config"]) + if mutation == "retain-canary": + remaining["single_node"]["agentic"].insert(0, point(24)) + elif mutation == "drop-another-point": + remaining["single_node"]["agentic"].pop() + else: + remaining["agentic_evals"] = [] + selected["remaining-search-space-config"] = json.dumps(remaining) + result, _ = run_gate(jobs, tmp_path, context(agentic_plan, selected)) + assert result.returncode != 0 + assert "remove only the selected canary" in result.stderr + + +@pytest.mark.parametrize( + "plan", + [ + {"single_node": {}, "multi_node": {}, "agentic_evals": [point(64, "dram")]}, + {"single_node": {"agentic": [point(8, **{"run-eval": True})]}}, + {}, + ], +) +def test_no_eligible_performance_requires_successful_explicit_empty_selection(jobs, tmp_path, plan): + selected = select(jobs, tmp_path, plan) + assert json.loads(selected["canary-config"]) == [] + assert json.loads(selected["multi-node-canary-config"]) == [] + result, remaining = run_gate(jobs, tmp_path, context(plan, selected)) + assert result.returncode == 0, result.stderr + assert remaining == json.loads(selected["remaining-search-space-config"]) + + +def test_single_node_has_precedence_over_lower_concurrency_multi_node(jobs, tmp_path): + plan = { + "single_node": {"agentic": [point(64), point(32)]}, + "multi_node": {"agentic": [point([1]), point([2])]}, + } + selected = select(jobs, tmp_path, plan) + assert json.loads(selected["canary-config"]) == [point(32)] + assert json.loads(selected["multi-node-canary-config"]) == [] + result, remaining = run_gate(jobs, tmp_path, context(plan, selected)) + assert result.returncode == 0, result.stderr + assert remaining["single_node"]["agentic"] == [point(64)] + assert remaining["multi_node"]["agentic"] == plan["multi_node"]["agentic"] + + +def test_multi_node_agentic_is_selected_when_single_candidates_are_absent(jobs, tmp_path): + plan = {"multi_node": {"agentic": [point([16]), point([4])]}} + selected = select(jobs, tmp_path, plan) + assert json.loads(selected["multi-node-canary-config"]) == [point([4])] + result, remaining = run_gate(jobs, tmp_path, context(plan, selected)) + assert result.returncode == 0, result.stderr + assert remaining["multi_node"]["agentic"] == [point([16])] + + +def test_multi_node_fixed_sequence_remains_outside_existing_canary_pool(jobs, tmp_path): + plan = {"multi_node": {"1k1k": [point([8])], "8k1k": [point([16])]}} + selected = select(jobs, tmp_path, plan) + assert json.loads(selected["multi-node-canary-config"]) == [] + result, remaining = run_gate(jobs, tmp_path, context(plan, selected)) + assert result.returncode == 0, result.stderr + assert remaining["multi_node"]["1k1k"] == plan["multi_node"]["1k1k"] + assert remaining["multi_node"]["8k1k"] == plan["multi_node"]["8k1k"] + + +def test_duplicate_identical_points_remove_one_occurrence(jobs, tmp_path): + plan = {"single_node": {"agentic": [point(8), point(8), point(16)]}} + selected = select(jobs, tmp_path, plan) + result, remaining = run_gate(jobs, tmp_path, context(plan, selected)) + assert result.returncode == 0, result.stderr + assert remaining["single_node"]["agentic"] == [point(8), point(16)] + + +def test_large_remaining_plan_passes_as_separate_raw_environment_values(jobs, tmp_path): + retained = point(16) + metadata = {"audit_padding": "x" * 80_000} + plan = { + "single_node": {"agentic": [point(8), retained]}, + "changelog_metadata": metadata, + } + selected = select(jobs, tmp_path, plan) + result, remaining = run_gate(jobs, tmp_path, context(plan, selected)) + assert result.returncode == 0, result.stderr + assert remaining["single_node"]["agentic"] == [retained] + assert remaining["changelog_metadata"] == metadata + + +def test_identical_points_in_multiple_groups_preserve_selector_removal_behavior(jobs, tmp_path): + plan = { + "single_node": { + "1k1k": [point(8), point(8)], + "8k1k": [point(8), point(16)], + "agentic": [point(32)], + }, + } + selected = select(jobs, tmp_path, plan) + result, remaining = run_gate(jobs, tmp_path, context(plan, selected)) + assert result.returncode == 0, result.stderr + assert remaining["single_node"] == { + "1k1k": [point(8)], + "8k1k": [point(16)], + "agentic": [point(32)], + } + + +@pytest.mark.parametrize("single,multi", [(None, None), ({}, None), (None, {})]) +def test_null_families_keep_existing_empty_selection_behavior(jobs, tmp_path, single, multi): + plan = {"single_node": single, "multi_node": multi} + selected = select(jobs, tmp_path, plan) + result, remaining = run_gate(jobs, tmp_path, context(plan, selected)) + assert result.returncode == 0, result.stderr + assert remaining == { + "single_node": {"1k1k": [], "8k1k": [], "agentic": []}, + "multi_node": {"agentic": []}, + } + + +def test_intentional_non_canary_mode_preserves_complete_plan(jobs, tmp_path, agentic_plan): + needs = { + "setup": { + "result": "success", + "outputs": {"search-space-config": json.dumps(agentic_plan)}, + }, + "canary-select": {"result": "skipped", "outputs": {}}, + "canary-sweep": {"result": "skipped"}, + "canary-multi-node-sweep": {"result": "skipped"}, + } + result, remaining = run_gate(jobs, tmp_path, needs, ["non-canary-full-sweep-enabled"]) + assert result.returncode == 0, result.stderr + assert remaining == agentic_plan + + +@pytest.mark.parametrize( + "labels", + [ + [], + ["agentx-fast"], + ["full-sweep-enabled", "non-canary-full-sweep-enabled"], + ], +) +def test_removed_or_conflicting_primary_labels_fail_closed(jobs, tmp_path, agentic_plan, labels): + needs = context(agentic_plan, select(jobs, tmp_path, agentic_plan)) + result, _ = run_gate(jobs, tmp_path, needs, labels) + assert result.returncode != 0 + assert "Exactly one primary sweep label" in result.stderr + + +@pytest.mark.parametrize("status", ["skipped", "failure", "cancelled"]) +def test_setup_must_succeed(jobs, tmp_path, agentic_plan, status): + needs = context(agentic_plan, select(jobs, tmp_path, agentic_plan)) + needs["setup"]["result"] = status + result, _ = run_gate(jobs, tmp_path, needs) + assert result.returncode != 0 + assert "Setup did not succeed" in result.stderr