nightly-tau2.yml

# Nightly τ²-bench A/B — multi-turn agentic benchmark comparing SMG's parsing
# frontend (SMG -> vLLM gRPC) against pure vLLM, same model/engine/checkpoint/
# sampling. Mirrors nightly-bfcl.yml's model matrix. Informational/non-blocking —
# never a merge gate.
#
# User-sim is the fixed gpt-5.2 model via the OpenAI API; agent arms expose an
# identical OpenAI /v1 endpoint so the τ²-bench harness can point at either and
# the ONLY variable is the frontend (tokenization + tool/reasoning parsing).
#
# Runs nightly across retail, airline, and telecom domains over a 6-model matrix
# (qwen3.6 + gpt-oss on H100; deepseek-v4, minimax-m2.7, kimi-k2.6, glm-5.2 on
# Blackwell). The 4 Blackwell legs are num_tasks-capped to bound GPU time and
# gpt-5.2 spend (tau2 is far heavier per-leg than bfcl: multi-turn + per-turn API
# spend). On PRs that touch this pipeline ALL legs run on a tiny retail / 1-trial /
# few-task subset — a quick "does each leg launch + parse + score" smoke. The heavy
# Blackwell legs are dominated by model-load time and serialized by a host lock, so
# a PR run is not fast, but it confirms every leg works end-to-end before merge.
name: Nightly tau2

on:
  schedule:
    - cron: "17 7 * * 2,4,6" # Tue/Thu/Sat 07:17 UTC = 00:17 PDT — alternates nights with nightly-bfcl so the two ~7h runs don't queue back-to-back on the shared Blackwell runner
  # PR sanity (disabled, like nightly-bfcl): when the tau2 pipeline itself changes
  # this would run ALL legs on a tiny retail / 1-trial / few-task subset — but the
  # 6-leg run (incl. the ~7h Blackwell legs) is too heavy for every pipeline PR.
  # Re-enable temporarily to smoke a pipeline change, or use workflow_dispatch.
  # pull_request:
  #   paths:
  #     - "scripts/tau2/**"
  #     - ".github/workflows/nightly-tau2.yml"
  workflow_dispatch:
    inputs:
      only:
        description: "Run only this matrix leg (qwen3.6|gpt-oss|deepseek-v4|minimax-m2.7|kimi-k2.6|glm-5.2); empty = all"
        required: false
        default: ""
      model:
        description: "Override HF model id for the selected leg; empty = matrix default"
        required: false
        default: ""
      domains:
        description: "Comma-separated tau2 domains"
        required: false
        default: "retail,airline,telecom"
      num_trials:
        description: "Trials per task (pass^k k value)"
        required: false
        default: "2"
      num_tasks:
        description: "Max tasks per domain (0 = all); empty = matrix per-leg default"
        required: false
        default: ""
      max_concurrency:
        description: "Concurrent tau2 simulations per arm"
        required: false
        default: "16"
      request_timeout:
        description: "Per-LiteLLM-request timeout (s) for the agent; 0 = off"
        required: false
        default: "300"
      run_timeout:
        description: "Per-domain `tau2 run` wall-clock cap (s); 0 = off"
        required: false
        default: "5400"
      vllm_tool_parser:
        description: "Override pure-vLLM --tool-call-parser; empty = matrix default"
        required: false
        default: ""
      smg_tool_parser:
        description: "Override SMG --tool-call-parser; empty = matrix default (empty also = SMG auto-detect)"
        required: false
        default: ""
      reasoning_parser:
        description: "Override reasoning parser for BOTH arms; empty = matrix defaults"
        required: false
        default: ""

concurrency:
  group: nightly-tau2-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
  contents: read

env:
  RUSTC_WRAPPER: sccache
  SCCACHE_GHA_ENABLED: "true"

jobs:
  # Build the smg wheel once (CPU-only Rust/maturin compile — no GPU needed).
  build-wheel:
    runs-on: k8s-runner-cpu
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v7
      - name: Cache wheel artifacts
        id: cache-wheel
        uses: actions/cache@v6
        with:
          path: |
            bindings/python/dist/*.whl
          key: nightly-wheel-${{ runner.os }}-${{ hashFiles('Cargo.lock', '**/Cargo.toml', 'crates/**/src/**', 'model_gateway/src/**', 'bindings/python/src/**') }}
          restore-keys: |
            nightly-wheel-${{ runner.os }}-
      - name: Setup Rust
        if: steps.cache-wheel.outputs.cache-hit != 'true'
        uses: ./.github/actions/setup-rust
      - name: Setup Python venv
        if: steps.cache-wheel.outputs.cache-hit != 'true'
        run: bash scripts/ci_setup_python_venv.sh
      - name: Build Python wheel
        if: steps.cache-wheel.outputs.cache-hit != 'true'
        run: bash scripts/ci_build_wheel.sh
      - name: Upload wheel artifact
        uses: actions/upload-artifact@v7
        with:
          name: smg-wheel
          path: bindings/python/dist/*.whl
          retention-days: 7

  # Compute the model matrix as JSON so the dispatch `only` filter (and the
  # PR-sanity H100-only subset) can depend on inputs — `matrix` is not available
  # in a job-level `if`.
  setup:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      matrix: ${{ steps.build.outputs.matrix }}
    steps:
      - id: build
        env:
          ONLY: ${{ github.event.inputs.only }}
        run: |
          set -euo pipefail
          python3 - <<'PY' >> "$GITHUB_OUTPUT"
          import json, os
          # max_model_len "auto": largest window that fits GPU memory (vLLM 0.24+),
          # capped at native max — multi-turn tau2 prompts can exceed a fixed 16k.
          # num_tasks: 0 = all (cheap H100 legs); 30/domain caps the heavy Blackwell
          # legs to bound GPU time + gpt-5.2 spend.
          legs = [
              # H100 legs (TP=2 per arm, both arms concurrent: GPUs 0,1 + 2,3).
              {"name": "qwen3.6", "runner": "4-gpu-h100", "tp": 2,
               "gpu_a": "0,1", "gpu_b": "2,3",
               "model": "Qwen/Qwen3.6-27B",
               "vllm_tool": "qwen3_xml", "vllm_reason": "qwen3",
               "smg_tool": "qwen_xml", "smg_reason": "qwen3",
               "vllm_extra": "", "gpu_mem": "0.85", "startup_timeout": "600",
               "model_cache": "/models", "arm_mode": "concurrent",
               "max_model_len": "auto", "num_tasks": 30},  # cap like Blackwell legs — dense 27B is slow; bounds runtime + skips the airline task-44 straggler
              # gpt-oss SMG arm passes NO tool parser (smg_tool empty) -> harmony auto-detect.
              {"name": "gpt-oss", "runner": "4-gpu-h100", "tp": 2,
               "gpu_a": "0,1", "gpu_b": "2,3",
               "model": "openai/gpt-oss-120b",
               "vllm_tool": "openai", "vllm_reason": "",
               "smg_tool": "", "smg_reason": "",
               "vllm_extra": "", "gpu_mem": "0.85", "startup_timeout": "600",
               "model_cache": "/models", "arm_mode": "concurrent",
               "max_model_len": "auto", "num_tasks": 0},
              # Blackwell legs (B200). Models pre-staged on NVMe at /raid/models/<id>.
              # Concurrent TP=4 per arm (0-3 + 4-7); num_tasks-capped.
              {"name": "deepseek-v4", "runner": "blackwell", "tp": 4,
               "gpu_a": "0,1,2,3", "gpu_b": "4,5,6,7",
               "model": "deepseek-ai/DeepSeek-V4-Flash",
               "vllm_tool": "deepseek_v4", "vllm_reason": "deepseek_v4",
               "smg_tool": "deepseek_v4", "smg_reason": "deepseek_v31",
               "vllm_extra": "--tokenizer-mode deepseek_v4 --trust-remote-code --kv-cache-dtype fp8 --block-size 256",
               "gpu_mem": "0.90", "startup_timeout": "2400", "model_cache": "/raid/models",
               "arm_mode": "concurrent", "max_model_len": "auto", "num_tasks": 30},
              {"name": "minimax-m2.7", "runner": "blackwell", "tp": 4,
               "gpu_a": "0,1,2,3", "gpu_b": "4,5,6,7",
               "model": "MiniMaxAI/MiniMax-M2.7",
               "vllm_tool": "minimax_m2", "vllm_reason": "minimax_m2",
               "smg_tool": "minimax_m2", "smg_reason": "minimax",
               "vllm_extra": "--trust-remote-code", "gpu_mem": "0.90", "startup_timeout": "2400",
               "model_cache": "/raid/models", "arm_mode": "concurrent",
               "max_model_len": "auto", "num_tasks": 30},
              {"name": "kimi-k2.6", "runner": "blackwell", "tp": 4,
               "gpu_a": "0,1,2,3", "gpu_b": "4,5,6,7",
               "model": "moonshotai/Kimi-K2.6",
               "vllm_tool": "kimi_k2", "vllm_reason": "kimi_k2",
               "smg_tool": "kimik2", "smg_reason": "kimi_thinking",  # K2.6 template prefills <think> (always-in-reasoning); kimi_k25 is always_in_reasoning=false -> mis-splits reasoning
               "vllm_extra": "--trust-remote-code", "gpu_mem": "0.90", "startup_timeout": "2400",
               "model_cache": "/raid/models", "arm_mode": "concurrent",
               "max_model_len": "auto", "num_tasks": 30},
              # GLM-5.2-FP8 (~744GB) needs the whole 8-GPU node, so it runs SEQUENTIAL
              # (arm A then arm B; gpu_b unused) unlike the concurrent half-node legs.
              {"name": "glm-5.2", "runner": "blackwell", "tp": 8,
               "gpu_a": "0,1,2,3,4,5,6,7", "gpu_b": "",
               "model": "zai-org/GLM-5.2-FP8",
               "vllm_tool": "glm47", "vllm_reason": "glm45",
               "smg_tool": "glm47_moe", "smg_reason": "glm45",
               "vllm_extra": "--trust-remote-code --kv-cache-dtype fp8_e4m3",
               "gpu_mem": "0.90", "startup_timeout": "3000", "model_cache": "/raid/models",
               "arm_mode": "sequential", "max_model_len": "auto", "num_tasks": 30},
          ]
          only = os.environ.get("ONLY", "")
          valid = [l["name"] for l in legs]
          if only:
              legs = [l for l in legs if l["name"] == only]
              if not legs:
                  raise SystemExit(f"unknown leg {only!r}; valid: {valid}")
          # All legs (incl. Blackwell) run on PRs too — the PR env below forces a
          # tiny retail / 1-trial / few-task subset, so each leg is just a quick
          # "does it launch + parse + score" smoke (model load dominates its time).
          print("matrix=" + json.dumps({"include": legs}))
          PY

  tau2-ab:
    name: tau2-ab ${{ matrix.model }} (${{ matrix.runner }} TP${{ matrix.tp }})
    needs: [build-wheel, setup]
    if: ${{ !cancelled() && needs.build-wheel.result == 'success' && needs.setup.result == 'success' }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
    runs-on: ${{ matrix.runner }}
    # Generous ceiling: multi-turn state simulation is slow, and the sequential
    # glm-5.2 leg scores two whole-node arms serially. Ceiling only; fast legs
    # finish early. --max-concurrency keeps the full run within budget.
    timeout-minutes: 360
    permissions:
      contents: read
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      # Per-runner model store: /models on h100, /raid/models (NVMe) on blackwell.
      ROUTER_LOCAL_MODEL_PATH: ${{ matrix.model_cache }}
      HF_HOME: ${{ matrix.model_cache }}
      HF_TOKEN: ${{ secrets.HF_TOKEN }}
      MODEL: ${{ github.event.inputs.model || matrix.model }}
      # PR = quick sanity subset; schedule/dispatch = full domain set.
      DOMAINS: ${{ github.event.inputs.domains || (github.event_name == 'pull_request' && 'retail' || 'retail,airline,telecom') }}
      # PR = single trial (fast); schedule/dispatch = 2 trials (pass^2).
      NUM_TRIALS: ${{ github.event.inputs.num_trials || (github.event_name == 'pull_request' && '1' || '2') }}
      # PR = 3 tasks (smoke); schedule/dispatch = the leg's matrix cap (0=all on H100,
      # 30 on the heavy Blackwell legs). Dispatch `num_tasks` overrides all legs.
      NUM_TASKS: ${{ github.event.inputs.num_tasks || (github.event_name == 'pull_request' && '3' || matrix.num_tasks) }}
      MAX_CONCURRENCY: ${{ github.event.inputs.max_concurrency || '16' }}
      # Fail-fast bounds (see COMMON below). Per-request cap 300s catches a single
      # degenerate generation; per-domain cap 5400s (90m) bounds a wedged `tau2 run`.
      REQUEST_TIMEOUT: ${{ github.event.inputs.request_timeout || '300' }}
      RUN_TIMEOUT: ${{ github.event.inputs.run_timeout || '5400' }}
    steps:
      - uses: actions/checkout@v7
      - uses: actions/download-artifact@v8
        with:
          name: smg-wheel
          path: wheel
      - name: Setup vLLM env (provides vllm + grpc servicer + smg)
        uses: ./.github/actions/setup-vllm
      - name: Install smg wheel + ninja (for vLLM torch.compile)
        run: |
          pip uninstall -y smg || true
          pip install wheel/*.whl
          pip install ninja
      - name: Install tau2-bench
        run: |
          # uv for fast venv management, then clone + sync tau2-bench (its own .venv).
          curl -LsSf https://astral.sh/uv/install.sh | sh
          export PATH="$HOME/.local/bin:$PATH"
          git clone https://github.com/sierra-research/tau2-bench
          (cd tau2-bench && uv sync)
      - name: Stage model weights
        # Skip if pre-staged locally at <cache>/<id> (e.g. blackwell NVMe); else
        # download just this leg's model (not the whole GPU tier).
        run: |
          if [ -d "$ROUTER_LOCAL_MODEL_PATH/$MODEL" ]; then
            echo "model present at $ROUTER_LOCAL_MODEL_PATH/$MODEL — skipping download"
          else
            bash scripts/ci_download_model.sh "$MODEL"
          fi
      - name: Launch arms + run τ²-bench A/B
        env:
          VLLM_BIN: vllm
          VLLM_PYTHON: python
          SMG_LAUNCH: "python -m smg.launch_router"
          TAU2_MODEL: ${{ env.MODEL }}
          TAU2_TP: ${{ matrix.tp }}
          TAU2_MAX_MODEL_LEN: ${{ matrix.max_model_len }}
          TAU2_GPU_MEM_UTIL: ${{ matrix.gpu_mem }}
          TAU2_VLLM_TOOL_PARSER: ${{ github.event.inputs.vllm_tool_parser || matrix.vllm_tool }}
          TAU2_SMG_TOOL_PARSER: ${{ github.event.inputs.smg_tool_parser || matrix.smg_tool }}
          TAU2_VLLM_REASONING_PARSER: ${{ github.event.inputs.reasoning_parser || matrix.vllm_reason }}
          TAU2_SMG_REASONING_PARSER: ${{ github.event.inputs.reasoning_parser || matrix.smg_reason }}
          TAU2_VLLM_EXTRA: ${{ matrix.vllm_extra }}
          TAU2_STARTUP_TIMEOUT: ${{ matrix.startup_timeout }}
          TAU2_RUN_DIR: ${{ runner.temp }}/tau2_run
          ARM_MODE: ${{ matrix.arm_mode }}
          GPU_A: ${{ matrix.gpu_a }}
          GPU_B: ${{ matrix.gpu_b }}
        run: |
          set -euo pipefail
          export PATH="$HOME/.local/bin:$PATH"
          # Auto-register the served model at $0 cost so litellm's per-response cost
          # lookup stops logging thousands of "This model isn't mapped yet" ERRORs
          # (self-hosted models aren't in litellm's price map). sitecustomize runs at
          # interpreter startup, inheriting into the tau2 subprocess's own venv.
          export PYTHONPATH="$PWD/scripts/tau2/costmap_shim${PYTHONPATH:+:$PYTHONPATH}"
          # Serialize the launch->serve->teardown phase PER HOST. Blackwell runner
          # agents share one bare-metal machine (host network + the same 8 GPUs), so
          # two legs at once collide on ports and GPU memory. A host-wide flock runs
          # them one at a time; the lock releases when this step's shell exits.
          #
          # CRITICAL: every launch_arms.sh call is invoked with `9>&-` so the
          # setsid-detached vLLM/SMG servers do NOT inherit lock FD 9 (a leaked
          # server would hold the lock forever and deadlock future jobs). With 9>&-,
          # only this step shell holds the lock; it always releases on step exit.
          exec 9>/tmp/tau2-host.lock
          echo "waiting for host lock..."
          flock -w 9000 9 || { echo "::error::host lock not acquired within 9000s"; exit 1; }
          echo "acquired host lock"
          # Pre-flight GPU nuke: clear leftovers from a prior cancelled leg whose
          # teardown was skipped. Safe: we hold the host lock, so no other tau2 leg runs.
          bash scripts/ci_killall_sglang.sh nuke_gpus 9>&- || true
          cleanup() {
            bash scripts/tau2/launch_arms.sh stop 9>&- || true
            bash scripts/ci_killall_sglang.sh nuke_gpus 9>&- || true
          }
          trap cleanup EXIT
          trap 'cleanup; exit 143' INT TERM

          TAU2_BIN="tau2-bench/.venv/bin/tau2"
          DATA_DIR="tau2-bench/data"
          COMMON=(
            --agent-model "$MODEL" --user-llm gpt-5.2
            --tau2 "$TAU2_BIN" --data-dir "$DATA_DIR"
            --domains "$DOMAINS" --num-trials "$NUM_TRIALS" --num-tasks "$NUM_TASKS"
            --max-concurrency "$MAX_CONCURRENCY"
            # Fail-fast bounds so one wedged simulation can't burn the whole 6h job
            # (a Qwen3.6-27B airline sim once hung ~5h on a single 77-min request).
            # --request-timeout caps each LiteLLM call; --run-timeout caps a whole
            # domain (worst case 3 domains/arm x 90m = 270m, under timeout-minutes:360).
            --request-timeout "$REQUEST_TIMEOUT" --run-timeout "$RUN_TIMEOUT"
          )
          if [ "$ARM_MODE" = sequential ]; then
            # Whole node per arm (TP=8), one at a time: score arm A, tear it down to
            # free the GPUs, score arm B, then diff the two saved score files.
            A_URL=$(TAU2_GPU="$GPU_A" bash scripts/tau2/launch_arms.sh a 9>&-)
            echo "arm A (vllm): $A_URL"
            python scripts/tau2/run_ab.py --score-arm "vllm=$A_URL" \
              --scores-out "$RUNNER_TEMP/scores_vllm.json" "${COMMON[@]}"
            bash scripts/tau2/launch_arms.sh stop 9>&-
            B_URL=$(TAU2_GPU="$GPU_A" bash scripts/tau2/launch_arms.sh b 9>&-)
            echo "arm B (smg): $B_URL"
            python scripts/tau2/run_ab.py --score-arm "smg=$B_URL" \
              --scores-out "$RUNNER_TEMP/scores_smg.json" "${COMMON[@]}"
            bash scripts/tau2/launch_arms.sh stop 9>&-
            python scripts/tau2/run_ab.py \
              --diff-baseline "$RUNNER_TEMP/scores_vllm.json" \
              --diff-candidate "$RUNNER_TEMP/scores_smg.json" \
              --num-trials "$NUM_TRIALS" --tolerance 0.02 \
              --out "$RUNNER_TEMP/tau2_ab.md" --json-out "$RUNNER_TEMP/tau2_ab.json" \
              || echo "TAU2_AB_REGRESSION=1" >> "$GITHUB_ENV"
          else
            # Concurrent: arm A (pure vLLM) on the first GPU half, arm B (SMG ->
            # vLLM gRPC) on the second; ports left unset so launch_arms.sh picks free.
            A_URL=$(TAU2_GPU="$GPU_A" bash scripts/tau2/launch_arms.sh a 9>&-)
            B_URL=$(TAU2_GPU="$GPU_B" bash scripts/tau2/launch_arms.sh b 9>&-)
            echo "arm A: $A_URL   arm B: $B_URL"
            python scripts/tau2/run_ab.py \
              --baseline "vllm=$A_URL" --candidate "smg=$B_URL" "${COMMON[@]}" \
              --tolerance 0.02 \
              --out "$RUNNER_TEMP/tau2_ab.md" --json-out "$RUNNER_TEMP/tau2_ab.json" \
              || echo "TAU2_AB_REGRESSION=1" >> "$GITHUB_ENV"
          fi
      - name: Render summary
        if: always()
        run: |
          {
            # Job name already shows the model + runner; keep this heading lean.
            echo "## τ²-bench A/B (domains: $DOMAINS)"
            cat "$RUNNER_TEMP/tau2_ab.md"
          } >> "$GITHUB_STEP_SUMMARY" || echo "no report produced for ${{ matrix.name }}" >> "$GITHUB_STEP_SUMMARY"
      - name: Collect tau2 transcripts
        if: always()
        run: |
          set -uo pipefail
          ROOT="tau2-bench/data/simulations"
          OUT="$RUNNER_TEMP/tau2_transcripts.tgz"
          if [ -d "$ROOT" ]; then
            if tar -czf "$OUT" -C "$ROOT" .; then
              echo "transcript tarball: $(du -h "$OUT" | cut -f1)"
              echo "contents (head):"; tar tzf "$OUT" | head -40 || true
            else
              echo "::warning::failed to create $OUT"
            fi
          else
            echo "no $ROOT"
          fi
      - name: Upload results (summary + scores + transcripts)
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: tau2-ab-${{ matrix.name }}-${{ github.run_id }}
          path: |
            ${{ runner.temp }}/tau2_ab.md
            ${{ runner.temp }}/tau2_ab.json
            ${{ runner.temp }}/tau2_transcripts.tgz
          if-no-files-found: warn
          retention-days: 30
      - name: Upload arm logs (failure only)
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: tau2-ab-${{ matrix.name }}-logs-${{ github.run_id }}
          path: ${{ runner.temp }}/tau2_run/*.log
          if-no-files-found: ignore
          retention-days: 14
      - name: Teardown arms (leg-scoped backstop)
        if: always()
        env:
          TAU2_RUN_DIR: ${{ runner.temp }}/tau2_run
        run: bash scripts/tau2/launch_arms.sh stop || true