atom-benchmark.yaml

name: ATOM Benchmark

# The run title IS the cadence label: the baseline lookup below matches on it to
# keep the nightly and weekly comparison chains apart. Both crons live in this
# workflow, so without that a weekly run would become the next nightly's
# baseline and silently blank out its whole regression table.
run-name: >-
  ${{ github.event_name == 'schedule'
      && (github.event.schedule == '37 4 * * 0' && 'weekly' || 'nightly')
      || format('manual ({0})', github.actor) }}

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

on:
  schedule:
    # Nightly at 00:12 Beijing time (16:12 UTC). Runs the 8k/1k grid only.
    - cron: '12 16 * * *'
    # Weekly at 12:37 Beijing time on Sunday (04:37 UTC). Runs the 1k/1k grid,
    # which is half the catalog's cells and was making the nightly too long.
    # Daytime and on a weekend, well clear of the 15:00-19:00 UTC cron cluster
    # the other benchmark workflows sit in.
    - cron: '37 4 * * 0'
  workflow_dispatch:
    inputs:
      publish_to_dashboard:
        description: "Upload results to the dashboard (off by default for manual runs; schedule always publishes)"
        type: boolean
        default: false
      deepseek-v4-pro:
        description: "Benchmark DeepSeek-V4-Pro"
        type: boolean
        default: true
      deepseek-v4-pro-eplb:
        description: "Benchmark DeepSeek-V4-Pro EPLB r0 + MegaMoE (8k1k, c=512/4096)"
        type: boolean
        default: true
      deepseek-v41-flash:
        description: "Benchmark DeepSeek-V4.1-Flash DSpark (TP4, c<=64)"
        type: boolean
        default: true
      deepseek-r1-0528-mxfp4:
        description: "Benchmark DeepSeek-R1-0528 MXFP4 (+ MXFP4-MTP3)"
        type: boolean
        default: true
      gpt-oss-120b:
        description: "Benchmark gpt-oss-120b"
        type: boolean
        default: true
      kimi-k3:
        description: "Benchmark Kimi-K3 (+ DSpark)"
        type: boolean
        default: true
      m3-mxfp8:
        description: "Benchmark MiniMax-M3-MXFP8 (+ EAGLE3)"
        type: boolean
        default: true
      m3-mxfp4:
        description: "Benchmark MiniMax-M3-MXFP4 (+ EAGLE3)"
        type: boolean
        default: true
      qwen35-397b-fp8:
        description: "Benchmark Qwen3.5-397B-A17B-FP8"
        type: boolean
        default: true
      qwen35-397b-mxfp4:
        description: "Benchmark Qwen3.5-397B-A17B-MXFP4"
        type: boolean
        default: true
      llama-3-3-70b-instruct-mxfp4:
        description: "Benchmark Llama-3.3-70B-Instruct-MXFP4"
        type: boolean
        default: true
      extra_args:
        description: "Extra arguments to pass to the ATOM server"
        type: string
      image:
        description: "Docker image; empty uses rocm/atom-dev:latest"
        type: string
        default: ""
      runner:
        description: "Optional runner label override for manual runs; leave empty to use model defaults"
        type: string
        default: ""
      enable_profiler:
        description: "Enable torch profiler to collect trace for performance debugging"
        type: boolean
        default: false
      enable_rtl:
        description: "Enable RTL (rocm-trace-lite) GPU kernel tracing for prefill/decode analysis"
        type: boolean
        default: false
      param_lists:
        description: |
          "Benchmark parameter lists. 
          Input as a single or multiple sets (comma-separated, semicolon between sets), 
          format: input_length,output_length,concurrency,random_range_ratio.
          Example (single set): 1024,1024,128,0.8
          Example (multiple sets): 1024,1024,128,0.8;2048,1024,256,0.7"
        type: string
        default: "1024,1024,128,0.8"
      atom_commit:
        description: "ATOM commit SHA to checkout; leave empty to use the workflow ref"
        type: string
        default: ""
      aiter_commit:
        description: "ROCm/aiter commit SHA, tag, or branch to reinstall before benchmarking; leave empty to keep the image version"
        type: string
        default: ""

permissions:
  actions: read
  contents: write
  issues: write

env:
  # Which slice of the catalog to build, and which chain to take a baseline
  # from. A manual run compares against the nightly, as it always has.
  CADENCE: ${{ github.event_name == 'schedule' && github.event.schedule == '37 4 * * 0' && 'weekly' || 'nightly' }}

jobs:
  # Single source of the benchmark matrix. Expands the structured catalog
  # (.github/benchmark/models.json) into fully-resolved per-run "cells" — one
  # matrix dimension — applying model selection and per-variant concurrency
  # bands in one place. See .github/scripts/{catalog,build_benchmark_matrix}.py
  # and .github/benchmark/README.md.
  build-matrix:
    name: Build benchmark matrix
    runs-on: ubuntu-latest
    outputs:
      configs_json: ${{ steps.build.outputs.configs_json }}
      has_cells: ${{ steps.build.outputs.has_cells }}
      checkout_sha: ${{ steps.revision.outputs.sha }}
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ inputs.atom_commit || github.sha }}
      - name: Record checked-out commit
        id: revision
        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
      - id: build
        env:
          EVENT_NAME: ${{ github.event_name }}
          INPUTS_JSON: ${{ toJson(inputs) }}
          CADENCE: ${{ env.CADENCE }}
        run: python3 .github/scripts/build_benchmark_matrix.py

  # Top-level fan-out: one matrix entry per (model variant × scenario) config.
  # Each invokes benchmark-tmpl.yml, which fans out over the config's
  # `concurrency` list. Two bounded matrices replace the former flat per-cell
  # matrix that overflowed GitHub's 256-job-per-matrix limit (278 cells). Every
  # (config × conc) cell still runs as its own parallel job; the caller job name
  # stays `benchmark` so downstream `needs:` are unchanged.
  benchmark:
    name: ${{ matrix.config.display }} ${{ matrix.config.scenario }}
    needs: [build-matrix]
    if: >-
      !cancelled()
      && needs.build-matrix.result == 'success'
      && needs.build-matrix.outputs.has_cells == 'true'
    strategy:
      fail-fast: false
      matrix:
        config: ${{ fromJson(needs.build-matrix.outputs.configs_json) }}
    uses: ./.github/workflows/benchmark-tmpl.yml
    secrets: inherit
    with:
      display: ${{ matrix.config.display }}
      prefix: ${{ matrix.config.prefix }}
      suffix: ${{ matrix.config.suffix }}
      model_path: ${{ matrix.config.model_path }}
      server_args: ${{ matrix.config.server_args }}
      bench_kind: ${{ matrix.config.bench_kind }}
      bench_args: ${{ matrix.config.bench_args }}
      env_vars: ${{ matrix.config.env_vars }}
      runner: ${{ (github.event_name == 'workflow_dispatch' && inputs.runner != '' && inputs.runner) || matrix.config.runner }}
      isl: ${{ matrix.config.isl }}
      osl: ${{ matrix.config.osl }}
      ratio: ${{ matrix.config.ratio }}
      ratio_str: ${{ matrix.config.ratio_str }}
      concurrency: ${{ matrix.config.concurrency }}
      image: ${{ inputs.image || 'rocm/atom-dev:latest' }}
      enable_profiler: ${{ inputs.enable_profiler || false }}
      enable_rtl: ${{ inputs.enable_rtl || false }}
      extra_args: ${{ inputs.extra_args || '' }}
      atom_commit: ${{ needs.build-matrix.outputs.checkout_sha }}
      aiter_commit: ${{ inputs.aiter_commit || '' }}

  summarize-benchmark-result:
    concurrency:
      group: gh-pages-deploy
      cancel-in-progress: false
    if: always()
    name: Summarize benchmark result
    needs: [benchmark]
    runs-on: ubuntu-latest
    env:
      # Per-step rather than a job-level `if`: the summary steps in this job
      # must run either way.
      PUBLISH_DASHBOARD: ${{ github.event_name != 'workflow_dispatch' || inputs.publish_to_dashboard }}
    outputs:
      has_regression: ${{ steps.check-regression.outputs.has_regression }}
    steps:
      - name: Checkout ATOM repo
        uses: actions/checkout@v6
        with:
          ref: ${{ inputs.atom_commit || github.ref }}

      - name: Download all benchmark results
        uses: actions/download-artifact@v8
        with:
          pattern: 'benchmark-*'
          merge-multiple: true
          path: .

      - name: Download baseline from previous run of the same cadence
        id: baseline
        run: |
          # Use the most recent completed run (regardless of success/failure)
          # because even failed runs have valid per-model benchmark artifacts.
          # Only requiring --status=success misses baselines when any single job fails.
          #
          # Matched on the run title, which `run-name` sets to the cadence: the
          # nightly and weekly crons run different grids out of this one
          # workflow, so comparing across them would leave every cell without a
          # counterpart. A manual run reads CADENCE as `nightly` and so keeps
          # comparing against the nightly, as before.
          #
          # limit=20 spans more than a week of nightlies, so a weekly run can
          # still reach the previous weekly seven runs back.
          PREV_RUN_ID=$(gh run list \
            --workflow="ATOM Benchmark" \
            --branch=main \
            --event=schedule \
            --limit=20 \
            --json databaseId,status,displayTitle \
            | jq -r --arg cadence "$CADENCE" \
                '[.[] | select(.status == "completed" and .displayTitle == $cadence)][0].databaseId // empty')

          if [ -n "$PREV_RUN_ID" ] && [ "$PREV_RUN_ID" != "${{ github.run_id }}" ]; then
            echo "Downloading baseline from run #$PREV_RUN_ID"
            mkdir -p /tmp/baseline
            # Only download benchmark result artifacts (prefix: benchmark-),
            # not profiler/regression traces which can be multi-GB.
            gh run download "$PREV_RUN_ID" --dir /tmp/baseline --pattern 'benchmark-*' || echo "::warning::Failed to download baseline artifacts"
            BASELINE_COUNT=$(find /tmp/baseline -name '*.json' | wc -l)
            echo "Downloaded $BASELINE_COUNT baseline files"
            echo "baseline_dir=/tmp/baseline" >> $GITHUB_OUTPUT
          else
            echo "No previous completed nightly run found"
            echo "baseline_dir=" >> $GITHUB_OUTPUT
          fi
        env:
          GH_TOKEN: ${{ github.token }}

      - name: List all benchmark results
        run: |
          echo "=== Current results ==="
          ls -la *.json 2>/dev/null || echo "No JSON files in current dir"
          if [ -d "/tmp/baseline" ]; then
            echo "=== Baseline results ==="
            find /tmp/baseline -name '*.json' | head -20
          fi

      - name: Summarize benchmark result
        id: check-regression
        run: |
          BASELINE_ARG=""
          if [ -n "${{ steps.baseline.outputs.baseline_dir }}" ]; then
            BASELINE_ARG="--baseline-dir ${{ steps.baseline.outputs.baseline_dir }}"
          fi
          .github/scripts/summarize.py . $BASELINE_ARG \
            --output-json regression_report.json \
            >> $GITHUB_STEP_SUMMARY || true

          # Check if regression was detected (exit code 2)
          if [ -f regression_report.json ]; then
            REG_COUNT=$(python3 -c "import json; print(json.load(open('regression_report.json'))['regression_count'])")
            if [ "$REG_COUNT" -gt 0 ]; then
              echo "has_regression=true" >> $GITHUB_OUTPUT
            else
              echo "has_regression=false" >> $GITHUB_OUTPUT
            fi
          else
            echo "has_regression=false" >> $GITHUB_OUTPUT
          fi

      - name: Upload regression report
        if: steps.check-regression.outputs.has_regression == 'true'
        uses: actions/upload-artifact@v7
        with:
          name: regression-report
          path: regression_report.json

      - name: Transform results for benchmark dashboard
        if: env.PUBLISH_DASHBOARD == 'true'
        run: |
          python3 .github/scripts/plugin_benchmark_to_dashboard.py \
            . \
            --output benchmark-action-input.json \
            --default-backend ATOM \
            --run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

      - name: Prepare existing dashboard data
        if: env.PUBLISH_DASHBOARD == 'true'
        id: prepare_dashboard_data
        uses: ./.github/actions/prepare-dashboard-data
        with:
          data-path: benchmark-dashboard/data.js

      - name: Store benchmark result to dashboard
        if: env.PUBLISH_DASHBOARD == 'true'
        uses: benchmark-action/github-action-benchmark@v1
        with:
          tool: customBiggerIsBetter
          output-file-path: benchmark-action-input.json
          gh-pages-branch: gh-pages
          benchmark-data-dir-path: benchmark-dashboard
          auto-push: false
          alert-threshold: "80%"
          comment-on-alert: true
          fail-on-alert: false
          max-items-in-chart: 300
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Validate dashboard data after write
        if: env.PUBLISH_DASHBOARD == 'true'
        uses: ./.github/actions/validate-dashboard-data
        with:
          data-path: benchmark-dashboard/data.js
          min-entries: ${{ steps.prepare_dashboard_data.outputs.entry_count }}

      - name: Deploy custom dashboard to gh-pages
        if: env.PUBLISH_DASHBOARD == 'true'
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          CURRENT_SHA=$(git rev-parse HEAD)
          # Generate prefix→display name mapping from the catalog
          python3 .github/scripts/dashboard_models_map.py \
            .github/benchmark/models.json /tmp/dashboard_models_map.js
          # Save dashboard assets before switching branches
          cp .github/dashboard/index.html /tmp/dashboard_index.html
          cp docs/assets/atom_logo.png /tmp/dashboard_logo.png
          cp docs/assets/atom_logo_mini.png /tmp/dashboard_logo_mini.png
          git fetch origin gh-pages
          git checkout gh-pages
          # Replace auto-generated index.html with custom dashboard + logo
          cp /tmp/dashboard_index.html benchmark-dashboard/index.html
          cp /tmp/dashboard_models_map.js benchmark-dashboard/models_map.js
          cp /tmp/dashboard_logo.png benchmark-dashboard/atom_logo.png
          cp /tmp/dashboard_logo_mini.png benchmark-dashboard/atom_logo_mini.png
          git add benchmark-dashboard/
          git diff --cached --quiet || git commit -m "Update benchmark data and dashboard"
          git push origin gh-pages
          git checkout "$CURRENT_SHA"

  # ---------- Generate regression matrix (lightweight, ubuntu) ----------
  generate-regression-matrix:
    if: always() && needs.summarize-benchmark-result.outputs.has_regression == 'true'
    name: Generate regression rerun matrix
    needs: [summarize-benchmark-result]
    runs-on: ubuntu-latest
    outputs:
      matrix_json: ${{ steps.gen.outputs.matrix_json }}
      has_matrix: ${{ steps.gen.outputs.has_matrix }}
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ inputs.atom_commit || github.ref }}

      - uses: actions/download-artifact@v8
        with:
          name: regression-report

      - name: Generate matrix from regression report
        id: gen
        run: |
          python3 .github/scripts/regression_rerun.py \
            regression_report.json \
            .github/benchmark/models.json \
            --output-matrix /tmp/matrix.json

          if [ -s /tmp/matrix.json ] && [ "$(cat /tmp/matrix.json)" != "[]" ]; then
            echo "matrix_json=$(cat /tmp/matrix.json)" >> $GITHUB_OUTPUT
            echo "has_matrix=true" >> $GITHUB_OUTPUT
            echo "=== Matrix cells ==="
            python3 -c "
          import json
          cells = json.load(open('/tmp/matrix.json'))
          for c in cells:
              configs = json.loads(c['configs'])
              print(f\"  {c['prefix']}: {len(configs)} config(s) on {c['runner']}\")
          "
          else
            echo "matrix_json=[]" >> $GITHUB_OUTPUT
            echo "has_matrix=false" >> $GITHUB_OUTPUT
            echo "No regression configs generated"
          fi

  # ---------- Regression re-run (GPU, matrix by model) ----------
  regression-rerun:
    if: >-
      !cancelled()
      && needs.generate-regression-matrix.outputs.has_matrix == 'true'
    name: Rerun ${{ matrix.cell.prefix }}
    needs: [generate-regression-matrix]
    strategy:
      fail-fast: false
      matrix:
        cell: ${{ fromJson(needs.generate-regression-matrix.outputs.matrix_json) }}
    runs-on: ${{ (github.event_name == 'workflow_dispatch' && inputs.runner != '' && inputs.runner) || matrix.cell.runner }}
    timeout-minutes: 60
    steps:
      - name: Kill all Docker containers
        run: |
          echo "=== Cleaning up containers on $(hostname) ==="
          containers=$(docker ps -q)
          if [ -n "$containers" ]; then
            docker kill $containers || true
          fi
          docker run --rm -v "${GITHUB_WORKSPACE:-$PWD}":/workspace -w /workspace --privileged rocm/pytorch:latest bash -lc "find /workspace -mindepth 1 -delete" || true

      - name: Checkout ATOM repo
        uses: actions/checkout@v6
        with:
          ref: ${{ inputs.atom_commit || github.ref }}

      - name: Docker Login
        uses: ./.github/actions/docker-auth
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Start container + download model
        uses: ./.github/actions/atom-bench-container
        with:
          image: ${{ inputs.image || 'rocm/atom-dev:latest' }}
          container-name: atom-regression
          model-path: ${{ matrix.cell.model_path }}
          runner: ${{ (github.event_name == 'workflow_dispatch' && inputs.runner != '' && inputs.runner) || matrix.cell.runner }}
          env-vars: ${{ matrix.cell.env_vars }}
          hf-token: ${{ secrets.AMD_HF_TOKEN }}
          download-required: "false"

      - name: Reinstall AITER from requested ref
        if: ${{ inputs.aiter_commit != '' }}
        env:
          CONTAINER_NAME: atom-regression
          AITER_GIT_REF: ${{ inputs.aiter_commit }}
        run: bash .github/scripts/install_aiter_from_ref.sh

      - name: Launch server
        env:
          SERVER_ARGS: ${{ matrix.cell.server_args }}
        run: |
          if [ -n "${MODEL_HOST_ROOT:-}" ]; then MODEL_DIR="/models/${{ matrix.cell.model_path }}"
          else MODEL_DIR="${{ matrix.cell.model_path }}"; fi

          # cell.env_vars are already injected into the container env via
          # --env-file in "Start CI container" — `docker exec` inherits them,
          # so no need to re-prefix them on the command line (the previous
          # inline form silently dropped the first var when env_vars contained
          # a literal newline).
          #
          # Launch via stdin so the container's bash parses the shell quoting in
          # SERVER_ARGS exactly once -- single-quoted JSON values survive intact
          # (e.g. --hf-overrides '{"use_index_cache": true}'). See "Run benchmark".
          echo "ENABLE_TORCH_PROFILER=1 \
            .github/scripts/atom_test.sh launch $MODEL_DIR $SERVER_ARGS" \
            | docker exec -i atom-regression bash -l

      - name: Run regression configs with profiler
        env:
          SERVER_ARGS: ${{ matrix.cell.server_args }}
        run: |
          set -uo pipefail  # no -e: allow per-config failure without aborting loop

          if [ -n "${MODEL_HOST_ROOT:-}" ]; then MODEL_DIR="/models/${{ matrix.cell.model_path }}"
          else MODEL_DIR="${{ matrix.cell.model_path }}"; fi

          FAIL_COUNT=0
          TOTAL_COUNT=0

          # Parse configs JSON into pipe-delimited lines for shell loop
          echo '${{ matrix.cell.configs }}' | python3 -c "
          import json, sys
          for c in json.load(sys.stdin):
              print('{isl}|{osl}|{conc}|{bench_args}|{prefix}'.format(**c))
          " > /tmp/regression_configs.txt

          while IFS='|' read -r ISL OSL CONC BENCH_ARGS PREFIX; do
            TOTAL_COUNT=$((TOTAL_COUNT + 1))
            TRACE_SUBDIR="${PREFIX}-${ISL}-${OSL}-${CONC}"

            docker exec atom-regression bash -lc "mkdir -p /app/trace/$TRACE_SUBDIR"

            # Record profiler stop count before benchmark (to detect new completions)
            STOP_COUNT_BEFORE=$(docker exec atom-regression grep -c "Profiler stopped." /tmp/atom_server.log 2>/dev/null) || STOP_COUNT_BEFORE=0

            echo "=== Profiling: $PREFIX ISL=$ISL OSL=$OSL CONC=$CONC ==="
            # cell.env_vars are already in the container env via --env-file
            # (see "Launch server"); docker exec inherits them, so do NOT
            # re-prefix them inline -- V4-Pro's env_vars contains a literal
            # newline that would break the bash -lc string / drop a var.
            if ! docker exec \
              -e SERVER_ARGS="$SERVER_ARGS" \
              atom-regression bash -lc "
              ENABLE_TORCH_PROFILER=1 \
              ISL=$ISL OSL=$OSL CONC=$CONC \
              RANDOM_RANGE_RATIO=0.8 \
              NUM_PROMPTS_OVERRIDE=$((CONC*2)) \
              RESULT_FILENAME=regression-${PREFIX}-${ISL}-${OSL}-${CONC} \
              BENCH_EXTRA_ARGS='$BENCH_ARGS' \
              .github/scripts/atom_test.sh benchmark $MODEL_DIR"; then
              echo "::warning::Benchmark failed for $PREFIX ISL=$ISL OSL=$OSL CONC=$CONC"
              FAIL_COUNT=$((FAIL_COUNT + 1))
              continue
            fi

            # Wait for engine core to finish exporting profiler trace (blocks inference)
            # See .claude/plan/profiler-async-refactor.md for long-term fix
            for i in $(seq 1 300); do
              STOP_COUNT=$(docker exec atom-regression grep -c "Profiler stopped." /tmp/atom_server.log 2>/dev/null) || STOP_COUNT=0
              [ "$STOP_COUNT" -gt "$STOP_COUNT_BEFORE" ] && echo "Profiler trace export done after ${i}s" && break
              [ "$i" -eq 1 ] && echo "Waiting for profiler trace export..."
              [ "$i" -eq 300 ] && echo "WARNING: Profiler did not finish after 300s, proceeding anyway"
              sleep 1
            done

            # Move trace files into per-config subdir
            docker exec atom-regression bash -lc "
              for d in /app/trace/rank_*; do
                [ -d \"\$d\" ] && mv \"\$d\" /app/trace/$TRACE_SUBDIR/ 2>/dev/null || true
              done"
          done < /tmp/regression_configs.txt

          echo "=== Regression re-run complete: $((TOTAL_COUNT - FAIL_COUNT))/$TOTAL_COUNT succeeded ==="
          if [ "$FAIL_COUNT" -gt 0 ]; then
            echo "::warning::$FAIL_COUNT/$TOTAL_COUNT configs failed"
          fi

      - name: Stop server
        if: always()
        run: |
          docker exec atom-regression bash -lc \
            "TORCH_PROFILER_DIR=/app/trace .github/scripts/atom_test.sh stop" || true

      - name: Copy profiler traces from container
        id: copy-traces
        if: always()
        run: |
          docker cp atom-regression:/app/trace ./regression-traces 2>/dev/null || echo "No traces found"
          TRACE_COUNT=$(find ./regression-traces -type f \( -name "*.json.gz" -o -name "*.pt.trace.*" \) 2>/dev/null | wc -l | tr -d ' ')
          echo "trace_count=$TRACE_COUNT" >> $GITHUB_OUTPUT
          if [ "$TRACE_COUNT" -gt 0 ]; then
            echo "Found $TRACE_COUNT trace files"
            ls -lhR ./regression-traces/
          else
            echo "::warning::No trace files found in container — profiler may not have produced output"
          fi

      - name: Upload regression traces
        if: always() && fromJSON(steps.copy-traces.outputs.trace_count) > 0
        uses: actions/upload-artifact@v7
        with:
          name: regression-traces-${{ github.run_id }}-${{ matrix.cell.prefix }}
          path: regression-traces/

      - name: Upload regression results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: regression-results-${{ github.run_id }}-${{ matrix.cell.prefix }}
          path: regression-*.json
          if-no-files-found: ignore

      - name: Clean Up
        if: always()
        run: |
          docker stop atom-regression || true
          docker rm atom-regression || true

  # ---------- Collect regression traces from all matrix cells ----------
  collect-regression-traces:
    if: always() && needs.regression-rerun.result != 'skipped'
    name: Collect regression traces
    needs: [regression-rerun]
    runs-on: ubuntu-latest
    steps:
      - name: Download all regression trace artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: regression-traces-${{ github.run_id }}-*
          merge-multiple: true
          path: regression-traces/
        continue-on-error: true

      - name: Upload combined regression traces
        uses: actions/upload-artifact@v7
        with:
          name: regression-traces-${{ github.run_id }}
          path: regression-traces/
          if-no-files-found: ignore

  # ---------- Profiler analysis (lightweight, runs on regression OR manual profiler) ----------
  profiler-analysis:
    if: >-
      always()
      && (needs.collect-regression-traces.result != 'cancelled'
          && needs.collect-regression-traces.result != 'skipped'
          || inputs.enable_profiler == true)
    name: Profiler trace analysis
    needs: [benchmark, summarize-benchmark-result, generate-regression-matrix, regression-rerun, collect-regression-traces]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout ATOM repo
        uses: actions/checkout@v6
        with:
          ref: ${{ inputs.atom_commit || github.ref }}

      - name: Download regression traces
        if: needs.collect-regression-traces.result == 'success'
        uses: actions/download-artifact@v8
        continue-on-error: true
        with:
          name: regression-traces-${{ github.run_id }}
          path: profiler-traces/

      - name: Download benchmark profiler traces
        if: needs.collect-regression-traces.result != 'success'
        uses: actions/download-artifact@v8
        continue-on-error: true
        with:
          pattern: profiler-traces-*
          merge-multiple: true
          path: profiler-traces/

      - name: Analyze traces
        run: |
          mkdir -p profiler-analysis
          pip install -q openpyxl 2>/dev/null || true

          # Find all config subdirectories containing rank_* dirs
          # Structure: profiler-traces/<config-name>/rank_0/*.gz
          CONFIG_DIRS=$(find profiler-traces -mindepth 1 -maxdepth 1 -type d 2>/dev/null || true)
          if [ -z "$CONFIG_DIRS" ]; then
            echo "No config directories found in profiler-traces/"
            ls -lhR profiler-traces/ 2>/dev/null || true
            exit 0
          fi

          for config_dir in $CONFIG_DIRS; do
            config=$(basename "$config_dir")
            echo ""
            echo "========== Config: $config =========="
            mkdir -p "profiler-analysis/$config"

            # Analyze rank_0 trace (representative for kernel breakdown + summary)
            RANK0_TRACE=$(find "$config_dir/rank_0" -name "*.pt.trace.json.gz" ! -name "capture_graph_*" 2>/dev/null | sort | tail -1)
            if [ -z "$RANK0_TRACE" ]; then
              echo "No rank_0 trace found for $config, skipping"
              continue
            fi

            echo "=== Analyzing: $RANK0_TRACE ==="
            cd "profiler-analysis/$config"
            python ../../tools/parse_trace.py "../../$RANK0_TRACE" 2>&1 | tee analysis.log || true
            python ../../tools/analyze_trace_summary.py "../../$RANK0_TRACE" \
              --output performance_summary.md 2>&1 || true
            if [ -f performance_summary.md ]; then
              echo "=== Performance Summary ($config) ==="
              cat performance_summary.md
            fi
            cd ../..
          done
          ls -laR profiler-analysis/ 2>/dev/null || true

      - name: Upload analysis results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: profiler-analysis-${{ github.run_id }}
          path: profiler-analysis/

      - name: Download regression report
        if: needs.summarize-benchmark-result.outputs.has_regression == 'true'
        uses: actions/download-artifact@v8
        with:
          name: regression-report
        continue-on-error: true

      - name: Create GitHub Issue for regression
        if: needs.summarize-benchmark-result.outputs.has_regression == 'true'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            let report;
            try {
              report = JSON.parse(fs.readFileSync('regression_report.json', 'utf8'));
            } catch (e) {
              console.log('No regression report found, skipping issue creation');
              return;
            }
            if (!report.regressions || report.regressions.length === 0) return;

            const regs = report.regressions.map(r => {
              const throughput = r.metrics.output_throughput || {};
              const tpot = r.metrics.mean_tpot_ms || {};
              return `| ${r.model} | ${r.isl}/${r.osl} | ${r.conc} | ${throughput.current?.toFixed(1) || 'N/A'} | ${throughput.baseline?.toFixed(1) || 'N/A'} | ${throughput.pct?.toFixed(1) || 'N/A'}% | ${tpot.current?.toFixed(2) || 'N/A'} | ${tpot.baseline?.toFixed(2) || 'N/A'} | ${tpot.pct?.toFixed(1) || 'N/A'}% |`;
            }).join('\n');

            let summary = 'Summary not available';
            try {
              const dirs = fs.readdirSync('profiler-analysis').filter(d =>
                fs.statSync(`profiler-analysis/${d}`).isDirectory());
              for (const d of dirs) {
                const p = `profiler-analysis/${d}/performance_summary.md`;
                if (fs.existsSync(p)) { summary = fs.readFileSync(p, 'utf8'); break; }
              }
            } catch(e) { /* ignore */ }

            const body = `## Performance Regression Detected

            **Commit:** \`${context.sha.substring(0, 8)}\`
            **Run:** ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}
            **Date:** ${report.timestamp}

            ### Regressed Configurations

            | Model | ISL/OSL | Conc | Tput (cur) | Tput (base) | Δ% | TPOT (cur) | TPOT (base) | Δ% |
            |-------|---------|------|-----------|------------|-----|-----------|------------|-----|
            ${regs}

            ### Performance Summary

            \`\`\`
            ${summary}
            \`\`\`

            ### Profiler Traces

            Download from [workflow artifacts](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).
            Open in [Perfetto UI](https://ui.perfetto.dev/) or Chrome \`chrome://tracing\` for analysis.

            ### Next Steps
            1. Download \`profiler-analysis-${context.runId}\` artifact
            2. Open trace files in Perfetto UI
            3. Compare kernel durations against previous traces
            4. Identify bottleneck changes
            `.replace(/^            /gm, '');

            await github.rest.issues.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              title: `[Perf Regression] ${report.regressions.length} config(s) regressed @ ${context.sha.substring(0, 8)}`,
              body: body,
              labels: ['performance', 'regression']
            });