Report Sweep (Throughput + MCP) #6
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Report Sweep (Throughput + MCP) | |
| # Fills the docs/reports/k80-*-by-family.md tables in one run. This is a MEASUREMENT | |
| # sweep, NOT a correctness gate — MCP T2 legitimately fails for weaker models, so this is | |
| # deliberately kept OUT of test-pipeline.yml (which must stay a fast pass/fail gate). | |
| # Loops every model x context serially on the single K80, thermal-guarded, and emits a | |
| # TSV + ready-to-paste report-row markdown as an artifact for a human to commit. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| suite: | |
| description: "What to sweep" | |
| type: choice | |
| options: ["both", "throughput", "mcp", "none"] | |
| default: "both" | |
| # "none" skips both default sweeps — use it with fit_map_models to run ONLY the fit-map. | |
| throughput_models: | |
| description: "Throughput models (space-separated)" | |
| type: string | |
| default: "deepseek-r1:14b deepseek-r1:32b gemma4:e4b gemma4:12b gemma4:31b gpt-oss:20b qwen3.6:27b qwen3.6:35b" | |
| mcp_models: | |
| description: "MCP models (space-separated; native tool support)" | |
| type: string | |
| default: "gemma4:e4b gemma4:12b gemma4:31b gpt-oss:20b qwen3.6:27b qwen3.6:35b" | |
| throughput_contexts: | |
| description: "Throughput context sizes (space-separated)" | |
| type: string | |
| default: "2048 4096 8192 16384" | |
| mcp_contexts: | |
| description: "MCP context sizes (space-separated)" | |
| type: string | |
| default: "8192 16384" | |
| mcp_tests: | |
| description: "MCP tests to run (T1=list projects, T2=list test suites w/ derived project_id)" | |
| type: string | |
| default: "T1 T2" | |
| num_predict: | |
| description: "Throughput tokens to generate" | |
| type: string | |
| default: "16" | |
| fit_map_models: | |
| description: "Fit-map models (space-separated; empty = skip the fit-map sweep). Records num_batch x context fit per model." | |
| type: string | |
| default: "" | |
| fit_map_num_batches: | |
| description: "Fit-map num_batch ladder (space-separated)" | |
| type: string | |
| default: "512 256 128" | |
| fit_map_contexts: | |
| description: "Fit-map context ladder (space-separated; each clipped to the model's native context via ollama show)" | |
| type: string | |
| default: "8192 65536 98304 131072 196608 262144" | |
| distractor_command: | |
| description: "MCP distractor server command (raises tool-menu difficulty; empty = none)" | |
| type: string | |
| default: "npx" | |
| distractor_args: | |
| description: "MCP distractor server args" | |
| type: string | |
| default: "@playwright/mcp@latest" | |
| runner_label: | |
| description: "Which GPU testbed to run on. sm37 = Tesla K80 (reference). sm75 = RTX 2060 (models must fit ~5.1 GiB)." | |
| required: false | |
| default: "sm37" | |
| type: choice | |
| options: | |
| - "sm37" | |
| - "sm75" | |
| env: | |
| OLLAMA_HOST: http://localhost:11434 | |
| jobs: | |
| sweep: | |
| name: Report Sweep | |
| runs-on: [self-hosted, "${{ inputs.runner_label || 'sm37' }}"] | |
| environment: cicd-1 | |
| timeout-minutes: 660 # ~11h: a full 13-model x (4 ctx tp + 4 mcp) sweep is long | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Identify host | |
| uses: ./.github/actions/identify-host | |
| - name: Validate model inputs | |
| env: | |
| TP_MODELS: ${{ inputs.throughput_models }} | |
| MCP_MODELS: ${{ inputs.mcp_models }} | |
| run: | | |
| for v in "$TP_MODELS" "$MCP_MODELS"; do | |
| if ! echo "$v" | grep -qE '^[a-zA-Z0-9:._/ -]+$'; then | |
| echo "ERROR: invalid characters in model list: $v" >&2; exit 1 | |
| fi | |
| done | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: Install test runner dependencies | |
| run: cd cicd/tests && npm ci | |
| - name: Reset sweep output dir | |
| # The self-hosted runner's /tmp/sweep-out persists across runs; without this, | |
| # aggregate-sweep.py globs stale tp_*/mcp_*/fitmap_* JSONs from earlier sweeps | |
| # into this run's SUMMARY.md (e.g. a suite=none fit-map run showing old | |
| # throughput/MCP tables). Start every sweep from an empty dir. | |
| run: rm -rf /tmp/sweep-out && mkdir -p /tmp/sweep-out | |
| - name: Throughput sweep | |
| if: ${{ inputs.suite == 'both' || inputs.suite == 'throughput' }} | |
| env: | |
| MODELS: ${{ inputs.throughput_models }} | |
| run: | | |
| set -o pipefail | |
| cd cicd/tests | |
| mkdir -p /tmp/sweep-out | |
| for CTX in ${{ inputs.throughput_contexts }}; do | |
| echo "::group::throughput @ ctx=$CTX" | |
| # one bench-throughput call measures ALL models at this context; a failure of one | |
| # cell shouldn't abort the sweep, so tolerate non-zero and keep the partial JSON. | |
| bash ../scripts/gpu-temp-guard.sh -- npx tsx src/cli.ts bench-throughput $MODELS \ | |
| --num-predict "${{ inputs.num_predict }}" \ | |
| --context "$CTX" \ | |
| --output "/tmp/sweep-out/tp_${CTX}.json" \ | |
| | tee -a "$GITHUB_STEP_SUMMARY" || echo "throughput ctx=$CTX returned non-zero (partial results kept)" | |
| echo "::endgroup::" | |
| done | |
| - name: MCP sweep | |
| if: ${{ inputs.suite == 'both' || inputs.suite == 'mcp' }} | |
| env: | |
| MODELS: ${{ inputs.mcp_models }} | |
| CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} | |
| TESTLINK_URL: ${{ secrets.TESTLINK_URL }} | |
| TESTLINK_API_KEY: ${{ secrets.TESTLINK_API_KEY }} | |
| run: | | |
| set -o pipefail | |
| cd cicd/tests | |
| mkdir -p /tmp/sweep-out | |
| for TEST in ${{ inputs.mcp_tests }}; do | |
| case "$TEST" in | |
| T1) PROMPT="List the projects."; ALLOW="list_projects" ;; | |
| T2) PROMPT="List the test suites in the ollama37 project."; ALLOW="list_projects,list_test_suites" ;; | |
| *) echo "unknown mcp test '$TEST' — skipping"; continue ;; | |
| esac | |
| for CTX in ${{ inputs.mcp_contexts }}; do | |
| echo "::group::mcp $TEST @ ctx=$CTX" | |
| bash ../scripts/gpu-temp-guard.sh -- npx tsx src/cli.ts test-mcp $MODELS \ | |
| --prompt "$PROMPT" \ | |
| --num-ctx "$CTX" \ | |
| --timeout "3600" \ | |
| --mcp-command "docker" \ | |
| --mcp-args "run --rm -i -e TESTLINK_URL -e TESTLINK_API_KEY dogkeeper886/testlink-mcp:latest" \ | |
| --mcp-env "TESTLINK_URL,TESTLINK_API_KEY" \ | |
| --distractor-command "${{ inputs.distractor_command }}" \ | |
| --distractor-args "${{ inputs.distractor_args }}" \ | |
| --verify-live --verify-allow "$ALLOW" --verify-server-name "testlink" \ | |
| --output "/tmp/sweep-out/mcp_${TEST}_${CTX}.json" \ | |
| | tee -a "$GITHUB_STEP_SUMMARY" || echo "mcp $TEST ctx=$CTX returned non-zero (partial results kept)" | |
| echo "::endgroup::" | |
| done | |
| done | |
| - name: MCP fit-map sweep | |
| # Opt-in (fit_map_models set) so the default sweep is byte-for-byte unchanged (K80 no-harm). | |
| # Per model: ollama show bounds the ctx ladder to native + gates on tool support (#446); | |
| # then loop num_batch x bounded-ctx, one test-mcp cell each. Judge is the structural check | |
| # (no --verify-live) so a judge/infra hiccup can't lose the VRAM/tps fit data — the fit | |
| # numbers (per-die VRAM + offload, #445) are captured regardless of the correctness verdict. | |
| if: ${{ inputs.fit_map_models != '' }} | |
| env: | |
| MODELS: ${{ inputs.fit_map_models }} | |
| TESTLINK_URL: ${{ secrets.TESTLINK_URL }} | |
| TESTLINK_API_KEY: ${{ secrets.TESTLINK_API_KEY }} | |
| run: | | |
| set -o pipefail | |
| cd cicd/tests | |
| mkdir -p /tmp/sweep-out | |
| for M in $MODELS; do | |
| read NATIVE_CTX TOOLS ARCH <<< "$(npx tsx src/cli.ts model-bounds "$M" 2>/dev/null)" | |
| echo "::group::fit-map $M (native_ctx=$NATIVE_CTX tools=$TOOLS arch=$ARCH)" | |
| if [ "$TOOLS" != "1" ]; then | |
| echo " $M reports no tool support — skipping (the fit-map uses the tool-call vehicle)" | |
| echo "::endgroup::"; continue | |
| fi | |
| SAFE=$(echo "$M" | tr ':/' '--') | |
| for NB in ${{ inputs.fit_map_num_batches }}; do | |
| for CTX in ${{ inputs.fit_map_contexts }}; do | |
| if [ "$NATIVE_CTX" != "0" ] && [ "$CTX" -gt "$NATIVE_CTX" ]; then | |
| echo " skip ctx=$CTX > native $NATIVE_CTX"; continue | |
| fi | |
| echo "--- $M ctx=$CTX nb=$NB ---" | |
| bash ../scripts/gpu-temp-guard.sh -- npx tsx src/cli.ts test-mcp "$M" \ | |
| --prompt "List the test suites in the ollama37 project." \ | |
| --num-ctx "$CTX" --num-batch "$NB" \ | |
| --timeout "3600" \ | |
| --mcp-command "docker" \ | |
| --mcp-args "run --rm -i -e TESTLINK_URL -e TESTLINK_API_KEY dogkeeper886/testlink-mcp:latest" \ | |
| --mcp-env "TESTLINK_URL,TESTLINK_API_KEY" \ | |
| --distractor-command "${{ inputs.distractor_command }}" \ | |
| --distractor-args "${{ inputs.distractor_args }}" \ | |
| --output "/tmp/sweep-out/fitmap_${CTX}_${NB}_${SAFE}.json" \ | |
| | tee -a "$GITHUB_STEP_SUMMARY" || echo "fit-map $M ctx=$CTX nb=$NB returned non-zero (partial results kept)" | |
| done | |
| done | |
| echo "::endgroup::" | |
| done | |
| - name: Unload models | |
| if: always() | |
| env: | |
| ALL_MODELS: ${{ inputs.throughput_models }} ${{ inputs.mcp_models }} ${{ inputs.fit_map_models }} | |
| run: | | |
| for M in $ALL_MODELS; do | |
| curl -s "${OLLAMA_HOST}/api/generate" -d "{\"model\":\"${M}\",\"keep_alive\":0}" || true | |
| done | |
| - name: Aggregate → TSV + report-row markdown | |
| if: always() | |
| run: | | |
| python3 cicd/scripts/aggregate-sweep.py /tmp/sweep-out /tmp/sweep-out/SUMMARY.md /tmp/sweep-out/results.tsv || \ | |
| echo "aggregation skipped/failed — raw JSONs are still uploaded" | |
| - name: Upload sweep results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: report-sweep-results | |
| path: /tmp/sweep-out |