test(picklescan): cover operator setitem protocol dispatch #826
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Benchmarks | |
| on: | |
| pull_request: | |
| paths: | |
| - "modelaudit/**" | |
| - "packages/modelaudit-picklescan/**" | |
| - "tests/benchmarks/**" | |
| - "tests/helpers/**" | |
| - "tests/conftest.py" | |
| - "tests/test_benchmark_report.py" | |
| - "scripts/benchmark_report.py" | |
| - "pyproject.toml" | |
| - "uv.lock" | |
| - ".github/workflows/perf.yml" | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - "modelaudit/**" | |
| - "packages/modelaudit-picklescan/**" | |
| - "tests/benchmarks/**" | |
| - "tests/helpers/**" | |
| - "tests/conftest.py" | |
| - "tests/test_benchmark_report.py" | |
| - "scripts/benchmark_report.py" | |
| - "pyproject.toml" | |
| - "uv.lock" | |
| - ".github/workflows/perf.yml" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| benchmarks: | |
| name: Benchmarks (Python 3.11) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 25 | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 | |
| with: | |
| enable-cache: true | |
| - name: Pin Python version | |
| run: | | |
| uv python pin 3.11 | |
| - name: Install Rust toolchain | |
| run: | | |
| rustup toolchain install stable --profile minimal | |
| rustup default stable | |
| - name: Run standalone picklescan Rust tests | |
| run: | | |
| cargo test --manifest-path packages/modelaudit-picklescan/Cargo.toml | |
| - name: Prepare benchmark temp directories | |
| id: paths | |
| run: | | |
| artifact_dir="$RUNNER_TEMP/modelaudit-benchmarks" | |
| base_worktree="$RUNNER_TEMP/modelaudit-base" | |
| echo "BENCHMARK_ARTIFACT_DIR=$artifact_dir" >> "$GITHUB_ENV" | |
| echo "BENCHMARK_BASE_WORKTREE=$base_worktree" >> "$GITHUB_ENV" | |
| echo "artifact_dir=$artifact_dir" >> "$GITHUB_OUTPUT" | |
| echo "base_worktree=$base_worktree" >> "$GITHUB_OUTPUT" | |
| rm -rf "$artifact_dir" "$base_worktree" | |
| mkdir -p "$artifact_dir" | |
| - name: Benchmark base commit | |
| if: github.event_name == 'pull_request' | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| run: | | |
| set -euo pipefail | |
| git worktree add --detach "$BENCHMARK_BASE_WORKTREE" "$BASE_SHA" | |
| if [ ! -f "$BENCHMARK_BASE_WORKTREE/tests/benchmarks/test_scan_benchmarks.py" ]; then | |
| echo "Base branch does not include the benchmark suite yet; skipping baseline run." | |
| exit 0 | |
| fi | |
| benchmark_tests=(tests/benchmarks/test_scan_benchmarks.py) | |
| if [ -f "$BENCHMARK_BASE_WORKTREE/tests/benchmarks/test_picklescan_benchmarks.py" ]; then | |
| benchmark_tests+=(tests/benchmarks/test_picklescan_benchmarks.py) | |
| fi | |
| uv run --directory "$BENCHMARK_BASE_WORKTREE" --python 3.11 --locked --with pytest-benchmark pytest \ | |
| "${benchmark_tests[@]}" \ | |
| --benchmark-json="$BENCHMARK_ARTIFACT_DIR/benchmark-base.json" \ | |
| -q | |
| - name: Benchmark current commit | |
| run: | | |
| # Keep this lane single-process; pytest-benchmark disables itself under xdist. | |
| benchmark_tests=(tests/benchmarks/test_scan_benchmarks.py) | |
| if [ -f tests/benchmarks/test_picklescan_benchmarks.py ]; then | |
| benchmark_tests+=(tests/benchmarks/test_picklescan_benchmarks.py) | |
| fi | |
| uv run --locked --with pytest-benchmark pytest \ | |
| "${benchmark_tests[@]}" \ | |
| --benchmark-json="$BENCHMARK_ARTIFACT_DIR/benchmark-head.json" \ | |
| -q | |
| - name: Compare against base | |
| id: compare | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| { | |
| echo "[Workflow run and artifacts](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" | |
| echo | |
| } > "$BENCHMARK_ARTIFACT_DIR/benchmark-comment.md" | |
| if [ -f "$BENCHMARK_ARTIFACT_DIR/benchmark-base.json" ]; then | |
| uv run --locked python scripts/benchmark_report.py \ | |
| --current "$BENCHMARK_ARTIFACT_DIR/benchmark-head.json" \ | |
| --baseline "$BENCHMARK_ARTIFACT_DIR/benchmark-base.json" \ | |
| --threshold 0.15 \ | |
| --summary-file "$BENCHMARK_ARTIFACT_DIR/benchmark-compare.md" | |
| else | |
| { | |
| echo "Base branch does not include the benchmark suite yet; showing current results only." | |
| echo | |
| } > "$BENCHMARK_ARTIFACT_DIR/benchmark-current.md" | |
| uv run --locked python scripts/benchmark_report.py \ | |
| --current "$BENCHMARK_ARTIFACT_DIR/benchmark-head.json" \ | |
| --summary-file "$BENCHMARK_ARTIFACT_DIR/benchmark-current-report.md" | |
| cat "$BENCHMARK_ARTIFACT_DIR/benchmark-current.md" >> "$BENCHMARK_ARTIFACT_DIR/benchmark-comment.md" | |
| cat "$BENCHMARK_ARTIFACT_DIR/benchmark-current-report.md" >> "$BENCHMARK_ARTIFACT_DIR/benchmark-comment.md" | |
| fi | |
| if [ -f "$BENCHMARK_ARTIFACT_DIR/benchmark-compare.md" ]; then | |
| cat "$BENCHMARK_ARTIFACT_DIR/benchmark-compare.md" >> "$BENCHMARK_ARTIFACT_DIR/benchmark-comment.md" | |
| fi | |
| cat "$BENCHMARK_ARTIFACT_DIR/benchmark-comment.md" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Summarize current results | |
| if: github.event_name != 'pull_request' | |
| run: | | |
| { | |
| echo "[Workflow run and artifacts](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" | |
| echo | |
| } > "$BENCHMARK_ARTIFACT_DIR/benchmark-summary.md" | |
| uv run --locked python scripts/benchmark_report.py \ | |
| --current "$BENCHMARK_ARTIFACT_DIR/benchmark-head.json" \ | |
| --summary-file "$BENCHMARK_ARTIFACT_DIR/benchmark-current.md" | |
| cat "$BENCHMARK_ARTIFACT_DIR/benchmark-current.md" >> "$BENCHMARK_ARTIFACT_DIR/benchmark-summary.md" | |
| cat "$BENCHMARK_ARTIFACT_DIR/benchmark-summary.md" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Comment benchmark summary on PR | |
| if: > | |
| always() && | |
| github.event_name == 'pull_request' && | |
| github.event.pull_request.head.repo.full_name == github.repository | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 | |
| env: | |
| COMMENT_BODY_PATH: ${{ steps.paths.outputs.artifact_dir }}/benchmark-comment.md | |
| with: | |
| script: | | |
| const fs = require("fs") | |
| if (!fs.existsSync(process.env.COMMENT_BODY_PATH)) { | |
| core.warning(`No benchmark comment body found at ${process.env.COMMENT_BODY_PATH}`) | |
| return | |
| } | |
| const marker = "<!-- modelaudit-perf-benchmarks -->" | |
| const body = `${marker}\n${fs.readFileSync(process.env.COMMENT_BODY_PATH, "utf8")}` | |
| const { owner, repo } = context.repo | |
| const issue_number = context.issue.number | |
| const comments = await github.paginate(github.rest.issues.listComments, { | |
| owner, | |
| repo, | |
| issue_number, | |
| per_page: 100, | |
| }) | |
| const existing = comments.find( | |
| (comment) => comment.user?.type === "Bot" && comment.body?.includes(marker), | |
| ) | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner, | |
| repo, | |
| comment_id: existing.id, | |
| body, | |
| }) | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number, | |
| body, | |
| }) | |
| } | |
| - name: Upload benchmark artifacts | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: benchmark-results-python-3.11 | |
| path: ${{ steps.paths.outputs.artifact_dir }} | |
| if-no-files-found: error | |
| retention-days: 14 | |
| - name: Cleanup benchmark base worktree | |
| if: always() && github.event_name == 'pull_request' | |
| run: | | |
| git worktree remove --force "$BENCHMARK_BASE_WORKTREE" || true |