workflows: report benchmark regressions back to pull request

Let's see if this works, it's a bit experimental at this point.

The twist comparred to how it's been done in EOPA (for example) is that
we're running the benchmarks post-merge, and report back if at the end
we find a failing check. This way, the PR goes green without having to
wait for the benchmarks, but there's still a connection between PR and
benchmark.

Signed-off-by: Stephan Renatus <stephan.renatus@gmail.com>

Author: srenatus

.github/workflows/benchmarks.yaml

@@ -74,6 +74,87 @@ jobs:
       publish: true
       publish_branch: benchmarks
 
+  regression-check:
+    permissions:
+      contents: read
+      pull-requests: write
+    name: Check for regressions
+    runs-on: ubuntu-24.04
+    needs: [check-changes, benchmarks]
+    if: ${{ needs.check-changes.outputs.go == 'true' }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - id: go_version
+        name: Read go version
+        run: echo "go_version=$(cat .go-version)" >> $GITHUB_OUTPUT
+      - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        with:
+          go-version: ${{ steps.go_version.outputs.go_version }}
+      - name: Fetch benchmark data
+        run: |
+          git fetch origin benchmarks
+          git show origin/benchmarks:benchmarks.json > benchmarks.json
+      - name: Split latest two runs
+        run: |
+          # Extract the two most recent runs into separate files
+          jq '.[0]' benchmarks.json | jq '[.]' > current.json
+          jq '.[1]' benchmarks.json | jq '[.]' > base.json
+      - name: Run regression checks
+        id: checks
+        continue-on-error: true
+        run: |
+          go run go.bobheadxi.dev/gobenchdata@v1 checks eval base.json current.json \
+            --checks.config build/gobenchdata-checks.yml \
+            --json report.json
+          go run go.bobheadxi.dev/gobenchdata@v1 checks report report.json
+      - name: Comment on PR if regression detected
+        if: ${{ steps.checks.outcome == 'failure' }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMIT_SHA: ${{ github.sha }}
+        run: |
+          PR_NUMBER=$(gh pr list --search "${COMMIT_SHA}" --state merged --json number --jq '.[0].number')
+
+          if [ -z "$PR_NUMBER" ] || [ "$PR_NUMBER" = "null" ]; then
+            echo "Could not find originating PR for commit ${COMMIT_SHA}"
+            exit 0
+          fi
+
+          REPORT=$(jq -r '
+            .checks[]
+            | select(.status != "pass")
+            | "| \(.name) | \(.status) | \(.diffs | length) benchmarks |"
+          ' report.json)
+
+          BODY=$(cat <<EOF
+          ## Benchmark Regression Detected
+
+          Commit \`${COMMIT_SHA}\` introduced a benchmark regression.
+
+          | Check | Status | Affected |
+          |-------|--------|----------|
+          ${REPORT}
+
+          <details>
+          <summary>Full report</summary>
+
+          \`\`\`json
+          $(jq . report.json)
+          \`\`\`
+
+          </details>
+
+          [Benchmarks Dashboard](https://open-policy-agent.github.io/opa/index.html)
+
+          _This comment was automatically generated by the benchmarks workflow._
+          EOF
+          )
+
+          gh pr comment "${PR_NUMBER}" --body "${BODY}"
+
   notebook:
     permissions:
       contents: write # we'll push to the `benchmarks` branch

build/gobenchdata-checks.yml

@@ -0,0 +1,8 @@
+checks:
+  - name: ns-per-op regression
+    description: Flag if NsPerOp regresses by more than 10%
+    package: ""
+    benchmarks: []
+    diff: (current.NsPerOp - base.NsPerOp) / base.NsPerOp * 100
+    thresholds:
+      max: 10