action.yml

# PR Quality Gate: protect your repo against AI slop.
#
# Checks derived from 76 PR outcomes across 38 repos.
# Each check corresponds to a pattern that predicted merge vs. closure.
# Any warning auto-closes the PR with a comment explaining why.
# Three-strike policy: 3+ gate closures from the same author = auto-ban without checks.
#
# Drop-in: works without any API keys. Optional Anthropic key improves
# the description depth check (Haiku, ~$0.001/PR).

name: 'PR Quality Gate'
description: 'Automatically block and ban AI slop PRs. Six checks, three-strike ban, zero config.'
inputs:
  github-token:
    description: 'GitHub token for API access'
    required: true
    default: ${{ github.token }}
  anthropic-api-key:
    description: 'Anthropic API key for description depth analysis (uses Haiku, ~$0.001/PR). Optional. Falls back to heuristics without it.'
    required: false
    default: ''
runs:
  using: 'composite'
  steps:
    - name: Run quality checks
      shell: bash
      env:
        GITHUB_TOKEN: ${{ inputs.github-token }}
        ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }}
        PR_NUMBER: ${{ github.event.pull_request.number }}
        PR_TITLE: ${{ github.event.pull_request.title }}
        PR_AUTHOR: ${{ github.event.pull_request.user.login }}
        PR_BASE: ${{ github.event.pull_request.base.ref }}
        REPO: ${{ github.repository }}
        COMMITS: ${{ github.event.pull_request.commits }}
        GITHUB_EVENT_PATH: ${{ github.event_path }}
      run: |
        set -euo pipefail

        PR_BODY=$(jq -r '.pull_request.body // ""' "$GITHUB_EVENT_PATH")

        RESULTS=""
        PASS_COUNT=0
        WARN_COUNT=0

        # 0. Three-strike ban: 3+ gate closures from same author across the org = auto-ban
        ORG=$(echo "$REPO" | cut -d'/' -f1)
        GATE_CLOSED_PRS=$(gh api graphql -f query="{ search(query: \"is:pr is:closed is:unmerged author:${PR_AUTHOR} org:${ORG}\", type: ISSUE, first: 100) { nodes { ... on PullRequest { number repository { nameWithOwner } } } } }" \
          --jq '.data.search.nodes[] | "\(.repository.nameWithOwner) \(.number)"' 2>/dev/null || true)

        PRIOR_STRIKES=0
        while IFS=' ' read -r strike_repo strike_number; do
          [ -z "$strike_repo" ] && continue
          HAS_GATE_COMMENT=$(gh api "repos/${strike_repo}/issues/${strike_number}/comments?per_page=10" \
            --jq '[.[] | select(.body | startswith("### PR Quality Gate"))] | length' 2>/dev/null || echo "0")
          if [ "$HAS_GATE_COMMENT" -gt 0 ]; then
            PRIOR_STRIKES=$((PRIOR_STRIKES + 1))
          fi
          if [ "$PRIOR_STRIKES" -ge 3 ]; then
            break
          fi
        done <<< "$GATE_CLOSED_PRS"

        if [ "$PRIOR_STRIKES" -ge 3 ]; then
          BAN_COMMENT=$(cat <<'BANEOF'
        ### PR Quality Gate: auto-closed

        This author has had 3+ PRs closed by quality gate checks across this org. Future PRs will be auto-closed without review.

        If you believe this is an error, open an issue to discuss.
        BANEOF
        )
          gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" -f body="$BAN_COMMENT" > /dev/null 2>&1
          gh api "repos/${REPO}/pulls/${PR_NUMBER}" -X PATCH -f state=closed > /dev/null 2>&1
          echo "PR Quality Gate: BANNED (${PRIOR_STRIKES} prior strikes)"
          exit 0
        fi

        # Standing check: has this author merged PRs to this repo before?
        PRIOR_MERGES=$(gh api "repos/${REPO}/pulls?state=closed&creator=${PR_AUTHOR}&per_page=100" --jq '[.[] | select(.merged_at != null)] | length' 2>/dev/null || echo "0")
        HAS_STANDING=false
        if [ "$PRIOR_MERGES" -ge 3 ]; then
          HAS_STANDING=true
        fi

        add_result() {
          local check="$1" status="$2" detail="$3"
          detail=$(echo "$detail" | sed 's/|/\\|/g')
          RESULTS="${RESULTS}| ${check} | ${status} | ${detail} |\n"
          case "$status" in
            pass) PASS_COUNT=$((PASS_COUNT + 1)) ;;
            warn) WARN_COUNT=$((WARN_COUNT + 1)) ;;
          esac
        }

        # 1a. Em-dash in title: near-dispositive. A 60-char human-typed title
        # does not naturally produce U+2014. It's an LLM output that wasn't
        # proofread. Treated as its own check so the verdict comment names
        # the strong signal separately from the weaker body-text check.
        if printf '%s' "$PR_TITLE" | grep -qE $'\xe2\x80\x94|\xe2\x80\x93\xe2\x80\x93|--'; then
          add_result "Em dash in title" "warn" "Em dash, double en-dash, or double hyphen in PR title. Human-typed titles do not produce U+2014; this is an unproofread LLM output."
        else
          add_result "Em dash in title" "pass" "No em dash in title"
        fi

        # 1b. Em-dash in body: weaker signal. Pasted source material can carry
        # legitimate em-dashes. Still flagged, still closes for first-timers.
        if printf '%s' "$PR_BODY" | grep -qE $'\xe2\x80\x94|\xe2\x80\x93\xe2\x80\x93| -- |--[a-zA-Z]|[a-zA-Z]--'; then
          add_result "Em dash in body" "warn" "Em dash, double en-dash, or double hyphen in PR body. Common in AI-generated prose."
        else
          add_result "Em dash in body" "pass" "No em dash in body"
        fi

        # 2. Description depth
        BODY_LEN=${#PR_BODY}
        if [ "$BODY_LEN" -lt 50 ]; then
          add_result "Description" "warn" "PR body is ${BODY_LEN} chars. Describe *why* this change is needed."
        elif [ -n "$ANTHROPIC_API_KEY" ]; then
          DIFF_SUMMARY=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.additions, .deletions, .changed_files' 2>/dev/null | tr '\n' '/' || echo "?/?/?")
          LLM_VERDICT=$(curl -s https://api.anthropic.com/v1/messages \
            -H "content-type: application/json" \
            -H "x-api-key: ${ANTHROPIC_API_KEY}" \
            -H "anthropic-version: 2023-06-01" \
            -d "$(jq -n \
              --arg body "$PR_BODY" \
              --arg title "$PR_TITLE" \
              --arg diff "$DIFF_SUMMARY" \
              '{
                model: "claude-haiku-4-5-20251001",
                max_tokens: 150,
                messages: [{
                  role: "user",
                  content: ("PR title: " + $title + "\nPR body: " + $body + "\nDiff stats: " + $diff + "\n\nDoes this PR description explain WHY the change is correct (root cause, design rationale), or does it only describe WHAT changed (restating the diff)? Answer exactly: WHY or WHAT, then one sentence explaining your judgment.")
                }]
              }')" 2>/dev/null | jq -r '.content[0].text // "ERROR"' 2>/dev/null || echo "ERROR")

          if echo "$LLM_VERDICT" | grep -qi "^WHAT"; then
            REASON=$(echo "$LLM_VERDICT" | head -1 | cut -c6-120)
            add_result "Description" "warn" "Describes *what* changed, not *why*. ${REASON}"
          elif echo "$LLM_VERDICT" | grep -qi "^WHY"; then
            add_result "Description" "pass" "Explains why"
          else
            add_result "Description" "pass" "Description present (LLM check inconclusive)"
          fi
        else
          # Heuristic fallback: no API key
          WHAT_SIGNALS=0
          if echo "$PR_BODY" | grep -qi "$(echo "$PR_TITLE" | sed 's/[^a-zA-Z ]//g' | head -c 30)"; then
            WHAT_SIGNALS=$((WHAT_SIGNALS + 1))
          fi
          if echo "$PR_BODY" | grep -qiE 'this (PR|pull request|change|commit) (adds|removes|updates|fixes|changes|modifies|implements)' && \
             ! echo "$PR_BODY" | grep -qiE 'because|root cause|the problem|the issue|the bug|rationale|the reason|this happens when'; then
            WHAT_SIGNALS=$((WHAT_SIGNALS + 1))
          fi
          PROSE_LINES=$(echo "$PR_BODY" | grep -cvE '^\s*[-*]|^\s*$|^#|^\|' || true)
          if [ "$PROSE_LINES" -eq 0 ] && [ "$BODY_LEN" -gt 50 ]; then
            WHAT_SIGNALS=$((WHAT_SIGNALS + 1))
          fi
          if [ "$WHAT_SIGNALS" -ge 2 ]; then
            add_result "Description" "warn" "Describes *what* changed, not *why*. Add root cause or rationale."
          else
            add_result "Description" "pass" "Description present"
          fi
        fi

        # 3. CONTRIBUTING.md compliance
        CONTRIBUTING=""
        for path in CONTRIBUTING.md .github/CONTRIBUTING.md; do
          CONTENT=$(gh api "repos/${REPO}/contents/${path}" --jq '.content' 2>/dev/null | base64 -d 2>/dev/null || true)
          if [ -n "$CONTENT" ]; then
            CONTRIBUTING="$CONTENT"
            break
          fi
        done

        if [ -n "$CONTRIBUTING" ]; then
          ISSUES=""

          TARGET_BRANCH=$(echo "$CONTRIBUTING" | grep -oiE '(submit|target|base|pr).{0,30}(main|master|develop|dev)' | head -1 || true)
          if [ -n "$TARGET_BRANCH" ]; then
            EXPECTED=$(echo "$TARGET_BRANCH" | grep -oE '(main|master|develop|dev)' | tail -1)
            if [ -n "$EXPECTED" ] && [ "$PR_BASE" != "$EXPECTED" ]; then
              ISSUES="${ISSUES}Target branch should be ${EXPECTED} (got ${PR_BASE}). "
            fi
          fi

          MAX_COMMITS=$(echo "$CONTRIBUTING" | grep -oiE '(max|limit|at most|no more than)[^0-9]{0,10}([0-9]+)[^0-9]{0,10}commit' | grep -oE '[0-9]+' | head -1 || true)
          if [ -n "$MAX_COMMITS" ] && [ "$COMMITS" -gt "$MAX_COMMITS" ]; then
            ISSUES="${ISSUES}${COMMITS} commits exceeds limit of ${MAX_COMMITS}. "
          fi

          if echo "$CONTRIBUTING" | grep -qiE 'AI|LLM|generative|copilot|chatgpt|ai.generated|ai.slop'; then
            AI_LINE=$(echo "$CONTRIBUTING" | grep -iE 'AI|LLM|generative|copilot|chatgpt|ai.generated|ai.slop' | head -1 | cut -c1-120)
            ISSUES="${ISSUES}AI policy detected: ${AI_LINE} "
          fi

          if [ -n "$ISSUES" ]; then
            add_result "CONTRIBUTING" "warn" "$ISSUES"
          else
            add_result "CONTRIBUTING" "pass" "Follows repo guidelines"
          fi
        else
          add_result "CONTRIBUTING" "pass" "No CONTRIBUTING.md found"
        fi

        # 3b. AGENTS.md compliance (AI/agent-specific repo policy)
        # Many repos now publish AGENTS.md alongside CONTRIBUTING.md to state
        # explicit rules for AI/agent contributions: blanket prohibitions,
        # required title markers (e.g. trailing 🤖), required disclosure fields.
        # Closures observed: openbao#3067 (missing 🤖 marker per their AGENTS.md),
        # kanidm#4339 (resubmission without reading AGENTS.md).
        AGENTS=""
        for path in AGENTS.md .github/AGENTS.md; do
          CONTENT=$(gh api "repos/${REPO}/contents/${path}" --jq '.content' 2>/dev/null | base64 -d 2>/dev/null || true)
          if [ -n "$CONTENT" ]; then
            AGENTS="$CONTENT"
            break
          fi
        done

        if [ -n "$AGENTS" ]; then
          AGENTS_ISSUES=""

          # Blanket prohibition: any phrasing that rejects AI-generated work outright.
          if echo "$AGENTS" | grep -qiE '(reject|prohibit|forbid|not? accept|do not (submit|open)|no (ai|llm|generative)).{0,80}(ai|llm|generative|generated|copilot|chatgpt|claude)'; then
            PROHIB_LINE=$(echo "$AGENTS" | grep -iE '(reject|prohibit|forbid|not? accept|do not (submit|open)|no (ai|llm|generative)).{0,80}(ai|llm|generative|generated|copilot|chatgpt|claude)' | head -1 | cut -c1-140)
            AGENTS_ISSUES="${AGENTS_ISSUES}AGENTS.md prohibits AI contributions: ${PROHIB_LINE} "
          fi

          # Required title marker: AGENTS.md often mandates a specific emoji or token in PR title.
          # Detect any emoji mentioned alongside "title" within ~80 chars.
          TITLE_MARKER_LINE=$(echo "$AGENTS" | grep -iE 'title.{0,80}(emoji|marker|🤖|✨|🧠|prefix|suffix|tag)|[(]?:robot|🤖|✨|🧠[)]?.{0,40}(title|pr.{0,5}title|end of)' | head -1 | cut -c1-140)
          if [ -n "$TITLE_MARKER_LINE" ]; then
            # Extract any non-ASCII char from the line as a candidate marker.
            CANDIDATE=$(printf '%s' "$TITLE_MARKER_LINE" | python3 -c "import sys,re; print(''.join(re.findall(r'[\U0001F300-\U0001FAFF✀-➿]', sys.stdin.read()))[:5])" 2>/dev/null || true)
            if [ -n "$CANDIDATE" ]; then
              if ! printf '%s' "$PR_TITLE" | grep -qF "$CANDIDATE"; then
                AGENTS_ISSUES="${AGENTS_ISSUES}AGENTS.md appears to require '${CANDIDATE}' marker in PR title (not found). "
              fi
            else
              AGENTS_ISSUES="${AGENTS_ISSUES}AGENTS.md mentions title marker requirement; verify PR title compliance: ${TITLE_MARKER_LINE} "
            fi
          fi

          # Required disclosure: if AGENTS.md asks contributors to disclose AI use, check the body.
          if echo "$AGENTS" | grep -qiE '(disclos|declar|state|note|mark).{0,60}(ai|llm|generative|copilot|chatgpt|assist)'; then
            if ! echo "$PR_BODY" | grep -qiE 'ai|llm|copilot|claude|chatgpt|assist|generated'; then
              AGENTS_ISSUES="${AGENTS_ISSUES}AGENTS.md may require AI-use disclosure; PR body has none. "
            fi
          fi

          if [ -n "$AGENTS_ISSUES" ]; then
            add_result "AGENTS" "warn" "$AGENTS_ISSUES"
          else
            add_result "AGENTS" "pass" "Follows AGENTS.md"
          fi
        else
          add_result "AGENTS" "pass" "No AGENTS.md found"
        fi

        # 4. Test presence
        DIFF_FILES=$(gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/files" --jq '.[].filename' 2>/dev/null || true)
        HAS_SOURCE=false
        HAS_TEST=false

        while IFS= read -r file; do
          [ -z "$file" ] && continue
          if echo "$file" | grep -qiE 'test|spec|_test\.|\.test\.|tests/'; then
            HAS_TEST=true
          elif echo "$file" | grep -qiE '\.(py|rs|go|ts|js|java|cpp|c|rb|swift|kt)$'; then
            HAS_SOURCE=true
          fi
        done <<< "$DIFF_FILES"

        IS_FIX=$(printf '%s' "$PR_TITLE" | grep -qiE '^fix|bug|patch|hotfix' && echo true || echo false)

        if [ "$HAS_SOURCE" = true ] && [ "$HAS_TEST" = false ]; then
          if [ "$IS_FIX" = true ]; then
            add_result "Tests" "warn" "Bug fix changes source files but adds no tests. A failing test on main proves the bug exists."
          else
            add_result "Tests" "warn" "Source files changed but no test files modified"
          fi
        else
          add_result "Tests" "pass" "Tests present or no source changes"
        fi

        # 5. Contributor velocity (org-scoped)
        # The signal we care about is "this author is flooding this maintainer
        # surface", not "this author is generally active on GitHub". A
        # first-timer to this repo who happens to open 12 PRs across other
        # repos is not the same as 12 PRs to repos sharing this maintainer
        # team. Scope to org to keep the two axes independent. Standing
        # (PRIOR_MERGES > 0) further exempts known-good contributors from
        # the velocity warn, since their pattern is already vetted here.
        if [[ "$OSTYPE" == "linux-gnu"* ]]; then
          SINCE=$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ)
        else
          SINCE=$(date -u -v-24H +%Y-%m-%dT%H:%M:%SZ)
        fi

        RECENT_PRS=$(gh api graphql -f query="{ search(query: \"is:pr author:${PR_AUTHOR} org:${ORG} created:>${SINCE}\", type: ISSUE, first: 1) { issueCount } }" --jq '.data.search.issueCount' 2>/dev/null || echo "0")

        if [ "${PRIOR_MERGES:-0}" -gt 0 ]; then
          add_result "Velocity" "pass" "${RECENT_PRS} PRs in last 24h to ${ORG} (standing here exempts)"
        elif [ "$RECENT_PRS" -gt 10 ]; then
          add_result "Velocity" "warn" "${RECENT_PRS} PRs opened in last 24h to ${ORG}. High-volume pattern on this maintainer surface."
        elif [ "$RECENT_PRS" -gt 5 ]; then
          add_result "Velocity" "warn" "${RECENT_PRS} PRs opened in last 24h to ${ORG}"
        else
          add_result "Velocity" "pass" "${RECENT_PRS} PRs in last 24h to ${ORG}"
        fi

        # 7. Emoji spam in title/body
        EMOJI_COUNT=$(printf '%s' "$PR_TITLE$PR_BODY" | python3 -c "
import sys, re
text = sys.stdin.read()
emojis = re.findall(r'[\U0001F300-\U0001F9FF\U00002702-\U000027B0\U0001FA00-\U0001FA6F\U0001FA70-\U0001FAFF]', text)
print(len(emojis))" 2>/dev/null || echo "0")

        if [ "$EMOJI_COUNT" -gt 3 ]; then
          add_result "Emoji" "warn" "${EMOJI_COUNT} emojis in PR text"
        else
          add_result "Emoji" "pass" "${EMOJI_COUNT} emojis"
        fi

        # 8. Account age (< 7 days is suspicious, but not disqualifying alone)
        ACCOUNT_CREATED=$(gh api "users/${PR_AUTHOR}" --jq '.created_at' 2>/dev/null || echo "")
        if [ -n "$ACCOUNT_CREATED" ]; then
          ACCOUNT_AGE_DAYS=$(python3 -c "
from datetime import datetime, timezone
created = datetime.fromisoformat('${ACCOUNT_CREATED}'.replace('Z','+00:00'))
now = datetime.now(timezone.utc)
print((now - created).days)" 2>/dev/null || echo "999")
          if [ "$ACCOUNT_AGE_DAYS" -lt 7 ]; then
            add_result "Account age" "warn" "Account is ${ACCOUNT_AGE_DAYS} days old"
          else
            add_result "Account age" "pass" "Account is ${ACCOUNT_AGE_DAYS} days old"
          fi
        fi

        # Build the comment
        HEADER="### PR Quality Gate"
        if [ "$WARN_COUNT" -gt 0 ]; then
          HEADER="${HEADER} (${WARN_COUNT} warning(s))"
        fi

        COMMENT=$(cat <<COMMENTEOF
        ${HEADER}

        | Check | Status | Detail |
        |-------|--------|--------|
        $(printf '%b' "$RESULTS")
        <details>
        <summary>About this check</summary>

        Each check corresponds to a pattern that predicted closure in [76 PR outcomes](https://github.com/kimjune01/sweep) across 38 repos. Authors with 3+ gate closures are auto-closed on sight.

        [Protect your repo against AI slop](https://github.com/kimjune01/sweep/blob/master/action.yml)
        </details>
        COMMENTEOF
        )

        # Post or update comment
        EXISTING=$(gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments" --jq '.[] | select(.body | startswith("### PR Quality Gate")) | .id' 2>/dev/null | head -1 || true)

        if [ -n "$EXISTING" ]; then
          gh api "repos/${REPO}/issues/comments/${EXISTING}" -X PATCH -f body="$COMMENT" > /dev/null 2>&1
        else
          gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" -f body="$COMMENT" > /dev/null 2>&1
        fi

        # Standing determines action: first-timers get closed, established contributors get warned
        if [ "$WARN_COUNT" -gt 0 ]; then
          if [ "$HAS_STANDING" = true ]; then
            echo "PR Quality Gate: ${WARN_COUNT} warning(s), advisory (${PRIOR_MERGES} prior merges)"
          else
            gh api "repos/${REPO}/pulls/${PR_NUMBER}" -X PATCH -f state=closed > /dev/null 2>&1
            echo "PR Quality Gate: CLOSED (${WARN_COUNT} warning(s), first-time contributor)"
          fi
        else
          echo "PR Quality Gate: PASSED (${PASS_COUNT} checks)"
        fi