From 2cb9b958e94a2d3b3d59c0cbc81a9129ab513c11 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:40:20 -0500 Subject: [PATCH 1/2] feat(klaud): status labels, close-to-stop, event wakes and clearer outcomes - Overlap review indexes only open Klaud PRs; other PRs never block a family. - Remove klaud-abort. A Klaud PR closed unmerged by anyone else stops the candidate, and the planner leaves its family alone for 7 days. - Display-only status labels (klaud-in-progress, klaud-parked, klaud-finished, klaud-infra, klaud-incompatible, klaud-error) set in code with the workflow token so label events start no sweep runs. - Rename outcomes: readiness-blocked -> missing-assets, uncertain -> inconclusive, unexpected-error -> no-outcome, capacity-deferred -> no-capacity, aborted -> closed-by-maintainer. Older records read back under the new names. - missing-assets and weights-missing cool down 7 days instead of 24 hours. - Planner waves every 4 hours. klaud-wake.yml starts a recovery-only pass as soon as a Klaud-owned run ends. - Finishing a maintainer-closed ready PR no longer tries to redraft it. --- .github/klaud-candidate-prompt.md | 13 +- .github/workflows/klaud-candidate.yml | 2 + .github/workflows/klaud-plan.yml | 46 ++- .github/workflows/klaud-wake.yml | 30 ++ inferencex-e2e/infx/klaud/__main__.py | 171 ++++----- inferencex-e2e/infx/klaud/github.py | 14 +- inferencex-e2e/infx/klaud/lifecycle.py | 148 +++++++- inferencex-e2e/infx/klaud/models.py | 42 ++- inferencex-e2e/infx/klaud/reporting.py | 12 +- .../infx/tests/klaud/test_klaud_github.py | 334 ++++++++++++------ 10 files changed, 539 insertions(+), 273 deletions(-) create mode 100644 .github/workflows/klaud-wake.yml diff --git a/.github/klaud-candidate-prompt.md b/.github/klaud-candidate-prompt.md index 2d7708f23e..522f0cc486 100644 --- a/.github/klaud-candidate-prompt.md +++ b/.github/klaud-candidate-prompt.md @@ -14,11 +14,12 @@ Treat candidate.source only as a published benchmark baseline. It is not authori for current repository state, upstream releases or image compatibility. Resolve the exact current family/image from the checkout, then independently inspect the actual bundled engine source, official upstream releases and available images; stop if retired, ambiguous, updated -or owned. Normalize registry `/` versus enroot `#` spelling when comparing the same image. +or owned by another Klaud Cold PR. Open PRs on other branches never block a candidate. +Normalize registry `/` versus enroot `#` spelling when comparing the same image. Use the canonical generator, inferencex-e2e/configs/runners.yaml and public OpenAPI/repository mappings for all points, exact cluster routes and physical node demand; never invent aliases or substitute sibling clusters. The planner already owns the family claim; leave claim refs -to the lifecycle helper. Recheck all open PRs before atomically creating the supplied +to the lifecycle helper. Recheck open Klaud Cold PRs (`klaud/auto-*`) before atomically creating the supplied exact candidate branch at its base SHA. An existing branch means stop, not takeover, unless KLAUD_RESUME=true. Create one draft PR after a real change, titled `[Klaud Cold] English / 简体中文`. @@ -40,7 +41,7 @@ Check changed defaults/semantics, renamed/removed options and hardware/CUDA/ROCm Release notes alone are insufficient; retain source links and provenance uncertainty in attempt comments. Pin mutable image tags by digest. Before GPU dispatch, inspect the actual launcher/recipe's effective image, staged weights/config/tokenizer and mount paths. -Missing assets are readiness-blocked; never map to an older image or different weights. +Missing assets mean outcome missing-assets; never map to an older image or different weights. When the model checkpoint itself is not staged on the target cluster, add `reason-code: weights-missing` so the planner pauses that model and precision on that hardware. Check the exact selected launch path for existing engine patches as well as proposed ones. @@ -56,9 +57,9 @@ overlays, monkey patches or forked/rebuilt engine wheels, including pre-existing If the shipped stack needs one, report incompatible. Preserve nodes:N, MTP chat templates, workspace permissions and artifact contracts; follow the CODEOWNER checklist. -Before every PR/branch mutation or cancellation, re-read labels. `klaud-abort` means a +Before every PR/branch mutation or cancellation, re-read the PR. If someone else closed it, a maintainer stopped this candidate: stop immediately without further changes; the job's -cleanup cancels your runs and closes the PR. Never add or remove that label yourself. +cleanup cancels your runs and deletes the branch. Never reopen it. The planner checks capacity once, when it selects a candidate; never check or defer for capacity inside this session. Klaud runs at the lowest scheduler priority: dispatch every @@ -187,7 +188,7 @@ other outcome or any run except the verified final sweep. Otherwise finish reports the failure/deferral, cancels owned work, confirms every job terminal, removes sweep labels, drafts/closes the PR, deletes the candidate branch and releases the family claim so a later wave may retry it. Pending cleanup means wait and retry finish. -Uncertainty is not incompatibility. Open PRs block the family until cleanup completes. +Uncertainty is not incompatibility. Your open PR blocks the family until cleanup completes. Without an owned PR, report without a placeholder or deleting someone else's claim. After finish succeeds, read verified $KLAUD_EVIDENCE/session-end.json and invoke diff --git a/.github/workflows/klaud-candidate.yml b/.github/workflows/klaud-candidate.yml index 41ce987e05..1dcdaec2e2 100644 --- a/.github/workflows/klaud-candidate.yml +++ b/.github/workflows/klaud-candidate.yml @@ -29,6 +29,7 @@ permissions: contents: read actions: read # Read workflow runs and artifacts. pull-requests: read # Read PR metadata. + issues: write # Set display-only Klaud status labels on its PR. jobs: claude: @@ -154,6 +155,7 @@ jobs: if: always() && steps.claude.outcome != 'skipped' env: GH_TOKEN: ${{ secrets.AGENT_PAT }} + KLAUD_STATUS_TOKEN: ${{ github.token }} run: uv run --project "$INFERENCEX_TOOLING_PROJECT" --locked python -P -m infx.klaud recover-current - name: Save sanitized candidate diagnostics id: diagnostics diff --git a/.github/workflows/klaud-plan.yml b/.github/workflows/klaud-plan.yml index 181e2bf5c9..bae53f5aa1 100644 --- a/.github/workflows/klaud-plan.yml +++ b/.github/workflows/klaud-plan.yml @@ -2,9 +2,15 @@ name: Klaud Cold auto-sweep run-name: >- ${{ inputs.resume-parent && format('Klaud Cold resume {0} {1} session {2}', inputs.resume-parent, inputs.resume-candidate, inputs.resume-session) - || github.event.schedule == '15,45 * * * *' && 'Klaud Cold wake' + || (github.event.schedule == '15,45 * * * *' || inputs.wake) && 'Klaud Cold wake' || 'Klaud Cold auto-sweep' }} +# klaud-wake.yml dispatches a wake whenever a Klaud-owned run ends; one runs and one waits, so a +# burst of endings collapses into a single recovery pass. Every other run is its own group. +concurrency: + group: ${{ inputs.wake && 'klaud-wake' || github.run_id }} + cancel-in-progress: false + on: # zizmor: ignore[concurrency-limits] workflow_dispatch: inputs: @@ -20,8 +26,12 @@ on: # zizmor: ignore[concurrency-limits] description: Number of the resumed session type: string default: '' + wake: + description: Recovery-only pass, dispatched when a Klaud-owned run ends + type: boolean + default: false schedule: - - cron: '0 */3 * * *' + - cron: '0 */4 * * *' - cron: '15,45 * * * *' permissions: @@ -41,6 +51,7 @@ jobs: contents: read actions: read # Read workflow runs and artifacts. pull-requests: read # Read PR metadata. + issues: write # Set display-only Klaud status labels on its PRs. outputs: tooling-ref: ${{ steps.checkout.outputs.commit }} blocked-families: ${{ steps.recover.outputs.blocked-families }} @@ -81,6 +92,7 @@ jobs: id: recover env: GH_TOKEN: ${{ secrets.AGENT_PAT }} # zizmor: ignore[secrets-outside-env] + KLAUD_STATUS_TOKEN: ${{ github.token }} run: | uv run --project "$INFERENCEX_PYTHON_PROJECT" --locked python -m infx.klaud recover --max-sessions "$MAX_CANDIDATES_PER_RUN" echo "blocked-families=$(jq -c . "$RUNNER_TEMP/klaud-recovery.json")" >> "$GITHUB_OUTPUT" @@ -88,17 +100,16 @@ jobs: review: name: Review candidate overlap needs: recover - if: github.event.schedule != '15,45 * * * *' + if: github.event.schedule != '15,45 * * * *' && !inputs.wake runs-on: ubuntu-latest permissions: contents: read actions: read # Read workflow runs and artifacts. pull-requests: read # Read PR metadata. - issues: read # Read klaud-abort labels and Klaud outcome comments. + issues: read # Read who closed Klaud PRs and Klaud outcome comments. env: REVIEW_BATCH_SIZE: '64' CANDIDATE_COOLDOWN_HOURS: '24' - STALE_PR_DAYS: '7' outputs: review: ${{ steps.review.outputs.structured_output }} outcome: ${{ steps.review.outcome }} @@ -135,7 +146,6 @@ jobs: --directory "$RUNNER_TEMP/klaud" --review-batch-size "$REVIEW_BATCH_SIZE" --cooldown-hours "$CANDIDATE_COOLDOWN_HOURS" - --stale-pr-days "$STALE_PR_DAYS" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: klaud-review-input @@ -180,9 +190,9 @@ jobs: --json-schema '${{ steps.prepare.outputs.review_schema }}' prompt: | Use Read on candidates.json and open-prs.json under ${{ runner.temp }}/klaud/. - Review candidates in the supplied shuffled order against current recipes and the open-PR index, - including drafts and arbitrary branches. The index omits PRs idle for ${{ env.STALE_PR_DAYS }} days - (other than Klaud candidate PRs); those no longer own any family. Changed-file paths are already fetched; + Review candidates in the supplied shuffled order against current recipes and the open-PR index. + The index lists only open Klaud candidate PRs (klaud/auto-* branches). Other PRs never own a + family: ignore them, even when they edit the same image or recipe. Changed-file paths are already fetched; inspect relevant PR bodies/diffs with direct gh pr view/diff commands. For API reads, use gh api --method GET. Use Read/Glob/Grep for local inspection, not shell file commands or Python. Run one allowed gh/git command per Bash call, without cd/env wrappers, @@ -198,12 +208,10 @@ jobs: a different family. For every proceed decision, resolve baseline-model to the exact display model accepted by the public benchmark API using the supplied OpenAPI schema. Target clusters and their capacity are resolved and checked in code, not by this review. - Mark duplicate when an open PR owns that family's image refresh or overlapping + Mark duplicate when an indexed Klaud PR owns that family's image refresh or overlapping compatibility work, even with another image tag; unrelated work on the same hardware - is not a duplicate. A cross-cutting PR (launcher, telemetry, refactor or shared-schema - work spanning many families) is a duplicate only if its diff changes this family's - image or the exact engine flags/environment a refresh edits; touching the same recipe - file for unrelated settings is not overlap. Return the supplied schema: candidate-id, decision + is not a duplicate; touching the same recipe file for unrelated settings is not overlap. + Return the supplied schema: candidate-id, decision (proceed/duplicate/uncertain), family (null if unresolved), pull-requests (overlapping PR numbers), baseline-model (required for proceed; otherwise null), reason (brief evidence). Missing evidence or ambiguity means uncertain. @@ -315,6 +323,11 @@ jobs: fail-fast: false matrix: candidate: ${{ fromJSON(needs.select.outputs.candidates) }} + permissions: + contents: read + actions: read # Read workflow runs and artifacts. + pull-requests: read # Read PR metadata. + issues: write # Set display-only Klaud status labels on its PR. uses: $/.github/workflows/klaud-candidate.yml with: candidate: ${{ matrix.candidate }} @@ -329,6 +342,11 @@ jobs: if: >- github.ref == 'refs/heads/main' && github.run_attempt == 1 && inputs.resume-parent != '' && github.actor == 'Klaud-Cold' + permissions: + contents: read + actions: read # Read workflow runs and artifacts. + pull-requests: read # Read PR metadata. + issues: write # Set display-only Klaud status labels on its PR. uses: $/.github/workflows/klaud-candidate.yml with: candidate: ${{ inputs.resume-candidate }} diff --git a/.github/workflows/klaud-wake.yml b/.github/workflows/klaud-wake.yml new file mode 100644 index 0000000000..adf373ff42 --- /dev/null +++ b/.github/workflows/klaud-wake.yml @@ -0,0 +1,30 @@ +name: Klaud Cold wake on run end +run-name: Klaud Cold wake on run end - ${{ github.event.workflow_run.display_title }} + +# A parked Klaud session resumes only after recovery sees its owned runs end. Rather than wait +# for the next :15/:45 wake, start a recovery-only planner pass as soon as a Klaud-owned +# targeted run (e2e Test - klaud-*) or PR sweep (klaud/auto-* branch) completes. No checkout and +# no event data reaches a shell; the event only gates the job. +on: # zizmor: ignore[concurrency-limits] + workflow_run: # zizmor: ignore[dangerous-triggers] runs no code from the triggering run + workflows: [End-to-End Tests, Run Sweep] + types: [completed] + +permissions: {} + +jobs: + wake: + name: Wake Klaud recovery + if: >- + github.event.workflow_run.head_repository.full_name == github.repository + && github.event.workflow_run.conclusion != 'skipped' + && (startsWith(github.event.workflow_run.display_title, 'e2e Test - klaud-') + || startsWith(github.event.workflow_run.head_branch, 'klaud/auto-')) + runs-on: ubuntu-latest + permissions: + actions: write # Dispatch the planner's recovery-only wake. + steps: + - name: Dispatch a recovery-only planner pass + env: + GH_TOKEN: ${{ github.token }} + run: gh workflow run klaud-plan.yml --repo "$GITHUB_REPOSITORY" --ref main -f wake=true diff --git a/inferencex-e2e/infx/klaud/__main__.py b/inferencex-e2e/infx/klaud/__main__.py index 7628e0ea01..f327281af1 100644 --- a/inferencex-e2e/infx/klaud/__main__.py +++ b/inferencex-e2e/infx/klaud/__main__.py @@ -28,7 +28,6 @@ read as github_read, ) from .models import ( - ABORT_LABEL, CandidateOutcome, OwnedCandidate, Ownership, @@ -44,7 +43,10 @@ from infx.clusters import RunnerInventory PRODUCERS = "producers.json" -RETRY_NEXT_WAVE = frozenset({"capacity-deferred", "agent-not-started", "aborted"}) +RETRY_NEXT_WAVE = frozenset({"no-capacity", "agent-not-started"}) +MAINTAINER_CLOSE_HOLD_DAYS = 7 +INFRA_OUTCOMES = frozenset({"missing-assets", "weights-missing"}) +INFRA_COOLDOWN_DAYS = 7 def observation_key(row: dict) -> tuple: @@ -214,15 +216,19 @@ def weights_key(source: dict) -> tuple: def recent_candidate_outcomes( repository: str, cooldown_hours: int ) -> tuple[dict[str, str | None], set[tuple]]: - """Return each candidate's latest outcome within ``cooldown_hours`` and missing-weights keys. + """Return each candidate's latest outcome still cooling down and missing-weights keys. - Resumed sessions record outcomes in later waves, so history reaches MAX_SESSIONS waves back. - The newest session artifact counts. Unreadable history means no cooldown. + Infrastructure outcomes (INFRA_OUTCOMES) cool down for INFRA_COOLDOWN_DAYS, others for + ``cooldown_hours``. Resumed sessions record outcomes in later waves, so history reaches + MAX_SESSIONS waves back. The newest session artifact counts. Unreadable history means no + cooldown. """ from .lifecycle import MAX_SESSIONS, WAKE_TITLE, WAVE_HOURS, resume_parent - cutoff = datetime.now(UTC) - timedelta(hours=cooldown_hours) - since = cutoff - timedelta(hours=MAX_SESSIONS * WAVE_HOURS) + now = datetime.now(UTC) + cutoff = now - timedelta(hours=cooldown_hours) + window = min(cutoff, now - timedelta(days=INFRA_COOLDOWN_DAYS)) + since = window - timedelta(hours=MAX_SESSIONS * WAVE_HOURS) try: runs = github_items( repository, @@ -261,7 +267,7 @@ def recent_candidate_outcomes( if ( not match or artifact.get("expired", True) - or utc(artifact["created_at"]) < cutoff + or utc(artifact["created_at"]) < window ): continue if match[1] not in latest or artifact["id"] > latest[match[1]][0]["id"]: @@ -271,8 +277,11 @@ def recent_candidate_outcomes( result: dict[str, str | None] = {} blocked: dict[int, set[str]] = {} for candidate, (artifact, run_id) in latest.items(): - result[candidate] = recorded_outcome(repository, artifact) - if result[candidate] == "weights-missing": + outcome = recorded_outcome(repository, artifact) + if outcome not in INFRA_OUTCOMES and utc(artifact["created_at"]) < cutoff: + continue + result[candidate] = outcome + if outcome == "weights-missing": blocked.setdefault(plans[run_id], set()).add(candidate) missing_weights = set() for plan_run, candidates in blocked.items(): @@ -333,10 +342,7 @@ def recorded_outcome(repository: str, artifact: dict) -> str | None: def order_by_cooldown(candidates: list[dict], recent: dict[str, str | None]) -> list[dict]: - """Return ``candidates`` never tried recently, then those whose recent outcome retries now. - - An abort is lifted by removing its label, so it never delays the next attempt by itself. - """ + """Return ``candidates`` never tried recently, then those whose recent outcome retries now.""" fresh = [candidate for candidate in candidates if candidate["id"] not in recent] deferred = [ candidate for candidate in candidates if recent.get(candidate["id"], "") in RETRY_NEXT_WAVE @@ -344,77 +350,50 @@ def order_by_cooldown(candidates: list[dict], recent: dict[str, str | None]) -> return fresh + deferred -def aborted_candidates(repository: str) -> set[str]: - """Candidates whose Klaud PR carries klaud-abort; the planner never retries them. +def maintainer_closed_families(repository: str, now: datetime) -> set[str]: + """Family prefixes of Klaud PRs someone else closed unmerged in the last hold window. - Removing the label returns the candidate to the pool at the planner's next scan. + Closing a Klaud PR stops that candidate, and the planner leaves its family alone for + MAINTAINER_CLOSE_HOLD_DAYS. Closed PRs are read newest-updated first until the window ends. """ from .lifecycle import BOT - aborted = set() - for issue in github_items(repository, f"issues?labels={ABORT_LABEL}&state=all&per_page=100"): - if "pull_request" not in issue or issue["user"]["login"] != BOT: - continue - pull = github_read(repository, f"pulls/{issue['number']}") - if match := re.fullmatch(r"klaud/auto-([0-9a-f]{16}-[0-9a-f]{16})", pull["head"]["ref"]): - aborted.add(match[1]) - return aborted + cutoff = now - timedelta(days=MAINTAINER_CLOSE_HOLD_DAYS) + held = set() + page = 1 + while True: + pulls = github_read( + repository, + f"pulls?state=closed&sort=updated&direction=desc&per_page=100&page={page}", + ) + for pull in pulls: + match = re.fullmatch(r"klaud/auto-([0-9a-f]{16})-[0-9a-f]{16}", pull["head"]["ref"]) + if ( + match + and pull["user"]["login"] == BOT + and not pull["merged_at"] + and utc(pull["closed_at"]) >= cutoff + and match[1] not in held + ): + closer = github_read(repository, f"issues/{pull['number']}").get("closed_by") or {} + if closer.get("login") != BOT: + held.add(match[1]) + if len(pulls) < 100 or utc(pulls[-1]["updated_at"]) < cutoff: + return held + page += 1 -def abort_released(repository: str, candidate_id: str) -> bool: - """Whether the candidate's latest Klaud PR ended in an abort whose label was since removed.""" - from urllib.parse import urlencode +def klaud_pulls(pulls: list[dict]) -> list[dict]: + """Return ``pulls`` on Klaud candidate branches. Other PRs never own a family.""" + return [pr for pr in pulls if re.match(r"^klaud[e]?/auto-", pr["head"]["ref"])] - from .lifecycle import BOT - query = urlencode( - { - "state": "closed", - "head": f"{repository.split('/', maxsplit=1)[0]}:klaud/auto-{candidate_id}", - "per_page": 100, - } - ) - try: - pulls = [ - pull - for pull in github_items(repository, "pulls?" + query) - if pull["user"]["login"] == BOT - ] - if not pulls: - return False - latest = max(pulls, key=lambda pull: pull["created_at"]) - if any(label["name"] == ABORT_LABEL for label in latest["labels"]): - return False - comments = github_items(repository, f"issues/{latest['number']}/comments?per_page=100") - for comment in reversed(comments): - if comment["user"]["login"] == BOT and comment["body"].startswith( - "", 1)[0]) - return record["outcome"]["outcome"] == "aborted" - except (OSError, ValueError, KeyError, TypeError, IndexError, subprocess.SubprocessError): - print(f"::warning::Klaud abort history unavailable for {candidate_id}; keeping cooldown") - return False - - -def active_pulls(pulls: list[dict], now: datetime, stale_days: int) -> list[dict]: - """Return ``pulls`` on Klaud branches or updated within ``stale_days`` before ``now``.""" - cutoff = now - timedelta(days=stale_days) - return [ - pr - for pr in pulls - if re.match(r"^klaud[e]?/auto-", pr["head"]["ref"]) - or datetime.fromisoformat(pr["updated_at"].replace("Z", "+00:00")) >= cutoff - ] - - -def plan( - root: Path, directory: Path, review_batch_size: int, cooldown_hours: int, stale_pr_days: int -) -> None: +def plan(root: Path, directory: Path, review_batch_size: int, cooldown_hours: int) -> None: """Write ``candidates.json`` and ``open-prs.json`` under ``directory`` for overlap review. - Unclaimed, unaborted feed candidates at ``root``'s HEAD are cooled down by - ``cooldown_hours`` and capped at ``review_batch_size``. PRs idle ``stale_pr_days`` drop out. + Unclaimed feed candidates at ``root``'s HEAD are cooled down by ``cooldown_hours``, held + while a maintainer recently closed their family's PR, and capped at ``review_batch_size``. + Only open Klaud PRs are indexed. """ from . import claims @@ -446,13 +425,11 @@ def plan( if recovery_file.exists(): blocked.update(json.loads(recovery_file.read_text())) candidates = [candidate for candidate in candidates if candidate["family"] not in blocked] - aborted = aborted_candidates(repository) - candidates = [candidate for candidate in candidates if candidate["id"] not in aborted] + held = maintainer_closed_families(repository, datetime.now(UTC)) + candidates = [ + candidate for candidate in candidates if candidate["id"].split("-", 1)[0] not in held + ] recent, missing_weights = recent_candidate_outcomes(repository, cooldown_hours) - for candidate in candidates: - held = candidate["id"] in recent and recent[candidate["id"]] not in RETRY_NEXT_WAVE - if held and abort_released(repository, candidate["id"]): - recent[candidate["id"]] = "aborted" candidates = [ candidate for candidate in candidates @@ -484,9 +461,7 @@ def plan( "branch": pr["head"]["ref"], "head": pr["head"]["sha"], } - for pr in active_pulls( - [pr for page in pulls for pr in page], datetime.now(UTC), stale_pr_days - ) + for pr in klaud_pulls([pr for page in pulls for pr in page]) ] if candidates: for pr in open_prs: @@ -639,7 +614,7 @@ def execution_diagnostics(path: Path) -> dict: def check_stop() -> dict: """Return the Stop hook decision, blocking unless a verified terminal outcome exists. - An aborted candidate never blocks, since the recover-current step cleans it up. A session + A PR a maintainer closed never blocks, since the recover-current step cleans it up. A session that parked may stop while a run it parked on is still live, or if it parked at its deadline. """ from .lifecycle import current_session, parked, terminal @@ -647,7 +622,7 @@ def check_stop() -> dict: try: session = current_session() pulls = session.pulls() - if pulls and session.aborted(pulls[0]): + if pulls and session.closed_by_maintainer(pulls[0]): return {} evidence = Path(os.environ["KLAUD_EVIDENCE"]) end = evidence / "session-end.json" @@ -832,7 +807,7 @@ def save_diagnostics( if failure is not None: observed = diagnostics.get("observed-session", {}) outcome = CandidateOutcome( - outcome="unexpected-error", + outcome="no-outcome", phase="unknown", pull_request=observed.get("pull-request"), run_ids=[run["id"] for run in observed.get("runs", [])], @@ -865,7 +840,7 @@ def save_diagnostics( "| --- | --- | --- | --- | --- | --- |\n" f"| {outcome.outcome} | {outcome.phase} | {reason} | {pr} | {repairs} | {runs} |\n" ) - return outcome.outcome != "unexpected-error" + return outcome.outcome != "no-outcome" def reviewed(contexts: list[dict]) -> tuple[PRReview, str | None]: @@ -1155,12 +1130,6 @@ def main() -> int: required=True, help="Hours a recently attempted candidate waits before another agent (1-168)", ) - prepare.add_argument( - "--stale-pr-days", - type=int, - required=True, - help="Days without activity after which another open PR stops owning a family (1-90)", - ) regeneration = commands.add_parser( "regenerate-producers", help="Regenerate reviewed candidates' baseline producer families; run without credentials", @@ -1326,7 +1295,7 @@ def main() -> int: return 0 if args.command == "recover-current": from . import claims - from .lifecycle import PendingCleanup, current_session, reconcile + from .lifecycle import PendingCleanup, current_session, reconcile, show_status session = current_session() try: @@ -1339,6 +1308,8 @@ def main() -> int: session.candidate, session.parent["id"], ) + finally: + show_status(session) return 0 if args.command == "resume-check": outputs = "".join( @@ -1417,15 +1388,7 @@ def main() -> int: parser.error("--review-batch-size must be between 1 and 256") if not 1 <= args.cooldown_hours <= 168: parser.error("--cooldown-hours must be between 1 and 168") - if not 1 <= args.stale_pr_days <= 90: - parser.error("--stale-pr-days must be between 1 and 90") - plan( - args.root, - args.directory, - args.review_batch_size, - args.cooldown_hours, - args.stale_pr_days, - ) + plan(args.root, args.directory, args.review_batch_size, args.cooldown_hours) return 0 if args.command == "regenerate-producers": regenerate_producers(args.directory) diff --git a/inferencex-e2e/infx/klaud/github.py b/inferencex-e2e/infx/klaud/github.py index aa7ea707ff..afbe2a5580 100644 --- a/inferencex-e2e/infx/klaud/github.py +++ b/inferencex-e2e/infx/klaud/github.py @@ -32,14 +32,22 @@ def items(repository: str, path: str, key: str | None = None) -> list[dict]: writes = 0 -def write(repository: str, path: str, method: str, payload: dict | None = None) -> dict: +def write( + repository: str, + path: str, + method: str, + payload: dict | None = None, + *, + token: str | None = None, +) -> dict | None: """Send ``payload`` to ``path`` with ``method`` and return the JSON reply. - Each call bumps the module ``writes`` counter so listings cached before it are read again. + ``token`` replaces the ambient gh token; such failures raise ``APIError``. Each call bumps + the module ``writes`` counter so listings cached before it are read again. """ global writes # noqa: PLW0603 writes += 1 - return github.api(repository, path, method=method, data=payload) + return github.api(repository, path, token, method=method, data=payload) def artifacts(repository: str, run_id: int) -> list[dict]: diff --git a/inferencex-e2e/infx/klaud/lifecycle.py b/inferencex-e2e/infx/klaud/lifecycle.py index e50ebf4150..a4e867a97b 100644 --- a/inferencex-e2e/infx/klaud/lifecycle.py +++ b/inferencex-e2e/infx/klaud/lifecycle.py @@ -20,7 +20,14 @@ from . import claims, github from .github import VerificationError -from .models import ABORT_LABEL, CandidateOutcome, OwnedCandidate, Ownership, identity, utc +from .models import ( + LEGACY_OUTCOMES, + CandidateOutcome, + OwnedCandidate, + Ownership, + identity, + utc, +) from .reporting import Prose, latest_next, translated from .validation import verify_sweep @@ -41,7 +48,7 @@ MAX_RUN_ATTEMPTS = 10 MAX_SESSIONS = 12 MAX_EXPIRED_SESSIONS = 4 -WAVE_HOURS = 3 +WAVE_HOURS = 4 RESUME_GIVE_UP_SECONDS = 24 * 60 * 60 SESSION_SECONDS = 330 * 60 PARK_AFTER_SECONDS = 300 * 60 @@ -50,6 +57,24 @@ WAKE_TITLE = "Klaud Cold wake" TIMEOUT_ANNOTATION = "exceeded the maximum execution time" +STATUS_LABELS = { + "klaud-in-progress": ("1d76db", "Klaud is working on this candidate or will resume it"), + "klaud-parked": ("fbca04", "Klaud parked until its queued or running benchmark runs end"), + "klaud-finished": ("0e8a16", "Klaud verified the full sweep; ready for review"), + "klaud-infra": ("d93f0b", "Stopped by cluster capacity or readiness (weights, mounts)"), + "klaud-incompatible": ("5319e7", "The new image cannot run this recipe without engine work"), + "klaud-error": ("b60205", "Klaud failed or could not decide; needs a look"), +} +OUTCOME_LABELS = { + "validated": "klaud-finished", + "no-capacity": "klaud-infra", + "missing-assets": "klaud-infra", + "incompatible": "klaud-incompatible", + "failed": "klaud-error", + "no-outcome": "klaud-error", + "inconclusive": "klaud-error", +} + def resume_title(parent: int, candidate: str, session: int) -> str: """The run name klaud-plan.yml gives the resume run of ``candidate``'s ``session``.""" @@ -278,9 +303,12 @@ def runs(self) -> list[dict]: raise VerificationError("Maintainer started runs after the session; leave them intact") return targeted + sweeps - def aborted(self, pull: dict) -> bool: - """Return True if ``pull`` carries the abort label.""" - return any(label["name"] == ABORT_LABEL for label in pull["labels"]) + def closed_by_maintainer(self, pull: dict) -> bool: + """Return True if someone other than Klaud closed ``pull`` without merging it.""" + if pull["state"] != "closed" or pull["merged_at"]: + return False + closer = github.read(self.repository, f"issues/{pull['number']}").get("closed_by") or {} + return closer.get("login") != BOT def refresh(self, pull: dict) -> dict: """Return fresh ``pull``, raising VerificationError if it moved, merged or changed hands.""" @@ -301,7 +329,10 @@ def check_parent(self) -> None: raise VerificationError("Session resumed; leave the active session intact") def report(self, pull: dict) -> dict | None: - """Return the outcome record of this session's latest bot report on ``pull``, or None.""" + """Return the outcome record of this session's latest bot report on ``pull``, or None. + + An outcome name from an older report is renamed to its current name. + """ comments = self.listing(f"issues/{pull['number']}/comments?per_page=100") matches = [ comment @@ -311,7 +342,10 @@ def report(self, pull: dict) -> dict | None: if not matches: return None body = max(matches, key=lambda c: c["id"])["body"] - return json.loads(body[len(self.marker) :].split("\n-->", 1)[0]) + record = json.loads(body[len(self.marker) :].split("\n-->", 1)[0]) + name = record["outcome"]["outcome"] + record["outcome"]["outcome"] = LEGACY_OUTCOMES.get(name, name) + return record def pending(self, pull: dict) -> CandidateOutcome | None: """Return the outcome of this session's pending cleanup request on ``pull``, or None. @@ -408,7 +442,7 @@ def verify( else: if pull and ( pull["state"] != "closed" - or not pull["draft"] + or (not pull["draft"] and outcome.outcome != "closed-by-maintainer") or any(label["name"] in SWEEP_LABELS for label in pull["labels"]) ): raise VerificationError("Failure/deferral PR cleanup is incomplete") @@ -507,7 +541,7 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: "DELETE", ) current = self.refresh(pull) - if not current["draft"]: + if current["state"] == "open" and not current["draft"]: subprocess_ready(self.repository, pull["number"]) self.refresh(pull) if current["state"] != "closed": @@ -557,8 +591,8 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: reason_en = f" · Reason: {outcome.reason_code}" if outcome.reason_code else "" reason_zh = f" · 原因:{outcome.reason_code}" if outcome.reason_code else "" why = ( - Prose(en="Stopped by the klaud-abort label.", zh="已由 klaud-abort 标签停止。") - if outcome.outcome == "aborted" + Prose(en="Closed by a maintainer.", zh="已由维护者关闭。") + if outcome.outcome == "closed-by-maintainer" else None if proof or self.recovering else latest_next(self, pull) @@ -574,12 +608,16 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: + ( "Full sweep verified; ready for review." if proof + else "PR closed; branch deleted." + if outcome.outcome == "closed-by-maintainer" else "PR closed; branch deleted for retry." ), f"**{outcome.outcome}**{reason_zh} · 修复次数:{repairs_zh} · 运行:{links} \n{why_zh}所有自有运行均已结束。" + ( "完整 sweep 已验证;已就绪,等待审查。" if proof + else "PR 已关闭;分支已删除。" + if outcome.outcome == "closed-by-maintainer" else "PR 已关闭;分支已删除,可重新尝试。" ), ) @@ -685,7 +723,6 @@ def resumable(session: Session, pull: dict | None) -> bool: """Whether a later session may continue this candidate's unfinished draft PR.""" return ( unfinished_draft(session, pull) - and not session.aborted(pull) and session.branch_exists() and not session.report(pull) and not session.pending(pull) @@ -832,7 +869,7 @@ def resumes(session: Session, pull: dict | None, now: datetime) -> bool: Every owned run must have ended, and the last started job must have run the gate step and parked, failed, timed out or been cancelled. Resuming stops RESUME_GIVE_UP_SECONDS after the later of that job's end and the last owned run's update, or when the session budget is - spent. Only klaud-abort stops a candidate otherwise. + spent. Closing the PR stops a candidate otherwise. """ if session.last_session_end() is None or not resumable(session, pull): return False @@ -966,6 +1003,83 @@ def summarize(repository: str, candidate: OwnedCandidate, parent: int, status: s ) +def status_label(session: Session, pull: dict) -> str | None: + """Return the display label for ``pull``'s current state, or None for no label. + + Derived only from the outcome record, pending cleanup, park markers and owned runs. Labels + are display-only: Klaud never reads them back. + """ + record = session.report(pull) + outcome = ( + CandidateOutcome.model_validate(record["outcome"]) if record else session.pending(pull) + ) + if outcome: + return OUTCOME_LABELS.get(outcome.outcome) + if not unfinished_draft(session, pull): + return None + if parked(session, pull, sessions_started(session)) is not None and any( + not terminal(run) for run in session.runs() + ): + return "klaud-parked" + return "klaud-in-progress" + + +def show_status(session: Session) -> None: + """Best effort: give ``session``'s unmerged PR exactly its status label. + + Writes with ``$KLAUD_STATUS_TOKEN``, the workflow token, whose label events start no + workflows, so no extra Run Sweep run joins the candidate's owned runs. Does nothing without + that token. + """ + token = os.environ.get("KLAUD_STATUS_TOKEN") + if not token: + return + try: + pulls = session.pulls() + if not pulls or pulls[0]["merged_at"]: + return + pull = pulls[0] + wanted = status_label(session, pull) + shown = {label["name"] for label in pull["labels"]} & STATUS_LABELS.keys() + if wanted and wanted not in shown: + color, description = STATUS_LABELS[wanted] + with contextlib.suppress(RuntimeError): + github.write( + session.repository, + "labels", + "POST", + {"name": wanted, "color": color, "description": description}, + token=token, + ) + github.write( + session.repository, + f"issues/{pull['number']}/labels", + "POST", + {"labels": [wanted]}, + token=token, + ) + for name in sorted(shown - {wanted}): + with contextlib.suppress(RuntimeError): + github.write( + session.repository, + f"issues/{pull['number']}/labels/{name}", + "DELETE", + token=token, + ) + except ( + OSError, + ValueError, + KeyError, + TypeError, + RuntimeError, + subprocess.SubprocessError, + ) as error: + print( + f"::warning::Klaud: status label of candidate {session.candidate.id} not updated: " + f"{type(error).__name__}." + ) + + def reconcile(session: Session, now: datetime) -> bool: """Settle interrupted ``session`` at ``now`` and return True if anything changed. @@ -991,10 +1105,10 @@ def reconcile(session: Session, now: datetime) -> bool: if not pull and not runs: return False pending = session.pending(pull) if pull else None - if pull and not pending and session.aborted(pull): + if pull and not pending and session.closed_by_maintainer(pull): session.finish( CandidateOutcome( - outcome="aborted", + outcome="closed-by-maintainer", phase="cleanup", pull_request=pull["number"], run_ids=[run["id"] for run in runs], @@ -1045,7 +1159,7 @@ def reconcile(session: Session, now: datetime) -> bool: if not session.recovering and resumable(session, pull) and within_budget(session, pull): raise PendingCleanup("Unfinished session kept for a later resume") outcome = pending or CandidateOutcome( - outcome="validated" if validated else "unexpected-error", + outcome="validated" if validated else "no-outcome", phase="final-sweep" if validated else "cleanup", pull_request=pull["number"] if pull else None, run_ids=[r["id"] for r in runs], @@ -1166,6 +1280,7 @@ def recover(max_sessions: int) -> None: status = "reconciled / 已收尾" if changed else None if status: summarize(repository, candidate, parent["id"], status) + show_status(session) try: running = len(running_sessions(repository)) except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError): @@ -1195,6 +1310,7 @@ def recover(max_sessions: int) -> None: resumed += 1 status = f"resumed / 已恢复 (session {number})" summarize(repository, session.candidate, run_id, status) + show_status(session) for owner, artifact in owners: if artifact and all( results[(owner.run_id, candidate.id)] for candidate in owner.candidates diff --git a/inferencex-e2e/infx/klaud/models.py b/inferencex-e2e/infx/klaud/models.py index 94c2a26f30..f0596f53c6 100644 --- a/inferencex-e2e/infx/klaud/models.py +++ b/inferencex-e2e/infx/klaud/models.py @@ -1,7 +1,4 @@ -"""Versioned local contracts, deliberately separate from external API schemas. - -``ABORT_LABEL`` is the maintainer label that stops a candidate now and blocks retries while set. -""" +"""Versioned local contracts, deliberately separate from external API schemas.""" from __future__ import annotations @@ -12,8 +9,6 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -ABORT_LABEL = "klaud-abort" - def utc(value: str | datetime) -> datetime: """Return an ISO timestamp or aware datetime as a UTC datetime, raising ValueError if naive.""" @@ -101,19 +96,28 @@ class Ownership(Contract): candidates: list[OwnedCandidate] = Field(max_length=256) +LEGACY_OUTCOMES = { + "readiness-blocked": "missing-assets", + "uncertain": "inconclusive", + "unexpected-error": "no-outcome", + "capacity-deferred": "no-capacity", + "aborted": "closed-by-maintainer", +} + + class CandidateOutcome(Contract): outcome: Literal[ "validated", - "capacity-deferred", - "readiness-blocked", + "no-capacity", + "missing-assets", "incompatible", "duplicate", "retired", "already-updated", - "uncertain", + "inconclusive", "failed", - "aborted", - "unexpected-error", + "closed-by-maintainer", + "no-outcome", ] phase: Literal["resolve", "baseline", "targeted", "final-sweep", "cleanup", "unknown"] pull_request: Annotated[int, Field(gt=0)] | None @@ -132,20 +136,28 @@ class CandidateOutcome(Contract): | None ) = None + @model_validator(mode="before") + @classmethod + def current_names(cls, data: Any) -> Any: + """Return ``data`` with an outcome name from older PR comments and artifacts renamed.""" + if isinstance(data, dict) and data.get("outcome") in LEGACY_OUTCOMES: + return {**data, "outcome": LEGACY_OUTCOMES[data["outcome"]]} + return data + @model_validator(mode="after") def consistent_outcome(self) -> CandidateOutcome: """Return the outcome, raising ValueError if phase, PR, runs or reason code disagree.""" - if self.outcome != "unexpected-error" and self.phase == "unknown": + if self.outcome != "no-outcome" and self.phase == "unknown": raise ValueError("Completed outcomes require a known phase") - if self.outcome in ("validated", "aborted") and self.pull_request is None: + if self.outcome in ("validated", "closed-by-maintainer") and self.pull_request is None: raise ValueError("This outcome requires a PR") if self.outcome == "validated" and (not self.run_ids or self.phase != "final-sweep"): raise ValueError("Validated requires final-sweep evidence") if len(set(self.run_ids)) != len(self.run_ids): raise ValueError("Run IDs must be distinct") if self.reason_code == "weights-missing": - if self.outcome != "readiness-blocked": - raise ValueError("weights-missing requires a readiness-blocked outcome") + if self.outcome != "missing-assets": + raise ValueError("weights-missing requires a missing-assets outcome") elif self.reason_code is not None and ( self.outcome != "failed" or self.phase != "baseline" ): diff --git a/inferencex-e2e/infx/klaud/reporting.py b/inferencex-e2e/infx/klaud/reporting.py index a61d3a9163..dc588574f1 100644 --- a/inferencex-e2e/infx/klaud/reporting.py +++ b/inferencex-e2e/infx/klaud/reporting.py @@ -31,7 +31,7 @@ from . import github from .github import VerificationError -from .models import ABORT_LABEL, Contract, identity, normalized_image +from .models import Contract, identity, normalized_image if TYPE_CHECKING: from infx.klaud.lifecycle import Session @@ -622,7 +622,7 @@ def baseline_table(baseline: Baseline, *, context: bool = True) -> str: def render_body(baseline: Baseline) -> str: - """Return the bilingual PR body section that presents ``baseline`` and the abort label.""" + """Return the bilingual PR body section that presents ``baseline`` and how to stop it.""" _, _, settings = point_layout(baseline.points) sources = ", ".join(f"[API {i + 1}]({url})" for i, url in enumerate(baseline.sources)) or "N/A" meta = " · ".join(part for part in (settings, f"Sources: {sources}") if part) @@ -631,9 +631,8 @@ def render_body(baseline: Baseline) -> str: + baseline_table(baseline, context=False) + "\n\n" + (eval_table(baseline.evals, None, compare=False) if baseline.evals else "**Eval:** N/A") - + f"\n\n**Stop:** Add the `{ABORT_LABEL}` label at any time to stop this candidate now; " - "Klaud cancels its runs, closes this PR and does not retry it. Remove the label to let " - "the planner pick it up again." + + "\n\n**Stop:** Close this PR at any time to stop this candidate now; Klaud cancels its " + "runs and deletes its branch." ) chinese = ( f"**目标:**{baseline.goal.zh} \n**基线:**{baseline.date} · `{baseline.image}` \n" @@ -641,8 +640,7 @@ def render_body(baseline: Baseline) -> str: .replace("Median latency", "中位延迟") .replace("Sources:", "来源:") + ";数值及异常说明见上表。 \n" - + f"**停止:**随时添加 `{ABORT_LABEL}` 标签即可立即停止该候选:Klaud 会取消其运行、" - "关闭此 PR 且不再重试。移除该标签后,规划器会在下次扫描时重新考虑它。" + + "**停止:**随时关闭此 PR 即可立即停止该候选:Klaud 会取消其运行并删除其分支。" ) return translated(english, chinese) diff --git a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py index 981274deb6..429e5d212c 100644 --- a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py +++ b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py @@ -12,7 +12,15 @@ from infx.klaud import __main__ as klaud from infx.klaud import api, claims, github, lifecycle, reporting, validation -from infx.klaud.models import CandidateOutcome, Feed, OwnedCandidate, Ownership, identity, stamp +from infx.klaud.models import ( + LEGACY_OUTCOMES, + CandidateOutcome, + Feed, + OwnedCandidate, + Ownership, + identity, + stamp, +) from infx.tests.historical_revision import ( AGENTX_MASTER, FAMILY, @@ -206,10 +214,10 @@ def test_candidate_outcome_rejects_raw_reason_and_preserves_legacy_receipts(): @pytest.mark.parametrize(("outcome", "phase", "reason", "valid"), [ - ("readiness-blocked", "targeted", "weights-missing", True), + ("missing-assets", "targeted", "weights-missing", True), ("failed", "targeted", "weights-missing", False), ("failed", "baseline", "weights-missing", False), - ("readiness-blocked", "targeted", "baseline-other", False), + ("missing-assets", "targeted", "baseline-other", False), ("failed", "baseline", "baseline-other", True), ]) # fmt: skip def test_reason_codes_pair_only_with_their_outcome(outcome, phase, reason, valid): @@ -277,12 +285,12 @@ def test_cooldown_skips_recent_blocks_across_bases_and_retries_capacity_deferral f"{digit * 16}-{digit * 16}" for digit in "1234567" ) outcomes = { - 1: ("readiness-blocked", "targeted", None), - 2: ("capacity-deferred", "targeted", None), + 1: ("missing-assets", "targeted", None), + 2: ("no-capacity", "targeted", None), 4: ("failed", "targeted", None), - 5: ("capacity-deferred", "targeted", None), - 7: ("unexpected-error", "unknown", None), - 8: ("readiness-blocked", "targeted", "weights-missing"), + 5: ("no-capacity", "targeted", None), + 7: ("no-outcome", "unknown", None), + 8: ("missing-assets", "targeted", "weights-missing"), } sources = { blocked: {"model": "dsr1", "precision": "fp8", "hardware": "b200"}, @@ -330,10 +338,10 @@ def download(_repository, artifact, destination): recent, missing_weights = klaud.recent_candidate_outcomes("example/project", 24) assert missing_weights == {("dsr1", "fp4", "b200")} assert recent == { - blocked: "readiness-blocked", + blocked: "missing-assets", unreadable: None, - deferred: "capacity-deferred", - superseded: "capacity-deferred", + deferred: "no-capacity", + superseded: "no-capacity", not_started: "agent-not-started", weightless: "weights-missing", } @@ -344,14 +352,17 @@ def download(_repository, artifact, destination): def test_cooldown_ages_each_candidates_newest_artifact_including_resumed_sessions(monkeypatch): - early, late, resumed = (f"{digit * 16}-{digit * 16}" for digit in "abc") + early, late, resumed, blocked, failed = (f"{digit * 16}-{digit * 16}" for digit in "abcde") artifacts = { # Started before the 24-hour cooldown began; a later wave resumed ``late``. 39: [(1, early, "2026-10-14T11:00:00Z"), (30, late, "2026-10-15T11:00:00Z")], # The listing names ``resumed``'s first session before its second. 41: [(20, resumed, "2026-10-15T05:00:00Z"), (21, resumed, "2026-10-15T10:00:00Z")], + # Days old: an infrastructure block still cools down, an ordinary failure does not. + 40: [(40, blocked, "2026-10-12T00:00:00Z"), (41, failed, "2026-10-12T00:00:00Z")], } - outcomes = {1: "failed", 30: "readiness-blocked", 20: "failed", 21: "capacity-deferred"} + outcomes = {1: "failed", 30: "missing-assets", 20: "failed", 21: "no-capacity", + 40: "missing-assets", 41: "failed"} queries = [] def items(_repository, path, _key): @@ -359,7 +370,8 @@ def items(_repository, path, _key): queries.append(path) return [{"id": 39, "created_at": "2026-10-14T06:00:00Z"}, {"id": 41, "created_at": "2026-10-15T00:00:00Z"}, - {"id": 40, "created_at": "2026-10-12T23:59:59Z"}] + {"id": 40, "created_at": "2026-10-11T23:59:59Z"}, + {"id": 38, "created_at": "2026-10-06T11:59:59Z"}] run_id = int(path.split("/")[2]) if run_id not in artifacts: pytest.fail(f"listed artifacts of run {run_id}") @@ -383,9 +395,10 @@ def now(cls, tz=None): monkeypatch.setattr(klaud, "download_json", download) assert klaud.recent_candidate_outcomes("example/project", 24) == ( - {late: "readiness-blocked", resumed: "capacity-deferred"}, set()) - # The cooldown plus the waves in which an older planner run may still resume sessions. - assert queries == ["actions/workflows/klaud-plan.yml/runs?created=>=2026-10-13"] + {late: "missing-assets", resumed: "no-capacity", blocked: "missing-assets"}, + set()) + # The infrastructure cooldown plus the waves in which an older planner run may still resume. + assert queries == ["actions/workflows/klaud-plan.yml/runs?created=>=2026-10-06"] def test_a_resumed_sessions_missing_weights_pause_its_model_from_the_planners_plan(monkeypatch): @@ -409,7 +422,7 @@ def download(_repository, artifact, destination): (destination / "candidates.json").write_text( json.dumps([{"id": candidate, "source": source}])) return - record = {"outcome": "readiness-blocked", "phase": "targeted", "pull-request": None, + record = {"outcome": "missing-assets", "phase": "targeted", "pull-request": None, "run-ids": [], "repairs-used": 0, "reason-code": "weights-missing"} (destination / "candidate-diagnostics.json").write_text( json.dumps({"candidate-outcome": record})) @@ -421,20 +434,18 @@ def download(_repository, artifact, destination): {candidate: "weights-missing"}, {("dsr1", "fp4", "b200")}) -def test_overlap_index_drops_idle_prs_but_keeps_klaud_candidate_prs(): - now = klaud.datetime(2026, 10, 14, tzinfo=klaud.UTC) - - def pull(number, branch, updated): - return {"number": number, "head": {"ref": branch}, "updated_at": updated} +def test_overlap_index_keeps_only_klaud_candidate_prs(): + def pull(number, branch): + return {"number": number, "head": {"ref": branch}} pulls = [ - pull(1, "feature/active", "2026-10-07T00:00:00Z"), - pull(2, "klaud/qwen3.5-nightly", "2026-10-06T23:59:59Z"), - pull(3, "klaud/auto-0123456789abcdef-0123456789abcdef", "2026-09-01T00:00:00Z"), - pull(4, "klaude/auto-fedcba9876543210-fedcba9876543210", "2026-09-01T00:00:00Z"), + pull(1, "feature/active"), + pull(2, "klaud/qwen3.5-nightly"), + pull(3, "klaud/auto-0123456789abcdef-0123456789abcdef"), + pull(4, "klaude/auto-fedcba9876543210-fedcba9876543210"), ] - assert [pr["number"] for pr in klaud.active_pulls(pulls, now, 7)] == [1, 3, 4] + assert [pr["number"] for pr in klaud.klaud_pulls(pulls)] == [3, 4] def test_capacity_is_idle_headroom_on_clusters_without_waiting_human_jobs(): @@ -603,7 +614,7 @@ def read(_repository, path): UNEXPECTED_ERROR = CandidateOutcome( - outcome="unexpected-error", phase="cleanup", pull_request=7, run_ids=[9], repairs_used=None, + outcome="no-outcome", phase="cleanup", pull_request=7, run_ids=[9], repairs_used=None, ) @@ -756,7 +767,7 @@ def items(_repository, path, _key=None): world = SimpleNamespace( session=lifecycle.Session("example/project", parent, candidate, recovering=True), pull=pull, reads=reads, comments=comments, writes=[], dispatches=[], finished=[], - released=[], start_resume=start_resume, sweep=sweep, + released=[], start_resume=start_resume, sweep=sweep, listings=listings, ) def write(_repository, path, method, payload=None): @@ -1266,7 +1277,7 @@ def test_diagnostics_verify_only_a_session_that_ended_parked( assert diagnostics["session-state"] == "parked" assert diagnostics["candidate-outcome"] is None else: - assert diagnostics["candidate-outcome"]["outcome"] == "unexpected-error" + assert diagnostics["candidate-outcome"]["outcome"] == "no-outcome" @pytest.mark.parametrize(("elapsed", "warned"), [ @@ -1291,15 +1302,20 @@ def now(cls, tz=None): assert "About 30 minutes remain" in hook["hookSpecificOutput"]["additionalContext"] +def close_by(world, closer): + """Mark the world's PR closed by ``closer``.""" + world.pull["state"] = "closed" + world.reads["issues/7"] = {"closed_by": {"login": closer}} + + @pytest.mark.parametrize("change", [ - "aborted", "closed", "outcome-recorded", "cleanup-pending", + "maintainer-closed", "outcome-recorded", "cleanup-pending", ]) # fmt: skip -def test_recovery_never_resumes_an_aborted_closed_or_recorded_pr(monkeypatch, change): - world = ended_session_sweep(monkeypatch, state="closed" if change == "closed" else "open", - sessions=("failure",)) - if change == "aborted": - world.pull["labels"].append({"name": lifecycle.ABORT_LABEL}) - elif change != "closed": +def test_recovery_never_resumes_a_closed_or_recorded_pr(monkeypatch, change): + world = ended_session_sweep(monkeypatch, sessions=("failure",)) + if change == "maintainer-closed": + close_by(world, "maintainer") + else: world.comments.append(recorded( world.session, "klaud-cleanup:" if change == "cleanup-pending" else "klaud-outcome:")) @@ -1308,27 +1324,142 @@ def test_recovery_never_resumes_an_aborted_closed_or_recorded_pr(monkeypatch, ch assert world.writes == [] -@pytest.mark.parametrize("conclusion", [None, "failure"], ids=["running", "finished"]) -def test_klaud_abort_closes_the_candidate_pr_as_aborted(monkeypatch, conclusion): +@pytest.mark.parametrize(("conclusion", "closer", "outcome"), [ + pytest.param(None, "maintainer", "closed-by-maintainer", id="running"), + pytest.param("failure", "maintainer", "closed-by-maintainer", id="finished"), + pytest.param("failure", "Klaud-Cold", "no-outcome", id="closed-by-klaud"), +]) # fmt: skip +def test_a_maintainer_closing_the_pr_finishes_the_candidate_as_closed_by_maintainer( + monkeypatch, conclusion, closer, outcome +): world = ended_session_sweep(monkeypatch, conclusion=conclusion, sessions=("failure",)) - world.pull["labels"].append({"name": lifecycle.ABORT_LABEL}) + close_by(world, closer) assert lifecycle.reconcile(world.session, NOW) is True - assert [outcome.outcome for outcome in world.finished] == ["aborted"] + assert [finished.outcome for finished in world.finished] == [outcome] assert world.writes == [] -def test_an_aborted_session_may_stop_without_its_own_outcome(tmp_path, monkeypatch): +REAL_FINISH = lifecycle.Session.finish + + +@pytest.mark.parametrize("draft", [True, False], ids=["draft", "ready-for-review"]) +def test_finishing_a_pr_a_maintainer_closed_leaves_it_closed_as_is(monkeypatch, draft): + world = ended_session_sweep(monkeypatch, conclusion="failure", sessions=("failure",)) + close_by(world, "maintainer") + world.pull.update(draft=draft, labels=[]) + world.reads[f"git/ref/heads/{world.session.branch}"] = {"object": {"sha": "c" * 40}} + readied = [] + monkeypatch.setattr(lifecycle, "subprocess_ready", lambda *args, **_: readied.append(args)) + monkeypatch.setattr(lifecycle.Session, "finish", REAL_FINISH) + serve = github.write + + def write(repository, path, method, payload=None): + if (method, path) == ("DELETE", f"git/refs/heads/{world.session.branch}"): + world.listings[f"git/matching-refs/heads/{world.session.branch}"] = [] + return serve(repository, path, method, payload) + + monkeypatch.setattr(github, "write", write) + + assert lifecycle.reconcile(world.session, NOW) is True + assert readied == [] + assert ("PATCH", "pulls/7") not in world.writes + assert world.session.report(world.pull)["outcome"]["outcome"] == "closed-by-maintainer" + assert "PR closed; branch deleted.\n" in world.comments[-1]["body"] + + +def test_a_session_whose_pr_a_maintainer_closed_may_stop_without_its_own_outcome( + tmp_path, monkeypatch +): world = ended_session_sweep(monkeypatch, conclusion=None, sessions=(None,)) - world.pull["labels"].append({"name": lifecycle.ABORT_LABEL}) + close_by(world, "maintainer") monkeypatch.setattr(lifecycle, "current_session", lambda: world.session) monkeypatch.setenv("KLAUD_EVIDENCE", str(tmp_path)) assert klaud.check_stop() == {} - world.pull["labels"].pop() + world.pull["state"] = "open" assert klaud.check_stop()["decision"] == "block" +def outcome_comment(session, outcome, marker="klaud-outcome:"): + """A Klaud comment recording ``outcome`` under ``session``'s ``marker``.""" + record = {"head": "c" * 40, "outcome": {**UNEXPECTED_ERROR.model_dump(by_alias=True), + "outcome": outcome, "phase": "targeted"}} + return {"id": 1, "user": {"login": "Klaud-Cold"}, "body": ( + session.marker.replace("klaud-outcome:", marker) + json.dumps(record) + "\n-->\nDone.")} + + +@pytest.mark.parametrize(("conclusion", "change", "label"), [ + pytest.param(None, None, "klaud-in-progress", id="run-live-without-park"), + pytest.param(None, "parked", "klaud-parked", id="parked-on-a-live-run"), + pytest.param("failure", "parked", "klaud-in-progress", id="parked-runs-ended"), + pytest.param("failure", "missing-assets", "klaud-infra", id="infra"), + pytest.param("failure", "incompatible", "klaud-incompatible", id="incompatible"), + pytest.param("failure", "inconclusive", "klaud-error", id="inconclusive"), + pytest.param("failure", "pending-failed", "klaud-error", id="cleanup-pending"), + pytest.param("failure", "retired", None, id="nothing-to-review"), + pytest.param("failure", "maintainer-closed", None, id="closed-without-outcome"), +]) # fmt: skip +def test_status_label_follows_outcome_park_and_owned_runs(monkeypatch, conclusion, change, label): + world = ended_session_sweep(monkeypatch, conclusion=conclusion, sessions=("failure",)) + if change == "parked": + world.comments.append(park_comment(world, 1)) + elif change == "maintainer-closed": + close_by(world, "maintainer") + elif change == "pending-failed": + world.comments.append(outcome_comment(world.session, "failed", "klaud-cleanup:")) + elif change: + world.comments.append(outcome_comment(world.session, change)) + + assert lifecycle.status_label(world.session, world.pull) == label + + +@pytest.mark.parametrize(("old", "new"), list(LEGACY_OUTCOMES.items())) +def test_outcomes_recorded_under_old_names_read_back_under_current_ones(monkeypatch, old, new): + world = ended_session_sweep(monkeypatch, sessions=("failure",)) + record = {**UNEXPECTED_ERROR.model_dump(by_alias=True), "outcome": old, "phase": "cleanup"} + world.comments.append({"id": 1, "user": {"login": "Klaud-Cold"}, "body": ( + world.session.marker + json.dumps({"head": "c" * 40, "outcome": record}) + "\n-->\n.")}) + + assert world.session.report(world.pull)["outcome"]["outcome"] == new + assert CandidateOutcome.model_validate(record).outcome == new + + +@pytest.mark.parametrize(("shown", "token", "merged", "writes"), [ + pytest.param(["klaud-in-progress"], "t", False, [ + ("POST", "labels"), ("POST", "issues/7/labels"), + ("DELETE", "issues/7/labels/klaud-in-progress"), + ], id="switches-to-parked"), + pytest.param(["klaud-parked"], "t", False, [], id="already-shown"), + pytest.param(["klaud-in-progress"], None, False, [], id="no-status-token"), + pytest.param(["klaud-in-progress"], "t", True, [], id="merged"), +]) # fmt: skip +def test_show_status_sets_one_label_with_the_workflow_token( + monkeypatch, shown, token, merged, writes +): + world = ended_session_sweep(monkeypatch, conclusion=None, sessions=("failure",)) + world.comments.append(park_comment(world, 1)) + world.pull["labels"] += [{"name": name} for name in shown] + world.pull["merged_at"] = "2026-10-15T00:00:00Z" if merged else None + if token: + monkeypatch.setenv("KLAUD_STATUS_TOKEN", token) + else: + monkeypatch.delenv("KLAUD_STATUS_TOKEN", raising=False) + sent = [] + + def write(_repository, path, method, payload=None, *, token=None): + sent.append((method, path, payload, token)) + + monkeypatch.setattr(github, "write", write) + + lifecycle.show_status(world.session) + + assert [(method, path) for method, path, _, _ in sent] == writes + assert all(sent_token == "t" for *_, sent_token in sent) + if writes: + assert sent[1][2] == {"labels": ["klaud-parked"]} + + def test_a_resumed_session_never_adopts_a_newer_runs_pr_on_its_branch(tmp_path, monkeypatch): world = ended_session_sweep(monkeypatch, conclusion="failure", sessions=("failure", "failure")) # A newer planner run opened this PR on the candidate branch after planner run 42 ended. @@ -1533,7 +1664,7 @@ def test_a_planner_job_runs_only_its_first_attempts_session_without_reading_gith id="ungranted-resume-run-not-counted"), pytest.param(("failure",) * lifecycle.MAX_SESSIONS + (None,), None, False, id="over-the-cap"), pytest.param(("failure", None), "outcome-recorded", False, id="outcome-recorded"), - pytest.param(("failure", None), "aborted", False, id="aborted-despite-a-grant"), + pytest.param(("failure", None), "maintainer-closed", False, id="closed-despite-a-grant"), pytest.param(("failure", None), "ungranted", False, id="no-grant"), pytest.param(("failure", None), "other-session", False, id="grant-for-another-session"), pytest.param(("failure", None), "forged", False, id="grant-from-someone-else"), @@ -1549,8 +1680,8 @@ def test_a_resume_run_starts_only_the_session_recovery_granted_it( author="maintainer" if change == "forged" else "Klaud-Cold")) if change == "outcome-recorded": world.comments.append(recorded(world.session)) - if change == "aborted": - world.pull["labels"].append({"name": lifecycle.ABORT_LABEL}) + if change == "maintainer-closed": + close_by(world, "maintainer") decision = str(resumes).lower() assert resume_check(tmp_path, monkeypatch, world, number) == ( @@ -2767,14 +2898,14 @@ def test_validated_outcome_requires_a_fail_fast_only_sweep(monkeypatch, labels, @pytest.mark.parametrize(("outcome", "latest", "chunked", "recovering", "shown"), [ - pytest.param("uncertain", None, False, False, False, id="no-own-attempt"), - pytest.param("uncertain", STOP, False, False, True, id="own-attempt"), - pytest.param("uncertain", STOP, True, False, True, id="chunked-own-attempt"), - pytest.param("uncertain", ("Traceback:\n
CUDA error
", "日志"), False, False, False, + pytest.param("inconclusive", None, False, False, False, id="no-own-attempt"), + pytest.param("inconclusive", STOP, False, False, True, id="own-attempt"), + pytest.param("inconclusive", STOP, True, False, True, id="chunked-own-attempt"), + pytest.param("inconclusive", ("Traceback:\n
CUDA error
", "日志"), False, False, False, id="raw-log"), pytest.param("validated", STOP, False, False, False, id="validated"), - pytest.param("uncertain", STOP, False, True, False, id="recovered-session"), - pytest.param("aborted", STOP, False, False, False, id="aborted-by-label"), + pytest.param("inconclusive", STOP, False, True, False, id="recovered-session"), + pytest.param("closed-by-maintainer", STOP, False, False, False, id="closed-by-maintainer"), ]) # fmt: skip def test_closing_report_states_why_from_the_latest_own_attempt( monkeypatch, outcome, latest, chunked, recovering, shown @@ -2795,14 +2926,14 @@ def test_closing_report_states_why_from_the_latest_own_attempt( ] runs = "[9](https://github.com/example/project/actions/runs/9)" done = { - "uncertain": ("PR closed; branch deleted for retry.", "PR 已关闭;分支已删除,可重新尝试。"), - "aborted": ("PR closed; branch deleted for retry.", "PR 已关闭;分支已删除,可重新尝试。"), + "inconclusive": ("PR closed; branch deleted for retry.", "PR 已关闭;分支已删除,可重新尝试。"), + "closed-by-maintainer": ("PR closed; branch deleted.", "PR 已关闭;分支已删除。"), "validated": ("Full sweep verified; ready for review.", "完整 sweep 已验证;已就绪,等待审查。"), }[outcome] why = (f"**Reason:** {latest[0]} \n", f"**原因:**{latest[1]} \n") if shown else ("", "") - if outcome == "aborted": - why = ("**Reason:** Stopped by the klaud-abort label. \n", - "**原因:**已由 klaud-abort 标签停止。 \n") + if outcome == "closed-by-maintainer": + why = ("**Reason:** Closed by a maintainer. \n", + "**原因:**已由维护者关闭。 \n") assert finish_report(monkeypatch, outcome, comments, recovering) == ( f"**{outcome}** · Repairs: 0 · Runs: {runs} \n{why[0]}All owned runs ended. {done[0]}" @@ -2813,7 +2944,7 @@ def test_closing_report_states_why_from_the_latest_own_attempt( def test_closing_report_translates_an_unknown_repair_count(monkeypatch): - report = finish_report(monkeypatch, "uncertain", [], recovering=True, repairs=None) + report = finish_report(monkeypatch, "inconclusive", [], recovering=True, repairs=None) english, chinese = report.split("中文") assert "· Repairs: unknown ·" in english @@ -2824,63 +2955,50 @@ def test_every_candidate_pr_body_says_how_to_stop_the_candidate(monkeypatch): body = reporting.render_body(frozen_dsr1_baseline(monkeypatch)) english, chinese = body.split("中文") - assert ("**Stop:** Add the `klaud-abort` label at any time to stop this candidate now; Klaud " - "cancels its runs, closes this PR and does not retry it. Remove the label to let the " - "planner pick it up again.") in english - assert ("**停止:**随时添加 `klaud-abort` 标签即可立即停止该候选:Klaud 会取消其运行、关闭此 PR " - "且不再重试。移除该标签后,规划器会在下次扫描时重新考虑它。") in chinese - + assert ("**Stop:** Close this PR at any time to stop this candidate now; Klaud cancels its " + "runs and deletes its branch.") in english + assert "**停止:**随时关闭此 PR 即可立即停止该候选:Klaud 会取消其运行并删除其分支。" in chinese -def test_the_planner_skips_aborted_candidates_until_the_label_is_removed(monkeypatch): - aborted, other = "1" * 16 + "-" + "2" * 16, "3" * 16 + "-" + "4" * 16 - issues = [ - {"number": 7, "user": {"login": "Klaud-Cold"}, "pull_request": {}}, - {"number": 8, "user": {"login": "maintainer"}, "pull_request": {}}, - {"number": 9, "user": {"login": "Klaud-Cold"}}, - ] - heads = {"pulls/7": f"klaud/auto-{aborted}", "pulls/8": f"klaud/auto-{other}"} - monkeypatch.setattr(klaud, "github_items", lambda _repository, path, _key=None: ( - issues if path == "issues?labels=klaud-abort&state=all&per_page=100" - else pytest.fail(path))) - monkeypatch.setattr(klaud, "github_read", lambda _repository, path: ( - {"head": {"ref": heads[path]}})) - assert klaud.aborted_candidates("example/project") == {aborted} +def test_the_planner_holds_a_family_for_a_week_after_a_maintainer_closes_its_pr(monkeypatch): + now = klaud.datetime(2026, 10, 25, tzinfo=klaud.UTC) + def pull(number, family, closed, *, merged=False, author="Klaud-Cold", branch=None): + return {"number": number, "user": {"login": author}, "closed_at": closed, + "updated_at": closed, "merged_at": closed if merged else None, + "head": {"ref": branch or f"klaud/auto-{family * 16}-{'0' * 16}"}} -@pytest.mark.parametrize(("labels", "outcome", "released"), [ - pytest.param([], "aborted", True, id="label-removed"), - pytest.param(["klaud-abort"], "aborted", False, id="still-labeled"), - pytest.param([], "uncertain", False, id="closed-for-another-reason"), -]) # fmt: skip -def test_removing_klaud_abort_lifts_the_cooldown_of_an_aborted_candidate( - monkeypatch, labels, outcome, released -): - candidate = "1" * 16 + "-" + "2" * 16 - record = {"head": "c" * 40, "outcome": {**UNEXPECTED_ERROR.model_dump(by_alias=True), - "outcome": outcome}} - comments = [ - {"id": 1, "user": {"login": "Klaud-Cold"}, "body": f"\nOlder."}, - {"id": 2, "user": {"login": "Klaud-Cold"}, - "body": f"\nLatest."}, + first = [ + pull(1, "a", "2026-10-24T00:00:00Z"), + pull(2, "b", "2026-10-24T00:00:00Z"), + pull(3, "c", "2026-10-23T00:00:00Z", merged=True), + pull(4, "d", "2026-10-22T00:00:00Z", author="maintainer"), + pull(5, "e", "2026-10-21T00:00:00Z", branch="feature/e"), + *[pull(10 + n, "f", "2026-10-20T00:00:00Z") for n in range(95)], + ] + second = [ + pull(200, "1", "2026-10-18T00:00:01Z"), + pull(201, "2", "2026-10-17T23:59:59Z"), ] - pulls = [{"number": 7, "user": {"login": "Klaud-Cold"}, "created_at": "2026-10-15T00:00:00Z", - "labels": [{"name": name} for name in labels]}] + closers = {1: "maintainer", 2: "Klaud-Cold", 4: "maintainer", 10: "maintainer", + 200: "maintainer", 201: "maintainer"} + pages = [] - def items(_repository, path, _key=None): - if path.startswith("pulls?state=closed&head=example%3Aklaud%2Fauto-"): - return pulls - if path == "issues/7/comments?per_page=100": - return comments + def read(_repository, path): + if path.startswith("pulls?state=closed&sort=updated&direction=desc&per_page=100&page="): + pages.append(int(path.rsplit("=", 1)[1])) + return {1: first, 2: second}[pages[-1]] + if path.startswith("issues/"): + return {"closed_by": {"login": closers[int(path.split("/")[1])]}} raise AssertionError(path) - monkeypatch.setattr(klaud, "github_items", items) + monkeypatch.setattr(klaud, "github_read", read) - assert klaud.abort_released("example/project", candidate) is released - assert [c["id"] for c in klaud.order_by_cooldown([{"id": candidate}], {candidate: "aborted"})] \ - == [candidate] + assert klaud.maintainer_closed_families("example/project", now) == { + "a" * 16, "f" * 16, "1" * 16} + assert pages == [1, 2] + candidate = "1" * 16 + "-" + "2" * 16 + assert klaud.order_by_cooldown([{"id": candidate}], {candidate: "closed-by-maintainer"}) == [] def test_producer_outside_the_local_clone_is_fetched_from_its_repository(tmp_path, monkeypatch): From a272bc7491d45a795b0b557741716501f2226b1b Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:41:31 -0500 Subject: [PATCH 2/2] ci(klaud): scope issues: write to the candidate job --- .github/workflows/klaud-candidate.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/klaud-candidate.yml b/.github/workflows/klaud-candidate.yml index 1dcdaec2e2..590391433b 100644 --- a/.github/workflows/klaud-candidate.yml +++ b/.github/workflows/klaud-candidate.yml @@ -29,12 +29,16 @@ permissions: contents: read actions: read # Read workflow runs and artifacts. pull-requests: read # Read PR metadata. - issues: write # Set display-only Klaud status labels on its PR. jobs: claude: name: Klaud Cold runs-on: ubuntu-latest + permissions: + contents: read + actions: read # Read workflow runs and artifacts. + pull-requests: read # Read PR metadata. + issues: write # Set display-only Klaud status labels on its PR. env: MAX_REPAIRS: '5' KLAUD_BRANCH: klaud/auto-${{ inputs.candidate }}